Files
gogobee/internal/plugin/link_thumbnail.go
prosolis ba7b20dfe5 url previews: stop dropping thumbnails on body cap and HEAD-403 CDNs
Two independent causes, both silent:

LimitedBody returned an error once the cap was hit, so scrapeOG failed the
whole goquery parse on any page over 2 MiB — even though og: tags live in
<head>, near the top. Hitting the cap is a truncation, not a failure: return
io.EOF and let the parser decide whether it found what it needed. Sized
against the pages actually posted here (n=14): og:title landed within the
first 7 KiB on twelve, worst case ~600 KiB.

validateImageURL HEAD-probed the image and bailed on non-200. Some publisher
CDNs (dims.apnews.com among them) answer HEAD with 403 while serving the same
URL over GET, so their thumbnails were always dropped. Probe with HEAD first,
fall back to a ranged GET asking for the first KiB. A 206 declares the full
size in Content-Range's "/119070" suffix rather than Content-Length, so the
tracking-pixel filter reads size from there.
2026-07-09 18:52:32 -07:00

241 lines
7.2 KiB
Go

package plugin
import (
"bytes"
"context"
"fmt"
"image"
_ "image/gif" // register GIF decoder for DecodeConfig
_ "image/jpeg" // register JPEG decoder for DecodeConfig
_ "image/png" // register PNG decoder for DecodeConfig
"io"
"log/slog"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"gogobee/internal/safehttp"
_ "golang.org/x/image/webp" // register WebP decoder for DecodeConfig
"maunium.net/go/mautrix/event"
"maunium.net/go/mautrix/id"
)
// thumbnailUserAgent identifies the bot when probing and downloading the
// preview image for a posted link.
const thumbnailUserAgent = "GogoBee/1.0 (+link thumbnail fetch)"
// thumbnailClient validates and downloads link-supplied image URLs. It routes
// through safehttp so a posted link can't steer fetches at loopback / RFC1918 /
// cloud-metadata IPs — the dial-time guard re-checks every redirect target too.
// The 10 MiB download ceiling below bounds memory regardless.
var thumbnailClient = safehttp.NewClient(15 * time.Second)
// resolveURL turns a possibly-relative ref into an absolute URL against base.
func resolveURL(base, ref string) string {
ref = strings.TrimSpace(ref)
if ref == "" {
return ""
}
b, err := url.Parse(base)
if err != nil {
return ref
}
r, err := url.Parse(ref)
if err != nil {
return ref
}
return b.ResolveReference(r).String()
}
// normalizeImageURL rewrites known CDN thumbnail URLs to a higher-resolution
// variant. Currently handles The Guardian's i.guim.co.uk, whose pages hand out
// narrow thumbnails. Signed URLs are left alone (re-signing isn't possible),
// as are unrecognized hosts.
func normalizeImageURL(raw string) string {
if raw == "" {
return raw
}
u, err := url.Parse(raw)
if err != nil || u.Host != "i.guim.co.uk" {
return raw
}
q := u.Query()
if q.Get("width") == "" || q.Get("s") != "" {
return raw
}
q.Set("width", "1200")
u.RawQuery = q.Encode()
return u.String()
}
// trackingPixelBytes is the size at or below which a declared image is assumed
// to be a tracking pixel rather than a real preview thumbnail.
const trackingPixelBytes = 5120
// declaredSize reports the full size of the resource a probe response describes,
// or -1 when the origin declares none. A ranged probe answers 206 with a
// Content-Length covering only the requested slice, so the total comes from
// Content-Range's "bytes 0-1023/119070" suffix instead.
func declaredSize(resp *http.Response) int64 {
if resp.StatusCode == http.StatusPartialContent {
cr := resp.Header.Get("Content-Range")
i := strings.LastIndexByte(cr, '/')
if i < 0 {
return -1
}
total, err := strconv.ParseInt(strings.TrimSpace(cr[i+1:]), 10, 64)
if err != nil {
return -1 // "*" or malformed: unknown, not empty
}
return total
}
if cl := resp.Header.Get("Content-Length"); cl != "" {
if size, err := strconv.ParseInt(cl, 10, 64); err == nil {
return size
}
}
return -1
}
// probeImage asks the origin about rawURL using method and reports its content
// type and full size. GET probes request only the first KiB, so falling back to
// one stays about as cheap as the HEAD it replaces.
func probeImage(method, rawURL string) (contentType string, size int64, ok bool) {
req, err := http.NewRequest(method, rawURL, nil)
if err != nil {
return "", -1, false
}
req.Header.Set("User-Agent", thumbnailUserAgent)
if method == http.MethodGet {
req.Header.Set("Range", "bytes=0-1023")
}
resp, err := thumbnailClient.Do(req)
if err != nil {
slog.Debug("thumbnail: image probe failed", "method", method, "url", rawURL, "err", err)
return "", -1, false
}
defer resp.Body.Close()
// Drain the sipped bytes so the connection can be reused.
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
slog.Debug("thumbnail: image probe rejected", "method", method, "url", rawURL, "status", resp.StatusCode)
return "", -1, false
}
return resp.Header.Get("Content-Type"), declaredSize(resp), true
}
// validateImageURL probes an image URL: it must be http(s), answer a probe, have
// an image/* content type, and (if a size is declared) exceed trackingPixelBytes.
// Returns false with no error on any failure.
//
// The probe is a HEAD first, falling back to a ranged GET: CDNs in front of some
// publishers (apnews's dims.apnews.com among them) answer HEAD with 403 while
// serving the very same URL over GET, and a HEAD-only check silently drops those
// thumbnails.
func validateImageURL(rawURL string) bool {
if rawURL == "" || safehttp.ValidateURL(rawURL) != nil {
return false
}
contentType, size, ok := probeImage(http.MethodHead, rawURL)
if !ok {
contentType, size, ok = probeImage(http.MethodGet, rawURL)
}
if !ok {
return false
}
if !strings.HasPrefix(contentType, "image/") {
slog.Debug("thumbnail: not an image", "url", rawURL, "content_type", contentType)
return false
}
if size >= 0 && size <= trackingPixelBytes {
slog.Debug("thumbnail: image too small, treating as tracking pixel", "url", rawURL, "size", size)
return false
}
return true
}
// SendImageFromURL downloads imageURL, uploads it to Matrix, and posts it as an
// m.image event in roomID. Returns an error on any failure so callers can fall
// back to a text-only preview. The fetch is SSRF-guarded and size-capped.
func (b *Base) SendImageFromURL(roomID id.RoomID, imageURL string) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", imageURL, nil)
if err != nil {
return fmt.Errorf("create image request: %w", err)
}
req.Header.Set("User-Agent", thumbnailUserAgent)
resp, err := thumbnailClient.Do(req)
if err != nil {
return fmt.Errorf("download image: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("image download status %d", resp.StatusCode)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024))
if err != nil {
return fmt.Errorf("read image: %w", err)
}
contentType := resp.Header.Get("Content-Type")
if i := strings.IndexByte(contentType, ';'); i >= 0 {
contentType = strings.TrimSpace(contentType[:i])
}
if contentType == "" {
contentType = "image/jpeg"
}
if !strings.HasPrefix(contentType, "image/") {
return fmt.Errorf("not an image: %s", contentType)
}
// Decode dimensions so clients render the image inline rather than as a
// downloadable file attachment.
var width, height int
if cfg, _, decErr := image.DecodeConfig(bytes.NewReader(data)); decErr == nil {
width, height = cfg.Width, cfg.Height
}
ext := ".jpg"
switch contentType {
case "image/png":
ext = ".png"
case "image/gif":
ext = ".gif"
case "image/webp":
ext = ".webp"
}
filename := "thumbnail" + ext
uri, err := b.UploadContent(data, contentType, filename)
if err != nil {
return fmt.Errorf("upload image: %w", err)
}
content := &event.MessageEventContent{
MsgType: event.MsgImage,
Body: filename,
FileName: filename,
URL: uri.CUString(),
Info: &event.FileInfo{
MimeType: contentType,
Size: len(data),
Width: width,
Height: height,
},
}
_, err = b.Client.SendMessageEvent(context.Background(), roomID, event.EventMessage, content)
return err
}