diff --git a/.env.example b/.env.example index 2fab177..e1510e2 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,11 @@ BOT_DISPLAY_NAME=GogoBee # Which rooms the bot posts scheduled content to (comma-separated room IDs) BROADCAST_ROOMS=!roomid:example.com +# The daily 08:00 WOTD auto-post is disabled by default. +# Set to true to enable it. The !wotd command and passive WOTD +# usage tracking work regardless of this setting. +ENABLE_WOTD_POST=false + # Admins who can run admin-only commands (comma-separated Matrix user IDs) ADMIN_USERS=@yourmxid:example.com diff --git a/internal/db/db.go b/internal/db/db.go index 0acb3f2..b985f47 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -333,6 +333,9 @@ func runMigrations(d *sql.DB) error { // engages when a fork / elite / boss / supply pinch actually // needs a decision. CAS-claim on this column gates re-entry. `ALTER TABLE dnd_expedition ADD COLUMN last_autorun_at DATETIME`, + // URL link previews now post the page's og:image/twitter:image + // thumbnail; cache it alongside the title/description. + `ALTER TABLE url_cache ADD COLUMN image_url TEXT NOT NULL DEFAULT ''`, } for _, stmt := range columnMigrations { if _, err := d.Exec(stmt); err != nil { @@ -1124,6 +1127,7 @@ CREATE TABLE IF NOT EXISTS url_cache ( url TEXT PRIMARY KEY, title TEXT DEFAULT '', description TEXT DEFAULT '', + image_url TEXT NOT NULL DEFAULT '', cached_at INTEGER DEFAULT (unixepoch()) ); @@ -2041,15 +2045,15 @@ func SeedSchedulerDefaults(d *sql.DB) error { name string cron string }{ - {"prefetch", "5 0 * * *"}, // 00:05 daily - {"maintenance", "0 3 * * *"}, // 03:00 daily - {"wotd", "0 8 * * *"}, // 08:00 daily - {"holidays", "0 7 * * *"}, // 07:00 daily - {"releases", "0 9 * * 1"}, // 09:00 Monday - {"birthday_check", "0 6 * * *"}, // 06:00 daily - {"anime_releases", "0 10 * * *"},// 10:00 daily - {"movie_releases", "0 11 * * *"},// 11:00 daily - {"concert_digest", "0 12 * * 0"},// 12:00 Sunday + {"prefetch", "5 0 * * *"}, // 00:05 daily + {"maintenance", "0 3 * * *"}, // 03:00 daily + {"wotd", "0 8 * * *"}, // 08:00 daily + {"holidays", "0 7 * * *"}, // 07:00 daily + {"releases", "0 9 * * 1"}, // 09:00 Monday + {"birthday_check", "0 6 * * *"}, // 06:00 daily + {"anime_releases", "0 10 * * *"}, // 10:00 daily + {"movie_releases", "0 11 * * *"}, // 11:00 daily + {"concert_digest", "0 12 * * 0"}, // 12:00 Sunday } stmt, err := d.Prepare(`INSERT OR IGNORE INTO scheduler_config (job_name, cron_expr) VALUES (?, ?)`) diff --git a/internal/plugin/link_thumbnail.go b/internal/plugin/link_thumbnail.go new file mode 100644 index 0000000..e247e46 --- /dev/null +++ b/internal/plugin/link_thumbnail.go @@ -0,0 +1,184 @@ +package plugin + +import ( + "bytes" + "context" + "fmt" + "image" + _ "image/gif" // register GIF decoder for DecodeConfig + _ "image/jpeg" // register JPEG decoder for DecodeConfig + _ "image/png" // register PNG decoder for DecodeConfig + "io" + "log/slog" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "gogobee/internal/safehttp" + + _ "golang.org/x/image/webp" // register WebP decoder for DecodeConfig + + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +// thumbnailUserAgent identifies the bot when probing and downloading the +// preview image for a posted link. +const thumbnailUserAgent = "GogoBee/1.0 (+link thumbnail fetch)" + +// thumbnailClient validates and downloads link-supplied image URLs. It routes +// through safehttp so a posted link can't steer fetches at loopback / RFC1918 / +// cloud-metadata IPs — the dial-time guard re-checks every redirect target too. +// The 10 MiB download ceiling below bounds memory regardless. +var thumbnailClient = safehttp.NewClient(15 * time.Second) + +// resolveURL turns a possibly-relative ref into an absolute URL against base. +func resolveURL(base, ref string) string { + ref = strings.TrimSpace(ref) + if ref == "" { + return "" + } + b, err := url.Parse(base) + if err != nil { + return ref + } + r, err := url.Parse(ref) + if err != nil { + return ref + } + return b.ResolveReference(r).String() +} + +// normalizeImageURL rewrites known CDN thumbnail URLs to a higher-resolution +// variant. Currently handles The Guardian's i.guim.co.uk, whose pages hand out +// narrow thumbnails. Signed URLs are left alone (re-signing isn't possible), +// as are unrecognized hosts. +func normalizeImageURL(raw string) string { + if raw == "" { + return raw + } + u, err := url.Parse(raw) + if err != nil || u.Host != "i.guim.co.uk" { + return raw + } + q := u.Query() + if q.Get("width") == "" || q.Get("s") != "" { + return raw + } + q.Set("width", "1200") + u.RawQuery = q.Encode() + return u.String() +} + +// validateImageURL HEAD-probes an image URL: it must be http(s), return 200, +// have an image/* content type, and (if a length is declared) exceed 5 KiB so +// tracking pixels are filtered. Returns false with no error on any failure. +func validateImageURL(rawURL string) bool { + if rawURL == "" || safehttp.ValidateURL(rawURL) != nil { + return false + } + req, err := http.NewRequest("HEAD", rawURL, nil) + if err != nil { + return false + } + req.Header.Set("User-Agent", thumbnailUserAgent) + + resp, err := thumbnailClient.Do(req) + if err != nil { + slog.Debug("thumbnail: image HEAD failed", "url", rawURL, "err", err) + return false + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return false + } + if !strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") { + return false + } + if cl := resp.Header.Get("Content-Length"); cl != "" { + if size, err := strconv.ParseInt(cl, 10, 64); err == nil && size <= 5120 { + return false // tracking pixel + } + } + return true +} + +// SendImageFromURL downloads imageURL, uploads it to Matrix, and posts it as an +// m.image event in roomID. Returns an error on any failure so callers can fall +// back to a text-only preview. The fetch is SSRF-guarded and size-capped. +func (b *Base) SendImageFromURL(roomID id.RoomID, imageURL string) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", imageURL, nil) + if err != nil { + return fmt.Errorf("create image request: %w", err) + } + req.Header.Set("User-Agent", thumbnailUserAgent) + + resp, err := thumbnailClient.Do(req) + if err != nil { + return fmt.Errorf("download image: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("image download status %d", resp.StatusCode) + } + + data, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024)) + if err != nil { + return fmt.Errorf("read image: %w", err) + } + + contentType := resp.Header.Get("Content-Type") + if i := strings.IndexByte(contentType, ';'); i >= 0 { + contentType = strings.TrimSpace(contentType[:i]) + } + if contentType == "" { + contentType = "image/jpeg" + } + if !strings.HasPrefix(contentType, "image/") { + return fmt.Errorf("not an image: %s", contentType) + } + + // Decode dimensions so clients render the image inline rather than as a + // downloadable file attachment. + var width, height int + if cfg, _, decErr := image.DecodeConfig(bytes.NewReader(data)); decErr == nil { + width, height = cfg.Width, cfg.Height + } + + ext := ".jpg" + switch contentType { + case "image/png": + ext = ".png" + case "image/gif": + ext = ".gif" + case "image/webp": + ext = ".webp" + } + filename := "thumbnail" + ext + + uri, err := b.UploadContent(data, contentType, filename) + if err != nil { + return fmt.Errorf("upload image: %w", err) + } + + content := &event.MessageEventContent{ + MsgType: event.MsgImage, + Body: filename, + FileName: filename, + URL: uri.CUString(), + Info: &event.FileInfo{ + MimeType: contentType, + Size: len(data), + Width: width, + Height: height, + }, + } + _, err = b.Client.SendMessageEvent(context.Background(), roomID, event.EventMessage, content) + return err +} diff --git a/internal/plugin/link_thumbnail_test.go b/internal/plugin/link_thumbnail_test.go new file mode 100644 index 0000000..094e343 --- /dev/null +++ b/internal/plugin/link_thumbnail_test.go @@ -0,0 +1,40 @@ +package plugin + +import "testing" + +func TestResolveURL(t *testing.T) { + cases := []struct { + base, ref, want string + }{ + {"https://e.com/news/story", "https://cdn.e.com/a.jpg", "https://cdn.e.com/a.jpg"}, + {"https://e.com/news/story", "//cdn.e.com/a.jpg", "https://cdn.e.com/a.jpg"}, + {"https://e.com/news/story", "/img/a.jpg", "https://e.com/img/a.jpg"}, + {"https://e.com/news/story", "a.jpg", "https://e.com/news/a.jpg"}, + {"https://e.com/news/story", "", ""}, + } + for _, c := range cases { + if got := resolveURL(c.base, c.ref); got != c.want { + t.Errorf("resolveURL(%q, %q) = %q, want %q", c.base, c.ref, got, c.want) + } + } +} + +func TestNormalizeImageURL(t *testing.T) { + cases := []struct { + name, in, want string + }{ + {"non-guardian passthrough", "https://e.com/a.jpg?width=140", "https://e.com/a.jpg?width=140"}, + {"signed left alone", "https://i.guim.co.uk/x.jpg?width=140&s=abc", "https://i.guim.co.uk/x.jpg?width=140&s=abc"}, + {"no width passthrough", "https://i.guim.co.uk/x.jpg", "https://i.guim.co.uk/x.jpg"}, + {"empty", "", ""}, + } + for _, c := range cases { + if got := normalizeImageURL(c.in); got != c.want { + t.Errorf("%s: normalizeImageURL(%q) = %q, want %q", c.name, c.in, got, c.want) + } + } + // Unsigned Guardian thumbnail gets bumped to width=1200. + if got := normalizeImageURL("https://i.guim.co.uk/x.jpg?width=140"); got != "https://i.guim.co.uk/x.jpg?width=1200" { + t.Errorf("normalizeImageURL unsigned = %q, want width=1200", got) + } +} diff --git a/internal/plugin/urls.go b/internal/plugin/urls.go index 1366666..89772ad 100644 --- a/internal/plugin/urls.go +++ b/internal/plugin/urls.go @@ -10,6 +10,7 @@ import ( "time" "gogobee/internal/db" + "gogobee/internal/safehttp" "github.com/PuerkitoBio/goquery" "maunium.net/go/mautrix" @@ -40,9 +41,10 @@ func NewURLsPlugin(client *mautrix.Client) *URLsPlugin { Base: NewBase(client), enabled: enabled, ignoreUsers: ignore, - httpClient: &http.Client{ - Timeout: 3 * time.Second, - }, + // Route page scrapes through safehttp so a posted link can't steer the + // fetch at loopback / RFC1918 / cloud-metadata IPs (re-checked on every + // redirect), and can't OOM the parser by streaming an unbounded body. + httpClient: safehttp.NewClient(8 * time.Second), } } @@ -90,12 +92,23 @@ func (p *URLsPlugin) OnMessage(ctx MessageContext) error { } func (p *URLsPlugin) previewURL(ctx MessageContext, targetURL string) { - title, desc, err := p.fetchPreview(targetURL) + title, desc, image, err := p.fetchPreview(targetURL) if err != nil { slog.Debug("urls: fetch preview failed", "url", targetURL, "err", err) return } + if title == "" && desc == "" && image == "" { + return + } + + // Post the thumbnail first (best-effort), so it renders above the text. + if image != "" && validateImageURL(image) { + if err := p.SendImageFromURL(ctx.RoomID, image); err != nil { + slog.Debug("urls: thumbnail post failed", "url", targetURL, "image", image, "err", err) + } + } + if title == "" && desc == "" { return } @@ -120,63 +133,69 @@ func (p *URLsPlugin) previewURL(ctx MessageContext, targetURL string) { } } -// fetchPreview retrieves og:title and og:description, checking cache first. -func (p *URLsPlugin) fetchPreview(rawURL string) (string, string, error) { +// fetchPreview retrieves og:title, og:description and og:image, checking cache first. +func (p *URLsPlugin) fetchPreview(rawURL string) (title, desc, image string, err error) { d := db.Get() now := time.Now().UTC().Unix() cacheTTL := int64(24 * 60 * 60) // Check cache - var title, desc string var cachedAt int64 - err := d.QueryRow( - `SELECT title, description, cached_at FROM url_cache WHERE url = ?`, rawURL, - ).Scan(&title, &desc, &cachedAt) + err = d.QueryRow( + `SELECT title, description, image_url, cached_at FROM url_cache WHERE url = ?`, rawURL, + ).Scan(&title, &desc, &image, &cachedAt) if err == nil && now-cachedAt < cacheTTL { - return title, desc, nil + return title, desc, image, nil } // Fetch from web - title, desc, err = p.scrapeOG(rawURL) + title, desc, image, err = p.scrapeOG(rawURL) if err != nil { - return "", "", err + return "", "", "", err } // Cache the result db.Exec("urls: cache write", - `INSERT INTO url_cache (url, title, description, cached_at) VALUES (?, ?, ?, ?) - ON CONFLICT(url) DO UPDATE SET title = ?, description = ?, cached_at = ?`, - rawURL, title, desc, now, title, desc, now, + `INSERT INTO url_cache (url, title, description, image_url, cached_at) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(url) DO UPDATE SET title = ?, description = ?, image_url = ?, cached_at = ?`, + rawURL, title, desc, image, now, title, desc, image, now, ) - return title, desc, nil + return title, desc, image, nil } -// scrapeOG fetches a URL and extracts og:title and og:description. -func (p *URLsPlugin) scrapeOG(rawURL string) (string, string, error) { +// scrapeOG fetches a URL and extracts og:title, og:description and og:image. +func (p *URLsPlugin) scrapeOG(rawURL string) (string, string, string, error) { + if err := safehttp.ValidateURL(rawURL); err != nil { + return "", "", "", err + } + req, err := http.NewRequest("GET", rawURL, nil) if err != nil { - return "", "", fmt.Errorf("create request: %w", err) + return "", "", "", fmt.Errorf("create request: %w", err) } req.Header.Set("User-Agent", "GogoBee Bot/1.0") + req.Header.Set("Accept", "text/html,application/xhtml+xml") resp, err := p.httpClient.Do(req) if err != nil { - return "", "", fmt.Errorf("fetch: %w", err) + return "", "", "", fmt.Errorf("fetch: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", "", fmt.Errorf("status %d", resp.StatusCode) + return "", "", "", fmt.Errorf("status %d", resp.StatusCode) } - doc, err := goquery.NewDocumentFromReader(resp.Body) + // Cap the parsed body at 2 MiB — og: tags live in , near the top. + doc, err := goquery.NewDocumentFromReader(safehttp.LimitedBody(resp.Body, 2*1024*1024)) if err != nil { - return "", "", fmt.Errorf("parse HTML: %w", err) + return "", "", "", fmt.Errorf("parse HTML: %w", err) } title := "" desc := "" + image := "" doc.Find("meta").Each(func(_ int, s *goquery.Selection) { prop, _ := s.Attr("property") @@ -186,6 +205,10 @@ func (p *URLsPlugin) scrapeOG(rawURL string) (string, string, error) { title = content case "og:description": desc = content + case "og:image:secure_url", "og:image:url", "og:image": + if image == "" && strings.TrimSpace(content) != "" { + image = content + } } }) @@ -205,5 +228,9 @@ func (p *URLsPlugin) scrapeOG(rawURL string) (string, string, error) { }) } - return strings.TrimSpace(title), strings.TrimSpace(desc), nil + if image != "" { + image = normalizeImageURL(resolveURL(rawURL, strings.TrimSpace(image))) + } + + return strings.TrimSpace(title), strings.TrimSpace(desc), image, nil } diff --git a/internal/safehttp/safehttp.go b/internal/safehttp/safehttp.go new file mode 100644 index 0000000..0f8ccfe --- /dev/null +++ b/internal/safehttp/safehttp.go @@ -0,0 +1,146 @@ +// Package safehttp provides an http.Client hardened against SSRF and +// memory-DoS via hostile upstreams. Every outbound fetch the bot makes +// against feed-supplied URLs (RSS articles, image hosts) should go through +// one of these clients so a malicious feed can't steer the bot at loopback, +// link-local, RFC1918, or cloud metadata IPs, and can't OOM the process by +// streaming an unbounded body. +package safehttp + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +// ErrBlockedHost is returned when a URL resolves to a non-public IP. +var ErrBlockedHost = errors.New("safehttp: blocked non-public host") + +// AllowPrivate, when true, disables the loopback/RFC1918 dial guard. It +// exists for tests that spin up httptest.NewServer on 127.0.0.1 — never +// set this in production. +var AllowPrivate bool + +// safeDialContext refuses connections to non-public IPs. It runs after +// DNS resolution, so a hostile DNS rebinding that returns 127.0.0.1 +// still gets blocked at dial time. +func safeDialContext(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + ips, err := (&net.Resolver{}).LookupIP(ctx, "ip", host) + if err != nil { + return nil, err + } + var allowed net.IP + for _, ip := range ips { + if AllowPrivate || isPublicIP(ip) { + allowed = ip + break + } + } + if allowed == nil { + return nil, fmt.Errorf("%w: %s", ErrBlockedHost, host) + } + d := &net.Dialer{Timeout: 5 * time.Second, KeepAlive: 30 * time.Second} + return d.DialContext(ctx, network, net.JoinHostPort(allowed.String(), port)) +} + +// isPublicIP reports whether ip is a globally routable unicast address. +// Rejects loopback, link-local, multicast, RFC1918, CGNAT, and the +// AWS/GCP/Azure metadata IPs 169.254.169.254 / fd00:ec2::254 (these +// already fall under link-local but spell it out for clarity). +func isPublicIP(ip net.IP) bool { + if ip == nil || ip.IsUnspecified() || ip.IsLoopback() || + ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || + ip.IsMulticast() || ip.IsPrivate() { + return false + } + // 100.64.0.0/10 (CGNAT) is not covered by IsPrivate on older Go. + if v4 := ip.To4(); v4 != nil { + if v4[0] == 100 && v4[1] >= 64 && v4[1] <= 127 { + return false + } + // 0.0.0.0/8 and friends. + if v4[0] == 0 { + return false + } + } + return true +} + +// ValidateURL returns nil if the URL is http(s) and parseable. It does +// not resolve DNS — the dial step does that — but it does reject bare +// schemes (file://, gopher://, etc.) before we even open a connection. +func ValidateURL(raw string) error { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return err + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("safehttp: unsupported scheme %q", u.Scheme) + } + if u.Host == "" { + return errors.New("safehttp: empty host") + } + return nil +} + +// NewClient returns an http.Client whose transport blocks non-public +// destinations at dial time, caps redirects at 5, and re-validates each +// redirect target's scheme. timeout is the per-request overall budget. +func NewClient(timeout time.Duration) *http.Client { + tr := &http.Transport{ + DialContext: safeDialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 32, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 5 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + ResponseHeaderTimeout: 10 * time.Second, + } + return &http.Client{ + Transport: tr, + Timeout: timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 5 { + return errors.New("safehttp: stopped after 5 redirects") + } + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + return fmt.Errorf("safehttp: unsupported redirect scheme %q", req.URL.Scheme) + } + return nil + }, + } +} + +// LimitedBody wraps r in a reader that errors once more than max bytes have +// been read. Use to cap how much of a response body downstream parsers +// (goquery, image.Decode) will ever see — a hostile origin streaming an +// endless body otherwise OOMs the process. +func LimitedBody(r io.Reader, max int64) io.Reader { + return &limitedReader{R: r, N: max} +} + +type limitedReader struct { + R io.Reader + N int64 +} + +func (l *limitedReader) Read(p []byte) (int, error) { + if l.N <= 0 { + return 0, fmt.Errorf("safehttp: response body exceeded cap") + } + if int64(len(p)) > l.N { + p = p[:l.N] + } + n, err := l.R.Read(p) + l.N -= int64(n) + return n, err +} diff --git a/main.go b/main.go index da8a03a..5d78995 100644 --- a/main.go +++ b/main.go @@ -406,8 +406,8 @@ func setupScheduledJobs( } }) - // WOTD post at 08:00 - if strings.ToLower(os.Getenv("DISABLE_WOTD_POST")) != "true" { + // WOTD post at 08:00 (disabled by default; opt in via ENABLE_WOTD_POST=true) + if strings.ToLower(os.Getenv("ENABLE_WOTD_POST")) == "true" { c.AddFunc("0 8 * * *", func() { slog.Info("scheduler: posting WOTD") for _, r := range rooms { @@ -415,7 +415,7 @@ func setupScheduledJobs( } }) } else { - slog.Info("scheduler: WOTD daily post disabled via DISABLE_WOTD_POST") + slog.Info("scheduler: WOTD daily post disabled (set ENABLE_WOTD_POST=true to enable)") } // Game releases Monday 09:00