mirror of
https://github.com/prosolis/gogobee.git
synced 2026-07-15 16:42:41 +00:00
Compare commits
1 Commits
link-thumb
...
4d35c0010e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4d35c0010e |
@@ -7,11 +7,6 @@ BOT_DISPLAY_NAME=GogoBee
|
||||
# Which rooms the bot posts scheduled content to (comma-separated room IDs)
|
||||
BROADCAST_ROOMS=!roomid:example.com
|
||||
|
||||
# The daily 08:00 WOTD auto-post is disabled by default.
|
||||
# Set to true to enable it. The !wotd command and passive WOTD
|
||||
# usage tracking work regardless of this setting.
|
||||
ENABLE_WOTD_POST=false
|
||||
|
||||
# Admins who can run admin-only commands (comma-separated Matrix user IDs)
|
||||
ADMIN_USERS=@yourmxid:example.com
|
||||
|
||||
|
||||
4
go.mod
4
go.mod
@@ -12,7 +12,7 @@ require (
|
||||
github.com/olebedev/when v1.1.0
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
|
||||
golang.org/x/image v0.40.0
|
||||
golang.org/x/image v0.43.0
|
||||
maunium.net/go/mautrix v0.28.0
|
||||
modernc.org/sqlite v1.50.1
|
||||
)
|
||||
@@ -40,7 +40,7 @@ require (
|
||||
golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a // indirect
|
||||
golang.org/x/net v0.54.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
golang.org/x/text v0.38.0 // indirect
|
||||
modernc.org/libc v1.72.3 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
|
||||
12
go.sum
12
go.sum
@@ -85,8 +85,8 @@ golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
|
||||
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
|
||||
golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a h1:+3jdDGGB8NGb1Zktc737jlt3/A5f6UlwSzmvqUuufxw=
|
||||
golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw=
|
||||
golang.org/x/image v0.40.0 h1:Tw4GyDXMo+daZN1znreBRC3VayR1aLFUyUEOLUdW1a8=
|
||||
golang.org/x/image v0.40.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA=
|
||||
golang.org/x/image v0.43.0 h1:FLxcP4ec2350nTfOC8ysKtqYSIFbk/QGjw1ZHNP4tsY=
|
||||
golang.org/x/image v0.43.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
@@ -112,8 +112,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -146,8 +146,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
|
||||
@@ -333,9 +333,6 @@ func runMigrations(d *sql.DB) error {
|
||||
// engages when a fork / elite / boss / supply pinch actually
|
||||
// needs a decision. CAS-claim on this column gates re-entry.
|
||||
`ALTER TABLE dnd_expedition ADD COLUMN last_autorun_at DATETIME`,
|
||||
// URL link previews now post the page's og:image/twitter:image
|
||||
// thumbnail; cache it alongside the title/description.
|
||||
`ALTER TABLE url_cache ADD COLUMN image_url TEXT NOT NULL DEFAULT ''`,
|
||||
}
|
||||
for _, stmt := range columnMigrations {
|
||||
if _, err := d.Exec(stmt); err != nil {
|
||||
@@ -1127,7 +1124,6 @@ CREATE TABLE IF NOT EXISTS url_cache (
|
||||
url TEXT PRIMARY KEY,
|
||||
title TEXT DEFAULT '',
|
||||
description TEXT DEFAULT '',
|
||||
image_url TEXT NOT NULL DEFAULT '',
|
||||
cached_at INTEGER DEFAULT (unixepoch())
|
||||
);
|
||||
|
||||
@@ -2045,15 +2041,15 @@ func SeedSchedulerDefaults(d *sql.DB) error {
|
||||
name string
|
||||
cron string
|
||||
}{
|
||||
{"prefetch", "5 0 * * *"}, // 00:05 daily
|
||||
{"maintenance", "0 3 * * *"}, // 03:00 daily
|
||||
{"wotd", "0 8 * * *"}, // 08:00 daily
|
||||
{"holidays", "0 7 * * *"}, // 07:00 daily
|
||||
{"releases", "0 9 * * 1"}, // 09:00 Monday
|
||||
{"birthday_check", "0 6 * * *"}, // 06:00 daily
|
||||
{"anime_releases", "0 10 * * *"}, // 10:00 daily
|
||||
{"movie_releases", "0 11 * * *"}, // 11:00 daily
|
||||
{"concert_digest", "0 12 * * 0"}, // 12:00 Sunday
|
||||
{"prefetch", "5 0 * * *"}, // 00:05 daily
|
||||
{"maintenance", "0 3 * * *"}, // 03:00 daily
|
||||
{"wotd", "0 8 * * *"}, // 08:00 daily
|
||||
{"holidays", "0 7 * * *"}, // 07:00 daily
|
||||
{"releases", "0 9 * * 1"}, // 09:00 Monday
|
||||
{"birthday_check", "0 6 * * *"}, // 06:00 daily
|
||||
{"anime_releases", "0 10 * * *"},// 10:00 daily
|
||||
{"movie_releases", "0 11 * * *"},// 11:00 daily
|
||||
{"concert_digest", "0 12 * * 0"},// 12:00 Sunday
|
||||
}
|
||||
|
||||
stmt, err := d.Prepare(`INSERT OR IGNORE INTO scheduler_config (job_name, cron_expr) VALUES (?, ?)`)
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif" // register GIF decoder for DecodeConfig
|
||||
_ "image/jpeg" // register JPEG decoder for DecodeConfig
|
||||
_ "image/png" // register PNG decoder for DecodeConfig
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gogobee/internal/safehttp"
|
||||
|
||||
_ "golang.org/x/image/webp" // register WebP decoder for DecodeConfig
|
||||
|
||||
"maunium.net/go/mautrix/event"
|
||||
"maunium.net/go/mautrix/id"
|
||||
)
|
||||
|
||||
// thumbnailUserAgent identifies the bot when probing and downloading the
|
||||
// preview image for a posted link.
|
||||
const thumbnailUserAgent = "GogoBee/1.0 (+link thumbnail fetch)"
|
||||
|
||||
// thumbnailClient validates and downloads link-supplied image URLs. It routes
|
||||
// through safehttp so a posted link can't steer fetches at loopback / RFC1918 /
|
||||
// cloud-metadata IPs — the dial-time guard re-checks every redirect target too.
|
||||
// The 10 MiB download ceiling below bounds memory regardless.
|
||||
var thumbnailClient = safehttp.NewClient(15 * time.Second)
|
||||
|
||||
// resolveURL turns a possibly-relative ref into an absolute URL against base.
|
||||
func resolveURL(base, ref string) string {
|
||||
ref = strings.TrimSpace(ref)
|
||||
if ref == "" {
|
||||
return ""
|
||||
}
|
||||
b, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return ref
|
||||
}
|
||||
r, err := url.Parse(ref)
|
||||
if err != nil {
|
||||
return ref
|
||||
}
|
||||
return b.ResolveReference(r).String()
|
||||
}
|
||||
|
||||
// normalizeImageURL rewrites known CDN thumbnail URLs to a higher-resolution
|
||||
// variant. Currently handles The Guardian's i.guim.co.uk, whose pages hand out
|
||||
// narrow thumbnails. Signed URLs are left alone (re-signing isn't possible),
|
||||
// as are unrecognized hosts.
|
||||
func normalizeImageURL(raw string) string {
|
||||
if raw == "" {
|
||||
return raw
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Host != "i.guim.co.uk" {
|
||||
return raw
|
||||
}
|
||||
q := u.Query()
|
||||
if q.Get("width") == "" || q.Get("s") != "" {
|
||||
return raw
|
||||
}
|
||||
q.Set("width", "1200")
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String()
|
||||
}
|
||||
|
||||
// validateImageURL HEAD-probes an image URL: it must be http(s), return 200,
|
||||
// have an image/* content type, and (if a length is declared) exceed 5 KiB so
|
||||
// tracking pixels are filtered. Returns false with no error on any failure.
|
||||
func validateImageURL(rawURL string) bool {
|
||||
if rawURL == "" || safehttp.ValidateURL(rawURL) != nil {
|
||||
return false
|
||||
}
|
||||
req, err := http.NewRequest("HEAD", rawURL, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
req.Header.Set("User-Agent", thumbnailUserAgent)
|
||||
|
||||
resp, err := thumbnailClient.Do(req)
|
||||
if err != nil {
|
||||
slog.Debug("thumbnail: image HEAD failed", "url", rawURL, "err", err)
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return false
|
||||
}
|
||||
if !strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") {
|
||||
return false
|
||||
}
|
||||
if cl := resp.Header.Get("Content-Length"); cl != "" {
|
||||
if size, err := strconv.ParseInt(cl, 10, 64); err == nil && size <= 5120 {
|
||||
return false // tracking pixel
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// SendImageFromURL downloads imageURL, uploads it to Matrix, and posts it as an
|
||||
// m.image event in roomID. Returns an error on any failure so callers can fall
|
||||
// back to a text-only preview. The fetch is SSRF-guarded and size-capped.
|
||||
func (b *Base) SendImageFromURL(roomID id.RoomID, imageURL string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", imageURL, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create image request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", thumbnailUserAgent)
|
||||
|
||||
resp, err := thumbnailClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download image: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("image download status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024))
|
||||
if err != nil {
|
||||
return fmt.Errorf("read image: %w", err)
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if i := strings.IndexByte(contentType, ';'); i >= 0 {
|
||||
contentType = strings.TrimSpace(contentType[:i])
|
||||
}
|
||||
if contentType == "" {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
if !strings.HasPrefix(contentType, "image/") {
|
||||
return fmt.Errorf("not an image: %s", contentType)
|
||||
}
|
||||
|
||||
// Decode dimensions so clients render the image inline rather than as a
|
||||
// downloadable file attachment.
|
||||
var width, height int
|
||||
if cfg, _, decErr := image.DecodeConfig(bytes.NewReader(data)); decErr == nil {
|
||||
width, height = cfg.Width, cfg.Height
|
||||
}
|
||||
|
||||
ext := ".jpg"
|
||||
switch contentType {
|
||||
case "image/png":
|
||||
ext = ".png"
|
||||
case "image/gif":
|
||||
ext = ".gif"
|
||||
case "image/webp":
|
||||
ext = ".webp"
|
||||
}
|
||||
filename := "thumbnail" + ext
|
||||
|
||||
uri, err := b.UploadContent(data, contentType, filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload image: %w", err)
|
||||
}
|
||||
|
||||
content := &event.MessageEventContent{
|
||||
MsgType: event.MsgImage,
|
||||
Body: filename,
|
||||
FileName: filename,
|
||||
URL: uri.CUString(),
|
||||
Info: &event.FileInfo{
|
||||
MimeType: contentType,
|
||||
Size: len(data),
|
||||
Width: width,
|
||||
Height: height,
|
||||
},
|
||||
}
|
||||
_, err = b.Client.SendMessageEvent(context.Background(), roomID, event.EventMessage, content)
|
||||
return err
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package plugin
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestResolveURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
base, ref, want string
|
||||
}{
|
||||
{"https://e.com/news/story", "https://cdn.e.com/a.jpg", "https://cdn.e.com/a.jpg"},
|
||||
{"https://e.com/news/story", "//cdn.e.com/a.jpg", "https://cdn.e.com/a.jpg"},
|
||||
{"https://e.com/news/story", "/img/a.jpg", "https://e.com/img/a.jpg"},
|
||||
{"https://e.com/news/story", "a.jpg", "https://e.com/news/a.jpg"},
|
||||
{"https://e.com/news/story", "", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := resolveURL(c.base, c.ref); got != c.want {
|
||||
t.Errorf("resolveURL(%q, %q) = %q, want %q", c.base, c.ref, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeImageURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, in, want string
|
||||
}{
|
||||
{"non-guardian passthrough", "https://e.com/a.jpg?width=140", "https://e.com/a.jpg?width=140"},
|
||||
{"signed left alone", "https://i.guim.co.uk/x.jpg?width=140&s=abc", "https://i.guim.co.uk/x.jpg?width=140&s=abc"},
|
||||
{"no width passthrough", "https://i.guim.co.uk/x.jpg", "https://i.guim.co.uk/x.jpg"},
|
||||
{"empty", "", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := normalizeImageURL(c.in); got != c.want {
|
||||
t.Errorf("%s: normalizeImageURL(%q) = %q, want %q", c.name, c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
// Unsigned Guardian thumbnail gets bumped to width=1200.
|
||||
if got := normalizeImageURL("https://i.guim.co.uk/x.jpg?width=140"); got != "https://i.guim.co.uk/x.jpg?width=1200" {
|
||||
t.Errorf("normalizeImageURL unsigned = %q, want width=1200", got)
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"time"
|
||||
|
||||
"gogobee/internal/db"
|
||||
"gogobee/internal/safehttp"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"maunium.net/go/mautrix"
|
||||
@@ -41,10 +40,9 @@ func NewURLsPlugin(client *mautrix.Client) *URLsPlugin {
|
||||
Base: NewBase(client),
|
||||
enabled: enabled,
|
||||
ignoreUsers: ignore,
|
||||
// Route page scrapes through safehttp so a posted link can't steer the
|
||||
// fetch at loopback / RFC1918 / cloud-metadata IPs (re-checked on every
|
||||
// redirect), and can't OOM the parser by streaming an unbounded body.
|
||||
httpClient: safehttp.NewClient(8 * time.Second),
|
||||
httpClient: &http.Client{
|
||||
Timeout: 3 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,23 +90,12 @@ func (p *URLsPlugin) OnMessage(ctx MessageContext) error {
|
||||
}
|
||||
|
||||
func (p *URLsPlugin) previewURL(ctx MessageContext, targetURL string) {
|
||||
title, desc, image, err := p.fetchPreview(targetURL)
|
||||
title, desc, err := p.fetchPreview(targetURL)
|
||||
if err != nil {
|
||||
slog.Debug("urls: fetch preview failed", "url", targetURL, "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
if title == "" && desc == "" && image == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// Post the thumbnail first (best-effort), so it renders above the text.
|
||||
if image != "" && validateImageURL(image) {
|
||||
if err := p.SendImageFromURL(ctx.RoomID, image); err != nil {
|
||||
slog.Debug("urls: thumbnail post failed", "url", targetURL, "image", image, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
if title == "" && desc == "" {
|
||||
return
|
||||
}
|
||||
@@ -133,69 +120,63 @@ func (p *URLsPlugin) previewURL(ctx MessageContext, targetURL string) {
|
||||
}
|
||||
}
|
||||
|
||||
// fetchPreview retrieves og:title, og:description and og:image, checking cache first.
|
||||
func (p *URLsPlugin) fetchPreview(rawURL string) (title, desc, image string, err error) {
|
||||
// fetchPreview retrieves og:title and og:description, checking cache first.
|
||||
func (p *URLsPlugin) fetchPreview(rawURL string) (string, string, error) {
|
||||
d := db.Get()
|
||||
now := time.Now().UTC().Unix()
|
||||
cacheTTL := int64(24 * 60 * 60)
|
||||
|
||||
// Check cache
|
||||
var title, desc string
|
||||
var cachedAt int64
|
||||
err = d.QueryRow(
|
||||
`SELECT title, description, image_url, cached_at FROM url_cache WHERE url = ?`, rawURL,
|
||||
).Scan(&title, &desc, &image, &cachedAt)
|
||||
err := d.QueryRow(
|
||||
`SELECT title, description, cached_at FROM url_cache WHERE url = ?`, rawURL,
|
||||
).Scan(&title, &desc, &cachedAt)
|
||||
if err == nil && now-cachedAt < cacheTTL {
|
||||
return title, desc, image, nil
|
||||
return title, desc, nil
|
||||
}
|
||||
|
||||
// Fetch from web
|
||||
title, desc, image, err = p.scrapeOG(rawURL)
|
||||
title, desc, err = p.scrapeOG(rawURL)
|
||||
if err != nil {
|
||||
return "", "", "", err
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
// Cache the result
|
||||
db.Exec("urls: cache write",
|
||||
`INSERT INTO url_cache (url, title, description, image_url, cached_at) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(url) DO UPDATE SET title = ?, description = ?, image_url = ?, cached_at = ?`,
|
||||
rawURL, title, desc, image, now, title, desc, image, now,
|
||||
`INSERT INTO url_cache (url, title, description, cached_at) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(url) DO UPDATE SET title = ?, description = ?, cached_at = ?`,
|
||||
rawURL, title, desc, now, title, desc, now,
|
||||
)
|
||||
|
||||
return title, desc, image, nil
|
||||
return title, desc, nil
|
||||
}
|
||||
|
||||
// scrapeOG fetches a URL and extracts og:title, og:description and og:image.
|
||||
func (p *URLsPlugin) scrapeOG(rawURL string) (string, string, string, error) {
|
||||
if err := safehttp.ValidateURL(rawURL); err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
|
||||
// scrapeOG fetches a URL and extracts og:title and og:description.
|
||||
func (p *URLsPlugin) scrapeOG(rawURL string) (string, string, error) {
|
||||
req, err := http.NewRequest("GET", rawURL, nil)
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("create request: %w", err)
|
||||
return "", "", fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "GogoBee Bot/1.0")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
||||
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("fetch: %w", err)
|
||||
return "", "", fmt.Errorf("fetch: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", "", "", fmt.Errorf("status %d", resp.StatusCode)
|
||||
return "", "", fmt.Errorf("status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Cap the parsed body at 2 MiB — og: tags live in <head>, near the top.
|
||||
doc, err := goquery.NewDocumentFromReader(safehttp.LimitedBody(resp.Body, 2*1024*1024))
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("parse HTML: %w", err)
|
||||
return "", "", fmt.Errorf("parse HTML: %w", err)
|
||||
}
|
||||
|
||||
title := ""
|
||||
desc := ""
|
||||
image := ""
|
||||
|
||||
doc.Find("meta").Each(func(_ int, s *goquery.Selection) {
|
||||
prop, _ := s.Attr("property")
|
||||
@@ -205,10 +186,6 @@ func (p *URLsPlugin) scrapeOG(rawURL string) (string, string, string, error) {
|
||||
title = content
|
||||
case "og:description":
|
||||
desc = content
|
||||
case "og:image:secure_url", "og:image:url", "og:image":
|
||||
if image == "" && strings.TrimSpace(content) != "" {
|
||||
image = content
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
@@ -228,9 +205,5 @@ func (p *URLsPlugin) scrapeOG(rawURL string) (string, string, string, error) {
|
||||
})
|
||||
}
|
||||
|
||||
if image != "" {
|
||||
image = normalizeImageURL(resolveURL(rawURL, strings.TrimSpace(image)))
|
||||
}
|
||||
|
||||
return strings.TrimSpace(title), strings.TrimSpace(desc), image, nil
|
||||
return strings.TrimSpace(title), strings.TrimSpace(desc), nil
|
||||
}
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
// Package safehttp provides an http.Client hardened against SSRF and
|
||||
// memory-DoS via hostile upstreams. Every outbound fetch the bot makes
|
||||
// against feed-supplied URLs (RSS articles, image hosts) should go through
|
||||
// one of these clients so a malicious feed can't steer the bot at loopback,
|
||||
// link-local, RFC1918, or cloud metadata IPs, and can't OOM the process by
|
||||
// streaming an unbounded body.
|
||||
package safehttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrBlockedHost is returned when a URL resolves to a non-public IP.
|
||||
var ErrBlockedHost = errors.New("safehttp: blocked non-public host")
|
||||
|
||||
// AllowPrivate, when true, disables the loopback/RFC1918 dial guard. It
|
||||
// exists for tests that spin up httptest.NewServer on 127.0.0.1 — never
|
||||
// set this in production.
|
||||
var AllowPrivate bool
|
||||
|
||||
// safeDialContext refuses connections to non-public IPs. It runs after
|
||||
// DNS resolution, so a hostile DNS rebinding that returns 127.0.0.1
|
||||
// still gets blocked at dial time.
|
||||
func safeDialContext(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ips, err := (&net.Resolver{}).LookupIP(ctx, "ip", host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var allowed net.IP
|
||||
for _, ip := range ips {
|
||||
if AllowPrivate || isPublicIP(ip) {
|
||||
allowed = ip
|
||||
break
|
||||
}
|
||||
}
|
||||
if allowed == nil {
|
||||
return nil, fmt.Errorf("%w: %s", ErrBlockedHost, host)
|
||||
}
|
||||
d := &net.Dialer{Timeout: 5 * time.Second, KeepAlive: 30 * time.Second}
|
||||
return d.DialContext(ctx, network, net.JoinHostPort(allowed.String(), port))
|
||||
}
|
||||
|
||||
// isPublicIP reports whether ip is a globally routable unicast address.
|
||||
// Rejects loopback, link-local, multicast, RFC1918, CGNAT, and the
|
||||
// AWS/GCP/Azure metadata IPs 169.254.169.254 / fd00:ec2::254 (these
|
||||
// already fall under link-local but spell it out for clarity).
|
||||
func isPublicIP(ip net.IP) bool {
|
||||
if ip == nil || ip.IsUnspecified() || ip.IsLoopback() ||
|
||||
ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
|
||||
ip.IsMulticast() || ip.IsPrivate() {
|
||||
return false
|
||||
}
|
||||
// 100.64.0.0/10 (CGNAT) is not covered by IsPrivate on older Go.
|
||||
if v4 := ip.To4(); v4 != nil {
|
||||
if v4[0] == 100 && v4[1] >= 64 && v4[1] <= 127 {
|
||||
return false
|
||||
}
|
||||
// 0.0.0.0/8 and friends.
|
||||
if v4[0] == 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ValidateURL returns nil if the URL is http(s) and parseable. It does
|
||||
// not resolve DNS — the dial step does that — but it does reject bare
|
||||
// schemes (file://, gopher://, etc.) before we even open a connection.
|
||||
func ValidateURL(raw string) error {
|
||||
u, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return fmt.Errorf("safehttp: unsupported scheme %q", u.Scheme)
|
||||
}
|
||||
if u.Host == "" {
|
||||
return errors.New("safehttp: empty host")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewClient returns an http.Client whose transport blocks non-public
|
||||
// destinations at dial time, caps redirects at 5, and re-validates each
|
||||
// redirect target's scheme. timeout is the per-request overall budget.
|
||||
func NewClient(timeout time.Duration) *http.Client {
|
||||
tr := &http.Transport{
|
||||
DialContext: safeDialContext,
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 32,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 5 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
ResponseHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
return &http.Client{
|
||||
Transport: tr,
|
||||
Timeout: timeout,
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 5 {
|
||||
return errors.New("safehttp: stopped after 5 redirects")
|
||||
}
|
||||
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
|
||||
return fmt.Errorf("safehttp: unsupported redirect scheme %q", req.URL.Scheme)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// LimitedBody wraps r in a reader that errors once more than max bytes have
|
||||
// been read. Use to cap how much of a response body downstream parsers
|
||||
// (goquery, image.Decode) will ever see — a hostile origin streaming an
|
||||
// endless body otherwise OOMs the process.
|
||||
func LimitedBody(r io.Reader, max int64) io.Reader {
|
||||
return &limitedReader{R: r, N: max}
|
||||
}
|
||||
|
||||
type limitedReader struct {
|
||||
R io.Reader
|
||||
N int64
|
||||
}
|
||||
|
||||
func (l *limitedReader) Read(p []byte) (int, error) {
|
||||
if l.N <= 0 {
|
||||
return 0, fmt.Errorf("safehttp: response body exceeded cap")
|
||||
}
|
||||
if int64(len(p)) > l.N {
|
||||
p = p[:l.N]
|
||||
}
|
||||
n, err := l.R.Read(p)
|
||||
l.N -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
6
main.go
6
main.go
@@ -406,8 +406,8 @@ func setupScheduledJobs(
|
||||
}
|
||||
})
|
||||
|
||||
// WOTD post at 08:00 (disabled by default; opt in via ENABLE_WOTD_POST=true)
|
||||
if strings.ToLower(os.Getenv("ENABLE_WOTD_POST")) == "true" {
|
||||
// WOTD post at 08:00
|
||||
if strings.ToLower(os.Getenv("DISABLE_WOTD_POST")) != "true" {
|
||||
c.AddFunc("0 8 * * *", func() {
|
||||
slog.Info("scheduler: posting WOTD")
|
||||
for _, r := range rooms {
|
||||
@@ -415,7 +415,7 @@ func setupScheduledJobs(
|
||||
}
|
||||
})
|
||||
} else {
|
||||
slog.Info("scheduler: WOTD daily post disabled (set ENABLE_WOTD_POST=true to enable)")
|
||||
slog.Info("scheduler: WOTD daily post disabled via DISABLE_WOTD_POST")
|
||||
}
|
||||
|
||||
// Game releases Monday 09:00
|
||||
|
||||
Reference in New Issue
Block a user