Fix SSRF, XSS, dedup, force-post, and DB hot-path issues

- Add internal/safehttp: hardened HTTP client (DNS-resolved dial guard
  blocking loopback/RFC1918/CGNAT/link-local, redirect re-validation,
  body-size cap) and rewire article/feed/thumb clients through it
- Cap goquery body at 5 MiB so a hostile origin can't OOM the process
- search.js: reject non-http(s) hrefs to block stored XSS via javascript:
- dedup: tracking-param key "CMP" was unreachable (lookup lowercases);
  fixed to "cmp" so CMP= is actually stripped from canonical URLs
- ForcePost: postItem now returns bool; on dedup-skip ForcePost returns
  false so !post falls back to DB lookup instead of silently consuming
- Bound reaction callbacks behind an 8-slot semaphore; drop overflow
- Add stories indexes on (channel, classified, seen_at DESC),
  (classified, seen_at DESC), and partial image_url to kill full scans
  in IsKnownImageURL and ORDER BY seen_at hot paths
- Surface FTS5 probe Scan error instead of swallowing it
This commit is contained in:
prosolis
2026-05-25 12:23:37 -07:00
parent e428f37c9e
commit 9e5ba5aafc
12 changed files with 234 additions and 19 deletions

View File

@@ -92,7 +92,10 @@ func (q *Queue) Wait() {
// ForcePost pops the next queued item for the given channel and posts it
// immediately, bypassing min-interval, burst cap, and daily cap. Dedup
// (canonical URL cooldown) still applies. Returns true if an item was sent.
// (canonical URL cooldown) still applies. Returns true only if a Matrix
// message was actually sent — a dedup-skip returns false so the caller
// can fall back (e.g., to a DB lookup) instead of silently consuming the
// queued item with no user-visible result.
func (q *Queue) ForcePost(channel string) bool {
q.mu.Lock()
items := q.queues[channel]
@@ -106,8 +109,7 @@ func (q *Queue) ForcePost(channel string) bool {
slog.Info("force-posting story on demand",
"guid", item.GUID, "channel", channel)
q.postItem(item)
return true
return q.postItem(item)
}
func (q *Queue) drain() {
@@ -202,7 +204,11 @@ func (q *Queue) PostNow(item QueueItem) {
q.postItem(item)
}
func (q *Queue) postItem(item QueueItem) {
// postItem returns true when a Matrix event was actually sent, false on
// any non-success path (dedup-skip, transport failure with or without
// retry, dead-letter). ForcePost uses the return value to decide whether
// to acknowledge the user-initiated !post or fall back to a DB lookup.
func (q *Queue) postItem(item QueueItem) bool {
// Last-mile dedup: if this canonical URL was already posted to this channel
// within the cooldown window, drop silently. Catches "same article, different
// GUID across feeds" and any race where two items slipped past ingest dedup.
@@ -215,7 +221,7 @@ func (q *Queue) postItem(item QueueItem) {
"url_canonical", canonical,
"cooldown_hours", q.config.DedupCooldownHours,
)
return
return false
}
story := &matrix.PostableStory{
@@ -237,7 +243,7 @@ func (q *Queue) postItem(item QueueItem) {
"channel", item.Channel,
"err", err,
)
return
return false
}
slog.Error("failed to post story, will retry",
"guid", item.GUID,
@@ -254,7 +260,7 @@ func (q *Queue) postItem(item QueueItem) {
q.mu.Lock()
q.queues[item.Channel] = append([]QueueItem{item}, q.queues[item.Channel]...)
q.mu.Unlock()
return
return false
}
// Record in post log (INSERT OR IGNORE via unique index)
@@ -265,4 +271,5 @@ func (q *Queue) postItem(item QueueItem) {
"channel", item.Channel,
"event_id", eventID,
)
return true
}