Fix SSRF, XSS, dedup, force-post, and DB hot-path issues
- Add internal/safehttp: hardened HTTP client (DNS-resolved dial guard blocking loopback/RFC1918/CGNAT/link-local, redirect re-validation, body-size cap) and rewire article/feed/thumb clients through it - Cap goquery body at 5 MiB so a hostile origin can't OOM the process - search.js: reject non-http(s) hrefs to block stored XSS via javascript: - dedup: tracking-param key "CMP" was unreachable (lookup lowercases); fixed to "cmp" so CMP= is actually stripped from canonical URLs - ForcePost: postItem now returns bool; on dedup-skip ForcePost returns false so !post falls back to DB lookup instead of silently consuming - Bound reaction callbacks behind an 8-slot semaphore; drop overflow - Add stories indexes on (channel, classified, seen_at DESC), (classified, seen_at DESC), and partial image_url to kill full scans in IsKnownImageURL and ORDER BY seen_at hot paths - Surface FTS5 probe Scan error instead of swallowing it
This commit is contained in:
@@ -92,7 +92,10 @@ func (q *Queue) Wait() {
|
||||
|
||||
// ForcePost pops the next queued item for the given channel and posts it
|
||||
// immediately, bypassing min-interval, burst cap, and daily cap. Dedup
|
||||
// (canonical URL cooldown) still applies. Returns true if an item was sent.
|
||||
// (canonical URL cooldown) still applies. Returns true only if a Matrix
|
||||
// message was actually sent — a dedup-skip returns false so the caller
|
||||
// can fall back (e.g., to a DB lookup) instead of silently consuming the
|
||||
// queued item with no user-visible result.
|
||||
func (q *Queue) ForcePost(channel string) bool {
|
||||
q.mu.Lock()
|
||||
items := q.queues[channel]
|
||||
@@ -106,8 +109,7 @@ func (q *Queue) ForcePost(channel string) bool {
|
||||
|
||||
slog.Info("force-posting story on demand",
|
||||
"guid", item.GUID, "channel", channel)
|
||||
q.postItem(item)
|
||||
return true
|
||||
return q.postItem(item)
|
||||
}
|
||||
|
||||
func (q *Queue) drain() {
|
||||
@@ -202,7 +204,11 @@ func (q *Queue) PostNow(item QueueItem) {
|
||||
q.postItem(item)
|
||||
}
|
||||
|
||||
func (q *Queue) postItem(item QueueItem) {
|
||||
// postItem returns true when a Matrix event was actually sent, false on
|
||||
// any non-success path (dedup-skip, transport failure with or without
|
||||
// retry, dead-letter). ForcePost uses the return value to decide whether
|
||||
// to acknowledge the user-initiated !post or fall back to a DB lookup.
|
||||
func (q *Queue) postItem(item QueueItem) bool {
|
||||
// Last-mile dedup: if this canonical URL was already posted to this channel
|
||||
// within the cooldown window, drop silently. Catches "same article, different
|
||||
// GUID across feeds" and any race where two items slipped past ingest dedup.
|
||||
@@ -215,7 +221,7 @@ func (q *Queue) postItem(item QueueItem) {
|
||||
"url_canonical", canonical,
|
||||
"cooldown_hours", q.config.DedupCooldownHours,
|
||||
)
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
story := &matrix.PostableStory{
|
||||
@@ -237,7 +243,7 @@ func (q *Queue) postItem(item QueueItem) {
|
||||
"channel", item.Channel,
|
||||
"err", err,
|
||||
)
|
||||
return
|
||||
return false
|
||||
}
|
||||
slog.Error("failed to post story, will retry",
|
||||
"guid", item.GUID,
|
||||
@@ -254,7 +260,7 @@ func (q *Queue) postItem(item QueueItem) {
|
||||
q.mu.Lock()
|
||||
q.queues[item.Channel] = append([]QueueItem{item}, q.queues[item.Channel]...)
|
||||
q.mu.Unlock()
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
// Record in post log (INSERT OR IGNORE via unique index)
|
||||
@@ -265,4 +271,5 @@ func (q *Queue) postItem(item QueueItem) {
|
||||
"channel", item.Channel,
|
||||
"event_id", eventID,
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user