Fix SSRF, XSS, dedup, force-post, and DB hot-path issues

- Add internal/safehttp: hardened HTTP client (DNS-resolved dial guard
  blocking loopback/RFC1918/CGNAT/link-local, redirect re-validation,
  body-size cap) and rewire article/feed/thumb clients through it
- Cap goquery body at 5 MiB so a hostile origin can't OOM the process
- search.js: reject non-http(s) hrefs to block stored XSS via javascript:
- dedup: tracking-param key "CMP" was unreachable (lookup lowercases);
  fixed to "cmp" so CMP= is actually stripped from canonical URLs
- ForcePost: postItem now returns bool; on dedup-skip ForcePost returns
  false so !post falls back to DB lookup instead of silently consuming
- Bound reaction callbacks behind an 8-slot semaphore; drop overflow
- Add stories indexes on (channel, classified, seen_at DESC),
  (classified, seen_at DESC), and partial image_url to kill full scans
  in IsKnownImageURL and ORDER BY seen_at hot paths
- Surface FTS5 probe Scan error instead of swallowing it
This commit is contained in:
prosolis
2026-05-25 12:23:37 -07:00
parent e428f37c9e
commit 9e5ba5aafc
12 changed files with 234 additions and 19 deletions

View File

@@ -10,8 +10,16 @@ import (
"time"
"github.com/PuerkitoBio/goquery"
"pete/internal/safehttp"
)
// articleBodyMax bounds how much of an article response goquery will
// ever buffer. ~5 MiB comfortably fits any real news article while
// preventing a hostile origin from OOMing the process by streaming an
// endless body within the request timeout window.
const articleBodyMax = 5 << 20
// resolveURL turns a possibly-relative URL into an absolute one using
// the base URL. Returns the raw input on parse failure.
func resolveURL(base, ref string) string {
@@ -74,7 +82,7 @@ func (m ArticleMeta) Gated() bool {
return m.Status == http.StatusPaymentRequired || m.Status == http.StatusForbidden
}
var articleClient = &http.Client{Timeout: 12 * time.Second}
var articleClient = safehttp.NewClient(12 * time.Second)
// FetchArticleMeta fetches an article URL with the default UA. If the result
// looks gated (explicit paywall signal, HTTP 402/403, or body too short) it
@@ -168,7 +176,7 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
return meta
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
doc, err := goquery.NewDocumentFromReader(safehttp.LimitedBody(resp.Body, articleBodyMax))
if err != nil {
return meta
}
@@ -352,7 +360,7 @@ func FetchArticleBody(articleURL string) string {
return ""
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
doc, err := goquery.NewDocumentFromReader(safehttp.LimitedBody(resp.Body, articleBodyMax))
if err != nil {
return ""
}