Fix SSRF, XSS, dedup, force-post, and DB hot-path issues
- Add internal/safehttp: hardened HTTP client (DNS-resolved dial guard blocking loopback/RFC1918/CGNAT/link-local, redirect re-validation, body-size cap) and rewire article/feed/thumb clients through it - Cap goquery body at 5 MiB so a hostile origin can't OOM the process - search.js: reject non-http(s) hrefs to block stored XSS via javascript: - dedup: tracking-param key "CMP" was unreachable (lookup lowercases); fixed to "cmp" so CMP= is actually stripped from canonical URLs - ForcePost: postItem now returns bool; on dedup-skip ForcePost returns false so !post falls back to DB lookup instead of silently consuming - Bound reaction callbacks behind an 8-slot semaphore; drop overflow - Add stories indexes on (channel, classified, seen_at DESC), (classified, seen_at DESC), and partial image_url to kill full scans in IsKnownImageURL and ORDER BY seen_at hot paths - Surface FTS5 probe Scan error instead of swallowing it
This commit is contained in:
@@ -10,8 +10,16 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
|
||||
"pete/internal/safehttp"
|
||||
)
|
||||
|
||||
// articleBodyMax bounds how much of an article response goquery will
|
||||
// ever buffer. ~5 MiB comfortably fits any real news article while
|
||||
// preventing a hostile origin from OOMing the process by streaming an
|
||||
// endless body within the request timeout window.
|
||||
const articleBodyMax = 5 << 20
|
||||
|
||||
// resolveURL turns a possibly-relative URL into an absolute one using
|
||||
// the base URL. Returns the raw input on parse failure.
|
||||
func resolveURL(base, ref string) string {
|
||||
@@ -74,7 +82,7 @@ func (m ArticleMeta) Gated() bool {
|
||||
return m.Status == http.StatusPaymentRequired || m.Status == http.StatusForbidden
|
||||
}
|
||||
|
||||
var articleClient = &http.Client{Timeout: 12 * time.Second}
|
||||
var articleClient = safehttp.NewClient(12 * time.Second)
|
||||
|
||||
// FetchArticleMeta fetches an article URL with the default UA. If the result
|
||||
// looks gated (explicit paywall signal, HTTP 402/403, or body too short) it
|
||||
@@ -168,7 +176,7 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
|
||||
return meta
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
doc, err := goquery.NewDocumentFromReader(safehttp.LimitedBody(resp.Body, articleBodyMax))
|
||||
if err != nil {
|
||||
return meta
|
||||
}
|
||||
@@ -352,7 +360,7 @@ func FetchArticleBody(articleURL string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
doc, err := goquery.NewDocumentFromReader(safehttp.LimitedBody(resp.Body, articleBodyMax))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -5,8 +5,16 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"pete/internal/safehttp"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// httptest servers bind to 127.0.0.1; allow the safe-dial check to
|
||||
// accept them in tests only.
|
||||
safehttp.AllowPrivate = true
|
||||
}
|
||||
|
||||
func TestFetchOGImage(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"html"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -12,11 +11,13 @@ import (
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
ext "github.com/mmcdole/gofeed/extensions"
|
||||
|
||||
"pete/internal/safehttp"
|
||||
)
|
||||
|
||||
const userAgent = "Pete/1.0 (newsbot; +https://github.com/reala-misaki/pete)"
|
||||
|
||||
var feedClient = &http.Client{Timeout: 30 * time.Second}
|
||||
var feedClient = safehttp.NewClient(30 * time.Second)
|
||||
|
||||
// FeedItem represents a parsed RSS item ready for routing.
|
||||
type FeedItem struct {
|
||||
|
||||
Reference in New Issue
Block a user