Fix SSRF, XSS, dedup, force-post, and DB hot-path issues

- Add internal/safehttp: hardened HTTP client (DNS-resolved dial guard blocking loopback/RFC1918/CGNAT/link-local, redirect re-validation, body-size cap) and rewire article/feed/thumb clients through it - Cap goquery body at 5 MiB so a hostile origin can't OOM the process - search.js: reject non-http(s) hrefs to block stored XSS via javascript: - dedup: tracking-param key "CMP" was unreachable (lookup lowercases); fixed to "cmp" so CMP= is actually stripped from canonical URLs - ForcePost: postItem now returns bool; on dedup-skip ForcePost returns false so !post falls back to DB lookup instead of silently consuming - Bound reaction callbacks behind an 8-slot semaphore; drop overflow - Add stories indexes on (channel, classified, seen_at DESC), (classified, seen_at DESC), and partial image_url to kill full scans in IsKnownImageURL and ORDER BY seen_at hot paths - Surface FTS5 probe Scan error instead of swallowing it
2026-05-25 12:23:37 -07:00
parent e428f37c9e
commit 9e5ba5aafc
12 changed files with 234 additions and 19 deletions
--- a/internal/safehttp/safehttp.go
+++ b/internal/safehttp/safehttp.go
@@ -0,0 +1,147 @@
+// Package safehttp provides an http.Client hardened against SSRF and
+// memory-DoS via hostile upstreams. Every outbound fetch the bot makes
+// against feed-supplied URLs (RSS feeds, article pages, image hosts)
+// should go through one of these clients so a malicious feed can't
+// steer Pete at loopback, link-local, RFC1918, or cloud metadata IPs,
+// and can't OOM the process by streaming an unbounded body.
+package safehttp
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// ErrBlockedHost is returned when a URL resolves to a non-public IP.
+var ErrBlockedHost = errors.New("safehttp: blocked non-public host")
+
+// AllowPrivate, when true, disables the loopback/RFC1918 dial guard. It
+// exists for tests that spin up httptest.NewServer on 127.0.0.1 — never
+// set this in production.
+var AllowPrivate bool
+
+// safeDialContext refuses connections to non-public IPs. It runs after
+// DNS resolution, so a hostile DNS rebinding that returns 127.0.0.1
+// still gets blocked at dial time.
+func safeDialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+	ips, err := (&net.Resolver{}).LookupIP(ctx, "ip", host)
+	if err != nil {
+		return nil, err
+	}
+	var allowed net.IP
+	for _, ip := range ips {
+		if AllowPrivate || isPublicIP(ip) {
+			allowed = ip
+			break
+		}
+	}
+	if allowed == nil {
+		return nil, fmt.Errorf("%w: %s", ErrBlockedHost, host)
+	}
+	d := &net.Dialer{Timeout: 5 * time.Second, KeepAlive: 30 * time.Second}
+	return d.DialContext(ctx, network, net.JoinHostPort(allowed.String(), port))
+}
+
+// isPublicIP reports whether ip is a globally routable unicast address.
+// Rejects loopback, link-local, multicast, RFC1918, CGNAT, and the
+// AWS/GCP/Azure metadata IPs 169.254.169.254 / fd00:ec2::254 (these
+// already fall under link-local but spell it out for clarity).
+func isPublicIP(ip net.IP) bool {
+	if ip == nil || ip.IsUnspecified() || ip.IsLoopback() ||
+		ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
+		ip.IsMulticast() || ip.IsPrivate() {
+		return false
+	}
+	// 100.64.0.0/10 (CGNAT) is not covered by IsPrivate on older Go.
+	if v4 := ip.To4(); v4 != nil {
+		if v4[0] == 100 && v4[1] >= 64 && v4[1] <= 127 {
+			return false
+		}
+		// 0.0.0.0/8, 192.0.0.0/24, 192.0.2.0/24, 198.18.0.0/15, 198.51.100.0/24, 203.0.113.0/24
+		if v4[0] == 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// ValidateURL returns nil if the URL is http(s) and parseable. It does
+// not resolve DNS — the dial step does that — but it does reject bare
+// schemes (file://, gopher://, etc.) before we even open a connection.
+func ValidateURL(raw string) error {
+	u, err := url.Parse(strings.TrimSpace(raw))
+	if err != nil {
+		return err
+	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return fmt.Errorf("safehttp: unsupported scheme %q", u.Scheme)
+	}
+	if u.Host == "" {
+		return errors.New("safehttp: empty host")
+	}
+	return nil
+}
+
+// NewClient returns an http.Client whose transport blocks non-public
+// destinations at dial time, caps redirects at 5, and re-validates each
+// redirect target's scheme. timeout is the per-request overall budget.
+func NewClient(timeout time.Duration) *http.Client {
+	tr := &http.Transport{
+		DialContext:           safeDialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          32,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   5 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		ResponseHeaderTimeout: 10 * time.Second,
+	}
+	return &http.Client{
+		Transport: tr,
+		Timeout:   timeout,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) >= 5 {
+				return errors.New("safehttp: stopped after 5 redirects")
+			}
+			if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
+				return fmt.Errorf("safehttp: unsupported redirect scheme %q", req.URL.Scheme)
+			}
+			return nil
+		},
+	}
+}
+
+// LimitedBody wraps r in an io.LimitReader that returns io.ErrUnexpectedEOF
+// once more than max bytes have been read. Use to cap how much of a
+// response body downstream parsers (goquery, gofeed, image.Decode) will
+// ever see — a hostile origin streaming an endless body otherwise OOMs
+// the process.
+func LimitedBody(r io.Reader, max int64) io.Reader {
+	return &limitedReader{R: r, N: max}
+}
+
+type limitedReader struct {
+	R io.Reader
+	N int64
+}
+
+func (l *limitedReader) Read(p []byte) (int, error) {
+	if l.N <= 0 {
+		return 0, fmt.Errorf("safehttp: response body exceeded cap")
+	}
+	if int64(len(p)) > l.N {
+		p = p[:l.N]
+	}
+	n, err := l.R.Read(p)
+	l.N -= int64(n)
+	return n, err
+}