Fix paywall false positives + add og:image fallbacks

- Body extractor falls back to <article>/<main> container text when
  <p> extraction is sparse, catching <br>-separated bodies (Phoronix).
- Detect Cloudflare bot-block / JS-challenge pages on 403/503 and
  treat them as transport failures rather than paywalls (Brooklyn Vegan).
- og:image extractor falls back to img.wp-post-image and the first
  content <img> in <article>/<main>, with lazy-load placeholder
  handling via data-src / data-lazy-src (Hardcore Gaming 101).
- New -backfill-paywall flag re-checks paywalled=1 rows with the
  current logic, clearing false positives and filling missing thumbs.

internal/ingestion/article.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"strings"
 	"time"
@@ -152,8 +153,15 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
 
 	meta := ArticleMeta{Status: resp.StatusCode}
 	// 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall).
-	// We still try to parse the body for whatever signals it contains.
+	// But many sites 403 scrapers via Cloudflare while serving humans fine —
+	// peek at the body and treat a Cloudflare challenge as a transport failure
+	// rather than a paywall, so we don't stamp readable articles.
 	if resp.StatusCode != http.StatusOK {
+		if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusServiceUnavailable {
+			if isCloudflareChallenge(resp) {
+				return ArticleMeta{FetchError: true}
+			}
+		}
 		if resp.StatusCode == http.StatusPaymentRequired {
 			meta.Paywalled = true
 		}
@@ -172,6 +180,40 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
 	return meta
 }
 
+// isCloudflareChallenge reports whether a non-200 response is Cloudflare's
+// bot-block or JS-challenge page rather than a real publisher gate. The
+// signatures are stable: cf-* response headers, the cf-error-details body
+// markup, and a handful of recognizable titles. We read at most 8 KiB so a
+// huge body can't stall the poller.
+func isCloudflareChallenge(resp *http.Response) bool {
+	if resp == nil {
+		return false
+	}
+	if resp.Header.Get("Server") == "cloudflare" || resp.Header.Get("CF-RAY") != "" || resp.Header.Get("cf-mitigated") != "" {
+		// Header alone isn't enough — plenty of real publishers sit behind
+		// Cloudflare. Confirm with a body marker below.
+	}
+	buf, err := io.ReadAll(io.LimitReader(resp.Body, 8192))
+	if err != nil {
+		return false
+	}
+	s := strings.ToLower(string(buf))
+	markers := []string{
+		"/cdn-cgi/styles/cf",
+		"cf-error-details",
+		"attention required! | cloudflare",
+		"<title>just a moment",
+		"sorry, you have been blocked",
+		"cloudflare ray id",
+	}
+	for _, m := range markers {
+		if strings.Contains(s, m) {
+			return true
+		}
+	}
+	return false
+}
+
 // detectPaywall checks the page for explicit gating signals that publishers
 // expose for Google News and crawlers. We treat the article as paywalled if:
 //   - <meta name="article:content_tier" content="metered|locked"> (Conde Nast,
@@ -318,10 +360,26 @@ func FetchArticleBody(articleURL string) string {
 }
 
 func extractBodyText(doc *goquery.Document) string {
-	sel := doc.Find("article p, main p")
-	if sel.Length() == 0 {
-		sel = doc.Find("p")
+	out := joinParagraphText(doc.Find("article p, main p"))
+	if len(out) < PaywallBodyThreshold {
+		if alt := joinParagraphText(doc.Find("p")); len(alt) > len(out) {
+			out = alt
+		}
 	}
+	// Sites like Phoronix use <br>-separated text inside <article> rather
+	// than <p> tags. Fall back to the container's raw text in that case.
+	if len(out) < PaywallBodyThreshold {
+		if alt := containerText(doc.Find("article, main").First()); len(alt) > len(out) {
+			out = alt
+		}
+	}
+	if len(out) > MaxBodyChars {
+		out = out[:MaxBodyChars]
+	}
+	return out
+}
+
+func joinParagraphText(sel *goquery.Selection) string {
 	var b strings.Builder
 	sel.Each(func(_ int, s *goquery.Selection) {
 		t := strings.TrimSpace(s.Text())
@@ -332,15 +390,17 @@ func extractBodyText(doc *goquery.Document) string {
 			b.WriteString("\n\n")
 		}
 		b.WriteString(t)
-		if b.Len() >= MaxBodyChars {
-			return
-		}
 	})
-	out := strings.TrimSpace(b.String())
-	if len(out) > MaxBodyChars {
-		out = out[:MaxBodyChars]
+	return strings.TrimSpace(b.String())
+}
+
+func containerText(sel *goquery.Selection) string {
+	if sel.Length() == 0 {
+		return ""
 	}
-	return out
+	// Collapse whitespace runs so <br>-laden markup doesn't produce a wall
+	// of blank lines, but preserve enough structure to look like prose.
+	return strings.Join(strings.Fields(sel.Text()), " ")
 }
 
 func extractOGImage(doc *goquery.Document, base string) string {
@@ -356,26 +416,99 @@ func extractOGImage(doc *goquery.Document, base string) string {
 			return resolveURL(base, strings.TrimSpace(v))
 		}
 	}
+	// Sites like Hardcore Gaming 101 publish no og:image — fall back to the
+	// WordPress featured image (.wp-post-image), then the first reasonably-
+	// sized <img> inside <article>/<main>.
+	if v, ok := doc.Find(`img.wp-post-image`).First().Attr("src"); ok {
+		if url := pickImageSrc(doc.Find(`img.wp-post-image`).First(), v); url != "" {
+			return resolveURL(base, url)
+		}
+	}
+	var found string
+	doc.Find(`article img, main img`).EachWithBreak(func(_ int, s *goquery.Selection) bool {
+		raw, _ := s.Attr("src")
+		url := pickImageSrc(s, raw)
+		if url == "" || !looksLikeContentImage(s, url) {
+			return true
+		}
+		found = url
+		return false
+	})
+	if found != "" {
+		return resolveURL(base, found)
+	}
 	return ""
 }
 
-// extractBodyChars concatenates the text of <p> tags inside <article> or
-// <main>, falling back to all <p> tags, and returns the trimmed length.
-func extractBodyChars(doc *goquery.Document) int {
-	sel := doc.Find("article p, main p")
-	if sel.Length() == 0 {
-		sel = doc.Find("p")
+// pickImageSrc returns the most useful image URL from an <img> selection,
+// preferring data-src/data-lazy-src (lazy-load) over src when src is a
+// placeholder (empty or data: URI).
+func pickImageSrc(s *goquery.Selection, src string) string {
+	src = strings.TrimSpace(src)
+	if !strings.HasPrefix(src, "data:") && src != "" {
+		return src
 	}
-	var b strings.Builder
-	sel.Each(func(_ int, s *goquery.Selection) {
-		t := strings.TrimSpace(s.Text())
-		if t == "" {
-			return
+	for _, attr := range []string{"data-src", "data-lazy-src", "data-original"} {
+		if v, ok := s.Attr(attr); ok {
+			v = strings.TrimSpace(v)
+			if v != "" && !strings.HasPrefix(v, "data:") {
+				return v
+			}
 		}
-		if b.Len() > 0 {
-			b.WriteByte(' ')
-		}
-		b.WriteString(t)
-	})
-	return len(strings.TrimSpace(b.String()))
+	}
+	return ""
+}
+
+// looksLikeContentImage filters out obvious non-article images: gravatars,
+// social-share icons, tracking pixels. Anything with declared dimensions
+// under 100px on either side is treated as chrome.
+func looksLikeContentImage(s *goquery.Selection, url string) bool {
+	low := strings.ToLower(url)
+	for _, bad := range []string{"gravatar.com", "/avatar/", "/icons/", "share-", "pixel.gif", "sprite"} {
+		if strings.Contains(low, bad) {
+			return false
+		}
+	}
+	if w := atoiAttr(s, "width"); w > 0 && w < 100 {
+		return false
+	}
+	if h := atoiAttr(s, "height"); h > 0 && h < 100 {
+		return false
+	}
+	return true
+}
+
+func atoiAttr(s *goquery.Selection, name string) int {
+	v, ok := s.Attr(name)
+	if !ok {
+		return 0
+	}
+	n := 0
+	for _, c := range strings.TrimSpace(v) {
+		if c < '0' || c > '9' {
+			break
+		}
+		n = n*10 + int(c-'0')
+	}
+	return n
+}
+
+// extractBodyChars returns the visible body length used to gauge whether a
+// page looks like a real article. It tries <p> tags inside <article>/<main>,
+// then all <p> tags, then the raw text inside <article>/<main> — that last
+// fallback catches sites like Phoronix that separate paragraphs with <br>
+// instead of <p>.
+func extractBodyChars(doc *goquery.Document) int {
+	best := len(joinParagraphText(doc.Find("article p, main p")))
+	if best < PaywallBodyThreshold {
+		if n := len(joinParagraphText(doc.Find("p"))); n > best {
+			best = n
+		}
+	}
+	if best < PaywallBodyThreshold {
+		if n := len(containerText(doc.Find("article, main").First())); n > best {
+			best = n
+		}
+	}
+	return best
 }