Fix paywall false positives + add og:image fallbacks

- Body extractor falls back to <article>/<main> container text when
  <p> extraction is sparse, catching <br>-separated bodies (Phoronix).
- Detect Cloudflare bot-block / JS-challenge pages on 403/503 and
  treat them as transport failures rather than paywalls (Brooklyn Vegan).
- og:image extractor falls back to img.wp-post-image and the first
  content <img> in <article>/<main>, with lazy-load placeholder
  handling via data-src / data-lazy-src (Hardcore Gaming 101).
- New -backfill-paywall flag re-checks paywalled=1 rows with the
  current logic, clearing false positives and filling missing thumbs.

internal/ingestion/article.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"strings"
 	"time"
@@ -152,8 +153,15 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
 
 	meta := ArticleMeta{Status: resp.StatusCode}
 	// 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall).
-	// We still try to parse the body for whatever signals it contains.
+	// But many sites 403 scrapers via Cloudflare while serving humans fine —
+	// peek at the body and treat a Cloudflare challenge as a transport failure
+	// rather than a paywall, so we don't stamp readable articles.
 	if resp.StatusCode != http.StatusOK {
+		if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusServiceUnavailable {
+			if isCloudflareChallenge(resp) {
+				return ArticleMeta{FetchError: true}
+			}
+		}
 		if resp.StatusCode == http.StatusPaymentRequired {
 			meta.Paywalled = true
 		}
@@ -172,6 +180,40 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
 	return meta
 }
 
+// isCloudflareChallenge reports whether a non-200 response is Cloudflare's
+// bot-block or JS-challenge page rather than a real publisher gate. The
+// signatures are stable: cf-* response headers, the cf-error-details body
+// markup, and a handful of recognizable titles. We read at most 8 KiB so a
+// huge body can't stall the poller.
+func isCloudflareChallenge(resp *http.Response) bool {
+	if resp == nil {
+		return false
+	}
+	if resp.Header.Get("Server") == "cloudflare" || resp.Header.Get("CF-RAY") != "" || resp.Header.Get("cf-mitigated") != "" {
+		// Header alone isn't enough — plenty of real publishers sit behind
+		// Cloudflare. Confirm with a body marker below.
+	}
+	buf, err := io.ReadAll(io.LimitReader(resp.Body, 8192))
+	if err != nil {
+		return false
+	}
+	s := strings.ToLower(string(buf))
+	markers := []string{
+		"/cdn-cgi/styles/cf",
+		"cf-error-details",
+		"attention required! | cloudflare",
+		"<title>just a moment",
+		"sorry, you have been blocked",
+		"cloudflare ray id",
+	}
+	for _, m := range markers {
+		if strings.Contains(s, m) {
+			return true
+		}
+	}
+	return false
+}
+
 // detectPaywall checks the page for explicit gating signals that publishers
 // expose for Google News and crawlers. We treat the article as paywalled if:
 //   - <meta name="article:content_tier" content="metered|locked"> (Conde Nast,
@@ -318,10 +360,26 @@ func FetchArticleBody(articleURL string) string {
 }
 
 func extractBodyText(doc *goquery.Document) string {
-	sel := doc.Find("article p, main p")
+	out := joinParagraphText(doc.Find("article p, main p"))
-	if sel.Length() == 0 {
+	if len(out) < PaywallBodyThreshold {
-		sel = doc.Find("p")
+		if alt := joinParagraphText(doc.Find("p")); len(alt) > len(out) {
+			out = alt
+		}
 	}
+	// Sites like Phoronix use <br>-separated text inside <article> rather
+	// than <p> tags. Fall back to the container's raw text in that case.
+	if len(out) < PaywallBodyThreshold {
+		if alt := containerText(doc.Find("article, main").First()); len(alt) > len(out) {
+			out = alt
+		}
+	}
+	if len(out) > MaxBodyChars {
+		out = out[:MaxBodyChars]
+	}
+	return out
+}
+
+func joinParagraphText(sel *goquery.Selection) string {
 	var b strings.Builder
 	sel.Each(func(_ int, s *goquery.Selection) {
 		t := strings.TrimSpace(s.Text())
@@ -332,15 +390,17 @@ func extractBodyText(doc *goquery.Document) string {
 			b.WriteString("\n\n")
 		}
 		b.WriteString(t)
-		if b.Len() >= MaxBodyChars {
-			return
-		}
 	})
-	out := strings.TrimSpace(b.String())
+	return strings.TrimSpace(b.String())
-	if len(out) > MaxBodyChars {
+}
-		out = out[:MaxBodyChars]
+
+func containerText(sel *goquery.Selection) string {
+	if sel.Length() == 0 {
+		return ""
 	}
-	return out
+	// Collapse whitespace runs so <br>-laden markup doesn't produce a wall
+	// of blank lines, but preserve enough structure to look like prose.
+	return strings.Join(strings.Fields(sel.Text()), " ")
 }
 
 func extractOGImage(doc *goquery.Document, base string) string {
@@ -356,26 +416,99 @@ func extractOGImage(doc *goquery.Document, base string) string {
 			return resolveURL(base, strings.TrimSpace(v))
 		}
 	}
+	// Sites like Hardcore Gaming 101 publish no og:image — fall back to the
+	// WordPress featured image (.wp-post-image), then the first reasonably-
+	// sized <img> inside <article>/<main>.
+	if v, ok := doc.Find(`img.wp-post-image`).First().Attr("src"); ok {
+		if url := pickImageSrc(doc.Find(`img.wp-post-image`).First(), v); url != "" {
+			return resolveURL(base, url)
+		}
+	}
+	var found string
+	doc.Find(`article img, main img`).EachWithBreak(func(_ int, s *goquery.Selection) bool {
+		raw, _ := s.Attr("src")
+		url := pickImageSrc(s, raw)
+		if url == "" || !looksLikeContentImage(s, url) {
+			return true
+		}
+		found = url
+		return false
+	})
+	if found != "" {
+		return resolveURL(base, found)
+	}
 	return ""
 }
 
-// extractBodyChars concatenates the text of <p> tags inside <article> or
+// pickImageSrc returns the most useful image URL from an <img> selection,
-// <main>, falling back to all <p> tags, and returns the trimmed length.
+// preferring data-src/data-lazy-src (lazy-load) over src when src is a
-func extractBodyChars(doc *goquery.Document) int {
+// placeholder (empty or data: URI).
-	sel := doc.Find("article p, main p")
+func pickImageSrc(s *goquery.Selection, src string) string {
-	if sel.Length() == 0 {
+	src = strings.TrimSpace(src)
-		sel = doc.Find("p")
+	if !strings.HasPrefix(src, "data:") && src != "" {
+		return src
 	}
-	var b strings.Builder
+	for _, attr := range []string{"data-src", "data-lazy-src", "data-original"} {
-	sel.Each(func(_ int, s *goquery.Selection) {
+		if v, ok := s.Attr(attr); ok {
-		t := strings.TrimSpace(s.Text())
+			v = strings.TrimSpace(v)
-		if t == "" {
+			if v != "" && !strings.HasPrefix(v, "data:") {
-			return
+				return v
+			}
 		}
-		if b.Len() > 0 {
+	}
-			b.WriteByte(' ')
+	return ""
-		}
+}
-		b.WriteString(t)
+
-	})
+// looksLikeContentImage filters out obvious non-article images: gravatars,
-	return len(strings.TrimSpace(b.String()))
+// social-share icons, tracking pixels. Anything with declared dimensions
+// under 100px on either side is treated as chrome.
+func looksLikeContentImage(s *goquery.Selection, url string) bool {
+	low := strings.ToLower(url)
+	for _, bad := range []string{"gravatar.com", "/avatar/", "/icons/", "share-", "pixel.gif", "sprite"} {
+		if strings.Contains(low, bad) {
+			return false
+		}
+	}
+	if w := atoiAttr(s, "width"); w > 0 && w < 100 {
+		return false
+	}
+	if h := atoiAttr(s, "height"); h > 0 && h < 100 {
+		return false
+	}
+	return true
+}
+
+func atoiAttr(s *goquery.Selection, name string) int {
+	v, ok := s.Attr(name)
+	if !ok {
+		return 0
+	}
+	n := 0
+	for _, c := range strings.TrimSpace(v) {
+		if c < '0' || c > '9' {
+			break
+		}
+		n = n*10 + int(c-'0')
+	}
+	return n
+}
+
+// extractBodyChars returns the visible body length used to gauge whether a
+// page looks like a real article. It tries <p> tags inside <article>/<main>,
+// then all <p> tags, then the raw text inside <article>/<main> — that last
+// fallback catches sites like Phoronix that separate paragraphs with <br>
+// instead of <p>.
+func extractBodyChars(doc *goquery.Document) int {
+	best := len(joinParagraphText(doc.Find("article p, main p")))
+	if best < PaywallBodyThreshold {
+		if n := len(joinParagraphText(doc.Find("p"))); n > best {
+			best = n
+		}
+	}
+	if best < PaywallBodyThreshold {
+		if n := len(containerText(doc.Find("article, main").First())); n > best {
+			best = n
+		}
+	}
+	return best
 }

main.go

@@ -27,6 +27,7 @@ func main() {
 	test := flag.Bool("test", false, "post one story to verify the full pipeline, then exit")
 	testSource := flag.String("test-source", "", "source name to use for -test (default: first enabled)")
 	local := flag.Bool("local", false, "web/RSS-only mode: poll feeds and serve the web UI, no Matrix login or posting")
+	backfillPaywall := flag.Bool("backfill-paywall", false, "re-check paywalled rows with current detection logic; clear false positives and fill missing thumbnails, then exit")
 	flag.Parse()
 
 	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
@@ -57,6 +58,11 @@ func main() {
 		return
 	}
 
+	if *backfillPaywall {
+		runBackfillPaywall()
+		return
+	}
+
 	// Test mode: post one story to verify the full pipeline, then exit
 	if *test {
 		runTest(cfg, *testSource)
@@ -373,3 +379,50 @@ func runTest(cfg *config.Config, sourceName string) {
 func timeNow() int64 {
 	return time.Now().Unix()
 }
+
+// runBackfillPaywall re-runs the current paywall detection logic against
+// every row marked paywalled=1. For each row we re-fetch the stored article
+// URL; if the new logic no longer considers it gated we clear the flag and,
+// when the row has no thumbnail, populate it from the live og:image.
+func runBackfillPaywall() {
+	rows, err := storage.Get().Query(`SELECT guid, article_url, image_url FROM stories WHERE paywalled = 1`)
+	if err != nil {
+		slog.Error("backfill: query failed", "err", err)
+		os.Exit(1)
+	}
+	type rec struct{ guid, url, img string }
+	var pending []rec
+	for rows.Next() {
+		var r rec
+		if err := rows.Scan(&r.guid, &r.url, &r.img); err != nil {
+			slog.Error("backfill: scan failed", "err", err)
+			continue
+		}
+		pending = append(pending, r)
+	}
+	rows.Close()
+	slog.Info("backfill: starting", "candidates", len(pending))
+
+	cleared, imaged, stillGated := 0, 0, 0
+	for _, r := range pending {
+		meta := ingestion.FetchArticleMeta(r.url)
+		gated := meta.Gated() || (meta.Fetched && meta.BodyChars < ingestion.PaywallBodyThreshold)
+		if gated {
+			stillGated++
+			slog.Info("backfill: still gated", "guid", r.guid, "status", meta.Status, "body_chars", meta.BodyChars)
+			continue
+		}
+		newImg := r.img
+		if newImg == "" && meta.ImageURL != "" {
+			newImg = meta.ImageURL
+			imaged++
+		}
+		if _, err := storage.Get().Exec(`UPDATE stories SET paywalled = 0, image_url = ? WHERE guid = ?`, newImg, r.guid); err != nil {
+			slog.Error("backfill: update failed", "guid", r.guid, "err", err)
+			continue
+		}
+		cleared++
+		slog.Info("backfill: cleared paywall flag", "guid", r.guid, "filled_image", newImg != r.img)
+	}
+	slog.Info("backfill complete", "cleared", cleared, "still_gated", stillGated, "thumbnails_filled", imaged)
+}