From b8dcaa2aa16c1ebbdbb7d045907dd8d75bf371bc Mon Sep 17 00:00:00 2001 From: prosolis <5590409+prosolis@users.noreply.github.com> Date: Mon, 25 May 2026 12:06:29 -0700 Subject: [PATCH] Fix paywall false positives + add og:image fallbacks - Body extractor falls back to
/
container text when

extraction is sparse, catching
-separated bodies (Phoronix). - Detect Cloudflare bot-block / JS-challenge pages on 403/503 and treat them as transport failures rather than paywalls (Brooklyn Vegan). - og:image extractor falls back to img.wp-post-image and the first content in

/
, with lazy-load placeholder handling via data-src / data-lazy-src (Hardcore Gaming 101). - New -backfill-paywall flag re-checks paywalled=1 rows with the current logic, clearing false positives and filling missing thumbs. --- internal/ingestion/article.go | 189 +++++++++++++++++++++++++++++----- main.go | 53 ++++++++++ 2 files changed, 214 insertions(+), 28 deletions(-) diff --git a/internal/ingestion/article.go b/internal/ingestion/article.go index 1efe39a..eea678b 100644 --- a/internal/ingestion/article.go +++ b/internal/ingestion/article.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "strings" "time" @@ -152,8 +153,15 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta { meta := ArticleMeta{Status: resp.StatusCode} // 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall). - // We still try to parse the body for whatever signals it contains. + // But many sites 403 scrapers via Cloudflare while serving humans fine — + // peek at the body and treat a Cloudflare challenge as a transport failure + // rather than a paywall, so we don't stamp readable articles. if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusServiceUnavailable { + if isCloudflareChallenge(resp) { + return ArticleMeta{FetchError: true} + } + } if resp.StatusCode == http.StatusPaymentRequired { meta.Paywalled = true } @@ -172,6 +180,40 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta { return meta } +// isCloudflareChallenge reports whether a non-200 response is Cloudflare's +// bot-block or JS-challenge page rather than a real publisher gate. The +// signatures are stable: cf-* response headers, the cf-error-details body +// markup, and a handful of recognizable titles. We read at most 8 KiB so a +// huge body can't stall the poller. +func isCloudflareChallenge(resp *http.Response) bool { + if resp == nil { + return false + } + if resp.Header.Get("Server") == "cloudflare" || resp.Header.Get("CF-RAY") != "" || resp.Header.Get("cf-mitigated") != "" { + // Header alone isn't enough — plenty of real publishers sit behind + // Cloudflare. Confirm with a body marker below. + } + buf, err := io.ReadAll(io.LimitReader(resp.Body, 8192)) + if err != nil { + return false + } + s := strings.ToLower(string(buf)) + markers := []string{ + "/cdn-cgi/styles/cf", + "cf-error-details", + "attention required! | cloudflare", + "just a moment", + "sorry, you have been blocked", + "cloudflare ray id", + } + for _, m := range markers { + if strings.Contains(s, m) { + return true + } + } + return false +} + // detectPaywall checks the page for explicit gating signals that publishers // expose for Google News and crawlers. We treat the article as paywalled if: // - <meta name="article:content_tier" content="metered|locked"> (Conde Nast, @@ -318,10 +360,26 @@ func FetchArticleBody(articleURL string) string { } func extractBodyText(doc *goquery.Document) string { - sel := doc.Find("article p, main p") - if sel.Length() == 0 { - sel = doc.Find("p") + out := joinParagraphText(doc.Find("article p, main p")) + if len(out) < PaywallBodyThreshold { + if alt := joinParagraphText(doc.Find("p")); len(alt) > len(out) { + out = alt + } } + // Sites like Phoronix use <br>-separated text inside <article> rather + // than <p> tags. Fall back to the container's raw text in that case. + if len(out) < PaywallBodyThreshold { + if alt := containerText(doc.Find("article, main").First()); len(alt) > len(out) { + out = alt + } + } + if len(out) > MaxBodyChars { + out = out[:MaxBodyChars] + } + return out +} + +func joinParagraphText(sel *goquery.Selection) string { var b strings.Builder sel.Each(func(_ int, s *goquery.Selection) { t := strings.TrimSpace(s.Text()) @@ -332,15 +390,17 @@ func extractBodyText(doc *goquery.Document) string { b.WriteString("\n\n") } b.WriteString(t) - if b.Len() >= MaxBodyChars { - return - } }) - out := strings.TrimSpace(b.String()) - if len(out) > MaxBodyChars { - out = out[:MaxBodyChars] + return strings.TrimSpace(b.String()) +} + +func containerText(sel *goquery.Selection) string { + if sel.Length() == 0 { + return "" } - return out + // Collapse whitespace runs so <br>-laden markup doesn't produce a wall + // of blank lines, but preserve enough structure to look like prose. + return strings.Join(strings.Fields(sel.Text()), " ") } func extractOGImage(doc *goquery.Document, base string) string { @@ -356,26 +416,99 @@ func extractOGImage(doc *goquery.Document, base string) string { return resolveURL(base, strings.TrimSpace(v)) } } + // Sites like Hardcore Gaming 101 publish no og:image — fall back to the + // WordPress featured image (.wp-post-image), then the first reasonably- + // sized <img> inside <article>/<main>. + if v, ok := doc.Find(`img.wp-post-image`).First().Attr("src"); ok { + if url := pickImageSrc(doc.Find(`img.wp-post-image`).First(), v); url != "" { + return resolveURL(base, url) + } + } + var found string + doc.Find(`article img, main img`).EachWithBreak(func(_ int, s *goquery.Selection) bool { + raw, _ := s.Attr("src") + url := pickImageSrc(s, raw) + if url == "" || !looksLikeContentImage(s, url) { + return true + } + found = url + return false + }) + if found != "" { + return resolveURL(base, found) + } return "" } -// extractBodyChars concatenates the text of <p> tags inside <article> or -// <main>, falling back to all <p> tags, and returns the trimmed length. -func extractBodyChars(doc *goquery.Document) int { - sel := doc.Find("article p, main p") - if sel.Length() == 0 { - sel = doc.Find("p") +// pickImageSrc returns the most useful image URL from an <img> selection, +// preferring data-src/data-lazy-src (lazy-load) over src when src is a +// placeholder (empty or data: URI). +func pickImageSrc(s *goquery.Selection, src string) string { + src = strings.TrimSpace(src) + if !strings.HasPrefix(src, "data:") && src != "" { + return src } - var b strings.Builder - sel.Each(func(_ int, s *goquery.Selection) { - t := strings.TrimSpace(s.Text()) - if t == "" { - return + for _, attr := range []string{"data-src", "data-lazy-src", "data-original"} { + if v, ok := s.Attr(attr); ok { + v = strings.TrimSpace(v) + if v != "" && !strings.HasPrefix(v, "data:") { + return v + } } - if b.Len() > 0 { - b.WriteByte(' ') - } - b.WriteString(t) - }) - return len(strings.TrimSpace(b.String())) + } + return "" +} + +// looksLikeContentImage filters out obvious non-article images: gravatars, +// social-share icons, tracking pixels. Anything with declared dimensions +// under 100px on either side is treated as chrome. +func looksLikeContentImage(s *goquery.Selection, url string) bool { + low := strings.ToLower(url) + for _, bad := range []string{"gravatar.com", "/avatar/", "/icons/", "share-", "pixel.gif", "sprite"} { + if strings.Contains(low, bad) { + return false + } + } + if w := atoiAttr(s, "width"); w > 0 && w < 100 { + return false + } + if h := atoiAttr(s, "height"); h > 0 && h < 100 { + return false + } + return true +} + +func atoiAttr(s *goquery.Selection, name string) int { + v, ok := s.Attr(name) + if !ok { + return 0 + } + n := 0 + for _, c := range strings.TrimSpace(v) { + if c < '0' || c > '9' { + break + } + n = n*10 + int(c-'0') + } + return n +} + +// extractBodyChars returns the visible body length used to gauge whether a +// page looks like a real article. It tries <p> tags inside <article>/<main>, +// then all <p> tags, then the raw text inside <article>/<main> — that last +// fallback catches sites like Phoronix that separate paragraphs with <br> +// instead of <p>. +func extractBodyChars(doc *goquery.Document) int { + best := len(joinParagraphText(doc.Find("article p, main p"))) + if best < PaywallBodyThreshold { + if n := len(joinParagraphText(doc.Find("p"))); n > best { + best = n + } + } + if best < PaywallBodyThreshold { + if n := len(containerText(doc.Find("article, main").First())); n > best { + best = n + } + } + return best } diff --git a/main.go b/main.go index 0e87cd9..c9485ac 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,7 @@ func main() { test := flag.Bool("test", false, "post one story to verify the full pipeline, then exit") testSource := flag.String("test-source", "", "source name to use for -test (default: first enabled)") local := flag.Bool("local", false, "web/RSS-only mode: poll feeds and serve the web UI, no Matrix login or posting") + backfillPaywall := flag.Bool("backfill-paywall", false, "re-check paywalled rows with current detection logic; clear false positives and fill missing thumbnails, then exit") flag.Parse() slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ @@ -57,6 +58,11 @@ func main() { return } + if *backfillPaywall { + runBackfillPaywall() + return + } + // Test mode: post one story to verify the full pipeline, then exit if *test { runTest(cfg, *testSource) @@ -373,3 +379,50 @@ func runTest(cfg *config.Config, sourceName string) { func timeNow() int64 { return time.Now().Unix() } + +// runBackfillPaywall re-runs the current paywall detection logic against +// every row marked paywalled=1. For each row we re-fetch the stored article +// URL; if the new logic no longer considers it gated we clear the flag and, +// when the row has no thumbnail, populate it from the live og:image. +func runBackfillPaywall() { + rows, err := storage.Get().Query(`SELECT guid, article_url, image_url FROM stories WHERE paywalled = 1`) + if err != nil { + slog.Error("backfill: query failed", "err", err) + os.Exit(1) + } + type rec struct{ guid, url, img string } + var pending []rec + for rows.Next() { + var r rec + if err := rows.Scan(&r.guid, &r.url, &r.img); err != nil { + slog.Error("backfill: scan failed", "err", err) + continue + } + pending = append(pending, r) + } + rows.Close() + slog.Info("backfill: starting", "candidates", len(pending)) + + cleared, imaged, stillGated := 0, 0, 0 + for _, r := range pending { + meta := ingestion.FetchArticleMeta(r.url) + gated := meta.Gated() || (meta.Fetched && meta.BodyChars < ingestion.PaywallBodyThreshold) + if gated { + stillGated++ + slog.Info("backfill: still gated", "guid", r.guid, "status", meta.Status, "body_chars", meta.BodyChars) + continue + } + newImg := r.img + if newImg == "" && meta.ImageURL != "" { + newImg = meta.ImageURL + imaged++ + } + if _, err := storage.Get().Exec(`UPDATE stories SET paywalled = 0, image_url = ? WHERE guid = ?`, newImg, r.guid); err != nil { + slog.Error("backfill: update failed", "guid", r.guid, "err", err) + continue + } + cleared++ + slog.Info("backfill: cleared paywall flag", "guid", r.guid, "filled_image", newImg != r.img) + } + slog.Info("backfill complete", "cleared", cleared, "still_gated", stillGated, "thumbnails_filled", imaged) +}