Fix paywall false positives + add og:image fallbacks
- Body extractor falls back to <article>/<main> container text when <p> extraction is sparse, catching <br>-separated bodies (Phoronix). - Detect Cloudflare bot-block / JS-challenge pages on 403/503 and treat them as transport failures rather than paywalls (Brooklyn Vegan). - og:image extractor falls back to img.wp-post-image and the first content <img> in <article>/<main>, with lazy-load placeholder handling via data-src / data-lazy-src (Hardcore Gaming 101). - New -backfill-paywall flag re-checks paywalled=1 rows with the current logic, clearing false positives and filling missing thumbs.
This commit is contained in:
prosolis
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -152,8 +153,15 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
|
||||
|
||||
meta := ArticleMeta{Status: resp.StatusCode}
|
||||
// 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall).
|
||||
// We still try to parse the body for whatever signals it contains.
|
||||
// But many sites 403 scrapers via Cloudflare while serving humans fine —
|
||||
// peek at the body and treat a Cloudflare challenge as a transport failure
|
||||
// rather than a paywall, so we don't stamp readable articles.
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusServiceUnavailable {
|
||||
if isCloudflareChallenge(resp) {
|
||||
return ArticleMeta{FetchError: true}
|
||||
}
|
||||
}
|
||||
if resp.StatusCode == http.StatusPaymentRequired {
|
||||
meta.Paywalled = true
|
||||
}
|
||||
@@ -172,6 +180,40 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
|
||||
return meta
|
||||
}
|
||||
|
||||
// isCloudflareChallenge reports whether a non-200 response is Cloudflare's
|
||||
// bot-block or JS-challenge page rather than a real publisher gate. The
|
||||
// signatures are stable: cf-* response headers, the cf-error-details body
|
||||
// markup, and a handful of recognizable titles. We read at most 8 KiB so a
|
||||
// huge body can't stall the poller.
|
||||
func isCloudflareChallenge(resp *http.Response) bool {
|
||||
if resp == nil {
|
||||
return false
|
||||
}
|
||||
if resp.Header.Get("Server") == "cloudflare" || resp.Header.Get("CF-RAY") != "" || resp.Header.Get("cf-mitigated") != "" {
|
||||
// Header alone isn't enough — plenty of real publishers sit behind
|
||||
// Cloudflare. Confirm with a body marker below.
|
||||
}
|
||||
buf, err := io.ReadAll(io.LimitReader(resp.Body, 8192))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s := strings.ToLower(string(buf))
|
||||
markers := []string{
|
||||
"/cdn-cgi/styles/cf",
|
||||
"cf-error-details",
|
||||
"attention required! | cloudflare",
|
||||
"<title>just a moment",
|
||||
"sorry, you have been blocked",
|
||||
"cloudflare ray id",
|
||||
}
|
||||
for _, m := range markers {
|
||||
if strings.Contains(s, m) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// detectPaywall checks the page for explicit gating signals that publishers
|
||||
// expose for Google News and crawlers. We treat the article as paywalled if:
|
||||
// - <meta name="article:content_tier" content="metered|locked"> (Conde Nast,
|
||||
@@ -318,10 +360,26 @@ func FetchArticleBody(articleURL string) string {
|
||||
}
|
||||
|
||||
func extractBodyText(doc *goquery.Document) string {
|
||||
sel := doc.Find("article p, main p")
|
||||
if sel.Length() == 0 {
|
||||
sel = doc.Find("p")
|
||||
out := joinParagraphText(doc.Find("article p, main p"))
|
||||
if len(out) < PaywallBodyThreshold {
|
||||
if alt := joinParagraphText(doc.Find("p")); len(alt) > len(out) {
|
||||
out = alt
|
||||
}
|
||||
}
|
||||
// Sites like Phoronix use <br>-separated text inside <article> rather
|
||||
// than <p> tags. Fall back to the container's raw text in that case.
|
||||
if len(out) < PaywallBodyThreshold {
|
||||
if alt := containerText(doc.Find("article, main").First()); len(alt) > len(out) {
|
||||
out = alt
|
||||
}
|
||||
}
|
||||
if len(out) > MaxBodyChars {
|
||||
out = out[:MaxBodyChars]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func joinParagraphText(sel *goquery.Selection) string {
|
||||
var b strings.Builder
|
||||
sel.Each(func(_ int, s *goquery.Selection) {
|
||||
t := strings.TrimSpace(s.Text())
|
||||
@@ -332,15 +390,17 @@ func extractBodyText(doc *goquery.Document) string {
|
||||
b.WriteString("\n\n")
|
||||
}
|
||||
b.WriteString(t)
|
||||
if b.Len() >= MaxBodyChars {
|
||||
return
|
||||
}
|
||||
})
|
||||
out := strings.TrimSpace(b.String())
|
||||
if len(out) > MaxBodyChars {
|
||||
out = out[:MaxBodyChars]
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
return out
|
||||
|
||||
func containerText(sel *goquery.Selection) string {
|
||||
if sel.Length() == 0 {
|
||||
return ""
|
||||
}
|
||||
// Collapse whitespace runs so <br>-laden markup doesn't produce a wall
|
||||
// of blank lines, but preserve enough structure to look like prose.
|
||||
return strings.Join(strings.Fields(sel.Text()), " ")
|
||||
}
|
||||
|
||||
func extractOGImage(doc *goquery.Document, base string) string {
|
||||
@@ -356,26 +416,99 @@ func extractOGImage(doc *goquery.Document, base string) string {
|
||||
return resolveURL(base, strings.TrimSpace(v))
|
||||
}
|
||||
}
|
||||
// Sites like Hardcore Gaming 101 publish no og:image — fall back to the
|
||||
// WordPress featured image (.wp-post-image), then the first reasonably-
|
||||
// sized <img> inside <article>/<main>.
|
||||
if v, ok := doc.Find(`img.wp-post-image`).First().Attr("src"); ok {
|
||||
if url := pickImageSrc(doc.Find(`img.wp-post-image`).First(), v); url != "" {
|
||||
return resolveURL(base, url)
|
||||
}
|
||||
}
|
||||
var found string
|
||||
doc.Find(`article img, main img`).EachWithBreak(func(_ int, s *goquery.Selection) bool {
|
||||
raw, _ := s.Attr("src")
|
||||
url := pickImageSrc(s, raw)
|
||||
if url == "" || !looksLikeContentImage(s, url) {
|
||||
return true
|
||||
}
|
||||
found = url
|
||||
return false
|
||||
})
|
||||
if found != "" {
|
||||
return resolveURL(base, found)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractBodyChars concatenates the text of <p> tags inside <article> or
|
||||
// <main>, falling back to all <p> tags, and returns the trimmed length.
|
||||
// pickImageSrc returns the most useful image URL from an <img> selection,
|
||||
// preferring data-src/data-lazy-src (lazy-load) over src when src is a
|
||||
// placeholder (empty or data: URI).
|
||||
func pickImageSrc(s *goquery.Selection, src string) string {
|
||||
src = strings.TrimSpace(src)
|
||||
if !strings.HasPrefix(src, "data:") && src != "" {
|
||||
return src
|
||||
}
|
||||
for _, attr := range []string{"data-src", "data-lazy-src", "data-original"} {
|
||||
if v, ok := s.Attr(attr); ok {
|
||||
v = strings.TrimSpace(v)
|
||||
if v != "" && !strings.HasPrefix(v, "data:") {
|
||||
return v
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// looksLikeContentImage filters out obvious non-article images: gravatars,
|
||||
// social-share icons, tracking pixels. Anything with declared dimensions
|
||||
// under 100px on either side is treated as chrome.
|
||||
func looksLikeContentImage(s *goquery.Selection, url string) bool {
|
||||
low := strings.ToLower(url)
|
||||
for _, bad := range []string{"gravatar.com", "/avatar/", "/icons/", "share-", "pixel.gif", "sprite"} {
|
||||
if strings.Contains(low, bad) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if w := atoiAttr(s, "width"); w > 0 && w < 100 {
|
||||
return false
|
||||
}
|
||||
if h := atoiAttr(s, "height"); h > 0 && h < 100 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func atoiAttr(s *goquery.Selection, name string) int {
|
||||
v, ok := s.Attr(name)
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, c := range strings.TrimSpace(v) {
|
||||
if c < '0' || c > '9' {
|
||||
break
|
||||
}
|
||||
n = n*10 + int(c-'0')
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// extractBodyChars returns the visible body length used to gauge whether a
|
||||
// page looks like a real article. It tries <p> tags inside <article>/<main>,
|
||||
// then all <p> tags, then the raw text inside <article>/<main> — that last
|
||||
// fallback catches sites like Phoronix that separate paragraphs with <br>
|
||||
// instead of <p>.
|
||||
func extractBodyChars(doc *goquery.Document) int {
|
||||
sel := doc.Find("article p, main p")
|
||||
if sel.Length() == 0 {
|
||||
sel = doc.Find("p")
|
||||
best := len(joinParagraphText(doc.Find("article p, main p")))
|
||||
if best < PaywallBodyThreshold {
|
||||
if n := len(joinParagraphText(doc.Find("p"))); n > best {
|
||||
best = n
|
||||
}
|
||||
var b strings.Builder
|
||||
sel.Each(func(_ int, s *goquery.Selection) {
|
||||
t := strings.TrimSpace(s.Text())
|
||||
if t == "" {
|
||||
return
|
||||
}
|
||||
if b.Len() > 0 {
|
||||
b.WriteByte(' ')
|
||||
if best < PaywallBodyThreshold {
|
||||
if n := len(containerText(doc.Find("article, main").First())); n > best {
|
||||
best = n
|
||||
}
|
||||
b.WriteString(t)
|
||||
})
|
||||
return len(strings.TrimSpace(b.String()))
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
53
main.go
53
main.go
@@ -27,6 +27,7 @@ func main() {
|
||||
test := flag.Bool("test", false, "post one story to verify the full pipeline, then exit")
|
||||
testSource := flag.String("test-source", "", "source name to use for -test (default: first enabled)")
|
||||
local := flag.Bool("local", false, "web/RSS-only mode: poll feeds and serve the web UI, no Matrix login or posting")
|
||||
backfillPaywall := flag.Bool("backfill-paywall", false, "re-check paywalled rows with current detection logic; clear false positives and fill missing thumbnails, then exit")
|
||||
flag.Parse()
|
||||
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
||||
@@ -57,6 +58,11 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
if *backfillPaywall {
|
||||
runBackfillPaywall()
|
||||
return
|
||||
}
|
||||
|
||||
// Test mode: post one story to verify the full pipeline, then exit
|
||||
if *test {
|
||||
runTest(cfg, *testSource)
|
||||
@@ -373,3 +379,50 @@ func runTest(cfg *config.Config, sourceName string) {
|
||||
func timeNow() int64 {
|
||||
return time.Now().Unix()
|
||||
}
|
||||
|
||||
// runBackfillPaywall re-runs the current paywall detection logic against
|
||||
// every row marked paywalled=1. For each row we re-fetch the stored article
|
||||
// URL; if the new logic no longer considers it gated we clear the flag and,
|
||||
// when the row has no thumbnail, populate it from the live og:image.
|
||||
func runBackfillPaywall() {
|
||||
rows, err := storage.Get().Query(`SELECT guid, article_url, image_url FROM stories WHERE paywalled = 1`)
|
||||
if err != nil {
|
||||
slog.Error("backfill: query failed", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
type rec struct{ guid, url, img string }
|
||||
var pending []rec
|
||||
for rows.Next() {
|
||||
var r rec
|
||||
if err := rows.Scan(&r.guid, &r.url, &r.img); err != nil {
|
||||
slog.Error("backfill: scan failed", "err", err)
|
||||
continue
|
||||
}
|
||||
pending = append(pending, r)
|
||||
}
|
||||
rows.Close()
|
||||
slog.Info("backfill: starting", "candidates", len(pending))
|
||||
|
||||
cleared, imaged, stillGated := 0, 0, 0
|
||||
for _, r := range pending {
|
||||
meta := ingestion.FetchArticleMeta(r.url)
|
||||
gated := meta.Gated() || (meta.Fetched && meta.BodyChars < ingestion.PaywallBodyThreshold)
|
||||
if gated {
|
||||
stillGated++
|
||||
slog.Info("backfill: still gated", "guid", r.guid, "status", meta.Status, "body_chars", meta.BodyChars)
|
||||
continue
|
||||
}
|
||||
newImg := r.img
|
||||
if newImg == "" && meta.ImageURL != "" {
|
||||
newImg = meta.ImageURL
|
||||
imaged++
|
||||
}
|
||||
if _, err := storage.Get().Exec(`UPDATE stories SET paywalled = 0, image_url = ? WHERE guid = ?`, newImg, r.guid); err != nil {
|
||||
slog.Error("backfill: update failed", "guid", r.guid, "err", err)
|
||||
continue
|
||||
}
|
||||
cleared++
|
||||
slog.Info("backfill: cleared paywall flag", "guid", r.guid, "filled_image", newImg != r.img)
|
||||
}
|
||||
slog.Info("backfill complete", "cleared", cleared, "still_gated", stillGated, "thumbnails_filled", imaged)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user