drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden paywall detection + render paywalled stamp on cards

Browse Source 
Bypass-UA retry (Googlebot + Google referer) for soft paywalls, JSON-LD
gating scoped to Article-typed nodes, HTTP 402 treated as explicit
paywall, Wayback freshness filter (30d cap), archive.today as secondary
archive fallback, and transport failures no longer trigger snapshot
swaps. When gating is detected and no archive workaround succeeds, the
story is stored with paywalled=1 and the web card renders a diagonal
red rubber-stamp overlay so readers know the link is gated.
This commit is contained in:
prosolis
2026-05-25 11:23:22 -07:00
parent 9bc8743b5e
commit 509a0fc7a7
11 changed files with 316 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
internal/ingestion/article.go
View File

@@ -48,54 +48,128 @@ func resolveURL(base, ref string) string {
// as paywalled / gated, and the caller should fall back to an archive snapshot.
const PaywallBodyThreshold = 500
// googlebotUA is what many metered publishers grant first-click access to.
// Re-tried automatically when the default-UA fetch looks gated.
const googlebotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
// ArticleMeta is what we can learn from fetching an article page directly.
type ArticleMeta struct {
ImageURL string // og:image or twitter:image, absolute URL
BodyChars int // length of extracted visible body text
Fetched bool // true if we got an HTTP 200 with HTML
Paywalled bool // true if the page explicitly declares gated access
ImageURL string // og:image or twitter:image, absolute URL
BodyChars int // length of extracted visible body text
Fetched bool // true if we got an HTTP 200 with HTML
FetchError bool // true if the fetch failed at the network/HTTP layer
Paywalled bool // true if the page explicitly declares gated access
Status int // last HTTP status seen (0 on transport error)
}
// Gated reports whether the response carries a strong gating signal: an
// explicit paywall meta/JSON-LD declaration, an HTTP 402 Payment Required,
// or a 403 Forbidden after the bypass retry. A short body alone is not
// considered gating — that's a heuristic used by callers separately.
func (m ArticleMeta) Gated() bool {
if m.Paywalled {
return true
}
return m.Status == http.StatusPaymentRequired || m.Status == http.StatusForbidden
}
var articleClient = &http.Client{Timeout: 12 * time.Second}
// FetchArticleMeta fetches an article URL and returns its og:image plus the
// length of the visible body text (concatenation of <p> tags under <article>
// or <main>, falling back to all <p> tags). Returns Fetched=false on any
// network/HTTP failure so callers can branch on accessibility.
// FetchArticleMeta fetches an article URL with the default UA. If the result
// looks gated (explicit paywall signal, HTTP 402/403, or body too short) it
// retries once with a Googlebot UA + Google referer — the combination most
// metered publishers grant first-click access to. Returns the best of the
// two attempts.
func FetchArticleMeta(articleURL string) ArticleMeta {
if articleURL == "" {
return ArticleMeta{}
}
first := fetchArticleMetaOnce(articleURL, userAgent, "")
if !shouldRetryAsBot(first) {
return first
}
second := fetchArticleMetaOnce(articleURL, googlebotUA, "https://www.google.com/")
return pickBetter(first, second)
}
// shouldRetryAsBot returns true when the first attempt looks gated or too
// thin to be the real article body.
func shouldRetryAsBot(m ArticleMeta) bool {
if m.FetchError {
return false // transport failure won't be fixed by a different UA
}
if m.Gated() {
return true
}
if m.Fetched && m.BodyChars < PaywallBodyThreshold {
return true
}
return false
}
// pickBetter chooses the more useful of two fetch attempts: prefer the one
// that isn't gated, then the one with more body, then the one that fetched
// at all.
func pickBetter(a, b ArticleMeta) ArticleMeta {
aGated, bGated := a.Gated(), b.Gated()
if aGated != bGated {
if bGated {
return a
}
return b
}
if a.Fetched != b.Fetched {
if b.Fetched {
return b
}
return a
}
if b.BodyChars > a.BodyChars {
return b
}
return a
}
func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
if err != nil {
return ArticleMeta{}
return ArticleMeta{FetchError: true}
}
req.Header.Set("User-Agent", userAgent)
req.Header.Set("User-Agent", ua)
req.Header.Set("Accept", "text/html,application/xhtml+xml")
if referer != "" {
req.Header.Set("Referer", referer)
}
resp, err := articleClient.Do(req)
if err != nil {
return ArticleMeta{}
return ArticleMeta{FetchError: true}
}
defer resp.Body.Close()
meta := ArticleMeta{Status: resp.StatusCode}
// 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall).
// We still try to parse the body for whatever signals it contains.
if resp.StatusCode != http.StatusOK {
return ArticleMeta{}
if resp.StatusCode == http.StatusPaymentRequired {
meta.Paywalled = true
}
return meta
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return ArticleMeta{}
return meta
}
return ArticleMeta{
ImageURL: extractOGImage(doc, articleURL),
BodyChars: extractBodyChars(doc),
Fetched: true,
Paywalled: detectPaywall(doc),
}
meta.Fetched = true
meta.ImageURL = extractOGImage(doc, articleURL)
meta.BodyChars = extractBodyChars(doc)
meta.Paywalled = detectPaywall(doc)
return meta
}
// detectPaywall checks the page for explicit gating signals that publishers
@@ -121,9 +195,11 @@ func detectPaywall(doc *goquery.Document) bool {
return gated
}
// jsonLDDeclaresGated returns true if the JSON-LD payload contains
// "isAccessibleForFree": false anywhere in its (possibly nested or arrayed)
// structure. Publishers vary wildly in shape, so we walk generically.
// jsonLDDeclaresGated returns true if the JSON-LD payload contains an
// Article-typed object (Article, NewsArticle, Report, BlogPosting, etc.)
// with "isAccessibleForFree" set falsy. We restrict to Article types so
// that embedded related-content or breadcrumb markup doesn't flip a free
// article to gated.
func jsonLDDeclaresGated(raw string) bool {
raw = strings.TrimSpace(raw)
if raw == "" {
@@ -136,21 +212,27 @@ func jsonLDDeclaresGated(raw string) bool {
return walkJSONLDForGated(v)
}
// articleSchemaTypes are the schema.org @type values we treat as "the main
// article" for paywall purposes.
var articleSchemaTypes = map[string]bool{
"article": true,
"newsarticle": true,
"report": true,
"reportagenewsarticle": true,
"blogposting": true,
"scholarlyarticle": true,
"techarticle": true,
"opinionnewsarticle": true,
"analysisnewsarticle": true,
"backgroundnewsarticle": true,
"reviewnewsarticle": true,
}
func walkJSONLDForGated(v any) bool {
switch x := v.(type) {
case map[string]any:
if raw, ok := x["isAccessibleForFree"]; ok {
switch r := raw.(type) {
case bool:
if !r {
return true
}
case string:
s := strings.ToLower(strings.TrimSpace(r))
if s == "false" || s == "no" {
return true
}
}
if isArticleType(x["@type"]) && declaresGated(x["isAccessibleForFree"]) {
return true
}
for _, vv := range x {
if walkJSONLDForGated(vv) {
@@ -167,6 +249,31 @@ func walkJSONLDForGated(v any) bool {
return false
}
func isArticleType(v any) bool {
switch t := v.(type) {
case string:
return articleSchemaTypes[strings.ToLower(strings.TrimSpace(t))]
case []any:
for _, e := range t {
if isArticleType(e) {
return true
}
}
}
return false
}
func declaresGated(v any) bool {
switch r := v.(type) {
case bool:
return !r
case string:
s := strings.ToLower(strings.TrimSpace(r))
return s == "false" || s == "no"
}
return false
}
// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
// that only care about the image. Returns "" when not found.
func FetchOGImage(articleURL string) string {

48
internal/ingestion/poller.go
View File

@@ -126,32 +126,47 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
continue
}
// Genuinely new story — fetch the article page once. We use it for:
// Genuinely new story — fetch the article page once (with an internal
// bypass-UA retry on gating signals). We use it for:
// 1. og:image fallback when the feed didn't give us one.
// 2. Paywall detection: visible body text below the threshold means
// the article is gated, so swap in a Wayback snapshot for both
// Pete (image) and the reader (posted link).
// 2. Paywall detection: explicit gating signals or a body too short
// to be a real article means we swap in an archive snapshot for
// both Pete (image) and the reader (posted link).
//
// Pure transport failures (FetchError) are NOT treated as gating —
// swapping in a stale snapshot for a transient blip is worse than
// posting the live link.
meta := FetchArticleMeta(originalURL)
paywalled := !meta.Fetched || meta.Paywalled || meta.BodyChars < PaywallBodyThreshold
if paywalled {
if snap := ResolveWayback(originalURL); snap != "" {
gated := meta.Gated() || (meta.Fetched && meta.BodyChars < PaywallBodyThreshold)
// paywalled tracks whether the link the user will click is still
// gated — i.e. gating was detected AND no archive workaround worked.
// When a snapshot swap succeeds, the reader gets a readable page, so
// we don't stamp it.
paywalled := false
if gated {
workedAround := false
if snap := resolveArchive(originalURL); snap != "" {
snapMeta := FetchArticleMeta(snap)
if snapMeta.Fetched {
items[i].ArticleURL = snap
if items[i].ImageURL == "" && snapMeta.ImageURL != "" {
items[i].ImageURL = snapMeta.ImageURL
}
slog.Info("paywall detected, using wayback snapshot",
workedAround = true
slog.Info("paywall detected, using archive snapshot",
"guid", items[i].GUID, "original", originalURL,
"snapshot", snap, "body_chars", meta.BodyChars)
"snapshot", snap, "body_chars", meta.BodyChars,
"status", meta.Status)
} else {
slog.Debug("paywall detected but snapshot fetch failed",
"guid", items[i].GUID, "original", originalURL, "snapshot", snap)
}
} else {
slog.Debug("paywall detected but no wayback snapshot available",
"guid", items[i].GUID, "url", originalURL, "body_chars", meta.BodyChars)
slog.Debug("paywall detected but no archive snapshot available",
"guid", items[i].GUID, "url", originalURL,
"body_chars", meta.BodyChars, "status", meta.Status)
}
paywalled = !workedAround
} else if items[i].ImageURL == "" && meta.ImageURL != "" {
items[i].ImageURL = meta.ImageURL
slog.Debug("og:image fallback used",
@@ -172,6 +187,7 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
URLCanonical: canonical,
HeadlineNorm: headlineNorm,
Source: items[i].Source,
Paywalled: paywalled,
SeenAt: time.Now().Unix(),
}); err != nil {
slog.Error("failed to insert story", "guid", items[i].GUID, "err", err)
@@ -188,3 +204,13 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
return nil
}
// resolveArchive tries Wayback first (well-behaved availability API, fresh
// snapshots filtered) and falls back to archive.today, which often has
// captures Wayback doesn't — especially for hard-walled publishers.
func resolveArchive(articleURL string) string {
if snap := ResolveWayback(articleURL); snap != "" {
return snap
}
return ResolveArchiveToday(articleURL)
}

88
internal/ingestion/wayback.go
View File

@@ -5,29 +5,40 @@ import (
"encoding/json"
"net/http"
"net/url"
"strings"
"time"
)
var waybackClient = &http.Client{Timeout: 10 * time.Second}
// maxSnapshotAge bounds how stale a Wayback snapshot can be before we treat
// it as useless for a freshly-published news article. The "closest" snapshot
// returned by the availability API can otherwise be years old.
const maxSnapshotAge = 30 * 24 * time.Hour
type waybackResp struct {
ArchivedSnapshots struct {
Closest struct {
Available bool `json:"available"`
URL string `json:"url"`
Status string `json:"status"`
Timestamp string `json:"timestamp"` // YYYYMMDDhhmmss
} `json:"closest"`
} `json:"archived_snapshots"`
}
// ResolveWayback asks the Internet Archive's Wayback availability API for the
// closest snapshot of the given URL. Returns the snapshot URL (https) or ""
// if no snapshot exists or the request fails.
// ResolveWayback asks the Internet Archive's Wayback availability API for
// the closest snapshot of the given URL. Returns the snapshot URL (https)
// or "" if no snapshot exists, the request fails, or the closest snapshot
// is older than maxSnapshotAge.
func ResolveWayback(articleURL string) string {
if articleURL == "" {
return ""
}
api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL)
// Anchor the lookup to "now" so we get the freshest snapshot rather
// than whichever happens to be Wayback's default closest match.
api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL) +
"&timestamp=" + time.Now().UTC().Format("20060102150405")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -54,9 +65,76 @@ func ResolveWayback(articleURL string) string {
if !snap.Available || snap.Status != "200" || snap.URL == "" {
return ""
}
if !snapshotFreshEnough(snap.Timestamp) {
return ""
}
// Wayback sometimes returns http:// even when https is available.
if len(snap.URL) > 7 && snap.URL[:7] == "http://" {
if strings.HasPrefix(snap.URL, "http://") {
return "https://" + snap.URL[7:]
}
return snap.URL
}
func snapshotFreshEnough(ts string) bool {
if ts == "" {
return false
}
t, err := time.Parse("20060102150405", ts)
if err != nil {
return false
}
return time.Since(t) <= maxSnapshotAge
}
var archiveTodayClient = &http.Client{
Timeout: 10 * time.Second,
// Don't follow the redirect — we just want the Location header pointing
// at the most recent capture.
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
}
// ResolveArchiveToday looks up the newest archive.today / archive.ph snapshot
// for the given URL. archive.ph has no public availability API, but its
// `/newest/<url>` endpoint redirects (HTTP 302) to the most recent capture
// when one exists, or returns a non-redirect response otherwise. Returns
// "" on any failure or when no snapshot exists.
func ResolveArchiveToday(articleURL string) string {
if articleURL == "" {
return ""
}
api := "https://archive.ph/newest/" + articleURL
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", api, nil)
if err != nil {
return ""
}
req.Header.Set("User-Agent", userAgent)
resp, err := archiveTodayClient.Do(req)
if err != nil {
return ""
}
defer resp.Body.Close()
if resp.StatusCode < 300 || resp.StatusCode >= 400 {
return ""
}
loc := strings.TrimSpace(resp.Header.Get("Location"))
if loc == "" {
return ""
}
// archive.ph occasionally returns a relative Location; absolutize it.
if strings.HasPrefix(loc, "/") {
loc = "https://archive.ph" + loc
}
// Bounce back the input as Location means "no snapshot, here's the form"
// — distinguish a real capture URL (contains /YYYY/ or a short hash path).
if strings.Contains(loc, "://archive.ph/") && !strings.Contains(loc, "://archive.ph/newest/") {
return loc
}
return ""
}