drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden paywall detection + render paywalled stamp on cards

Browse Source 
Bypass-UA retry (Googlebot + Google referer) for soft paywalls, JSON-LD
gating scoped to Article-typed nodes, HTTP 402 treated as explicit
paywall, Wayback freshness filter (30d cap), archive.today as secondary
archive fallback, and transport failures no longer trigger snapshot
swaps. When gating is detected and no archive workaround succeeds, the
story is stored with paywalled=1 and the web card renders a diagonal
red rubber-stamp overlay so readers know the link is gated.
This commit is contained in:
prosolis
2026-05-25 11:23:22 -07:00
parent 9bc8743b5e
commit 509a0fc7a7
11 changed files with 316 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
internal/ingestion/poller.go
View File

@@ -126,32 +126,47 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
continue
}
// Genuinely new story — fetch the article page once. We use it for:
// Genuinely new story — fetch the article page once (with an internal
// bypass-UA retry on gating signals). We use it for:
// 1. og:image fallback when the feed didn't give us one.
// 2. Paywall detection: visible body text below the threshold means
// the article is gated, so swap in a Wayback snapshot for both
// Pete (image) and the reader (posted link).
// 2. Paywall detection: explicit gating signals or a body too short
// to be a real article means we swap in an archive snapshot for
// both Pete (image) and the reader (posted link).
//
// Pure transport failures (FetchError) are NOT treated as gating —
// swapping in a stale snapshot for a transient blip is worse than
// posting the live link.
meta := FetchArticleMeta(originalURL)
paywalled := !meta.Fetched || meta.Paywalled || meta.BodyChars < PaywallBodyThreshold
if paywalled {
if snap := ResolveWayback(originalURL); snap != "" {
gated := meta.Gated() || (meta.Fetched && meta.BodyChars < PaywallBodyThreshold)
// paywalled tracks whether the link the user will click is still
// gated — i.e. gating was detected AND no archive workaround worked.
// When a snapshot swap succeeds, the reader gets a readable page, so
// we don't stamp it.
paywalled := false
if gated {
workedAround := false
if snap := resolveArchive(originalURL); snap != "" {
snapMeta := FetchArticleMeta(snap)
if snapMeta.Fetched {
items[i].ArticleURL = snap
if items[i].ImageURL == "" && snapMeta.ImageURL != "" {
items[i].ImageURL = snapMeta.ImageURL
}
slog.Info("paywall detected, using wayback snapshot",
workedAround = true
slog.Info("paywall detected, using archive snapshot",
"guid", items[i].GUID, "original", originalURL,
"snapshot", snap, "body_chars", meta.BodyChars)
"snapshot", snap, "body_chars", meta.BodyChars,
"status", meta.Status)
} else {
slog.Debug("paywall detected but snapshot fetch failed",
"guid", items[i].GUID, "original", originalURL, "snapshot", snap)
}
} else {
slog.Debug("paywall detected but no wayback snapshot available",
"guid", items[i].GUID, "url", originalURL, "body_chars", meta.BodyChars)
slog.Debug("paywall detected but no archive snapshot available",
"guid", items[i].GUID, "url", originalURL,
"body_chars", meta.BodyChars, "status", meta.Status)
}
paywalled = !workedAround
} else if items[i].ImageURL == "" && meta.ImageURL != "" {
items[i].ImageURL = meta.ImageURL
slog.Debug("og:image fallback used",
@@ -172,6 +187,7 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
URLCanonical: canonical,
HeadlineNorm: headlineNorm,
Source: items[i].Source,
Paywalled: paywalled,
SeenAt: time.Now().Unix(),
}); err != nil {
slog.Error("failed to insert story", "guid", items[i].GUID, "err", err)
@@ -188,3 +204,13 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
return nil
}
// resolveArchive tries Wayback first (well-behaved availability API, fresh
// snapshots filtered) and falls back to archive.today, which often has
// captures Wayback doesn't — especially for hard-walled publishers.
func resolveArchive(articleURL string) string {
if snap := ResolveWayback(articleURL); snap != "" {
return snap
}
return ResolveArchiveToday(articleURL)
}