drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden paywall detection + render paywalled stamp on cards

Browse Source 
Bypass-UA retry (Googlebot + Google referer) for soft paywalls, JSON-LD
gating scoped to Article-typed nodes, HTTP 402 treated as explicit
paywall, Wayback freshness filter (30d cap), archive.today as secondary
archive fallback, and transport failures no longer trigger snapshot
swaps. When gating is detected and no archive workaround succeeds, the
story is stored with paywalled=1 and the web card renders a diagonal
red rubber-stamp overlay so readers know the link is gated.
This commit is contained in:
prosolis
2026-05-25 11:23:22 -07:00
parent 9bc8743b5e
commit 509a0fc7a7
11 changed files with 316 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
internal/ingestion/article.go
View File

@@ -48,54 +48,128 @@ func resolveURL(base, ref string) string {
// as paywalled / gated, and the caller should fall back to an archive snapshot.
const PaywallBodyThreshold = 500
// googlebotUA is what many metered publishers grant first-click access to.
// Re-tried automatically when the default-UA fetch looks gated.
const googlebotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
// ArticleMeta is what we can learn from fetching an article page directly.
type ArticleMeta struct {
ImageURL string // og:image or twitter:image, absolute URL
BodyChars int // length of extracted visible body text
Fetched bool // true if we got an HTTP 200 with HTML
Paywalled bool // true if the page explicitly declares gated access
ImageURL string // og:image or twitter:image, absolute URL
BodyChars int // length of extracted visible body text
Fetched bool // true if we got an HTTP 200 with HTML
FetchError bool // true if the fetch failed at the network/HTTP layer
Paywalled bool // true if the page explicitly declares gated access
Status int // last HTTP status seen (0 on transport error)
}
// Gated reports whether the response carries a strong gating signal: an
// explicit paywall meta/JSON-LD declaration, an HTTP 402 Payment Required,
// or a 403 Forbidden after the bypass retry. A short body alone is not
// considered gating — that's a heuristic used by callers separately.
func (m ArticleMeta) Gated() bool {
if m.Paywalled {
return true
}
return m.Status == http.StatusPaymentRequired || m.Status == http.StatusForbidden
}
var articleClient = &http.Client{Timeout: 12 * time.Second}
// FetchArticleMeta fetches an article URL and returns its og:image plus the
// length of the visible body text (concatenation of <p> tags under <article>
// or <main>, falling back to all <p> tags). Returns Fetched=false on any
// network/HTTP failure so callers can branch on accessibility.
// FetchArticleMeta fetches an article URL with the default UA. If the result
// looks gated (explicit paywall signal, HTTP 402/403, or body too short) it
// retries once with a Googlebot UA + Google referer — the combination most
// metered publishers grant first-click access to. Returns the best of the
// two attempts.
func FetchArticleMeta(articleURL string) ArticleMeta {
if articleURL == "" {
return ArticleMeta{}
}
first := fetchArticleMetaOnce(articleURL, userAgent, "")
if !shouldRetryAsBot(first) {
return first
}
second := fetchArticleMetaOnce(articleURL, googlebotUA, "https://www.google.com/")
return pickBetter(first, second)
}
// shouldRetryAsBot returns true when the first attempt looks gated or too
// thin to be the real article body.
func shouldRetryAsBot(m ArticleMeta) bool {
if m.FetchError {
return false // transport failure won't be fixed by a different UA
}
if m.Gated() {
return true
}
if m.Fetched && m.BodyChars < PaywallBodyThreshold {
return true
}
return false
}
// pickBetter chooses the more useful of two fetch attempts: prefer the one
// that isn't gated, then the one with more body, then the one that fetched
// at all.
func pickBetter(a, b ArticleMeta) ArticleMeta {
aGated, bGated := a.Gated(), b.Gated()
if aGated != bGated {
if bGated {
return a
}
return b
}
if a.Fetched != b.Fetched {
if b.Fetched {
return b
}
return a
}
if b.BodyChars > a.BodyChars {
return b
}
return a
}
func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
if err != nil {
return ArticleMeta{}
return ArticleMeta{FetchError: true}
}
req.Header.Set("User-Agent", userAgent)
req.Header.Set("User-Agent", ua)
req.Header.Set("Accept", "text/html,application/xhtml+xml")
if referer != "" {
req.Header.Set("Referer", referer)
}
resp, err := articleClient.Do(req)
if err != nil {
return ArticleMeta{}
return ArticleMeta{FetchError: true}
}
defer resp.Body.Close()
meta := ArticleMeta{Status: resp.StatusCode}
// 402 is unambiguous gating; 403 often is too (e.g. NYT-style hard wall).
// We still try to parse the body for whatever signals it contains.
if resp.StatusCode != http.StatusOK {
return ArticleMeta{}
if resp.StatusCode == http.StatusPaymentRequired {
meta.Paywalled = true
}
return meta
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return ArticleMeta{}
return meta
}
return ArticleMeta{
ImageURL: extractOGImage(doc, articleURL),
BodyChars: extractBodyChars(doc),
Fetched: true,
Paywalled: detectPaywall(doc),
}
meta.Fetched = true
meta.ImageURL = extractOGImage(doc, articleURL)
meta.BodyChars = extractBodyChars(doc)
meta.Paywalled = detectPaywall(doc)
return meta
}
// detectPaywall checks the page for explicit gating signals that publishers
@@ -121,9 +195,11 @@ func detectPaywall(doc *goquery.Document) bool {
return gated
}
// jsonLDDeclaresGated returns true if the JSON-LD payload contains
// "isAccessibleForFree": false anywhere in its (possibly nested or arrayed)
// structure. Publishers vary wildly in shape, so we walk generically.
// jsonLDDeclaresGated returns true if the JSON-LD payload contains an
// Article-typed object (Article, NewsArticle, Report, BlogPosting, etc.)
// with "isAccessibleForFree" set falsy. We restrict to Article types so
// that embedded related-content or breadcrumb markup doesn't flip a free
// article to gated.
func jsonLDDeclaresGated(raw string) bool {
raw = strings.TrimSpace(raw)
if raw == "" {
@@ -136,21 +212,27 @@ func jsonLDDeclaresGated(raw string) bool {
return walkJSONLDForGated(v)
}
// articleSchemaTypes are the schema.org @type values we treat as "the main
// article" for paywall purposes.
var articleSchemaTypes = map[string]bool{
"article": true,
"newsarticle": true,
"report": true,
"reportagenewsarticle": true,
"blogposting": true,
"scholarlyarticle": true,
"techarticle": true,
"opinionnewsarticle": true,
"analysisnewsarticle": true,
"backgroundnewsarticle": true,
"reviewnewsarticle": true,
}
func walkJSONLDForGated(v any) bool {
switch x := v.(type) {
case map[string]any:
if raw, ok := x["isAccessibleForFree"]; ok {
switch r := raw.(type) {
case bool:
if !r {
return true
}
case string:
s := strings.ToLower(strings.TrimSpace(r))
if s == "false" || s == "no" {
return true
}
}
if isArticleType(x["@type"]) && declaresGated(x["isAccessibleForFree"]) {
return true
}
for _, vv := range x {
if walkJSONLDForGated(vv) {
@@ -167,6 +249,31 @@ func walkJSONLDForGated(v any) bool {
return false
}
func isArticleType(v any) bool {
switch t := v.(type) {
case string:
return articleSchemaTypes[strings.ToLower(strings.TrimSpace(t))]
case []any:
for _, e := range t {
if isArticleType(e) {
return true
}
}
}
return false
}
func declaresGated(v any) bool {
switch r := v.(type) {
case bool:
return !r
case string:
s := strings.ToLower(strings.TrimSpace(r))
return s == "false" || s == "no"
}
return false
}
// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
// that only care about the image. Returns "" when not found.
func FetchOGImage(articleURL string) string {