drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Drop LWN subscriber-only articles instead of stamping paywalled

Browse Source 
LWN's subscriber-only articles have no public version reachable by
archive snapshots or bypass UAs, so stamping them as paywalled produces
clicks that always dead-end. Detect the marker text and skip ingestion
entirely.
This commit is contained in:
prosolis
2026-05-25 14:31:08 -07:00
parent dd99f2c7ab
commit 83b1216541
3 changed files with 78 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
internal/ingestion/article.go
View File

@@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"net/url"
"strings" "strings"
"time" "time"
@@ -68,6 +69,11 @@ type ArticleMeta struct {
Fetched bool // true if we got an HTTP 200 with HTML Fetched bool // true if we got an HTTP 200 with HTML
FetchError bool // true if the fetch failed at the network/HTTP layer FetchError bool // true if the fetch failed at the network/HTTP layer
Paywalled bool // true if the page explicitly declares gated access Paywalled bool // true if the page explicitly declares gated access
// SubscriberOnly is true when the source publishes the article in a
// form that genuinely has no public version — no archive snapshot, no
// bypass UA, nothing. Callers should drop these from view entirely
// rather than stamp them as paywalled.
SubscriberOnly bool
Status int // last HTTP status seen (0 on transport error) Status int // last HTTP status seen (0 on transport error)
} }
@@ -107,6 +113,9 @@ func shouldRetryAsBot(m ArticleMeta) bool {
if m.FetchError { if m.FetchError {
return false // transport failure won't be fixed by a different UA return false // transport failure won't be fixed by a different UA
} }
if m.SubscriberOnly {
return false // genuinely subscriber-only; no UA changes that
}
if m.Gated() { if m.Gated() {
return true return true
} }
@@ -185,9 +194,27 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
meta.ImageURL = extractOGImage(doc, articleURL) meta.ImageURL = extractOGImage(doc, articleURL)
meta.BodyChars = extractBodyChars(doc) meta.BodyChars = extractBodyChars(doc)
meta.Paywalled = detectPaywall(doc) meta.Paywalled = detectPaywall(doc)
meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
return meta return meta
} }
// detectSubscriberOnly returns true for sources known to publish articles
// with no public version reachable by any workaround we have (archive
// snapshots, bypass UAs, etc). Currently only LWN's subscriber-only
// articles, which display a fixed marker in the article body.
func detectSubscriberOnly(articleURL string, doc *goquery.Document) bool {
u, err := url.Parse(articleURL)
if err != nil {
return false
}
host := strings.ToLower(u.Hostname())
if host != "lwn.net" && !strings.HasSuffix(host, ".lwn.net") {
return false
}
body := strings.ToLower(doc.Find("body").Text())
return strings.Contains(body, "available only to lwn subscribers")
}
// isCloudflareChallenge reports whether a non-200 response is Cloudflare's // isCloudflareChallenge reports whether a non-200 response is Cloudflare's
// bot-block or JS-challenge page rather than a real publisher gate. The // bot-block or JS-challenge page rather than a real publisher gate. The
// signatures are stable: cf-* response headers, the cf-error-details body // signatures are stable: cf-* response headers, the cf-error-details body

39
internal/ingestion/og_test.go
View File

@@ -104,3 +104,42 @@ func TestExtractBodyCharsANNLayout(t *testing.T) {
t.Fatalf("extractBodyChars = %d, want >= %d (ANN-style body was incorrectly treated as paywalled)", n, PaywallBodyThreshold) t.Fatalf("extractBodyChars = %d, want >= %d (ANN-style body was incorrectly treated as paywalled)", n, PaywallBodyThreshold)
} }
} }
func TestDetectSubscriberOnly(t *testing.T) {
cases := []struct {
name string
url string
body string
want bool
}{
{
name: "lwn subscriber-only marker",
url: "https://lwn.net/Articles/999999/",
body: `<html><body><p>The page you have requested is available only to LWN subscribers. Please log in.</p></body></html>`,
want: true,
},
{
name: "lwn free article",
url: "https://lwn.net/Articles/123456/",
body: `<html><body><article><p>Full free article body here.</p></article></body></html>`,
want: false,
},
{
name: "non-lwn host with same marker text is ignored",
url: "https://example.com/x",
body: `<html><body>available only to lwn subscribers</body></html>`,
want: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(tc.body))
if err != nil {
t.Fatalf("parse: %v", err)
}
if got := detectSubscriberOnly(tc.url, doc); got != tc.want {
t.Fatalf("detectSubscriberOnly = %v, want %v", got, tc.want)
}
})
}
}

11
internal/ingestion/poller.go
View File

@@ -137,6 +137,17 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
// swapping in a stale snapshot for a transient blip is worse than // swapping in a stale snapshot for a transient blip is worse than
// posting the live link. // posting the live link.
meta := FetchArticleMeta(originalURL) meta := FetchArticleMeta(originalURL)
// Some sources (e.g. LWN's subscriber-only articles) have no public
// version reachable by any workaround. Drop these from view entirely
// rather than stamping them as paywalled.
if meta.SubscriberOnly {
// No DB row written, so the next poll will re-fetch and re-detect
// this URL — fine in practice since LWN's feed is tiny and the
// alternative (a tombstone row) leaks into every story query.
slog.Info("dropping subscriber-only story",
"guid", items[i].GUID, "url", originalURL, "source", src.Name)
continue
}
gated := meta.Gated() || (meta.Fetched && meta.BodyChars < PaywallBodyThreshold) gated := meta.Gated() || (meta.Fetched && meta.BodyChars < PaywallBodyThreshold)
// paywalled tracks whether the link the user will click is still // paywalled tracks whether the link the user will click is still
// gated — i.e. gating was detected AND no archive workaround worked. // gated — i.e. gating was detected AND no archive workaround worked.