From 83b121654146d0dfb328b2d270db09bcc6c13e72 Mon Sep 17 00:00:00 2001
From: prosolis <5590409+prosolis@users.noreply.github.com>
Date: Mon, 25 May 2026 14:31:08 -0700
Subject: [PATCH] Drop LWN subscriber-only articles instead of stamping
paywalled
LWN's subscriber-only articles have no public version reachable by
archive snapshots or bypass UAs, so stamping them as paywalled produces
clicks that always dead-end. Detect the marker text and skip ingestion
entirely.
---
internal/ingestion/article.go | 29 +++++++++++++++++++++++++-
internal/ingestion/og_test.go | 39 +++++++++++++++++++++++++++++++++++
internal/ingestion/poller.go | 11 ++++++++++
3 files changed, 78 insertions(+), 1 deletion(-)
diff --git a/internal/ingestion/article.go b/internal/ingestion/article.go
index 8d84345..0a36c8e 100644
--- a/internal/ingestion/article.go
+++ b/internal/ingestion/article.go
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net/http"
+ "net/url"
"strings"
"time"
@@ -68,7 +69,12 @@ type ArticleMeta struct {
Fetched bool // true if we got an HTTP 200 with HTML
FetchError bool // true if the fetch failed at the network/HTTP layer
Paywalled bool // true if the page explicitly declares gated access
- Status int // last HTTP status seen (0 on transport error)
+ // SubscriberOnly is true when the source publishes the article in a
+ // form that genuinely has no public version — no archive snapshot, no
+ // bypass UA, nothing. Callers should drop these from view entirely
+ // rather than stamp them as paywalled.
+ SubscriberOnly bool
+ Status int // last HTTP status seen (0 on transport error)
}
// Gated reports whether the response carries a strong gating signal: an
@@ -107,6 +113,9 @@ func shouldRetryAsBot(m ArticleMeta) bool {
if m.FetchError {
return false // transport failure won't be fixed by a different UA
}
+ if m.SubscriberOnly {
+ return false // genuinely subscriber-only; no UA changes that
+ }
if m.Gated() {
return true
}
@@ -185,9 +194,27 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
meta.ImageURL = extractOGImage(doc, articleURL)
meta.BodyChars = extractBodyChars(doc)
meta.Paywalled = detectPaywall(doc)
+ meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
return meta
}
+// detectSubscriberOnly returns true for sources known to publish articles
+// with no public version reachable by any workaround we have (archive
+// snapshots, bypass UAs, etc). Currently only LWN's subscriber-only
+// articles, which display a fixed marker in the article body.
+func detectSubscriberOnly(articleURL string, doc *goquery.Document) bool {
+ u, err := url.Parse(articleURL)
+ if err != nil {
+ return false
+ }
+ host := strings.ToLower(u.Hostname())
+ if host != "lwn.net" && !strings.HasSuffix(host, ".lwn.net") {
+ return false
+ }
+ body := strings.ToLower(doc.Find("body").Text())
+ return strings.Contains(body, "available only to lwn subscribers")
+}
+
// isCloudflareChallenge reports whether a non-200 response is Cloudflare's
// bot-block or JS-challenge page rather than a real publisher gate. The
// signatures are stable: cf-* response headers, the cf-error-details body
diff --git a/internal/ingestion/og_test.go b/internal/ingestion/og_test.go
index c673ca1..fe4a4ed 100644
--- a/internal/ingestion/og_test.go
+++ b/internal/ingestion/og_test.go
@@ -104,3 +104,42 @@ func TestExtractBodyCharsANNLayout(t *testing.T) {
t.Fatalf("extractBodyChars = %d, want >= %d (ANN-style body was incorrectly treated as paywalled)", n, PaywallBodyThreshold)
}
}
+
+func TestDetectSubscriberOnly(t *testing.T) {
+ cases := []struct {
+ name string
+ url string
+ body string
+ want bool
+ }{
+ {
+ name: "lwn subscriber-only marker",
+ url: "https://lwn.net/Articles/999999/",
+ body: `The page you have requested is available only to LWN subscribers. Please log in.
`,
+ want: true,
+ },
+ {
+ name: "lwn free article",
+ url: "https://lwn.net/Articles/123456/",
+ body: `Full free article body here.
`,
+ want: false,
+ },
+ {
+ name: "non-lwn host with same marker text is ignored",
+ url: "https://example.com/x",
+ body: `available only to lwn subscribers`,
+ want: false,
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ doc, err := goquery.NewDocumentFromReader(strings.NewReader(tc.body))
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+ if got := detectSubscriberOnly(tc.url, doc); got != tc.want {
+ t.Fatalf("detectSubscriberOnly = %v, want %v", got, tc.want)
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/poller.go b/internal/ingestion/poller.go
index 58ee624..eafe4ab 100644
--- a/internal/ingestion/poller.go
+++ b/internal/ingestion/poller.go
@@ -137,6 +137,17 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
// swapping in a stale snapshot for a transient blip is worse than
// posting the live link.
meta := FetchArticleMeta(originalURL)
+ // Some sources (e.g. LWN's subscriber-only articles) have no public
+ // version reachable by any workaround. Drop these from view entirely
+ // rather than stamping them as paywalled.
+ if meta.SubscriberOnly {
+ // No DB row written, so the next poll will re-fetch and re-detect
+ // this URL — fine in practice since LWN's feed is tiny and the
+ // alternative (a tombstone row) leaks into every story query.
+ slog.Info("dropping subscriber-only story",
+ "guid", items[i].GUID, "url", originalURL, "source", src.Name)
+ continue
+ }
gated := meta.Gated() || (meta.Fetched && meta.BodyChars < PaywallBodyThreshold)
// paywalled tracks whether the link the user will click is still
// gated — i.e. gating was detected AND no archive workaround worked.