Drop LWN subscriber-only articles instead of stamping paywalled

LWN's subscriber-only articles have no public version reachable by archive snapshots or bypass UAs, so stamping them as paywalled produces clicks that always dead-end. Detect the marker text and skip ingestion entirely.
2026-05-25 14:31:08 -07:00
parent dd99f2c7ab
commit 83b1216541
3 changed files with 78 additions and 1 deletions
--- a/internal/ingestion/og_test.go
+++ b/internal/ingestion/og_test.go
@@ -104,3 +104,42 @@ func TestExtractBodyCharsANNLayout(t *testing.T) {
 		t.Fatalf("extractBodyChars = %d, want >= %d (ANN-style body was incorrectly treated as paywalled)", n, PaywallBodyThreshold)
 	}
 }
+
+func TestDetectSubscriberOnly(t *testing.T) {
+	cases := []struct {
+		name string
+		url  string
+		body string
+		want bool
+	}{
+		{
+			name: "lwn subscriber-only marker",
+			url:  "https://lwn.net/Articles/999999/",
+			body: `<html><body><p>The page you have requested is available only to LWN subscribers. Please log in.</p></body></html>`,
+			want: true,
+		},
+		{
+			name: "lwn free article",
+			url:  "https://lwn.net/Articles/123456/",
+			body: `<html><body><article><p>Full free article body here.</p></article></body></html>`,
+			want: false,
+		},
+		{
+			name: "non-lwn host with same marker text is ignored",
+			url:  "https://example.com/x",
+			body: `<html><body>available only to lwn subscribers</body></html>`,
+			want: false,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			doc, err := goquery.NewDocumentFromReader(strings.NewReader(tc.body))
+			if err != nil {
+				t.Fatalf("parse: %v", err)
+			}
+			if got := detectSubscriberOnly(tc.url, doc); got != tc.want {
+				t.Fatalf("detectSubscriberOnly = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}