diff --git a/internal/ingestion/article.go b/internal/ingestion/article.go index 2f05c8a..dccd737 100644 --- a/internal/ingestion/article.go +++ b/internal/ingestion/article.go @@ -193,12 +193,18 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta { meta.Fetched = true meta.ImageURL = extractOGImage(doc, articleURL) + // Paywall detection reads +

Burnham is on the cusp of becoming the U.K.'s seventh prime minister in 10 years and will enter Number 10 later this month after being elected unopposed.

+` + doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) + if err != nil { + t.Fatalf("parse: %v", err) + } + stripNonContent(doc) + out := extractBodyText(doc) + if strings.Contains(out, "datawrapper-height") || strings.Contains(out, "addEventListener") { + t.Fatalf("inline script leaked into body text:\n%s", out) + } + if !strings.Contains(out, "net trust rating") || !strings.Contains(out, "seventh prime minister") { + t.Fatalf("prose paragraphs missing from body text:\n%s", out) + } +} + func TestDetectSubscriberOnly(t *testing.T) { cases := []struct { name string diff --git a/internal/ingestion/parser.go b/internal/ingestion/parser.go index 227dc2e..0b92207 100644 --- a/internal/ingestion/parser.go +++ b/internal/ingestion/parser.go @@ -46,6 +46,12 @@ var ( wsRe = regexp.MustCompile(`\s+`) imgSrcRe = regexp.MustCompile(`(?i)]+src=["']([^"']+)["']`) + // nonContentElemRe strips whole ` + + `

Burnham is on the cusp of becoming prime minister.

` + gotEmbed := extractContentText(embed) + if strings.Contains(gotEmbed, "datawrapper") || strings.Contains(gotEmbed, "addEventListener") || strings.Contains(gotEmbed, "querySelectorAll") { + t.Errorf("inline script leaked into content text:\n%s", gotEmbed) + } + for _, want := range []string{"Net trust rating", "prime minister"} { + if !strings.Contains(gotEmbed, want) { + t.Errorf("prose %q missing from content text:\n%s", want, gotEmbed) + } + } + + // The same guard applies to descriptions/ledes. + ledeIn := `Trust gap widens. North vs south.` + if gotLede := extractLede(ledeIn); strings.Contains(gotLede, "track") { + t.Errorf("inline script leaked into lede: %q", gotLede) + } }