Strip inline scripts from reader body extraction

Datawrapper embed resizers (and other inline scripts) were leaking into
reader mode as literal text. Two extraction paths were affected:

- parser.go extractContentText/extractLede stripped only script tags via
  htmlTagRe, leaving the JS body behind. This is the path that usually
  wins for feeds shipping full content:encoded (e.g. Politico).
- article.go goquery paths call .Text(), which concatenates script source.

Both now drop whole <script>/<style>/<noscript> elements before pulling
text. Paywall detection (JSON-LD) still runs before the goquery strip.
This commit is contained in:
prosolis
2026-07-07 21:33:35 -07:00
parent ff3a0e87be
commit 2ea5f7a6f7
4 changed files with 79 additions and 3 deletions

View File

@@ -193,12 +193,18 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
meta.Fetched = true
meta.ImageURL = extractOGImage(doc, articleURL)
// Paywall detection reads <script type="application/ld+json"> and the LWN
// marker, so run it before we strip non-content nodes below.
meta.Paywalled = detectPaywall(doc)
meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
// Strip <script>/<style>/etc. before pulling body text — goquery's .Text()
// includes the source of inline scripts (e.g. Datawrapper embed resizers),
// which would otherwise leak verbatim into reader mode.
stripNonContent(doc)
// One body extraction serves both purposes: the text feeds reader mode and
// its length is the paywall/thin-body signal.
meta.BodyText = extractBodyText(doc)
meta.BodyChars = len(meta.BodyText)
meta.Paywalled = detectPaywall(doc)
meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
return meta
}
@@ -395,9 +401,19 @@ func FetchArticleBody(articleURL string) string {
if err != nil {
return ""
}
stripNonContent(doc)
return extractBodyText(doc)
}
// stripNonContent removes nodes whose text content is never article prose but
// which goquery's .Text() would otherwise concatenate into the extracted body:
// inline scripts (Datawrapper/embed resizers, analytics), CSS, and the fallback
// markup inside <noscript>/<template>. Must run after any logic that inspects
// these nodes (e.g. JSON-LD paywall detection).
func stripNonContent(doc *goquery.Document) {
doc.Find("script, style, noscript, template").Remove()
}
func extractBodyText(doc *goquery.Document) string {
out := joinParagraphText(doc.Find("article p, main p"))
if len(out) < PaywallBodyThreshold {