From 2ea5f7a6f744ceb8dc926007d2248e58bbdab994 Mon Sep 17 00:00:00 2001
From: prosolis <5590409+prosolis@users.noreply.github.com>
Date: Tue, 7 Jul 2026 21:33:35 -0700
Subject: [PATCH] Strip inline scripts from reader body extraction

Datawrapper embed resizers (and other inline scripts) were leaking into
reader mode as literal text. Two extraction paths were affected:

- parser.go extractContentText/extractLede stripped only script tags via
  htmlTagRe, leaving the JS body behind. This is the path that usually
  wins for feeds shipping full content:encoded (e.g. Politico).
- article.go goquery paths call .Text(), which concatenates script source.

Both now drop whole <script>/<style>/<noscript> elements before pulling
text. Paywall detection (JSON-LD) still runs before the goquery strip.
---
 internal/ingestion/article.go     | 20 ++++++++++++++++++--
 internal/ingestion/og_test.go     | 25 +++++++++++++++++++++++++
 internal/ingestion/parser.go      | 15 ++++++++++++++-
 internal/ingestion/parser_test.go | 22 ++++++++++++++++++++++
 4 files changed, 79 insertions(+), 3 deletions(-)
diff --git a/internal/ingestion/article.go b/internal/ingestion/article.go
index 2f05c8a..dccd737 100644
--- a/internal/ingestion/article.go
+++ b/internal/ingestion/article.go
@@ -193,12 +193,18 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
 
 	meta.Fetched = true
 	meta.ImageURL = extractOGImage(doc, articleURL)
+	// Paywall detection reads <script type="application/ld+json"> and the LWN
+	// marker, so run it before we strip non-content nodes below.
+	meta.Paywalled = detectPaywall(doc)
+	meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
+	// Strip <script>/<style>/etc. before pulling body text — goquery's .Text()
+	// includes the source of inline scripts (e.g. Datawrapper embed resizers),
+	// which would otherwise leak verbatim into reader mode.
+	stripNonContent(doc)
 	// One body extraction serves both purposes: the text feeds reader mode and
 	// its length is the paywall/thin-body signal.
 	meta.BodyText = extractBodyText(doc)
 	meta.BodyChars = len(meta.BodyText)
-	meta.Paywalled = detectPaywall(doc)
-	meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
 	return meta
 }
 
@@ -395,9 +401,19 @@ func FetchArticleBody(articleURL string) string {
 	if err != nil {
 		return ""
 	}
+	stripNonContent(doc)
 	return extractBodyText(doc)
 }
 
+// stripNonContent removes nodes whose text content is never article prose but
+// which goquery's .Text() would otherwise concatenate into the extracted body:
+// inline scripts (Datawrapper/embed resizers, analytics), CSS, and the fallback
+// markup inside <noscript>/<template>. Must run after any logic that inspects
+// these nodes (e.g. JSON-LD paywall detection).
+func stripNonContent(doc *goquery.Document) {
+	doc.Find("script, style, noscript, template").Remove()
+}
+
 func extractBodyText(doc *goquery.Document) string {
 	out := joinParagraphText(doc.Find("article p, main p"))
 	if len(out) < PaywallBodyThreshold {
diff --git a/internal/ingestion/og_test.go b/internal/ingestion/og_test.go
index fe4a4ed..f6c1c08 100644
--- a/internal/ingestion/og_test.go
+++ b/internal/ingestion/og_test.go
@@ -105,6 +105,31 @@ func TestExtractBodyCharsANNLayout(t *testing.T) {
 	}
 }
 
+// TestExtractBodyTextStripsInlineScript guards against inline embed scripts
+// (Datawrapper iframe resizers, analytics) leaking into reader mode. goquery's
+// .Text() concatenates the source of <script> nodes, so extraction must strip
+// them first. The script here sits between two prose paragraphs, mirroring the
+// Politico/Datawrapper layout that surfaced the bug.
+func TestExtractBodyTextStripsInlineScript(t *testing.T) {
+	body := `<html><body><article>
+<p>While he has an overall net trust rating of +11 percent nationally, this masks a large regional gap between the north and south of the country.</p>
+<figure><script type="text/javascript">!function(){"use strict";window.addEventListener("message",(function(a){if(void 0!==a.data["datawrapper-height"]){var e=document.querySelectorAll("iframe");for(var t in a.data["datawrapper-height"])for(var r=0;r<e.length;r++);}}))}();</script></figure>
+<p>Burnham is on the cusp of becoming the U.K.'s seventh prime minister in 10 years and will enter Number 10 later this month after being elected unopposed.</p>
+</article></body></html>`
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
+	if err != nil {
+		t.Fatalf("parse: %v", err)
+	}
+	stripNonContent(doc)
+	out := extractBodyText(doc)
+	if strings.Contains(out, "datawrapper-height") || strings.Contains(out, "addEventListener") {
+		t.Fatalf("inline script leaked into body text:\n%s", out)
+	}
+	if !strings.Contains(out, "net trust rating") || !strings.Contains(out, "seventh prime minister") {
+		t.Fatalf("prose paragraphs missing from body text:\n%s", out)
+	}
+}
+
 func TestDetectSubscriberOnly(t *testing.T) {
 	cases := []struct {
 		name string
diff --git a/internal/ingestion/parser.go b/internal/ingestion/parser.go
index 227dc2e..0b92207 100644
--- a/internal/ingestion/parser.go
+++ b/internal/ingestion/parser.go
@@ -46,6 +46,12 @@ var (
 	wsRe      = regexp.MustCompile(`\s+`)
 	imgSrcRe  = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
 
+	// nonContentElemRe strips whole <script>/<style>/<noscript> elements —
+	// tags AND their text — before the generic tag strip runs. htmlTagRe alone
+	// removes only the tags, which would leave inline JavaScript (e.g. a
+	// Datawrapper embed resizer) as literal text in reader mode.
+	nonContentElemRe = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script\s*>|<style\b[^>]*>.*?</style\s*>|<noscript\b[^>]*>.*?</noscript\s*>`)
+
 	// Used by extractContentText to preserve paragraph structure when turning
 	// content:encoded HTML into plain text.
 	blockCloseRe  = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|blockquote|article|section|figure|figcaption|ul|ol|table|tr|pre)>`)
@@ -149,6 +155,9 @@ func extractLede(desc string) string {
 	if desc == "" {
 		return ""
 	}
+	// Drop <script>/<style>/<noscript> (tags and their inner text) before the
+	// generic tag strip, which would otherwise leave the inner text behind.
+	desc = nonContentElemRe.ReplaceAllString(desc, " ")
 	// Replace tags with a space so adjacent blocks like </p><p> don't fuse words.
 	text := htmlTagRe.ReplaceAllString(desc, " ")
 	text = html.UnescapeString(text)
@@ -165,7 +174,11 @@ func extractContentText(raw string) string {
 	if strings.TrimSpace(raw) == "" {
 		return ""
 	}
-	s := brRe.ReplaceAllString(raw, "\n")
+	// Remove <script>/<style>/<noscript> elements whole (tags + inner text)
+	// first — otherwise htmlTagRe below strips only the tags and leaves inline
+	// JavaScript (e.g. a Datawrapper embed resizer) as literal reader text.
+	s := nonContentElemRe.ReplaceAllString(raw, "\n")
+	s = brRe.ReplaceAllString(s, "\n")
 	s = blockCloseRe.ReplaceAllString(s, "\n\n")
 	s = htmlTagRe.ReplaceAllString(s, "")
 	s = html.UnescapeString(s)
diff --git a/internal/ingestion/parser_test.go b/internal/ingestion/parser_test.go
index cde079a..5a9fee2 100644
--- a/internal/ingestion/parser_test.go
+++ b/internal/ingestion/parser_test.go
@@ -94,4 +94,26 @@ func TestExtractContentText(t *testing.T) {
 	if n := len(extractContentText(long)); n > maxContentChars {
 		t.Errorf("capped length = %d, want <= %d", n, maxContentChars)
 	}
+
+	// Inline embed scripts (Datawrapper resizers, analytics) must be dropped
+	// whole — tags and body — not left as literal reader text. htmlTagRe alone
+	// strips only the tags, so extractContentText has to remove the element first.
+	embed := `<p>Net trust rating of +11 percent nationally.</p>` +
+		`<figure><script type="text/javascript">!function(){"use strict";window.addEventListener("message",(function(a){var e=document.querySelectorAll("iframe");}))}();</script></figure>` +
+		`<p>Burnham is on the cusp of becoming prime minister.</p>`
+	gotEmbed := extractContentText(embed)
+	if strings.Contains(gotEmbed, "datawrapper") || strings.Contains(gotEmbed, "addEventListener") || strings.Contains(gotEmbed, "querySelectorAll") {
+		t.Errorf("inline script leaked into content text:\n%s", gotEmbed)
+	}
+	for _, want := range []string{"Net trust rating", "prime minister"} {
+		if !strings.Contains(gotEmbed, want) {
+			t.Errorf("prose %q missing from content text:\n%s", want, gotEmbed)
+		}
+	}
+
+	// The same guard applies to descriptions/ledes.
+	ledeIn := `Trust gap widens.<script>track("x");</script> North vs south.`
+	if gotLede := extractLede(ledeIn); strings.Contains(gotLede, "track") {
+		t.Errorf("inline script leaked into lede: %q", gotLede)
+	}
 }
