Strip inline scripts from reader body extraction

Datawrapper embed resizers (and other inline scripts) were leaking into
reader mode as literal text. Two extraction paths were affected:

- parser.go extractContentText/extractLede stripped only script tags via
  htmlTagRe, leaving the JS body behind. This is the path that usually
  wins for feeds shipping full content:encoded (e.g. Politico).
- article.go goquery paths call .Text(), which concatenates script source.

Both now drop whole <script>/<style>/<noscript> elements before pulling
text. Paywall detection (JSON-LD) still runs before the goquery strip.
This commit is contained in:
prosolis
2026-07-07 21:33:35 -07:00
parent ff3a0e87be
commit 2ea5f7a6f7
4 changed files with 79 additions and 3 deletions

View File

@@ -94,4 +94,26 @@ func TestExtractContentText(t *testing.T) {
if n := len(extractContentText(long)); n > maxContentChars {
t.Errorf("capped length = %d, want <= %d", n, maxContentChars)
}
// Inline embed scripts (Datawrapper resizers, analytics) must be dropped
// whole — tags and body — not left as literal reader text. htmlTagRe alone
// strips only the tags, so extractContentText has to remove the element first.
embed := `<p>Net trust rating of +11 percent nationally.</p>` +
`<figure><script type="text/javascript">!function(){"use strict";window.addEventListener("message",(function(a){var e=document.querySelectorAll("iframe");}))}();</script></figure>` +
`<p>Burnham is on the cusp of becoming prime minister.</p>`
gotEmbed := extractContentText(embed)
if strings.Contains(gotEmbed, "datawrapper") || strings.Contains(gotEmbed, "addEventListener") || strings.Contains(gotEmbed, "querySelectorAll") {
t.Errorf("inline script leaked into content text:\n%s", gotEmbed)
}
for _, want := range []string{"Net trust rating", "prime minister"} {
if !strings.Contains(gotEmbed, want) {
t.Errorf("prose %q missing from content text:\n%s", want, gotEmbed)
}
}
// The same guard applies to descriptions/ledes.
ledeIn := `Trust gap widens.<script>track("x");</script> North vs south.`
if gotLede := extractLede(ledeIn); strings.Contains(gotLede, "track") {
t.Errorf("inline script leaked into lede: %q", gotLede)
}
}