Strip inline scripts from reader body extraction
Datawrapper embed resizers (and other inline scripts) were leaking into reader mode as literal text. Two extraction paths were affected: - parser.go extractContentText/extractLede stripped only script tags via htmlTagRe, leaving the JS body behind. This is the path that usually wins for feeds shipping full content:encoded (e.g. Politico). - article.go goquery paths call .Text(), which concatenates script source. Both now drop whole <script>/<style>/<noscript> elements before pulling text. Paywall detection (JSON-LD) still runs before the goquery strip.
This commit is contained in:
@@ -94,4 +94,26 @@ func TestExtractContentText(t *testing.T) {
|
||||
if n := len(extractContentText(long)); n > maxContentChars {
|
||||
t.Errorf("capped length = %d, want <= %d", n, maxContentChars)
|
||||
}
|
||||
|
||||
// Inline embed scripts (Datawrapper resizers, analytics) must be dropped
|
||||
// whole — tags and body — not left as literal reader text. htmlTagRe alone
|
||||
// strips only the tags, so extractContentText has to remove the element first.
|
||||
embed := `<p>Net trust rating of +11 percent nationally.</p>` +
|
||||
`<figure><script type="text/javascript">!function(){"use strict";window.addEventListener("message",(function(a){var e=document.querySelectorAll("iframe");}))}();</script></figure>` +
|
||||
`<p>Burnham is on the cusp of becoming prime minister.</p>`
|
||||
gotEmbed := extractContentText(embed)
|
||||
if strings.Contains(gotEmbed, "datawrapper") || strings.Contains(gotEmbed, "addEventListener") || strings.Contains(gotEmbed, "querySelectorAll") {
|
||||
t.Errorf("inline script leaked into content text:\n%s", gotEmbed)
|
||||
}
|
||||
for _, want := range []string{"Net trust rating", "prime minister"} {
|
||||
if !strings.Contains(gotEmbed, want) {
|
||||
t.Errorf("prose %q missing from content text:\n%s", want, gotEmbed)
|
||||
}
|
||||
}
|
||||
|
||||
// The same guard applies to descriptions/ledes.
|
||||
ledeIn := `Trust gap widens.<script>track("x");</script> North vs south.`
|
||||
if gotLede := extractLede(ledeIn); strings.Contains(gotLede, "track") {
|
||||
t.Errorf("inline script leaked into lede: %q", gotLede)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user