drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Strip inline scripts from reader body extraction

Browse Source 
Datawrapper embed resizers (and other inline scripts) were leaking into
reader mode as literal text. Two extraction paths were affected:

- parser.go extractContentText/extractLede stripped only script tags via
  htmlTagRe, leaving the JS body behind. This is the path that usually
  wins for feeds shipping full content:encoded (e.g. Politico).
- article.go goquery paths call .Text(), which concatenates script source.

Both now drop whole <script>/<style>/<noscript> elements before pulling
text. Paywall detection (JSON-LD) still runs before the goquery strip.
This commit is contained in:
prosolis
2026-07-07 21:33:35 -07:00
parent ff3a0e87be
commit 2ea5f7a6f7
4 changed files with 79 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
internal/ingestion/article.go
View File

@@ -193,12 +193,18 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
meta.Fetched = true
meta.ImageURL = extractOGImage(doc, articleURL)
// Paywall detection reads <script type="application/ld+json"> and the LWN
// marker, so run it before we strip non-content nodes below.
meta.Paywalled = detectPaywall(doc)
meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
// Strip <script>/<style>/etc. before pulling body text — goquery's .Text()
// includes the source of inline scripts (e.g. Datawrapper embed resizers),
// which would otherwise leak verbatim into reader mode.
stripNonContent(doc)
// One body extraction serves both purposes: the text feeds reader mode and
// its length is the paywall/thin-body signal.
meta.BodyText = extractBodyText(doc)
meta.BodyChars = len(meta.BodyText)
meta.Paywalled = detectPaywall(doc)
meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
return meta
}
@@ -395,9 +401,19 @@ func FetchArticleBody(articleURL string) string {
if err != nil {
return ""
}
stripNonContent(doc)
return extractBodyText(doc)
}
// stripNonContent removes nodes whose text content is never article prose but
// which goquery's .Text() would otherwise concatenate into the extracted body:
// inline scripts (Datawrapper/embed resizers, analytics), CSS, and the fallback
// markup inside <noscript>/<template>. Must run after any logic that inspects
// these nodes (e.g. JSON-LD paywall detection).
func stripNonContent(doc *goquery.Document) {
doc.Find("script, style, noscript, template").Remove()
}
func extractBodyText(doc *goquery.Document) string {
out := joinParagraphText(doc.Find("article p, main p"))
if len(out) < PaywallBodyThreshold {

25
internal/ingestion/og_test.go
View File

@@ -105,6 +105,31 @@ func TestExtractBodyCharsANNLayout(t *testing.T) {
}
}
// TestExtractBodyTextStripsInlineScript guards against inline embed scripts
// (Datawrapper iframe resizers, analytics) leaking into reader mode. goquery's
// .Text() concatenates the source of <script> nodes, so extraction must strip
// them first. The script here sits between two prose paragraphs, mirroring the
// Politico/Datawrapper layout that surfaced the bug.
func TestExtractBodyTextStripsInlineScript(t *testing.T) {
body := `<html><body><article>
<p>While he has an overall net trust rating of +11 percent nationally, this masks a large regional gap between the north and south of the country.</p>
<figure><script type="text/javascript">!function(){"use strict";window.addEventListener("message",(function(a){if(void 0!==a.data["datawrapper-height"]){var e=document.querySelectorAll("iframe");for(var t in a.data["datawrapper-height"])for(var r=0;r<e.length;r++);}}))}();</script></figure>
<p>Burnham is on the cusp of becoming the U.K.'s seventh prime minister in 10 years and will enter Number 10 later this month after being elected unopposed.</p>
</article></body></html>`
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
if err != nil {
t.Fatalf("parse: %v", err)
}
stripNonContent(doc)
out := extractBodyText(doc)
if strings.Contains(out, "datawrapper-height") || strings.Contains(out, "addEventListener") {
t.Fatalf("inline script leaked into body text:\n%s", out)
}
if !strings.Contains(out, "net trust rating") || !strings.Contains(out, "seventh prime minister") {
t.Fatalf("prose paragraphs missing from body text:\n%s", out)
}
}
func TestDetectSubscriberOnly(t *testing.T) {
cases := []struct {
name string

15
internal/ingestion/parser.go
View File

@@ -46,6 +46,12 @@ var (
wsRe = regexp.MustCompile(`\s+`)
imgSrcRe = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
// nonContentElemRe strips whole <script>/<style>/<noscript> elements —
// tags AND their text — before the generic tag strip runs. htmlTagRe alone
// removes only the tags, which would leave inline JavaScript (e.g. a
// Datawrapper embed resizer) as literal text in reader mode.
nonContentElemRe = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script\s*>|<style\b[^>]*>.*?</style\s*>|<noscript\b[^>]*>.*?</noscript\s*>`)
// Used by extractContentText to preserve paragraph structure when turning
// content:encoded HTML into plain text.
blockCloseRe = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|blockquote|article|section|figure|figcaption|ul|ol|table|tr|pre)>`)
@@ -149,6 +155,9 @@ func extractLede(desc string) string {
if desc == "" {
return ""
}
// Drop <script>/<style>/<noscript> (tags and their inner text) before the
// generic tag strip, which would otherwise leave the inner text behind.
desc = nonContentElemRe.ReplaceAllString(desc, " ")
// Replace tags with a space so adjacent blocks like </p><p> don't fuse words.
text := htmlTagRe.ReplaceAllString(desc, " ")
text = html.UnescapeString(text)
@@ -165,7 +174,11 @@ func extractContentText(raw string) string {
if strings.TrimSpace(raw) == "" {
return ""
}
s := brRe.ReplaceAllString(raw, "\n")
// Remove <script>/<style>/<noscript> elements whole (tags + inner text)
// first — otherwise htmlTagRe below strips only the tags and leaves inline
// JavaScript (e.g. a Datawrapper embed resizer) as literal reader text.
s := nonContentElemRe.ReplaceAllString(raw, "\n")
s = brRe.ReplaceAllString(s, "\n")
s = blockCloseRe.ReplaceAllString(s, "\n\n")
s = htmlTagRe.ReplaceAllString(s, "")
s = html.UnescapeString(s)

22
internal/ingestion/parser_test.go
View File

@@ -94,4 +94,26 @@ func TestExtractContentText(t *testing.T) {
if n := len(extractContentText(long)); n > maxContentChars {
t.Errorf("capped length = %d, want <= %d", n, maxContentChars)
}
// Inline embed scripts (Datawrapper resizers, analytics) must be dropped
// whole — tags and body — not left as literal reader text. htmlTagRe alone
// strips only the tags, so extractContentText has to remove the element first.
embed := `<p>Net trust rating of +11 percent nationally.</p>` +
`<figure><script type="text/javascript">!function(){"use strict";window.addEventListener("message",(function(a){var e=document.querySelectorAll("iframe");}))}();</script></figure>` +
`<p>Burnham is on the cusp of becoming prime minister.</p>`
gotEmbed := extractContentText(embed)
if strings.Contains(gotEmbed, "datawrapper") || strings.Contains(gotEmbed, "addEventListener") || strings.Contains(gotEmbed, "querySelectorAll") {
t.Errorf("inline script leaked into content text:\n%s", gotEmbed)
}
for _, want := range []string{"Net trust rating", "prime minister"} {
if !strings.Contains(gotEmbed, want) {
t.Errorf("prose %q missing from content text:\n%s", want, gotEmbed)
}
}
// The same guard applies to descriptions/ledes.
ledeIn := `Trust gap widens.<script>track("x");</script> North vs south.`
if gotLede := extractLede(ledeIn); strings.Contains(gotLede, "track") {
t.Errorf("inline script leaked into lede: %q", gotLede)
}
}