Strip inline scripts from reader body extraction

Datawrapper embed resizers (and other inline scripts) were leaking into
reader mode as literal text. Two extraction paths were affected:

- parser.go extractContentText/extractLede stripped only script tags via
  htmlTagRe, leaving the JS body behind. This is the path that usually
  wins for feeds shipping full content:encoded (e.g. Politico).
- article.go goquery paths call .Text(), which concatenates script source.

Both now drop whole <script>/<style>/<noscript> elements before pulling
text. Paywall detection (JSON-LD) still runs before the goquery strip.
This commit is contained in:
prosolis
2026-07-07 21:33:35 -07:00
parent ff3a0e87be
commit 2ea5f7a6f7
4 changed files with 79 additions and 3 deletions

View File

@@ -46,6 +46,12 @@ var (
wsRe = regexp.MustCompile(`\s+`)
imgSrcRe = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
// nonContentElemRe strips whole <script>/<style>/<noscript> elements —
// tags AND their text — before the generic tag strip runs. htmlTagRe alone
// removes only the tags, which would leave inline JavaScript (e.g. a
// Datawrapper embed resizer) as literal text in reader mode.
nonContentElemRe = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script\s*>|<style\b[^>]*>.*?</style\s*>|<noscript\b[^>]*>.*?</noscript\s*>`)
// Used by extractContentText to preserve paragraph structure when turning
// content:encoded HTML into plain text.
blockCloseRe = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|blockquote|article|section|figure|figcaption|ul|ol|table|tr|pre)>`)
@@ -149,6 +155,9 @@ func extractLede(desc string) string {
if desc == "" {
return ""
}
// Drop <script>/<style>/<noscript> (tags and their inner text) before the
// generic tag strip, which would otherwise leave the inner text behind.
desc = nonContentElemRe.ReplaceAllString(desc, " ")
// Replace tags with a space so adjacent blocks like </p><p> don't fuse words.
text := htmlTagRe.ReplaceAllString(desc, " ")
text = html.UnescapeString(text)
@@ -165,7 +174,11 @@ func extractContentText(raw string) string {
if strings.TrimSpace(raw) == "" {
return ""
}
s := brRe.ReplaceAllString(raw, "\n")
// Remove <script>/<style>/<noscript> elements whole (tags + inner text)
// first — otherwise htmlTagRe below strips only the tags and leaves inline
// JavaScript (e.g. a Datawrapper embed resizer) as literal reader text.
s := nonContentElemRe.ReplaceAllString(raw, "\n")
s = brRe.ReplaceAllString(s, "\n")
s = blockCloseRe.ReplaceAllString(s, "\n\n")
s = htmlTagRe.ReplaceAllString(s, "")
s = html.UnescapeString(s)