Strip inline scripts from reader body extraction
Datawrapper embed resizers (and other inline scripts) were leaking into reader mode as literal text. Two extraction paths were affected: - parser.go extractContentText/extractLede stripped only script tags via htmlTagRe, leaving the JS body behind. This is the path that usually wins for feeds shipping full content:encoded (e.g. Politico). - article.go goquery paths call .Text(), which concatenates script source. Both now drop whole <script>/<style>/<noscript> elements before pulling text. Paywall detection (JSON-LD) still runs before the goquery strip.
This commit is contained in:
@@ -46,6 +46,12 @@ var (
|
||||
wsRe = regexp.MustCompile(`\s+`)
|
||||
imgSrcRe = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
|
||||
|
||||
// nonContentElemRe strips whole <script>/<style>/<noscript> elements —
|
||||
// tags AND their text — before the generic tag strip runs. htmlTagRe alone
|
||||
// removes only the tags, which would leave inline JavaScript (e.g. a
|
||||
// Datawrapper embed resizer) as literal text in reader mode.
|
||||
nonContentElemRe = regexp.MustCompile(`(?is)<script\b[^>]*>.*?</script\s*>|<style\b[^>]*>.*?</style\s*>|<noscript\b[^>]*>.*?</noscript\s*>`)
|
||||
|
||||
// Used by extractContentText to preserve paragraph structure when turning
|
||||
// content:encoded HTML into plain text.
|
||||
blockCloseRe = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|blockquote|article|section|figure|figcaption|ul|ol|table|tr|pre)>`)
|
||||
@@ -149,6 +155,9 @@ func extractLede(desc string) string {
|
||||
if desc == "" {
|
||||
return ""
|
||||
}
|
||||
// Drop <script>/<style>/<noscript> (tags and their inner text) before the
|
||||
// generic tag strip, which would otherwise leave the inner text behind.
|
||||
desc = nonContentElemRe.ReplaceAllString(desc, " ")
|
||||
// Replace tags with a space so adjacent blocks like </p><p> don't fuse words.
|
||||
text := htmlTagRe.ReplaceAllString(desc, " ")
|
||||
text = html.UnescapeString(text)
|
||||
@@ -165,7 +174,11 @@ func extractContentText(raw string) string {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return ""
|
||||
}
|
||||
s := brRe.ReplaceAllString(raw, "\n")
|
||||
// Remove <script>/<style>/<noscript> elements whole (tags + inner text)
|
||||
// first — otherwise htmlTagRe below strips only the tags and leaves inline
|
||||
// JavaScript (e.g. a Datawrapper embed resizer) as literal reader text.
|
||||
s := nonContentElemRe.ReplaceAllString(raw, "\n")
|
||||
s = brRe.ReplaceAllString(s, "\n")
|
||||
s = blockCloseRe.ReplaceAllString(s, "\n\n")
|
||||
s = htmlTagRe.ReplaceAllString(s, "")
|
||||
s = html.UnescapeString(s)
|
||||
|
||||
Reference in New Issue
Block a user