Strip inline scripts from reader body extraction
Datawrapper embed resizers (and other inline scripts) were leaking into reader mode as literal text. Two extraction paths were affected: - parser.go extractContentText/extractLede stripped only script tags via htmlTagRe, leaving the JS body behind. This is the path that usually wins for feeds shipping full content:encoded (e.g. Politico). - article.go goquery paths call .Text(), which concatenates script source. Both now drop whole <script>/<style>/<noscript> elements before pulling text. Paywall detection (JSON-LD) still runs before the goquery strip.
This commit is contained in:
@@ -193,12 +193,18 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
|
||||
|
||||
meta.Fetched = true
|
||||
meta.ImageURL = extractOGImage(doc, articleURL)
|
||||
// Paywall detection reads <script type="application/ld+json"> and the LWN
|
||||
// marker, so run it before we strip non-content nodes below.
|
||||
meta.Paywalled = detectPaywall(doc)
|
||||
meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
|
||||
// Strip <script>/<style>/etc. before pulling body text — goquery's .Text()
|
||||
// includes the source of inline scripts (e.g. Datawrapper embed resizers),
|
||||
// which would otherwise leak verbatim into reader mode.
|
||||
stripNonContent(doc)
|
||||
// One body extraction serves both purposes: the text feeds reader mode and
|
||||
// its length is the paywall/thin-body signal.
|
||||
meta.BodyText = extractBodyText(doc)
|
||||
meta.BodyChars = len(meta.BodyText)
|
||||
meta.Paywalled = detectPaywall(doc)
|
||||
meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
|
||||
return meta
|
||||
}
|
||||
|
||||
@@ -395,9 +401,19 @@ func FetchArticleBody(articleURL string) string {
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
stripNonContent(doc)
|
||||
return extractBodyText(doc)
|
||||
}
|
||||
|
||||
// stripNonContent removes nodes whose text content is never article prose but
|
||||
// which goquery's .Text() would otherwise concatenate into the extracted body:
|
||||
// inline scripts (Datawrapper/embed resizers, analytics), CSS, and the fallback
|
||||
// markup inside <noscript>/<template>. Must run after any logic that inspects
|
||||
// these nodes (e.g. JSON-LD paywall detection).
|
||||
func stripNonContent(doc *goquery.Document) {
|
||||
doc.Find("script, style, noscript, template").Remove()
|
||||
}
|
||||
|
||||
func extractBodyText(doc *goquery.Document) string {
|
||||
out := joinParagraphText(doc.Find("article p, main p"))
|
||||
if len(out) < PaywallBodyThreshold {
|
||||
|
||||
Reference in New Issue
Block a user