2ea5f7a6f7
Datawrapper embed resizers (and other inline scripts) were leaking into reader mode as literal text. Two extraction paths were affected: - parser.go extractContentText/extractLede stripped only script tags via htmlTagRe, leaving the JS body behind. This is the path that usually wins for feeds shipping full content:encoded (e.g. Politico). - article.go goquery paths call .Text(), which concatenates script source. Both now drop whole <script>/<style>/<noscript> elements before pulling text. Paywall detection (JSON-LD) still runs before the goquery strip.
171 lines
6.5 KiB
Go
171 lines
6.5 KiB
Go
package ingestion
|
|
|
|
import (
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
|
|
"pete/internal/safehttp"
|
|
)
|
|
|
|
func init() {
|
|
// httptest servers bind to 127.0.0.1; allow the safe-dial check to
|
|
// accept them in tests only.
|
|
safehttp.AllowPrivate = true
|
|
}
|
|
|
|
func TestFetchOGImage(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
body string
|
|
want string // suffix match
|
|
}{
|
|
{
|
|
name: "og:image property",
|
|
body: `<html><head><meta property="og:image" content="https://example.com/lead.jpg"></head></html>`,
|
|
want: "https://example.com/lead.jpg",
|
|
},
|
|
{
|
|
name: "content before property",
|
|
body: `<html><head><meta content="https://example.com/swap.jpg" property="og:image"/></head></html>`,
|
|
want: "https://example.com/swap.jpg",
|
|
},
|
|
{
|
|
name: "twitter:image fallback",
|
|
body: `<html><head><meta name="twitter:image" content="https://example.com/twit.jpg"></head></html>`,
|
|
want: "https://example.com/twit.jpg",
|
|
},
|
|
{
|
|
name: "relative URL resolved",
|
|
body: `<html><head><meta property="og:image" content="/img/lead.jpg"></head></html>`,
|
|
want: "/img/lead.jpg",
|
|
},
|
|
{
|
|
name: "no og tags",
|
|
body: `<html><head><title>nothing</title></head></html>`,
|
|
want: "",
|
|
},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html")
|
|
w.Write([]byte(c.body))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
got := FetchOGImage(srv.URL + "/article")
|
|
if c.want == "" {
|
|
if got != "" {
|
|
t.Errorf("expected empty, got %q", got)
|
|
}
|
|
return
|
|
}
|
|
if !strings.HasSuffix(got, c.want) {
|
|
t.Errorf("got %q, want suffix %q", got, c.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFirstImgSrc(t *testing.T) {
|
|
in := `<p>Body <img alt="x" src="https://cdn.example.com/lead.png" width="600"/> more</p>`
|
|
if got := firstImgSrc(in); got != "https://cdn.example.com/lead.png" {
|
|
t.Errorf("got %q", got)
|
|
}
|
|
if got := firstImgSrc(""); got != "" {
|
|
t.Errorf("empty input should yield empty, got %q", got)
|
|
}
|
|
}
|
|
|
|
// TestExtractBodyCharsANNLayout regresses a false-positive paywall flag on
|
|
// Anime News Network: their layout has no <article>/<main> tags and most of
|
|
// the prose lives outside <p> tags, inside <div class="KonaBody"> containers.
|
|
// Before the fix, body-length came out below PaywallBodyThreshold and the
|
|
// poller flagged the story as gated.
|
|
func TestExtractBodyCharsANNLayout(t *testing.T) {
|
|
body := `<html><body>
|
|
<div id="maincontent" class="maincontent news article">
|
|
<h1>Yen Press Licenses Wistoria</h1>
|
|
<div id="content-zone"><div class="news"><div class="text-zone"><div class="KonaBody">
|
|
<div class="meat"><p>Yen Press announced on Friday during its panel that it has licensed seven manga and two novel series for release in December 2026 and winter 2027.</p>
|
|
Snowmelt and Agapanthus is a manga series being released in the coming season; the publisher described it as a romance story with supernatural undertones, drawn in a delicate watercolor style that should appeal to fans of slice-of-life fantasy. Witch and Cat follows a young magic apprentice and her talking familiar through a series of low-stakes adventures across a sleepy countryside town, leaning into cozy atmosphere over plot. Sinful Is the Angel Who Loves is positioned for an older audience and runs in the publisher's mature imprint. Each release will be available in print and digital across English-language markets, with the first volumes scheduled for the December 2026 wave.
|
|
</div>
|
|
</div></div></div></div>
|
|
</div></body></html>`
|
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
|
|
if err != nil {
|
|
t.Fatalf("parse: %v", err)
|
|
}
|
|
if n := extractBodyChars(doc); n < PaywallBodyThreshold {
|
|
t.Fatalf("extractBodyChars = %d, want >= %d (ANN-style body was incorrectly treated as paywalled)", n, PaywallBodyThreshold)
|
|
}
|
|
}
|
|
|
|
// TestExtractBodyTextStripsInlineScript guards against inline embed scripts
|
|
// (Datawrapper iframe resizers, analytics) leaking into reader mode. goquery's
|
|
// .Text() concatenates the source of <script> nodes, so extraction must strip
|
|
// them first. The script here sits between two prose paragraphs, mirroring the
|
|
// Politico/Datawrapper layout that surfaced the bug.
|
|
func TestExtractBodyTextStripsInlineScript(t *testing.T) {
|
|
body := `<html><body><article>
|
|
<p>While he has an overall net trust rating of +11 percent nationally, this masks a large regional gap between the north and south of the country.</p>
|
|
<figure><script type="text/javascript">!function(){"use strict";window.addEventListener("message",(function(a){if(void 0!==a.data["datawrapper-height"]){var e=document.querySelectorAll("iframe");for(var t in a.data["datawrapper-height"])for(var r=0;r<e.length;r++);}}))}();</script></figure>
|
|
<p>Burnham is on the cusp of becoming the U.K.'s seventh prime minister in 10 years and will enter Number 10 later this month after being elected unopposed.</p>
|
|
</article></body></html>`
|
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
|
|
if err != nil {
|
|
t.Fatalf("parse: %v", err)
|
|
}
|
|
stripNonContent(doc)
|
|
out := extractBodyText(doc)
|
|
if strings.Contains(out, "datawrapper-height") || strings.Contains(out, "addEventListener") {
|
|
t.Fatalf("inline script leaked into body text:\n%s", out)
|
|
}
|
|
if !strings.Contains(out, "net trust rating") || !strings.Contains(out, "seventh prime minister") {
|
|
t.Fatalf("prose paragraphs missing from body text:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestDetectSubscriberOnly(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
url string
|
|
body string
|
|
want bool
|
|
}{
|
|
{
|
|
name: "lwn subscriber-only marker",
|
|
url: "https://lwn.net/Articles/999999/",
|
|
body: `<html><body><p>The page you have requested is available only to LWN subscribers. Please log in.</p></body></html>`,
|
|
want: true,
|
|
},
|
|
{
|
|
name: "lwn free article",
|
|
url: "https://lwn.net/Articles/123456/",
|
|
body: `<html><body><article><p>Full free article body here.</p></article></body></html>`,
|
|
want: false,
|
|
},
|
|
{
|
|
name: "non-lwn host with same marker text is ignored",
|
|
url: "https://example.com/x",
|
|
body: `<html><body>available only to lwn subscribers</body></html>`,
|
|
want: false,
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(tc.body))
|
|
if err != nil {
|
|
t.Fatalf("parse: %v", err)
|
|
}
|
|
if got := detectSubscriberOnly(tc.url, doc); got != tc.want {
|
|
t.Fatalf("detectSubscriberOnly = %v, want %v", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|