drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix paywall false positive on Anime News Network

Browse Source 
ANN wraps article bodies in <div class="KonaBody"> with no semantic
<article>/<main> tags, so body-length extraction fell below the 500-char
threshold and the poller flagged stories as gated. Broaden the container
fallback to also try itemprop="articleBody", common content-container
classes, and ANN's KonaBody.
This commit is contained in:
prosolis
2026-05-25 13:12:50 -07:00
parent 9e5ba5aafc
commit dd99f2c7ab
2 changed files with 59 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
internal/ingestion/article.go
View File

@@ -375,9 +375,11 @@ func extractBodyText(doc *goquery.Document) string {
}
}
// Sites like Phoronix use <br>-separated text inside <article> rather
// than <p> tags. Fall back to the container's raw text in that case.
// than <p> tags, and sites like Anime News Network skip semantic
// article/main tags entirely and put the body in a custom div. Fall
// back to the raw text of the longest known container.
if len(out) < PaywallBodyThreshold {
if alt := containerText(doc.Find("article, main").First()); len(alt) > len(out) {
if alt := bestContainerText(doc); len(alt) > len(out) {
out = alt
}
}
@@ -387,6 +389,34 @@ func extractBodyText(doc *goquery.Document) string {
return out
}
// bodyContainerSelectors is the list of selectors we try when <article>/<main>
// aren't present or are too thin. Ordered loosely from semantic to generic;
// we take the longest match across all of them.
var bodyContainerSelectors = []string{
"article",
"main",
`[itemprop="articleBody"]`,
`[role="main"]`,
".article-body",
".article-content",
".entry-content",
".post-content",
".story-body",
".KonaBody", // Anime News Network
"#maincontent",
"#content",
}
func bestContainerText(doc *goquery.Document) string {
var best string
for _, sel := range bodyContainerSelectors {
if t := containerText(doc.Find(sel).First()); len(t) > len(best) {
best = t
}
}
return best
}
func joinParagraphText(sel *goquery.Selection) string {
var b strings.Builder
sel.Each(func(_ int, s *goquery.Selection) {
@@ -514,7 +544,7 @@ func extractBodyChars(doc *goquery.Document) int {
}
}
if best < PaywallBodyThreshold {
if n := len(containerText(doc.Find("article, main").First())); n > best {
if n := len(bestContainerText(doc)); n > best {
best = n
}
}

26
internal/ingestion/og_test.go
View File

@@ -6,6 +6,8 @@ import (
"strings"
"testing"
"github.com/PuerkitoBio/goquery"
"pete/internal/safehttp"
)
@@ -78,3 +80,27 @@ func TestFirstImgSrc(t *testing.T) {
t.Errorf("empty input should yield empty, got %q", got)
}
}
// TestExtractBodyCharsANNLayout regresses a false-positive paywall flag on
// Anime News Network: their layout has no <article>/<main> tags and most of
// the prose lives outside <p> tags, inside <div class="KonaBody"> containers.
// Before the fix, body-length came out below PaywallBodyThreshold and the
// poller flagged the story as gated.
func TestExtractBodyCharsANNLayout(t *testing.T) {
body := `<html><body>
<div id="maincontent" class="maincontent news article">
<h1>Yen Press Licenses Wistoria</h1>
<div id="content-zone"><div class="news"><div class="text-zone"><div class="KonaBody">
<div class="meat"><p>Yen Press announced on Friday during its panel that it has licensed seven manga and two novel series for release in December 2026 and winter 2027.</p>
Snowmelt and Agapanthus is a manga series being released in the coming season; the publisher described it as a romance story with supernatural undertones, drawn in a delicate watercolor style that should appeal to fans of slice-of-life fantasy. Witch and Cat follows a young magic apprentice and her talking familiar through a series of low-stakes adventures across a sleepy countryside town, leaning into cozy atmosphere over plot. Sinful Is the Angel Who Loves is positioned for an older audience and runs in the publisher's mature imprint. Each release will be available in print and digital across English-language markets, with the first volumes scheduled for the December 2026 wave.
</div>
</div></div></div></div>
</div></body></html>`
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
if err != nil {
t.Fatalf("parse: %v", err)
}
if n := extractBodyChars(doc); n < PaywallBodyThreshold {
t.Fatalf("extractBodyChars = %d, want >= %d (ANN-style body was incorrectly treated as paywalled)", n, PaywallBodyThreshold)
}
}