Fix paywall false positive on Anime News Network

ANN wraps article bodies in <div class="KonaBody"> with no semantic
<article>/<main> tags, so body-length extraction fell below the 500-char
threshold and the poller flagged stories as gated. Broaden the container
fallback to also try itemprop="articleBody", common content-container
classes, and ANN's KonaBody.
This commit is contained in:
prosolis
2026-05-25 13:12:50 -07:00
parent 9e5ba5aafc
commit dd99f2c7ab
2 changed files with 59 additions and 3 deletions

View File

@@ -375,9 +375,11 @@ func extractBodyText(doc *goquery.Document) string {
}
}
// Sites like Phoronix use <br>-separated text inside <article> rather
// than <p> tags. Fall back to the container's raw text in that case.
// than <p> tags, and sites like Anime News Network skip semantic
// article/main tags entirely and put the body in a custom div. Fall
// back to the raw text of the longest known container.
if len(out) < PaywallBodyThreshold {
if alt := containerText(doc.Find("article, main").First()); len(alt) > len(out) {
if alt := bestContainerText(doc); len(alt) > len(out) {
out = alt
}
}
@@ -387,6 +389,34 @@ func extractBodyText(doc *goquery.Document) string {
return out
}
// bodyContainerSelectors is the list of selectors we try when <article>/<main>
// aren't present or are too thin. Ordered loosely from semantic to generic;
// we take the longest match across all of them.
var bodyContainerSelectors = []string{
"article",
"main",
`[itemprop="articleBody"]`,
`[role="main"]`,
".article-body",
".article-content",
".entry-content",
".post-content",
".story-body",
".KonaBody", // Anime News Network
"#maincontent",
"#content",
}
func bestContainerText(doc *goquery.Document) string {
var best string
for _, sel := range bodyContainerSelectors {
if t := containerText(doc.Find(sel).First()); len(t) > len(best) {
best = t
}
}
return best
}
func joinParagraphText(sel *goquery.Selection) string {
var b strings.Builder
sel.Each(func(_ int, s *goquery.Selection) {
@@ -514,7 +544,7 @@ func extractBodyChars(doc *goquery.Document) int {
}
}
if best < PaywallBodyThreshold {
if n := len(containerText(doc.Find("article, main").First())); n > best {
if n := len(bestContainerText(doc)); n > best {
best = n
}
}