Fix paywall false positive on Anime News Network
ANN wraps article bodies in <div class="KonaBody"> with no semantic <article>/<main> tags, so body-length extraction fell below the 500-char threshold and the poller flagged stories as gated. Broaden the container fallback to also try itemprop="articleBody", common content-container classes, and ANN's KonaBody.
This commit is contained in:
@@ -375,9 +375,11 @@ func extractBodyText(doc *goquery.Document) string {
|
||||
}
|
||||
}
|
||||
// Sites like Phoronix use <br>-separated text inside <article> rather
|
||||
// than <p> tags. Fall back to the container's raw text in that case.
|
||||
// than <p> tags, and sites like Anime News Network skip semantic
|
||||
// article/main tags entirely and put the body in a custom div. Fall
|
||||
// back to the raw text of the longest known container.
|
||||
if len(out) < PaywallBodyThreshold {
|
||||
if alt := containerText(doc.Find("article, main").First()); len(alt) > len(out) {
|
||||
if alt := bestContainerText(doc); len(alt) > len(out) {
|
||||
out = alt
|
||||
}
|
||||
}
|
||||
@@ -387,6 +389,34 @@ func extractBodyText(doc *goquery.Document) string {
|
||||
return out
|
||||
}
|
||||
|
||||
// bodyContainerSelectors is the list of selectors we try when <article>/<main>
|
||||
// aren't present or are too thin. Ordered loosely from semantic to generic;
|
||||
// we take the longest match across all of them.
|
||||
var bodyContainerSelectors = []string{
|
||||
"article",
|
||||
"main",
|
||||
`[itemprop="articleBody"]`,
|
||||
`[role="main"]`,
|
||||
".article-body",
|
||||
".article-content",
|
||||
".entry-content",
|
||||
".post-content",
|
||||
".story-body",
|
||||
".KonaBody", // Anime News Network
|
||||
"#maincontent",
|
||||
"#content",
|
||||
}
|
||||
|
||||
func bestContainerText(doc *goquery.Document) string {
|
||||
var best string
|
||||
for _, sel := range bodyContainerSelectors {
|
||||
if t := containerText(doc.Find(sel).First()); len(t) > len(best) {
|
||||
best = t
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func joinParagraphText(sel *goquery.Selection) string {
|
||||
var b strings.Builder
|
||||
sel.Each(func(_ int, s *goquery.Selection) {
|
||||
@@ -514,7 +544,7 @@ func extractBodyChars(doc *goquery.Document) int {
|
||||
}
|
||||
}
|
||||
if best < PaywallBodyThreshold {
|
||||
if n := len(containerText(doc.Find("article, main").First())); n > best {
|
||||
if n := len(bestContainerText(doc)); n > best {
|
||||
best = n
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user