rather
- // than tags. Fall back to the container's raw text in that case.
+ // than
tags, and sites like Anime News Network skip semantic
+ // article/main tags entirely and put the body in a custom div. Fall
+ // back to the raw text of the longest known container.
if len(out) < PaywallBodyThreshold {
- if alt := containerText(doc.Find("article, main").First()); len(alt) > len(out) {
+ if alt := bestContainerText(doc); len(alt) > len(out) {
out = alt
}
}
@@ -387,6 +389,34 @@ func extractBodyText(doc *goquery.Document) string {
return out
}
+// bodyContainerSelectors is the list of selectors we try when /
+// aren't present or are too thin. Ordered loosely from semantic to generic;
+// we take the longest match across all of them.
+var bodyContainerSelectors = []string{
+ "article",
+ "main",
+ `[itemprop="articleBody"]`,
+ `[role="main"]`,
+ ".article-body",
+ ".article-content",
+ ".entry-content",
+ ".post-content",
+ ".story-body",
+ ".KonaBody", // Anime News Network
+ "#maincontent",
+ "#content",
+}
+
+func bestContainerText(doc *goquery.Document) string {
+ var best string
+ for _, sel := range bodyContainerSelectors {
+ if t := containerText(doc.Find(sel).First()); len(t) > len(best) {
+ best = t
+ }
+ }
+ return best
+}
+
func joinParagraphText(sel *goquery.Selection) string {
var b strings.Builder
sel.Each(func(_ int, s *goquery.Selection) {
@@ -514,7 +544,7 @@ func extractBodyChars(doc *goquery.Document) int {
}
}
if best < PaywallBodyThreshold {
- if n := len(containerText(doc.Find("article, main").First())); n > best {
+ if n := len(bestContainerText(doc)); n > best {
best = n
}
}
diff --git a/internal/ingestion/og_test.go b/internal/ingestion/og_test.go
index 9941951..c673ca1 100644
--- a/internal/ingestion/og_test.go
+++ b/internal/ingestion/og_test.go
@@ -6,6 +6,8 @@ import (
"strings"
"testing"
+ "github.com/PuerkitoBio/goquery"
+
"pete/internal/safehttp"
)
@@ -78,3 +80,27 @@ func TestFirstImgSrc(t *testing.T) {
t.Errorf("empty input should yield empty, got %q", got)
}
}
+
+// TestExtractBodyCharsANNLayout regresses a false-positive paywall flag on
+// Anime News Network: their layout has no / tags and most of
+// the prose lives outside tags, inside
containers.
+// Before the fix, body-length came out below PaywallBodyThreshold and the
+// poller flagged the story as gated.
+func TestExtractBodyCharsANNLayout(t *testing.T) {
+ body := `
+
+
Yen Press Licenses Wistoria
+
+
Yen Press announced on Friday during its panel that it has licensed seven manga and two novel series for release in December 2026 and winter 2027.
+ Snowmelt and Agapanthus is a manga series being released in the coming season; the publisher described it as a romance story with supernatural undertones, drawn in a delicate watercolor style that should appeal to fans of slice-of-life fantasy. Witch and Cat follows a young magic apprentice and her talking familiar through a series of low-stakes adventures across a sleepy countryside town, leaning into cozy atmosphere over plot. Sinful Is the Angel Who Loves is positioned for an older audience and runs in the publisher's mature imprint. Each release will be available in print and digital across English-language markets, with the first volumes scheduled for the December 2026 wave.
+
+
+
`
+ doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+ if n := extractBodyChars(doc); n < PaywallBodyThreshold {
+ t.Fatalf("extractBodyChars = %d, want >= %d (ANN-style body was incorrectly treated as paywalled)", n, PaywallBodyThreshold)
+ }
+}