tags. Fall back to the container's raw text in that case.
+ if len(out) < PaywallBodyThreshold {
+ if alt := containerText(doc.Find("article, main").First()); len(alt) > len(out) {
+ out = alt
+ }
+ }
+ if len(out) > MaxBodyChars {
+ out = out[:MaxBodyChars]
+ }
+ return out
+}
+
+func joinParagraphText(sel *goquery.Selection) string {
var b strings.Builder
sel.Each(func(_ int, s *goquery.Selection) {
t := strings.TrimSpace(s.Text())
@@ -332,15 +390,17 @@ func extractBodyText(doc *goquery.Document) string {
b.WriteString("\n\n")
}
b.WriteString(t)
- if b.Len() >= MaxBodyChars {
- return
- }
})
- out := strings.TrimSpace(b.String())
- if len(out) > MaxBodyChars {
- out = out[:MaxBodyChars]
+ return strings.TrimSpace(b.String())
+}
+
+func containerText(sel *goquery.Selection) string {
+ if sel.Length() == 0 {
+ return ""
}
- return out
+ // Collapse whitespace runs so tags inside tags, and returns the trimmed length.
-func extractBodyChars(doc *goquery.Document) int {
- sel := doc.Find("article p, main p")
- if sel.Length() == 0 {
- sel = doc.Find("p")
+// pickImageSrc returns the most useful image URL from an tags inside tags, then the raw text inside .
+func extractBodyChars(doc *goquery.Document) int {
+ best := len(joinParagraphText(doc.Find("article p, main p")))
+ if best < PaywallBodyThreshold {
+ if n := len(joinParagraphText(doc.Find("p"))); n > best {
+ best = n
+ }
+ }
+ if best < PaywallBodyThreshold {
+ if n := len(containerText(doc.Find("article, main").First())); n > best {
+ best = n
+ }
+ }
+ return best
}
diff --git a/main.go b/main.go
index 0e87cd9..c9485ac 100644
--- a/main.go
+++ b/main.go
@@ -27,6 +27,7 @@ func main() {
test := flag.Bool("test", false, "post one story to verify the full pipeline, then exit")
testSource := flag.String("test-source", "", "source name to use for -test (default: first enabled)")
local := flag.Bool("local", false, "web/RSS-only mode: poll feeds and serve the web UI, no Matrix login or posting")
+ backfillPaywall := flag.Bool("backfill-paywall", false, "re-check paywalled rows with current detection logic; clear false positives and fill missing thumbnails, then exit")
flag.Parse()
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
@@ -57,6 +58,11 @@ func main() {
return
}
+ if *backfillPaywall {
+ runBackfillPaywall()
+ return
+ }
+
// Test mode: post one story to verify the full pipeline, then exit
if *test {
runTest(cfg, *testSource)
@@ -373,3 +379,50 @@ func runTest(cfg *config.Config, sourceName string) {
func timeNow() int64 {
return time.Now().Unix()
}
+
+// runBackfillPaywall re-runs the current paywall detection logic against
+// every row marked paywalled=1. For each row we re-fetch the stored article
+// URL; if the new logic no longer considers it gated we clear the flag and,
+// when the row has no thumbnail, populate it from the live og:image.
+func runBackfillPaywall() {
+ rows, err := storage.Get().Query(`SELECT guid, article_url, image_url FROM stories WHERE paywalled = 1`)
+ if err != nil {
+ slog.Error("backfill: query failed", "err", err)
+ os.Exit(1)
+ }
+ type rec struct{ guid, url, img string }
+ var pending []rec
+ for rows.Next() {
+ var r rec
+ if err := rows.Scan(&r.guid, &r.url, &r.img); err != nil {
+ slog.Error("backfill: scan failed", "err", err)
+ continue
+ }
+ pending = append(pending, r)
+ }
+ rows.Close()
+ slog.Info("backfill: starting", "candidates", len(pending))
+
+ cleared, imaged, stillGated := 0, 0, 0
+ for _, r := range pending {
+ meta := ingestion.FetchArticleMeta(r.url)
+ gated := meta.Gated() || (meta.Fetched && meta.BodyChars < ingestion.PaywallBodyThreshold)
+ if gated {
+ stillGated++
+ slog.Info("backfill: still gated", "guid", r.guid, "status", meta.Status, "body_chars", meta.BodyChars)
+ continue
+ }
+ newImg := r.img
+ if newImg == "" && meta.ImageURL != "" {
+ newImg = meta.ImageURL
+ imaged++
+ }
+ if _, err := storage.Get().Exec(`UPDATE stories SET paywalled = 0, image_url = ? WHERE guid = ?`, newImg, r.guid); err != nil {
+ slog.Error("backfill: update failed", "guid", r.guid, "err", err)
+ continue
+ }
+ cleared++
+ slog.Info("backfill: cleared paywall flag", "guid", r.guid, "filled_image", newImg != r.img)
+ }
+ slog.Info("backfill complete", "cleared", cleared, "still_gated", stillGated, "thumbnails_filled", imaged)
+}
-laden markup doesn't produce a wall
+ // of blank lines, but preserve enough structure to look like prose.
+ return strings.Join(strings.Fields(sel.Text()), " ")
}
func extractOGImage(doc *goquery.Document, base string) string {
@@ -356,26 +416,99 @@ func extractOGImage(doc *goquery.Document, base string) string {
return resolveURL(base, strings.TrimSpace(v))
}
}
+ // Sites like Hardcore Gaming 101 publish no og:image — fall back to the
+ // WordPress featured image (.wp-post-image), then the first reasonably-
+ // sized inside
selection,
+// preferring data-src/data-lazy-src (lazy-load) over src when src is a
+// placeholder (empty or data: URI).
+func pickImageSrc(s *goquery.Selection, src string) string {
+ src = strings.TrimSpace(src)
+ if !strings.HasPrefix(src, "data:") && src != "" {
+ return src
}
- var b strings.Builder
- sel.Each(func(_ int, s *goquery.Selection) {
- t := strings.TrimSpace(s.Text())
- if t == "" {
- return
+ for _, attr := range []string{"data-src", "data-lazy-src", "data-original"} {
+ if v, ok := s.Attr(attr); ok {
+ v = strings.TrimSpace(v)
+ if v != "" && !strings.HasPrefix(v, "data:") {
+ return v
+ }
}
- if b.Len() > 0 {
- b.WriteByte(' ')
- }
- b.WriteString(t)
- })
- return len(strings.TrimSpace(b.String()))
+ }
+ return ""
+}
+
+// looksLikeContentImage filters out obvious non-article images: gravatars,
+// social-share icons, tracking pixels. Anything with declared dimensions
+// under 100px on either side is treated as chrome.
+func looksLikeContentImage(s *goquery.Selection, url string) bool {
+ low := strings.ToLower(url)
+ for _, bad := range []string{"gravatar.com", "/avatar/", "/icons/", "share-", "pixel.gif", "sprite"} {
+ if strings.Contains(low, bad) {
+ return false
+ }
+ }
+ if w := atoiAttr(s, "width"); w > 0 && w < 100 {
+ return false
+ }
+ if h := atoiAttr(s, "height"); h > 0 && h < 100 {
+ return false
+ }
+ return true
+}
+
+func atoiAttr(s *goquery.Selection, name string) int {
+ v, ok := s.Attr(name)
+ if !ok {
+ return 0
+ }
+ n := 0
+ for _, c := range strings.TrimSpace(v) {
+ if c < '0' || c > '9' {
+ break
+ }
+ n = n*10 + int(c-'0')
+ }
+ return n
+}
+
+// extractBodyChars returns the visible body length used to gauge whether a
+// page looks like a real article. It tries
+// instead of