From e26b69e43f5e74187b969ac994ebad8bf4bf080b Mon Sep 17 00:00:00 2001
From: prosolis <5590409+prosolis@users.noreply.github.com>
Date: Fri, 22 May 2026 17:53:54 -0700
Subject: [PATCH] Paywall detection: swap to Wayback snapshot when article body
is gated
Fetch each new article once and measure visible text. If body is below
500 chars (or fetch fails), resolve a Wayback snapshot via the
archive.org/wayback/available API and use that URL for both the og:image
fallback and the posted link. Dedup keys stay derived from the original
URL so paywalled/non-paywalled hits collide as before.
- New: internal/ingestion/article.go (FetchArticleMeta via goquery)
- New: internal/ingestion/wayback.go (ResolveWayback)
- Removed: internal/ingestion/og.go (folded into article.go)
- poller.go: dedup first, then one article fetch, then snapshot fallback
---
go.mod | 2 +-
internal/ingestion/article.go | 139 ++++++++++++++++++++++++++++++++++
internal/ingestion/og.go | 98 ------------------------
internal/ingestion/poller.go | 47 +++++++++---
internal/ingestion/wayback.go | 62 +++++++++++++++
5 files changed, 238 insertions(+), 110 deletions(-)
create mode 100644 internal/ingestion/article.go
delete mode 100644 internal/ingestion/og.go
create mode 100644 internal/ingestion/wayback.go
diff --git a/go.mod b/go.mod
index dfd488e..d73b160 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@ module pete
go 1.25.0
require (
+ github.com/PuerkitoBio/goquery v1.12.0
github.com/mmcdole/gofeed v1.3.0
gopkg.in/yaml.v3 v3.0.1
maunium.net/go/mautrix v0.28.0
@@ -11,7 +12,6 @@ require (
require (
filippo.io/edwards25519 v1.2.0 // indirect
- github.com/PuerkitoBio/goquery v1.12.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/google/uuid v1.6.0 // indirect
diff --git a/internal/ingestion/article.go b/internal/ingestion/article.go
new file mode 100644
index 0000000..29836a8
--- /dev/null
+++ b/internal/ingestion/article.go
@@ -0,0 +1,139 @@
+package ingestion
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/PuerkitoBio/goquery"
+)
+
+// resolveURL turns a possibly-relative URL into an absolute one using
+// the base URL. Returns the raw input on parse failure.
+func resolveURL(base, ref string) string {
+ ref = strings.TrimSpace(ref)
+ if ref == "" {
+ return ""
+ }
+ if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
+ return ref
+ }
+ if strings.HasPrefix(ref, "//") {
+ if i := strings.Index(base, "://"); i > 0 {
+ return base[:i+1] + ref
+ }
+ return "https:" + ref
+ }
+ i := strings.Index(base, "://")
+ if i < 0 {
+ return ref
+ }
+ rest := base[i+3:]
+ slash := strings.Index(rest, "/")
+ if slash < 0 {
+ return fmt.Sprintf("%s%s", base, ref)
+ }
+ host := base[:i+3+slash]
+ if strings.HasPrefix(ref, "/") {
+ return host + ref
+ }
+ return host + "/" + ref
+}
+
+// PaywallBodyThreshold is the minimum visible body length (in characters)
+// for an article to be considered accessible. Anything below this is treated
+// as paywalled / gated, and the caller should fall back to an archive snapshot.
+const PaywallBodyThreshold = 500
+
+// ArticleMeta is what we can learn from fetching an article page directly.
+type ArticleMeta struct {
+ ImageURL string // og:image or twitter:image, absolute URL
+ BodyChars int // length of extracted visible body text
+ Fetched bool // true if we got an HTTP 200 with HTML
+}
+
+var articleClient = &http.Client{Timeout: 12 * time.Second}
+
+// FetchArticleMeta fetches an article URL and returns its og:image plus the
+// length of the visible body text (concatenation of
tags under
+// or , falling back to all tags). Returns Fetched=false on any
+// network/HTTP failure so callers can branch on accessibility.
+func FetchArticleMeta(articleURL string) ArticleMeta {
+ if articleURL == "" {
+ return ArticleMeta{}
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
+ if err != nil {
+ return ArticleMeta{}
+ }
+ req.Header.Set("User-Agent", userAgent)
+ req.Header.Set("Accept", "text/html,application/xhtml+xml")
+
+ resp, err := articleClient.Do(req)
+ if err != nil {
+ return ArticleMeta{}
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return ArticleMeta{}
+ }
+
+ doc, err := goquery.NewDocumentFromReader(resp.Body)
+ if err != nil {
+ return ArticleMeta{}
+ }
+
+ return ArticleMeta{
+ ImageURL: extractOGImage(doc, articleURL),
+ BodyChars: extractBodyChars(doc),
+ Fetched: true,
+ }
+}
+
+// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
+// that only care about the image. Returns "" when not found.
+func FetchOGImage(articleURL string) string {
+ return FetchArticleMeta(articleURL).ImageURL
+}
+
+func extractOGImage(doc *goquery.Document, base string) string {
+ selectors := []string{
+ `meta[property="og:image:secure_url"]`,
+ `meta[property="og:image:url"]`,
+ `meta[property="og:image"]`,
+ `meta[name="twitter:image:src"]`,
+ `meta[name="twitter:image"]`,
+ }
+ for _, sel := range selectors {
+ if v, ok := doc.Find(sel).First().Attr("content"); ok && strings.TrimSpace(v) != "" {
+ return resolveURL(base, strings.TrimSpace(v))
+ }
+ }
+ return ""
+}
+
+// extractBodyChars concatenates the text of
tags inside or
+// , falling back to all tags, and returns the trimmed length.
+func extractBodyChars(doc *goquery.Document) int {
+ sel := doc.Find("article p, main p")
+ if sel.Length() == 0 {
+ sel = doc.Find("p")
+ }
+ var b strings.Builder
+ sel.Each(func(_ int, s *goquery.Selection) {
+ t := strings.TrimSpace(s.Text())
+ if t == "" {
+ return
+ }
+ if b.Len() > 0 {
+ b.WriteByte(' ')
+ }
+ b.WriteString(t)
+ })
+ return len(strings.TrimSpace(b.String()))
+}
diff --git a/internal/ingestion/og.go b/internal/ingestion/og.go
deleted file mode 100644
index d2899bf..0000000
--- a/internal/ingestion/og.go
+++ /dev/null
@@ -1,98 +0,0 @@
-package ingestion
-
-import (
- "context"
- "fmt"
- "html"
- "io"
- "net/http"
- "regexp"
- "strings"
- "time"
-)
-
-// Match in either attribute order,
-// and the twitter:image variant.
-var metaImageRe = regexp.MustCompile(
- `(?is)]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["'][^>]*content\s*=\s*["']([^"']+)["']` +
- `|]*content\s*=\s*["']([^"']+)["'][^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["']`)
-
-var ogClient = &http.Client{Timeout: 8 * time.Second}
-
-// FetchOGImage fetches the article URL and returns the first og:image or
-// twitter:image found in the . Returns "" on any failure — callers
-// should treat this as best-effort.
-func FetchOGImage(articleURL string) string {
- if articleURL == "" {
- return ""
- }
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
- if err != nil {
- return ""
- }
- req.Header.Set("User-Agent", userAgent)
- req.Header.Set("Accept", "text/html,application/xhtml+xml")
-
- resp, err := ogClient.Do(req)
- if err != nil {
- return ""
- }
- defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- return ""
- }
-
- // og:image lives in ; cap read at 512KB to avoid pulling whole articles.
- body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
- if err != nil {
- return ""
- }
-
- m := metaImageRe.FindSubmatch(body)
- if len(m) == 0 {
- return ""
- }
- // One of the two capture groups will be non-empty depending on attr order.
- for i := 1; i < len(m); i++ {
- if len(m[i]) > 0 {
- return resolveURL(articleURL, html.UnescapeString(string(m[i])))
- }
- }
- return ""
-}
-
-// resolveURL turns a possibly-relative image URL into an absolute one using
-// the article URL as the base. Returns the raw input on parse failure.
-func resolveURL(base, ref string) string {
- ref = strings.TrimSpace(ref)
- if ref == "" {
- return ""
- }
- if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
- return ref
- }
- if strings.HasPrefix(ref, "//") {
- if i := strings.Index(base, "://"); i > 0 {
- return base[:i+1] + ref
- }
- return "https:" + ref
- }
- // relative path — naive join: scheme://host + ref
- i := strings.Index(base, "://")
- if i < 0 {
- return ref
- }
- rest := base[i+3:]
- slash := strings.Index(rest, "/")
- if slash < 0 {
- return fmt.Sprintf("%s%s", base, ref)
- }
- host := base[:i+3+slash]
- if strings.HasPrefix(ref, "/") {
- return host + ref
- }
- return host + "/" + ref
-}
diff --git a/internal/ingestion/poller.go b/internal/ingestion/poller.go
index 57c1e87..c2453c0 100644
--- a/internal/ingestion/poller.go
+++ b/internal/ingestion/poller.go
@@ -106,17 +106,10 @@ func (p *Poller) pollOnceWithErr(src config.SourceConfig) error {
continue
}
- // Last-resort image fallback: feed gave us nothing, scrape og:image
- // from the article page. Only runs for genuinely new items.
- if items[i].ImageURL == "" {
- if og := FetchOGImage(items[i].ArticleURL); og != "" {
- items[i].ImageURL = og
- slog.Debug("og:image fallback used",
- "guid", items[i].GUID, "url", items[i].ArticleURL, "image", og)
- }
- }
-
- canonical := dedup.CanonicalURL(items[i].ArticleURL)
+ // Dedup checks first — canonical and headline use the ORIGINAL URL so
+ // they stay stable whether we end up swapping to a Wayback snapshot.
+ originalURL := items[i].ArticleURL
+ canonical := dedup.CanonicalURL(originalURL)
headlineNorm := dedup.NormalizeHeadline(items[i].Headline)
if storage.IsCanonicalSeen(canonical) {
@@ -130,6 +123,38 @@ func (p *Poller) pollOnceWithErr(src config.SourceConfig) error {
continue
}
+ // Genuinely new story — fetch the article page once. We use it for:
+ // 1. og:image fallback when the feed didn't give us one.
+ // 2. Paywall detection: visible body text below the threshold means
+ // the article is gated, so swap in a Wayback snapshot for both
+ // Pete (image) and the reader (posted link).
+ meta := FetchArticleMeta(originalURL)
+ paywalled := !meta.Fetched || meta.BodyChars < PaywallBodyThreshold
+ if paywalled {
+ if snap := ResolveWayback(originalURL); snap != "" {
+ snapMeta := FetchArticleMeta(snap)
+ if snapMeta.Fetched {
+ items[i].ArticleURL = snap
+ if items[i].ImageURL == "" && snapMeta.ImageURL != "" {
+ items[i].ImageURL = snapMeta.ImageURL
+ }
+ slog.Info("paywall detected, using wayback snapshot",
+ "guid", items[i].GUID, "original", originalURL,
+ "snapshot", snap, "body_chars", meta.BodyChars)
+ } else {
+ slog.Debug("paywall detected but snapshot fetch failed",
+ "guid", items[i].GUID, "original", originalURL, "snapshot", snap)
+ }
+ } else {
+ slog.Debug("paywall detected but no wayback snapshot available",
+ "guid", items[i].GUID, "url", originalURL, "body_chars", meta.BodyChars)
+ }
+ } else if items[i].ImageURL == "" && meta.ImageURL != "" {
+ items[i].ImageURL = meta.ImageURL
+ slog.Debug("og:image fallback used",
+ "guid", items[i].GUID, "url", originalURL, "image", meta.ImageURL)
+ }
+
// Stamp source metadata onto the item
items[i].Source = src.Name
items[i].FeedHint = src.FeedHint
diff --git a/internal/ingestion/wayback.go b/internal/ingestion/wayback.go
new file mode 100644
index 0000000..82015e8
--- /dev/null
+++ b/internal/ingestion/wayback.go
@@ -0,0 +1,62 @@
+package ingestion
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/url"
+ "time"
+)
+
+var waybackClient = &http.Client{Timeout: 10 * time.Second}
+
+type waybackResp struct {
+ ArchivedSnapshots struct {
+ Closest struct {
+ Available bool `json:"available"`
+ URL string `json:"url"`
+ Status string `json:"status"`
+ } `json:"closest"`
+ } `json:"archived_snapshots"`
+}
+
+// ResolveWayback asks the Internet Archive's Wayback availability API for the
+// closest snapshot of the given URL. Returns the snapshot URL (https) or ""
+// if no snapshot exists or the request fails.
+func ResolveWayback(articleURL string) string {
+ if articleURL == "" {
+ return ""
+ }
+ api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL)
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, "GET", api, nil)
+ if err != nil {
+ return ""
+ }
+ req.Header.Set("User-Agent", userAgent)
+
+ resp, err := waybackClient.Do(req)
+ if err != nil {
+ return ""
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return ""
+ }
+
+ var wr waybackResp
+ if err := json.NewDecoder(resp.Body).Decode(&wr); err != nil {
+ return ""
+ }
+ snap := wr.ArchivedSnapshots.Closest
+ if !snap.Available || snap.Status != "200" || snap.URL == "" {
+ return ""
+ }
+ // Wayback sometimes returns http:// even when https is available.
+ if len(snap.URL) > 7 && snap.URL[:7] == "http://" {
+ return "https://" + snap.URL[7:]
+ }
+ return snap.URL
+}