Paywall detection: swap to Wayback snapshot when article body is gated

Fetch each new article once and measure visible <p> text. If body is below
500 chars (or fetch fails), resolve a Wayback snapshot via the
archive.org/wayback/available API and use that URL for both the og:image
fallback and the posted link. Dedup keys stay derived from the original
URL so paywalled/non-paywalled hits collide as before.

- New: internal/ingestion/article.go (FetchArticleMeta via goquery)
- New: internal/ingestion/wayback.go (ResolveWayback)
- Removed: internal/ingestion/og.go (folded into article.go)
- poller.go: dedup first, then one article fetch, then snapshot fallback

go.mod

@@ -3,6 +3,7 @@ module pete
 go 1.25.0
 
 require (
+	github.com/PuerkitoBio/goquery v1.12.0
 	github.com/mmcdole/gofeed v1.3.0
 	gopkg.in/yaml.v3 v3.0.1
 	maunium.net/go/mautrix v0.28.0
@@ -11,7 +12,6 @@ require (
 
 require (
 	filippo.io/edwards25519 v1.2.0 // indirect
-	github.com/PuerkitoBio/goquery v1.12.0 // indirect
 	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect

internal/ingestion/article.go

@@ -0,0 +1,139 @@
+package ingestion
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// resolveURL turns a possibly-relative URL into an absolute one using
+// the base URL. Returns the raw input on parse failure.
+func resolveURL(base, ref string) string {
+	ref = strings.TrimSpace(ref)
+	if ref == "" {
+		return ""
+	}
+	if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
+		return ref
+	}
+	if strings.HasPrefix(ref, "//") {
+		if i := strings.Index(base, "://"); i > 0 {
+			return base[:i+1] + ref
+		}
+		return "https:" + ref
+	}
+	i := strings.Index(base, "://")
+	if i < 0 {
+		return ref
+	}
+	rest := base[i+3:]
+	slash := strings.Index(rest, "/")
+	if slash < 0 {
+		return fmt.Sprintf("%s%s", base, ref)
+	}
+	host := base[:i+3+slash]
+	if strings.HasPrefix(ref, "/") {
+		return host + ref
+	}
+	return host + "/" + ref
+}
+
+// PaywallBodyThreshold is the minimum visible body length (in characters)
+// for an article to be considered accessible. Anything below this is treated
+// as paywalled / gated, and the caller should fall back to an archive snapshot.
+const PaywallBodyThreshold = 500
+
+// ArticleMeta is what we can learn from fetching an article page directly.
+type ArticleMeta struct {
+	ImageURL  string // og:image or twitter:image, absolute URL
+	BodyChars int    // length of extracted visible body text
+	Fetched   bool   // true if we got an HTTP 200 with HTML
+}
+
+var articleClient = &http.Client{Timeout: 12 * time.Second}
+
+// FetchArticleMeta fetches an article URL and returns its og:image plus the
+// length of the visible body text (concatenation of <p> tags under <article>
+// or <main>, falling back to all <p> tags). Returns Fetched=false on any
+// network/HTTP failure so callers can branch on accessibility.
+func FetchArticleMeta(articleURL string) ArticleMeta {
+	if articleURL == "" {
+		return ArticleMeta{}
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
+	if err != nil {
+		return ArticleMeta{}
+	}
+	req.Header.Set("User-Agent", userAgent)
+	req.Header.Set("Accept", "text/html,application/xhtml+xml")
+
+	resp, err := articleClient.Do(req)
+	if err != nil {
+		return ArticleMeta{}
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return ArticleMeta{}
+	}
+
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return ArticleMeta{}
+	}
+
+	return ArticleMeta{
+		ImageURL:  extractOGImage(doc, articleURL),
+		BodyChars: extractBodyChars(doc),
+		Fetched:   true,
+	}
+}
+
+// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
+// that only care about the image. Returns "" when not found.
+func FetchOGImage(articleURL string) string {
+	return FetchArticleMeta(articleURL).ImageURL
+}
+
+func extractOGImage(doc *goquery.Document, base string) string {
+	selectors := []string{
+		`meta[property="og:image:secure_url"]`,
+		`meta[property="og:image:url"]`,
+		`meta[property="og:image"]`,
+		`meta[name="twitter:image:src"]`,
+		`meta[name="twitter:image"]`,
+	}
+	for _, sel := range selectors {
+		if v, ok := doc.Find(sel).First().Attr("content"); ok && strings.TrimSpace(v) != "" {
+			return resolveURL(base, strings.TrimSpace(v))
+		}
+	}
+	return ""
+}
+
+// extractBodyChars concatenates the text of <p> tags inside <article> or
+// <main>, falling back to all <p> tags, and returns the trimmed length.
+func extractBodyChars(doc *goquery.Document) int {
+	sel := doc.Find("article p, main p")
+	if sel.Length() == 0 {
+		sel = doc.Find("p")
+	}
+	var b strings.Builder
+	sel.Each(func(_ int, s *goquery.Selection) {
+		t := strings.TrimSpace(s.Text())
+		if t == "" {
+			return
+		}
+		if b.Len() > 0 {
+			b.WriteByte(' ')
+		}
+		b.WriteString(t)
+	})
+	return len(strings.TrimSpace(b.String()))
+}

internal/ingestion/og.go

@@ -1,98 +0,0 @@
-package ingestion
-
-import (
-	"context"
-	"fmt"
-	"html"
-	"io"
-	"net/http"
-	"regexp"
-	"strings"
-	"time"
-)
-
-// Match <meta property="og:image" content="..."> in either attribute order,
-// and the twitter:image variant.
-var metaImageRe = regexp.MustCompile(
-	`(?is)<meta\s+[^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["'][^>]*content\s*=\s*["']([^"']+)["']` +
-		`|<meta\s+[^>]*content\s*=\s*["']([^"']+)["'][^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["']`)
-
-var ogClient = &http.Client{Timeout: 8 * time.Second}
-
-// FetchOGImage fetches the article URL and returns the first og:image or
-// twitter:image found in the <head>. Returns "" on any failure — callers
-// should treat this as best-effort.
-func FetchOGImage(articleURL string) string {
-	if articleURL == "" {
-		return ""
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
-	if err != nil {
-		return ""
-	}
-	req.Header.Set("User-Agent", userAgent)
-	req.Header.Set("Accept", "text/html,application/xhtml+xml")
-
-	resp, err := ogClient.Do(req)
-	if err != nil {
-		return ""
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return ""
-	}
-
-	// og:image lives in <head>; cap read at 512KB to avoid pulling whole articles.
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
-	if err != nil {
-		return ""
-	}
-
-	m := metaImageRe.FindSubmatch(body)
-	if len(m) == 0 {
-		return ""
-	}
-	// One of the two capture groups will be non-empty depending on attr order.
-	for i := 1; i < len(m); i++ {
-		if len(m[i]) > 0 {
-			return resolveURL(articleURL, html.UnescapeString(string(m[i])))
-		}
-	}
-	return ""
-}
-
-// resolveURL turns a possibly-relative image URL into an absolute one using
-// the article URL as the base. Returns the raw input on parse failure.
-func resolveURL(base, ref string) string {
-	ref = strings.TrimSpace(ref)
-	if ref == "" {
-		return ""
-	}
-	if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
-		return ref
-	}
-	if strings.HasPrefix(ref, "//") {
-		if i := strings.Index(base, "://"); i > 0 {
-			return base[:i+1] + ref
-		}
-		return "https:" + ref
-	}
-	// relative path — naive join: scheme://host + ref
-	i := strings.Index(base, "://")
-	if i < 0 {
-		return ref
-	}
-	rest := base[i+3:]
-	slash := strings.Index(rest, "/")
-	if slash < 0 {
-		return fmt.Sprintf("%s%s", base, ref)
-	}
-	host := base[:i+3+slash]
-	if strings.HasPrefix(ref, "/") {
-		return host + ref
-	}
-	return host + "/" + ref
-}

internal/ingestion/poller.go

@@ -106,17 +106,10 @@ func (p *Poller) pollOnceWithErr(src config.SourceConfig) error {
 			continue
 		}
 
-		// Last-resort image fallback: feed gave us nothing, scrape og:image
+		// Dedup checks first — canonical and headline use the ORIGINAL URL so
-		// from the article page. Only runs for genuinely new items.
+		// they stay stable whether we end up swapping to a Wayback snapshot.
-		if items[i].ImageURL == "" {
+		originalURL := items[i].ArticleURL
-			if og := FetchOGImage(items[i].ArticleURL); og != "" {
+		canonical := dedup.CanonicalURL(originalURL)
-				items[i].ImageURL = og
-				slog.Debug("og:image fallback used",
-					"guid", items[i].GUID, "url", items[i].ArticleURL, "image", og)
-			}
-		}
-
-		canonical := dedup.CanonicalURL(items[i].ArticleURL)
 		headlineNorm := dedup.NormalizeHeadline(items[i].Headline)
 
 		if storage.IsCanonicalSeen(canonical) {
@@ -130,6 +123,38 @@ func (p *Poller) pollOnceWithErr(src config.SourceConfig) error {
 			continue
 		}
 
+		// Genuinely new story — fetch the article page once. We use it for:
+		//   1. og:image fallback when the feed didn't give us one.
+		//   2. Paywall detection: visible body text below the threshold means
+		//      the article is gated, so swap in a Wayback snapshot for both
+		//      Pete (image) and the reader (posted link).
+		meta := FetchArticleMeta(originalURL)
+		paywalled := !meta.Fetched || meta.BodyChars < PaywallBodyThreshold
+		if paywalled {
+			if snap := ResolveWayback(originalURL); snap != "" {
+				snapMeta := FetchArticleMeta(snap)
+				if snapMeta.Fetched {
+					items[i].ArticleURL = snap
+					if items[i].ImageURL == "" && snapMeta.ImageURL != "" {
+						items[i].ImageURL = snapMeta.ImageURL
+					}
+					slog.Info("paywall detected, using wayback snapshot",
+						"guid", items[i].GUID, "original", originalURL,
+						"snapshot", snap, "body_chars", meta.BodyChars)
+				} else {
+					slog.Debug("paywall detected but snapshot fetch failed",
+						"guid", items[i].GUID, "original", originalURL, "snapshot", snap)
+				}
+			} else {
+				slog.Debug("paywall detected but no wayback snapshot available",
+					"guid", items[i].GUID, "url", originalURL, "body_chars", meta.BodyChars)
+			}
+		} else if items[i].ImageURL == "" && meta.ImageURL != "" {
+			items[i].ImageURL = meta.ImageURL
+			slog.Debug("og:image fallback used",
+				"guid", items[i].GUID, "url", originalURL, "image", meta.ImageURL)
+		}
+
 		// Stamp source metadata onto the item
 		items[i].Source = src.Name
 		items[i].FeedHint = src.FeedHint

internal/ingestion/wayback.go

@@ -0,0 +1,62 @@
+package ingestion
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+var waybackClient = &http.Client{Timeout: 10 * time.Second}
+
+type waybackResp struct {
+	ArchivedSnapshots struct {
+		Closest struct {
+			Available bool   `json:"available"`
+			URL       string `json:"url"`
+			Status    string `json:"status"`
+		} `json:"closest"`
+	} `json:"archived_snapshots"`
+}
+
+// ResolveWayback asks the Internet Archive's Wayback availability API for the
+// closest snapshot of the given URL. Returns the snapshot URL (https) or ""
+// if no snapshot exists or the request fails.
+func ResolveWayback(articleURL string) string {
+	if articleURL == "" {
+		return ""
+	}
+	api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, "GET", api, nil)
+	if err != nil {
+		return ""
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	resp, err := waybackClient.Do(req)
+	if err != nil {
+		return ""
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return ""
+	}
+
+	var wr waybackResp
+	if err := json.NewDecoder(resp.Body).Decode(&wr); err != nil {
+		return ""
+	}
+	snap := wr.ArchivedSnapshots.Closest
+	if !snap.Available || snap.Status != "200" || snap.URL == "" {
+		return ""
+	}
+	// Wayback sometimes returns http:// even when https is available.
+	if len(snap.URL) > 7 && snap.URL[:7] == "http://" {
+		return "https://" + snap.URL[7:]
+	}
+	return snap.URL
+}