Paywall detection: swap to Wayback snapshot when article body is gated

Fetch each new article once and measure visible <p> text. If body is below
500 chars (or fetch fails), resolve a Wayback snapshot via the
archive.org/wayback/available API and use that URL for both the og:image
fallback and the posted link. Dedup keys stay derived from the original
URL so paywalled/non-paywalled hits collide as before.

- New: internal/ingestion/article.go (FetchArticleMeta via goquery)
- New: internal/ingestion/wayback.go (ResolveWayback)
- Removed: internal/ingestion/og.go (folded into article.go)
- poller.go: dedup first, then one article fetch, then snapshot fallback

internal/ingestion/article.go

@@ -0,0 +1,139 @@
+package ingestion
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// resolveURL turns a possibly-relative URL into an absolute one using
+// the base URL. Returns the raw input on parse failure.
+func resolveURL(base, ref string) string {
+	ref = strings.TrimSpace(ref)
+	if ref == "" {
+		return ""
+	}
+	if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
+		return ref
+	}
+	if strings.HasPrefix(ref, "//") {
+		if i := strings.Index(base, "://"); i > 0 {
+			return base[:i+1] + ref
+		}
+		return "https:" + ref
+	}
+	i := strings.Index(base, "://")
+	if i < 0 {
+		return ref
+	}
+	rest := base[i+3:]
+	slash := strings.Index(rest, "/")
+	if slash < 0 {
+		return fmt.Sprintf("%s%s", base, ref)
+	}
+	host := base[:i+3+slash]
+	if strings.HasPrefix(ref, "/") {
+		return host + ref
+	}
+	return host + "/" + ref
+}
+
+// PaywallBodyThreshold is the minimum visible body length (in characters)
+// for an article to be considered accessible. Anything below this is treated
+// as paywalled / gated, and the caller should fall back to an archive snapshot.
+const PaywallBodyThreshold = 500
+
+// ArticleMeta is what we can learn from fetching an article page directly.
+type ArticleMeta struct {
+	ImageURL  string // og:image or twitter:image, absolute URL
+	BodyChars int    // length of extracted visible body text
+	Fetched   bool   // true if we got an HTTP 200 with HTML
+}
+
+var articleClient = &http.Client{Timeout: 12 * time.Second}
+
+// FetchArticleMeta fetches an article URL and returns its og:image plus the
+// length of the visible body text (concatenation of <p> tags under <article>
+// or <main>, falling back to all <p> tags). Returns Fetched=false on any
+// network/HTTP failure so callers can branch on accessibility.
+func FetchArticleMeta(articleURL string) ArticleMeta {
+	if articleURL == "" {
+		return ArticleMeta{}
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil)
+	if err != nil {
+		return ArticleMeta{}
+	}
+	req.Header.Set("User-Agent", userAgent)
+	req.Header.Set("Accept", "text/html,application/xhtml+xml")
+
+	resp, err := articleClient.Do(req)
+	if err != nil {
+		return ArticleMeta{}
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return ArticleMeta{}
+	}
+
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return ArticleMeta{}
+	}
+
+	return ArticleMeta{
+		ImageURL:  extractOGImage(doc, articleURL),
+		BodyChars: extractBodyChars(doc),
+		Fetched:   true,
+	}
+}
+
+// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers
+// that only care about the image. Returns "" when not found.
+func FetchOGImage(articleURL string) string {
+	return FetchArticleMeta(articleURL).ImageURL
+}
+
+func extractOGImage(doc *goquery.Document, base string) string {
+	selectors := []string{
+		`meta[property="og:image:secure_url"]`,
+		`meta[property="og:image:url"]`,
+		`meta[property="og:image"]`,
+		`meta[name="twitter:image:src"]`,
+		`meta[name="twitter:image"]`,
+	}
+	for _, sel := range selectors {
+		if v, ok := doc.Find(sel).First().Attr("content"); ok && strings.TrimSpace(v) != "" {
+			return resolveURL(base, strings.TrimSpace(v))
+		}
+	}
+	return ""
+}
+
+// extractBodyChars concatenates the text of <p> tags inside <article> or
+// <main>, falling back to all <p> tags, and returns the trimmed length.
+func extractBodyChars(doc *goquery.Document) int {
+	sel := doc.Find("article p, main p")
+	if sel.Length() == 0 {
+		sel = doc.Find("p")
+	}
+	var b strings.Builder
+	sel.Each(func(_ int, s *goquery.Selection) {
+		t := strings.TrimSpace(s.Text())
+		if t == "" {
+			return
+		}
+		if b.Len() > 0 {
+			b.WriteByte(' ')
+		}
+		b.WriteString(t)
+	})
+	return len(strings.TrimSpace(b.String()))
+}