diff --git a/go.mod b/go.mod index dfd488e..d73b160 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module pete go 1.25.0 require ( + github.com/PuerkitoBio/goquery v1.12.0 github.com/mmcdole/gofeed v1.3.0 gopkg.in/yaml.v3 v3.0.1 maunium.net/go/mautrix v0.28.0 @@ -11,7 +12,6 @@ require ( require ( filippo.io/edwards25519 v1.2.0 // indirect - github.com/PuerkitoBio/goquery v1.12.0 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect diff --git a/internal/ingestion/article.go b/internal/ingestion/article.go new file mode 100644 index 0000000..29836a8 --- /dev/null +++ b/internal/ingestion/article.go @@ -0,0 +1,139 @@ +package ingestion + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" +) + +// resolveURL turns a possibly-relative URL into an absolute one using +// the base URL. Returns the raw input on parse failure. +func resolveURL(base, ref string) string { + ref = strings.TrimSpace(ref) + if ref == "" { + return "" + } + if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") { + return ref + } + if strings.HasPrefix(ref, "//") { + if i := strings.Index(base, "://"); i > 0 { + return base[:i+1] + ref + } + return "https:" + ref + } + i := strings.Index(base, "://") + if i < 0 { + return ref + } + rest := base[i+3:] + slash := strings.Index(rest, "/") + if slash < 0 { + return fmt.Sprintf("%s%s", base, ref) + } + host := base[:i+3+slash] + if strings.HasPrefix(ref, "/") { + return host + ref + } + return host + "/" + ref +} + +// PaywallBodyThreshold is the minimum visible body length (in characters) +// for an article to be considered accessible. Anything below this is treated +// as paywalled / gated, and the caller should fall back to an archive snapshot. +const PaywallBodyThreshold = 500 + +// ArticleMeta is what we can learn from fetching an article page directly. +type ArticleMeta struct { + ImageURL string // og:image or twitter:image, absolute URL + BodyChars int // length of extracted visible body text + Fetched bool // true if we got an HTTP 200 with HTML +} + +var articleClient = &http.Client{Timeout: 12 * time.Second} + +// FetchArticleMeta fetches an article URL and returns its og:image plus the +// length of the visible body text (concatenation of

tags under

+// or
, falling back to all

tags). Returns Fetched=false on any +// network/HTTP failure so callers can branch on accessibility. +func FetchArticleMeta(articleURL string) ArticleMeta { + if articleURL == "" { + return ArticleMeta{} + } + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil) + if err != nil { + return ArticleMeta{} + } + req.Header.Set("User-Agent", userAgent) + req.Header.Set("Accept", "text/html,application/xhtml+xml") + + resp, err := articleClient.Do(req) + if err != nil { + return ArticleMeta{} + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return ArticleMeta{} + } + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return ArticleMeta{} + } + + return ArticleMeta{ + ImageURL: extractOGImage(doc, articleURL), + BodyChars: extractBodyChars(doc), + Fetched: true, + } +} + +// FetchOGImage is a thin wrapper around FetchArticleMeta kept for callers +// that only care about the image. Returns "" when not found. +func FetchOGImage(articleURL string) string { + return FetchArticleMeta(articleURL).ImageURL +} + +func extractOGImage(doc *goquery.Document, base string) string { + selectors := []string{ + `meta[property="og:image:secure_url"]`, + `meta[property="og:image:url"]`, + `meta[property="og:image"]`, + `meta[name="twitter:image:src"]`, + `meta[name="twitter:image"]`, + } + for _, sel := range selectors { + if v, ok := doc.Find(sel).First().Attr("content"); ok && strings.TrimSpace(v) != "" { + return resolveURL(base, strings.TrimSpace(v)) + } + } + return "" +} + +// extractBodyChars concatenates the text of

tags inside

or +//
, falling back to all

tags, and returns the trimmed length. +func extractBodyChars(doc *goquery.Document) int { + sel := doc.Find("article p, main p") + if sel.Length() == 0 { + sel = doc.Find("p") + } + var b strings.Builder + sel.Each(func(_ int, s *goquery.Selection) { + t := strings.TrimSpace(s.Text()) + if t == "" { + return + } + if b.Len() > 0 { + b.WriteByte(' ') + } + b.WriteString(t) + }) + return len(strings.TrimSpace(b.String())) +} diff --git a/internal/ingestion/og.go b/internal/ingestion/og.go deleted file mode 100644 index d2899bf..0000000 --- a/internal/ingestion/og.go +++ /dev/null @@ -1,98 +0,0 @@ -package ingestion - -import ( - "context" - "fmt" - "html" - "io" - "net/http" - "regexp" - "strings" - "time" -) - -// Match in either attribute order, -// and the twitter:image variant. -var metaImageRe = regexp.MustCompile( - `(?is)]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["'][^>]*content\s*=\s*["']([^"']+)["']` + - `|]*content\s*=\s*["']([^"']+)["'][^>]*(?:property|name)\s*=\s*["'](?:og:image(?::secure_url|:url)?|twitter:image(?::src)?)["']`) - -var ogClient = &http.Client{Timeout: 8 * time.Second} - -// FetchOGImage fetches the article URL and returns the first og:image or -// twitter:image found in the . Returns "" on any failure — callers -// should treat this as best-effort. -func FetchOGImage(articleURL string) string { - if articleURL == "" { - return "" - } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, "GET", articleURL, nil) - if err != nil { - return "" - } - req.Header.Set("User-Agent", userAgent) - req.Header.Set("Accept", "text/html,application/xhtml+xml") - - resp, err := ogClient.Do(req) - if err != nil { - return "" - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "" - } - - // og:image lives in ; cap read at 512KB to avoid pulling whole articles. - body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024)) - if err != nil { - return "" - } - - m := metaImageRe.FindSubmatch(body) - if len(m) == 0 { - return "" - } - // One of the two capture groups will be non-empty depending on attr order. - for i := 1; i < len(m); i++ { - if len(m[i]) > 0 { - return resolveURL(articleURL, html.UnescapeString(string(m[i]))) - } - } - return "" -} - -// resolveURL turns a possibly-relative image URL into an absolute one using -// the article URL as the base. Returns the raw input on parse failure. -func resolveURL(base, ref string) string { - ref = strings.TrimSpace(ref) - if ref == "" { - return "" - } - if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") { - return ref - } - if strings.HasPrefix(ref, "//") { - if i := strings.Index(base, "://"); i > 0 { - return base[:i+1] + ref - } - return "https:" + ref - } - // relative path — naive join: scheme://host + ref - i := strings.Index(base, "://") - if i < 0 { - return ref - } - rest := base[i+3:] - slash := strings.Index(rest, "/") - if slash < 0 { - return fmt.Sprintf("%s%s", base, ref) - } - host := base[:i+3+slash] - if strings.HasPrefix(ref, "/") { - return host + ref - } - return host + "/" + ref -} diff --git a/internal/ingestion/poller.go b/internal/ingestion/poller.go index 57c1e87..c2453c0 100644 --- a/internal/ingestion/poller.go +++ b/internal/ingestion/poller.go @@ -106,17 +106,10 @@ func (p *Poller) pollOnceWithErr(src config.SourceConfig) error { continue } - // Last-resort image fallback: feed gave us nothing, scrape og:image - // from the article page. Only runs for genuinely new items. - if items[i].ImageURL == "" { - if og := FetchOGImage(items[i].ArticleURL); og != "" { - items[i].ImageURL = og - slog.Debug("og:image fallback used", - "guid", items[i].GUID, "url", items[i].ArticleURL, "image", og) - } - } - - canonical := dedup.CanonicalURL(items[i].ArticleURL) + // Dedup checks first — canonical and headline use the ORIGINAL URL so + // they stay stable whether we end up swapping to a Wayback snapshot. + originalURL := items[i].ArticleURL + canonical := dedup.CanonicalURL(originalURL) headlineNorm := dedup.NormalizeHeadline(items[i].Headline) if storage.IsCanonicalSeen(canonical) { @@ -130,6 +123,38 @@ func (p *Poller) pollOnceWithErr(src config.SourceConfig) error { continue } + // Genuinely new story — fetch the article page once. We use it for: + // 1. og:image fallback when the feed didn't give us one. + // 2. Paywall detection: visible body text below the threshold means + // the article is gated, so swap in a Wayback snapshot for both + // Pete (image) and the reader (posted link). + meta := FetchArticleMeta(originalURL) + paywalled := !meta.Fetched || meta.BodyChars < PaywallBodyThreshold + if paywalled { + if snap := ResolveWayback(originalURL); snap != "" { + snapMeta := FetchArticleMeta(snap) + if snapMeta.Fetched { + items[i].ArticleURL = snap + if items[i].ImageURL == "" && snapMeta.ImageURL != "" { + items[i].ImageURL = snapMeta.ImageURL + } + slog.Info("paywall detected, using wayback snapshot", + "guid", items[i].GUID, "original", originalURL, + "snapshot", snap, "body_chars", meta.BodyChars) + } else { + slog.Debug("paywall detected but snapshot fetch failed", + "guid", items[i].GUID, "original", originalURL, "snapshot", snap) + } + } else { + slog.Debug("paywall detected but no wayback snapshot available", + "guid", items[i].GUID, "url", originalURL, "body_chars", meta.BodyChars) + } + } else if items[i].ImageURL == "" && meta.ImageURL != "" { + items[i].ImageURL = meta.ImageURL + slog.Debug("og:image fallback used", + "guid", items[i].GUID, "url", originalURL, "image", meta.ImageURL) + } + // Stamp source metadata onto the item items[i].Source = src.Name items[i].FeedHint = src.FeedHint diff --git a/internal/ingestion/wayback.go b/internal/ingestion/wayback.go new file mode 100644 index 0000000..82015e8 --- /dev/null +++ b/internal/ingestion/wayback.go @@ -0,0 +1,62 @@ +package ingestion + +import ( + "context" + "encoding/json" + "net/http" + "net/url" + "time" +) + +var waybackClient = &http.Client{Timeout: 10 * time.Second} + +type waybackResp struct { + ArchivedSnapshots struct { + Closest struct { + Available bool `json:"available"` + URL string `json:"url"` + Status string `json:"status"` + } `json:"closest"` + } `json:"archived_snapshots"` +} + +// ResolveWayback asks the Internet Archive's Wayback availability API for the +// closest snapshot of the given URL. Returns the snapshot URL (https) or "" +// if no snapshot exists or the request fails. +func ResolveWayback(articleURL string) string { + if articleURL == "" { + return "" + } + api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", api, nil) + if err != nil { + return "" + } + req.Header.Set("User-Agent", userAgent) + + resp, err := waybackClient.Do(req) + if err != nil { + return "" + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "" + } + + var wr waybackResp + if err := json.NewDecoder(resp.Body).Decode(&wr); err != nil { + return "" + } + snap := wr.ArchivedSnapshots.Closest + if !snap.Available || snap.Status != "200" || snap.URL == "" { + return "" + } + // Wayback sometimes returns http:// even when https is available. + if len(snap.URL) > 7 && snap.URL[:7] == "http://" { + return "https://" + snap.URL[7:] + } + return snap.URL +}