Paywall detection: swap to Wayback snapshot when article body is gated

Fetch each new article once and measure visible <p> text. If body is below
500 chars (or fetch fails), resolve a Wayback snapshot via the
archive.org/wayback/available API and use that URL for both the og:image
fallback and the posted link. Dedup keys stay derived from the original
URL so paywalled/non-paywalled hits collide as before.

- New: internal/ingestion/article.go (FetchArticleMeta via goquery)
- New: internal/ingestion/wayback.go (ResolveWayback)
- Removed: internal/ingestion/og.go (folded into article.go)
- poller.go: dedup first, then one article fetch, then snapshot fallback
This commit is contained in:
prosolis
2026-05-22 17:53:54 -07:00
parent ca6663c051
commit e26b69e43f
5 changed files with 238 additions and 110 deletions

View File

@@ -0,0 +1,62 @@
package ingestion
import (
"context"
"encoding/json"
"net/http"
"net/url"
"time"
)
var waybackClient = &http.Client{Timeout: 10 * time.Second}
type waybackResp struct {
ArchivedSnapshots struct {
Closest struct {
Available bool `json:"available"`
URL string `json:"url"`
Status string `json:"status"`
} `json:"closest"`
} `json:"archived_snapshots"`
}
// ResolveWayback asks the Internet Archive's Wayback availability API for the
// closest snapshot of the given URL. Returns the snapshot URL (https) or ""
// if no snapshot exists or the request fails.
func ResolveWayback(articleURL string) string {
if articleURL == "" {
return ""
}
api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", api, nil)
if err != nil {
return ""
}
req.Header.Set("User-Agent", userAgent)
resp, err := waybackClient.Do(req)
if err != nil {
return ""
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return ""
}
var wr waybackResp
if err := json.NewDecoder(resp.Body).Decode(&wr); err != nil {
return ""
}
snap := wr.ArchivedSnapshots.Closest
if !snap.Available || snap.Status != "200" || snap.URL == "" {
return ""
}
// Wayback sometimes returns http:// even when https is available.
if len(snap.URL) > 7 && snap.URL[:7] == "http://" {
return "https://" + snap.URL[7:]
}
return snap.URL
}