Paywall detection: swap to Wayback snapshot when article body is gated
Fetch each new article once and measure visible <p> text. If body is below 500 chars (or fetch fails), resolve a Wayback snapshot via the archive.org/wayback/available API and use that URL for both the og:image fallback and the posted link. Dedup keys stay derived from the original URL so paywalled/non-paywalled hits collide as before. - New: internal/ingestion/article.go (FetchArticleMeta via goquery) - New: internal/ingestion/wayback.go (ResolveWayback) - Removed: internal/ingestion/og.go (folded into article.go) - poller.go: dedup first, then one article fetch, then snapshot fallback
This commit is contained in:
62
internal/ingestion/wayback.go
Normal file
62
internal/ingestion/wayback.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package ingestion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
var waybackClient = &http.Client{Timeout: 10 * time.Second}
|
||||
|
||||
type waybackResp struct {
|
||||
ArchivedSnapshots struct {
|
||||
Closest struct {
|
||||
Available bool `json:"available"`
|
||||
URL string `json:"url"`
|
||||
Status string `json:"status"`
|
||||
} `json:"closest"`
|
||||
} `json:"archived_snapshots"`
|
||||
}
|
||||
|
||||
// ResolveWayback asks the Internet Archive's Wayback availability API for the
|
||||
// closest snapshot of the given URL. Returns the snapshot URL (https) or ""
|
||||
// if no snapshot exists or the request fails.
|
||||
func ResolveWayback(articleURL string) string {
|
||||
if articleURL == "" {
|
||||
return ""
|
||||
}
|
||||
api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", api, nil)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
|
||||
resp, err := waybackClient.Do(req)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return ""
|
||||
}
|
||||
|
||||
var wr waybackResp
|
||||
if err := json.NewDecoder(resp.Body).Decode(&wr); err != nil {
|
||||
return ""
|
||||
}
|
||||
snap := wr.ArchivedSnapshots.Closest
|
||||
if !snap.Available || snap.Status != "200" || snap.URL == "" {
|
||||
return ""
|
||||
}
|
||||
// Wayback sometimes returns http:// even when https is available.
|
||||
if len(snap.URL) > 7 && snap.URL[:7] == "http://" {
|
||||
return "https://" + snap.URL[7:]
|
||||
}
|
||||
return snap.URL
|
||||
}
|
||||
Reference in New Issue
Block a user