e26b69e43f
Fetch each new article once and measure visible <p> text. If body is below 500 chars (or fetch fails), resolve a Wayback snapshot via the archive.org/wayback/available API and use that URL for both the og:image fallback and the posted link. Dedup keys stay derived from the original URL so paywalled/non-paywalled hits collide as before. - New: internal/ingestion/article.go (FetchArticleMeta via goquery) - New: internal/ingestion/wayback.go (ResolveWayback) - Removed: internal/ingestion/og.go (folded into article.go) - poller.go: dedup first, then one article fetch, then snapshot fallback
63 lines
1.5 KiB
Go
63 lines
1.5 KiB
Go
package ingestion
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/url"
|
|
"time"
|
|
)
|
|
|
|
var waybackClient = &http.Client{Timeout: 10 * time.Second}
|
|
|
|
type waybackResp struct {
|
|
ArchivedSnapshots struct {
|
|
Closest struct {
|
|
Available bool `json:"available"`
|
|
URL string `json:"url"`
|
|
Status string `json:"status"`
|
|
} `json:"closest"`
|
|
} `json:"archived_snapshots"`
|
|
}
|
|
|
|
// ResolveWayback asks the Internet Archive's Wayback availability API for the
|
|
// closest snapshot of the given URL. Returns the snapshot URL (https) or ""
|
|
// if no snapshot exists or the request fails.
|
|
func ResolveWayback(articleURL string) string {
|
|
if articleURL == "" {
|
|
return ""
|
|
}
|
|
api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "GET", api, nil)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
req.Header.Set("User-Agent", userAgent)
|
|
|
|
resp, err := waybackClient.Do(req)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return ""
|
|
}
|
|
|
|
var wr waybackResp
|
|
if err := json.NewDecoder(resp.Body).Decode(&wr); err != nil {
|
|
return ""
|
|
}
|
|
snap := wr.ArchivedSnapshots.Closest
|
|
if !snap.Available || snap.Status != "200" || snap.URL == "" {
|
|
return ""
|
|
}
|
|
// Wayback sometimes returns http:// even when https is available.
|
|
if len(snap.URL) > 7 && snap.URL[:7] == "http://" {
|
|
return "https://" + snap.URL[7:]
|
|
}
|
|
return snap.URL
|
|
}
|