package ingestion import ( "context" "encoding/json" "net/http" "net/url" "strings" "time" "pete/internal/safehttp" ) var waybackClient = safehttp.NewClient(10 * time.Second) // maxSnapshotAge bounds how stale a Wayback snapshot can be before we treat // it as useless for a freshly-published news article. The "closest" snapshot // returned by the availability API can otherwise be years old. const maxSnapshotAge = 30 * 24 * time.Hour type waybackResp struct { ArchivedSnapshots struct { Closest struct { Available bool `json:"available"` URL string `json:"url"` Status string `json:"status"` Timestamp string `json:"timestamp"` // YYYYMMDDhhmmss } `json:"closest"` } `json:"archived_snapshots"` } // ResolveWayback asks the Internet Archive's Wayback availability API for // the closest snapshot of the given URL. Returns the snapshot URL (https) // or "" if no snapshot exists, the request fails, or the closest snapshot // is older than maxSnapshotAge. func ResolveWayback(articleURL string) string { if articleURL == "" { return "" } // Anchor the lookup to "now" so we get the freshest snapshot rather // than whichever happens to be Wayback's default closest match. api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL) + "×tamp=" + time.Now().UTC().Format("20060102150405") ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, "GET", api, nil) if err != nil { return "" } req.Header.Set("User-Agent", userAgent) resp, err := waybackClient.Do(req) if err != nil { return "" } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return "" } var wr waybackResp if err := json.NewDecoder(resp.Body).Decode(&wr); err != nil { return "" } snap := wr.ArchivedSnapshots.Closest if !snap.Available || snap.Status != "200" || snap.URL == "" { return "" } if !snapshotFreshEnough(snap.Timestamp) { return "" } // Wayback sometimes returns http:// even when https is available. if strings.HasPrefix(snap.URL, "http://") { return "https://" + snap.URL[7:] } return snap.URL } func snapshotFreshEnough(ts string) bool { if ts == "" { return false } t, err := time.Parse("20060102150405", ts) if err != nil { return false } return time.Since(t) <= maxSnapshotAge } // archiveTodayClient routes through safehttp for the dial-time SSRF guard, but // overrides CheckRedirect so we read the 302 Location header (the most recent // capture) instead of following it. var archiveTodayClient = func() *http.Client { c := safehttp.NewClient(10 * time.Second) c.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } return c }() // ResolveArchiveToday looks up the newest archive.today / archive.ph snapshot // for the given URL. archive.ph has no public availability API, but its // `/newest/` endpoint redirects (HTTP 302) to the most recent capture // when one exists, or returns a non-redirect response otherwise. Returns // "" on any failure or when no snapshot exists. func ResolveArchiveToday(articleURL string) string { if articleURL == "" { return "" } api := "https://archive.ph/newest/" + articleURL ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, "GET", api, nil) if err != nil { return "" } req.Header.Set("User-Agent", userAgent) resp, err := archiveTodayClient.Do(req) if err != nil { return "" } defer resp.Body.Close() if resp.StatusCode < 300 || resp.StatusCode >= 400 { return "" } loc := strings.TrimSpace(resp.Header.Get("Location")) if loc == "" { return "" } // archive.ph occasionally returns a relative Location; absolutize it. if strings.HasPrefix(loc, "/") { loc = "https://archive.ph" + loc } // Bounce back the input as Location means "no snapshot, here's the form" // — distinguish a real capture URL (contains /YYYY/ or a short hash path). if strings.Contains(loc, "://archive.ph/") && !strings.Contains(loc, "://archive.ph/newest/") { return loc } return "" }