package ingestion import ( "context" "encoding/json" "net/http" "net/url" "time" ) var waybackClient = &http.Client{Timeout: 10 * time.Second} type waybackResp struct { ArchivedSnapshots struct { Closest struct { Available bool `json:"available"` URL string `json:"url"` Status string `json:"status"` } `json:"closest"` } `json:"archived_snapshots"` } // ResolveWayback asks the Internet Archive's Wayback availability API for the // closest snapshot of the given URL. Returns the snapshot URL (https) or "" // if no snapshot exists or the request fails. func ResolveWayback(articleURL string) string { if articleURL == "" { return "" } api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, "GET", api, nil) if err != nil { return "" } req.Header.Set("User-Agent", userAgent) resp, err := waybackClient.Do(req) if err != nil { return "" } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return "" } var wr waybackResp if err := json.NewDecoder(resp.Body).Decode(&wr); err != nil { return "" } snap := wr.ArchivedSnapshots.Closest if !snap.Available || snap.Status != "200" || snap.URL == "" { return "" } // Wayback sometimes returns http:// even when https is available. if len(snap.URL) > 7 && snap.URL[:7] == "http://" { return "https://" + snap.URL[7:] } return snap.URL }