Pete/internal/ingestion/wayback.go

package ingestion

import (
	"context"
	"encoding/json"
	"net/http"
	"net/url"
	"strings"
	"time"
)

var waybackClient = &http.Client{Timeout: 10 * time.Second}

// maxSnapshotAge bounds how stale a Wayback snapshot can be before we treat
// it as useless for a freshly-published news article. The "closest" snapshot
// returned by the availability API can otherwise be years old.
const maxSnapshotAge = 30 * 24 * time.Hour

type waybackResp struct {
	ArchivedSnapshots struct {
		Closest struct {
			Available bool   `json:"available"`
			URL       string `json:"url"`
			Status    string `json:"status"`
			Timestamp string `json:"timestamp"` // YYYYMMDDhhmmss
		} `json:"closest"`
	} `json:"archived_snapshots"`
}

// ResolveWayback asks the Internet Archive's Wayback availability API for
// the closest snapshot of the given URL. Returns the snapshot URL (https)
// or "" if no snapshot exists, the request fails, or the closest snapshot
// is older than maxSnapshotAge.
func ResolveWayback(articleURL string) string {
	if articleURL == "" {
		return ""
	}
	// Anchor the lookup to "now" so we get the freshest snapshot rather
	// than whichever happens to be Wayback's default closest match.
	api := "https://archive.org/wayback/available?url=" + url.QueryEscape(articleURL) +
		"&timestamp=" + time.Now().UTC().Format("20060102150405")
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", api, nil)
	if err != nil {
		return ""
	}
	req.Header.Set("User-Agent", userAgent)

	resp, err := waybackClient.Do(req)
	if err != nil {
		return ""
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return ""
	}

	var wr waybackResp
	if err := json.NewDecoder(resp.Body).Decode(&wr); err != nil {
		return ""
	}
	snap := wr.ArchivedSnapshots.Closest
	if !snap.Available || snap.Status != "200" || snap.URL == "" {
		return ""
	}
	if !snapshotFreshEnough(snap.Timestamp) {
		return ""
	}
	// Wayback sometimes returns http:// even when https is available.
	if strings.HasPrefix(snap.URL, "http://") {
		return "https://" + snap.URL[7:]
	}
	return snap.URL
}

func snapshotFreshEnough(ts string) bool {
	if ts == "" {
		return false
	}
	t, err := time.Parse("20060102150405", ts)
	if err != nil {
		return false
	}
	return time.Since(t) <= maxSnapshotAge
}

var archiveTodayClient = &http.Client{
	Timeout: 10 * time.Second,
	// Don't follow the redirect — we just want the Location header pointing
	// at the most recent capture.
	CheckRedirect: func(*http.Request, []*http.Request) error {
		return http.ErrUseLastResponse
	},
}

// ResolveArchiveToday looks up the newest archive.today / archive.ph snapshot
// for the given URL. archive.ph has no public availability API, but its
// `/newest/<url>` endpoint redirects (HTTP 302) to the most recent capture
// when one exists, or returns a non-redirect response otherwise. Returns
// "" on any failure or when no snapshot exists.
func ResolveArchiveToday(articleURL string) string {
	if articleURL == "" {
		return ""
	}
	api := "https://archive.ph/newest/" + articleURL
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", api, nil)
	if err != nil {
		return ""
	}
	req.Header.Set("User-Agent", userAgent)

	resp, err := archiveTodayClient.Do(req)
	if err != nil {
		return ""
	}
	defer resp.Body.Close()

	if resp.StatusCode < 300 || resp.StatusCode >= 400 {
		return ""
	}
	loc := strings.TrimSpace(resp.Header.Get("Location"))
	if loc == "" {
		return ""
	}
	// archive.ph occasionally returns a relative Location; absolutize it.
	if strings.HasPrefix(loc, "/") {
		loc = "https://archive.ph" + loc
	}
	// Bounce back the input as Location means "no snapshot, here's the form"
	// — distinguish a real capture URL (contains /YYYY/ or a short hash path).
	if strings.Contains(loc, "://archive.ph/") && !strings.Contains(loc, "://archive.ph/newest/") {
		return loc
	}
	return ""
}