package ingestion import ( "log/slog" "net/http" "net/url" "strconv" "strings" "time" ) // NormalizeImageURL rewrites known CDN thumbnail URLs to a higher-resolution // variant. Currently handles Guardian's i.guim.co.uk, whose RSS feeds hand out // 140-px-wide thumbnails by default. Unrecognized hosts pass through unchanged. func NormalizeImageURL(raw string) string { if raw == "" { return raw } u, err := url.Parse(raw) if err != nil || u.Host != "i.guim.co.uk" { return raw } q := u.Query() if q.Get("width") == "" { return raw } // The Guardian signs each (width, quality, fit, …) combination with `s=`. // Changing width without re-signing yields a 401 "invalid signature", // so leave signed URLs alone — the parser is now responsible for picking // the widest media:content variant up front. if q.Get("s") != "" { return raw } q.Set("width", "1200") u.RawQuery = q.Encode() return u.String() } var imageClient = &http.Client{ Timeout: 10 * time.Second, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= 3 { return http.ErrUseLastResponse } return nil }, } // ValidateImageURL checks that an image URL returns a valid image response. // Returns false (with no error) on any failure — story posts without image. func ValidateImageURL(url string) bool { if url == "" { return false } req, err := http.NewRequest("HEAD", url, nil) if err != nil { slog.Debug("image validation: bad url", "url", url, "err", err) return false } req.Header.Set("User-Agent", userAgent) resp, err := imageClient.Do(req) if err != nil { slog.Warn("image validation: request failed", "url", url, "err", err) return false } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { slog.Warn("image validation: bad status", "url", url, "status", resp.StatusCode) return false } contentType := resp.Header.Get("Content-Type") if !strings.HasPrefix(contentType, "image/") { slog.Warn("image validation: not an image", "url", url, "content_type", contentType) return false } // Filter tracking pixels: Content-Length must be > 5KB if present if cl := resp.Header.Get("Content-Length"); cl != "" { size, err := strconv.ParseInt(cl, 10, 64) if err == nil && size <= 5120 { slog.Warn("image validation: small image (≤5KB), skipping", "url", url, "size", size) return false } } return true }