0111a1b06d
- Add -local flag: web/RSS-only mode that skips Matrix login and posting, so the web UI can be exercised against live feeds without credentials - Add Makefile (build/local/seed/test/clean) that handles Tailwind + go build in one shot - Fix Guardian thumbnails: NormalizeImageURL was rewriting width=1200 onto signed i.guim.co.uk URLs, invalidating the s= signature and returning 401. Leave signed URLs alone and pick the widest media:content variant up front instead - Use pete.avif as the header logo, favicon, and footer mark; drop the unused leaf.svg
92 lines
2.4 KiB
Go
92 lines
2.4 KiB
Go
package ingestion
|
|
|
|
import (
|
|
"log/slog"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// NormalizeImageURL rewrites known CDN thumbnail URLs to a higher-resolution
|
|
// variant. Currently handles Guardian's i.guim.co.uk, whose RSS feeds hand out
|
|
// 140-px-wide thumbnails by default. Unrecognized hosts pass through unchanged.
|
|
func NormalizeImageURL(raw string) string {
|
|
if raw == "" {
|
|
return raw
|
|
}
|
|
u, err := url.Parse(raw)
|
|
if err != nil || u.Host != "i.guim.co.uk" {
|
|
return raw
|
|
}
|
|
q := u.Query()
|
|
if q.Get("width") == "" {
|
|
return raw
|
|
}
|
|
// The Guardian signs each (width, quality, fit, …) combination with `s=`.
|
|
// Changing width without re-signing yields a 401 "invalid signature",
|
|
// so leave signed URLs alone — the parser is now responsible for picking
|
|
// the widest media:content variant up front.
|
|
if q.Get("s") != "" {
|
|
return raw
|
|
}
|
|
q.Set("width", "1200")
|
|
u.RawQuery = q.Encode()
|
|
return u.String()
|
|
}
|
|
|
|
var imageClient = &http.Client{
|
|
Timeout: 10 * time.Second,
|
|
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
|
if len(via) >= 3 {
|
|
return http.ErrUseLastResponse
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
|
|
// ValidateImageURL checks that an image URL returns a valid image response.
|
|
// Returns false (with no error) on any failure — story posts without image.
|
|
func ValidateImageURL(url string) bool {
|
|
if url == "" {
|
|
return false
|
|
}
|
|
|
|
req, err := http.NewRequest("HEAD", url, nil)
|
|
if err != nil {
|
|
slog.Debug("image validation: bad url", "url", url, "err", err)
|
|
return false
|
|
}
|
|
req.Header.Set("User-Agent", userAgent)
|
|
|
|
resp, err := imageClient.Do(req)
|
|
if err != nil {
|
|
slog.Warn("image validation: request failed", "url", url, "err", err)
|
|
return false
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
slog.Warn("image validation: bad status", "url", url, "status", resp.StatusCode)
|
|
return false
|
|
}
|
|
|
|
contentType := resp.Header.Get("Content-Type")
|
|
if !strings.HasPrefix(contentType, "image/") {
|
|
slog.Warn("image validation: not an image", "url", url, "content_type", contentType)
|
|
return false
|
|
}
|
|
|
|
// Filter tracking pixels: Content-Length must be > 5KB if present
|
|
if cl := resp.Header.Get("Content-Length"); cl != "" {
|
|
size, err := strconv.ParseInt(cl, 10, 64)
|
|
if err == nil && size <= 5120 {
|
|
slog.Warn("image validation: small image (≤5KB), skipping", "url", url, "size", size)
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|