d5d0656dba
- Route image-validation, Matrix image-download, and wayback/archive.today fetches through safehttp so feed-controlled URLs can't reach loopback/ RFC1918/cloud-metadata IPs (incl. via redirects). - Cap feed body size with safehttp.LimitedBody (16 MiB) to prevent OOM. - Fail closed if matrix.pickle_key is unset/<16 chars; drop the hardcoded "pete_pickle_key" default that silently weakened E2EE-at-rest. - Gate !post behind a matrix.admins allowlist; empty = disabled (channel rooms are public, so empty must not mean anyone). - Reject /\host open-redirect bypass in post-login safeNext. - Allowlist http(s) schemes for the Matrix link href; escape JSON embedded in inline <script> (prefs/sources blobs).
89 lines
2.5 KiB
Go
89 lines
2.5 KiB
Go
package ingestion
|
|
|
|
import (
|
|
"log/slog"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"pete/internal/safehttp"
|
|
)
|
|
|
|
// NormalizeImageURL rewrites known CDN thumbnail URLs to a higher-resolution
|
|
// variant. Currently handles Guardian's i.guim.co.uk, whose RSS feeds hand out
|
|
// 140-px-wide thumbnails by default. Unrecognized hosts pass through unchanged.
|
|
func NormalizeImageURL(raw string) string {
|
|
if raw == "" {
|
|
return raw
|
|
}
|
|
u, err := url.Parse(raw)
|
|
if err != nil || u.Host != "i.guim.co.uk" {
|
|
return raw
|
|
}
|
|
q := u.Query()
|
|
if q.Get("width") == "" {
|
|
return raw
|
|
}
|
|
// The Guardian signs each (width, quality, fit, …) combination with `s=`.
|
|
// Changing width without re-signing yields a 401 "invalid signature",
|
|
// so leave signed URLs alone — the parser is now responsible for picking
|
|
// the widest media:content variant up front.
|
|
if q.Get("s") != "" {
|
|
return raw
|
|
}
|
|
q.Set("width", "1200")
|
|
u.RawQuery = q.Encode()
|
|
return u.String()
|
|
}
|
|
|
|
// imageClient validates feed-supplied image URLs. It routes through safehttp
|
|
// so a hostile feed can't steer the HEAD probe at loopback / RFC1918 / cloud
|
|
// metadata IPs — the dial-time guard also re-checks every redirect target.
|
|
var imageClient = safehttp.NewClient(10 * time.Second)
|
|
|
|
// ValidateImageURL checks that an image URL returns a valid image response.
|
|
// Returns false (with no error) on any failure — story posts without image.
|
|
func ValidateImageURL(url string) bool {
|
|
if url == "" {
|
|
return false
|
|
}
|
|
|
|
req, err := http.NewRequest("HEAD", url, nil)
|
|
if err != nil {
|
|
slog.Debug("image validation: bad url", "url", url, "err", err)
|
|
return false
|
|
}
|
|
req.Header.Set("User-Agent", userAgent)
|
|
|
|
resp, err := imageClient.Do(req)
|
|
if err != nil {
|
|
slog.Warn("image validation: request failed", "url", url, "err", err)
|
|
return false
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
slog.Warn("image validation: bad status", "url", url, "status", resp.StatusCode)
|
|
return false
|
|
}
|
|
|
|
contentType := resp.Header.Get("Content-Type")
|
|
if !strings.HasPrefix(contentType, "image/") {
|
|
slog.Warn("image validation: not an image", "url", url, "content_type", contentType)
|
|
return false
|
|
}
|
|
|
|
// Filter tracking pixels: Content-Length must be > 5KB if present
|
|
if cl := resp.Header.Get("Content-Length"); cl != "" {
|
|
size, err := strconv.ParseInt(cl, 10, 64)
|
|
if err == nil && size <= 5120 {
|
|
slog.Warn("image validation: small image (≤5KB), skipping", "url", url, "size", size)
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|