afe2ef996b
- !post falls back to newest unposted story for the channel when the in-memory queue is empty (the steady state under round-robin). - Accept ❓️/❔️ (U+FE0F variation selector) as question reactions — the bare codepoints alone missed clients that render the colored emoji. - Rewrite Guardian i.guim.co.uk thumbnails to width=1200 so we stop rejecting real images as "tracking pixels"; relabel the size warning. - Log decrypt failures and reactions on events not in post_log so future silent drops surface instead of vanishing.
85 lines
2.1 KiB
Go
85 lines
2.1 KiB
Go
package ingestion
|
|
|
|
import (
|
|
"log/slog"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// NormalizeImageURL rewrites known CDN thumbnail URLs to a higher-resolution
|
|
// variant. Currently handles Guardian's i.guim.co.uk, whose RSS feeds hand out
|
|
// 140-px-wide thumbnails by default. Unrecognized hosts pass through unchanged.
|
|
func NormalizeImageURL(raw string) string {
|
|
if raw == "" {
|
|
return raw
|
|
}
|
|
u, err := url.Parse(raw)
|
|
if err != nil || u.Host != "i.guim.co.uk" {
|
|
return raw
|
|
}
|
|
q := u.Query()
|
|
if q.Get("width") == "" {
|
|
return raw
|
|
}
|
|
q.Set("width", "1200")
|
|
u.RawQuery = q.Encode()
|
|
return u.String()
|
|
}
|
|
|
|
var imageClient = &http.Client{
|
|
Timeout: 10 * time.Second,
|
|
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
|
if len(via) >= 3 {
|
|
return http.ErrUseLastResponse
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
|
|
// ValidateImageURL checks that an image URL returns a valid image response.
|
|
// Returns false (with no error) on any failure — story posts without image.
|
|
func ValidateImageURL(url string) bool {
|
|
if url == "" {
|
|
return false
|
|
}
|
|
|
|
req, err := http.NewRequest("HEAD", url, nil)
|
|
if err != nil {
|
|
slog.Debug("image validation: bad url", "url", url, "err", err)
|
|
return false
|
|
}
|
|
req.Header.Set("User-Agent", userAgent)
|
|
|
|
resp, err := imageClient.Do(req)
|
|
if err != nil {
|
|
slog.Warn("image validation: request failed", "url", url, "err", err)
|
|
return false
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
slog.Warn("image validation: bad status", "url", url, "status", resp.StatusCode)
|
|
return false
|
|
}
|
|
|
|
contentType := resp.Header.Get("Content-Type")
|
|
if !strings.HasPrefix(contentType, "image/") {
|
|
slog.Warn("image validation: not an image", "url", url, "content_type", contentType)
|
|
return false
|
|
}
|
|
|
|
// Filter tracking pixels: Content-Length must be > 5KB if present
|
|
if cl := resp.Header.Get("Content-Length"); cl != "" {
|
|
size, err := strconv.ParseInt(cl, 10, 64)
|
|
if err == nil && size <= 5120 {
|
|
slog.Warn("image validation: small image (≤5KB), skipping", "url", url, "size", size)
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|