Add per-source user_agent override for WAF-gated feeds

The Portugal News moved its RSS behind AWS WAF, which 405s Pete's honest
bot UA (confirmed: bot UA 0/6, browser UA 6/6 from the server IP). Add an
optional per-source user_agent that falls back to the default bot UA, and
set The Portugal News to a browser UA. Not load-related — 30-min polls and
the IP isn't banned.

config.example.toml

@@ -65,6 +65,9 @@ tier = 1
 poll_interval_minutes = 20
 direct_route = "politics"
 enabled = true
+# Optional: override the User-Agent for this feed only. Leave unset to use
+# Pete's honest bot UA; set a browser string for feeds whose WAF blocks bots.
+# user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"
 
 [[sources]]
 name = "The Guardian — Politics"

internal/config/config.go

@@ -91,6 +91,10 @@ type SourceConfig struct {
 	// present and does not match (prefix). Useful for multilingual feeds
 	// like Politico Europe that publish English + French side-by-side.
 	Language string `toml:"language"`
+	// UserAgent overrides the User-Agent sent when fetching this feed. Empty
+	// uses Pete's honest default bot UA. Set a browser-like string only for
+	// sources whose WAF (e.g. AWS WAF on The Portugal News) blocks bot UAs.
+	UserAgent string `toml:"user_agent"`
 }
 
 func Load(path string) (*Config, error) {

internal/ingestion/parser.go

@@ -45,9 +45,9 @@ var (
 	imgSrcRe  = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
 )
 
-
+// FetchFeed fetches and parses an RSS/Atom feed, returning items. ua overrides
-// FetchFeed fetches and parses an RSS/Atom feed, returning items.
+// the User-Agent header; pass "" to use Pete's honest default bot UA.
-func FetchFeed(feedURL string) ([]FeedItem, error) {
+func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
@@ -55,7 +55,10 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("User-Agent", userAgent)
+	if ua == "" {
+		ua = userAgent
+	}
+	req.Header.Set("User-Agent", ua)
 	resp, err := feedClient.Do(req)
 	if err != nil {
 		return nil, err

internal/ingestion/poller.go

@@ -95,7 +95,7 @@ func (p *Poller) pollOnce(ctx context.Context, src config.SourceConfig) {
 }
 
 func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) error {
-	items, err := FetchFeed(src.FeedURL)
+	items, err := FetchFeed(src.FeedURL, src.UserAgent)
 	if err != nil {
 		return err
 	}

main.go

@@ -283,7 +283,7 @@ func runSeed(cfg *config.Config) {
 		if !src.Enabled {
 			continue
 		}
-		items, err := ingestion.FetchFeed(src.FeedURL)
+		items, err := ingestion.FetchFeed(src.FeedURL, src.UserAgent)
 		if err != nil {
 			slog.Error("seed: feed fetch failed", "source", src.Name, "err", err)
 			continue
@@ -323,7 +323,7 @@ func runTest(cfg *config.Config, sourceName string) {
 		if sourceName != "" && s.Name != sourceName {
 			continue
 		}
-		items, err := ingestion.FetchFeed(s.FeedURL)
+		items, err := ingestion.FetchFeed(s.FeedURL, s.UserAgent)
 		if err != nil {
 			slog.Error("test: feed fetch failed", "source", s.Name, "err", err)
 			continue