From 4bdf9a7615c1c21ea4a5efb49dd5b1708b5c44cc Mon Sep 17 00:00:00 2001
From: prosolis <5590409+prosolis@users.noreply.github.com>
Date: Sun, 21 Jun 2026 16:40:05 -0700
Subject: [PATCH] Add per-source user_agent override for WAF-gated feeds
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Portugal News moved its RSS behind AWS WAF, which 405s Pete's honest
bot UA (confirmed: bot UA 0/6, browser UA 6/6 from the server IP). Add an
optional per-source user_agent that falls back to the default bot UA, and
set The Portugal News to a browser UA. Not load-related — 30-min polls and
the IP isn't banned.
---
 config.example.toml          |  3 +++
 internal/config/config.go    |  4 ++++
 internal/ingestion/parser.go | 13 ++++++++-----
 internal/ingestion/poller.go |  2 +-
 main.go                      |  4 ++--
 5 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/config.example.toml b/config.example.toml
index 71b7ce0..0f0446c 100644
--- a/config.example.toml
+++ b/config.example.toml
@@ -65,6 +65,9 @@ tier = 1
 poll_interval_minutes = 20
 direct_route = "politics"
 enabled = true
+# Optional: override the User-Agent for this feed only. Leave unset to use
+# Pete's honest bot UA; set a browser string for feeds whose WAF blocks bots.
+# user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0"
 
 [[sources]]
 name = "The Guardian — Politics"
diff --git a/internal/config/config.go b/internal/config/config.go
index 8df1e65..d710a1a 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -91,6 +91,10 @@ type SourceConfig struct {
 	// present and does not match (prefix). Useful for multilingual feeds
 	// like Politico Europe that publish English + French side-by-side.
 	Language string `toml:"language"`
+	// UserAgent overrides the User-Agent sent when fetching this feed. Empty
+	// uses Pete's honest default bot UA. Set a browser-like string only for
+	// sources whose WAF (e.g. AWS WAF on The Portugal News) blocks bot UAs.
+	UserAgent string `toml:"user_agent"`
 }
 
 func Load(path string) (*Config, error) {
diff --git a/internal/ingestion/parser.go b/internal/ingestion/parser.go
index 2872432..89bf8df 100644
--- a/internal/ingestion/parser.go
+++ b/internal/ingestion/parser.go
@@ -36,7 +36,7 @@ type FeedItem struct {
 	Source      string
 	DirectRoute string
 	Language    string // per-item <language> (or dc:language) when present; "" otherwise
-	PublishedAt int64 // unix seconds from RSS pubDate / Atom published (or updated as fallback); 0 if absent or unparseable
+	PublishedAt int64  // unix seconds from RSS pubDate / Atom published (or updated as fallback); 0 if absent or unparseable
 }
 
 var (
@@ -45,9 +45,9 @@ var (
 	imgSrcRe  = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
 )
 
-
-// FetchFeed fetches and parses an RSS/Atom feed, returning items.
-func FetchFeed(feedURL string) ([]FeedItem, error) {
+// FetchFeed fetches and parses an RSS/Atom feed, returning items. ua overrides
+// the User-Agent header; pass "" to use Pete's honest default bot UA.
+func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
@@ -55,7 +55,10 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("User-Agent", userAgent)
+	if ua == "" {
+		ua = userAgent
+	}
+	req.Header.Set("User-Agent", ua)
 	resp, err := feedClient.Do(req)
 	if err != nil {
 		return nil, err
diff --git a/internal/ingestion/poller.go b/internal/ingestion/poller.go
index 23ad646..f511974 100644
--- a/internal/ingestion/poller.go
+++ b/internal/ingestion/poller.go
@@ -95,7 +95,7 @@ func (p *Poller) pollOnce(ctx context.Context, src config.SourceConfig) {
 }
 
 func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) error {
-	items, err := FetchFeed(src.FeedURL)
+	items, err := FetchFeed(src.FeedURL, src.UserAgent)
 	if err != nil {
 		return err
 	}
diff --git a/main.go b/main.go
index d432874..ce30e3f 100644
--- a/main.go
+++ b/main.go
@@ -283,7 +283,7 @@ func runSeed(cfg *config.Config) {
 		if !src.Enabled {
 			continue
 		}
-		items, err := ingestion.FetchFeed(src.FeedURL)
+		items, err := ingestion.FetchFeed(src.FeedURL, src.UserAgent)
 		if err != nil {
 			slog.Error("seed: feed fetch failed", "source", src.Name, "err", err)
 			continue
@@ -323,7 +323,7 @@ func runTest(cfg *config.Config, sourceName string) {
 		if sourceName != "" && s.Name != sourceName {
 			continue
 		}
-		items, err := ingestion.FetchFeed(s.FeedURL)
+		items, err := ingestion.FetchFeed(s.FeedURL, s.UserAgent)
 		if err != nil {
 			slog.Error("test: feed fetch failed", "source", s.Name, "err", err)
 			continue