Add per-source user_agent override for WAF-gated feeds

The Portugal News moved its RSS behind AWS WAF, which 405s Pete's honest
bot UA (confirmed: bot UA 0/6, browser UA 6/6 from the server IP). Add an
optional per-source user_agent that falls back to the default bot UA, and
set The Portugal News to a browser UA. Not load-related — 30-min polls and
the IP isn't banned.
This commit is contained in:
prosolis
2026-06-21 16:40:05 -07:00
parent d5d0656dba
commit 4bdf9a7615
5 changed files with 18 additions and 8 deletions

View File

@@ -36,7 +36,7 @@ type FeedItem struct {
Source string
DirectRoute string
Language string // per-item <language> (or dc:language) when present; "" otherwise
PublishedAt int64 // unix seconds from RSS pubDate / Atom published (or updated as fallback); 0 if absent or unparseable
PublishedAt int64 // unix seconds from RSS pubDate / Atom published (or updated as fallback); 0 if absent or unparseable
}
var (
@@ -45,9 +45,9 @@ var (
imgSrcRe = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
)
// FetchFeed fetches and parses an RSS/Atom feed, returning items.
func FetchFeed(feedURL string) ([]FeedItem, error) {
// FetchFeed fetches and parses an RSS/Atom feed, returning items. ua overrides
// the User-Agent header; pass "" to use Pete's honest default bot UA.
func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
@@ -55,7 +55,10 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", userAgent)
if ua == "" {
ua = userAgent
}
req.Header.Set("User-Agent", ua)
resp, err := feedClient.Do(req)
if err != nil {
return nil, err

View File

@@ -95,7 +95,7 @@ func (p *Poller) pollOnce(ctx context.Context, src config.SourceConfig) {
}
func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) error {
items, err := FetchFeed(src.FeedURL)
items, err := FetchFeed(src.FeedURL, src.UserAgent)
if err != nil {
return err
}