drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add per-source user_agent override for WAF-gated feeds

Browse Source 
The Portugal News moved its RSS behind AWS WAF, which 405s Pete's honest
bot UA (confirmed: bot UA 0/6, browser UA 6/6 from the server IP). Add an
optional per-source user_agent that falls back to the default bot UA, and
set The Portugal News to a browser UA. Not load-related — 30-min polls and
the IP isn't banned.
This commit is contained in:
prosolis
2026-06-21 16:40:05 -07:00
parent d5d0656dba
commit 4bdf9a7615
5 changed files with 18 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
main.go
View File

@@ -283,7 +283,7 @@ func runSeed(cfg *config.Config) {
if !src.Enabled {
continue
}
items, err := ingestion.FetchFeed(src.FeedURL)
items, err := ingestion.FetchFeed(src.FeedURL, src.UserAgent)
if err != nil {
slog.Error("seed: feed fetch failed", "source", src.Name, "err", err)
continue
@@ -323,7 +323,7 @@ func runTest(cfg *config.Config, sourceName string) {
if sourceName != "" && s.Name != sourceName {
continue
}
items, err := ingestion.FetchFeed(s.FeedURL)
items, err := ingestion.FetchFeed(s.FeedURL, s.UserAgent)
if err != nil {
slog.Error("test: feed fetch failed", "source", s.Name, "err", err)
continue