Add per-source user_agent override for WAF-gated feeds
The Portugal News moved its RSS behind AWS WAF, which 405s Pete's honest bot UA (confirmed: bot UA 0/6, browser UA 6/6 from the server IP). Add an optional per-source user_agent that falls back to the default bot UA, and set The Portugal News to a browser UA. Not load-related — 30-min polls and the IP isn't banned.
This commit is contained in:
@@ -36,7 +36,7 @@ type FeedItem struct {
|
||||
Source string
|
||||
DirectRoute string
|
||||
Language string // per-item <language> (or dc:language) when present; "" otherwise
|
||||
PublishedAt int64 // unix seconds from RSS pubDate / Atom published (or updated as fallback); 0 if absent or unparseable
|
||||
PublishedAt int64 // unix seconds from RSS pubDate / Atom published (or updated as fallback); 0 if absent or unparseable
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -45,9 +45,9 @@ var (
|
||||
imgSrcRe = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
|
||||
)
|
||||
|
||||
|
||||
// FetchFeed fetches and parses an RSS/Atom feed, returning items.
|
||||
func FetchFeed(feedURL string) ([]FeedItem, error) {
|
||||
// FetchFeed fetches and parses an RSS/Atom feed, returning items. ua overrides
|
||||
// the User-Agent header; pass "" to use Pete's honest default bot UA.
|
||||
func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
@@ -55,7 +55,10 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
if ua == "" {
|
||||
ua = userAgent
|
||||
}
|
||||
req.Header.Set("User-Agent", ua)
|
||||
resp, err := feedClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -95,7 +95,7 @@ func (p *Poller) pollOnce(ctx context.Context, src config.SourceConfig) {
|
||||
}
|
||||
|
||||
func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) error {
|
||||
items, err := FetchFeed(src.FeedURL)
|
||||
items, err := FetchFeed(src.FeedURL, src.UserAgent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user