diff --git a/internal/config/config.go b/internal/config/config.go index 6f0169f..8a8f2a2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -70,6 +70,10 @@ type SourceConfig struct { PollIntervalMinutes int `toml:"poll_interval_minutes"` DirectRoute string `toml:"direct_route"` Enabled bool `toml:"enabled"` + // Language, when set, drops feed items whose per-item language tag is + // present and does not match (prefix). Useful for multilingual feeds + // like Politico Europe that publish English + French side-by-side. + Language string `toml:"language"` } func Load(path string) (*Config, error) { diff --git a/internal/ingestion/poller.go b/internal/ingestion/poller.go index 926e7ef..23ad646 100644 --- a/internal/ingestion/poller.go +++ b/internal/ingestion/poller.go @@ -3,6 +3,7 @@ package ingestion import ( "context" "log/slog" + "strings" "sync" "time" @@ -109,6 +110,17 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e continue } + // Drop items in languages this source isn't configured to surface. + // Items without a language tag pass through — only filter when the + // feed explicitly marks an item with a non-matching language. + if src.Language != "" && items[i].Language != "" && + !strings.HasPrefix(items[i].Language, strings.ToLower(src.Language)) { + slog.Info("dropping story (language filter)", + "guid", items[i].GUID, "source", src.Name, + "item_lang", items[i].Language, "want", src.Language) + continue + } + // Dedup checks first — canonical and headline use the ORIGINAL URL so // they stay stable whether we end up swapping to a Wayback snapshot. originalURL := items[i].ArticleURL