Add per-source language filter

When a source sets language = "en", drop items whose per-item
language tag is present and doesn't prefix-match. Items without
a language tag pass through unchanged. Politico Europe is the
motivating case — same headlines appear in en, fr, and de.
This commit is contained in:
prosolis
2026-05-26 23:03:12 -07:00
parent 7d469cf8c5
commit 78fc3ef811
2 changed files with 16 additions and 0 deletions

View File

@@ -70,6 +70,10 @@ type SourceConfig struct {
PollIntervalMinutes int `toml:"poll_interval_minutes"`
DirectRoute string `toml:"direct_route"`
Enabled bool `toml:"enabled"`
// Language, when set, drops feed items whose per-item language tag is
// present and does not match (prefix). Useful for multilingual feeds
// like Politico Europe that publish English + French side-by-side.
Language string `toml:"language"`
}
func Load(path string) (*Config, error) {

View File

@@ -3,6 +3,7 @@ package ingestion
import (
"context"
"log/slog"
"strings"
"sync"
"time"
@@ -109,6 +110,17 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
continue
}
// Drop items in languages this source isn't configured to surface.
// Items without a language tag pass through — only filter when the
// feed explicitly marks an item with a non-matching language.
if src.Language != "" && items[i].Language != "" &&
!strings.HasPrefix(items[i].Language, strings.ToLower(src.Language)) {
slog.Info("dropping story (language filter)",
"guid", items[i].GUID, "source", src.Name,
"item_lang", items[i].Language, "want", src.Language)
continue
}
// Dedup checks first — canonical and headline use the ORIGINAL URL so
// they stay stable whether we end up swapping to a Wayback snapshot.
originalURL := items[i].ArticleURL