Add per-source language filter
When a source sets language = "en", drop items whose per-item language tag is present and doesn't prefix-match. Items without a language tag pass through unchanged. Politico Europe is the motivating case — same headlines appear in en, fr, and de.
This commit is contained in:
@@ -70,6 +70,10 @@ type SourceConfig struct {
|
||||
PollIntervalMinutes int `toml:"poll_interval_minutes"`
|
||||
DirectRoute string `toml:"direct_route"`
|
||||
Enabled bool `toml:"enabled"`
|
||||
// Language, when set, drops feed items whose per-item language tag is
|
||||
// present and does not match (prefix). Useful for multilingual feeds
|
||||
// like Politico Europe that publish English + French side-by-side.
|
||||
Language string `toml:"language"`
|
||||
}
|
||||
|
||||
func Load(path string) (*Config, error) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package ingestion
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -109,6 +110,17 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
continue
|
||||
}
|
||||
|
||||
// Drop items in languages this source isn't configured to surface.
|
||||
// Items without a language tag pass through — only filter when the
|
||||
// feed explicitly marks an item with a non-matching language.
|
||||
if src.Language != "" && items[i].Language != "" &&
|
||||
!strings.HasPrefix(items[i].Language, strings.ToLower(src.Language)) {
|
||||
slog.Info("dropping story (language filter)",
|
||||
"guid", items[i].GUID, "source", src.Name,
|
||||
"item_lang", items[i].Language, "want", src.Language)
|
||||
continue
|
||||
}
|
||||
|
||||
// Dedup checks first — canonical and headline use the ORIGINAL URL so
|
||||
// they stay stable whether we end up swapping to a Wayback snapshot.
|
||||
originalURL := items[i].ArticleURL
|
||||
|
||||
Reference in New Issue
Block a user