Add per-source language filter
When a source sets language = "en", drop items whose per-item language tag is present and doesn't prefix-match. Items without a language tag pass through unchanged. Politico Europe is the motivating case — same headlines appear in en, fr, and de.
This commit is contained in:
@@ -70,6 +70,10 @@ type SourceConfig struct {
|
|||||||
PollIntervalMinutes int `toml:"poll_interval_minutes"`
|
PollIntervalMinutes int `toml:"poll_interval_minutes"`
|
||||||
DirectRoute string `toml:"direct_route"`
|
DirectRoute string `toml:"direct_route"`
|
||||||
Enabled bool `toml:"enabled"`
|
Enabled bool `toml:"enabled"`
|
||||||
|
// Language, when set, drops feed items whose per-item language tag is
|
||||||
|
// present and does not match (prefix). Useful for multilingual feeds
|
||||||
|
// like Politico Europe that publish English + French side-by-side.
|
||||||
|
Language string `toml:"language"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func Load(path string) (*Config, error) {
|
func Load(path string) (*Config, error) {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package ingestion
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -109,6 +110,17 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Drop items in languages this source isn't configured to surface.
|
||||||
|
// Items without a language tag pass through — only filter when the
|
||||||
|
// feed explicitly marks an item with a non-matching language.
|
||||||
|
if src.Language != "" && items[i].Language != "" &&
|
||||||
|
!strings.HasPrefix(items[i].Language, strings.ToLower(src.Language)) {
|
||||||
|
slog.Info("dropping story (language filter)",
|
||||||
|
"guid", items[i].GUID, "source", src.Name,
|
||||||
|
"item_lang", items[i].Language, "want", src.Language)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
// Dedup checks first — canonical and headline use the ORIGINAL URL so
|
// Dedup checks first — canonical and headline use the ORIGINAL URL so
|
||||||
// they stay stable whether we end up swapping to a Wayback snapshot.
|
// they stay stable whether we end up swapping to a Wayback snapshot.
|
||||||
originalURL := items[i].ArticleURL
|
originalURL := items[i].ArticleURL
|
||||||
|
|||||||
Reference in New Issue
Block a user