Add per-source language filter
When a source sets language = "en", drop items whose per-item language tag is present and doesn't prefix-match. Items without a language tag pass through unchanged. Politico Europe is the motivating case — same headlines appear in en, fr, and de.
This commit is contained in:
@@ -3,6 +3,7 @@ package ingestion
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -109,6 +110,17 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
|
||||
continue
|
||||
}
|
||||
|
||||
// Drop items in languages this source isn't configured to surface.
|
||||
// Items without a language tag pass through — only filter when the
|
||||
// feed explicitly marks an item with a non-matching language.
|
||||
if src.Language != "" && items[i].Language != "" &&
|
||||
!strings.HasPrefix(items[i].Language, strings.ToLower(src.Language)) {
|
||||
slog.Info("dropping story (language filter)",
|
||||
"guid", items[i].GUID, "source", src.Name,
|
||||
"item_lang", items[i].Language, "want", src.Language)
|
||||
continue
|
||||
}
|
||||
|
||||
// Dedup checks first — canonical and headline use the ORIGINAL URL so
|
||||
// they stay stable whether we end up swapping to a Wayback snapshot.
|
||||
originalURL := items[i].ArticleURL
|
||||
|
||||
Reference in New Issue
Block a user