diff --git a/internal/ingestion/parser.go b/internal/ingestion/parser.go
index d3bceb8..19c7f55 100644
--- a/internal/ingestion/parser.go
+++ b/internal/ingestion/parser.go
@@ -28,11 +28,13 @@ type FeedItem struct {
ArticleURL string
Source string
DirectRoute string
+ Language string // per-item ]+src=["']([^"']+)["']`)
)
@@ -59,6 +61,7 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
Lede: extractLede(item.Description),
ImageURL: NormalizeImageURL(extractImageURL(item)),
ArticleURL: strings.TrimSpace(item.Link),
+ Language: itemLanguage(item),
PublishedAt: itemPublished(item),
}
if fi.GUID == "" || fi.ArticleURL == "" {
@@ -71,6 +74,22 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
return items, nil
}
+// itemLanguage returns the per-item language tag if the feed provides one.
+// Politico Europe puts a raw
don't fuse words. + text := htmlTagRe.ReplaceAllString(desc, " ") text = html.UnescapeString(text) + text = wsRe.ReplaceAllString(text, " ") return strings.TrimSpace(text) } diff --git a/internal/ingestion/parser_test.go b/internal/ingestion/parser_test.go index 54a36eb..62b7d71 100644 --- a/internal/ingestion/parser_test.go +++ b/internal/ingestion/parser_test.go @@ -48,6 +48,11 @@ func TestExtractLede(t *testing.T) { "It's a new era—one of change.", "It's a new era\u2014one of change.", }, + { + "adjacent block tags inject a space", + "
...end of moments
Clarence B Jones, a former...
", + "...end of moments Clarence B Jones, a former...", + }, { "tags and entities combined", "Oil prices rose & gas fell by >5%.
",