diff --git a/internal/ingestion/parser.go b/internal/ingestion/parser.go index d3bceb8..19c7f55 100644 --- a/internal/ingestion/parser.go +++ b/internal/ingestion/parser.go @@ -28,11 +28,13 @@ type FeedItem struct { ArticleURL string Source string DirectRoute string + Language string // per-item (or dc:language) when present; "" otherwise PublishedAt int64 // unix seconds from RSS pubDate / Atom published (or updated as fallback); 0 if absent or unparseable } var ( htmlTagRe = regexp.MustCompile(`<[^>]*>`) + wsRe = regexp.MustCompile(`\s+`) imgSrcRe = regexp.MustCompile(`(?i)]+src=["']([^"']+)["']`) ) @@ -59,6 +61,7 @@ func FetchFeed(feedURL string) ([]FeedItem, error) { Lede: extractLede(item.Description), ImageURL: NormalizeImageURL(extractImageURL(item)), ArticleURL: strings.TrimSpace(item.Link), + Language: itemLanguage(item), PublishedAt: itemPublished(item), } if fi.GUID == "" || fi.ArticleURL == "" { @@ -71,6 +74,22 @@ func FetchFeed(feedURL string) ([]FeedItem, error) { return items, nil } +// itemLanguage returns the per-item language tag if the feed provides one. +// Politico Europe puts a raw inside each ; gofeed parks +// unknown item-level elements in item.Custom. We also check the standard +// dc:language extension for feeds that use Dublin Core. +func itemLanguage(item *gofeed.Item) string { + if item.Custom != nil { + if v, ok := item.Custom["language"]; ok { + return strings.ToLower(strings.TrimSpace(v)) + } + } + if item.DublinCoreExt != nil && len(item.DublinCoreExt.Language) > 0 { + return strings.ToLower(strings.TrimSpace(item.DublinCoreExt.Language[0])) + } + return "" +} + func itemGUID(item *gofeed.Item) string { if item.GUID != "" { return item.GUID @@ -96,8 +115,10 @@ func extractLede(desc string) string { if desc == "" { return "" } - text := htmlTagRe.ReplaceAllString(desc, "") + // Replace tags with a space so adjacent blocks like

don't fuse words. + text := htmlTagRe.ReplaceAllString(desc, " ") text = html.UnescapeString(text) + text = wsRe.ReplaceAllString(text, " ") return strings.TrimSpace(text) } diff --git a/internal/ingestion/parser_test.go b/internal/ingestion/parser_test.go index 54a36eb..62b7d71 100644 --- a/internal/ingestion/parser_test.go +++ b/internal/ingestion/parser_test.go @@ -48,6 +48,11 @@ func TestExtractLede(t *testing.T) { "It's a new era—one of change.", "It's a new era\u2014one of change.", }, + { + "adjacent block tags inject a space", + "

...end of moments

Clarence B Jones, a former...

", + "...end of moments Clarence B Jones, a former...", + }, { "tags and entities combined", "

Oil prices rose & gas fell by >5%.

",