From 7d469cf8c5dccc0f0789e9090c99a65a724b4563 Mon Sep 17 00:00:00 2001
From: prosolis <5590409+prosolis@users.noreply.github.com>
Date: Tue, 26 May 2026 23:02:46 -0700
Subject: [PATCH] Fix glued-together RSS ledes; parse per-item language

Two ingestion changes:

- extractLede replaced HTML tags with empty string, so adjacent
  block tags like </p><p> fused words across paragraphs. Replace
  tags with a space and collapse whitespace.
- Pull each item's <language> tag (or dc:language) into FeedItem
  so the poller can filter on it. Politico Europe publishes the
  same story in en / fr / de side-by-side and we want to keep
  only one language per source.
---
 internal/ingestion/parser.go      | 23 ++++++++++++++++++++++-
 internal/ingestion/parser_test.go |  5 +++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/internal/ingestion/parser.go b/internal/ingestion/parser.go
index d3bceb8..19c7f55 100644
--- a/internal/ingestion/parser.go
+++ b/internal/ingestion/parser.go
@@ -28,11 +28,13 @@ type FeedItem struct {
 	ArticleURL  string
 	Source      string
 	DirectRoute string
+	Language    string // per-item <language> (or dc:language) when present; "" otherwise
 	PublishedAt int64 // unix seconds from RSS pubDate / Atom published (or updated as fallback); 0 if absent or unparseable
 }
 
 var (
 	htmlTagRe = regexp.MustCompile(`<[^>]*>`)
+	wsRe      = regexp.MustCompile(`\s+`)
 	imgSrcRe  = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
 )
 
@@ -59,6 +61,7 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
 			Lede:        extractLede(item.Description),
 			ImageURL:    NormalizeImageURL(extractImageURL(item)),
 			ArticleURL:  strings.TrimSpace(item.Link),
+			Language:    itemLanguage(item),
 			PublishedAt: itemPublished(item),
 		}
 		if fi.GUID == "" || fi.ArticleURL == "" {
@@ -71,6 +74,22 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
 	return items, nil
 }
 
+// itemLanguage returns the per-item language tag if the feed provides one.
+// Politico Europe puts a raw <language> inside each <item>; gofeed parks
+// unknown item-level elements in item.Custom. We also check the standard
+// dc:language extension for feeds that use Dublin Core.
+func itemLanguage(item *gofeed.Item) string {
+	if item.Custom != nil {
+		if v, ok := item.Custom["language"]; ok {
+			return strings.ToLower(strings.TrimSpace(v))
+		}
+	}
+	if item.DublinCoreExt != nil && len(item.DublinCoreExt.Language) > 0 {
+		return strings.ToLower(strings.TrimSpace(item.DublinCoreExt.Language[0]))
+	}
+	return ""
+}
+
 func itemGUID(item *gofeed.Item) string {
 	if item.GUID != "" {
 		return item.GUID
@@ -96,8 +115,10 @@ func extractLede(desc string) string {
 	if desc == "" {
 		return ""
 	}
-	text := htmlTagRe.ReplaceAllString(desc, "")
+	// Replace tags with a space so adjacent blocks like </p><p> don't fuse words.
+	text := htmlTagRe.ReplaceAllString(desc, " ")
 	text = html.UnescapeString(text)
+	text = wsRe.ReplaceAllString(text, " ")
 	return strings.TrimSpace(text)
 }
 
diff --git a/internal/ingestion/parser_test.go b/internal/ingestion/parser_test.go
index 54a36eb..62b7d71 100644
--- a/internal/ingestion/parser_test.go
+++ b/internal/ingestion/parser_test.go
@@ -48,6 +48,11 @@ func TestExtractLede(t *testing.T) {
 			"It&#39;s a new era&#8212;one of change.",
 			"It's a new era\u2014one of change.",
 		},
+		{
+			"adjacent block tags inject a space",
+			"<p>...end of moments</p><p>Clarence B Jones, a former...</p>",
+			"...end of moments Clarence B Jones, a former...",
+		},
 		{
 			"tags and entities combined",
 			"<p>Oil prices rose &amp; gas fell by &gt;5%.</p>",