Fix glued-together RSS ledes; parse per-item language

Two ingestion changes:

- extractLede replaced HTML tags with empty string, so adjacent
  block tags like </p><p> fused words across paragraphs. Replace
  tags with a space and collapse whitespace.
- Pull each item's <language> tag (or dc:language) into FeedItem
  so the poller can filter on it. Politico Europe publishes the
  same story in en / fr / de side-by-side and we want to keep
  only one language per source.

internal/ingestion/parser.go

@@ -28,11 +28,13 @@ type FeedItem struct {
 	ArticleURL  string
 	Source      string
 	DirectRoute string
+	Language    string // per-item <language> (or dc:language) when present; "" otherwise
 	PublishedAt int64 // unix seconds from RSS pubDate / Atom published (or updated as fallback); 0 if absent or unparseable
 }
 
 var (
 	htmlTagRe = regexp.MustCompile(`<[^>]*>`)
+	wsRe      = regexp.MustCompile(`\s+`)
 	imgSrcRe  = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
 )
 
@@ -59,6 +61,7 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
 			Lede:        extractLede(item.Description),
 			ImageURL:    NormalizeImageURL(extractImageURL(item)),
 			ArticleURL:  strings.TrimSpace(item.Link),
+			Language:    itemLanguage(item),
 			PublishedAt: itemPublished(item),
 		}
 		if fi.GUID == "" || fi.ArticleURL == "" {
@@ -71,6 +74,22 @@ func FetchFeed(feedURL string) ([]FeedItem, error) {
 	return items, nil
 }
 
+// itemLanguage returns the per-item language tag if the feed provides one.
+// Politico Europe puts a raw <language> inside each <item>; gofeed parks
+// unknown item-level elements in item.Custom. We also check the standard
+// dc:language extension for feeds that use Dublin Core.
+func itemLanguage(item *gofeed.Item) string {
+	if item.Custom != nil {
+		if v, ok := item.Custom["language"]; ok {
+			return strings.ToLower(strings.TrimSpace(v))
+		}
+	}
+	if item.DublinCoreExt != nil && len(item.DublinCoreExt.Language) > 0 {
+		return strings.ToLower(strings.TrimSpace(item.DublinCoreExt.Language[0]))
+	}
+	return ""
+}
+
 func itemGUID(item *gofeed.Item) string {
 	if item.GUID != "" {
 		return item.GUID
@@ -96,8 +115,10 @@ func extractLede(desc string) string {
 	if desc == "" {
 		return ""
 	}
-	text := htmlTagRe.ReplaceAllString(desc, "")
+	// Replace tags with a space so adjacent blocks like </p><p> don't fuse words.
+	text := htmlTagRe.ReplaceAllString(desc, " ")
 	text = html.UnescapeString(text)
+	text = wsRe.ReplaceAllString(text, " ")
 	return strings.TrimSpace(text)
 }
 

internal/ingestion/parser_test.go

@@ -48,6 +48,11 @@ func TestExtractLede(t *testing.T) {
 			"It's a new era—one of change.",
 			"It's a new era\u2014one of change.",
 		},
+		{
+			"adjacent block tags inject a space",
+			"<p>...end of moments</p><p>Clarence B Jones, a former...</p>",
+			"...end of moments Clarence B Jones, a former...",
+		},
 		{
 			"tags and entities combined",
 			"<p>Oil prices rose &amp; gas fell by &gt;5%.</p>",