7d469cf8c5
Two ingestion changes: - extractLede replaced HTML tags with empty string, so adjacent block tags like </p><p> fused words across paragraphs. Replace tags with a space and collapse whitespace. - Pull each item's <language> tag (or dc:language) into FeedItem so the poller can filter on it. Politico Europe publishes the same story in en / fr / de side-by-side and we want to keep only one language per source.
72 lines
1.8 KiB
Go
72 lines
1.8 KiB
Go
package ingestion
|
|
|
|
import "testing"
|
|
|
|
func TestExtractLede(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
input string
|
|
want string
|
|
}{
|
|
{
|
|
"plain text verbatim",
|
|
"The president signed a new bill. More details followed in the afternoon session.",
|
|
"The president signed a new bill. More details followed in the afternoon session.",
|
|
},
|
|
{
|
|
"single sentence",
|
|
"Breaking news from the capital.",
|
|
"Breaking news from the capital.",
|
|
},
|
|
{
|
|
"empty",
|
|
"",
|
|
"",
|
|
},
|
|
{
|
|
"html stripped",
|
|
"<p>The company announced a <strong>major</strong> acquisition. Shares rose 5%.</p>",
|
|
"The company announced a major acquisition. Shares rose 5%.",
|
|
},
|
|
{
|
|
"nested html",
|
|
"<div><p>Apple revealed the iPhone 16. It features a new chip.</p></div>",
|
|
"Apple revealed the iPhone 16. It features a new chip.",
|
|
},
|
|
{
|
|
"whitespace trimmed",
|
|
" \n Some news happened today. More to come. \n ",
|
|
"Some news happened today. More to come.",
|
|
},
|
|
{
|
|
"html entities decoded",
|
|
"The U.S. & EU agreed on a "landmark" deal worth >$1bn.",
|
|
`The U.S. & EU agreed on a "landmark" deal worth >$1bn.`,
|
|
},
|
|
{
|
|
"numeric entities",
|
|
"It's a new era—one of change.",
|
|
"It's a new era\u2014one of change.",
|
|
},
|
|
{
|
|
"adjacent block tags inject a space",
|
|
"<p>...end of moments</p><p>Clarence B Jones, a former...</p>",
|
|
"...end of moments Clarence B Jones, a former...",
|
|
},
|
|
{
|
|
"tags and entities combined",
|
|
"<p>Oil prices rose & gas fell by >5%.</p>",
|
|
"Oil prices rose & gas fell by >5%.",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := extractLede(tt.input)
|
|
if got != tt.want {
|
|
t.Errorf("extractLede(%q) = %q, want %q", tt.input, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|