Initial commit

internal/ingestion/parser_test.go

@@ -0,0 +1,66 @@
+package ingestion
+
+import "testing"
+
+func TestExtractLede(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  string
+	}{
+		{
+			"plain text verbatim",
+			"The president signed a new bill. More details followed in the afternoon session.",
+			"The president signed a new bill. More details followed in the afternoon session.",
+		},
+		{
+			"single sentence",
+			"Breaking news from the capital.",
+			"Breaking news from the capital.",
+		},
+		{
+			"empty",
+			"",
+			"",
+		},
+		{
+			"html stripped",
+			"<p>The company announced a <strong>major</strong> acquisition. Shares rose 5%.</p>",
+			"The company announced a major acquisition. Shares rose 5%.",
+		},
+		{
+			"nested html",
+			"<div><p>Apple revealed the iPhone 16. It features a new chip.</p></div>",
+			"Apple revealed the iPhone 16. It features a new chip.",
+		},
+		{
+			"whitespace trimmed",
+			"  \n  Some news happened today. More to come.  \n  ",
+			"Some news happened today. More to come.",
+		},
+		{
+			"html entities decoded",
+			"The U.S. &amp; EU agreed on a &quot;landmark&quot; deal worth &gt;$1bn.",
+			`The U.S. & EU agreed on a "landmark" deal worth >$1bn.`,
+		},
+		{
+			"numeric entities",
+			"It&#39;s a new era&#8212;one of change.",
+			"It's a new era\u2014one of change.",
+		},
+		{
+			"tags and entities combined",
+			"<p>Oil prices rose &amp; gas fell by &gt;5%.</p>",
+			"Oil prices rose & gas fell by >5%.",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := extractLede(tt.input)
+			if got != tt.want {
+				t.Errorf("extractLede(%q) = %q, want %q", tt.input, got, tt.want)
+			}
+		})
+	}
+}