Pete/internal/ingestion/parser_test.go

package ingestion

import (
	"strings"
	"testing"
)

func TestExtractLede(t *testing.T) {
	tests := []struct {
		name  string
		input string
		want  string
	}{
		{
			"plain text verbatim",
			"The president signed a new bill. More details followed in the afternoon session.",
			"The president signed a new bill. More details followed in the afternoon session.",
		},
		{
			"single sentence",
			"Breaking news from the capital.",
			"Breaking news from the capital.",
		},
		{
			"empty",
			"",
			"",
		},
		{
			"html stripped",
			"<p>The company announced a <strong>major</strong> acquisition. Shares rose 5%.</p>",
			"The company announced a major acquisition. Shares rose 5%.",
		},
		{
			"nested html",
			"<div><p>Apple revealed the iPhone 16. It features a new chip.</p></div>",
			"Apple revealed the iPhone 16. It features a new chip.",
		},
		{
			"whitespace trimmed",
			"  \n  Some news happened today. More to come.  \n  ",
			"Some news happened today. More to come.",
		},
		{
			"html entities decoded",
			"The U.S. &amp; EU agreed on a &quot;landmark&quot; deal worth &gt;$1bn.",
			`The U.S. & EU agreed on a "landmark" deal worth >$1bn.`,
		},
		{
			"numeric entities",
			"It&#39;s a new era&#8212;one of change.",
			"It's a new era\u2014one of change.",
		},
		{
			"adjacent block tags inject a space",
			"<p>...end of moments</p><p>Clarence B Jones, a former...</p>",
			"...end of moments Clarence B Jones, a former...",
		},
		{
			"tags and entities combined",
			"<p>Oil prices rose &amp; gas fell by &gt;5%.</p>",
			"Oil prices rose & gas fell by >5%.",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := extractLede(tt.input)
			if got != tt.want {
				t.Errorf("extractLede(%q) = %q, want %q", tt.input, got, tt.want)
			}
		})
	}
}

func TestExtractContentText(t *testing.T) {
	if got := extractContentText(""); got != "" {
		t.Errorf("empty input = %q, want empty", got)
	}
	if got := extractContentText("   \n  "); got != "" {
		t.Errorf("whitespace-only input = %q, want empty", got)
	}

	in := `<p>First paragraph with <a href="/x">a link</a> &amp; entity.</p>` +
		`<p>Second line one.<br>line two.</p><ul><li>item</li></ul>`
	got := extractContentText(in)
	want := "First paragraph with a link & entity.\n\nSecond line one.\nline two.\n\nitem"
	if got != want {
		t.Errorf("extractContentText:\n got  %q\n want %q", got, want)
	}

	// Output is capped so a runaway body can't bloat the row.
	long := "<p>" + strings.Repeat("x", maxContentChars+500) + "</p>"
	if n := len(extractContentText(long)); n > maxContentChars {
		t.Errorf("capped length = %d, want <= %d", n, maxContentChars)
	}

	// Inline embed scripts (Datawrapper resizers, analytics) must be dropped
	// whole — tags and body — not left as literal reader text. htmlTagRe alone
	// strips only the tags, so extractContentText has to remove the element first.
	embed := `<p>Net trust rating of +11 percent nationally.</p>` +
		`<figure><script type="text/javascript">!function(){"use strict";window.addEventListener("message",(function(a){var e=document.querySelectorAll("iframe");}))}();</script></figure>` +
		`<p>Burnham is on the cusp of becoming prime minister.</p>`
	gotEmbed := extractContentText(embed)
	if strings.Contains(gotEmbed, "datawrapper") || strings.Contains(gotEmbed, "addEventListener") || strings.Contains(gotEmbed, "querySelectorAll") {
		t.Errorf("inline script leaked into content text:\n%s", gotEmbed)
	}
	for _, want := range []string{"Net trust rating", "prime minister"} {
		if !strings.Contains(gotEmbed, want) {
			t.Errorf("prose %q missing from content text:\n%s", want, gotEmbed)
		}
	}

	// The same guard applies to descriptions/ledes.
	ledeIn := `Trust gap widens.<script>track("x");</script> North vs south.`
	if gotLede := extractLede(ledeIn); strings.Contains(gotLede, "track") {
		t.Errorf("inline script leaked into lede: %q", gotLede)
	}
}