55aa167151
Reader mode presents the stories on a page one at a time in a focused overlay, marking each read as it's shown. Left/right arrows (or the header book button / `f`) page through them; read stories dim on the grid. Read state is device-local in localStorage. Backing this required actually capturing article bodies, which Pete wasn't doing — it kept only the RSS <description> lede and discarded content:encoded: - stories.content column (idempotent migration; old rows fall back to lede) - parser keeps content:encoded as paragraph-preserving text - article fetch already done for paywall detection now also returns its body, so ingest stores the richer of feed-content vs scraped body with no extra request (prefers the archive snapshot body for paywalled stories) - GET /api/article?id= serves the stored text; card queries now select id and expose it as data-id for the reader Tests cover content extraction, the storage round-trip, and the article endpoint + card rendering end to end.
98 lines
2.6 KiB
Go
98 lines
2.6 KiB
Go
package ingestion
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestExtractLede(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
input string
|
|
want string
|
|
}{
|
|
{
|
|
"plain text verbatim",
|
|
"The president signed a new bill. More details followed in the afternoon session.",
|
|
"The president signed a new bill. More details followed in the afternoon session.",
|
|
},
|
|
{
|
|
"single sentence",
|
|
"Breaking news from the capital.",
|
|
"Breaking news from the capital.",
|
|
},
|
|
{
|
|
"empty",
|
|
"",
|
|
"",
|
|
},
|
|
{
|
|
"html stripped",
|
|
"<p>The company announced a <strong>major</strong> acquisition. Shares rose 5%.</p>",
|
|
"The company announced a major acquisition. Shares rose 5%.",
|
|
},
|
|
{
|
|
"nested html",
|
|
"<div><p>Apple revealed the iPhone 16. It features a new chip.</p></div>",
|
|
"Apple revealed the iPhone 16. It features a new chip.",
|
|
},
|
|
{
|
|
"whitespace trimmed",
|
|
" \n Some news happened today. More to come. \n ",
|
|
"Some news happened today. More to come.",
|
|
},
|
|
{
|
|
"html entities decoded",
|
|
"The U.S. & EU agreed on a "landmark" deal worth >$1bn.",
|
|
`The U.S. & EU agreed on a "landmark" deal worth >$1bn.`,
|
|
},
|
|
{
|
|
"numeric entities",
|
|
"It's a new era—one of change.",
|
|
"It's a new era\u2014one of change.",
|
|
},
|
|
{
|
|
"adjacent block tags inject a space",
|
|
"<p>...end of moments</p><p>Clarence B Jones, a former...</p>",
|
|
"...end of moments Clarence B Jones, a former...",
|
|
},
|
|
{
|
|
"tags and entities combined",
|
|
"<p>Oil prices rose & gas fell by >5%.</p>",
|
|
"Oil prices rose & gas fell by >5%.",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := extractLede(tt.input)
|
|
if got != tt.want {
|
|
t.Errorf("extractLede(%q) = %q, want %q", tt.input, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestExtractContentText(t *testing.T) {
|
|
if got := extractContentText(""); got != "" {
|
|
t.Errorf("empty input = %q, want empty", got)
|
|
}
|
|
if got := extractContentText(" \n "); got != "" {
|
|
t.Errorf("whitespace-only input = %q, want empty", got)
|
|
}
|
|
|
|
in := `<p>First paragraph with <a href="/x">a link</a> & entity.</p>` +
|
|
`<p>Second line one.<br>line two.</p><ul><li>item</li></ul>`
|
|
got := extractContentText(in)
|
|
want := "First paragraph with a link & entity.\n\nSecond line one.\nline two.\n\nitem"
|
|
if got != want {
|
|
t.Errorf("extractContentText:\n got %q\n want %q", got, want)
|
|
}
|
|
|
|
// Output is capped so a runaway body can't bloat the row.
|
|
long := "<p>" + strings.Repeat("x", maxContentChars+500) + "</p>"
|
|
if n := len(extractContentText(long)); n > maxContentChars {
|
|
t.Errorf("capped length = %d, want <= %d", n, maxContentChars)
|
|
}
|
|
}
|