Add feed/reader mode with full-article capture at ingest

Reader mode presents the stories on a page one at a time in a focused
overlay, marking each read as it's shown. Left/right arrows (or the header
book button / `f`) page through them; read stories dim on the grid. Read
state is device-local in localStorage.

Backing this required actually capturing article bodies, which Pete wasn't
doing — it kept only the RSS <description> lede and discarded content:encoded:

- stories.content column (idempotent migration; old rows fall back to lede)
- parser keeps content:encoded as paragraph-preserving text
- article fetch already done for paywall detection now also returns its body,
  so ingest stores the richer of feed-content vs scraped body with no extra
  request (prefers the archive snapshot body for paywalled stories)
- GET /api/article?id= serves the stored text; card queries now select id and
  expose it as data-id for the reader

Tests cover content extraction, the storage round-trip, and the article
endpoint + card rendering end to end.

internal/ingestion/article.go

@@ -65,6 +65,7 @@ const googlebotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.
 // ArticleMeta is what we can learn from fetching an article page directly.
 type ArticleMeta struct {
 	ImageURL   string // og:image or twitter:image, absolute URL
+	BodyText   string // extracted visible article body text (capped at MaxBodyChars)
 	BodyChars  int    // length of extracted visible body text
 	Fetched    bool   // true if we got an HTTP 200 with HTML
 	FetchError bool   // true if the fetch failed at the network/HTTP layer
@@ -192,7 +193,10 @@ func fetchArticleMetaOnce(articleURL, ua, referer string) ArticleMeta {
 
 	meta.Fetched = true
 	meta.ImageURL = extractOGImage(doc, articleURL)
-	meta.BodyChars = extractBodyChars(doc)
+	// One body extraction serves both purposes: the text feeds reader mode and
+	// its length is the paywall/thin-body signal.
+	meta.BodyText = extractBodyText(doc)
+	meta.BodyChars = len(meta.BodyText)
 	meta.Paywalled = detectPaywall(doc)
 	meta.SubscriberOnly = detectSubscriberOnly(articleURL, doc)
 	return meta

internal/ingestion/parser.go

@@ -31,6 +31,7 @@ type FeedItem struct {
 	GUID        string
 	Headline    string
 	Lede        string
+	Content     string // full article text from content:encoded, block structure preserved; "" when the feed ships only a snippet
 	ImageURL    string
 	ArticleURL  string
 	Source      string
@@ -43,8 +44,19 @@ var (
 	htmlTagRe = regexp.MustCompile(`<[^>]*>`)
 	wsRe      = regexp.MustCompile(`\s+`)
 	imgSrcRe  = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
+
+	// Used by extractContentText to preserve paragraph structure when turning
+	// content:encoded HTML into plain text.
+	blockCloseRe  = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|blockquote|article|section|figure|figcaption|ul|ol|table|tr|pre)>`)
+	brRe          = regexp.MustCompile(`(?i)<br\s*/?>`)
+	hspaceRe      = regexp.MustCompile(`[ \t\f\r]+`)
+	manyNewlineRe = regexp.MustCompile(`\n{3,}`)
 )
 
+// maxContentChars bounds the stored article text. Generous enough for any real
+// article while keeping row size and the reader payload sane.
+const maxContentChars = 20000
+
 // FetchFeed fetches and parses an RSS/Atom feed, returning items. ua overrides
 // the User-Agent header; pass "" to use Pete's honest default bot UA.
 func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
@@ -79,6 +91,7 @@ func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
 			GUID:        itemGUID(item),
 			Headline:    strings.TrimSpace(item.Title),
 			Lede:        extractLede(item.Description),
+			Content:     extractContentText(item.Content),
 			ImageURL:    NormalizeImageURL(extractImageURL(item)),
 			ArticleURL:  strings.TrimSpace(item.Link),
 			Language:    itemLanguage(item),
@@ -142,6 +155,31 @@ func extractLede(desc string) string {
 	return strings.TrimSpace(text)
 }
 
+// extractContentText converts a feed's content:encoded HTML into plain text
+// with paragraph breaks preserved, for reader mode. Block-level boundaries
+// (</p>, </div>, <br>, list items, headings…) become newlines before the
+// remaining tags are stripped, so the reader can re-wrap the text into
+// paragraphs. Returns "" when the feed carried no content:encoded.
+func extractContentText(raw string) string {
+	if strings.TrimSpace(raw) == "" {
+		return ""
+	}
+	s := brRe.ReplaceAllString(raw, "\n")
+	s = blockCloseRe.ReplaceAllString(s, "\n\n")
+	s = htmlTagRe.ReplaceAllString(s, "")
+	s = html.UnescapeString(s)
+	lines := strings.Split(s, "\n")
+	for i, ln := range lines {
+		lines[i] = strings.TrimSpace(hspaceRe.ReplaceAllString(ln, " "))
+	}
+	s = manyNewlineRe.ReplaceAllString(strings.Join(lines, "\n"), "\n\n")
+	s = strings.TrimSpace(s)
+	if len(s) > maxContentChars {
+		s = s[:maxContentChars]
+	}
+	return s
+}
+
 // extractImageURL tries to find an image URL from feed item metadata.
 // Order: media:content/thumbnail, enclosures, item.Image, then <img> tags
 // scraped from content:encoded / description (catches feeds like Ars that

internal/ingestion/parser_test.go

@@ -1,6 +1,9 @@
 package ingestion
 
-import "testing"
+import (
+	"strings"
+	"testing"
+)
 
 func TestExtractLede(t *testing.T) {
 	tests := []struct {
@@ -69,3 +72,26 @@ func TestExtractLede(t *testing.T) {
 		})
 	}
 }
+
+func TestExtractContentText(t *testing.T) {
+	if got := extractContentText(""); got != "" {
+		t.Errorf("empty input = %q, want empty", got)
+	}
+	if got := extractContentText("   \n  "); got != "" {
+		t.Errorf("whitespace-only input = %q, want empty", got)
+	}
+
+	in := `<p>First paragraph with <a href="/x">a link</a> &amp; entity.</p>` +
+		`<p>Second line one.<br>line two.</p><ul><li>item</li></ul>`
+	got := extractContentText(in)
+	want := "First paragraph with a link & entity.\n\nSecond line one.\nline two.\n\nitem"
+	if got != want {
+		t.Errorf("extractContentText:\n got  %q\n want %q", got, want)
+	}
+
+	// Output is capped so a runaway body can't bloat the row.
+	long := "<p>" + strings.Repeat("x", maxContentChars+500) + "</p>"
+	if n := len(extractContentText(long)); n > maxContentChars {
+		t.Errorf("capped length = %d, want <= %d", n, maxContentChars)
+	}
+}

internal/ingestion/poller.go

@@ -160,6 +160,10 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
 				"guid", items[i].GUID, "url", originalURL, "source", src.Name)
 			continue
 		}
+		// bodyText is the article text we may store for reader mode. It starts
+		// as the body scraped during the fetch above and is upgraded to the
+		// snapshot's body if we end up swapping in an archive snapshot.
+		bodyText := meta.BodyText
 		gated := meta.Gated() || (meta.Fetched && meta.BodyChars < PaywallBodyThreshold)
 		// paywalled tracks whether the link the user will click is still
 		// gated — i.e. gating was detected AND no archive workaround worked.
@@ -175,6 +179,9 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
 					if items[i].ImageURL == "" && snapMeta.ImageURL != "" {
 						items[i].ImageURL = snapMeta.ImageURL
 					}
+					if snapMeta.BodyText != "" {
+						bodyText = snapMeta.BodyText
+					}
 					workedAround = true
 					slog.Info("paywall detected, using archive snapshot",
 						"guid", items[i].GUID, "original", originalURL,
@@ -209,10 +216,18 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
 		if publishedAt > seenAt {
 			publishedAt = seenAt
 		}
+		// Reader-mode body: prefer whichever source gave us more complete text.
+		// Feeds that ship full content:encoded usually win; feeds that only
+		// syndicate a snippet fall back to the scraped article body.
+		content := items[i].Content
+		if len(bodyText) > len(content) {
+			content = bodyText
+		}
 		if err := storage.InsertStory(&storage.Story{
 			GUID:         items[i].GUID,
 			Headline:     items[i].Headline,
 			Lede:         items[i].Lede,
+			Content:      content,
 			ImageURL:     items[i].ImageURL,
 			ArticleURL:   items[i].ArticleURL,
 			URLCanonical: canonical,