Add feed/reader mode with full-article capture at ingest

Reader mode presents the stories on a page one at a time in a focused
overlay, marking each read as it's shown. Left/right arrows (or the header
book button / `f`) page through them; read stories dim on the grid. Read
state is device-local in localStorage.

Backing this required actually capturing article bodies, which Pete wasn't
doing — it kept only the RSS <description> lede and discarded content:encoded:

- stories.content column (idempotent migration; old rows fall back to lede)
- parser keeps content:encoded as paragraph-preserving text
- article fetch already done for paywall detection now also returns its body,
  so ingest stores the richer of feed-content vs scraped body with no extra
  request (prefers the archive snapshot body for paywalled stories)
- GET /api/article?id= serves the stored text; card queries now select id and
  expose it as data-id for the reader

Tests cover content extraction, the storage round-trip, and the article
endpoint + card rendering end to end.
This commit is contained in:
prosolis
2026-07-06 22:46:14 -07:00
parent 410f8dda0a
commit 55aa167151
17 changed files with 803 additions and 13 deletions

View File

@@ -31,6 +31,7 @@ type FeedItem struct {
GUID string
Headline string
Lede string
Content string // full article text from content:encoded, block structure preserved; "" when the feed ships only a snippet
ImageURL string
ArticleURL string
Source string
@@ -43,8 +44,19 @@ var (
htmlTagRe = regexp.MustCompile(`<[^>]*>`)
wsRe = regexp.MustCompile(`\s+`)
imgSrcRe = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
// Used by extractContentText to preserve paragraph structure when turning
// content:encoded HTML into plain text.
blockCloseRe = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|blockquote|article|section|figure|figcaption|ul|ol|table|tr|pre)>`)
brRe = regexp.MustCompile(`(?i)<br\s*/?>`)
hspaceRe = regexp.MustCompile(`[ \t\f\r]+`)
manyNewlineRe = regexp.MustCompile(`\n{3,}`)
)
// maxContentChars bounds the stored article text. Generous enough for any real
// article while keeping row size and the reader payload sane.
const maxContentChars = 20000
// FetchFeed fetches and parses an RSS/Atom feed, returning items. ua overrides
// the User-Agent header; pass "" to use Pete's honest default bot UA.
func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
@@ -79,6 +91,7 @@ func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
GUID: itemGUID(item),
Headline: strings.TrimSpace(item.Title),
Lede: extractLede(item.Description),
Content: extractContentText(item.Content),
ImageURL: NormalizeImageURL(extractImageURL(item)),
ArticleURL: strings.TrimSpace(item.Link),
Language: itemLanguage(item),
@@ -142,6 +155,31 @@ func extractLede(desc string) string {
return strings.TrimSpace(text)
}
// extractContentText converts a feed's content:encoded HTML into plain text
// with paragraph breaks preserved, for reader mode. Block-level boundaries
// (</p>, </div>, <br>, list items, headings…) become newlines before the
// remaining tags are stripped, so the reader can re-wrap the text into
// paragraphs. Returns "" when the feed carried no content:encoded.
func extractContentText(raw string) string {
if strings.TrimSpace(raw) == "" {
return ""
}
s := brRe.ReplaceAllString(raw, "\n")
s = blockCloseRe.ReplaceAllString(s, "\n\n")
s = htmlTagRe.ReplaceAllString(s, "")
s = html.UnescapeString(s)
lines := strings.Split(s, "\n")
for i, ln := range lines {
lines[i] = strings.TrimSpace(hspaceRe.ReplaceAllString(ln, " "))
}
s = manyNewlineRe.ReplaceAllString(strings.Join(lines, "\n"), "\n\n")
s = strings.TrimSpace(s)
if len(s) > maxContentChars {
s = s[:maxContentChars]
}
return s
}
// extractImageURL tries to find an image URL from feed item metadata.
// Order: media:content/thumbnail, enclosures, item.Image, then <img> tags
// scraped from content:encoded / description (catches feeds like Ars that