Add feed/reader mode with full-article capture at ingest
Reader mode presents the stories on a page one at a time in a focused overlay, marking each read as it's shown. Left/right arrows (or the header book button / `f`) page through them; read stories dim on the grid. Read state is device-local in localStorage. Backing this required actually capturing article bodies, which Pete wasn't doing — it kept only the RSS <description> lede and discarded content:encoded: - stories.content column (idempotent migration; old rows fall back to lede) - parser keeps content:encoded as paragraph-preserving text - article fetch already done for paywall detection now also returns its body, so ingest stores the richer of feed-content vs scraped body with no extra request (prefers the archive snapshot body for paywalled stories) - GET /api/article?id= serves the stored text; card queries now select id and expose it as data-id for the reader Tests cover content extraction, the storage round-trip, and the article endpoint + card rendering end to end.
This commit is contained in:
@@ -31,6 +31,7 @@ type FeedItem struct {
|
||||
GUID string
|
||||
Headline string
|
||||
Lede string
|
||||
Content string // full article text from content:encoded, block structure preserved; "" when the feed ships only a snippet
|
||||
ImageURL string
|
||||
ArticleURL string
|
||||
Source string
|
||||
@@ -43,8 +44,19 @@ var (
|
||||
htmlTagRe = regexp.MustCompile(`<[^>]*>`)
|
||||
wsRe = regexp.MustCompile(`\s+`)
|
||||
imgSrcRe = regexp.MustCompile(`(?i)<img[^>]+src=["']([^"']+)["']`)
|
||||
|
||||
// Used by extractContentText to preserve paragraph structure when turning
|
||||
// content:encoded HTML into plain text.
|
||||
blockCloseRe = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|blockquote|article|section|figure|figcaption|ul|ol|table|tr|pre)>`)
|
||||
brRe = regexp.MustCompile(`(?i)<br\s*/?>`)
|
||||
hspaceRe = regexp.MustCompile(`[ \t\f\r]+`)
|
||||
manyNewlineRe = regexp.MustCompile(`\n{3,}`)
|
||||
)
|
||||
|
||||
// maxContentChars bounds the stored article text. Generous enough for any real
|
||||
// article while keeping row size and the reader payload sane.
|
||||
const maxContentChars = 20000
|
||||
|
||||
// FetchFeed fetches and parses an RSS/Atom feed, returning items. ua overrides
|
||||
// the User-Agent header; pass "" to use Pete's honest default bot UA.
|
||||
func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
|
||||
@@ -79,6 +91,7 @@ func FetchFeed(feedURL, ua string) ([]FeedItem, error) {
|
||||
GUID: itemGUID(item),
|
||||
Headline: strings.TrimSpace(item.Title),
|
||||
Lede: extractLede(item.Description),
|
||||
Content: extractContentText(item.Content),
|
||||
ImageURL: NormalizeImageURL(extractImageURL(item)),
|
||||
ArticleURL: strings.TrimSpace(item.Link),
|
||||
Language: itemLanguage(item),
|
||||
@@ -142,6 +155,31 @@ func extractLede(desc string) string {
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
// extractContentText converts a feed's content:encoded HTML into plain text
|
||||
// with paragraph breaks preserved, for reader mode. Block-level boundaries
|
||||
// (</p>, </div>, <br>, list items, headings…) become newlines before the
|
||||
// remaining tags are stripped, so the reader can re-wrap the text into
|
||||
// paragraphs. Returns "" when the feed carried no content:encoded.
|
||||
func extractContentText(raw string) string {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return ""
|
||||
}
|
||||
s := brRe.ReplaceAllString(raw, "\n")
|
||||
s = blockCloseRe.ReplaceAllString(s, "\n\n")
|
||||
s = htmlTagRe.ReplaceAllString(s, "")
|
||||
s = html.UnescapeString(s)
|
||||
lines := strings.Split(s, "\n")
|
||||
for i, ln := range lines {
|
||||
lines[i] = strings.TrimSpace(hspaceRe.ReplaceAllString(ln, " "))
|
||||
}
|
||||
s = manyNewlineRe.ReplaceAllString(strings.Join(lines, "\n"), "\n\n")
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) > maxContentChars {
|
||||
s = s[:maxContentChars]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// extractImageURL tries to find an image URL from feed item metadata.
|
||||
// Order: media:content/thumbnail, enclosures, item.Image, then <img> tags
|
||||
// scraped from content:encoded / description (catches feeds like Ars that
|
||||
|
||||
Reference in New Issue
Block a user