drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add feed/reader mode with full-article capture at ingest

Browse Source 
Reader mode presents the stories on a page one at a time in a focused
overlay, marking each read as it's shown. Left/right arrows (or the header
book button / `f`) page through them; read stories dim on the grid. Read
state is device-local in localStorage.

Backing this required actually capturing article bodies, which Pete wasn't
doing — it kept only the RSS <description> lede and discarded content:encoded:

- stories.content column (idempotent migration; old rows fall back to lede)
- parser keeps content:encoded as paragraph-preserving text
- article fetch already done for paywall detection now also returns its body,
  so ingest stores the richer of feed-content vs scraped body with no extra
  request (prefers the archive snapshot body for paywalled stories)
- GET /api/article?id= serves the stored text; card queries now select id and
  expose it as data-id for the reader

Tests cover content extraction, the storage round-trip, and the article
endpoint + card rendering end to end.
This commit is contained in:
prosolis
2026-07-06 22:46:14 -07:00
parent 410f8dda0a
commit 55aa167151
17 changed files with 803 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
internal/storage/queries.go
View File

@@ -38,13 +38,28 @@ func InsertStory(s *Story) error {
publishedAt = s.PublishedAt
}
_, err := Get().Exec(
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt,
`INSERT INTO stories (guid, headline, lede, content, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt,
)
return err
}
// GetStoryReaderText returns the stored full article text and lede for a single
// story, for reader mode. found is false when no story has that id.
func GetStoryReaderText(id int64) (content, lede string, found bool, err error) {
var c sql.NullString
var l sql.NullString
row := Get().QueryRow(`SELECT content, lede FROM stories WHERE id = ?`, id)
switch err = row.Scan(&c, &l); {
case errors.Is(err, sql.ErrNoRows):
return "", "", false, nil
case err != nil:
return "", "", false, err
}
return c.String, l.String, true, nil
}
// nullIfEmpty returns a SQL NULL for empty strings so partial unique indexes
// (WHERE col IS NOT NULL) can hold multiple "unknown" rows without conflict.
func nullIfEmpty(s string) any {
@@ -236,7 +251,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
// channels (_discarded, _duplicate) are excluded.
func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories s
WHERE s.classified = 1
@@ -250,7 +265,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
@@ -262,7 +277,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
// channels, newest first. Sentinel channels are excluded.
func ListAllClassified(limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories s
WHERE s.classified = 1
@@ -277,7 +292,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
@@ -289,7 +304,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
// Matrix, ordered by post time (newest first). Posted is always true.
func ListRecentlyPosted(limit int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at
FROM stories s
JOIN post_log p ON p.guid = s.guid
GROUP BY s.guid
@@ -302,7 +317,7 @@ func ListRecentlyPosted(limit int) ([]Story, error) {
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil {
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil {
return nil, err
}
s.Posted = true