drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add feed/reader mode with full-article capture at ingest

Browse Source 
Reader mode presents the stories on a page one at a time in a focused
overlay, marking each read as it's shown. Left/right arrows (or the header
book button / `f`) page through them; read stories dim on the grid. Read
state is device-local in localStorage.

Backing this required actually capturing article bodies, which Pete wasn't
doing — it kept only the RSS <description> lede and discarded content:encoded:

- stories.content column (idempotent migration; old rows fall back to lede)
- parser keeps content:encoded as paragraph-preserving text
- article fetch already done for paywall detection now also returns its body,
  so ingest stores the richer of feed-content vs scraped body with no extra
  request (prefers the archive snapshot body for paywalled stories)
- GET /api/article?id= serves the stored text; card queries now select id and
  expose it as data-id for the reader

Tests cover content extraction, the storage round-trip, and the article
endpoint + card rendering end to end.
This commit is contained in:
prosolis
2026-07-06 22:46:14 -07:00
parent 410f8dda0a
commit 55aa167151
17 changed files with 803 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/storage/db.go
View File

@@ -81,6 +81,10 @@ func runMigrations(d *sql.DB) error {
addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
// content holds the full article text (feed content:encoded when present,
// else the body scraped during paywall detection) for reader mode. Stories
// ingested before this column existed simply have NULL and fall back to lede.
addColumnIfMissing(d, "stories", "content", "TEXT")
addColumnIfMissing(d, "stories", "published_at", "INTEGER")
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
addColumnIfMissing(d, "post_log", "forced", "INTEGER NOT NULL DEFAULT 0")

33
internal/storage/queries.go
View File

@@ -38,13 +38,28 @@ func InsertStory(s *Story) error {
publishedAt = s.PublishedAt
}
_, err := Get().Exec(
`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt,
`INSERT INTO stories (guid, headline, lede, content, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt,
)
return err
}
// GetStoryReaderText returns the stored full article text and lede for a single
// story, for reader mode. found is false when no story has that id.
func GetStoryReaderText(id int64) (content, lede string, found bool, err error) {
var c sql.NullString
var l sql.NullString
row := Get().QueryRow(`SELECT content, lede FROM stories WHERE id = ?`, id)
switch err = row.Scan(&c, &l); {
case errors.Is(err, sql.ErrNoRows):
return "", "", false, nil
case err != nil:
return "", "", false, err
}
return c.String, l.String, true, nil
}
// nullIfEmpty returns a SQL NULL for empty strings so partial unique indexes
// (WHERE col IS NOT NULL) can hold multiple "unknown" rows without conflict.
func nullIfEmpty(s string) any {
@@ -236,7 +251,7 @@ func GetNewestPostableStoryByChannel(channel string) (*Story, error) {
// channels (_discarded, _duplicate) are excluded.
func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories s
WHERE s.classified = 1
@@ -250,7 +265,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
@@ -262,7 +277,7 @@ func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error)
// channels, newest first. Sentinel channels are excluded.
func ListAllClassified(limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories s
WHERE s.classified = 1
@@ -277,7 +292,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
@@ -289,7 +304,7 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
// Matrix, ordered by post time (newest first). Posted is always true.
func ListRecentlyPosted(limit int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at
FROM stories s
JOIN post_log p ON p.guid = s.guid
GROUP BY s.guid
@@ -302,7 +317,7 @@ func ListRecentlyPosted(limit int) ([]Story, error) {
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil {
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil {
return nil, err
}
s.Posted = true

1
internal/storage/schema.go
View File

@@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS stories (
guid TEXT UNIQUE NOT NULL,
headline TEXT NOT NULL,
lede TEXT,
content TEXT,
image_url TEXT,
article_url TEXT NOT NULL,
url_canonical TEXT,

61
internal/storage/storage_test.go
View File

@@ -61,6 +61,67 @@ func TestInsertStoryAndGUIDSeen(t *testing.T) {
}
}
func TestGetStoryReaderText(t *testing.T) {
setupTestDB(t)
s := &Story{
GUID: "reader-guid",
Headline: "Reader Headline",
Lede: "Short lede.",
Content: "First paragraph.\n\nSecond paragraph.",
ArticleURL: "https://example.com/reader",
Source: "Src",
SeenAt: 1700000000,
}
if err := InsertStory(s); err != nil {
t.Fatal(err)
}
var id int64
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, s.GUID).Scan(&id); err != nil {
t.Fatal(err)
}
content, lede, found, err := GetStoryReaderText(id)
if err != nil {
t.Fatal(err)
}
if !found {
t.Fatal("expected story to be found")
}
if content != s.Content {
t.Errorf("content = %q, want %q", content, s.Content)
}
if lede != s.Lede {
t.Errorf("lede = %q, want %q", lede, s.Lede)
}
// A story ingested before content capture (no content) is still found, with
// an empty content string so the reader falls back to the lede.
bare := &Story{GUID: "bare-guid", Headline: "H", Lede: "Only a lede.", ArticleURL: "https://a.com", Source: "S", SeenAt: 1}
if err := InsertStory(bare); err != nil {
t.Fatal(err)
}
var bareID int64
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, bare.GUID).Scan(&bareID); err != nil {
t.Fatal(err)
}
content, lede, found, err = GetStoryReaderText(bareID)
if err != nil || !found {
t.Fatalf("bare story: found=%v err=%v", found, err)
}
if content != "" {
t.Errorf("bare content = %q, want empty", content)
}
if lede != "Only a lede." {
t.Errorf("bare lede = %q", lede)
}
// Unknown id reports not-found rather than erroring.
if _, _, found, err = GetStoryReaderText(999999); err != nil || found {
t.Errorf("unknown id: found=%v err=%v, want found=false err=nil", found, err)
}
}
func TestInsertStoryDuplicateGUID(t *testing.T) {
setupTestDB(t)

1
internal/storage/types.go
View File

@@ -6,6 +6,7 @@ type Story struct {
GUID string
Headline string
Lede string
Content string // full article text for reader mode (feed content:encoded or scraped body); "" when unavailable
ImageURL string
ArticleURL string
URLCanonical string