diff --git a/internal/storage/db.go b/internal/storage/db.go index 8f0a205..9cf9ca8 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -85,6 +85,11 @@ func runMigrations(d *sql.DB) error { // else the body scraped during paywall detection) for reader mode. Stories // ingested before this column existed simply have NULL and fall back to lede. addColumnIfMissing(d, "stories", "content", "TEXT") + // content_chars caches the character count of content so the "N min read" + // chip never has to LENGTH() the full body on the hot listing path. Filled at + // insert time; the backfill below populates rows that predate the column. + addColumnIfMissing(d, "stories", "content_chars", "INTEGER NOT NULL DEFAULT 0") + backfillContentChars(d) addColumnIfMissing(d, "stories", "published_at", "INTEGER") addColumnIfMissing(d, "post_log", "url_canonical", "TEXT") addColumnIfMissing(d, "post_log", "forced", "INTEGER NOT NULL DEFAULT 0") @@ -154,6 +159,20 @@ func exec(label, query string, args ...any) { } } +// backfillContentChars populates content_chars for rows carrying a body but a +// zero count — i.e. stories ingested before the column existed. LENGTH() counts +// characters (code points) for TEXT, matching the utf8.RuneCountInString done at +// insert. After the first run this matches no rows (bodied stories are set, +// bodyless ones stay 0 and are filtered by content IS NOT NULL), so it's a cheap +// startup no-op thereafter. +func backfillContentChars(d *sql.DB) { + if _, err := d.Exec( + `UPDATE stories SET content_chars = LENGTH(content) + WHERE content_chars = 0 AND content IS NOT NULL AND content <> ''`); err != nil { + slog.Error("backfill content_chars failed", "err", err) + } +} + func addColumnIfMissing(d *sql.DB, table, column, columnType string) { q := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, columnType) if _, err := d.Exec(q); err != nil { diff --git a/internal/storage/queries.go b/internal/storage/queries.go index 46f2e87..cdd41d0 100644 --- a/internal/storage/queries.go +++ b/internal/storage/queries.go @@ -7,6 +7,7 @@ import ( "log/slog" "strings" "time" + "unicode/utf8" ) func nowUnix() int64 { @@ -38,9 +39,9 @@ func InsertStory(s *Story) error { publishedAt = s.PublishedAt } _, err := Get().Exec( - `INSERT INTO stories (guid, headline, lede, content, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt, + `INSERT INTO stories (guid, headline, lede, content, content_chars, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), utf8.RuneCountInString(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt, ) return err } @@ -414,8 +415,10 @@ func TrendingStories(limit int, sinceDay int64) ([]Story, error) { // StoryContentLengths returns the captured article text length (in characters) // for the given story ids, as a map keyed by id. Used to estimate a "N min read" -// chip without transferring the full body: only LENGTH(content) crosses the -// wire. Ids with no captured content are absent (treated as zero by callers). +// chip. It reads the precomputed content_chars column rather than LENGTH()-ing +// the full body, so the hot listing path never scans article text. Ids with no +// captured content have content_chars = 0 and are absent from the map (treated +// as zero by callers). func StoryContentLengths(ids []int64) map[int64]int { out := make(map[int64]int, len(ids)) if len(ids) == 0 { @@ -423,8 +426,8 @@ func StoryContentLengths(ids []int64) map[int64]int { } ph, args := intInClause(ids) rows, err := Get().Query( - `SELECT id, LENGTH(content) FROM stories - WHERE id IN (`+ph+`) AND content IS NOT NULL AND content <> ''`, args...) + `SELECT id, content_chars FROM stories + WHERE id IN (`+ph+`) AND content_chars > 0`, args...) if err != nil { slog.Error("story content lengths query failed", "err", err) return out diff --git a/internal/storage/schema.go b/internal/storage/schema.go index cc55c4c..f4962b9 100644 --- a/internal/storage/schema.go +++ b/internal/storage/schema.go @@ -7,6 +7,7 @@ CREATE TABLE IF NOT EXISTS stories ( headline TEXT NOT NULL, lede TEXT, content TEXT, + content_chars INTEGER NOT NULL DEFAULT 0, image_url TEXT, article_url TEXT NOT NULL, url_canonical TEXT,