Precompute content_chars to drop per-render body scans

The N-min-read chip derived reading time via LENGTH(content) over the
full article-body TEXT column on every listing render. LENGTH can't use
an index, so SQLite read each row's whole body per request on the hottest
path. Cache the character count in a content_chars column filled at insert
time (backfilled for existing rows), and point StoryContentLengths at it.
This commit is contained in:
prosolis
2026-07-07 22:41:41 -07:00
parent 8f9fcc45f3
commit 74aa578a2d
3 changed files with 30 additions and 7 deletions

View File

@@ -85,6 +85,11 @@ func runMigrations(d *sql.DB) error {
// else the body scraped during paywall detection) for reader mode. Stories
// ingested before this column existed simply have NULL and fall back to lede.
addColumnIfMissing(d, "stories", "content", "TEXT")
// content_chars caches the character count of content so the "N min read"
// chip never has to LENGTH() the full body on the hot listing path. Filled at
// insert time; the backfill below populates rows that predate the column.
addColumnIfMissing(d, "stories", "content_chars", "INTEGER NOT NULL DEFAULT 0")
backfillContentChars(d)
addColumnIfMissing(d, "stories", "published_at", "INTEGER")
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
addColumnIfMissing(d, "post_log", "forced", "INTEGER NOT NULL DEFAULT 0")
@@ -154,6 +159,20 @@ func exec(label, query string, args ...any) {
}
}
// backfillContentChars populates content_chars for rows carrying a body but a
// zero count — i.e. stories ingested before the column existed. LENGTH() counts
// characters (code points) for TEXT, matching the utf8.RuneCountInString done at
// insert. After the first run this matches no rows (bodied stories are set,
// bodyless ones stay 0 and are filtered by content IS NOT NULL), so it's a cheap
// startup no-op thereafter.
func backfillContentChars(d *sql.DB) {
if _, err := d.Exec(
`UPDATE stories SET content_chars = LENGTH(content)
WHERE content_chars = 0 AND content IS NOT NULL AND content <> ''`); err != nil {
slog.Error("backfill content_chars failed", "err", err)
}
}
func addColumnIfMissing(d *sql.DB, table, column, columnType string) {
q := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, columnType)
if _, err := d.Exec(q); err != nil {

View File

@@ -7,6 +7,7 @@ import (
"log/slog"
"strings"
"time"
"unicode/utf8"
)
func nowUnix() int64 {
@@ -38,9 +39,9 @@ func InsertStory(s *Story) error {
publishedAt = s.PublishedAt
}
_, err := Get().Exec(
`INSERT INTO stories (guid, headline, lede, content, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt,
`INSERT INTO stories (guid, headline, lede, content, content_chars, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), utf8.RuneCountInString(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt,
)
return err
}
@@ -414,8 +415,10 @@ func TrendingStories(limit int, sinceDay int64) ([]Story, error) {
// StoryContentLengths returns the captured article text length (in characters)
// for the given story ids, as a map keyed by id. Used to estimate a "N min read"
// chip without transferring the full body: only LENGTH(content) crosses the
// wire. Ids with no captured content are absent (treated as zero by callers).
// chip. It reads the precomputed content_chars column rather than LENGTH()-ing
// the full body, so the hot listing path never scans article text. Ids with no
// captured content have content_chars = 0 and are absent from the map (treated
// as zero by callers).
func StoryContentLengths(ids []int64) map[int64]int {
out := make(map[int64]int, len(ids))
if len(ids) == 0 {
@@ -423,8 +426,8 @@ func StoryContentLengths(ids []int64) map[int64]int {
}
ph, args := intInClause(ids)
rows, err := Get().Query(
`SELECT id, LENGTH(content) FROM stories
WHERE id IN (`+ph+`) AND content IS NOT NULL AND content <> ''`, args...)
`SELECT id, content_chars FROM stories
WHERE id IN (`+ph+`) AND content_chars > 0`, args...)
if err != nil {
slog.Error("story content lengths query failed", "err", err)
return out

View File

@@ -7,6 +7,7 @@ CREATE TABLE IF NOT EXISTS stories (
headline TEXT NOT NULL,
lede TEXT,
content TEXT,
content_chars INTEGER NOT NULL DEFAULT 0,
image_url TEXT,
article_url TEXT NOT NULL,
url_canonical TEXT,