drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Precompute content_chars to drop per-render body scans

Browse Source 
The N-min-read chip derived reading time via LENGTH(content) over the
full article-body TEXT column on every listing render. LENGTH can't use
an index, so SQLite read each row's whole body per request on the hottest
path. Cache the character count in a content_chars column filled at insert
time (backfilled for existing rows), and point StoryContentLengths at it.
This commit is contained in:
prosolis
2026-07-07 22:41:41 -07:00
parent 8f9fcc45f3
commit 74aa578a2d
3 changed files with 30 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/storage/queries.go
View File

@@ -7,6 +7,7 @@ import (
"log/slog"
"strings"
"time"
"unicode/utf8"
)
func nowUnix() int64 {
@@ -38,9 +39,9 @@ func InsertStory(s *Story) error {
publishedAt = s.PublishedAt
}
_, err := Get().Exec(
`INSERT INTO stories (guid, headline, lede, content, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt,
`INSERT INTO stories (guid, headline, lede, content, content_chars, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), utf8.RuneCountInString(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt,
)
return err
}
@@ -414,8 +415,10 @@ func TrendingStories(limit int, sinceDay int64) ([]Story, error) {
// StoryContentLengths returns the captured article text length (in characters)
// for the given story ids, as a map keyed by id. Used to estimate a "N min read"
// chip without transferring the full body: only LENGTH(content) crosses the
// wire. Ids with no captured content are absent (treated as zero by callers).
// chip. It reads the precomputed content_chars column rather than LENGTH()-ing
// the full body, so the hot listing path never scans article text. Ids with no
// captured content have content_chars = 0 and are absent from the map (treated
// as zero by callers).
func StoryContentLengths(ids []int64) map[int64]int {
out := make(map[int64]int, len(ids))
if len(ids) == 0 {
@@ -423,8 +426,8 @@ func StoryContentLengths(ids []int64) map[int64]int {
}
ph, args := intInClause(ids)
rows, err := Get().Query(
`SELECT id, LENGTH(content) FROM stories
WHERE id IN (`+ph+`) AND content IS NOT NULL AND content <> ''`, args...)
`SELECT id, content_chars FROM stories
WHERE id IN (`+ph+`) AND content_chars > 0`, args...)
if err != nil {
slog.Error("story content lengths query failed", "err", err)
return out