Precompute content_chars to drop per-render body scans
The N-min-read chip derived reading time via LENGTH(content) over the full article-body TEXT column on every listing render. LENGTH can't use an index, so SQLite read each row's whole body per request on the hottest path. Cache the character count in a content_chars column filled at insert time (backfilled for existing rows), and point StoryContentLengths at it.
This commit is contained in:
@@ -85,6 +85,11 @@ func runMigrations(d *sql.DB) error {
|
||||
// else the body scraped during paywall detection) for reader mode. Stories
|
||||
// ingested before this column existed simply have NULL and fall back to lede.
|
||||
addColumnIfMissing(d, "stories", "content", "TEXT")
|
||||
// content_chars caches the character count of content so the "N min read"
|
||||
// chip never has to LENGTH() the full body on the hot listing path. Filled at
|
||||
// insert time; the backfill below populates rows that predate the column.
|
||||
addColumnIfMissing(d, "stories", "content_chars", "INTEGER NOT NULL DEFAULT 0")
|
||||
backfillContentChars(d)
|
||||
addColumnIfMissing(d, "stories", "published_at", "INTEGER")
|
||||
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
|
||||
addColumnIfMissing(d, "post_log", "forced", "INTEGER NOT NULL DEFAULT 0")
|
||||
@@ -154,6 +159,20 @@ func exec(label, query string, args ...any) {
|
||||
}
|
||||
}
|
||||
|
||||
// backfillContentChars populates content_chars for rows carrying a body but a
|
||||
// zero count — i.e. stories ingested before the column existed. LENGTH() counts
|
||||
// characters (code points) for TEXT, matching the utf8.RuneCountInString done at
|
||||
// insert. After the first run this matches no rows (bodied stories are set,
|
||||
// bodyless ones stay 0 and are filtered by content IS NOT NULL), so it's a cheap
|
||||
// startup no-op thereafter.
|
||||
func backfillContentChars(d *sql.DB) {
|
||||
if _, err := d.Exec(
|
||||
`UPDATE stories SET content_chars = LENGTH(content)
|
||||
WHERE content_chars = 0 AND content IS NOT NULL AND content <> ''`); err != nil {
|
||||
slog.Error("backfill content_chars failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func addColumnIfMissing(d *sql.DB, table, column, columnType string) {
|
||||
q := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, columnType)
|
||||
if _, err := d.Exec(q); err != nil {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func nowUnix() int64 {
|
||||
@@ -38,9 +39,9 @@ func InsertStory(s *Story) error {
|
||||
publishedAt = s.PublishedAt
|
||||
}
|
||||
_, err := Get().Exec(
|
||||
`INSERT INTO stories (guid, headline, lede, content, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt,
|
||||
`INSERT INTO stories (guid, headline, lede, content, content_chars, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), utf8.RuneCountInString(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -414,8 +415,10 @@ func TrendingStories(limit int, sinceDay int64) ([]Story, error) {
|
||||
|
||||
// StoryContentLengths returns the captured article text length (in characters)
|
||||
// for the given story ids, as a map keyed by id. Used to estimate a "N min read"
|
||||
// chip without transferring the full body: only LENGTH(content) crosses the
|
||||
// wire. Ids with no captured content are absent (treated as zero by callers).
|
||||
// chip. It reads the precomputed content_chars column rather than LENGTH()-ing
|
||||
// the full body, so the hot listing path never scans article text. Ids with no
|
||||
// captured content have content_chars = 0 and are absent from the map (treated
|
||||
// as zero by callers).
|
||||
func StoryContentLengths(ids []int64) map[int64]int {
|
||||
out := make(map[int64]int, len(ids))
|
||||
if len(ids) == 0 {
|
||||
@@ -423,8 +426,8 @@ func StoryContentLengths(ids []int64) map[int64]int {
|
||||
}
|
||||
ph, args := intInClause(ids)
|
||||
rows, err := Get().Query(
|
||||
`SELECT id, LENGTH(content) FROM stories
|
||||
WHERE id IN (`+ph+`) AND content IS NOT NULL AND content <> ''`, args...)
|
||||
`SELECT id, content_chars FROM stories
|
||||
WHERE id IN (`+ph+`) AND content_chars > 0`, args...)
|
||||
if err != nil {
|
||||
slog.Error("story content lengths query failed", "err", err)
|
||||
return out
|
||||
|
||||
@@ -7,6 +7,7 @@ CREATE TABLE IF NOT EXISTS stories (
|
||||
headline TEXT NOT NULL,
|
||||
lede TEXT,
|
||||
content TEXT,
|
||||
content_chars INTEGER NOT NULL DEFAULT 0,
|
||||
image_url TEXT,
|
||||
article_url TEXT NOT NULL,
|
||||
url_canonical TEXT,
|
||||
|
||||
Reference in New Issue
Block a user