package storage import ( "database/sql" "encoding/json" "errors" "log/slog" "strings" "time" "unicode/utf8" ) func nowUnix() int64 { return time.Now().Unix() } // IsGUIDSeen checks if a GUID has already been ingested. func IsGUIDSeen(guid string) bool { var count int if err := Get().QueryRow(`SELECT COUNT(*) FROM stories WHERE guid = ?`, guid).Scan(&count); err != nil { slog.Error("IsGUIDSeen query failed", "guid", guid, "err", err) return true // fail closed: assume seen to prevent duplicates } return count > 0 } // InsertStory inserts a new story record. func InsertStory(s *Story) error { classified := 0 if s.Classified { classified = 1 } paywalled := 0 if s.Paywalled { paywalled = 1 } var publishedAt any if s.PublishedAt > 0 { publishedAt = s.PublishedAt } _, err := Get().Exec( `INSERT INTO stories (guid, headline, lede, content, content_chars, image_url, article_url, url_canonical, headline_norm, source, platforms, channel, classified, paywalled, seen_at, published_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, s.GUID, s.Headline, s.Lede, nullIfEmpty(s.Content), utf8.RuneCountInString(s.Content), s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.Platforms, s.Channel, classified, paywalled, s.SeenAt, publishedAt, ) return err } // GetStoryReaderText returns the stored full article text and lede for a single // story, for reader mode. found is false when no story has that id. The query // mirrors the visible-story filter (classified, non-sentinel channel) so the // public /api/article endpoint can't be enumerated to pull the captured bodies // of discarded or not-yet-classified stories that never surface in the UI. func GetStoryReaderText(id int64) (content, lede string, found bool, err error) { var c sql.NullString var l sql.NullString row := Get().QueryRow( `SELECT content, lede FROM stories WHERE id = ? AND classified = 1 AND channel NOT IN ('_discarded', '_duplicate')`, id) switch err = row.Scan(&c, &l); { case errors.Is(err, sql.ErrNoRows): return "", "", false, nil case err != nil: return "", "", false, err } return c.String, l.String, true, nil } // nullIfEmpty returns a SQL NULL for empty strings so partial unique indexes // (WHERE col IS NOT NULL) can hold multiple "unknown" rows without conflict. func nullIfEmpty(s string) any { if s == "" { return nil } return s } // IsCanonicalSeen reports whether any story with this canonical URL exists. // Empty input always returns false (no canonical = no dedup possible). func IsCanonicalSeen(canonical string) bool { if canonical == "" { return false } var n int if err := Get().QueryRow(`SELECT COUNT(*) FROM stories WHERE url_canonical = ?`, canonical).Scan(&n); err != nil { slog.Error("IsCanonicalSeen query failed", "err", err) return true // fail closed } return n > 0 } // IsHeadlineSeen reports whether a same-source story with this normalized // headline already exists. Empty inputs return false. func IsHeadlineSeen(source, headlineNorm string) bool { if source == "" || headlineNorm == "" { return false } var n int if err := Get().QueryRow( `SELECT COUNT(*) FROM stories WHERE source = ? AND headline_norm = ?`, source, headlineNorm).Scan(&n); err != nil { slog.Error("IsHeadlineSeen query failed", "err", err) return true } return n > 0 } // WasCanonicalPostedRecently reports whether the canonical URL was posted to // the given channel within `cooldownSeconds`. Empty canonical returns false. func WasCanonicalPostedRecently(canonical, channel string, cooldownSeconds int64) bool { if canonical == "" || cooldownSeconds <= 0 { return false } cutoff := nowUnix() - cooldownSeconds var n int if err := Get().QueryRow( `SELECT COUNT(*) FROM post_log WHERE url_canonical = ? AND channel = ? AND posted_at >= ?`, canonical, channel, cutoff).Scan(&n); err != nil { slog.Error("WasCanonicalPostedRecently query failed", "err", err) return true // fail closed } return n > 0 } // MarkClassified marks a story as successfully classified and sets its channel. func MarkClassified(guid, channel, platforms string) { exec("mark classified", `UPDATE stories SET classified = 1, channel = ?, platforms = ? WHERE guid = ?`, channel, platforms, guid) } // GetStoryByGUID returns the full story record for a GUID, or nil if not found. func GetStoryByGUID(guid string) (*Story, error) { row := Get().QueryRow( `SELECT guid, headline, lede, image_url, article_url, source, platforms, channel, seen_at FROM stories WHERE guid = ?`, guid) var s Story if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil { return nil, err } return &s, nil } // InsertPostLog records that a story was posted to a channel. // Uses OR IGNORE to prevent duplicate posts for the same story+channel. // forced=true marks the row as a manual !post override so it doesn't count // against the global daily cap. func InsertPostLog(guid, channel, eventID, urlCanonical string, forced bool) { f := 0 if forced { f = 1 } exec("insert post_log", `INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at, forced) VALUES (?, ?, ?, ?, ?, ?)`, guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix(), f) } // GetLastPostTime returns the unix timestamp of the most recent post to a channel. func GetLastPostTime(channel string) int64 { var t int64 if err := Get().QueryRow(`SELECT COALESCE(MAX(posted_at), 0) FROM post_log WHERE channel = ?`, channel).Scan(&t); err != nil { slog.Error("GetLastPostTime query failed", "channel", channel, "err", err) return nowUnix() // fail closed: pretend we just posted to prevent flooding } return t } // CountAllPostsInWindow counts posts across ALL channels within a time window. // Used for the global daily cap. Excludes forced (!post) entries so manual // overrides don't eat into the auto-rotation budget. func CountAllPostsInWindow(windowStart int64) int { var count int if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE posted_at >= ? AND forced = 0`, windowStart).Scan(&count); err != nil { slog.Error("CountAllPostsInWindow query failed", "err", err) return 1<<31 - 1 // fail closed: huge number prevents posting } return count } // CountPostsInWindow counts posts to a channel within a time window. func CountPostsInWindow(channel string, windowStart int64) int { var count int if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE channel = ? AND posted_at >= ?`, channel, windowStart).Scan(&count); err != nil { slog.Error("CountPostsInWindow query failed", "channel", channel, "err", err) return 999 // fail closed: pretend burst cap reached to prevent flooding } return count } // LookupPostGUID finds the story GUID for a given Matrix event ID. func LookupPostGUID(eventID string) (guid, channel string, found bool) { err := Get().QueryRow( `SELECT guid, channel FROM post_log WHERE event_id = ? LIMIT 1`, eventID).Scan(&guid, &channel) if err != nil { return "", "", false } return guid, channel, true } // InsertReaction records a reaction on a posted story. // Uses OR IGNORE to deduplicate if Matrix delivers the same reaction twice. func InsertReaction(postGUID, channel, eventID, emoji, userID string, reactedAt int64) { exec("insert reaction", `INSERT OR IGNORE INTO reactions (post_guid, channel, event_id, emoji, user_id, reacted_at) VALUES (?, ?, ?, ?, ?, ?)`, postGUID, channel, eventID, emoji, userID, reactedAt) } // GetNewestPostableStory returns the newest classified story from a source // that has a real (non-sentinel) channel and has not yet been posted (no // row in post_log). Returns (nil, nil) when nothing qualifies. func GetNewestPostableStory(source string) (*Story, error) { row := Get().QueryRow( `SELECT guid, headline, lede, image_url, article_url, source, platforms, channel, seen_at FROM stories WHERE classified = 1 AND source = ? AND channel IS NOT NULL AND channel NOT IN ('_discarded', '_duplicate') AND guid NOT IN (SELECT guid FROM post_log) ORDER BY COALESCE(published_at, seen_at) DESC LIMIT 1`, source) var s Story if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } return nil, err } return &s, nil } // GetNewestPostableStoryByChannel returns the newest classified story routed // to the given channel that has not yet been posted. Used by !post to satisfy // on-demand requests in round-robin mode (where the in-memory queue is empty // between ticks). Returns (nil, nil) when nothing qualifies. func GetNewestPostableStoryByChannel(channel string) (*Story, error) { row := Get().QueryRow( `SELECT guid, headline, lede, image_url, article_url, source, platforms, channel, seen_at FROM stories WHERE classified = 1 AND channel = ? AND guid NOT IN (SELECT guid FROM post_log) ORDER BY COALESCE(published_at, seen_at) DESC LIMIT 1`, channel) var s Story if err := row.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.SeenAt); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } return nil, err } return &s, nil } // ListClassifiedByChannel returns up to `limit` classified stories routed to a // real channel, newest first, with optional offset for pagination. Sentinel // channels (_discarded, _duplicate) are excluded. func ListClassifiedByChannel(channel string, limit, offset int) ([]Story, error) { rows, err := Get().Query( `SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at, EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted FROM stories s WHERE s.classified = 1 AND s.channel = ? ORDER BY COALESCE(s.published_at, s.seen_at) DESC LIMIT ? OFFSET ?`, channel, limit, offset) if err != nil { return nil, err } defer rows.Close() var out []Story for rows.Next() { var s Story if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil { return nil, err } out = append(out, s) } return out, rows.Err() } // ListAllClassified returns up to `limit` classified stories across all real // channels, newest first. Sentinel channels are excluded. func ListAllClassified(limit, offset int) ([]Story, error) { rows, err := Get().Query( `SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at, EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted FROM stories s WHERE s.classified = 1 AND s.channel IS NOT NULL AND s.channel NOT IN ('_discarded', '_duplicate') ORDER BY COALESCE(s.published_at, s.seen_at) DESC LIMIT ? OFFSET ?`, limit, offset) if err != nil { return nil, err } defer rows.Close() var out []Story for rows.Next() { var s Story if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil { return nil, err } out = append(out, s) } return out, rows.Err() } // ListForFeed returns up to `limit` classified stories for the outbound RSS and // JSON feeds, newest first. Unlike the web list queries it also selects the full // `content` and `published_at`, which the feeds need for content:encoded bodies // and real pubDates. channel == "" spans all real channels; otherwise it scopes // to that one channel. Sentinel channels are always excluded. func ListForFeed(channel string, limit int) ([]Story, error) { const cols = `s.id, s.guid, s.headline, s.lede, s.content, s.image_url, s.article_url, s.url_canonical, s.source, s.channel, s.seen_at, s.published_at` var ( rows *sql.Rows err error ) if channel == "" { rows, err = Get().Query( `SELECT `+cols+` FROM stories s WHERE s.classified = 1 AND s.channel IS NOT NULL AND s.channel NOT IN ('_discarded', '_duplicate') ORDER BY COALESCE(s.published_at, s.seen_at) DESC LIMIT ?`, limit) } else { rows, err = Get().Query( `SELECT `+cols+` FROM stories s WHERE s.classified = 1 AND s.channel = ? ORDER BY COALESCE(s.published_at, s.seen_at) DESC LIMIT ?`, channel, limit) } if err != nil { return nil, err } defer rows.Close() var out []Story for rows.Next() { var s Story var content, canonical sql.NullString var published sql.NullInt64 if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &content, &s.ImageURL, &s.ArticleURL, &canonical, &s.Source, &s.Channel, &s.SeenAt, &published); err != nil { return nil, err } s.Content = content.String s.URLCanonical = canonical.String s.PublishedAt = published.Int64 out = append(out, s) } return out, rows.Err() } // ListRecentlyPosted returns up to `limit` stories that have been posted to // Matrix, ordered by post time (newest first). Posted is always true. func ListRecentlyPosted(limit int) ([]Story, error) { rows, err := Get().Query( `SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at FROM stories s JOIN post_log p ON p.guid = s.guid GROUP BY s.guid ORDER BY MAX(p.posted_at) DESC LIMIT ?`, limit) if err != nil { return nil, err } defer rows.Close() var out []Story for rows.Next() { var s Story if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt); err != nil { return nil, err } s.Posted = true out = append(out, s) } return out, rows.Err() } // TrendingStories returns the most-read classified stories (real channels only) // counting views recorded on or after sinceDay (a unix day number), newest-ish // as a tiebreak. Used for the home page's "popular this week" rail. Stories with // no views in the window don't appear, so the rail is empty until reads exist. func TrendingStories(limit int, sinceDay int64) ([]Story, error) { rows, err := Get().Query( `SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at, EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted FROM stories s JOIN story_views v ON v.story_id = s.id WHERE s.classified = 1 AND s.channel IS NOT NULL AND s.channel NOT IN ('_discarded', '_duplicate') AND v.day >= ? GROUP BY s.id ORDER BY SUM(v.views) DESC, COALESCE(s.published_at, s.seen_at) DESC LIMIT ?`, sinceDay, limit) if err != nil { return nil, err } defer rows.Close() var out []Story for rows.Next() { var s Story if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil { return nil, err } out = append(out, s) } return out, rows.Err() } // StoryContentLengths returns the captured article text length (in characters) // for the given story ids, as a map keyed by id. Used to estimate a "N min read" // chip. It reads the precomputed content_chars column rather than LENGTH()-ing // the full body, so the hot listing path never scans article text. Ids with no // captured content have content_chars = 0 and are absent from the map (treated // as zero by callers). func StoryContentLengths(ids []int64) map[int64]int { out := make(map[int64]int, len(ids)) if len(ids) == 0 { return out } ph, args := intInClause(ids) rows, err := Get().Query( `SELECT id, content_chars FROM stories WHERE id IN (`+ph+`) AND content_chars > 0`, args...) if err != nil { slog.Error("story content lengths query failed", "err", err) return out } defer rows.Close() for rows.Next() { var id, n int64 if err := rows.Scan(&id, &n); err != nil { slog.Error("scan content length failed", "err", err) continue } out[id] = int(n) } if err := rows.Err(); err != nil { slog.Error("story content lengths iteration failed", "err", err) } return out } // CountClassifiedByChannel returns how many classified stories exist for a channel. func CountClassifiedByChannel(channel string) (int, error) { var n int err := Get().QueryRow( `SELECT COUNT(*) FROM stories WHERE classified = 1 AND channel = ?`, channel).Scan(&n) return n, err } // IsKnownImageURL reports whether any story has this exact image_url. Used // by the thumbnail proxy to guard against arbitrary URL fetches. func IsKnownImageURL(url string) bool { if url == "" { return false } var n int if err := Get().QueryRow(`SELECT COUNT(*) FROM stories WHERE image_url = ? LIMIT 1`, url).Scan(&n); err != nil { slog.Error("IsKnownImageURL query failed", "err", err) return false } return n > 0 } // GetRoundRobinState returns the last channel posted by the round-robin // scheduler and the timestamp of that tick. Empty string + 0 if no state yet. func GetRoundRobinState() (lastChannel string, lastTickAt int64, err error) { row := Get().QueryRow(`SELECT last_channel, last_tick_at FROM round_robin_state WHERE id = 1`) var lc *string if scanErr := row.Scan(&lc, &lastTickAt); scanErr != nil { if errors.Is(scanErr, sql.ErrNoRows) { return "", 0, nil } return "", 0, scanErr } if lc != nil { lastChannel = *lc } return lastChannel, lastTickAt, nil } // SetRoundRobinState persists the last-posted channel and tick timestamp. func SetRoundRobinState(lastChannel string, tickAt int64) { exec("set round_robin_state", `INSERT INTO round_robin_state (id, last_channel, last_tick_at) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET last_channel = excluded.last_channel, last_tick_at = excluded.last_tick_at`, lastChannel, tickAt) } // SearchStories runs an FTS5 query against headline + lede and returns the // best-ranked classified stories (real channels only), newest-ish first via // bm25. The user query is tokenized and each token becomes a prefix match; // FTS5 special characters are stripped so user input cannot break syntax. func SearchStories(query string, limit int) ([]Story, error) { fts := buildFTSQuery(query) if fts == "" { return nil, nil } rows, err := Get().Query( `SELECT s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at, EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted FROM stories_fts f JOIN stories s ON s.id = f.rowid WHERE f.stories_fts MATCH ? AND s.classified = 1 AND s.channel IS NOT NULL AND s.channel NOT IN ('_discarded', '_duplicate') ORDER BY bm25(stories_fts) ASC, s.seen_at DESC LIMIT ?`, fts, limit) if err != nil { return nil, err } defer rows.Close() var out []Story for rows.Next() { var s Story if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil { return nil, err } out = append(out, s) } return out, rows.Err() } func buildFTSQuery(raw string) string { var tokens []string var cur strings.Builder flush := func() { if cur.Len() > 0 { tokens = append(tokens, cur.String()) cur.Reset() } } for _, r := range raw { switch { case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': cur.WriteRune(r) case r > 127: cur.WriteRune(r) default: flush() } } flush() for i, t := range tokens { tokens[i] = "\"" + t + "\"*" } return strings.Join(tokens, " ") } // MarshalPlatforms converts a string slice to a JSON array string for storage. func MarshalPlatforms(platforms []string) string { if len(platforms) == 0 { return "[]" } data, err := json.Marshal(platforms) if err != nil { slog.Error("marshal platforms failed", "err", err) return "[]" } return string(data) } // UnmarshalPlatforms converts a JSON array string back to a string slice. func UnmarshalPlatforms(raw string) []string { if raw == "" || raw == "[]" { return nil } var platforms []string if err := json.Unmarshal([]byte(raw), &platforms); err != nil { slog.Error("unmarshal platforms failed", "err", err, "raw", raw) return nil } return platforms }