Pete/internal/storage/db.go

package storage

import (
	"context"
	"database/sql"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	_ "modernc.org/sqlite"
)

var (
	mu       sync.RWMutex
	globalDB *sql.DB
)

// Init opens (or creates) the SQLite database and runs migrations.
func Init(dbPath string) error {
	mu.Lock()
	defer mu.Unlock()

	if globalDB != nil {
		return nil
	}

	dir := filepath.Dir(dbPath)
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return fmt.Errorf("create data dir: %w", err)
	}

	d, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(ON)")
	if err != nil {
		return fmt.Errorf("open database: %w", err)
	}

	d.SetMaxOpenConns(1)

	if err := runMigrations(d); err != nil {
		return fmt.Errorf("run migrations: %w", err)
	}

	globalDB = d
	slog.Info("database initialized", "path", dbPath)
	return nil
}

// Get returns the global database handle. Panics if Init was not called.
func Get() *sql.DB {
	mu.RLock()
	db := globalDB
	mu.RUnlock()
	if db == nil {
		panic("storage.Get() called before storage.Init()")
	}
	return db
}

// Close closes the global database handle.
func Close() error {
	mu.Lock()
	defer mu.Unlock()
	if globalDB != nil {
		err := globalDB.Close()
		globalDB = nil
		return err
	}
	return nil
}

func runMigrations(d *sql.DB) error {
	if _, err := d.Exec(schema); err != nil {
		return fmt.Errorf("create schema: %w", err)
	}

	// Idempotent column adds for DBs created before the dedup columns existed.
	// SQLite errors with "duplicate column name" when the column is already there;
	// we swallow that specifically.
	addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
	addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
	addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
	// content holds the full article text (feed content:encoded when present,
	// else the body scraped during paywall detection) for reader mode. Stories
	// ingested before this column existed simply have NULL and fall back to lede.
	addColumnIfMissing(d, "stories", "content", "TEXT")
	// content_chars caches the character count of content so the "N min read"
	// chip never has to LENGTH() the full body on the hot listing path. Filled at
	// insert time; the backfill below populates rows that predate the column.
	addColumnIfMissing(d, "stories", "content_chars", "INTEGER NOT NULL DEFAULT 0")
	backfillContentChars(d)
	addColumnIfMissing(d, "stories", "published_at", "INTEGER")
	addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
	addColumnIfMissing(d, "post_log", "forced", "INTEGER NOT NULL DEFAULT 0")
	addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")
	// Feed alerts. last_new_at is when a source last produced a story Pete kept;
	// the two *_alerted_at stamps remember that the owner has already been told a
	// feed is down or has gone quiet, so an alert fires once per episode and
	// survives a restart. See backfillLastNewAt for what existing rows start with.
	addColumnIfMissing(d, "source_health", "last_new_at", "INTEGER")
	addColumnIfMissing(d, "source_health", "down_alerted_at", "INTEGER")
	addColumnIfMissing(d, "source_health", "quiet_alerted_at", "INTEGER")
	backfillLastNewAt(d)
	// Occupancy of a shared table. Rows written before the casino went multiplayer
	// are solo games and read as NULL, which is exactly what they are.
	addColumnIfMissing(d, "game_live_hands", "table_id", "TEXT")
	// The public detail sheet (stats + equipped gear) for an adventurer's
	// click-through page. Rides the roster snapshot; NULL on rows pushed by a
	// gogobee build that predates the detail page.
	addColumnIfMissing(d, "adventure_roster", "detail_json", "TEXT")
	// The noun a fact is about (a mischief bounty, a found treasure's name). Facts
	// recorded before the treasure_found event existed carry NULL, which is right:
	// they had no such noun to keep.
	addColumnIfMissing(d, "adventure_events", "stakes", "TEXT")
	// The run behind a dispatch that *ended* one. NULL on every fact filed before
	// the run report existed and on every fact that isn't the end of an
	// expedition; both simply render without the "read the run" link.
	addColumnIfMissing(d, "adventure_events", "run_id", "TEXT")
	// The liveblog's late-arriving prose and the column that carries it. Both are
	// in their tables' CREATE TABLE — those tables have never shipped — so these
	// two adds exist only for a database that already ran an earlier build of the
	// run-liveblog branch.
	addColumnIfMissing(d, "adventure_run", "summary", "TEXT NOT NULL DEFAULT ''")
	addColumnIfMissing(d, "adventure_run_beat", "prose", "TEXT NOT NULL DEFAULT ''")
	// Ask 7: upgrade orders carry a target tier for the 5 standard equipment slots.
	addColumnIfMissing(d, "equip_orders", "tier", "INTEGER NOT NULL DEFAULT 0")
	// W5b: the three verbs that take arguments (which zone, which loadout, how
	// many days of sitting) carry them as one small JSON object. W5a's two verbs
	// take none, so an existing row gets '' and reads back as no params — which is
	// exactly what extract and siege_join mean.
	addColumnIfMissing(d, "adventure_orders", "params", "TEXT NOT NULL DEFAULT ''")
	// Adventure alerts. A subscription made before they existed knows only the OIDC
	// subject, and the adventure ownership join needs the Matrix localpart — so an
	// existing row gets "" here and is skipped for owner-scoped alerts until the
	// browser re-subscribes, which it does on every page load that has push on.
	// Realm-wide alerts (the Siege) need no localpart and work immediately.
	addColumnIfMissing(d, "push_subscriptions", "user_localpart", "TEXT NOT NULL DEFAULT ''")
	// The adventure watermark is deliberately separate from last_notified_at: the
	// digest and the alerts run on different clocks (6 hours vs 2 minutes), and
	// sharing one column would let whichever ran last decide what the other had
	// already seen. 0 on a pre-existing row is corrected to "now" on the first
	// pass rather than replaying every dispatch Pete has ever stored.
	addColumnIfMissing(d, "push_subscriptions", "last_adv_notified_at", "INTEGER NOT NULL DEFAULT 0")

	// FTS5 virtual tables don't support IF NOT EXISTS reliably.
	// Check sqlite_master before creating.
	var ftsExists int
	if err := d.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='stories_fts'`).Scan(&ftsExists); err != nil {
		return fmt.Errorf("probe FTS5 table: %w", err)
	}
	if ftsExists == 0 {
		if _, err := d.Exec(ftsSchema); err != nil {
			return fmt.Errorf("create FTS5 table: %w", err)
		}
		if _, err := d.Exec(ftsTriggers); err != nil {
			return fmt.Errorf("create FTS5 triggers: %w", err)
		}
		slog.Info("created FTS5 search index")
	}

	return nil
}

// StoryRetention is how long a story is kept after Pete first saw it. It is also
// how long Pete remembers having seen it: the stories row is the only record of
// its GUID, canonical URL and headline, so the poller ignores feed items older
// than this rather than meet them again as strangers once their row is gone.
const StoryRetention = 30 * 24 * time.Hour

// StartMaintenance runs RunMaintenance now and then once a day until ctx ends.
// It used to run only at startup, so on a service that went weeks between
// deploys nothing was pruned, and then everything was, all at once.
func StartMaintenance(ctx context.Context) {
	RunMaintenance()
	go func() {
		t := time.NewTicker(24 * time.Hour)
		defer t.Stop()
		for {
			select {
			case <-ctx.Done():
				return
			case <-t.C:
				RunMaintenance()
			}
		}
	}()
}

// RunMaintenance prunes stale data.
func RunMaintenance() {
	// Prune old stories and their post logs / reactions. A story someone has
	// bookmarked is kept however old it is: the orphan sweep below would
	// otherwise take the bookmark with it, and a saved story that quietly
	// vanishes after a month is exactly what saving it was meant to prevent.
	storyCutoff := nowUnix() - int64(StoryRetention/time.Second)
	exec("prune old stories",
		`DELETE FROM stories WHERE seen_at < ? AND classified = 1
		   AND id NOT IN (SELECT story_id FROM user_story_state WHERE bookmarked_at IS NOT NULL)`, storyCutoff)
	exec("prune old post_log",
		`DELETE FROM post_log WHERE posted_at < ?`, storyCutoff)
	exec("prune old reactions",
		`DELETE FROM reactions WHERE reacted_at < ?`, storyCutoff)

	// Drop per-user read/bookmark rows whose story has been pruned above, so the
	// table can't accumulate dangling references as stories age out.
	exec("prune orphan user_story_state",
		`DELETE FROM user_story_state WHERE story_id NOT IN (SELECT id FROM stories)`)

	// Same for per-story view counts once their story has aged out.
	exec("prune orphan story_views",
		`DELETE FROM story_views WHERE story_id NOT IN (SELECT id FROM stories)`)

	// Daily unique tokens are only useful for the recent window; their salts are
	// long gone. page_views is kept forever (tiny aggregate, all-time totals).
	exec("prune old daily_visitors",
		`DELETE FROM daily_visitors WHERE day < ?`, unixDay()-30)

	// Finished expedition logs. Kept for longer than the page shows them (the
	// adventurer page hides a run six hours after it ends) because the dispatch
	// that announced the run outlives the run, and a dead link from a story to
	// its own log is worse than a log nobody reads. A run still walking is never
	// pruned however old it looks — see PruneRuns for why.
	if err := PruneRuns(nowUnix() - int64(14*86400)); err != nil {
		slog.Error("db exec failed", "op", "prune finished runs", "err", err)
	}

	exec("wal checkpoint", "PRAGMA wal_checkpoint(TRUNCATE)")
	exec("optimize", "PRAGMA optimize")
}

// exec is a fire-and-forget helper that logs errors. Several callers run it from
// background goroutines (metrics, view counts), which can outlive a Close() — so
// unlike Get() it must not panic on a nil handle: it simply skips the write.
func exec(label, query string, args ...any) {
	mu.RLock()
	db := globalDB
	mu.RUnlock()
	if db == nil {
		slog.Warn("db exec skipped: no database", "op", label)
		return
	}
	if _, err := db.Exec(query, args...); err != nil {
		slog.Error("db exec failed", "op", label, "err", err)
	}
}

// backfillContentChars populates content_chars for rows carrying a body but a
// zero count — i.e. stories ingested before the column existed. LENGTH() counts
// characters (code points) for TEXT, matching the utf8.RuneCountInString done at
// insert. After the first run this matches no rows (bodied stories are set,
// bodyless ones stay 0 and are filtered by content IS NOT NULL), so it's a cheap
// startup no-op thereafter.
func backfillContentChars(d *sql.DB) {
	if _, err := d.Exec(
		`UPDATE stories SET content_chars = LENGTH(content)
		 WHERE content_chars = 0 AND content IS NOT NULL AND content <> ''`); err != nil {
		slog.Error("backfill content_chars failed", "err", err)
	}
}

func addColumnIfMissing(d *sql.DB, table, column, columnType string) {
	q := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, columnType)
	if _, err := d.Exec(q); err != nil {
		// SQLite returns "duplicate column name" when the column already exists.
		if !strings.Contains(err.Error(), "duplicate column name") {
			slog.Error("alter table failed", "table", table, "column", column, "err", err)
		}
	}
}