Initial commit

2026-05-22 17:25:27 -07:00
commit 652d6dfa38
40 changed files with 5855 additions and 0 deletions
--- a/internal/storage/db.go
+++ b/internal/storage/db.go
@@ -0,0 +1,140 @@
+package storage
+
+import (
+	"database/sql"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	_ "modernc.org/sqlite"
+)
+
+var (
+	mu       sync.RWMutex
+	globalDB *sql.DB
+)
+
+// Init opens (or creates) the SQLite database and runs migrations.
+func Init(dbPath string) error {
+	mu.Lock()
+	defer mu.Unlock()
+
+	if globalDB != nil {
+		return nil
+	}
+
+	dir := filepath.Dir(dbPath)
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("create data dir: %w", err)
+	}
+
+	d, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(ON)")
+	if err != nil {
+		return fmt.Errorf("open database: %w", err)
+	}
+
+	d.SetMaxOpenConns(1)
+
+	if err := runMigrations(d); err != nil {
+		return fmt.Errorf("run migrations: %w", err)
+	}
+
+	globalDB = d
+	slog.Info("database initialized", "path", dbPath)
+	return nil
+}
+
+// Get returns the global database handle. Panics if Init was not called.
+func Get() *sql.DB {
+	mu.RLock()
+	db := globalDB
+	mu.RUnlock()
+	if db == nil {
+		panic("storage.Get() called before storage.Init()")
+	}
+	return db
+}
+
+// Close closes the global database handle.
+func Close() error {
+	mu.Lock()
+	defer mu.Unlock()
+	if globalDB != nil {
+		err := globalDB.Close()
+		globalDB = nil
+		return err
+	}
+	return nil
+}
+
+func runMigrations(d *sql.DB) error {
+	if _, err := d.Exec(schema); err != nil {
+		return fmt.Errorf("create schema: %w", err)
+	}
+
+	// Idempotent column adds for DBs created before the dedup columns existed.
+	// SQLite errors with "duplicate column name" when the column is already there;
+	// we swallow that specifically.
+	addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
+	addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
+	addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
+
+	// FTS5 virtual tables don't support IF NOT EXISTS reliably.
+	// Check sqlite_master before creating.
+	var ftsExists int
+	d.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='stories_fts'`).Scan(&ftsExists)
+	if ftsExists == 0 {
+		if _, err := d.Exec(ftsSchema); err != nil {
+			return fmt.Errorf("create FTS5 table: %w", err)
+		}
+		if _, err := d.Exec(ftsTriggers); err != nil {
+			return fmt.Errorf("create FTS5 triggers: %w", err)
+		}
+		slog.Info("created FTS5 search index")
+	}
+
+	return nil
+}
+
+// RunMaintenance prunes stale data. Called periodically.
+func RunMaintenance(recentWindowHours, classificationLogDays int) {
+	recentCutoff := nowUnix() - int64(recentWindowHours*3600)
+	exec("prune recent_headlines",
+		`DELETE FROM recent_headlines WHERE seen_at < ?`, recentCutoff)
+
+	logCutoff := nowUnix() - int64(classificationLogDays*86400)
+	exec("prune classification_log",
+		`DELETE FROM classification_log WHERE logged_at < ?`, logCutoff)
+
+	// Prune old stories (30 days) and their post logs / reactions
+	storyCutoff := nowUnix() - int64(30*86400)
+	exec("prune old stories",
+		`DELETE FROM stories WHERE seen_at < ? AND classified = 1`, storyCutoff)
+	exec("prune old post_log",
+		`DELETE FROM post_log WHERE posted_at < ?`, storyCutoff)
+	exec("prune old reactions",
+		`DELETE FROM reactions WHERE reacted_at < ?`, storyCutoff)
+
+	exec("wal checkpoint", "PRAGMA wal_checkpoint(TRUNCATE)")
+	exec("optimize", "PRAGMA optimize")
+}
+
+// exec is a fire-and-forget helper that logs errors.
+func exec(label, query string, args ...any) {
+	if _, err := Get().Exec(query, args...); err != nil {
+		slog.Error("db exec failed", "op", label, "err", err)
+	}
+}
+
+func addColumnIfMissing(d *sql.DB, table, column, columnType string) {
+	q := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, columnType)
+	if _, err := d.Exec(q); err != nil {
+		// SQLite returns "duplicate column name" when the column already exists.
+		if !strings.Contains(err.Error(), "duplicate column name") {
+			slog.Error("alter table failed", "table", table, "column", column, "err", err)
+		}
+	}
+}
--- a/internal/storage/queries.go
+++ b/internal/storage/queries.go
@@ -0,0 +1,225 @@
+package storage
+
+import (
+	"encoding/json"
+	"log/slog"
+	"time"
+)
+
+func nowUnix() int64 {
+	return time.Now().Unix()
+}
+
+// IsGUIDSeen checks if a GUID has already been ingested.
+func IsGUIDSeen(guid string) bool {
+	var count int
+	if err := Get().QueryRow(`SELECT COUNT(*) FROM stories WHERE guid = ?`, guid).Scan(&count); err != nil {
+		slog.Error("IsGUIDSeen query failed", "guid", guid, "err", err)
+		return true // fail closed: assume seen to prevent duplicates
+	}
+	return count > 0
+}
+
+// InsertStory inserts a new story record.
+func InsertStory(s *Story) error {
+	classified := 0
+	if s.Classified {
+		classified = 1
+	}
+	_, err := Get().Exec(
+		`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, feed_hint, platforms, channel, classified, seen_at)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.FeedHint, s.Platforms, s.Channel, classified, s.SeenAt,
+	)
+	return err
+}
+
+// nullIfEmpty returns a SQL NULL for empty strings so partial unique indexes
+// (WHERE col IS NOT NULL) can hold multiple "unknown" rows without conflict.
+func nullIfEmpty(s string) any {
+	if s == "" {
+		return nil
+	}
+	return s
+}
+
+// IsCanonicalSeen reports whether any story with this canonical URL exists.
+// Empty input always returns false (no canonical = no dedup possible).
+func IsCanonicalSeen(canonical string) bool {
+	if canonical == "" {
+		return false
+	}
+	var n int
+	if err := Get().QueryRow(`SELECT COUNT(*) FROM stories WHERE url_canonical = ?`, canonical).Scan(&n); err != nil {
+		slog.Error("IsCanonicalSeen query failed", "err", err)
+		return true // fail closed
+	}
+	return n > 0
+}
+
+// IsHeadlineSeen reports whether a same-source story with this normalized
+// headline already exists. Empty inputs return false.
+func IsHeadlineSeen(source, headlineNorm string) bool {
+	if source == "" || headlineNorm == "" {
+		return false
+	}
+	var n int
+	if err := Get().QueryRow(
+		`SELECT COUNT(*) FROM stories WHERE source = ? AND headline_norm = ?`,
+		source, headlineNorm).Scan(&n); err != nil {
+		slog.Error("IsHeadlineSeen query failed", "err", err)
+		return true
+	}
+	return n > 0
+}
+
+// WasCanonicalPostedRecently reports whether the canonical URL was posted to
+// the given channel within `cooldownSeconds`. Empty canonical returns false.
+func WasCanonicalPostedRecently(canonical, channel string, cooldownSeconds int64) bool {
+	if canonical == "" || cooldownSeconds <= 0 {
+		return false
+	}
+	cutoff := nowUnix() - cooldownSeconds
+	var n int
+	if err := Get().QueryRow(
+		`SELECT COUNT(*) FROM post_log WHERE url_canonical = ? AND channel = ? AND posted_at >= ?`,
+		canonical, channel, cutoff).Scan(&n); err != nil {
+		slog.Error("WasCanonicalPostedRecently query failed", "err", err)
+		return true // fail closed
+	}
+	return n > 0
+}
+
+// MarkClassified marks a story as successfully classified and sets its channel.
+func MarkClassified(guid, channel, platforms string) {
+	exec("mark classified",
+		`UPDATE stories SET classified = 1, channel = ?, platforms = ? WHERE guid = ?`,
+		channel, platforms, guid)
+}
+
+// GetUnclassifiedStories returns stories from a specific source that were ingested but not yet classified.
+func GetUnclassifiedStories(source string) ([]Story, error) {
+	rows, err := Get().Query(
+		`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, seen_at
+		 FROM stories WHERE classified = 0 AND source = ? ORDER BY seen_at ASC LIMIT 20`, source)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var stories []Story
+	for rows.Next() {
+		var s Story
+		if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.SeenAt); err != nil {
+			return nil, err
+		}
+		stories = append(stories, s)
+	}
+	return stories, rows.Err()
+}
+
+// InsertRecentHeadline adds a headline to the dedup context window.
+func InsertRecentHeadline(guid, headline, source string) {
+	exec("insert recent_headline",
+		`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
+		guid, headline, source, nowUnix())
+}
+
+// GetRecentHeadlines returns the most recent headlines for LLM dedup context, capped at limit.
+func GetRecentHeadlines(limit int) ([]RecentHeadline, error) {
+	rows, err := Get().Query(
+		`SELECT guid, headline, source, seen_at FROM recent_headlines ORDER BY seen_at DESC LIMIT ?`, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var headlines []RecentHeadline
+	for rows.Next() {
+		var h RecentHeadline
+		if err := rows.Scan(&h.GUID, &h.Headline, &h.Source, &h.SeenAt); err != nil {
+			return nil, err
+		}
+		headlines = append(headlines, h)
+	}
+	return headlines, rows.Err()
+}
+
+// InsertClassificationLog stores a classification result for tuning review.
+func InsertClassificationLog(guid, resultJSON string) {
+	exec("insert classification_log",
+		`INSERT OR REPLACE INTO classification_log (guid, result, logged_at) VALUES (?, ?, ?)`,
+		guid, resultJSON, nowUnix())
+}
+
+// InsertPostLog records that a story was posted to a channel.
+// Uses OR IGNORE to prevent duplicate posts for the same story+channel.
+func InsertPostLog(guid, channel, eventID, urlCanonical string) {
+	exec("insert post_log",
+		`INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at) VALUES (?, ?, ?, ?, ?)`,
+		guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix())
+}
+
+// GetLastPostTime returns the unix timestamp of the most recent post to a channel.
+func GetLastPostTime(channel string) int64 {
+	var t int64
+	if err := Get().QueryRow(`SELECT COALESCE(MAX(posted_at), 0) FROM post_log WHERE channel = ?`, channel).Scan(&t); err != nil {
+		slog.Error("GetLastPostTime query failed", "channel", channel, "err", err)
+		return nowUnix() // fail closed: pretend we just posted to prevent flooding
+	}
+	return t
+}
+
+// CountPostsInWindow counts posts to a channel within a time window.
+func CountPostsInWindow(channel string, windowStart int64) int {
+	var count int
+	if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE channel = ? AND posted_at >= ?`, channel, windowStart).Scan(&count); err != nil {
+		slog.Error("CountPostsInWindow query failed", "channel", channel, "err", err)
+		return 999 // fail closed: pretend burst cap reached to prevent flooding
+	}
+	return count
+}
+
+// LookupPostGUID finds the story GUID for a given Matrix event ID.
+func LookupPostGUID(eventID string) (guid, channel string, found bool) {
+	err := Get().QueryRow(
+		`SELECT guid, channel FROM post_log WHERE event_id = ? LIMIT 1`, eventID).Scan(&guid, &channel)
+	if err != nil {
+		return "", "", false
+	}
+	return guid, channel, true
+}
+
+// InsertReaction records a reaction on a posted story.
+// Uses OR IGNORE to deduplicate if Matrix delivers the same reaction twice.
+func InsertReaction(postGUID, channel, eventID, emoji, userID string, reactedAt int64) {
+	exec("insert reaction",
+		`INSERT OR IGNORE INTO reactions (post_guid, channel, event_id, emoji, user_id, reacted_at) VALUES (?, ?, ?, ?, ?, ?)`,
+		postGUID, channel, eventID, emoji, userID, reactedAt)
+}
+
+// MarshalPlatforms converts a string slice to a JSON array string for storage.
+func MarshalPlatforms(platforms []string) string {
+	if len(platforms) == 0 {
+		return "[]"
+	}
+	data, err := json.Marshal(platforms)
+	if err != nil {
+		slog.Error("marshal platforms failed", "err", err)
+		return "[]"
+	}
+	return string(data)
+}
+
+// UnmarshalPlatforms converts a JSON array string back to a string slice.
+func UnmarshalPlatforms(raw string) []string {
+	if raw == "" || raw == "[]" {
+		return nil
+	}
+	var platforms []string
+	if err := json.Unmarshal([]byte(raw), &platforms); err != nil {
+		slog.Error("unmarshal platforms failed", "err", err, "raw", raw)
+		return nil
+	}
+	return platforms
+}
--- a/internal/storage/schema.go
+++ b/internal/storage/schema.go
@@ -0,0 +1,93 @@
+package storage
+
+const schema = `
+CREATE TABLE IF NOT EXISTS stories (
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    guid          TEXT UNIQUE NOT NULL,
+    headline      TEXT NOT NULL,
+    lede          TEXT,
+    image_url     TEXT,
+    article_url   TEXT NOT NULL,
+    url_canonical TEXT,
+    headline_norm TEXT,
+    source        TEXT NOT NULL,
+    feed_hint     TEXT,
+    platforms     TEXT,
+    channel       TEXT,
+    classified    INTEGER NOT NULL DEFAULT 0,
+    seen_at       INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS recent_headlines (
+    guid       TEXT PRIMARY KEY,
+    headline   TEXT NOT NULL,
+    source     TEXT NOT NULL,
+    seen_at    INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS classification_log (
+    guid       TEXT PRIMARY KEY,
+    result     TEXT NOT NULL,
+    logged_at  INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS post_log (
+    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+    guid          TEXT NOT NULL,
+    channel       TEXT NOT NULL,
+    event_id      TEXT,
+    url_canonical TEXT,
+    posted_at     INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS reactions (
+    id         INTEGER PRIMARY KEY AUTOINCREMENT,
+    post_guid  TEXT NOT NULL,
+    channel    TEXT NOT NULL,
+    event_id   TEXT NOT NULL,
+    emoji      TEXT NOT NULL,
+    user_id    TEXT NOT NULL,
+    reacted_at INTEGER NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_post_log_guid_channel ON post_log(guid, channel);
+CREATE INDEX IF NOT EXISTS idx_post_log_event_id ON post_log(event_id);
+CREATE INDEX IF NOT EXISTS idx_post_log_channel_posted ON post_log(channel, posted_at);
+CREATE INDEX IF NOT EXISTS idx_post_log_canonical_channel ON post_log(url_canonical, channel, posted_at);
+CREATE INDEX IF NOT EXISTS idx_stories_classified_source ON stories(classified, source);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_url_canonical ON stories(url_canonical) WHERE url_canonical IS NOT NULL AND url_canonical <> '';
+CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_source_headline_norm ON stories(source, headline_norm) WHERE headline_norm IS NOT NULL AND headline_norm <> '';
+CREATE INDEX IF NOT EXISTS idx_reactions_post_guid ON reactions(post_guid);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_reactions_event_id ON reactions(event_id);
+`
+
+const ftsSchema = `
+CREATE VIRTUAL TABLE stories_fts USING fts5(
+    guid UNINDEXED,
+    headline,
+    lede,
+    source UNINDEXED,
+    platforms UNINDEXED,
+    content='stories',
+    content_rowid='id'
+);
+`
+
+const ftsTriggers = `
+CREATE TRIGGER stories_fts_insert AFTER INSERT ON stories BEGIN
+    INSERT INTO stories_fts(rowid, guid, headline, lede, source, platforms)
+    VALUES (new.id, new.guid, new.headline, new.lede, new.source, new.platforms);
+END;
+
+CREATE TRIGGER stories_fts_delete AFTER DELETE ON stories BEGIN
+    INSERT INTO stories_fts(stories_fts, rowid, guid, headline, lede, source, platforms)
+    VALUES ('delete', old.id, old.guid, old.headline, old.lede, old.source, old.platforms);
+END;
+
+CREATE TRIGGER stories_fts_update AFTER UPDATE ON stories BEGIN
+    INSERT INTO stories_fts(stories_fts, rowid, guid, headline, lede, source, platforms)
+    VALUES ('delete', old.id, old.guid, old.headline, old.lede, old.source, old.platforms);
+    INSERT INTO stories_fts(rowid, guid, headline, lede, source, platforms)
+    VALUES (new.id, new.guid, new.headline, new.lede, new.source, new.platforms);
+END;
+`
--- a/internal/storage/storage_test.go
+++ b/internal/storage/storage_test.go
@@ -0,0 +1,405 @@
+package storage
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+)
+
+func setupTestDB(t *testing.T) {
+	t.Helper()
+
+	// Reset global state
+	mu.Lock()
+	if globalDB != nil {
+		globalDB.Close()
+		globalDB = nil
+	}
+	mu.Unlock()
+
+	dbPath := filepath.Join(t.TempDir(), "test.db")
+	if err := Init(dbPath); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { Close() })
+}
+
+func TestInitAndGet(t *testing.T) {
+	setupTestDB(t)
+	db := Get()
+	if db == nil {
+		t.Fatal("Get() returned nil after Init()")
+	}
+}
+
+func TestIsGUIDSeen_NotSeen(t *testing.T) {
+	setupTestDB(t)
+	if IsGUIDSeen("never-seen-guid") {
+		t.Error("expected false for unseen GUID")
+	}
+}
+
+func TestInsertStoryAndGUIDSeen(t *testing.T) {
+	setupTestDB(t)
+
+	s := &Story{
+		GUID:       "test-guid-1",
+		Headline:   "Test Headline",
+		Lede:       "Test lede sentence.",
+		ArticleURL: "https://example.com/article",
+		Source:     "Test Source",
+		FeedHint:   "tech",
+		SeenAt:     1700000000,
+	}
+	if err := InsertStory(s); err != nil {
+		t.Fatal(err)
+	}
+
+	if !IsGUIDSeen("test-guid-1") {
+		t.Error("expected true after insert")
+	}
+	if IsGUIDSeen("other-guid") {
+		t.Error("expected false for different GUID")
+	}
+}
+
+func TestInsertStoryDuplicateGUID(t *testing.T) {
+	setupTestDB(t)
+
+	s := &Story{
+		GUID: "dup-guid", Headline: "H1", ArticleURL: "https://a.com", Source: "S", SeenAt: 1,
+	}
+	if err := InsertStory(s); err != nil {
+		t.Fatal(err)
+	}
+	// Second insert with same GUID should fail
+	err := InsertStory(s)
+	if err == nil {
+		t.Error("expected error on duplicate GUID insert")
+	}
+}
+
+func TestMarkClassifiedAndGetUnclassified(t *testing.T) {
+	setupTestDB(t)
+
+	InsertStory(&Story{
+		GUID: "g1", Headline: "H1", ArticleURL: "https://a.com", Source: "S", SeenAt: 1,
+	})
+	InsertStory(&Story{
+		GUID: "g2", Headline: "H2", ArticleURL: "https://b.com", Source: "S", SeenAt: 2,
+	})
+
+	unclassified, err := GetUnclassifiedStories("S")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(unclassified) != 2 {
+		t.Fatalf("unclassified = %d, want 2", len(unclassified))
+	}
+
+	MarkClassified("g1", "tech", "[]")
+
+	unclassified, err = GetUnclassifiedStories("S")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(unclassified) != 1 {
+		t.Fatalf("unclassified = %d, want 1", len(unclassified))
+	}
+	if unclassified[0].GUID != "g2" {
+		t.Errorf("remaining = %q, want g2", unclassified[0].GUID)
+	}
+}
+
+func TestRecentHeadlines(t *testing.T) {
+	setupTestDB(t)
+
+	// Insert with explicit timestamps to ensure ordering
+	db := Get()
+	db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
+		"g1", "Headline One", "Source A", 1000)
+	db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
+		"g2", "Headline Two", "Source B", 2000)
+	db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
+		"g3", "Headline Three", "Source A", 3000)
+
+	headlines, err := GetRecentHeadlines(2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(headlines) != 2 {
+		t.Fatalf("headlines = %d, want 2", len(headlines))
+	}
+	// Should be most recent first
+	if headlines[0].GUID != "g3" {
+		t.Errorf("first = %q, want g3", headlines[0].GUID)
+	}
+	if headlines[1].GUID != "g2" {
+		t.Errorf("second = %q, want g2", headlines[1].GUID)
+	}
+}
+
+func TestPostLogAndLookup(t *testing.T) {
+	setupTestDB(t)
+
+	InsertPostLog("story-1", "tech", "$event1:example.org", "")
+	InsertPostLog("story-2", "politics", "$event2:example.org", "")
+
+	guid, channel, found := LookupPostGUID("$event1:example.org")
+	if !found {
+		t.Fatal("expected to find event")
+	}
+	if guid != "story-1" || channel != "tech" {
+		t.Errorf("got guid=%q channel=%q", guid, channel)
+	}
+
+	_, _, found = LookupPostGUID("$nonexistent:example.org")
+	if found {
+		t.Error("expected not found for unknown event")
+	}
+}
+
+func TestLastPostTimeAndCountInWindow(t *testing.T) {
+	setupTestDB(t)
+
+	if got := GetLastPostTime("tech"); got != 0 {
+		t.Errorf("empty channel last post = %d, want 0", got)
+	}
+
+	// Insert posts with specific timestamps
+	Get().Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`,
+		"g1", "tech", "$e1", 1000)
+	Get().Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`,
+		"g2", "tech", "$e2", 2000)
+	Get().Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`,
+		"g3", "politics", "$e3", 1500)
+
+	if got := GetLastPostTime("tech"); got != 2000 {
+		t.Errorf("tech last post = %d, want 2000", got)
+	}
+
+	count := CountPostsInWindow("tech", 500)
+	if count != 2 {
+		t.Errorf("tech posts in window = %d, want 2", count)
+	}
+
+	count = CountPostsInWindow("tech", 1500)
+	if count != 1 {
+		t.Errorf("tech posts after 1500 = %d, want 1", count)
+	}
+
+	count = CountPostsInWindow("politics", 0)
+	if count != 1 {
+		t.Errorf("politics posts = %d, want 1", count)
+	}
+}
+
+func TestReactions(t *testing.T) {
+	setupTestDB(t)
+
+	InsertReaction("story-1", "tech", "$react1", "📰", "@user:example.org", 1000)
+	InsertReaction("story-1", "tech", "$react2", "👍", "@user2:example.org", 1001)
+
+	var count int
+	Get().QueryRow(`SELECT COUNT(*) FROM reactions WHERE post_guid = ?`, "story-1").Scan(&count)
+	if count != 2 {
+		t.Errorf("reactions = %d, want 2", count)
+	}
+}
+
+func TestMarshalUnmarshalPlatforms(t *testing.T) {
+	tests := []struct {
+		name  string
+		input []string
+		json  string
+		back  []string
+	}{
+		{"nil", nil, "[]", nil},
+		{"empty", []string{}, "[]", nil},
+		{"single", []string{"pc"}, `["pc"]`, []string{"pc"}},
+		{"multi", []string{"nintendo-switch", "pc"}, `["nintendo-switch","pc"]`, []string{"nintendo-switch", "pc"}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := MarshalPlatforms(tt.input)
+			if got != tt.json {
+				t.Errorf("Marshal(%v) = %q, want %q", tt.input, got, tt.json)
+			}
+			back := UnmarshalPlatforms(got)
+			if len(back) != len(tt.back) {
+				t.Errorf("Unmarshal roundtrip: %v, want %v", back, tt.back)
+			}
+		})
+	}
+}
+
+func TestInsertPostLog_DuplicateIgnored(t *testing.T) {
+	setupTestDB(t)
+
+	InsertPostLog("story-1", "tech", "$event1:example.org", "")
+	// Same guid+channel should be silently ignored
+	InsertPostLog("story-1", "tech", "$event1-retry:example.org", "")
+
+	var count int
+	Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ? AND channel = ?`, "story-1", "tech").Scan(&count)
+	if count != 1 {
+		t.Errorf("expected 1 post_log entry, got %d", count)
+	}
+
+	// Different channel should be allowed
+	InsertPostLog("story-1", "politics", "$event2:example.org", "")
+	Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ?`, "story-1").Scan(&count)
+	if count != 2 {
+		t.Errorf("expected 2 post_log entries across channels, got %d", count)
+	}
+}
+
+func TestInsertReaction_DuplicateIgnored(t *testing.T) {
+	setupTestDB(t)
+
+	InsertReaction("story-1", "tech", "$react1", "📰", "@user:example.org", 1000)
+	// Same event_id should be silently ignored
+	InsertReaction("story-1", "tech", "$react1", "📰", "@user:example.org", 1001)
+
+	var count int
+	Get().QueryRow(`SELECT COUNT(*) FROM reactions WHERE event_id = ?`, "$react1").Scan(&count)
+	if count != 1 {
+		t.Errorf("expected 1 reaction, got %d", count)
+	}
+}
+
+func TestGetUnclassifiedStories_ScopedBySource(t *testing.T) {
+	setupTestDB(t)
+
+	InsertStory(&Story{GUID: "g1", Headline: "H1", ArticleURL: "https://a.com", Source: "SourceA", SeenAt: 1})
+	InsertStory(&Story{GUID: "g2", Headline: "H2", ArticleURL: "https://b.com", Source: "SourceB", SeenAt: 2})
+	InsertStory(&Story{GUID: "g3", Headline: "H3", ArticleURL: "https://c.com", Source: "SourceA", SeenAt: 3})
+
+	got, err := GetUnclassifiedStories("SourceA")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 2 {
+		t.Fatalf("expected 2 stories for SourceA, got %d", len(got))
+	}
+
+	got, err = GetUnclassifiedStories("SourceB")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 {
+		t.Fatalf("expected 1 story for SourceB, got %d", len(got))
+	}
+
+	got, err = GetUnclassifiedStories("SourceC")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("expected 0 stories for SourceC, got %d", len(got))
+	}
+}
+
+func TestGetUnclassifiedStories_Limit20(t *testing.T) {
+	setupTestDB(t)
+
+	for i := 0; i < 30; i++ {
+		InsertStory(&Story{
+			GUID: fmt.Sprintf("g%d", i), Headline: fmt.Sprintf("H%d", i),
+			ArticleURL: "https://a.com", Source: "S", SeenAt: int64(i),
+		})
+	}
+
+	got, err := GetUnclassifiedStories("S")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 20 {
+		t.Fatalf("expected 20 (capped), got %d", len(got))
+	}
+}
+
+func TestRunMaintenance(t *testing.T) {
+	setupTestDB(t)
+	db := Get()
+
+	now := nowUnix()
+	old := now - 100*86400 // 100 days ago
+
+	// Insert old + recent stories
+	InsertStory(&Story{GUID: "old", Headline: "Old", ArticleURL: "https://a.com", Source: "S", Classified: true, SeenAt: old})
+	InsertStory(&Story{GUID: "new", Headline: "New", ArticleURL: "https://b.com", Source: "S", Classified: true, SeenAt: now})
+	MarkClassified("old", "tech", "[]")
+	MarkClassified("new", "tech", "[]")
+
+	// Insert old + recent post logs
+	db.Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`, "old", "tech", "$e1", old)
+	db.Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`, "new", "tech", "$e2", now)
+
+	// Insert old + recent reactions
+	InsertReaction("old", "tech", "$r1", "👍", "@u:x", old)
+	InsertReaction("new", "tech", "$r2", "👍", "@u:x", now)
+
+	// Insert old + recent headlines
+	db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`, "old", "Old", "S", old)
+	db.Exec(`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`, "new", "New", "S", now)
+
+	RunMaintenance(24, 7)
+
+	// Old stories should be pruned
+	var count int
+	db.QueryRow(`SELECT COUNT(*) FROM stories`).Scan(&count)
+	if count != 1 {
+		t.Errorf("stories: expected 1, got %d", count)
+	}
+
+	db.QueryRow(`SELECT COUNT(*) FROM post_log`).Scan(&count)
+	if count != 1 {
+		t.Errorf("post_log: expected 1, got %d", count)
+	}
+
+	db.QueryRow(`SELECT COUNT(*) FROM reactions`).Scan(&count)
+	if count != 1 {
+		t.Errorf("reactions: expected 1, got %d", count)
+	}
+
+	db.QueryRow(`SELECT COUNT(*) FROM recent_headlines`).Scan(&count)
+	if count != 1 {
+		t.Errorf("recent_headlines: expected 1, got %d", count)
+	}
+}
+
+func TestFTS5Search(t *testing.T) {
+	setupTestDB(t)
+
+	InsertStory(&Story{
+		GUID: "g1", Headline: "Elden Ring DLC breaks sales records", Lede: "FromSoftware celebrates milestone.",
+		ArticleURL: "https://a.com", Source: "S", SeenAt: 1,
+	})
+	InsertStory(&Story{
+		GUID: "g2", Headline: "Apple unveils new MacBook Pro", Lede: "M5 chip delivers impressive gains.",
+		ArticleURL: "https://b.com", Source: "S", SeenAt: 2,
+	})
+
+	// Search for Elden Ring
+	rows, err := Get().Query(`
+		SELECT s.guid, s.headline FROM stories s
+		JOIN stories_fts f ON s.id = f.rowid
+		WHERE stories_fts MATCH 'Elden Ring'
+		ORDER BY rank`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rows.Close()
+
+	var results []string
+	for rows.Next() {
+		var guid, headline string
+		rows.Scan(&guid, &headline)
+		results = append(results, guid)
+	}
+	if len(results) != 1 || results[0] != "g1" {
+		t.Errorf("FTS5 search results = %v, want [g1]", results)
+	}
+}
--- a/internal/storage/types.go
+++ b/internal/storage/types.go
@@ -0,0 +1,27 @@
+package storage
+
+// Story is the core record for an ingested news item.
+type Story struct {
+	ID         int64
+	GUID       string
+	Headline   string
+	Lede       string
+	ImageURL     string
+	ArticleURL   string
+	URLCanonical string
+	HeadlineNorm string
+	Source       string
+	FeedHint   string
+	Platforms  string // JSON array e.g. '["nintendo-switch","multi"]'
+	Channel    string
+	Classified bool
+	SeenAt     int64
+}
+
+// RecentHeadline is a lightweight record for the LLM dedup context window.
+type RecentHeadline struct {
+	GUID     string
+	Headline string
+	Source   string
+	SeenAt   int64
+}