Files
Pete/internal/storage/db.go
prosolis 9e5ba5aafc Fix SSRF, XSS, dedup, force-post, and DB hot-path issues
- Add internal/safehttp: hardened HTTP client (DNS-resolved dial guard
  blocking loopback/RFC1918/CGNAT/link-local, redirect re-validation,
  body-size cap) and rewire article/feed/thumb clients through it
- Cap goquery body at 5 MiB so a hostile origin can't OOM the process
- search.js: reject non-http(s) hrefs to block stored XSS via javascript:
- dedup: tracking-param key "CMP" was unreachable (lookup lowercases);
  fixed to "cmp" so CMP= is actually stripped from canonical URLs
- ForcePost: postItem now returns bool; on dedup-skip ForcePost returns
  false so !post falls back to DB lookup instead of silently consuming
- Bound reaction callbacks behind an 8-slot semaphore; drop overflow
- Add stories indexes on (channel, classified, seen_at DESC),
  (classified, seen_at DESC), and partial image_url to kill full scans
  in IsKnownImageURL and ORDER BY seen_at hot paths
- Surface FTS5 probe Scan error instead of swallowing it
2026-05-25 12:23:37 -07:00

137 lines
3.6 KiB
Go

package storage
import (
"database/sql"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"sync"
_ "modernc.org/sqlite"
)
var (
mu sync.RWMutex
globalDB *sql.DB
)
// Init opens (or creates) the SQLite database and runs migrations.
func Init(dbPath string) error {
mu.Lock()
defer mu.Unlock()
if globalDB != nil {
return nil
}
dir := filepath.Dir(dbPath)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("create data dir: %w", err)
}
d, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(ON)")
if err != nil {
return fmt.Errorf("open database: %w", err)
}
d.SetMaxOpenConns(1)
if err := runMigrations(d); err != nil {
return fmt.Errorf("run migrations: %w", err)
}
globalDB = d
slog.Info("database initialized", "path", dbPath)
return nil
}
// Get returns the global database handle. Panics if Init was not called.
func Get() *sql.DB {
mu.RLock()
db := globalDB
mu.RUnlock()
if db == nil {
panic("storage.Get() called before storage.Init()")
}
return db
}
// Close closes the global database handle.
func Close() error {
mu.Lock()
defer mu.Unlock()
if globalDB != nil {
err := globalDB.Close()
globalDB = nil
return err
}
return nil
}
func runMigrations(d *sql.DB) error {
if _, err := d.Exec(schema); err != nil {
return fmt.Errorf("create schema: %w", err)
}
// Idempotent column adds for DBs created before the dedup columns existed.
// SQLite errors with "duplicate column name" when the column is already there;
// we swallow that specifically.
addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")
// FTS5 virtual tables don't support IF NOT EXISTS reliably.
// Check sqlite_master before creating.
var ftsExists int
if err := d.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='stories_fts'`).Scan(&ftsExists); err != nil {
return fmt.Errorf("probe FTS5 table: %w", err)
}
if ftsExists == 0 {
if _, err := d.Exec(ftsSchema); err != nil {
return fmt.Errorf("create FTS5 table: %w", err)
}
if _, err := d.Exec(ftsTriggers); err != nil {
return fmt.Errorf("create FTS5 triggers: %w", err)
}
slog.Info("created FTS5 search index")
}
return nil
}
// RunMaintenance prunes stale data. Called periodically.
func RunMaintenance() {
// Prune old stories (30 days) and their post logs / reactions
storyCutoff := nowUnix() - int64(30*86400)
exec("prune old stories",
`DELETE FROM stories WHERE seen_at < ? AND classified = 1`, storyCutoff)
exec("prune old post_log",
`DELETE FROM post_log WHERE posted_at < ?`, storyCutoff)
exec("prune old reactions",
`DELETE FROM reactions WHERE reacted_at < ?`, storyCutoff)
exec("wal checkpoint", "PRAGMA wal_checkpoint(TRUNCATE)")
exec("optimize", "PRAGMA optimize")
}
// exec is a fire-and-forget helper that logs errors.
func exec(label, query string, args ...any) {
if _, err := Get().Exec(query, args...); err != nil {
slog.Error("db exec failed", "op", label, "err", err)
}
}
func addColumnIfMissing(d *sql.DB, table, column, columnType string) {
q := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, columnType)
if _, err := d.Exec(q); err != nil {
// SQLite returns "duplicate column name" when the column already exists.
if !strings.Contains(err.Error(), "duplicate column name") {
slog.Error("alter table failed", "table", table, "column", column, "err", err)
}
}
}