509a0fc7a7
Bypass-UA retry (Googlebot + Google referer) for soft paywalls, JSON-LD gating scoped to Article-typed nodes, HTTP 402 treated as explicit paywall, Wayback freshness filter (30d cap), archive.today as secondary archive fallback, and transport failures no longer trigger snapshot swaps. When gating is detected and no archive workaround succeeds, the story is stored with paywalled=1 and the web card renders a diagonal red rubber-stamp overlay so readers know the link is gated.
135 lines
3.6 KiB
Go
135 lines
3.6 KiB
Go
package storage
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
var (
|
|
mu sync.RWMutex
|
|
globalDB *sql.DB
|
|
)
|
|
|
|
// Init opens (or creates) the SQLite database and runs migrations.
|
|
func Init(dbPath string) error {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
|
|
if globalDB != nil {
|
|
return nil
|
|
}
|
|
|
|
dir := filepath.Dir(dbPath)
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
return fmt.Errorf("create data dir: %w", err)
|
|
}
|
|
|
|
d, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(ON)")
|
|
if err != nil {
|
|
return fmt.Errorf("open database: %w", err)
|
|
}
|
|
|
|
d.SetMaxOpenConns(1)
|
|
|
|
if err := runMigrations(d); err != nil {
|
|
return fmt.Errorf("run migrations: %w", err)
|
|
}
|
|
|
|
globalDB = d
|
|
slog.Info("database initialized", "path", dbPath)
|
|
return nil
|
|
}
|
|
|
|
// Get returns the global database handle. Panics if Init was not called.
|
|
func Get() *sql.DB {
|
|
mu.RLock()
|
|
db := globalDB
|
|
mu.RUnlock()
|
|
if db == nil {
|
|
panic("storage.Get() called before storage.Init()")
|
|
}
|
|
return db
|
|
}
|
|
|
|
// Close closes the global database handle.
|
|
func Close() error {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if globalDB != nil {
|
|
err := globalDB.Close()
|
|
globalDB = nil
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func runMigrations(d *sql.DB) error {
|
|
if _, err := d.Exec(schema); err != nil {
|
|
return fmt.Errorf("create schema: %w", err)
|
|
}
|
|
|
|
// Idempotent column adds for DBs created before the dedup columns existed.
|
|
// SQLite errors with "duplicate column name" when the column is already there;
|
|
// we swallow that specifically.
|
|
addColumnIfMissing(d, "stories", "url_canonical", "TEXT")
|
|
addColumnIfMissing(d, "stories", "headline_norm", "TEXT")
|
|
addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
|
|
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
|
|
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")
|
|
|
|
// FTS5 virtual tables don't support IF NOT EXISTS reliably.
|
|
// Check sqlite_master before creating.
|
|
var ftsExists int
|
|
d.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='stories_fts'`).Scan(&ftsExists)
|
|
if ftsExists == 0 {
|
|
if _, err := d.Exec(ftsSchema); err != nil {
|
|
return fmt.Errorf("create FTS5 table: %w", err)
|
|
}
|
|
if _, err := d.Exec(ftsTriggers); err != nil {
|
|
return fmt.Errorf("create FTS5 triggers: %w", err)
|
|
}
|
|
slog.Info("created FTS5 search index")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// RunMaintenance prunes stale data. Called periodically.
|
|
func RunMaintenance() {
|
|
// Prune old stories (30 days) and their post logs / reactions
|
|
storyCutoff := nowUnix() - int64(30*86400)
|
|
exec("prune old stories",
|
|
`DELETE FROM stories WHERE seen_at < ? AND classified = 1`, storyCutoff)
|
|
exec("prune old post_log",
|
|
`DELETE FROM post_log WHERE posted_at < ?`, storyCutoff)
|
|
exec("prune old reactions",
|
|
`DELETE FROM reactions WHERE reacted_at < ?`, storyCutoff)
|
|
|
|
exec("wal checkpoint", "PRAGMA wal_checkpoint(TRUNCATE)")
|
|
exec("optimize", "PRAGMA optimize")
|
|
}
|
|
|
|
// exec is a fire-and-forget helper that logs errors.
|
|
func exec(label, query string, args ...any) {
|
|
if _, err := Get().Exec(query, args...); err != nil {
|
|
slog.Error("db exec failed", "op", label, "err", err)
|
|
}
|
|
}
|
|
|
|
func addColumnIfMissing(d *sql.DB, table, column, columnType string) {
|
|
q := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, columnType)
|
|
if _, err := d.Exec(q); err != nil {
|
|
// SQLite returns "duplicate column name" when the column already exists.
|
|
if !strings.Contains(err.Error(), "duplicate column name") {
|
|
slog.Error("alter table failed", "table", table, "column", column, "err", err)
|
|
}
|
|
}
|
|
}
|