dreamdict/internal/dictionary/db.go

package dictionary

import (
	"database/sql"
	"fmt"
	"strings"

	_ "modernc.org/sqlite"
)

const schemaVersion = "2"

const schemaSQL = `
CREATE TABLE IF NOT EXISTS words (
    id        INTEGER PRIMARY KEY AUTOINCREMENT,
    word      TEXT    NOT NULL,
    lang      TEXT    NOT NULL,
    pos       TEXT,
    frequency  INTEGER DEFAULT 0,
    difficulty REAL DEFAULT NULL,
    UNIQUE(word, lang)
);

CREATE INDEX IF NOT EXISTS idx_words_lang     ON words(lang);
CREATE INDEX IF NOT EXISTS idx_words_lang_pos ON words(lang, pos);
CREATE INDEX IF NOT EXISTS idx_words_word     ON words(word);

CREATE TABLE IF NOT EXISTS definitions (
    id       INTEGER PRIMARY KEY AUTOINCREMENT,
    word_id  INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
    pos      TEXT,
    gloss    TEXT    NOT NULL,
    source   TEXT    NOT NULL,
    priority INTEGER NOT NULL DEFAULT 99,
    UNIQUE(word_id, gloss)
);

CREATE INDEX IF NOT EXISTS idx_definitions_word_id          ON definitions(word_id);
CREATE INDEX IF NOT EXISTS idx_definitions_word_id_priority ON definitions(word_id, priority);

CREATE TABLE IF NOT EXISTS synonyms (
    id       INTEGER PRIMARY KEY AUTOINCREMENT,
    word_id  INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
    synonym  TEXT    NOT NULL,
    source   TEXT    NOT NULL,
    UNIQUE(word_id, synonym)
);

CREATE INDEX IF NOT EXISTS idx_synonyms_word_id ON synonyms(word_id);

CREATE TABLE IF NOT EXISTS translations (
    id          INTEGER PRIMARY KEY AUTOINCREMENT,
    word_id     INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
    translation TEXT    NOT NULL,
    target_lang TEXT    NOT NULL,
    source      TEXT    NOT NULL,
    UNIQUE(word_id, translation, target_lang)
);

CREATE INDEX IF NOT EXISTS idx_translations_word_id ON translations(word_id);

CREATE TABLE IF NOT EXISTS meta (
    key   TEXT PRIMARY KEY,
    value TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS antonyms (
    id       INTEGER PRIMARY KEY AUTOINCREMENT,
    word_id  INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
    antonym  TEXT    NOT NULL,
    source   TEXT    NOT NULL,
    UNIQUE(word_id, antonym)
);

CREATE INDEX IF NOT EXISTS idx_antonyms_word_id ON antonyms(word_id);

CREATE TABLE IF NOT EXISTS synsets (
    id        INTEGER PRIMARY KEY AUTOINCREMENT,
    synset_id TEXT NOT NULL UNIQUE,
    pos       TEXT NOT NULL
);

CREATE TABLE IF NOT EXISTS word_synsets (
    word_id   INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
    synset_id INTEGER NOT NULL REFERENCES synsets(id) ON DELETE CASCADE,
    source    TEXT NOT NULL,
    PRIMARY KEY (word_id, synset_id)
);

CREATE INDEX IF NOT EXISTS idx_word_synsets_word_id   ON word_synsets(word_id);
CREATE INDEX IF NOT EXISTS idx_word_synsets_synset_id ON word_synsets(synset_id);

CREATE TABLE IF NOT EXISTS pronunciations (
    id       INTEGER PRIMARY KEY AUTOINCREMENT,
    word_id  INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
    format   TEXT NOT NULL,
    value    TEXT NOT NULL,
    source   TEXT NOT NULL,
    UNIQUE(word_id, format, source)
);

CREATE INDEX IF NOT EXISTS idx_pronunciations_word_id ON pronunciations(word_id);

CREATE TABLE IF NOT EXISTS etymology (
    id         INTEGER PRIMARY KEY AUTOINCREMENT,
    word_id    INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
    text       TEXT NOT NULL,
    source     TEXT NOT NULL,
    UNIQUE(word_id, source)
);

CREATE INDEX IF NOT EXISTS idx_etymology_word_id ON etymology(word_id);
`

func openDB(dbPath string) (*sql.DB, error) {
	db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(wal)&_pragma=foreign_keys(on)")
	if err != nil {
		return nil, fmt.Errorf("dictionary: open db: %w", err)
	}
	db.SetMaxOpenConns(1) // SQLite write serialization for import
	if err := db.Ping(); err != nil {
		db.Close()
		return nil, fmt.Errorf("dictionary: ping db: %w", err)
	}
	return db, nil
}

func openReadOnlyDB(dbPath string) (*sql.DB, error) {
	db, err := sql.Open("sqlite", dbPath+
		"?mode=ro"+
		"&_pragma=journal_mode(wal)"+
		"&_pragma=foreign_keys(on)"+
		"&_pragma=cache_size(-64000)"+
		"&_pragma=mmap_size(268435456)"+
		"&_pragma=temp_store(memory)")
	if err != nil {
		return nil, fmt.Errorf("dictionary: open db (ro): %w", err)
	}
	// Pin to one connection so pragmas are consistent. WAL mode still
	// allows concurrent reads from the same connection in Go's model.
	db.SetMaxOpenConns(1)
	if err := db.Ping(); err != nil {
		db.Close()
		return nil, fmt.Errorf("dictionary: ping db: %w", err)
	}
	return db, nil
}

func BootstrapSchema(db *sql.DB) error {
	_, err := db.Exec(schemaSQL)
	if err != nil {
		return fmt.Errorf("dictionary: bootstrap schema: %w", err)
	}

	// Schema migrations for databases created before v2.
	// ALTER TABLE ADD COLUMN is a no-op if the column already exists in SQLite
	// when we catch the "duplicate column" error.
	migrations := []string{
		"ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL",
		"ALTER TABLE pronunciations ADD COLUMN cmu_tail TEXT",
	}
	for _, m := range migrations {
		if _, err := db.Exec(m); err != nil {
			// Ignore "duplicate column name" — means migration already applied
			if !isDuplicateColumn(err) {
				return fmt.Errorf("dictionary: migration: %w", err)
			}
		}
	}

	return nil
}

// Migrate opens the database read-write, applies any pending schema migrations,
// and populates derived data (e.g. CMU rhyme tails). Returns the number of
// CMU tails populated (0 if already up to date).
func Migrate(dbPath string) (int, error) {
	db, err := openDB(dbPath)
	if err != nil {
		return 0, err
	}
	defer db.Close()

	if err := BootstrapSchema(db); err != nil {
		return 0, err
	}

	return PopulateCMUTails(db)
}

func isDuplicateColumn(err error) bool {
	return err != nil && strings.Contains(err.Error(), "duplicate column")
}

// PopulateCMUTails computes the rhyme tail for all CMU pronunciation entries
// that don't already have one. The tail is the phoneme sequence from the last
// stressed vowel onward (e.g. "K AE1 T" → "AE1 T").
func PopulateCMUTails(db *sql.DB) (int, error) {
	// Create index if it doesn't exist.
	db.Exec("CREATE INDEX IF NOT EXISTS idx_pronunciations_cmu_tail ON pronunciations(cmu_tail)")

	rows, err := db.Query(
		`SELECT id, value FROM pronunciations WHERE format = 'cmu' AND (cmu_tail IS NULL OR cmu_tail = '')`)
	if err != nil {
		return 0, fmt.Errorf("dictionary: populate cmu tails: %w", err)
	}

	type entry struct {
		id    int
		value string
	}
	var entries []entry
	for rows.Next() {
		var e entry
		if err := rows.Scan(&e.id, &e.value); err != nil {
			rows.Close()
			return 0, fmt.Errorf("dictionary: populate cmu tails scan: %w", err)
		}
		entries = append(entries, e)
	}
	rows.Close()

	if len(entries) == 0 {
		return 0, nil
	}

	tx, err := db.Begin()
	if err != nil {
		return 0, fmt.Errorf("dictionary: populate cmu tails begin: %w", err)
	}
	defer tx.Rollback()

	stmt, err := tx.Prepare("UPDATE pronunciations SET cmu_tail = ? WHERE id = ?")
	if err != nil {
		return 0, fmt.Errorf("dictionary: populate cmu tails prepare: %w", err)
	}
	defer stmt.Close()

	count := 0
	for _, e := range entries {
		tail := cmuTail(e.value)
		if tail == "" {
			continue
		}
		if _, err := stmt.Exec(tail, e.id); err != nil {
			return 0, fmt.Errorf("dictionary: populate cmu tails update: %w", err)
		}
		count++
	}

	if err := tx.Commit(); err != nil {
		return 0, fmt.Errorf("dictionary: populate cmu tails commit: %w", err)
	}
	return count, nil
}

// cmuTail extracts the phoneme sequence from the last stressed vowel onward.
// e.g. "K AE1 T" → "AE1 T", "IH0 F EH1 M ER0 AH0 L" → "EH1 M ER0 AH0 L"
func cmuTail(phonemes string) string {
	parts := strings.Fields(phonemes)
	lastStressed := -1
	for i, p := range parts {
		if len(p) >= 2 && p[len(p)-1] == '1' {
			lastStressed = i
		}
	}
	// Fall back to last vowel with any stress marker.
	if lastStressed == -1 {
		for i, p := range parts {
			if len(p) >= 2 && (p[len(p)-1] == '0' || p[len(p)-1] == '2') {
				lastStressed = i
			}
		}
	}
	if lastStressed == -1 {
		return ""
	}
	return strings.Join(parts[lastStressed:], " ")
}