Add Phase 2+3 features: antonyms, backing, pronunciation, etymology, difficulty, affix expansion
New endpoints: /antonyms, /backing, /pronunciation, /etymology with difficulty scoring on /random. Cross-language synset backing links French/Portuguese words to English equivalents via WordNet 3.0 synset IDs (matching WOLF and OMW offsets). New loaders: Hunspell affix expansion (fr, pt-PT), English programmatic inflector, CMU Pronouncing Dictionary, SUBTLEX-US frequency, CETEMPúblico frequency, Open Multilingual Wordnet (Portuguese), and SQL-based difficulty scoring. Schema v2 adds tables: antonyms, synsets, word_synsets, pronunciations, etymology with migration support for existing databases. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -3,11 +3,12 @@ package dictionary
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const schemaVersion = "1"
|
||||
const schemaVersion = "2"
|
||||
|
||||
const schemaSQL = `
|
||||
CREATE TABLE IF NOT EXISTS words (
|
||||
@@ -15,7 +16,8 @@ CREATE TABLE IF NOT EXISTS words (
|
||||
word TEXT NOT NULL,
|
||||
lang TEXT NOT NULL,
|
||||
pos TEXT,
|
||||
frequency INTEGER DEFAULT 0,
|
||||
frequency INTEGER DEFAULT 0,
|
||||
difficulty REAL DEFAULT NULL,
|
||||
UNIQUE(word, lang)
|
||||
);
|
||||
|
||||
@@ -61,6 +63,53 @@ CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS antonyms (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
|
||||
antonym TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
UNIQUE(word_id, antonym)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_antonyms_word_id ON antonyms(word_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS synsets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
synset_id TEXT NOT NULL UNIQUE,
|
||||
pos TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS word_synsets (
|
||||
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
|
||||
synset_id INTEGER NOT NULL REFERENCES synsets(id) ON DELETE CASCADE,
|
||||
source TEXT NOT NULL,
|
||||
PRIMARY KEY (word_id, synset_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_word_synsets_word_id ON word_synsets(word_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_word_synsets_synset_id ON word_synsets(synset_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pronunciations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
|
||||
format TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
UNIQUE(word_id, format, source)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pronunciations_word_id ON pronunciations(word_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS etymology (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
UNIQUE(word_id, source)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_etymology_word_id ON etymology(word_id);
|
||||
`
|
||||
|
||||
func openDB(dbPath string) (*sql.DB, error) {
|
||||
@@ -77,11 +126,19 @@ func openDB(dbPath string) (*sql.DB, error) {
|
||||
}
|
||||
|
||||
func openReadOnlyDB(dbPath string) (*sql.DB, error) {
|
||||
db, err := sql.Open("sqlite", dbPath+"?mode=ro&_pragma=journal_mode(wal)&_pragma=foreign_keys(on)")
|
||||
db, err := sql.Open("sqlite", dbPath+
|
||||
"?mode=ro"+
|
||||
"&_pragma=journal_mode(wal)"+
|
||||
"&_pragma=foreign_keys(on)"+
|
||||
"&_pragma=cache_size(-64000)"+
|
||||
"&_pragma=mmap_size(268435456)"+
|
||||
"&_pragma=temp_store(memory)")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: open db (ro): %w", err)
|
||||
}
|
||||
// No MaxOpenConns limit — WAL mode supports concurrent reads
|
||||
// Pin to one connection so pragmas are consistent. WAL mode still
|
||||
// allows concurrent reads from the same connection in Go's model.
|
||||
db.SetMaxOpenConns(1)
|
||||
if err := db.Ping(); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("dictionary: ping db: %w", err)
|
||||
@@ -94,5 +151,25 @@ func BootstrapSchema(db *sql.DB) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("dictionary: bootstrap schema: %w", err)
|
||||
}
|
||||
|
||||
// Schema migrations for databases created before v2.
|
||||
// ALTER TABLE ADD COLUMN is a no-op if the column already exists in SQLite
|
||||
// when we catch the "duplicate column" error.
|
||||
migrations := []string{
|
||||
"ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL",
|
||||
}
|
||||
for _, m := range migrations {
|
||||
if _, err := db.Exec(m); err != nil {
|
||||
// Ignore "duplicate column name" — means migration already applied
|
||||
if !isDuplicateColumn(err) {
|
||||
return fmt.Errorf("dictionary: migration: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isDuplicateColumn(err error) bool {
|
||||
return err != nil && strings.Contains(err.Error(), "duplicate column")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user