Add Phase 2+3 features: antonyms, backing, pronunciation, etymology, difficulty, affix expansion

New endpoints: /antonyms, /backing, /pronunciation, /etymology with difficulty
scoring on /random. Cross-language synset backing links French/Portuguese words
to English equivalents via WordNet 3.0 synset IDs (matching WOLF and OMW offsets).

New loaders: Hunspell affix expansion (fr, pt-PT), English programmatic inflector,
CMU Pronouncing Dictionary, SUBTLEX-US frequency, CETEMPúblico frequency,
Open Multilingual Wordnet (Portuguese), and SQL-based difficulty scoring.

Schema v2 adds tables: antonyms, synsets, word_synsets, pronunciations, etymology
with migration support for existing databases.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
prosolis
2026-04-02 01:01:06 -07:00
parent d5e4ce0d36
commit 207086da08
22 changed files with 3102 additions and 120 deletions

View File

@@ -3,11 +3,12 @@ package dictionary
import (
"database/sql"
"fmt"
"strings"
_ "modernc.org/sqlite"
)
const schemaVersion = "1"
const schemaVersion = "2"
const schemaSQL = `
CREATE TABLE IF NOT EXISTS words (
@@ -15,7 +16,8 @@ CREATE TABLE IF NOT EXISTS words (
word TEXT NOT NULL,
lang TEXT NOT NULL,
pos TEXT,
frequency INTEGER DEFAULT 0,
frequency INTEGER DEFAULT 0,
difficulty REAL DEFAULT NULL,
UNIQUE(word, lang)
);
@@ -61,6 +63,53 @@ CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS antonyms (
id INTEGER PRIMARY KEY AUTOINCREMENT,
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
antonym TEXT NOT NULL,
source TEXT NOT NULL,
UNIQUE(word_id, antonym)
);
CREATE INDEX IF NOT EXISTS idx_antonyms_word_id ON antonyms(word_id);
CREATE TABLE IF NOT EXISTS synsets (
id INTEGER PRIMARY KEY AUTOINCREMENT,
synset_id TEXT NOT NULL UNIQUE,
pos TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS word_synsets (
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
synset_id INTEGER NOT NULL REFERENCES synsets(id) ON DELETE CASCADE,
source TEXT NOT NULL,
PRIMARY KEY (word_id, synset_id)
);
CREATE INDEX IF NOT EXISTS idx_word_synsets_word_id ON word_synsets(word_id);
CREATE INDEX IF NOT EXISTS idx_word_synsets_synset_id ON word_synsets(synset_id);
CREATE TABLE IF NOT EXISTS pronunciations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
format TEXT NOT NULL,
value TEXT NOT NULL,
source TEXT NOT NULL,
UNIQUE(word_id, format, source)
);
CREATE INDEX IF NOT EXISTS idx_pronunciations_word_id ON pronunciations(word_id);
CREATE TABLE IF NOT EXISTS etymology (
id INTEGER PRIMARY KEY AUTOINCREMENT,
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
text TEXT NOT NULL,
source TEXT NOT NULL,
UNIQUE(word_id, source)
);
CREATE INDEX IF NOT EXISTS idx_etymology_word_id ON etymology(word_id);
`
func openDB(dbPath string) (*sql.DB, error) {
@@ -77,11 +126,19 @@ func openDB(dbPath string) (*sql.DB, error) {
}
func openReadOnlyDB(dbPath string) (*sql.DB, error) {
db, err := sql.Open("sqlite", dbPath+"?mode=ro&_pragma=journal_mode(wal)&_pragma=foreign_keys(on)")
db, err := sql.Open("sqlite", dbPath+
"?mode=ro"+
"&_pragma=journal_mode(wal)"+
"&_pragma=foreign_keys(on)"+
"&_pragma=cache_size(-64000)"+
"&_pragma=mmap_size(268435456)"+
"&_pragma=temp_store(memory)")
if err != nil {
return nil, fmt.Errorf("dictionary: open db (ro): %w", err)
}
// No MaxOpenConns limit — WAL mode supports concurrent reads
// Pin to one connection so pragmas are consistent. WAL mode still
// allows concurrent reads from the same connection in Go's model.
db.SetMaxOpenConns(1)
if err := db.Ping(); err != nil {
db.Close()
return nil, fmt.Errorf("dictionary: ping db: %w", err)
@@ -94,5 +151,25 @@ func BootstrapSchema(db *sql.DB) error {
if err != nil {
return fmt.Errorf("dictionary: bootstrap schema: %w", err)
}
// Schema migrations for databases created before v2.
// ALTER TABLE ADD COLUMN is a no-op if the column already exists in SQLite
// when we catch the "duplicate column" error.
migrations := []string{
"ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL",
}
for _, m := range migrations {
if _, err := db.Exec(m); err != nil {
// Ignore "duplicate column name" — means migration already applied
if !isDuplicateColumn(err) {
return fmt.Errorf("dictionary: migration: %w", err)
}
}
}
return nil
}
func isDuplicateColumn(err error) bool {
return err != nil && strings.Contains(err.Error(), "duplicate column")
}