Add Phase 2+3 features: antonyms, backing, pronunciation, etymology, difficulty, affix expansion

New endpoints: /antonyms, /backing, /pronunciation, /etymology with difficulty scoring on /random. Cross-language synset backing links French/Portuguese words to English equivalents via WordNet 3.0 synset IDs (matching WOLF and OMW offsets). New loaders: Hunspell affix expansion (fr, pt-PT), English programmatic inflector, CMU Pronouncing Dictionary, SUBTLEX-US frequency, CETEMPúblico frequency, Open Multilingual Wordnet (Portuguese), and SQL-based difficulty scoring. Schema v2 adds tables: antonyms, synsets, word_synsets, pronunciations, etymology with migration support for existing databases. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-02 01:01:06 -07:00
parent d5e4ce0d36
commit 207086da08
22 changed files with 3102 additions and 120 deletions
--- a/internal/dictionary/bench_test.go
+++ b/internal/dictionary/bench_test.go
@@ -0,0 +1,151 @@
+package dictionary
+
+import (
+	"database/sql"
+	"fmt"
+	"testing"
+
+	_ "modernc.org/sqlite"
+)
+
+func setupBenchDB(b *testing.B) (*Dictionary, *sql.DB) {
+	b.Helper()
+	db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(on)")
+	if err != nil {
+		b.Fatal(err)
+	}
+	db.SetMaxOpenConns(1)
+	if err := BootstrapSchema(db); err != nil {
+		b.Fatal(err)
+	}
+
+	// Seed a realistic-ish dataset: 1000 words with definitions, synonyms, translations
+	exec := func(q string, args ...any) {
+		if _, err := db.Exec(q, args...); err != nil {
+			b.Fatalf("seed: %s: %v", q, err)
+		}
+	}
+
+	exec("INSERT INTO meta (key, value) VALUES ('schema_version', '1')")
+	exec("INSERT INTO meta (key, value) VALUES ('imported_at', '2025-01-01T00:00:00Z')")
+
+	poses := []string{"noun", "verb", "adjective", "adverb"}
+	langs := []string{"en", "fr", "pt-PT"}
+
+	for i := 1; i <= 1000; i++ {
+		lang := langs[i%len(langs)]
+		pos := poses[i%len(poses)]
+		word := fmt.Sprintf("word%04d", i)
+		exec("INSERT INTO words (id, word, lang, pos, frequency) VALUES (?, ?, ?, ?, ?)",
+			i, word, lang, pos, i%100)
+
+		// 2 definitions per word
+		exec("INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (?, ?, ?, 'wordnet', 10)",
+			i, pos, fmt.Sprintf("definition A of %s", word))
+		exec("INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (?, ?, ?, 'wiktionary', 20)",
+			i, pos, fmt.Sprintf("definition B of %s", word))
+
+		// 2 synonyms
+		exec("INSERT INTO synonyms (word_id, synonym, source) VALUES (?, ?, 'wordnet')",
+			i, fmt.Sprintf("syn1_%s", word))
+		exec("INSERT INTO synonyms (word_id, synonym, source) VALUES (?, ?, 'wordnet')",
+			i, fmt.Sprintf("syn2_%s", word))
+
+		// Translation for en words
+		if lang == "en" {
+			exec("INSERT INTO translations (word_id, translation, target_lang, source) VALUES (?, ?, 'fr', 'wiktionary')",
+				i, fmt.Sprintf("fr_%s", word))
+		}
+	}
+
+	return NewFromDB(db), db
+}
+
+func BenchmarkIsValidWord(b *testing.B) {
+	d, db := setupBenchDB(b)
+	defer db.Close()
+
+	b.Run("hit", func(b *testing.B) {
+		for b.Loop() {
+			d.IsValidWord("word0003", "en")
+		}
+	})
+	b.Run("miss", func(b *testing.B) {
+		for b.Loop() {
+			d.IsValidWord("nonexistent", "en")
+		}
+	})
+}
+
+func BenchmarkRandomWord(b *testing.B) {
+	d, db := setupBenchDB(b)
+	defer db.Close()
+
+	b.Run("no_filter", func(b *testing.B) {
+		for b.Loop() {
+			d.RandomWord("en", Options{})
+		}
+	})
+	b.Run("with_filters", func(b *testing.B) {
+		for b.Loop() {
+			d.RandomWord("en", Options{POS: "noun", MinLength: 4, MaxLength: 10})
+		}
+	})
+}
+
+func BenchmarkDefine(b *testing.B) {
+	d, db := setupBenchDB(b)
+	defer db.Close()
+
+	b.Run("with_defs", func(b *testing.B) {
+		for b.Loop() {
+			d.Define("word0003", "en")
+		}
+	})
+	b.Run("no_defs", func(b *testing.B) {
+		for b.Loop() {
+			d.Define("nonexistent", "en")
+		}
+	})
+}
+
+func BenchmarkSynonyms(b *testing.B) {
+	d, db := setupBenchDB(b)
+	defer db.Close()
+
+	b.Run("with_syns", func(b *testing.B) {
+		for b.Loop() {
+			d.Synonyms("word0003", "en")
+		}
+	})
+	b.Run("no_syns", func(b *testing.B) {
+		for b.Loop() {
+			d.Synonyms("nonexistent", "en")
+		}
+	})
+}
+
+func BenchmarkTranslate(b *testing.B) {
+	d, db := setupBenchDB(b)
+	defer db.Close()
+
+	b.Run("with_trans", func(b *testing.B) {
+		for b.Loop() {
+			d.Translate("word0003", "en", "fr")
+		}
+	})
+	b.Run("no_trans", func(b *testing.B) {
+		for b.Loop() {
+			d.Translate("nonexistent", "en", "fr")
+		}
+	})
+}
+
+func BenchmarkWordCount(b *testing.B) {
+	d, db := setupBenchDB(b)
+	defer db.Close()
+
+	for b.Loop() {
+		d.WordCount()
+	}
+}
--- a/internal/dictionary/db.go
+++ b/internal/dictionary/db.go
@@ -3,11 +3,12 @@ package dictionary
 import (
 	"database/sql"
 	"fmt"
+	"strings"

 	_ "modernc.org/sqlite"
 )

-const schemaVersion = "1"
+const schemaVersion = "2"

 const schemaSQL = `
 CREATE TABLE IF NOT EXISTS words (
@@ -15,7 +16,8 @@ CREATE TABLE IF NOT EXISTS words (
    word      TEXT    NOT NULL,
    lang      TEXT    NOT NULL,
    pos       TEXT,
-    frequency INTEGER DEFAULT 0,
+    frequency  INTEGER DEFAULT 0,
+    difficulty REAL DEFAULT NULL,
    UNIQUE(word, lang)
 );

@@ -61,6 +63,53 @@ CREATE TABLE IF NOT EXISTS meta (
    key   TEXT PRIMARY KEY,
    value TEXT NOT NULL
 );
+
+CREATE TABLE IF NOT EXISTS antonyms (
+    id       INTEGER PRIMARY KEY AUTOINCREMENT,
+    word_id  INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
+    antonym  TEXT    NOT NULL,
+    source   TEXT    NOT NULL,
+    UNIQUE(word_id, antonym)
+);
+
+CREATE INDEX IF NOT EXISTS idx_antonyms_word_id ON antonyms(word_id);
+
+CREATE TABLE IF NOT EXISTS synsets (
+    id        INTEGER PRIMARY KEY AUTOINCREMENT,
+    synset_id TEXT NOT NULL UNIQUE,
+    pos       TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS word_synsets (
+    word_id   INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
+    synset_id INTEGER NOT NULL REFERENCES synsets(id) ON DELETE CASCADE,
+    source    TEXT NOT NULL,
+    PRIMARY KEY (word_id, synset_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_word_synsets_word_id   ON word_synsets(word_id);
+CREATE INDEX IF NOT EXISTS idx_word_synsets_synset_id ON word_synsets(synset_id);
+
+CREATE TABLE IF NOT EXISTS pronunciations (
+    id       INTEGER PRIMARY KEY AUTOINCREMENT,
+    word_id  INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
+    format   TEXT NOT NULL,
+    value    TEXT NOT NULL,
+    source   TEXT NOT NULL,
+    UNIQUE(word_id, format, source)
+);
+
+CREATE INDEX IF NOT EXISTS idx_pronunciations_word_id ON pronunciations(word_id);
+
+CREATE TABLE IF NOT EXISTS etymology (
+    id         INTEGER PRIMARY KEY AUTOINCREMENT,
+    word_id    INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
+    text       TEXT NOT NULL,
+    source     TEXT NOT NULL,
+    UNIQUE(word_id, source)
+);
+
+CREATE INDEX IF NOT EXISTS idx_etymology_word_id ON etymology(word_id);
 `

 func openDB(dbPath string) (*sql.DB, error) {
@@ -77,11 +126,19 @@ func openDB(dbPath string) (*sql.DB, error) {
 }

 func openReadOnlyDB(dbPath string) (*sql.DB, error) {
-	db, err := sql.Open("sqlite", dbPath+"?mode=ro&_pragma=journal_mode(wal)&_pragma=foreign_keys(on)")
+	db, err := sql.Open("sqlite", dbPath+
+		"?mode=ro"+
+		"&_pragma=journal_mode(wal)"+
+		"&_pragma=foreign_keys(on)"+
+		"&_pragma=cache_size(-64000)"+
+		"&_pragma=mmap_size(268435456)"+
+		"&_pragma=temp_store(memory)")
 	if err != nil {
 		return nil, fmt.Errorf("dictionary: open db (ro): %w", err)
 	}
-	// No MaxOpenConns limit — WAL mode supports concurrent reads
+	// Pin to one connection so pragmas are consistent. WAL mode still
+	// allows concurrent reads from the same connection in Go's model.
+	db.SetMaxOpenConns(1)
 	if err := db.Ping(); err != nil {
 		db.Close()
 		return nil, fmt.Errorf("dictionary: ping db: %w", err)
@@ -94,5 +151,25 @@ func BootstrapSchema(db *sql.DB) error {
 	if err != nil {
 		return fmt.Errorf("dictionary: bootstrap schema: %w", err)
 	}
+
+	// Schema migrations for databases created before v2.
+	// ALTER TABLE ADD COLUMN is a no-op if the column already exists in SQLite
+	// when we catch the "duplicate column" error.
+	migrations := []string{
+		"ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL",
+	}
+	for _, m := range migrations {
+		if _, err := db.Exec(m); err != nil {
+			// Ignore "duplicate column name" — means migration already applied
+			if !isDuplicateColumn(err) {
+				return fmt.Errorf("dictionary: migration: %w", err)
+			}
+		}
+	}
+
 	return nil
 }
+
+func isDuplicateColumn(err error) bool {
+	return err != nil && strings.Contains(err.Error(), "duplicate column")
+}
--- a/internal/dictionary/dict.go
+++ b/internal/dictionary/dict.go
@@ -22,10 +22,12 @@ func ValidLang(lang string) bool {
 }

 type Options struct {
-	MinLength    int
-	MaxLength    int
-	POS          string // "", "noun", "verb", "adjective", "adverb"
-	MinFrequency int    // 0 = no filter
+	MinLength     int
+	MaxLength     int
+	POS           string  // "", "noun", "verb", "adjective", "adverb"
+	MinFrequency  int     // 0 = no filter
+	MinDifficulty float64 // 0.0 = no filter
+	MaxDifficulty float64 // 0.0 = no filter
 }

 type Definition struct {
@@ -35,6 +37,18 @@ type Definition struct {
 	Priority int    `json:"priority"`
 }

+type EnglishEquivalent struct {
+	Word       string `json:"word"`
+	Definition string `json:"definition"`
+	Synset     string `json:"synset"`
+}
+
+type Pronunciation struct {
+	Format string `json:"format"`
+	Value  string `json:"value"`
+	Source string `json:"source"`
+}
+
 type Dictionary struct {
 	db *sql.DB
 }
--- a/internal/dictionary/dict_test.go
+++ b/internal/dictionary/dict_test.go
@@ -89,30 +89,30 @@ func TestRandomWord(t *testing.T) {
 	d := NewFromDB(db)

 	// Basic random word
-	word, err := d.RandomWord("en", Options{})
+	result, err := d.RandomWord("en", Options{})
 	if err != nil {
 		t.Fatalf("RandomWord: %v", err)
 	}
-	if word == "" {
+	if result.Word == "" {
 		t.Error("RandomWord returned empty string")
 	}

 	// With POS filter
-	word, err = d.RandomWord("en", Options{POS: "adjective"})
+	result, err = d.RandomWord("en", Options{POS: "adjective"})
 	if err != nil {
 		t.Fatalf("RandomWord with POS: %v", err)
 	}
-	if word != "happy" {
-		t.Errorf("RandomWord(adjective) = %q, want happy", word)
+	if result.Word != "happy" {
+		t.Errorf("RandomWord(adjective) = %q, want happy", result.Word)
 	}

 	// With length filters
-	word, err = d.RandomWord("en", Options{MinLength: 4, MaxLength: 5})
+	result, err = d.RandomWord("en", Options{MinLength: 4, MaxLength: 5})
 	if err != nil {
 		t.Fatalf("RandomWord with length: %v", err)
 	}
-	if len(word) < 4 || len(word) > 5 {
-		t.Errorf("RandomWord length %d not in [4,5]", len(word))
+	if len(result.Word) < 4 || len(result.Word) > 5 {
+		t.Errorf("RandomWord length %d not in [4,5]", len(result.Word))
 	}

 	// No match
--- a/internal/dictionary/query.go
+++ b/internal/dictionary/query.go
@@ -18,8 +18,13 @@ func (d *Dictionary) IsValidWord(word, lang string) (bool, error) {
 	return exists, nil
 }

-func (d *Dictionary) RandomWord(lang string, opts Options) (string, error) {
-	query := "SELECT word FROM words WHERE lang = ?"
+type RandomResult struct {
+	Word       string   `json:"word"`
+	Difficulty *float64 `json:"difficulty,omitempty"`
+}
+
+func (d *Dictionary) RandomWord(lang string, opts Options) (RandomResult, error) {
+	query := "SELECT word, difficulty FROM words WHERE lang = ?"
 	args := []any{lang}

 	if opts.POS != "" {
@@ -38,18 +43,30 @@ func (d *Dictionary) RandomWord(lang string, opts Options) (string, error) {
 		query += " AND frequency >= ?"
 		args = append(args, opts.MinFrequency)
 	}
+	if opts.MinDifficulty > 0 {
+		query += " AND difficulty >= ?"
+		args = append(args, opts.MinDifficulty)
+	}
+	if opts.MaxDifficulty > 0 {
+		query += " AND difficulty <= ?"
+		args = append(args, opts.MaxDifficulty)
+	}

 	query += " ORDER BY RANDOM() LIMIT 1"

-	var word string
-	err := d.db.QueryRow(query, args...).Scan(&word)
+	var result RandomResult
+	var diff sql.NullFloat64
+	err := d.db.QueryRow(query, args...).Scan(&result.Word, &diff)
 	if err == sql.ErrNoRows {
-		return "", ErrNoMatch
+		return RandomResult{}, ErrNoMatch
 	}
 	if err != nil {
-		return "", fmt.Errorf("dictionary: random word: %w", err)
+		return RandomResult{}, fmt.Errorf("dictionary: random word: %w", err)
 	}
-	return word, nil
+	if diff.Valid {
+		result.Difficulty = &diff.Float64
+	}
+	return result, nil
 }

 func (d *Dictionary) Define(word, lang string) ([]Definition, error) {
@@ -175,6 +192,44 @@ func (d *Dictionary) Frequency(word, lang string) (int, error) {
 	return freq, nil
 }

+// FrequencyBatch returns frequency scores for multiple words in a language.
+// Words not found or without frequency data are returned with a score of 0.
+func (d *Dictionary) FrequencyBatch(words []string, lang string) (map[string]int, error) {
+	if len(words) == 0 {
+		return map[string]int{}, nil
+	}
+
+	placeholders := make([]string, len(words))
+	args := make([]any, 0, len(words)+1)
+	args = append(args, lang)
+	for i, w := range words {
+		placeholders[i] = "?"
+		args = append(args, strings.ToLower(w))
+	}
+
+	query := fmt.Sprintf(
+		"SELECT word, COALESCE(frequency, 0) FROM words WHERE lang = ? AND word IN (%s)",
+		strings.Join(placeholders, ","),
+	)
+
+	rows, err := d.db.Query(query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("dictionary: frequency batch: %w", err)
+	}
+	defer rows.Close()
+
+	result := make(map[string]int, len(words))
+	for rows.Next() {
+		var word string
+		var freq int
+		if err := rows.Scan(&word, &freq); err != nil {
+			return nil, fmt.Errorf("dictionary: frequency batch scan: %w", err)
+		}
+		result[word] = freq
+	}
+	return result, rows.Err()
+}
+
 func (d *Dictionary) Meta(key string) (string, error) {
 	var val string
 	err := d.db.QueryRow("SELECT value FROM meta WHERE key = ?", key).Scan(&val)
@@ -187,6 +242,143 @@ func (d *Dictionary) Meta(key string) (string, error) {
 	return val, nil
 }

+func (d *Dictionary) Antonyms(word, lang string) ([]string, error) {
+	rows, err := d.db.Query(`
+		SELECT DISTINCT a.antonym
+		FROM antonyms a
+		JOIN words w ON w.id = a.word_id
+		WHERE w.word = ? AND w.lang = ?
+		ORDER BY a.antonym`,
+		strings.ToLower(word), lang,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("dictionary: antonyms: %w", err)
+	}
+	defer rows.Close()
+
+	var ants []string
+	for rows.Next() {
+		var a string
+		if err := rows.Scan(&a); err != nil {
+			return nil, fmt.Errorf("dictionary: antonyms scan: %w", err)
+		}
+		ants = append(ants, a)
+	}
+	if ants == nil {
+		ants = []string{}
+	}
+	return ants, rows.Err()
+}
+
+// EnglishBacking returns English words that share a synset with the given
+// non-English word, providing semantic equivalents with WordNet definitions.
+func (d *Dictionary) EnglishBacking(word, lang string) ([]EnglishEquivalent, error) {
+	// Find synsets for this word, then find English words in those synsets,
+	// along with their WordNet definitions.
+	rows, err := d.db.Query(`
+		SELECT DISTINCT ew.word,
+			COALESCE(
+				(SELECT d.gloss FROM definitions d
+				 WHERE d.word_id = ew.id AND d.source = 'wordnet' AND d.pos = s.pos
+				 ORDER BY d.priority ASC LIMIT 1),
+				(SELECT d.gloss FROM definitions d
+				 WHERE d.word_id = ew.id AND d.source = 'wordnet'
+				 ORDER BY d.priority ASC LIMIT 1),
+				''
+			) AS gloss,
+			s.synset_id
+		FROM words w
+		JOIN word_synsets ws ON ws.word_id = w.id
+		JOIN synsets s ON s.id = ws.synset_id
+		JOIN word_synsets ews ON ews.synset_id = s.id
+		JOIN words ew ON ew.id = ews.word_id AND ew.lang = 'en'
+		WHERE w.word = ? AND w.lang = ?
+		ORDER BY s.synset_id, ew.word`,
+		strings.ToLower(word), lang,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("dictionary: english backing: %w", err)
+	}
+	defer rows.Close()
+
+	var results []EnglishEquivalent
+	seen := make(map[string]bool)
+	for rows.Next() {
+		var eq EnglishEquivalent
+		if err := rows.Scan(&eq.Word, &eq.Definition, &eq.Synset); err != nil {
+			return nil, fmt.Errorf("dictionary: english backing scan: %w", err)
+		}
+		key := eq.Word + "|" + eq.Synset
+		if !seen[key] {
+			seen[key] = true
+			results = append(results, eq)
+		}
+	}
+	if results == nil {
+		results = []EnglishEquivalent{}
+	}
+
+	// Fallback to translations table if no synset matches
+	if len(results) == 0 {
+		trans, err := d.Translate(word, lang, "en")
+		if err != nil {
+			return nil, err
+		}
+		for _, t := range trans {
+			results = append(results, EnglishEquivalent{Word: t})
+		}
+	}
+
+	return results, rows.Err()
+}
+
+func (d *Dictionary) Pronunciation(word, lang string) ([]Pronunciation, error) {
+	rows, err := d.db.Query(`
+		SELECT p.format, p.value, p.source
+		FROM pronunciations p
+		JOIN words w ON w.id = p.word_id
+		WHERE w.word = ? AND w.lang = ?
+		ORDER BY p.source, p.format`,
+		strings.ToLower(word), lang,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("dictionary: pronunciation: %w", err)
+	}
+	defer rows.Close()
+
+	var prons []Pronunciation
+	for rows.Next() {
+		var p Pronunciation
+		if err := rows.Scan(&p.Format, &p.Value, &p.Source); err != nil {
+			return nil, fmt.Errorf("dictionary: pronunciation scan: %w", err)
+		}
+		prons = append(prons, p)
+	}
+	if prons == nil {
+		prons = []Pronunciation{}
+	}
+	return prons, rows.Err()
+}
+
+func (d *Dictionary) Etymology(word, lang string) (string, error) {
+	var text string
+	err := d.db.QueryRow(`
+		SELECT e.text
+		FROM etymology e
+		JOIN words w ON w.id = e.word_id
+		WHERE w.word = ? AND w.lang = ?
+		LIMIT 1`,
+		strings.ToLower(word), lang,
+	).Scan(&text)
+	if err == sql.ErrNoRows {
+		return "", nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("dictionary: etymology: %w", err)
+	}
+	return text, nil
+}
+
 func (d *Dictionary) DefCount() (map[string]int, error) {
 	rows, err := d.db.Query(`
 		SELECT w.lang, COUNT(*)