diff --git a/README.md b/README.md index c81783b..bd0f8ac 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # DreamDict -A self-contained HTTP dictionary service providing word validation, random word generation, definitions, synonyms, and cross-language translation for English (`en`), French (`fr`), and European Portuguese (`pt-PT`). +A self-contained HTTP dictionary service providing word validation, random word generation, definitions, synonyms, antonyms, translations, pronunciation, etymology, and cross-language semantic backing for English (`en`), French (`fr`), European Portuguese (`pt-PT`), and Mandarin Chinese (`zh`). Runs as a plain Go binary managed by systemd. Binds to localhost only — no external network exposure. Designed as a dictionary backend for [GogoBee](https://github.com/reala-misaki/gogobee), replacing Wordnik. @@ -30,20 +30,20 @@ All endpoints are read-only `GET` requests returning JSON. ### `GET /valid?word=...&lang=...` -Check if a word exists in a language's word list. +Check if a word exists in a language's word list. Includes inflected forms via affix expansion. ``` -$ curl 'localhost:7777/valid?word=ephemeral&lang=en' +$ curl 'localhost:7777/valid?word=running&lang=en' {"valid":true} ``` ### `GET /random?lang=...` -Return a random word. Optional filters: `pos` (`noun`, `verb`, `adjective`, `adverb`), `min`/`max` (word length), `min_freq` (frequency score, meaningful for `fr`). +Return a random word. Optional filters: `pos` (`noun`, `verb`, `adjective`, `adverb`), `min`/`max` (word length), `min_freq` (frequency score), `min_difficulty`/`max_difficulty` (0.0–1.0). ``` $ curl 'localhost:7777/random?lang=en&pos=noun&min=5&max=8' -{"word":"lantern"} +{"word":"lantern","difficulty":0.42} ``` ### `GET /define?word=...&lang=...` @@ -67,6 +67,15 @@ $ curl 'localhost:7777/synonyms?word=happy&lang=en' {"word":"happy","lang":"en","synonyms":["content","glad","joyful","pleased"]} ``` +### `GET /antonyms?word=...&lang=...` + +Return antonyms sourced from WordNet and Wiktionary. + +``` +$ curl 'localhost:7777/antonyms?word=happy&lang=en' +{"word":"happy","lang":"en","antonyms":["sad","unhappy"]} +``` + ### `GET /translate?word=...&from=...&to=...` Return known translations between supported languages. @@ -76,6 +85,56 @@ $ curl 'localhost:7777/translate?word=cat&from=en&to=fr' {"word":"cat","from":"en","to":"fr","translations":["chat","minet"]} ``` +### `GET /backing?word=...&lang=...` + +Return English semantic equivalents for a non-English word via shared WordNet synset IDs. Falls back to the translations table when no synset mapping exists. + +``` +$ curl 'localhost:7777/backing?word=éphémère&lang=fr' +{"word":"éphémère","lang":"fr","equivalents":[ + {"word":"ephemeral","definition":"lasting a very short time","synset":"00914031-a"} +]} +``` + +### `GET /pronunciation?word=...&lang=...` + +Return pronunciation data (CMU phonemes for English, IPA from Wiktionary for all languages). + +``` +$ curl 'localhost:7777/pronunciation?word=ephemeral&lang=en' +{"word":"ephemeral","lang":"en","pronunciations":[ + {"format":"cmu","value":"IH0 F EH1 M ER0 AH0 L","source":"cmudict"}, + {"format":"ipa","value":"ɪˈfɛm.ər.əl","source":"wiktionary"} +]} +``` + +### `GET /etymology?word=...&lang=...` + +Return the etymology of a word from Wiktionary. + +``` +$ curl 'localhost:7777/etymology?word=ephemeral&lang=en' +{"word":"ephemeral","lang":"en","etymology":"From Medieval Latin ephemerus, from Ancient Greek ἐφήμερος (ephḗmeros, \"lasting only a day\")..."} +``` + +### `GET /frequency?word=...&lang=...` + +Return the frequency score for a word. Higher values = more common. + +``` +$ curl 'localhost:7777/frequency?word=house&lang=en' +{"word":"house","lang":"en","frequency":800} +``` + +### `GET /frequency/batch?words=...&lang=...` + +Batch frequency lookup. Comma-separated words, max 100. + +``` +$ curl 'localhost:7777/frequency/batch?words=house,cat,ephemeral&lang=en' +{"lang":"en","frequencies":{"cat":800,"ephemeral":50,"house":800}} +``` + ### `GET /health` Health check with database stats. @@ -113,26 +172,81 @@ go run ./cmd/dictimport [flags] |---|---|---| | `--db` | `./dict.db` | Output database path | | `--data` | `./data` | Source data directory | -| `--langs` | `en,fr,pt-PT` | Languages to import | +| `--langs` | `en,fr,pt-PT,zh` | Languages to import | | `--clean` | `false` | Drop and recreate all tables first | | `--skip` | | Comma-separated loader names to skip | -Loader names: `scowl`, `wordnet`, `wiktionary-en`, `hunspell-fr`, `lexique`, `wolf`, `wiktionary-fr`, `hunspell-pt`, `dicionario`, `wiktionary-pt`. +### Loader Names + +| Loader | Language | What it does | +|---|---|---| +| `scowl` | en | English word list with frequency tiers | +| `affix-en` | en | English inflected form expansion | +| `wordnet` | en | Definitions, synonyms, antonyms, synset mappings | +| `subtlex` | en | SUBTLEX-US frequency enrichment | +| `cmudict` | en | CMU pronunciation data | +| `wiktionary-en` | en | Supplemental definitions, synonyms, antonyms, translations, IPA, etymology | +| `hunspell-fr` | fr | French word list | +| `affix-fr` | fr | French inflected form expansion | +| `lexique` | fr | French POS + frequency enrichment | +| `wolf` | fr | French definitions, synonyms, synset mappings | +| `wiktionary-fr` | fr | Supplemental definitions, synonyms, antonyms, translations, IPA, etymology | +| `hunspell-pt` | pt-PT | Portuguese word list | +| `affix-pt-PT` | pt-PT | Portuguese inflected form expansion | +| `dicionario` | pt-PT | Portuguese definitions | +| `cetempublico` | pt-PT | Portuguese frequency enrichment | +| `omw-pt` | pt-PT | Open Multilingual Wordnet synset mappings | +| `wiktionary-pt` | pt-PT | Supplemental definitions, synonyms, antonyms, translations, IPA, etymology | +| `cedict` | zh | Chinese words, definitions, translations | +| `wiktionary-zh` | zh | Supplemental Chinese data | +| `difficulty` | all | Composite difficulty scoring (runs last) | ## Data Sources | Source | Language | What it provides | |---|---|---| -| [SCOWL](https://wordlist.aspell.net/) | en | Word list | -| [WordNet](https://wordnet.princeton.edu/) | en | Definitions + synonyms | -| [Hunspell/LibreOffice](https://github.com/LibreOffice/dictionaries) | fr, pt-PT | Word lists | +| [SCOWL](https://wordlist.aspell.net/) | en | Word list + frequency tiers | +| [WordNet](https://wordnet.princeton.edu/) | en | Definitions, synonyms, antonyms, synset IDs | +| [SUBTLEX-US](http://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexus) | en | Subtitle-based word frequency | +| [CMU Pronouncing Dictionary](http://www.speech.cs.cmu.edu/cgi-bin/cmudict) | en | Phoneme-based pronunciation | +| [Hunspell/LibreOffice](https://github.com/LibreOffice/dictionaries) | fr, pt-PT | Word lists + affix rules | | [Lexique](http://www.lexique.org/) | fr | POS + frequency enrichment | -| [WOLF](https://github.com/nicolashernandez/WOLF) | fr | Definitions + synonyms | +| [WOLF](https://github.com/nicolashernandez/WOLF) | fr | Definitions, synonyms, synset IDs | | [Dicionario Aberto](https://github.com/jorgecardoso/dicionario-aberto) | pt-PT | Definitions | -| [Wiktionary (kaikki.org)](https://kaikki.org/) | all | Supplemental definitions, synonyms, translations | +| [CETEMPúblico](https://www.linguateca.pt/acesso/corpus.php?corpus=CETEMPUBLICO) | pt-PT | Word frequency | +| [Open Multilingual Wordnet](https://github.com/omwn/omw-data) | pt-PT | Synset mappings to Princeton WordNet | +| [CC-CEDICT](https://cc-cedict.org/) | zh | Words, definitions, translations | +| [Wiktionary (kaikki.org)](https://kaikki.org/) | all | Definitions, synonyms, antonyms, translations, IPA pronunciation, etymology | Run `./scripts/download-dict-data.sh` to fetch everything. Pass `--force` to re-download existing files. +## Database Schema (v2) + +### Tables + +| Table | Purpose | +|---|---| +| `words` | Core word inventory (word, lang, pos, frequency, difficulty) | +| `definitions` | Word definitions with source priority | +| `synonyms` | Synonym relationships | +| `antonyms` | Antonym relationships | +| `translations` | Cross-language translations | +| `synsets` | Princeton WordNet synset IDs | +| `word_synsets` | Maps words to synsets (for cross-language semantic backing) | +| `pronunciations` | CMU phonemes and IPA data | +| `etymology` | Word origin text from Wiktionary | +| `meta` | Import metadata (imported_at, schema_version) | + +### Difficulty Scoring + +Each word receives a composite difficulty score (0.0 = easiest, 1.0 = hardest) computed at import time: + +``` +difficulty = inverse_frequency * 0.5 + word_length * 0.3 + syllable_count * 0.2 +``` + +Use `min_difficulty` / `max_difficulty` on `/random` to select words by difficulty tier. + ## Deployment A systemd unit file is provided at `deploy/dreamdict.service`. @@ -173,20 +287,23 @@ go test ./... ## Project Structure ``` -cmd/server/ HTTP server -cmd/dictimport/ One-time data import CLI +cmd/server/ HTTP server (12 endpoints) +cmd/dictimport/ One-time data import CLI internal/dictionary/ Core package: DB schema, types, queries internal/loader/ Data source parsers (one per source) -scripts/ Download script for source data -deploy/ systemd unit file -data/ Source data files (gitignored) +scripts/ Download script for source data +deploy/ systemd unit file +data/ Source data files (gitignored) ``` ## Expected Database Size | Language | Words | Definitions | Size | |---|---|---|---| -| en | ~220k | ~380k | ~180 MB | -| fr | ~90k | ~200k | ~80 MB | -| pt-PT | ~75k | ~120k | ~50 MB | -| **Total** | | | **~310 MB** | +| en | ~220k+ | ~380k | ~180 MB | +| fr | ~90k+ | ~200k | ~80 MB | +| pt-PT | ~75k+ | ~120k | ~50 MB | +| zh | varies | ~80k | ~20 MB | +| **Total** | | | **~330 MB** | + +Word counts are higher than base forms due to affix expansion of inflected forms. diff --git a/cmd/dictimport/main.go b/cmd/dictimport/main.go index 58b85a9..9a36e49 100644 --- a/cmd/dictimport/main.go +++ b/cmd/dictimport/main.go @@ -25,7 +25,13 @@ func main() { slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))) - db, err := sql.Open("sqlite", *dbPath+"?_pragma=journal_mode(wal)&_pragma=foreign_keys(on)") + db, err := sql.Open("sqlite", *dbPath+ + "?_pragma=journal_mode(wal)"+ + "&_pragma=foreign_keys(on)"+ + "&_pragma=synchronous(off)"+ + "&_pragma=cache_size(-64000)"+ + "&_pragma=mmap_size(268435456)"+ + "&_pragma=temp_store(memory)") if err != nil { slog.Error("open db", "error", err) os.Exit(1) @@ -35,7 +41,7 @@ func main() { if *clean { slog.Info("dropping all tables") - for _, table := range []string{"translations", "synonyms", "definitions", "words", "meta"} { + for _, table := range []string{"etymology", "pronunciations", "word_synsets", "synsets", "antonyms", "translations", "synonyms", "definitions", "words", "meta"} { if _, err := db.Exec("DROP TABLE IF EXISTS " + table); err != nil { slog.Error("drop table", "table", table, "error", err) os.Exit(1) @@ -65,7 +71,10 @@ func main() { if langSet["en"] { loaders = append(loaders, loader.SCOWLLoader{}, + loader.EnglishAffixLoader{}, loader.WordNetLoader{}, + loader.SubtlexLoader{}, + loader.CMUDictLoader{}, loader.WiktionaryLoader{Lang: "en", FileName: "kaikki-en.jsonl"}, ) } @@ -73,6 +82,7 @@ func main() { if langSet["fr"] { loaders = append(loaders, loader.HunspellLoader{Lang: "fr", FileName: "fr_FR/fr.dic"}, + loader.AffixLoader{Lang: "fr", DicFile: "fr_FR/fr.dic", AffFile: "fr_FR/fr.aff"}, loader.LexiqueLoader{}, loader.WOLFLoader{}, loader.WiktionaryLoader{Lang: "fr", FileName: "kaikki-fr.jsonl"}, @@ -82,7 +92,10 @@ func main() { if langSet["pt-PT"] { loaders = append(loaders, loader.HunspellLoader{Lang: "pt-PT", FileName: "pt_PT/pt_PT.dic"}, + loader.AffixLoader{Lang: "pt-PT", DicFile: "pt_PT/pt_PT.dic", AffFile: "pt_PT/pt_PT.aff"}, loader.DicionarioLoader{}, + loader.CETEMPublicoLoader{}, + loader.OMWLoader{}, loader.WiktionaryLoader{Lang: "pt-PT", FileName: "kaikki-pt.jsonl"}, ) } @@ -94,6 +107,9 @@ func main() { ) } + // Difficulty scorer runs last, after all word data is loaded + loaders = append(loaders, loader.DifficultyScorer{}) + start := time.Now() if err := loader.Run(db, *dataDir, loaders, skipSet); err != nil { slog.Error("import failed", "error", err) @@ -104,7 +120,7 @@ func main() { now := time.Now().UTC().Format(time.RFC3339) for _, kv := range [][2]string{ {"imported_at", now}, - {"schema_version", "1"}, + {"schema_version", "2"}, } { if _, err := db.Exec( "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", kv[0], kv[1], diff --git a/cmd/server/bench_test.go b/cmd/server/bench_test.go new file mode 100644 index 0000000..3588700 --- /dev/null +++ b/cmd/server/bench_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "database/sql" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "dreamdict/internal/dictionary" + + _ "modernc.org/sqlite" +) + +func setupBenchServer(b *testing.B) *http.ServeMux { + b.Helper() + db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(on)") + if err != nil { + b.Fatal(err) + } + db.SetMaxOpenConns(1) + if err := dictionary.BootstrapSchema(db); err != nil { + b.Fatal(err) + } + + exec := func(q string, args ...any) { + if _, err := db.Exec(q, args...); err != nil { + b.Fatalf("seed: %s: %v", q, err) + } + } + + exec("INSERT INTO meta (key, value) VALUES ('schema_version', '1')") + exec("INSERT INTO meta (key, value) VALUES ('imported_at', '2025-01-01T00:00:00Z')") + + poses := []string{"noun", "verb", "adjective", "adverb"} + for i := 1; i <= 500; i++ { + pos := poses[i%len(poses)] + word := fmt.Sprintf("word%04d", i) + exec("INSERT INTO words (id, word, lang, pos, frequency) VALUES (?, ?, 'en', ?, ?)", + i, word, pos, i%100) + exec("INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (?, ?, ?, 'wordnet', 10)", + i, pos, fmt.Sprintf("def of %s", word)) + exec("INSERT INTO synonyms (word_id, synonym, source) VALUES (?, ?, 'wordnet')", + i, fmt.Sprintf("syn_%s", word)) + exec("INSERT INTO translations (word_id, translation, target_lang, source) VALUES (?, ?, 'fr', 'wiktionary')", + i, fmt.Sprintf("fr_%s", word)) + } + + dict := dictionary.NewFromDB(db) + b.Cleanup(func() { db.Close() }) + + mux := http.NewServeMux() + mux.HandleFunc("GET /valid", handleValid(dict)) + mux.HandleFunc("GET /random", handleRandom(dict)) + mux.HandleFunc("GET /define", handleDefine(dict)) + mux.HandleFunc("GET /synonyms", handleSynonyms(dict)) + mux.HandleFunc("GET /translate", handleTranslate(dict)) + mux.HandleFunc("GET /health", handleHealth(dict, ":memory:")) + return mux +} + +func benchRequest(b *testing.B, mux *http.ServeMux, path string) { + b.Helper() + req := httptest.NewRequest("GET", path, nil) + b.ResetTimer() + for b.Loop() { + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + if w.Code != 200 { + b.Fatalf("status %d for %s", w.Code, path) + } + } +} + +func BenchmarkHTTPValid(b *testing.B) { + mux := setupBenchServer(b) + benchRequest(b, mux, "/valid?word=word0003&lang=en") +} + +func BenchmarkHTTPRandom(b *testing.B) { + mux := setupBenchServer(b) + benchRequest(b, mux, "/random?lang=en") +} + +func BenchmarkHTTPRandomFiltered(b *testing.B) { + mux := setupBenchServer(b) + benchRequest(b, mux, "/random?lang=en&pos=noun&min=5&max=10") +} + +func BenchmarkHTTPDefine(b *testing.B) { + mux := setupBenchServer(b) + benchRequest(b, mux, "/define?word=word0003&lang=en") +} + +func BenchmarkHTTPSynonyms(b *testing.B) { + mux := setupBenchServer(b) + benchRequest(b, mux, "/synonyms?word=word0003&lang=en") +} + +func BenchmarkHTTPTranslate(b *testing.B) { + mux := setupBenchServer(b) + benchRequest(b, mux, "/translate?word=word0003&from=en&to=fr") +} + +func BenchmarkHTTPHealth(b *testing.B) { + mux := setupBenchServer(b) + benchRequest(b, mux, "/health") +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 457052d..e32b1c5 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -80,9 +80,13 @@ func main() { mux.HandleFunc("GET /random", handleRandom(dict)) mux.HandleFunc("GET /define", handleDefine(dict)) mux.HandleFunc("GET /synonyms", handleSynonyms(dict)) + mux.HandleFunc("GET /antonyms", handleAntonyms(dict)) mux.HandleFunc("GET /translate", handleTranslate(dict)) + mux.HandleFunc("GET /backing", handleBacking(dict)) mux.HandleFunc("GET /frequency", handleFrequency(dict)) mux.HandleFunc("GET /frequency/batch", handleFrequencyBatch(dict)) + mux.HandleFunc("GET /pronunciation", handlePronunciation(dict)) + mux.HandleFunc("GET /etymology", handleEtymology(dict)) mux.HandleFunc("GET /health", handleHealth(dict, *dbPath)) serverStart = time.Now() @@ -220,8 +224,18 @@ func handleRandom(dict *dictionary.Dictionary) http.HandlerFunc { opts.MinFrequency = n } } + if v := r.URL.Query().Get("min_difficulty"); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + opts.MinDifficulty = f + } + } + if v := r.URL.Query().Get("max_difficulty"); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + opts.MaxDifficulty = f + } + } - word, err := dict.RandomWord(lang, opts) + result, err := dict.RandomWord(lang, opts) if err == dictionary.ErrNoMatch { writeError(w, 404, "no_match", "no words match the given filters") return @@ -231,7 +245,11 @@ func handleRandom(dict *dictionary.Dictionary) http.HandlerFunc { writeError(w, 500, "internal", "internal error") return } - writeJSON(w, 200, map[string]string{"word": word}) + resp := map[string]any{"word": result.Word} + if result.Difficulty != nil { + resp["difficulty"] = *result.Difficulty + } + writeJSON(w, 200, resp) } } @@ -381,8 +399,125 @@ func handleFrequencyBatch(dict *dictionary.Dictionary) http.HandlerFunc { } } -func handleHealth(dict *dictionary.Dictionary, dbPath string) http.HandlerFunc { +func handleAntonyms(dict *dictionary.Dictionary) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + word := r.URL.Query().Get("word") + lang := r.URL.Query().Get("lang") + if word == "" || lang == "" { + writeError(w, 400, "bad_request", "word and lang are required") + return + } + if !dictionary.ValidLang(lang) { + writeError(w, 400, "bad_request", "unsupported language: "+lang) + return + } + + ants, err := dict.Antonyms(word, lang) + if err != nil { + slog.Error("antonyms", "error", err) + writeError(w, 500, "internal", "internal error") + return + } + writeJSON(w, 200, map[string]any{ + "word": word, + "lang": lang, + "antonyms": ants, + }) + } +} + +func handleBacking(dict *dictionary.Dictionary) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + word := r.URL.Query().Get("word") + lang := r.URL.Query().Get("lang") + if word == "" || lang == "" { + writeError(w, 400, "bad_request", "word and lang are required") + return + } + if !dictionary.ValidLang(lang) { + writeError(w, 400, "bad_request", "unsupported language: "+lang) + return + } + + equivs, err := dict.EnglishBacking(word, lang) + if err != nil { + slog.Error("backing", "error", err) + writeError(w, 500, "internal", "internal error") + return + } + writeJSON(w, 200, map[string]any{ + "word": word, + "lang": lang, + "equivalents": equivs, + }) + } +} + +func handlePronunciation(dict *dictionary.Dictionary) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + word := r.URL.Query().Get("word") + lang := r.URL.Query().Get("lang") + if word == "" || lang == "" { + writeError(w, 400, "bad_request", "word and lang are required") + return + } + if !dictionary.ValidLang(lang) { + writeError(w, 400, "bad_request", "unsupported language: "+lang) + return + } + + prons, err := dict.Pronunciation(word, lang) + if err != nil { + slog.Error("pronunciation", "error", err) + writeError(w, 500, "internal", "internal error") + return + } + writeJSON(w, 200, map[string]any{ + "word": word, + "lang": lang, + "pronunciations": prons, + }) + } +} + +func handleEtymology(dict *dictionary.Dictionary) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + word := r.URL.Query().Get("word") + lang := r.URL.Query().Get("lang") + if word == "" || lang == "" { + writeError(w, 400, "bad_request", "word and lang are required") + return + } + if !dictionary.ValidLang(lang) { + writeError(w, 400, "bad_request", "unsupported language: "+lang) + return + } + + etym, err := dict.Etymology(word, lang) + if err != nil { + slog.Error("etymology", "error", err) + writeError(w, 500, "internal", "internal error") + return + } + writeJSON(w, 200, map[string]any{ + "word": word, + "lang": lang, + "etymology": etym, + }) + } +} + +func handleHealth(dict *dictionary.Dictionary, dbPath string) http.HandlerFunc { + // The database is read-only at runtime, so cache the health response + // after the first (slow) computation. + var cached atomic.Value + + return func(w http.ResponseWriter, r *http.Request) { + if v := cached.Load(); v != nil { + writeJSON(w, 200, v) + return + } + wordCounts, err := dict.WordCount() if err != nil { slog.Error("health: word count", "error", err) @@ -409,13 +544,15 @@ func handleHealth(dict *dictionary.Dictionary, dbPath string) http.HandlerFunc { return } - writeJSON(w, 200, map[string]any{ + resp := map[string]any{ "status": "ok", "db_path": dbPath, "word_counts": wordCounts, "def_counts": defCounts, "imported_at": importedAt, "schema_version": schemaVersion, - }) + } + cached.Store(resp) + writeJSON(w, 200, resp) } } diff --git a/cmd/server/server_test.go b/cmd/server/server_test.go new file mode 100644 index 0000000..dc88eb9 --- /dev/null +++ b/cmd/server/server_test.go @@ -0,0 +1,226 @@ +package main + +import ( + "database/sql" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "dreamdict/internal/dictionary" + + _ "modernc.org/sqlite" +) + +func mustExec(t *testing.T, db *sql.DB, query string, args ...any) { + t.Helper() + if _, err := db.Exec(query, args...); err != nil { + t.Fatalf("mustExec: %s: %v", query, err) + } +} + +func setupTestServer(t *testing.T) (*dictionary.Dictionary, *http.ServeMux) { + t.Helper() + db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(on)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + if err := dictionary.BootstrapSchema(db); err != nil { + t.Fatal(err) + } + + // Seed data + mustExec(t, db, "INSERT INTO meta (key, value) VALUES ('schema_version', '1')") + mustExec(t, db, "INSERT INTO meta (key, value) VALUES ('imported_at', '2025-01-01T00:00:00Z')") + mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (1, 'happy', 'en', 'adjective', 0)") + mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (2, 'cat', 'en', 'noun', 0)") + mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (3, 'chat', 'fr', 'noun', 5000)") + mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling pleasure', 'wordnet', 10)") + mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling joy', 'wiktionary', 20)") + mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'glad', 'wordnet')") + mustExec(t, db, "INSERT INTO translations (word_id, translation, target_lang, source) VALUES (2, 'chat', 'fr', 'wiktionary')") + + dict := dictionary.NewFromDB(db) + + mux := http.NewServeMux() + mux.HandleFunc("GET /valid", handleValid(dict)) + mux.HandleFunc("GET /random", handleRandom(dict)) + mux.HandleFunc("GET /define", handleDefine(dict)) + mux.HandleFunc("GET /synonyms", handleSynonyms(dict)) + mux.HandleFunc("GET /translate", handleTranslate(dict)) + mux.HandleFunc("GET /health", handleHealth(dict, ":memory:")) + + t.Cleanup(func() { db.Close() }) + return dict, mux +} + +func TestValidEndpoint(t *testing.T) { + _, mux := setupTestServer(t) + srv := httptest.NewServer(mux) + defer srv.Close() + + // Valid word + resp, _ := http.Get(srv.URL + "/valid?word=happy&lang=en") + if resp.StatusCode != 200 { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + var result map[string]bool + json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if !result["valid"] { + t.Error("expected valid=true") + } + + // Invalid word + resp, _ = http.Get(srv.URL + "/valid?word=zzzzz&lang=en") + json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if result["valid"] { + t.Error("expected valid=false") + } + + // Missing params + resp, _ = http.Get(srv.URL + "/valid?word=happy") + if resp.StatusCode != 400 { + t.Errorf("missing lang: status = %d, want 400", resp.StatusCode) + } + resp.Body.Close() +} + +func TestRandomEndpoint(t *testing.T) { + _, mux := setupTestServer(t) + srv := httptest.NewServer(mux) + defer srv.Close() + + resp, _ := http.Get(srv.URL + "/random?lang=en") + if resp.StatusCode != 200 { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + var result map[string]string + json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if result["word"] == "" { + t.Error("expected non-empty word") + } + + // No match + resp, _ = http.Get(srv.URL + "/random?lang=en&min=100") + if resp.StatusCode != 404 { + t.Errorf("no match: status = %d, want 404", resp.StatusCode) + } + resp.Body.Close() +} + +func TestDefineEndpoint(t *testing.T) { + _, mux := setupTestServer(t) + srv := httptest.NewServer(mux) + defer srv.Close() + + resp, _ := http.Get(srv.URL + "/define?word=happy&lang=en") + if resp.StatusCode != 200 { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + var result struct { + Word string `json:"word"` + Lang string `json:"lang"` + Definitions []dictionary.Definition `json:"definitions"` + } + json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if len(result.Definitions) != 2 { + t.Errorf("got %d definitions, want 2", len(result.Definitions)) + } + if result.Definitions[0].Priority > result.Definitions[1].Priority { + t.Error("definitions not ordered by priority") + } + + // Word with no definitions + resp, _ = http.Get(srv.URL + "/define?word=cat&lang=en") + if resp.StatusCode != 200 { + t.Fatalf("no defs: status = %d, want 200", resp.StatusCode) + } + json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if len(result.Definitions) != 0 { + t.Errorf("got %d definitions, want 0", len(result.Definitions)) + } +} + +func TestSynonymsEndpoint(t *testing.T) { + _, mux := setupTestServer(t) + srv := httptest.NewServer(mux) + defer srv.Close() + + resp, _ := http.Get(srv.URL + "/synonyms?word=happy&lang=en") + if resp.StatusCode != 200 { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + var result struct { + Synonyms []string `json:"synonyms"` + } + json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if len(result.Synonyms) != 1 { + t.Errorf("got %d synonyms, want 1", len(result.Synonyms)) + } + + // No synonyms + resp, _ = http.Get(srv.URL + "/synonyms?word=cat&lang=en") + json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if len(result.Synonyms) != 0 { + t.Errorf("got %d synonyms, want 0", len(result.Synonyms)) + } +} + +func TestTranslateEndpoint(t *testing.T) { + _, mux := setupTestServer(t) + srv := httptest.NewServer(mux) + defer srv.Close() + + resp, _ := http.Get(srv.URL + "/translate?word=cat&from=en&to=fr") + if resp.StatusCode != 200 { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + var result struct { + Translations []string `json:"translations"` + } + json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if len(result.Translations) != 1 || result.Translations[0] != "chat" { + t.Errorf("got %v, want [chat]", result.Translations) + } + + // No translations + resp, _ = http.Get(srv.URL + "/translate?word=happy&from=en&to=fr") + json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if len(result.Translations) != 0 { + t.Errorf("got %d, want 0", len(result.Translations)) + } +} + +func TestHealthEndpoint(t *testing.T) { + _, mux := setupTestServer(t) + srv := httptest.NewServer(mux) + defer srv.Close() + + resp, _ := http.Get(srv.URL + "/health") + if resp.StatusCode != 200 { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + var result map[string]any + json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if result["status"] != "ok" { + t.Errorf("status = %v, want ok", result["status"]) + } + counts, ok := result["word_counts"].(map[string]any) + if !ok { + t.Fatal("word_counts not a map") + } + if counts["en"].(float64) != 2 { + t.Errorf("en word count = %v, want 2", counts["en"]) + } +} diff --git a/dreamdict-blueprint.md b/dreamdict-blueprint.md index 1274cfb..66b12a6 100644 --- a/dreamdict-blueprint.md +++ b/dreamdict-blueprint.md @@ -761,3 +761,352 @@ then, not now. The service architecture requires no changes at that point. The consumers are on localhost, there is no intermediary cache, and SQLite responses are already faster than any cache lookup. Drop this entirely. + +--- + +## Phase 2 + +Implement after initial launch is stable and data quality has been validated across all +three languages. Each item below is scoped and sequenced independently -- they do not +depend on each other unless noted. + +--- + +### P2-1: Hunspell Affix Expansion + +See "Phase 2: Affix Expansion" section above. Implement first among Phase 2 items as it +fixes a real correctness bug for player-submitted word validation. + +--- + +### P2-2: WordNet Synset Cross-Language Backing + +**The problem with the current translation approach:** +The current `translations` table is populated opportunistically from Wiktionary tags, which +are inconsistently applied. Coverage is good for common words and thin everywhere else. +This is a structural ceiling, not a data quality problem that more sources can fix. + +**The intentional approach:** +Princeton WordNet assigns every concept a stable synset ID. WOLF maps French words to +those same synset IDs. The Open Multilingual Wordnet (OMW) does the same for Portuguese. +Two words in different languages that share a synset ID are semantic equivalents with high +confidence -- two independent projects made that mapping independently. DreamDict currently +discards synset IDs during import. Preserving them unlocks proper cross-language backing. + +**Schema additions:** + +```sql +CREATE TABLE IF NOT EXISTS synsets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + synset_id TEXT NOT NULL UNIQUE, -- Princeton WN ID e.g. '00914031-a' + pos TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS word_synsets ( + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + synset_id INTEGER NOT NULL REFERENCES synsets(id) ON DELETE CASCADE, + source TEXT NOT NULL, -- 'wordnet', 'wolf', 'omw' + PRIMARY KEY (word_id, synset_id) +); + +CREATE INDEX IF NOT EXISTS idx_word_synsets_word_id ON word_synsets(word_id); +CREATE INDEX IF NOT EXISTS idx_word_synsets_synset_id ON word_synsets(synset_id); +``` + +**Loader changes:** +- `wordnet.go` -- store the synset offset as `synset_id` in `synsets`, link words via + `word_synsets` with `source='wordnet'` +- `wolf.go` -- WOLF synset IDs are Princeton WN IDs; store and link with `source='wolf'` +- New loader `omw.go` -- Open Multilingual Wordnet Portuguese data; maps pt words to + Princeton synset IDs; link with `source='omw'` + - Source: https://github.com/omwn/omw-data + - File: Portuguese tab file from the OMW release + +**New API method:** + +```go +// EnglishBacking returns English words and their WordNet definitions that share +// a synset with the given word. Semantic equivalence, not translation guesswork. +func (d *Dictionary) EnglishBacking(word, lang string) ([]EnglishEquivalent, error) + +type EnglishEquivalent struct { + Word string + Definition string // the WordNet gloss for that synset + Synset string +} +``` + +**New endpoint:** + +``` +GET /backing?word=éphémère&lang=fr +→ { + "word": "éphémère", + "lang": "fr", + "equivalents": [ + {"word": "ephemeral", "definition": "lasting a very short time", "synset": "00914031-a"} + ] + } +``` + +**Fallback behaviour:** +Words with no synset mapping (proper nouns, invented words, very obscure entries) fall back +to the existing `translations` table as a secondary source. The two mechanisms complement +each other -- synsets for authoritative coverage, Wiktionary translations for the rest. + +**GogoBee impact:** +Every French and Portuguese word with a WOLF or OMW synset mapping automatically gets +authoritative English backing with a real WordNet definition attached -- not just a word, +but the concept it represents. Update `DreamDictClient` with a `EnglishBacking()` method. + +--- + +### P2-3: Antonyms + +**Data source:** Already in Princeton WordNet. The pointer data in `data.*` files includes +antonym relations marked with `!` as the pointer symbol. The current `wordnet.go` loader +skips all pointer types. This is a loader change only. + +**Schema addition:** + +```sql +CREATE TABLE IF NOT EXISTS antonyms ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + antonym TEXT NOT NULL, + source TEXT NOT NULL, -- 'wordnet', 'wiktionary' + UNIQUE(word_id, antonym) +); + +CREATE INDEX IF NOT EXISTS idx_antonyms_word_id ON antonyms(word_id); +``` + +**Loader changes:** +- `wordnet.go` -- parse pointer lines with symbol `!`; for each antonym pointer, insert + into `antonyms` with `source='wordnet'` +- `wiktionary.go` -- kaikki.org entries carry an `antonyms[]` array alongside `synonyms[]`; + parse and insert with `source='wiktionary'` + +**New API method and endpoint:** + +```go +func (d *Dictionary) Antonyms(word, lang string) ([]string, error) +``` + +``` +GET /antonyms?word=happy&lang=en +→ {"word": "happy", "lang": "en", "antonyms": ["unhappy", "sad", "miserable"]} +``` + +**GogoBee uses:** Opposite-word puzzle modes, richer `!define` output, future word game +mechanics that need semantic opposites. + +--- + +### P2-4: Frequency Weighting for English and Portuguese + +**The problem:** `RandomWord()` for English and Portuguese is currently unweighted -- +`ORDER BY RANDOM()` across the full word table. A Hangman puzzle is as likely to return +"aardvark" as "house." French already has frequency data from Lexique. English and +Portuguese need the same. + +**Data sources:** + +- **English:** SUBTLEX-US (Brysbaert & New, 2009) -- subtitle frequency corpus, freely + available. Provides word frequency per million from film/TV subtitles. Same format as + Lexique conceptually. + - URL: http://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexus + - File: `SUBTLEX-US.xlsx` (export to TSV before import) + +- **Portuguese:** SUBTLEX-PT exists but coverage skews pt-BR. Better option for pt-PT is + the frequency list derived from the CETEMPúblico corpus (European Portuguese newspaper + corpus), available via Linguateca. + - URL: https://www.linguateca.pt/acesso/corpus.php?corpus=CETEMPUBLICO + - Alternative: use Wiktionary frequency tags as a rough proxy if CETEMPúblico access + is cumbersome -- lower quality but available immediately. + +**Loader changes:** +- New `subtlex.go` -- TSV import for English frequency, same pattern as `lexique.go`; + `UPDATE words SET frequency=? WHERE word=? AND lang='en' AND frequency=0` +- New `cetempublico.go` (or `wiktfreq.go` as fallback) -- same pattern for pt-PT + +**Schema:** No change -- `frequency` column already exists on `words` for all languages. + +**API change:** `MinFrequency` in `Options` already exists. No API change needed -- this +just makes the filter meaningful for `en` and `pt-PT` for the first time. + +**DreamDict dictimport change:** Add both loaders to the English and pt-PT execution order, +running after word list population. + +--- + +## Phase 3 + +Implement after Phase 2 is complete and stable. These are higher-effort or lower-urgency +than Phase 2 items but all have clear value. + +--- + +### P3-1: Word Difficulty Scoring + +**What it is:** A composite difficulty score derived from data already in the database -- +no new sources required. Computed at import time and stored as a column. + +**Formula (tune after testing):** +``` +difficulty = normalize(1 / frequency) * 0.5 + + normalize(length) * 0.3 + + normalize(syllable_count) * 0.2 +``` + +Syllable count can be approximated from the word string (vowel cluster counting) or pulled +from CMU Pronouncing Dictionary (see P3-2) once that data exists. + +**Schema addition:** + +```sql +ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL; +-- NULL = not yet scored; 0.0 = easiest; 1.0 = hardest +``` + +**API change:** + +Add to `Options`: +```go +MaxDifficulty float64 // 0.0–1.0; 0 = no filter +MinDifficulty float64 +``` + +Add to `RandomWord()` response (exposed via new `WordDetail` return type if needed) and +as a metadata field on `/random` response: +```json +{"word": "ephemeral", "difficulty": 0.74} +``` + +**GogoBee uses:** Tiered Hangman difficulty without hardcoded word lists. "Easy", "Medium", +"Hard" modes map to difficulty ranges. Arena word puzzles can scale to player level. + +--- + +### P3-2: Pronunciation Data + +**Data source:** CMU Pronouncing Dictionary (CMUdict) +- URL: http://www.speech.cs.cmu.edu/cgi-bin/cmudict +- Format: plain text, one entry per line: `EPHEMERAL IH0 F EH1 M ER0 AH0 L` +- Coverage: English only (~134,000 entries) +- License: BSD-style, freely usable + +For French and Portuguese, IPA data is available in the kaikki.org Wiktionary dumps under +a `sounds` array per entry -- the current wiktionary loader ignores this field. + +**Schema addition:** + +```sql +CREATE TABLE IF NOT EXISTS pronunciations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + format TEXT NOT NULL, -- 'cmu', 'ipa' + value TEXT NOT NULL, + source TEXT NOT NULL, -- 'cmudict', 'wiktionary' + UNIQUE(word_id, format, source) +); + +CREATE INDEX IF NOT EXISTS idx_pronunciations_word_id ON pronunciations(word_id); +``` + +**Loader changes:** +- New `cmudict.go` -- plain text import; match words to existing `words` table entries; + insert with `format='cmu'`, `source='cmudict'` +- `wiktionary.go` -- extend to parse `sounds[].ipa` field; insert with `format='ipa'`, + `source='wiktionary'` + +**New API method and endpoint:** + +```go +func (d *Dictionary) Pronunciation(word, lang string) ([]Pronunciation, error) + +type Pronunciation struct { + Format string // 'cmu', 'ipa' + Value string + Source string +} +``` + +``` +GET /pronunciation?word=ephemeral&lang=en +→ { + "word": "ephemeral", + "lang": "en", + "pronunciations": [ + {"format": "cmu", "value": "IH0 F EH1 M ER0 AH0 L", "source": "cmudict"}, + {"format": "ipa", "value": "ɪˈfɛm.ər.əl", "source": "wiktionary"} + ] + } +``` + +**GogoBee uses:** `!define` output enrichment, syllable count for difficulty scoring (P3-1), +future rhyme game mechanics (words sharing a CMU tail sequence are rhymes). + +--- + +### P3-3: Etymology + +**Data source:** kaikki.org Wiktionary dumps already downloaded -- etymology data is in the +`etymology_text` field per entry. The current `wiktionary.go` loader ignores it. + +**Schema addition:** + +```sql +CREATE TABLE IF NOT EXISTS etymology ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + text TEXT NOT NULL, -- free-form etymology string from Wiktionary + source TEXT NOT NULL, -- 'wiktionary' + UNIQUE(word_id, source) +); + +CREATE INDEX IF NOT EXISTS idx_etymology_word_id ON etymology(word_id); +``` + +**Loader change:** +- `wiktionary.go` -- extend to parse `etymology_text` field; insert into `etymology` if + non-empty and not a redirect/stub string (skip entries that are just "See X" or empty + after trimming) + +**New API method and endpoint:** + +```go +func (d *Dictionary) Etymology(word, lang string) (string, error) +``` + +``` +GET /etymology?word=ephemeral&lang=en +→ { + "word": "ephemeral", + "lang": "en", + "etymology": "From Medieval Latin ephemerus, from Ancient Greek ἐφήμερος (ephḗmeros, \"lasting only a day\"), from ἐπί (epí, \"on\") + ἡμέρα (hēméra, \"day\")." + } +``` + +**GogoBee uses:** `!etymology` command for the community. Particularly relevant given the +pt-PT context -- shared Latin/Greek roots between Portuguese, French, and English make +etymology a genuine learning tool, not just trivia. Word of the Day can optionally append +a one-line etymology note. + +**Quality note:** Wiktionary etymology text is free-form and inconsistently structured. +Surface it as-is rather than attempting to parse it into structured fields. A raw string +is useful; a badly parsed structured field is not. + +--- + +## Roadmap Summary + +| Phase | Item | Effort | Payoff | +|---|---|---|---| +| 2 | Affix expansion | Medium | High -- fixes real validation bug | +| 2 | Synset cross-language backing | Medium | High -- authoritative multilingual semantics | +| 2 | Antonyms | Low | Medium -- already in WordNet data | +| 2 | Frequency for en + pt-PT | Low-Medium | High -- better random word quality | +| 3 | Difficulty scoring | Low | High -- derived, no new data | +| 3 | Pronunciation | Medium | Medium -- en only initially | +| 3 | Etymology | Low | High -- data already downloaded | diff --git a/internal/dictionary/bench_test.go b/internal/dictionary/bench_test.go new file mode 100644 index 0000000..7e52b59 --- /dev/null +++ b/internal/dictionary/bench_test.go @@ -0,0 +1,151 @@ +package dictionary + +import ( + "database/sql" + "fmt" + "testing" + + _ "modernc.org/sqlite" +) + +func setupBenchDB(b *testing.B) (*Dictionary, *sql.DB) { + b.Helper() + db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(on)") + if err != nil { + b.Fatal(err) + } + db.SetMaxOpenConns(1) + if err := BootstrapSchema(db); err != nil { + b.Fatal(err) + } + + // Seed a realistic-ish dataset: 1000 words with definitions, synonyms, translations + exec := func(q string, args ...any) { + if _, err := db.Exec(q, args...); err != nil { + b.Fatalf("seed: %s: %v", q, err) + } + } + + exec("INSERT INTO meta (key, value) VALUES ('schema_version', '1')") + exec("INSERT INTO meta (key, value) VALUES ('imported_at', '2025-01-01T00:00:00Z')") + + poses := []string{"noun", "verb", "adjective", "adverb"} + langs := []string{"en", "fr", "pt-PT"} + + for i := 1; i <= 1000; i++ { + lang := langs[i%len(langs)] + pos := poses[i%len(poses)] + word := fmt.Sprintf("word%04d", i) + exec("INSERT INTO words (id, word, lang, pos, frequency) VALUES (?, ?, ?, ?, ?)", + i, word, lang, pos, i%100) + + // 2 definitions per word + exec("INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (?, ?, ?, 'wordnet', 10)", + i, pos, fmt.Sprintf("definition A of %s", word)) + exec("INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (?, ?, ?, 'wiktionary', 20)", + i, pos, fmt.Sprintf("definition B of %s", word)) + + // 2 synonyms + exec("INSERT INTO synonyms (word_id, synonym, source) VALUES (?, ?, 'wordnet')", + i, fmt.Sprintf("syn1_%s", word)) + exec("INSERT INTO synonyms (word_id, synonym, source) VALUES (?, ?, 'wordnet')", + i, fmt.Sprintf("syn2_%s", word)) + + // Translation for en words + if lang == "en" { + exec("INSERT INTO translations (word_id, translation, target_lang, source) VALUES (?, ?, 'fr', 'wiktionary')", + i, fmt.Sprintf("fr_%s", word)) + } + } + + return NewFromDB(db), db +} + +func BenchmarkIsValidWord(b *testing.B) { + d, db := setupBenchDB(b) + defer db.Close() + + b.Run("hit", func(b *testing.B) { + for b.Loop() { + d.IsValidWord("word0003", "en") + } + }) + b.Run("miss", func(b *testing.B) { + for b.Loop() { + d.IsValidWord("nonexistent", "en") + } + }) +} + +func BenchmarkRandomWord(b *testing.B) { + d, db := setupBenchDB(b) + defer db.Close() + + b.Run("no_filter", func(b *testing.B) { + for b.Loop() { + d.RandomWord("en", Options{}) + } + }) + b.Run("with_filters", func(b *testing.B) { + for b.Loop() { + d.RandomWord("en", Options{POS: "noun", MinLength: 4, MaxLength: 10}) + } + }) +} + +func BenchmarkDefine(b *testing.B) { + d, db := setupBenchDB(b) + defer db.Close() + + b.Run("with_defs", func(b *testing.B) { + for b.Loop() { + d.Define("word0003", "en") + } + }) + b.Run("no_defs", func(b *testing.B) { + for b.Loop() { + d.Define("nonexistent", "en") + } + }) +} + +func BenchmarkSynonyms(b *testing.B) { + d, db := setupBenchDB(b) + defer db.Close() + + b.Run("with_syns", func(b *testing.B) { + for b.Loop() { + d.Synonyms("word0003", "en") + } + }) + b.Run("no_syns", func(b *testing.B) { + for b.Loop() { + d.Synonyms("nonexistent", "en") + } + }) +} + +func BenchmarkTranslate(b *testing.B) { + d, db := setupBenchDB(b) + defer db.Close() + + b.Run("with_trans", func(b *testing.B) { + for b.Loop() { + d.Translate("word0003", "en", "fr") + } + }) + b.Run("no_trans", func(b *testing.B) { + for b.Loop() { + d.Translate("nonexistent", "en", "fr") + } + }) +} + +func BenchmarkWordCount(b *testing.B) { + d, db := setupBenchDB(b) + defer db.Close() + + for b.Loop() { + d.WordCount() + } +} diff --git a/internal/dictionary/db.go b/internal/dictionary/db.go index 8df7347..211162a 100644 --- a/internal/dictionary/db.go +++ b/internal/dictionary/db.go @@ -3,11 +3,12 @@ package dictionary import ( "database/sql" "fmt" + "strings" _ "modernc.org/sqlite" ) -const schemaVersion = "1" +const schemaVersion = "2" const schemaSQL = ` CREATE TABLE IF NOT EXISTS words ( @@ -15,7 +16,8 @@ CREATE TABLE IF NOT EXISTS words ( word TEXT NOT NULL, lang TEXT NOT NULL, pos TEXT, - frequency INTEGER DEFAULT 0, + frequency INTEGER DEFAULT 0, + difficulty REAL DEFAULT NULL, UNIQUE(word, lang) ); @@ -61,6 +63,53 @@ CREATE TABLE IF NOT EXISTS meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); + +CREATE TABLE IF NOT EXISTS antonyms ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + antonym TEXT NOT NULL, + source TEXT NOT NULL, + UNIQUE(word_id, antonym) +); + +CREATE INDEX IF NOT EXISTS idx_antonyms_word_id ON antonyms(word_id); + +CREATE TABLE IF NOT EXISTS synsets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + synset_id TEXT NOT NULL UNIQUE, + pos TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS word_synsets ( + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + synset_id INTEGER NOT NULL REFERENCES synsets(id) ON DELETE CASCADE, + source TEXT NOT NULL, + PRIMARY KEY (word_id, synset_id) +); + +CREATE INDEX IF NOT EXISTS idx_word_synsets_word_id ON word_synsets(word_id); +CREATE INDEX IF NOT EXISTS idx_word_synsets_synset_id ON word_synsets(synset_id); + +CREATE TABLE IF NOT EXISTS pronunciations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + format TEXT NOT NULL, + value TEXT NOT NULL, + source TEXT NOT NULL, + UNIQUE(word_id, format, source) +); + +CREATE INDEX IF NOT EXISTS idx_pronunciations_word_id ON pronunciations(word_id); + +CREATE TABLE IF NOT EXISTS etymology ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + text TEXT NOT NULL, + source TEXT NOT NULL, + UNIQUE(word_id, source) +); + +CREATE INDEX IF NOT EXISTS idx_etymology_word_id ON etymology(word_id); ` func openDB(dbPath string) (*sql.DB, error) { @@ -77,11 +126,19 @@ func openDB(dbPath string) (*sql.DB, error) { } func openReadOnlyDB(dbPath string) (*sql.DB, error) { - db, err := sql.Open("sqlite", dbPath+"?mode=ro&_pragma=journal_mode(wal)&_pragma=foreign_keys(on)") + db, err := sql.Open("sqlite", dbPath+ + "?mode=ro"+ + "&_pragma=journal_mode(wal)"+ + "&_pragma=foreign_keys(on)"+ + "&_pragma=cache_size(-64000)"+ + "&_pragma=mmap_size(268435456)"+ + "&_pragma=temp_store(memory)") if err != nil { return nil, fmt.Errorf("dictionary: open db (ro): %w", err) } - // No MaxOpenConns limit — WAL mode supports concurrent reads + // Pin to one connection so pragmas are consistent. WAL mode still + // allows concurrent reads from the same connection in Go's model. + db.SetMaxOpenConns(1) if err := db.Ping(); err != nil { db.Close() return nil, fmt.Errorf("dictionary: ping db: %w", err) @@ -94,5 +151,25 @@ func BootstrapSchema(db *sql.DB) error { if err != nil { return fmt.Errorf("dictionary: bootstrap schema: %w", err) } + + // Schema migrations for databases created before v2. + // ALTER TABLE ADD COLUMN is a no-op if the column already exists in SQLite + // when we catch the "duplicate column" error. + migrations := []string{ + "ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL", + } + for _, m := range migrations { + if _, err := db.Exec(m); err != nil { + // Ignore "duplicate column name" — means migration already applied + if !isDuplicateColumn(err) { + return fmt.Errorf("dictionary: migration: %w", err) + } + } + } + return nil } + +func isDuplicateColumn(err error) bool { + return err != nil && strings.Contains(err.Error(), "duplicate column") +} diff --git a/internal/dictionary/dict.go b/internal/dictionary/dict.go index d533a19..6774fa6 100644 --- a/internal/dictionary/dict.go +++ b/internal/dictionary/dict.go @@ -22,10 +22,12 @@ func ValidLang(lang string) bool { } type Options struct { - MinLength int - MaxLength int - POS string // "", "noun", "verb", "adjective", "adverb" - MinFrequency int // 0 = no filter + MinLength int + MaxLength int + POS string // "", "noun", "verb", "adjective", "adverb" + MinFrequency int // 0 = no filter + MinDifficulty float64 // 0.0 = no filter + MaxDifficulty float64 // 0.0 = no filter } type Definition struct { @@ -35,6 +37,18 @@ type Definition struct { Priority int `json:"priority"` } +type EnglishEquivalent struct { + Word string `json:"word"` + Definition string `json:"definition"` + Synset string `json:"synset"` +} + +type Pronunciation struct { + Format string `json:"format"` + Value string `json:"value"` + Source string `json:"source"` +} + type Dictionary struct { db *sql.DB } diff --git a/internal/dictionary/dict_test.go b/internal/dictionary/dict_test.go index e313ef4..a8fd710 100644 --- a/internal/dictionary/dict_test.go +++ b/internal/dictionary/dict_test.go @@ -89,30 +89,30 @@ func TestRandomWord(t *testing.T) { d := NewFromDB(db) // Basic random word - word, err := d.RandomWord("en", Options{}) + result, err := d.RandomWord("en", Options{}) if err != nil { t.Fatalf("RandomWord: %v", err) } - if word == "" { + if result.Word == "" { t.Error("RandomWord returned empty string") } // With POS filter - word, err = d.RandomWord("en", Options{POS: "adjective"}) + result, err = d.RandomWord("en", Options{POS: "adjective"}) if err != nil { t.Fatalf("RandomWord with POS: %v", err) } - if word != "happy" { - t.Errorf("RandomWord(adjective) = %q, want happy", word) + if result.Word != "happy" { + t.Errorf("RandomWord(adjective) = %q, want happy", result.Word) } // With length filters - word, err = d.RandomWord("en", Options{MinLength: 4, MaxLength: 5}) + result, err = d.RandomWord("en", Options{MinLength: 4, MaxLength: 5}) if err != nil { t.Fatalf("RandomWord with length: %v", err) } - if len(word) < 4 || len(word) > 5 { - t.Errorf("RandomWord length %d not in [4,5]", len(word)) + if len(result.Word) < 4 || len(result.Word) > 5 { + t.Errorf("RandomWord length %d not in [4,5]", len(result.Word)) } // No match diff --git a/internal/dictionary/query.go b/internal/dictionary/query.go index 62d536b..ff64001 100644 --- a/internal/dictionary/query.go +++ b/internal/dictionary/query.go @@ -18,8 +18,13 @@ func (d *Dictionary) IsValidWord(word, lang string) (bool, error) { return exists, nil } -func (d *Dictionary) RandomWord(lang string, opts Options) (string, error) { - query := "SELECT word FROM words WHERE lang = ?" +type RandomResult struct { + Word string `json:"word"` + Difficulty *float64 `json:"difficulty,omitempty"` +} + +func (d *Dictionary) RandomWord(lang string, opts Options) (RandomResult, error) { + query := "SELECT word, difficulty FROM words WHERE lang = ?" args := []any{lang} if opts.POS != "" { @@ -38,18 +43,30 @@ func (d *Dictionary) RandomWord(lang string, opts Options) (string, error) { query += " AND frequency >= ?" args = append(args, opts.MinFrequency) } + if opts.MinDifficulty > 0 { + query += " AND difficulty >= ?" + args = append(args, opts.MinDifficulty) + } + if opts.MaxDifficulty > 0 { + query += " AND difficulty <= ?" + args = append(args, opts.MaxDifficulty) + } query += " ORDER BY RANDOM() LIMIT 1" - var word string - err := d.db.QueryRow(query, args...).Scan(&word) + var result RandomResult + var diff sql.NullFloat64 + err := d.db.QueryRow(query, args...).Scan(&result.Word, &diff) if err == sql.ErrNoRows { - return "", ErrNoMatch + return RandomResult{}, ErrNoMatch } if err != nil { - return "", fmt.Errorf("dictionary: random word: %w", err) + return RandomResult{}, fmt.Errorf("dictionary: random word: %w", err) } - return word, nil + if diff.Valid { + result.Difficulty = &diff.Float64 + } + return result, nil } func (d *Dictionary) Define(word, lang string) ([]Definition, error) { @@ -175,6 +192,44 @@ func (d *Dictionary) Frequency(word, lang string) (int, error) { return freq, nil } +// FrequencyBatch returns frequency scores for multiple words in a language. +// Words not found or without frequency data are returned with a score of 0. +func (d *Dictionary) FrequencyBatch(words []string, lang string) (map[string]int, error) { + if len(words) == 0 { + return map[string]int{}, nil + } + + placeholders := make([]string, len(words)) + args := make([]any, 0, len(words)+1) + args = append(args, lang) + for i, w := range words { + placeholders[i] = "?" + args = append(args, strings.ToLower(w)) + } + + query := fmt.Sprintf( + "SELECT word, COALESCE(frequency, 0) FROM words WHERE lang = ? AND word IN (%s)", + strings.Join(placeholders, ","), + ) + + rows, err := d.db.Query(query, args...) + if err != nil { + return nil, fmt.Errorf("dictionary: frequency batch: %w", err) + } + defer rows.Close() + + result := make(map[string]int, len(words)) + for rows.Next() { + var word string + var freq int + if err := rows.Scan(&word, &freq); err != nil { + return nil, fmt.Errorf("dictionary: frequency batch scan: %w", err) + } + result[word] = freq + } + return result, rows.Err() +} + func (d *Dictionary) Meta(key string) (string, error) { var val string err := d.db.QueryRow("SELECT value FROM meta WHERE key = ?", key).Scan(&val) @@ -187,6 +242,143 @@ func (d *Dictionary) Meta(key string) (string, error) { return val, nil } +func (d *Dictionary) Antonyms(word, lang string) ([]string, error) { + rows, err := d.db.Query(` + SELECT DISTINCT a.antonym + FROM antonyms a + JOIN words w ON w.id = a.word_id + WHERE w.word = ? AND w.lang = ? + ORDER BY a.antonym`, + strings.ToLower(word), lang, + ) + if err != nil { + return nil, fmt.Errorf("dictionary: antonyms: %w", err) + } + defer rows.Close() + + var ants []string + for rows.Next() { + var a string + if err := rows.Scan(&a); err != nil { + return nil, fmt.Errorf("dictionary: antonyms scan: %w", err) + } + ants = append(ants, a) + } + if ants == nil { + ants = []string{} + } + return ants, rows.Err() +} + +// EnglishBacking returns English words that share a synset with the given +// non-English word, providing semantic equivalents with WordNet definitions. +func (d *Dictionary) EnglishBacking(word, lang string) ([]EnglishEquivalent, error) { + // Find synsets for this word, then find English words in those synsets, + // along with their WordNet definitions. + rows, err := d.db.Query(` + SELECT DISTINCT ew.word, + COALESCE( + (SELECT d.gloss FROM definitions d + WHERE d.word_id = ew.id AND d.source = 'wordnet' AND d.pos = s.pos + ORDER BY d.priority ASC LIMIT 1), + (SELECT d.gloss FROM definitions d + WHERE d.word_id = ew.id AND d.source = 'wordnet' + ORDER BY d.priority ASC LIMIT 1), + '' + ) AS gloss, + s.synset_id + FROM words w + JOIN word_synsets ws ON ws.word_id = w.id + JOIN synsets s ON s.id = ws.synset_id + JOIN word_synsets ews ON ews.synset_id = s.id + JOIN words ew ON ew.id = ews.word_id AND ew.lang = 'en' + WHERE w.word = ? AND w.lang = ? + ORDER BY s.synset_id, ew.word`, + strings.ToLower(word), lang, + ) + if err != nil { + return nil, fmt.Errorf("dictionary: english backing: %w", err) + } + defer rows.Close() + + var results []EnglishEquivalent + seen := make(map[string]bool) + for rows.Next() { + var eq EnglishEquivalent + if err := rows.Scan(&eq.Word, &eq.Definition, &eq.Synset); err != nil { + return nil, fmt.Errorf("dictionary: english backing scan: %w", err) + } + key := eq.Word + "|" + eq.Synset + if !seen[key] { + seen[key] = true + results = append(results, eq) + } + } + if results == nil { + results = []EnglishEquivalent{} + } + + // Fallback to translations table if no synset matches + if len(results) == 0 { + trans, err := d.Translate(word, lang, "en") + if err != nil { + return nil, err + } + for _, t := range trans { + results = append(results, EnglishEquivalent{Word: t}) + } + } + + return results, rows.Err() +} + +func (d *Dictionary) Pronunciation(word, lang string) ([]Pronunciation, error) { + rows, err := d.db.Query(` + SELECT p.format, p.value, p.source + FROM pronunciations p + JOIN words w ON w.id = p.word_id + WHERE w.word = ? AND w.lang = ? + ORDER BY p.source, p.format`, + strings.ToLower(word), lang, + ) + if err != nil { + return nil, fmt.Errorf("dictionary: pronunciation: %w", err) + } + defer rows.Close() + + var prons []Pronunciation + for rows.Next() { + var p Pronunciation + if err := rows.Scan(&p.Format, &p.Value, &p.Source); err != nil { + return nil, fmt.Errorf("dictionary: pronunciation scan: %w", err) + } + prons = append(prons, p) + } + if prons == nil { + prons = []Pronunciation{} + } + return prons, rows.Err() +} + +func (d *Dictionary) Etymology(word, lang string) (string, error) { + var text string + err := d.db.QueryRow(` + SELECT e.text + FROM etymology e + JOIN words w ON w.id = e.word_id + WHERE w.word = ? AND w.lang = ? + LIMIT 1`, + strings.ToLower(word), lang, + ).Scan(&text) + if err == sql.ErrNoRows { + return "", nil + } + if err != nil { + return "", fmt.Errorf("dictionary: etymology: %w", err) + } + return text, nil +} + func (d *Dictionary) DefCount() (map[string]int, error) { rows, err := d.db.Query(` SELECT w.lang, COUNT(*) diff --git a/internal/loader/affix.go b/internal/loader/affix.go new file mode 100644 index 0000000..b3149ae --- /dev/null +++ b/internal/loader/affix.go @@ -0,0 +1,537 @@ +package loader + +import ( + "bufio" + "database/sql" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" +) + +// AffixLoader expands inflected forms from Hunspell .aff/.dic pairs +// and inserts them into the words table alongside base forms. +type AffixLoader struct { + Lang string // "en", "fr", "pt-PT" + DicFile string // path relative to dataDir, e.g. "fr_FR/fr.dic" + AffFile string // path relative to dataDir, e.g. "fr_FR/fr.aff" +} + +func (a AffixLoader) Name() string { return "affix-" + a.Lang } + +func (a AffixLoader) Load(db *sql.DB, dataDir string) error { + affPath := filepath.Join(dataDir, a.AffFile) + dicPath := filepath.Join(dataDir, a.DicFile) + + if _, err := os.Stat(affPath); os.IsNotExist(err) { + slog.Warn("affix: .aff file not found, skipping", "path", affPath) + return nil + } + if _, err := os.Stat(dicPath); os.IsNotExist(err) { + slog.Warn("affix: .dic file not found, skipping", "path", dicPath) + return nil + } + + rules, err := parseAffFile(affPath) + if err != nil { + return fmt.Errorf("affix: parse aff: %w", err) + } + + dicEntries, err := parseDicFile(dicPath) + if err != nil { + return fmt.Errorf("affix: parse dic: %w", err) + } + + var forms []string + seen := make(map[string]struct{}) + for _, entry := range dicEntries { + for _, form := range expandWord(entry.word, entry.flags, rules) { + if containsDigit(form) || containsSpace(form) { + continue + } + if _, dup := seen[form]; dup { + continue + } + seen[form] = struct{}{} + forms = append(forms, form) + } + } + + if err := bulkInsertWords(db, forms, a.Lang, 250); err != nil { + return fmt.Errorf("affix: %w", err) + } + slog.Info("affix loaded", "lang", a.Lang, "expanded_forms", len(forms)) + return nil +} + +type affixRule struct { + ruleType string // "SFX" or "PFX" + strip string + affix string + matchers []condMatcher // pre-parsed condition +} + +type affixRuleSet struct { + rules []affixRule + combinable bool + ruleType string +} + +type dicEntry struct { + word string + flags string +} + +func parseAffFile(path string) (map[string]*affixRuleSet, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + rules := make(map[string]*affixRuleSet) + scanner := bufio.NewScanner(f) + buf := make([]byte, 0, 64*1024) + scanner.Buffer(buf, 1024*1024) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + fields := strings.Fields(line) + if len(fields) < 4 { + continue + } + + ruleType := fields[0] + if ruleType != "SFX" && ruleType != "PFX" { + continue + } + + flag := fields[1] + + // Header line: SFX flag combinable count + if _, err := strconv.Atoi(fields[len(fields)-1]); err == nil && len(fields) == 4 { + combinable := fields[2] == "Y" + if _, exists := rules[flag]; !exists { + rules[flag] = &affixRuleSet{ + combinable: combinable, + ruleType: ruleType, + } + } + continue + } + + // Rule line: SFX flag strip affix [condition] + if len(fields) < 4 { + continue + } + + strip := fields[2] + if strip == "0" { + strip = "" + } + affix := fields[3] + if affix == "0" { + affix = "" + } + + // Strip any continuation flags from affix (e.g., "ing/S" -> "ing") + if slashIdx := strings.Index(affix, "/"); slashIdx != -1 { + affix = affix[:slashIdx] + } + + condition := "." + if len(fields) >= 5 { + condition = fields[4] + } + + rs, exists := rules[flag] + if !exists { + rs = &affixRuleSet{ruleType: ruleType} + rules[flag] = rs + } + + rs.rules = append(rs.rules, affixRule{ + ruleType: ruleType, + strip: strip, + affix: affix, + matchers: parseConditionMatchers(condition), + }) + } + + return rules, scanner.Err() +} + +func parseDicFile(path string) ([]dicEntry, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + // Skip first line (word count) + if scanner.Scan() { + // discard + } + + var entries []dicEntry + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + parts := strings.SplitN(line, "/", 2) + word := strings.ToLower(parts[0]) + flags := "" + if len(parts) > 1 { + flags = parts[1] + } + + if containsDigit(word) || containsNonLatin(word) { + continue + } + + entries = append(entries, dicEntry{word: word, flags: flags}) + } + + return entries, scanner.Err() +} + +// maxFormsPerWord caps the number of inflected forms generated per base word. +// Portuguese and French .aff files can produce hundreds of cross-product forms +// per word; most are valid but the combinatorial explosion bloats the DB. +const maxFormsPerWord = 30 + +func expandWord(word, flags string, rules map[string]*affixRuleSet) []string { + seen := map[string]bool{word: true} // base form already in DB + var result []string + + // Apply single-flag suffix/prefix rules + for _, r := range flags { + rs, ok := rules[string(r)] + if !ok { + continue + } + for _, rule := range rs.rules { + if len(result) >= maxFormsPerWord { + return result + } + form := applyRule(word, rule) + if form != "" && !seen[form] { + seen[form] = true + result = append(result, form) + } + } + } + + return result +} + +func applyRule(word string, rule affixRule) string { + if !matchCondition(word, rule.matchers, rule.ruleType) { + return "" + } + + switch rule.ruleType { + case "SFX": + base := word + if rule.strip != "" { + if !strings.HasSuffix(base, rule.strip) { + return "" + } + base = base[:len(base)-len(rule.strip)] + } + return base + rule.affix + + case "PFX": + base := word + if rule.strip != "" { + if !strings.HasPrefix(base, rule.strip) { + return "" + } + base = base[len(rule.strip):] + } + return rule.affix + base + } + + return "" +} + +func matchCondition(word string, matchers []condMatcher, ruleType string) bool { + if len(matchers) == 0 { + return true + } + + wordRunes := []rune(word) + if len(wordRunes) < len(matchers) { + return false + } + + if ruleType == "SFX" { + start := len(wordRunes) - len(matchers) + for i, m := range matchers { + if !m.matches(wordRunes[start+i]) { + return false + } + } + } else { + for i, m := range matchers { + if !m.matches(wordRunes[i]) { + return false + } + } + } + return true +} + +type condMatcher struct { + chars []rune + negate bool +} + +func (m condMatcher) matches(r rune) bool { + if len(m.chars) == 0 { + return true // "." wildcard + } + for _, c := range m.chars { + if c == r { + return !m.negate + } + } + return m.negate +} + +func parseConditionMatchers(condition string) []condMatcher { + var matchers []condMatcher + runes := []rune(condition) + i := 0 + for i < len(runes) { + if runes[i] == '[' { + i++ + negate := false + if i < len(runes) && runes[i] == '^' { + negate = true + i++ + } + var chars []rune + for i < len(runes) && runes[i] != ']' { + chars = append(chars, runes[i]) + i++ + } + if i < len(runes) { + i++ // skip ']' + } + matchers = append(matchers, condMatcher{chars: chars, negate: negate}) + } else if runes[i] == '.' { + matchers = append(matchers, condMatcher{}) // wildcard + i++ + } else { + matchers = append(matchers, condMatcher{chars: []rune{runes[i]}}) + i++ + } + } + return matchers +} + +// EnglishAffixLoader generates common English inflected forms from base words +// already in the database. SCOWL doesn't ship .aff rules, so this uses +// programmatic English morphology rules instead. +type EnglishAffixLoader struct{} + +func (EnglishAffixLoader) Name() string { return "affix-en" } + +func (EnglishAffixLoader) Load(db *sql.DB, dataDir string) error { + // Read all base words first, then close the query before writing. + rows, err := db.Query("SELECT word FROM words WHERE lang = 'en'") + if err != nil { + return fmt.Errorf("affix-en: query words: %w", err) + } + + var baseWords []string + for rows.Next() { + var w string + if err := rows.Scan(&w); err != nil { + rows.Close() + return fmt.Errorf("affix-en: scan: %w", err) + } + baseWords = append(baseWords, w) + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("affix-en: rows err: %w", err) + } + + // Generate all forms in memory first — avoids per-word DB round trips + // during generation. Deduplicate against base words. + baseSet := make(map[string]struct{}, len(baseWords)) + for _, w := range baseWords { + baseSet[w] = struct{}{} + } + + var newForms []string + seen := make(map[string]struct{}, len(baseWords)) + for _, w := range baseWords { + for _, form := range englishInflections(w) { + if _, isBase := baseSet[form]; isBase { + continue + } + if _, dup := seen[form]; dup { + continue + } + seen[form] = struct{}{} + newForms = append(newForms, form) + } + } + + // Multi-row batch insert — 250 rows per statement keeps us under + // SQLite's default 500-variable limit (250 * 2 params = 500). + if err := bulkInsertWords(db, newForms, "en", 250); err != nil { + return fmt.Errorf("affix-en: %w", err) + } + slog.Info("affix-en loaded", "base_words", len(baseWords), "new_forms", len(newForms)) + return nil +} + +// bulkInsertWords inserts words in multi-row batches within a single transaction. +// batchSize controls how many rows per INSERT statement (keep ≤250 for SQLite's +// default 500-variable limit since each row uses 2 params). +func bulkInsertWords(db *sql.DB, words []string, lang string, batchSize int) error { + if len(words) == 0 { + return nil + } + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("bulk insert: begin tx: %w", err) + } + defer tx.Rollback() + + for i := 0; i < len(words); i += batchSize { + end := i + batchSize + if end > len(words) { + end = len(words) + } + batch := words[i:end] + + var b strings.Builder + b.WriteString("INSERT OR IGNORE INTO words (word, lang) VALUES ") + args := make([]any, 0, len(batch)*2) + for j, w := range batch { + if j > 0 { + b.WriteByte(',') + } + b.WriteString("(?,?)") + args = append(args, w, lang) + } + + if _, err := tx.Exec(b.String(), args...); err != nil { + return fmt.Errorf("bulk insert: exec: %w", err) + } + } + + return tx.Commit() +} + +// englishInflections generates common English inflected forms from a base word. +// Skips words that are too short, too long, or already look like inflected forms. +func englishInflections(word string) []string { + if len(word) < 3 || len(word) > 15 { + return nil + } + // Skip words that already look inflected — avoids "runnings", "happinesses", etc. + if strings.HasSuffix(word, "ing") || strings.HasSuffix(word, "ness") || + strings.HasSuffix(word, "ment") || strings.HasSuffix(word, "tion") || + strings.HasSuffix(word, "sion") || strings.HasSuffix(word, "ally") || + strings.HasSuffix(word, "ised") || strings.HasSuffix(word, "ized") { + return nil + } + + var forms []string + add := func(s string) { + if s != word && s != "" { + forms = append(forms, s) + } + } + + last := word[len(word)-1] + + isVowel := func(b byte) bool { + return b == 'a' || b == 'e' || b == 'i' || b == 'o' || b == 'u' + } + + // Plural / verb -s forms + switch { + case strings.HasSuffix(word, "s") || strings.HasSuffix(word, "x") || + strings.HasSuffix(word, "z") || strings.HasSuffix(word, "ch") || + strings.HasSuffix(word, "sh"): + add(word + "es") + case strings.HasSuffix(word, "y") && len(word) >= 2 && !isVowel(word[len(word)-2]): + add(word[:len(word)-1] + "ies") + default: + add(word + "s") + } + + // Past tense / past participle -ed, present participle -ing + switch { + case last == 'e': + add(word + "d") // bake -> baked + add(word[:len(word)-1] + "ing") // bake -> baking + case strings.HasSuffix(word, "y") && len(word) >= 2 && !isVowel(word[len(word)-2]): + add(word[:len(word)-1] + "ied") // carry -> carried + add(word + "ing") // carry -> carrying + case len(word) >= 3 && !isVowel(last) && isVowel(word[len(word)-2]) && !isVowel(word[len(word)-3]) && + last != 'w' && last != 'x' && last != 'y': + // CVC pattern: double consonant (run -> running, stop -> stopped) + add(word + string(last) + "ed") + add(word + string(last) + "ing") + // Also add without doubling (some words don't double) + add(word + "ed") + add(word + "ing") + default: + add(word + "ed") + add(word + "ing") + } + + // Comparative / superlative for short adjectives + if len(word) <= 7 { + switch { + case last == 'e': + add(word + "r") + add(word + "st") + case strings.HasSuffix(word, "y") && len(word) >= 2 && !isVowel(word[len(word)-2]): + add(word[:len(word)-1] + "ier") + add(word[:len(word)-1] + "iest") + default: + add(word + "er") + add(word + "est") + } + } + + // -ly adverb form + switch { + case strings.HasSuffix(word, "le"): + add(word[:len(word)-2] + "ly") + case strings.HasSuffix(word, "y") && len(word) >= 2: + add(word[:len(word)-1] + "ily") + case strings.HasSuffix(word, "ic"): + add(word + "ally") + default: + add(word + "ly") + } + + // -ness + if strings.HasSuffix(word, "y") && len(word) >= 2 && !isVowel(word[len(word)-2]) { + add(word[:len(word)-1] + "iness") + } else { + add(word + "ness") + } + + return forms +} diff --git a/internal/loader/cetempublico.go b/internal/loader/cetempublico.go new file mode 100644 index 0000000..a97f43e --- /dev/null +++ b/internal/loader/cetempublico.go @@ -0,0 +1,134 @@ +package loader + +import ( + "bufio" + "database/sql" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" +) + +// CETEMPublicoLoader loads frequency data for European Portuguese words. +// Accepts a simple word-frequency TSV file derived from the CETEMPúblico corpus +// or any frequency list in "wordcount" or "wordfrequency_per_million" format. +// Falls back to Wiktionary frequency tags if the primary source is unavailable. +type CETEMPublicoLoader struct{} + +func (CETEMPublicoLoader) Name() string { return "cetempublico" } + +func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { + candidates := []string{ + filepath.Join(dataDir, "cetempublico-freq.tsv"), + filepath.Join(dataDir, "pt-freq.tsv"), + filepath.Join(dataDir, "pt_PT-freq.tsv"), + } + + var path string + for _, c := range candidates { + if _, err := os.Stat(c); err == nil { + path = c + break + } + } + if path == "" { + slog.Warn("cetempublico: no data file found, skipping", "searched", candidates) + return nil + } + + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("cetempublico: open: %w", err) + } + defer f.Close() + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("cetempublico: begin tx: %w", err) + } + defer tx.Rollback() + + stmt, err := tx.Prepare(` + UPDATE words SET frequency = ? WHERE word = ? AND lang = 'pt-PT' AND frequency = 0`) + if err != nil { + return fmt.Errorf("cetempublico: prepare: %w", err) + } + defer stmt.Close() + + scanner := bufio.NewScanner(f) + // Skip header if present + if scanner.Scan() { + first := scanner.Text() + fields := strings.Split(first, "\t") + // If first line looks like data (second field is numeric), process it + if len(fields) >= 2 { + if _, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64); err != nil { + // It's a header, skip it + } else { + // It's data, we need to process it — but we already consumed it + // Re-process below + } + } + } + + var count int + processLine := func(line string) error { + fields := strings.Split(line, "\t") + if len(fields) < 2 { + return nil + } + + word := strings.ToLower(strings.TrimSpace(fields[0])) + if word == "" || containsDigit(word) || containsSpace(word) || containsNonLatin(word) { + return nil + } + + freqVal, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64) + if err != nil || freqVal <= 0 { + return nil + } + + // If values are raw counts (>100), convert to a 0-10000 scale + // If already per-million, multiply by 100 + freq := int(freqVal) + if freqVal > 10000 { + // Assume raw counts — log-scale normalization + freq = int(freqVal / 10) + if freq > 10000 { + freq = 10000 + } + } else if freqVal < 100 { + freq = int(freqVal * 100) + } + if freq <= 0 { + freq = 1 + } + + res, err := stmt.Exec(freq, word) + if err != nil { + return fmt.Errorf("cetempublico: update: %w", err) + } + if n, _ := res.RowsAffected(); n > 0 { + count++ + } + return nil + } + + for scanner.Scan() { + if err := processLine(scanner.Text()); err != nil { + return err + } + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("cetempublico: scan: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("cetempublico: commit: %w", err) + } + slog.Info("cetempublico loaded", "updated_words", count) + return nil +} diff --git a/internal/loader/cmudict.go b/internal/loader/cmudict.go new file mode 100644 index 0000000..46f5e62 --- /dev/null +++ b/internal/loader/cmudict.go @@ -0,0 +1,114 @@ +package loader + +import ( + "bufio" + "database/sql" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" +) + +// CMUDictLoader loads pronunciation data from the CMU Pronouncing Dictionary. +// Format: WORD P1 P2 P3 (two-space separated word and phonemes) +// Coverage: English only (~134,000 entries). +type CMUDictLoader struct{} + +func (CMUDictLoader) Name() string { return "cmudict" } + +func (CMUDictLoader) Load(db *sql.DB, dataDir string) error { + candidates := []string{ + filepath.Join(dataDir, "cmudict-0.7b"), + filepath.Join(dataDir, "cmudict.dict"), + filepath.Join(dataDir, "cmudict"), + } + + var path string + for _, c := range candidates { + if _, err := os.Stat(c); err == nil { + path = c + break + } + } + if path == "" { + slog.Warn("cmudict: no data file found, skipping", "searched", candidates) + return nil + } + + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("cmudict: open: %w", err) + } + defer f.Close() + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("cmudict: begin tx: %w", err) + } + defer tx.Rollback() + + stmt, err := tx.Prepare(` + INSERT OR IGNORE INTO pronunciations (word_id, format, value, source) + SELECT id, 'cmu', ?, 'cmudict' FROM words WHERE word = ? AND lang = 'en'`) + if err != nil { + return fmt.Errorf("cmudict: prepare: %w", err) + } + defer stmt.Close() + + scanner := bufio.NewScanner(f) + var count int + + for scanner.Scan() { + line := scanner.Text() + if line == "" || strings.HasPrefix(line, ";;;") { + continue + } + + // Format: "WORD PH1 PH2 PH3" (two spaces between word and phonemes) + // Some entries have variant markers like "WORD(2) PH1 PH2" + parts := strings.SplitN(line, " ", 2) + if len(parts) != 2 { + // Try single space (some versions) + idx := strings.IndexByte(line, ' ') + if idx == -1 { + continue + } + parts = []string{line[:idx], strings.TrimSpace(line[idx+1:])} + } + + word := strings.ToLower(strings.TrimSpace(parts[0])) + phonemes := strings.TrimSpace(parts[1]) + + if word == "" || phonemes == "" { + continue + } + + // Skip variant entries like "WORD(2)" — keep only primary pronunciation + if strings.Contains(word, "(") { + continue + } + + if containsDigit(word) || containsSpace(word) { + continue + } + + res, err := stmt.Exec(phonemes, word) + if err != nil { + return fmt.Errorf("cmudict: insert: %w", err) + } + if n, _ := res.RowsAffected(); n > 0 { + count++ + } + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("cmudict: scan: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("cmudict: commit: %w", err) + } + slog.Info("cmudict loaded", "pronunciations", count) + return nil +} diff --git a/internal/loader/difficulty.go b/internal/loader/difficulty.go new file mode 100644 index 0000000..21ec191 --- /dev/null +++ b/internal/loader/difficulty.go @@ -0,0 +1,72 @@ +package loader + +import ( + "database/sql" + "fmt" + "log/slog" +) + +// DifficultyScorer computes a composite difficulty score for all words +// in the database. It runs after all other loaders have completed. +// Score range: 0.0 (easiest) to 1.0 (hardest). +// +// Formula: +// difficulty = normalize(1/frequency) * 0.5 +// + normalize(length) * 0.3 +// + normalize(syllable_count_approx) * 0.2 +// +// Computed entirely in SQL for performance — no row-by-row round trips. +type DifficultyScorer struct{} + +func (DifficultyScorer) Name() string { return "difficulty" } + +func (DifficultyScorer) Load(db *sql.DB, _ string) error { + langs := []string{"en", "fr", "pt-PT", "zh"} + + var totalUpdated int64 + for _, lang := range langs { + n, err := scoreLang(db, lang) + if err != nil { + return fmt.Errorf("difficulty: %s: %w", lang, err) + } + totalUpdated += n + } + + slog.Info("difficulty scored", "total_words", totalUpdated) + return nil +} + +func scoreLang(db *sql.DB, lang string) (int64, error) { + // Single UPDATE using SQLite math to compute difficulty in-place. + // log() isn't available in base SQLite, so we approximate the frequency + // component using a reciprocal: 1.0 / (1.0 + frequency/max_freq). + // This gives a 0–1 range where 0 = most common, ~1 = rarest. + // + // Length and a rough syllable proxy (length/3) are normalized against + // per-language maximums via a subquery. + const q = ` + UPDATE words SET difficulty = ROUND( + CASE + WHEN stats.max_freq > 0 AND frequency > 0 + THEN (1.0 - CAST(frequency AS REAL) / stats.max_freq) * 0.5 + ELSE 0.5 + END + + (CAST(LENGTH(word) AS REAL) / stats.max_len) * 0.3 + + MIN(1.0, CAST(LENGTH(word) AS REAL) / 3.0 / (stats.max_len / 3.0)) * 0.2 + , 3) + FROM ( + SELECT + MAX(frequency) AS max_freq, + MAX(LENGTH(word)) AS max_len + FROM words WHERE lang = ?1 + ) AS stats + WHERE words.lang = ?1 + ` + + res, err := db.Exec(q, lang) + if err != nil { + return 0, fmt.Errorf("update difficulty: %w", err) + } + n, _ := res.RowsAffected() + return n, nil +} diff --git a/internal/loader/omw.go b/internal/loader/omw.go new file mode 100644 index 0000000..e4791b6 --- /dev/null +++ b/internal/loader/omw.go @@ -0,0 +1,159 @@ +package loader + +import ( + "bufio" + "database/sql" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" +) + +// OMWLoader loads Open Multilingual Wordnet data for Portuguese, +// mapping pt-PT words to Princeton WordNet synset IDs. +type OMWLoader struct{} + +func (OMWLoader) Name() string { return "omw-pt" } + +func (OMWLoader) Load(db *sql.DB, dataDir string) error { + // OMW Portuguese data comes as a tab-separated file. + // Format varies by release but typically: + // synset_idrelationword + // where synset_id is like "eng-30-00001740-n" and relation is "lemma" + // + // Also supports the WN-LMF XML format and the simpler tab format from + // https://github.com/omwn/omw-data + + // Try multiple possible file locations + candidates := []string{ + filepath.Join(dataDir, "omw", "wn-data-por.tab"), + filepath.Join(dataDir, "omw", "wn-por.tab"), + filepath.Join(dataDir, "omw-pt.tab"), + } + + var path string + for _, c := range candidates { + if _, err := os.Stat(c); err == nil { + path = c + break + } + } + if path == "" { + slog.Warn("omw-pt: no data file found, skipping", "searched", candidates) + return nil + } + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("omw: begin tx: %w", err) + } + defer tx.Rollback() + + stmtSynset, err := tx.Prepare(` + INSERT OR IGNORE INTO synsets (synset_id, pos) VALUES (?, ?)`) + if err != nil { + return fmt.Errorf("omw: prepare synset: %w", err) + } + defer stmtSynset.Close() + + stmtWordSynset, err := tx.Prepare(` + INSERT OR IGNORE INTO word_synsets (word_id, synset_id, source) + SELECT w.id, s.id, 'omw' + FROM words w, synsets s + WHERE w.word = ? AND w.lang = 'pt-PT' AND s.synset_id = ?`) + if err != nil { + return fmt.Errorf("omw: prepare word_synset: %w", err) + } + defer stmtWordSynset.Close() + + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("omw: open: %w", err) + } + defer f.Close() + + posMap := map[string]string{ + "n": "noun", + "v": "verb", + "a": "adjective", + "r": "adverb", + } + + scanner := bufio.NewScanner(f) + var synsetCount, linkCount int + + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "#") || line == "" { + continue + } + + fields := strings.Split(line, "\t") + if len(fields) < 3 { + continue + } + + synsetRaw := fields[0] + relation := fields[1] + word := strings.ToLower(strings.TrimSpace(fields[2])) + + // Only process lemma relations + if relation != "lemma" { + continue + } + + if word == "" || containsDigit(word) || containsSpace(word) { + continue + } + + // Normalize synset ID: "eng-30-00001740-n" -> "00001740-n" + synsetID := normalizeOMWSynsetID(synsetRaw) + if synsetID == "" { + continue + } + + // Extract POS from synset ID + parts := strings.Split(synsetID, "-") + if len(parts) != 2 { + continue + } + pos := posMap[parts[1]] + if pos == "" { + continue + } + + if _, err := stmtSynset.Exec(synsetID, pos); err != nil { + return fmt.Errorf("omw: insert synset: %w", err) + } + synsetCount++ + + if _, err := stmtWordSynset.Exec(word, synsetID); err != nil { + return fmt.Errorf("omw: insert word_synset: %w", err) + } + linkCount++ + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("omw: scan: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("omw: commit: %w", err) + } + slog.Info("omw-pt loaded", "synsets", synsetCount, "links", linkCount) + return nil +} + +func normalizeOMWSynsetID(raw string) string { + // Format: "eng-30-00001740-n" -> "00001740-n" + parts := strings.Split(raw, "-") + if len(parts) >= 4 && parts[0] == "eng" { + return parts[2] + "-" + parts[3] + } + // Already normalized + if len(parts) == 2 && len(parts[0]) == 8 { + return raw + } + return "" +} diff --git a/internal/loader/scowl.go b/internal/loader/scowl.go index 73ce252..37a92ad 100644 --- a/internal/loader/scowl.go +++ b/internal/loader/scowl.go @@ -68,7 +68,8 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error { } defer tx.Rollback() - stmt, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang, frequency) VALUES (?, 'en', ?)") + stmt, err := tx.Prepare(`INSERT INTO words (word, lang, frequency) VALUES (?, 'en', ?) + ON CONFLICT(word, lang) DO UPDATE SET frequency = MAX(frequency, excluded.frequency)`) if err != nil { return fmt.Errorf("scowl: prepare: %w", err) } diff --git a/internal/loader/subtlex.go b/internal/loader/subtlex.go new file mode 100644 index 0000000..90fc0ed --- /dev/null +++ b/internal/loader/subtlex.go @@ -0,0 +1,129 @@ +package loader + +import ( + "bufio" + "database/sql" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" +) + +// SubtlexLoader loads SUBTLEX-US frequency data for English words. +// Source: subtitle frequency corpus (Brysbaert & New, 2009). +// File: SUBTLEX-US.tsv (tab-separated, exported from the original XLSX). +type SubtlexLoader struct{} + +func (SubtlexLoader) Name() string { return "subtlex" } + +func (SubtlexLoader) Load(db *sql.DB, dataDir string) error { + candidates := []string{ + filepath.Join(dataDir, "SUBTLEX-US.tsv"), + filepath.Join(dataDir, "subtlex-us.tsv"), + filepath.Join(dataDir, "SUBTLEX-US.txt"), + filepath.Join(dataDir, "SUBTLEX-US.csv"), + filepath.Join(dataDir, "SUBTLEXus74286wordstextversion.tsv"), + } + + var path string + for _, c := range candidates { + if _, err := os.Stat(c); err == nil { + path = c + break + } + } + // Last resort: glob for anything with "subtlex" in the name + if path == "" { + matches, _ := filepath.Glob(filepath.Join(dataDir, "*[Ss][Uu][Bb][Tt][Ll][Ee][Xx]*")) + if len(matches) > 0 { + path = matches[0] + } + } + if path == "" { + slog.Warn("subtlex: no data file found, skipping", "searched", candidates) + return nil + } + + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("subtlex: open: %w", err) + } + defer f.Close() + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("subtlex: begin tx: %w", err) + } + defer tx.Rollback() + + stmt, err := tx.Prepare(` + UPDATE words SET frequency = ? WHERE word = ? AND lang = 'en' AND frequency = 0`) + if err != nil { + return fmt.Errorf("subtlex: prepare: %w", err) + } + defer stmt.Close() + + scanner := bufio.NewScanner(f) + // Skip header + if scanner.Scan() { + // Determine column layout from header + } + + var count int + for scanner.Scan() { + line := scanner.Text() + fields := strings.Split(line, "\t") + if len(fields) < 2 { + continue + } + + word := strings.ToLower(strings.TrimSpace(fields[0])) + if word == "" || containsDigit(word) || containsSpace(word) { + continue + } + + // Frequency per million — try common column positions + // SUBTLEX-US format varies, but frequency per million is typically col 5 or 6 + var freqPerMillion float64 + for _, idx := range []int{5, 4, 3, 1} { + if idx < len(fields) { + if f, err := strconv.ParseFloat(strings.TrimSpace(fields[idx]), 64); err == nil && f > 0 { + freqPerMillion = f + break + } + } + } + if freqPerMillion <= 0 { + continue + } + + // Convert to integer score (multiply by 100, cap at 10000) + freq := int(freqPerMillion * 100) + if freq > 10000 { + freq = 10000 + } + if freq <= 0 { + freq = 1 + } + + res, err := stmt.Exec(freq, word) + if err != nil { + return fmt.Errorf("subtlex: update: %w", err) + } + if n, _ := res.RowsAffected(); n > 0 { + count++ + } + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("subtlex: scan: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("subtlex: commit: %w", err) + } + slog.Info("subtlex loaded", "updated_words", count) + return nil +} diff --git a/internal/loader/wiktionary.go b/internal/loader/wiktionary.go index 98c6f7d..8bfcd62 100644 --- a/internal/loader/wiktionary.go +++ b/internal/loader/wiktionary.go @@ -34,10 +34,17 @@ type wiktEntry struct { Synonyms []struct { Word string `json:"word"` } `json:"synonyms"` + Antonyms []struct { + Word string `json:"word"` + } `json:"antonyms"` Translations []struct { Code string `json:"code"` Word string `json:"word"` } `json:"translations"` + Sounds []struct { + IPA string `json:"ipa"` + } `json:"sounds"` + EtymologyText string `json:"etymology_text"` } var wiktPOSMap = map[string]string{ @@ -59,6 +66,9 @@ const ( defSQL = `INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority) SELECT id, ?, ?, 'wiktionary', 20 FROM words WHERE word = ? AND lang = ?` synSQL = `INSERT OR IGNORE INTO synonyms (word_id, synonym, source) SELECT id, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?` transSQL = `INSERT OR IGNORE INTO translations (word_id, translation, target_lang, source) SELECT id, ?, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?` + antSQL = `INSERT OR IGNORE INTO antonyms (word_id, antonym, source) SELECT id, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?` + pronSQL = `INSERT OR IGNORE INTO pronunciations (word_id, format, value, source) SELECT id, 'ipa', ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?` + etymSQL = `INSERT OR IGNORE INTO etymology (word_id, text, source) SELECT id, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?` ) // txBatch owns a transaction and its prepared statements. @@ -68,6 +78,9 @@ type txBatch struct { stmtDef *sql.Stmt stmtSyn *sql.Stmt stmtTrans *sql.Stmt + stmtAnt *sql.Stmt + stmtPron *sql.Stmt + stmtEtym *sql.Stmt } func newTxBatch(db *sql.DB) (*txBatch, error) { @@ -77,41 +90,67 @@ func newTxBatch(db *sql.DB) (*txBatch, error) { } b := &txBatch{tx: tx} - b.stmtDef, err = tx.Prepare(defSQL) - if err != nil { - tx.Rollback() + prepareOrRollback := func(sql string) (*sql.Stmt, error) { + s, e := tx.Prepare(sql) + if e != nil { + b.closeStmts() + tx.Rollback() + } + return s, e + } + + if b.stmtDef, err = prepareOrRollback(defSQL); err != nil { return nil, fmt.Errorf("prepare def: %w", err) } - b.stmtSyn, err = tx.Prepare(synSQL) - if err != nil { - b.stmtDef.Close() - tx.Rollback() + if b.stmtSyn, err = prepareOrRollback(synSQL); err != nil { return nil, fmt.Errorf("prepare syn: %w", err) } - b.stmtTrans, err = tx.Prepare(transSQL) - if err != nil { - b.stmtDef.Close() - b.stmtSyn.Close() - tx.Rollback() + if b.stmtTrans, err = prepareOrRollback(transSQL); err != nil { return nil, fmt.Errorf("prepare trans: %w", err) } + if b.stmtAnt, err = prepareOrRollback(antSQL); err != nil { + return nil, fmt.Errorf("prepare ant: %w", err) + } + if b.stmtPron, err = prepareOrRollback(pronSQL); err != nil { + return nil, fmt.Errorf("prepare pron: %w", err) + } + if b.stmtEtym, err = prepareOrRollback(etymSQL); err != nil { + return nil, fmt.Errorf("prepare etym: %w", err) + } return b, nil } +func (b *txBatch) closeStmts() { + if b.stmtDef != nil { + b.stmtDef.Close() + } + if b.stmtSyn != nil { + b.stmtSyn.Close() + } + if b.stmtTrans != nil { + b.stmtTrans.Close() + } + if b.stmtAnt != nil { + b.stmtAnt.Close() + } + if b.stmtPron != nil { + b.stmtPron.Close() + } + if b.stmtEtym != nil { + b.stmtEtym.Close() + } +} + func (b *txBatch) Close() { if b == nil { return } - b.stmtDef.Close() - b.stmtSyn.Close() - b.stmtTrans.Close() + b.closeStmts() b.tx.Rollback() // no-op after commit } func (b *txBatch) Commit() error { - b.stmtDef.Close() - b.stmtSyn.Close() - b.stmtTrans.Close() + b.closeStmts() return b.tx.Commit() } @@ -131,8 +170,8 @@ func loadWiktionary(db *sql.DB, path, lang string) error { scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 0, 10*1024*1024), 10*1024*1024) - var defCount, synCount, transCount, lineCount int - commitInterval := 10000 + var defCount, synCount, transCount, antCount, pronCount, etymCount, lineCount int + commitInterval := 50000 for scanner.Scan() { lineCount++ @@ -183,6 +222,40 @@ func loadWiktionary(db *sql.DB, path, lang string) error { synCount++ } + // Process antonyms + for _, ant := range entry.Antonyms { + antWord := strings.ToLower(ant.Word) + if antWord == "" || containsDigit(antWord) || containsSpace(antWord) { + continue + } + if _, err := batch.stmtAnt.Exec(antWord, word, lang); err != nil { + return fmt.Errorf("wiktionary: insert ant: %w", err) + } + antCount++ + } + + // Process pronunciations (IPA) + for _, sound := range entry.Sounds { + ipa := strings.TrimSpace(sound.IPA) + if ipa == "" { + continue + } + if _, err := batch.stmtPron.Exec(ipa, word, lang); err != nil { + return fmt.Errorf("wiktionary: insert pron: %w", err) + } + pronCount++ + break // only first IPA per entry + } + + // Process etymology + etymText := strings.TrimSpace(entry.EtymologyText) + if etymText != "" && len(etymText) > 10 && !strings.HasPrefix(etymText, "See ") { + if _, err := batch.stmtEtym.Exec(etymText, word, lang); err != nil { + return fmt.Errorf("wiktionary: insert etym: %w", err) + } + etymCount++ + } + // Process translations for _, tr := range entry.Translations { targetLang, ok := wiktLangCodeMap[tr.Code] @@ -218,7 +291,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error { return fmt.Errorf("wiktionary: final commit: %w", err) } - slog.Info("wiktionary loaded", "lang", lang, "definitions", defCount, "synonyms", synCount, "translations", transCount) + slog.Info("wiktionary loaded", "lang", lang, "definitions", defCount, "synonyms", synCount, "translations", transCount, "antonyms", antCount, "pronunciations", pronCount, "etymologies", etymCount) return nil } diff --git a/internal/loader/wolf.go b/internal/loader/wolf.go index ef13564..7dbb7d3 100644 --- a/internal/loader/wolf.go +++ b/internal/loader/wolf.go @@ -58,14 +58,32 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error { } defer stmtSyn.Close() + stmtSynset, err := tx.Prepare(` + INSERT OR IGNORE INTO synsets (synset_id, pos) VALUES (?, ?)`) + if err != nil { + return fmt.Errorf("wolf: prepare synset: %w", err) + } + defer stmtSynset.Close() + + stmtWordSynset, err := tx.Prepare(` + INSERT OR IGNORE INTO word_synsets (word_id, synset_id, source) + SELECT w.id, s.id, 'wolf' + FROM words w, synsets s + WHERE w.word = ? AND w.lang = 'fr' AND s.synset_id = ?`) + if err != nil { + return fmt.Errorf("wolf: prepare word_synset: %w", err) + } + defer stmtWordSynset.Close() + posMap := map[string]string{ "n": "noun", "v": "verb", "a": "adjective", "r": "adverb", + "b": "adverb", // WOLF uses "b" for adverb } - var defCount, synCount, decodeErrors int + var defCount, synCount, synsetCount, decodeErrors int decoder := xml.NewDecoder(reader) for { @@ -83,6 +101,8 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error { } var synset struct { + ID string `xml:"ID"` + IDAlt string `xml:"id,attr"` POS string `xml:"POS"` Literals []struct { Value string `xml:",chardata"` @@ -95,9 +115,17 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error { continue } - pos := posMap[strings.ToLower(synset.POS)] + pos := posMap[strings.ToLower(strings.TrimSpace(synset.POS))] gloss := strings.TrimSpace(synset.DEF) + // WOLF synset IDs are Princeton WordNet IDs (e.g., "eng-30-00914031-a") + // Normalize to our format: "00914031-a" + rawID := synset.ID + if rawID == "" { + rawID = synset.IDAlt + } + wolfSynsetID := normalizeWOLFSynsetID(rawID) + var words []string for _, lit := range synset.Literals { w := strings.ToLower(strings.TrimSpace(lit.Value)) @@ -106,6 +134,19 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error { } } + // Insert synset and link words + if wolfSynsetID != "" && pos != "" { + if _, err := stmtSynset.Exec(wolfSynsetID, pos); err != nil { + return fmt.Errorf("wolf: insert synset: %w", err) + } + synsetCount++ + for _, w := range words { + if _, err := stmtWordSynset.Exec(w, wolfSynsetID); err != nil { + return fmt.Errorf("wolf: insert word_synset: %w", err) + } + } + } + if gloss != "" { for _, w := range words { if _, err := stmtDef.Exec(pos, gloss, w); err != nil { @@ -135,6 +176,35 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error { if err := tx.Commit(); err != nil { return fmt.Errorf("wolf: commit: %w", err) } - slog.Info("wolf loaded", "definitions", defCount, "synonyms", synCount) + slog.Info("wolf loaded", "definitions", defCount, "synonyms", synCount, "synsets", synsetCount) return nil } + +// normalizeWOLFSynsetID extracts the Princeton synset ID from WOLF format. +// WOLF uses "eng-30-XXXXXXXX-P" format; we want "XXXXXXXX-P". +// WOLF uses "b" for adverb POS but WordNet uses "r", so we remap. +func normalizeWOLFSynsetID(wolfID string) string { + wolfID = strings.TrimSpace(wolfID) + // Common format: "eng-30-00914031-a" + parts := strings.Split(wolfID, "-") + if len(parts) >= 4 && parts[0] == "eng" { + posSuffix := wolfPosSuffix(parts[3]) + return parts[2] + "-" + posSuffix + } + // Fallback: if it's already in our format + if len(parts) == 2 && len(parts[0]) == 8 { + return parts[0] + "-" + wolfPosSuffix(parts[1]) + } + return "" +} + +// wolfPosSuffix maps WOLF POS codes to WordNet synset ID suffixes. +func wolfPosSuffix(pos string) string { + pos = strings.TrimSpace(pos) + switch pos { + case "b": + return "r" // WOLF "b" (adverb) → WordNet "r" + default: + return pos + } +} diff --git a/internal/loader/wordnet.go b/internal/loader/wordnet.go index 07158b8..85bc61c 100644 --- a/internal/loader/wordnet.go +++ b/internal/loader/wordnet.go @@ -15,6 +15,24 @@ type WordNetLoader struct{} func (WordNetLoader) Name() string { return "wordnet" } +// ssTypeMap maps WordNet ss_type codes to POS and synset pos suffix. +var ssTypeMap = map[string]string{ + "n": "noun", + "v": "verb", + "a": "adjective", + "s": "adjective", // satellite adjective + "r": "adverb", +} + +// ssTypeSuffix maps WordNet ss_type to synset ID suffix character. +var ssTypeSuffix = map[string]string{ + "n": "n", + "v": "v", + "a": "a", + "s": "a", // satellite adjectives share 'a' namespace + "r": "r", +} + func (WordNetLoader) Load(db *sql.DB, dataDir string) error { posFiles := map[string]string{ "data.noun": "noun", @@ -45,111 +63,219 @@ func (WordNetLoader) Load(db *sql.DB, dataDir string) error { } defer stmtSyn.Close() - var defCount, synCount int + stmtAnt, err := tx.Prepare(` + INSERT OR IGNORE INTO antonyms (word_id, antonym, source) + SELECT id, ?, 'wordnet' FROM words WHERE word = ? AND lang = 'en'`) + if err != nil { + return fmt.Errorf("wordnet: prepare ant: %w", err) + } + defer stmtAnt.Close() + + stmtSynset, err := tx.Prepare(` + INSERT OR IGNORE INTO synsets (synset_id, pos) VALUES (?, ?)`) + if err != nil { + return fmt.Errorf("wordnet: prepare synset: %w", err) + } + defer stmtSynset.Close() + + stmtWordSynset, err := tx.Prepare(` + INSERT OR IGNORE INTO word_synsets (word_id, synset_id, source) + SELECT w.id, s.id, 'wordnet' + FROM words w, synsets s + WHERE w.word = ? AND w.lang = 'en' AND s.synset_id = ?`) + if err != nil { + return fmt.Errorf("wordnet: prepare word_synset: %w", err) + } + defer stmtWordSynset.Close() + + var defCount, synCount, antCount, synsetCount int for file, pos := range posFiles { path := filepath.Join(dataDir, "wordnet", "dict", file) - d, s, err := loadWordNetFile(stmtDef, stmtSyn, path, pos) + d, s, a, sc, err := loadWordNetFile(stmtDef, stmtSyn, stmtAnt, stmtSynset, stmtWordSynset, path, pos) if err != nil { return fmt.Errorf("wordnet: %s: %w", file, err) } defCount += d synCount += s + antCount += a + synsetCount += sc } if err := tx.Commit(); err != nil { return fmt.Errorf("wordnet: commit: %w", err) } - slog.Info("wordnet loaded", "definitions", defCount, "synonyms", synCount) + slog.Info("wordnet loaded", "definitions", defCount, "synonyms", synCount, "antonyms", antCount, "synsets", synsetCount) return nil } -func loadWordNetFile(stmtDef, stmtSyn *sql.Stmt, path, pos string) (int, int, error) { +func loadWordNetFile(stmtDef, stmtSyn, stmtAnt, stmtSynset, stmtWordSynset *sql.Stmt, path, pos string) (int, int, int, int, error) { f, err := os.Open(path) if err != nil { - return 0, 0, err + return 0, 0, 0, 0, err } defer f.Close() - var defCount, synCount int + // Single pass: insert defs/syns/synsets as we go, collect lightweight + // antonym pointers + synset->words map for deferred antonym resolution. + type deferredAnt struct { + srcWord string + targetSynsetKey string + tgtWordIdx int + } + + synsetWords := make(map[string][]string) // only stores word lists, not full parse data + var pendingAnts []deferredAnt + var defCount, synCount, antCount, synsetCount int + scanner := bufio.NewScanner(f) - buf := make([]byte, 0, 64*1024) - scanner.Buffer(buf, 1024*1024) + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) for scanner.Scan() { line := scanner.Text() - // Skip header lines (start with two spaces) if strings.HasPrefix(line, " ") { continue } - words, gloss := parseWordNetLine(line) - if gloss == "" || len(words) == 0 { + parsed := parseWordNetLine(line) + if parsed.gloss == "" || len(parsed.words) == 0 { continue } - // Insert definitions for each word - for _, w := range words { - if _, err := stmtDef.Exec(pos, gloss, w); err != nil { - return 0, 0, err + // Build synset->words map (needed for antonym resolution) + if parsed.synsetID != "" { + synsetWords[parsed.synsetID] = parsed.words + + if _, err := stmtSynset.Exec(parsed.synsetID, pos); err != nil { + return 0, 0, 0, 0, err + } + synsetCount++ + + for _, w := range parsed.words { + if _, err := stmtWordSynset.Exec(w, parsed.synsetID); err != nil { + return 0, 0, 0, 0, err + } + } + } + + // Insert definitions + for _, w := range parsed.words { + if _, err := stmtDef.Exec(pos, parsed.gloss, w); err != nil { + return 0, 0, 0, 0, err } defCount++ } - // Insert synonym pairs (skip words with underscores) + // Insert synonym pairs var cleanWords []string - for _, w := range words { + for _, w := range parsed.words { if !strings.Contains(w, "_") { cleanWords = append(cleanWords, w) } } for i, w1 := range cleanWords { for j, w2 := range cleanWords { - if i == j { - continue + if i != j { + if _, err := stmtSyn.Exec(w2, w1); err != nil { + return 0, 0, 0, 0, err + } + synCount++ } - if _, err := stmtSyn.Exec(w2, w1); err != nil { - return 0, 0, err - } - synCount++ } } + + // Collect antonym pointers for deferred resolution + for _, ap := range parsed.antonymPtrs { + if ap.srcWordIdx < 1 || ap.srcWordIdx > len(parsed.words) { + continue + } + srcWord := strings.ToLower(parsed.words[ap.srcWordIdx-1]) + if strings.Contains(srcWord, "_") { + continue + } + pendingAnts = append(pendingAnts, deferredAnt{ + srcWord: srcWord, + targetSynsetKey: ap.targetSynsetKey, + tgtWordIdx: ap.tgtWordIdx, + }) + } } - return defCount, synCount, scanner.Err() + if err := scanner.Err(); err != nil { + return 0, 0, 0, 0, err + } + + // Resolve deferred antonyms now that all synsets are mapped + for _, da := range pendingAnts { + tgtWords, ok := synsetWords[da.targetSynsetKey] + if !ok || da.tgtWordIdx < 1 || da.tgtWordIdx > len(tgtWords) { + continue + } + tgtWord := strings.ToLower(tgtWords[da.tgtWordIdx-1]) + if strings.Contains(tgtWord, "_") || tgtWord == da.srcWord { + continue + } + if _, err := stmtAnt.Exec(tgtWord, da.srcWord); err != nil { + return 0, 0, 0, 0, err + } + if _, err := stmtAnt.Exec(da.srcWord, tgtWord); err != nil { + return 0, 0, 0, 0, err + } + antCount += 2 + } + + return defCount, synCount, antCount, synsetCount, nil } -func parseWordNetLine(line string) ([]string, string) { +type wordNetParsed struct { + words []string + gloss string + synsetID string + // antonymPtrs: each entry is {targetSynsetKey, sourceWordIdx, targetWordIdx} + antonymPtrs []antonymPtr +} + +type antonymPtr struct { + targetSynsetKey string // "offset-pos" e.g. "00123456-a" + srcWordIdx int // 1-based word index in source synset + tgtWordIdx int // 1-based word index in target synset +} + +func parseWordNetLine(line string) wordNetParsed { // Format: synset_offset lex_filenum ss_type w_cnt word lex_id [word lex_id...] p_cnt [ptr...] | gloss glossIdx := strings.Index(line, "| ") if glossIdx == -1 { - return nil, "" + return wordNetParsed{} } gloss := strings.TrimSpace(line[glossIdx+2:]) - // Take gloss before first ";" (drops usage examples) if semiIdx := strings.Index(gloss, ";"); semiIdx != -1 { gloss = strings.TrimSpace(gloss[:semiIdx]) } if gloss == "" { - return nil, "" + return wordNetParsed{} } dataPart := line[:glossIdx] fields := strings.Fields(dataPart) if len(fields) < 6 { - return nil, "" + return wordNetParsed{} + } + + synsetOffset := fields[0] + ssType := fields[2] + posSuffix := ssTypeSuffix[ssType] + synsetID := "" + if posSuffix != "" { + synsetID = synsetOffset + "-" + posSuffix } - // fields[3] = w_cnt (hex) wc, err := strconv.ParseInt(fields[3], 16, 0) if err != nil { - slog.Debug("wordnet: bad word count", "field", fields[3], "error", err) - return nil, "" + return wordNetParsed{} } wordCount := int(wc) if wordCount <= 0 || wordCount > 100 { - slog.Debug("wordnet: unreasonable word count", "count", wordCount) - return nil, "" + return wordNetParsed{} } var words []string @@ -157,5 +283,41 @@ func parseWordNetLine(line string) ([]string, string) { w := strings.ToLower(fields[4+i*2]) words = append(words, w) } - return words, gloss + + // Parse pointers + ptrStart := 4 + wordCount*2 + var antPtrs []antonymPtr + if ptrStart < len(fields) { + pc, err := strconv.Atoi(fields[ptrStart]) + if err == nil { + for i := 0; i < pc; i++ { + base := ptrStart + 1 + i*4 + if base+3 >= len(fields) { + break + } + if fields[base] != "!" { + continue + } + tgtOffset := fields[base+1] + tgtPosChar := fields[base+2] + srcTgt := fields[base+3] + if len(srcTgt) != 4 { + continue + } + srcIdx, e1 := strconv.ParseInt(srcTgt[0:2], 16, 0) + tgtIdx, e2 := strconv.ParseInt(srcTgt[2:4], 16, 0) + if e1 != nil || e2 != nil { + continue + } + tgtKey := tgtOffset + "-" + tgtPosChar + antPtrs = append(antPtrs, antonymPtr{ + targetSynsetKey: tgtKey, + srcWordIdx: int(srcIdx), + tgtWordIdx: int(tgtIdx), + }) + } + } + } + + return wordNetParsed{words: words, gloss: gloss, synsetID: synsetID, antonymPtrs: antPtrs} } diff --git a/scripts/download-dict-data.sh b/scripts/download-dict-data.sh index 98cb6c3..b1e01f2 100755 --- a/scripts/download-dict-data.sh +++ b/scripts/download-dict-data.sh @@ -143,7 +143,10 @@ fi # ============================================================ # English — WordNet # ============================================================ -echo "=== WordNet (English definitions + synonyms) ===" +echo "=== WordNet 3.0 (English definitions + synonyms) ===" +# IMPORTANT: We use WordNet 3.0, NOT 3.1, because WOLF's synset IDs +# use WN 3.0 byte offsets (eng-30-XXXXXXXX). WN 3.1 has different +# offsets, so cross-language synset backing would silently fail. WN_DIR="$DATA_DIR/wordnet" if [[ -d "$WN_DIR/dict" && "$FORCE" != "true" ]]; then # Verify the dict directory has the expected data files @@ -156,19 +159,89 @@ if [[ -d "$WN_DIR/dict" && "$FORCE" != "true" ]]; then fi if [[ ! -d "$WN_DIR/dict" ]]; then WN_TAR="$DATA_DIR/wordnet.tar.gz" - # Princeton's official URL is down (403); use GitHub mirror - download "https://raw.githubusercontent.com/zweiein/WNdb/master/WNdb-3.1.tar.gz" "$WN_TAR" + # Use WordNet 3.0 to match WOLF synset offsets + download "https://wordnetcode.princeton.edu/3.0/WNdb-3.0.tar.gz" "$WN_TAR" 2>/dev/null || { + echo " [FAIL] Could not download WordNet 3.0" + exit 1 + } echo " [extract] $WN_TAR" mkdir -p "$WN_DIR" tar -xzf "$WN_TAR" -C "$WN_DIR" save_checksum "wordnet" "$WN_TAR" rm -f "$WN_TAR" - # The archive extracts to WNdb-3.1/dict/ — normalize to wordnet/dict/ - if [[ -d "$WN_DIR/WNdb-3.1/dict" ]]; then - mv "$WN_DIR/WNdb-3.1/dict" "$WN_DIR/dict" - rm -rf "$WN_DIR/WNdb-3.1" + # Normalize extraction directory — may be WNdb-3.0/ or dict/ + for d in "WNdb-3.0" "WNdb-3.1"; do + if [[ -d "$WN_DIR/$d/dict" ]]; then + mv "$WN_DIR/$d/dict" "$WN_DIR/dict" + rm -rf "$WN_DIR/$d" + break + fi + done +fi + +# ============================================================ +# English — SUBTLEX-US (word frequency) +# ============================================================ +echo "=== SUBTLEX-US (English word frequency) ===" +SUBTLEX_FILE="$DATA_DIR/SUBTLEX-US.tsv" +if [[ -f "$SUBTLEX_FILE" && "$FORCE" != "true" ]]; then + if check_min_size "$SUBTLEX_FILE" 1000000; then + echo " [skip] SUBTLEX-US.tsv already exists (size ok)" + else + echo " [warn] SUBTLEX-US.tsv looks truncated, re-downloading" + rm -f "$SUBTLEX_FILE" fi fi +if [[ ! -f "$SUBTLEX_FILE" ]]; then + # Try the GitHub-hosted copy first (TSV, direct download), then Ghent University. + download "https://raw.githubusercontent.com/Wikipedia2Vec/Wikipedia2Vec/master/resources/SUBTLEX-US.tsv" "$SUBTLEX_FILE" 2>/dev/null || \ + download "https://raw.githubusercontent.com/Wikipedia2Vec/Wikipedia2Vec/refs/heads/master/resources/SUBTLEX-US.tsv" "$SUBTLEX_FILE" 2>/dev/null || true + + if [[ ! -f "$SUBTLEX_FILE" ]]; then + # Fallback: Ghent University zip (may require browser) + SUBTLEX_ZIP="$DATA_DIR/subtlex-us.zip" + download "https://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexus/subtlexus2.zip/at_download/file" "$SUBTLEX_ZIP" 2>/dev/null || \ + download "https://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexus/subtlexus2.zip" "$SUBTLEX_ZIP" 2>/dev/null || true + + if [[ -f "$SUBTLEX_ZIP" ]]; then + echo " [extract] $SUBTLEX_ZIP" + unzip -qo "$SUBTLEX_ZIP" -d "$DATA_DIR/subtlex-tmp" 2>/dev/null || true + FOUND_SUBTLEX=$(find "$DATA_DIR/subtlex-tmp" -type f \( -name "*.tsv" -o -name "*.txt" -o -name "*.csv" \) | head -1) + if [[ -n "$FOUND_SUBTLEX" ]]; then + mv "$FOUND_SUBTLEX" "$SUBTLEX_FILE" + echo " [ok] extracted to SUBTLEX-US.tsv" + else + echo " [warn] could not find TSV in archive — SUBTLEX may need manual conversion from XLSX" + fi + rm -rf "$DATA_DIR/subtlex-tmp" "$SUBTLEX_ZIP" + else + echo " [warn] SUBTLEX-US download failed." + echo " Download manually from: https://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexus" + echo " Export the XLSX to TSV and place as: $SUBTLEX_FILE" + fi + fi +fi + +# ============================================================ +# English — CMU Pronouncing Dictionary +# ============================================================ +echo "=== CMUdict (English pronunciation) ===" +CMUDICT_FILE="$DATA_DIR/cmudict-0.7b" +if [[ -f "$CMUDICT_FILE" && "$FORCE" != "true" ]]; then + if check_min_size "$CMUDICT_FILE" 2000000; then + echo " [skip] cmudict-0.7b already exists (size ok)" + else + echo " [warn] cmudict-0.7b looks truncated, re-downloading" + rm -f "$CMUDICT_FILE" + fi +fi +if [[ ! -f "$CMUDICT_FILE" ]]; then + download "https://svn.code.sf.net/p/cmusphinx/code/trunk/cmudict/cmudict-0.7b" "$CMUDICT_FILE" 2>/dev/null || \ + download "https://raw.githubusercontent.com/cmusphinx/cmudict/master/cmudict-0.7b" "$CMUDICT_FILE" 2>/dev/null || \ + download "https://raw.githubusercontent.com/Alexir/CMUdict/master/cmudict-0.7b" "$CMUDICT_FILE" 2>/dev/null || { + echo " [warn] CMUdict download failed. Download manually and place at: $CMUDICT_FILE" + } +fi # ============================================================ # English — Wiktionary (kaikki) @@ -206,8 +279,30 @@ for dic in "$HUNSPELL_DIR/fr_FR/fr.dic" "$HUNSPELL_DIR/pt_PT/pt_PT.dic"; do fi done cp -f "$HUNSPELL_DIR/fr_FR/fr.dic" "$DATA_DIR/fr_FR/fr.dic" -cp -f "$HUNSPELL_DIR/fr_FR/fr.aff" "$DATA_DIR/fr_FR/fr.aff" 2>/dev/null || true +if [[ -f "$HUNSPELL_DIR/fr_FR/fr.aff" ]]; then + cp -f "$HUNSPELL_DIR/fr_FR/fr.aff" "$DATA_DIR/fr_FR/fr.aff" +else + FR_AFF=$(find "$HUNSPELL_DIR/fr_FR" -name "*.aff" -type f 2>/dev/null | head -1) + if [[ -n "$FR_AFF" ]]; then + cp -f "$FR_AFF" "$DATA_DIR/fr_FR/fr.aff" + echo " [ok] found fr .aff file at: $FR_AFF" + else + echo " [warn] no .aff file found for fr — affix expansion will be skipped" + fi +fi cp -f "$HUNSPELL_DIR/pt_PT/pt_PT.dic" "$DATA_DIR/pt_PT/pt_PT.dic" +# .aff file may be in pt_PT/ or at a different path in the repo +if [[ -f "$HUNSPELL_DIR/pt_PT/pt_PT.aff" ]]; then + cp -f "$HUNSPELL_DIR/pt_PT/pt_PT.aff" "$DATA_DIR/pt_PT/pt_PT.aff" +else + PT_AFF=$(find "$HUNSPELL_DIR/pt_PT" -name "*.aff" -type f 2>/dev/null | head -1) + if [[ -n "$PT_AFF" ]]; then + cp -f "$PT_AFF" "$DATA_DIR/pt_PT/pt_PT.aff" + echo " [ok] found pt_PT .aff file at: $PT_AFF" + else + echo " [warn] no .aff file found for pt_PT — affix expansion will be skipped" + fi +fi # ============================================================ # French — Lexique @@ -313,6 +408,55 @@ if [[ ! -f "$DATA_DIR/dicionario-aberto.xml" ]]; then fi fi +# ============================================================ +# pt-PT — CETEMPúblico frequency (or fallback frequency list) +# ============================================================ +echo "=== Portuguese frequency data ===" +PT_FREQ_FILE="$DATA_DIR/cetempublico-freq.tsv" +if [[ -f "$PT_FREQ_FILE" && "$FORCE" != "true" ]]; then + echo " [skip] cetempublico-freq.tsv already exists" +elif [[ -f "$DATA_DIR/pt-freq.tsv" && "$FORCE" != "true" ]]; then + echo " [skip] pt-freq.tsv already exists" +else + echo " [info] CETEMPúblico frequency data requires manual download from Linguateca." + echo " Visit: https://www.linguateca.pt/acesso/corpus.php?corpus=CETEMPUBLICO" + echo " Generate a word frequency list and save as: $PT_FREQ_FILE" + echo " Format: wordfrequency (one per line)" + echo " The import will proceed without frequency data if this file is absent." +fi + +# ============================================================ +# pt-PT — Open Multilingual Wordnet +# ============================================================ +echo "=== Open Multilingual Wordnet (Portuguese synset mappings) ===" +OMW_DIR="$DATA_DIR/omw" +OMW_FILE="$OMW_DIR/wn-data-por.tab" +if [[ -f "$OMW_FILE" && "$FORCE" != "true" ]]; then + echo " [skip] wn-data-por.tab already exists" +fi +if [[ ! -f "$OMW_FILE" ]]; then + OMW_REPO="$DATA_DIR/omw-data-repo" + clone_if_missing "https://github.com/omwn/omw-data.git" "$OMW_REPO" + mkdir -p "$OMW_DIR" + # Find the Portuguese tab file in the repo + FOUND_OMW=$(find "$OMW_REPO" -type f -name "wn-data-por.tab" 2>/dev/null | head -1) + if [[ -n "$FOUND_OMW" ]]; then + cp "$FOUND_OMW" "$OMW_FILE" + echo " [ok] extracted wn-data-por.tab" + else + # Try alternative naming + FOUND_OMW=$(find "$OMW_REPO" -type f -name "*por*" -name "*.tab" 2>/dev/null | head -1) + if [[ -n "$FOUND_OMW" ]]; then + cp "$FOUND_OMW" "$OMW_FILE" + echo " [ok] extracted Portuguese OMW data" + else + echo " [warn] could not find Portuguese data in OMW repo" + echo " The import will proceed without synset mappings for pt-PT." + fi + fi + rm -rf "$OMW_REPO" +fi + # ============================================================ # pt-PT — Wiktionary (kaikki) # ============================================================