From 704c06f41de465755348636e4bd7bda82fc2c158 Mon Sep 17 00:00:00 2001 From: prosolis <5590409+prosolis@users.noreply.github.com> Date: Sat, 4 Apr 2026 15:21:27 -0700 Subject: [PATCH] Add regional variant tagging (us/gb) for English words - SCOWL loader now loads american-words and british-words alongside english-words, tagging each with variant "us", "gb", or NULL (common) - Words appearing in both American and British lists get NULL variant - Add variant column to words table with schema migration - API: /define returns variant field for English, /random and /words accept ?variant=us|gb filter - Add total word count to server status line - Fix rows.Close leak in PopulateCMUTails, add rows.Err() check - Add tests for PopulateCMUTails, cmuTail, TotalWordCount - Expand test seed data and assertions to cover pt-PT definitions, synonyms, and validity checks - Update README with variant docs, /difficulty, /words, /rhyme endpoints, and current database statistics Co-Authored-By: Claude Opus 4.6 --- README.md | 69 ++++++++++---- cmd/server/main.go | 116 ++++++++++++++++------- cmd/server/server_test.go | 50 ++++++++++ internal/dictionary/db.go | 8 +- internal/dictionary/dict.go | 1 + internal/dictionary/dict_test.go | 155 +++++++++++++++++++++++++++++++ internal/dictionary/query.go | 87 +++++++++++++++++ internal/loader/scowl.go | 73 +++++++++++---- 8 files changed, 488 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index 91ebc0a..7e68002 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ curl 'localhost:7777/valid?word=running&lang=en' ### `GET /random` -Return a random word. Optional filters: `pos` (`noun`, `verb`, `adjective`, `adverb`), `min`/`max` (word length), `min_freq` (frequency score), `min_difficulty`/`max_difficulty` (0.0–1.0). +Return a random word. Optional filters: `pos` (`noun`, `verb`, `adjective`, `adverb`), `min`/`max` (word length), `min_freq` (frequency score), `min_difficulty`/`max_difficulty` (0.0–1.0), `variant` (`us` or `gb`, English only). ```bash curl 'localhost:7777/random?lang=en&pos=noun&min=5&max=8&max_difficulty=0.5' @@ -73,16 +73,17 @@ curl 'localhost:7777/random?lang=en&pos=noun&min=5&max=8&max_difficulty=0.5' ### `GET /define` -Return all definitions ordered by source priority. Curated sources (WordNet, WOLF, Dicionário Aberto) appear before Wiktionary supplemental definitions. +Return all definitions ordered by source priority. Curated sources (WordNet, WOLF, Dicionário Aberto) appear before Wiktionary supplemental definitions. For English words, the response includes a `variant` field (`"us"`, `"gb"`, or omitted for common English) indicating regional spelling. ```bash -curl 'localhost:7777/define?word=ephemeral&lang=en' +curl 'localhost:7777/define?word=ardor&lang=en' { - "word": "ephemeral", + "word": "ardor", "lang": "en", + "variant": "us", "definitions": [ - {"pos":"adjective","gloss":"lasting for a very short time","source":"wordnet","priority":10}, - {"pos":"adjective","gloss":"lasting only for a day","source":"wiktionary","priority":20} + {"pos":"noun","gloss":"a feeling of strong eagerness","source":"wordnet","priority":10}, + {"pos":"noun","gloss":"feelings of great warmth and intensity","source":"wordnet","priority":10} ] } ``` @@ -176,6 +177,33 @@ curl 'localhost:7777/frequency/batch?words=house,cat,ephemeral&lang=en' {"lang":"en","frequencies":{"cat":800,"ephemeral":50,"house":800}} ``` +### `GET /difficulty` + +Return the difficulty score for a word (0.0 = easiest, 1.0 = hardest). + +```bash +curl 'localhost:7777/difficulty?word=ephemeral&lang=en' +{"word":"ephemeral","lang":"en","difficulty":0.72} +``` + +### `GET /words` + +Return a list of words matching filters. Same filter options as `/random` plus `variant`. Returns up to 20,000 results. + +```bash +curl 'localhost:7777/words?lang=en&min=5&max=5&variant=gb&min_freq=500' +{"lang":"en","count":42,"words":["colour","fibre","honour",...]} +``` + +### `GET /rhyme` + +Return English rhyming words based on CMU phoneme tail matching. Optional `limit` parameter (default 10). + +```bash +curl 'localhost:7777/rhyme?word=cat&limit=5' +{"word":"cat","rhymes":["bat","flat","hat","mat","sat"]} +``` + ### `GET /health` Health check with database statistics. Always unauthenticated — safe for monitoring tools. @@ -185,9 +213,9 @@ curl localhost:7777/health { "status": "ok", "db_path": "./dict.db", - "word_counts": {"en":214832,"fr":82104,"pt-PT":71088,"zh":120000}, - "def_counts": {"en":380000,"fr":200000,"pt-PT":120000,"zh":80000}, - "imported_at": "2026-03-28T14:22:00Z", + "word_counts": {"en":136615,"fr":56096,"pt-PT":136300,"zh":120883}, + "def_counts": {"en":361640,"fr":137738,"pt-PT":84577,"zh":199929}, + "imported_at": "2026-04-04T21:31:17Z", "schema_version": "2" } ``` @@ -248,7 +276,7 @@ go run ./cmd/dictimport [flags] | Loader | Language | What it provides | |---|---|---| -| `scowl` | en | Word list with frequency tiers | +| `scowl` | en | Word list with frequency tiers and regional variant tagging (us/gb) | | `affix-en` | en | Inflected form expansion | | `wordnet` | en | Definitions, synonyms, antonyms, synset IDs | | `subtlex` | en | SUBTLEX-US subtitle frequency data | @@ -267,6 +295,7 @@ go run ./cmd/dictimport [flags] | `wiktionary-pt` | pt-PT | Supplemental definitions, synonyms, antonyms, translations, IPA, etymology (filtered to European Portuguese entries) | | `cedict` | zh | Words, definitions, translations via CC-CEDICT | | `wiktionary-zh` | zh | Supplemental Chinese data | +| `prune-orphans` | all | Removes ghost words with no definitions, frequency, or references | | `difficulty` | all | Composite difficulty scoring — runs last | --- @@ -276,7 +305,7 @@ go run ./cmd/dictimport [flags] | Source | Language | What it provides | |---|---|---| | [SCOWL](https://wordlist.aspell.net/) | en | Word list + frequency tiers | -| [WordNet 3.1](https://wordnet.princeton.edu/) | en | Definitions, synonyms, antonyms, synset IDs | +| [WordNet 3.0](https://wordnet.princeton.edu/) | en | Definitions, synonyms, antonyms, synset IDs | | [SUBTLEX-US](http://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexus) | en | Subtitle-based word frequency | | [CMU Pronouncing Dictionary](http://www.speech.cs.cmu.edu/cgi-bin/cmudict) | en | Phoneme pronunciation | | [Hunspell/LibreOffice](https://github.com/LibreOffice/dictionaries) | fr, pt-PT | Word lists + affix rules | @@ -310,7 +339,7 @@ Schema version 2. Full schema in `internal/dictionary/db.go`. | Table | Purpose | |---|---| -| `words` | Core word inventory: word, lang, pos, frequency, difficulty | +| `words` | Core word inventory: word, lang, pos, frequency, difficulty, variant | | `definitions` | Definitions with source and priority | | `synonyms` | Synonym relationships | | `antonyms` | Antonym relationships | @@ -393,15 +422,15 @@ data/ Source data files (gitignored) ## Expected Database Size -| Language | Words | Definitions | Size | -|---|---|---|---| -| en | ~220k+ | ~380k | ~180 MB | -| fr | ~90k+ | ~200k | ~80 MB | -| pt-PT | ~75k+ | ~120k | ~50 MB | -| zh | varies | ~80k | ~20 MB | -| **Total** | | | **~330 MB** | +| Language | Words | Definitions | +|---|---|---| +| en | ~137k | ~362k | +| fr | ~56k | ~138k | +| pt-PT | ~136k | ~85k | +| zh | ~121k | ~200k | +| **Total** | **~450k** | **~785k** | -Word counts exceed base form counts due to affix expansion of inflected forms. +English words include ~3,800 US-only and ~3,900 GB-only regional variants. Word counts include inflected forms from affix expansion, with ghost words pruned automatically. --- diff --git a/cmd/server/main.go b/cmd/server/main.go index a302264..e1ee1cf 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -103,9 +103,15 @@ func main() { mux.HandleFunc("GET /pronunciation", handlePronunciation(dict)) mux.HandleFunc("GET /etymology", handleEtymology(dict)) mux.HandleFunc("GET /difficulty", handleDifficulty(dict)) + mux.HandleFunc("GET /words", handleWords(dict)) mux.HandleFunc("GET /rhyme", handleRhyme(dict)) mux.HandleFunc("GET /health", handleHealth(dict, *dbPath)) + // Cache total word count for status line + if wc, err := dict.TotalWordCount(); err == nil { + totalWords.Store(uint64(wc)) + } + serverStart = time.Now() addr := fmt.Sprintf("%s:%d", *host, *port) @@ -154,6 +160,7 @@ func (r *statusRecorder) WriteHeader(code int) { var ( queryCount atomic.Uint64 errorCount atomic.Uint64 + totalWords atomic.Uint64 serverStart time.Time lastPath atomic.Value // stores string ) @@ -172,8 +179,9 @@ func loggingMiddleware(next http.Handler) http.Handler { rps := float64(n) / time.Since(serverStart).Seconds() errs := errorCount.Load() last, _ := lastPath.Load().(string) - fmt.Fprintf(os.Stdout, "\r\033[2KQueries: %d | Errors: %d | Req/s: %.1f | Uptime: %s | Last: %s", - n, errs, rps, uptime, last) + words := totalWords.Load() + fmt.Fprintf(os.Stdout, "\r\033[2KWords: %dk | Queries: %d | Errors: %d | Req/s: %.1f | Uptime: %s | Last: %s", + words/1000, n, errs, rps, uptime, last) slog.Debug("request", "method", r.Method, "path", r.URL.Path, @@ -201,6 +209,39 @@ func authMiddleware(token string, next http.Handler) http.Handler { }) } +func parseOptions(r *http.Request) dictionary.Options { + opts := dictionary.Options{ + POS: r.URL.Query().Get("pos"), + Variant: r.URL.Query().Get("variant"), + } + if v := r.URL.Query().Get("min"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + opts.MinLength = n + } + } + if v := r.URL.Query().Get("max"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + opts.MaxLength = n + } + } + if v := r.URL.Query().Get("min_freq"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + opts.MinFrequency = n + } + } + if v := r.URL.Query().Get("min_difficulty"); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + opts.MinDifficulty = f + } + } + if v := r.URL.Query().Get("max_difficulty"); v != "" { + if f, err := strconv.ParseFloat(v, 64); err == nil { + opts.MaxDifficulty = f + } + } + return opts +} + func writeJSON(w http.ResponseWriter, status int, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) @@ -246,34 +287,7 @@ func handleRandom(dict *dictionary.Dictionary) http.HandlerFunc { return } - opts := dictionary.Options{ - POS: r.URL.Query().Get("pos"), - } - if v := r.URL.Query().Get("min"); v != "" { - if n, err := strconv.Atoi(v); err == nil { - opts.MinLength = n - } - } - if v := r.URL.Query().Get("max"); v != "" { - if n, err := strconv.Atoi(v); err == nil { - opts.MaxLength = n - } - } - if v := r.URL.Query().Get("min_freq"); v != "" { - if n, err := strconv.Atoi(v); err == nil { - opts.MinFrequency = n - } - } - if v := r.URL.Query().Get("min_difficulty"); v != "" { - if f, err := strconv.ParseFloat(v, 64); err == nil { - opts.MinDifficulty = f - } - } - if v := r.URL.Query().Get("max_difficulty"); v != "" { - if f, err := strconv.ParseFloat(v, 64); err == nil { - opts.MaxDifficulty = f - } - } + opts := parseOptions(r) result, err := dict.RandomWord(lang, opts) if err == dictionary.ErrNoMatch { @@ -293,6 +307,38 @@ func handleRandom(dict *dictionary.Dictionary) http.HandlerFunc { } } +func handleWords(dict *dictionary.Dictionary) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + lang := r.URL.Query().Get("lang") + if lang == "" { + writeError(w, 400, "bad_request", "lang is required") + return + } + if !dictionary.ValidLang(lang) { + writeError(w, 400, "bad_request", "unsupported language: "+lang) + return + } + + opts := parseOptions(r) + + words, err := dict.Words(lang, opts) + if err == dictionary.ErrNoMatch { + writeError(w, 404, "no_match", "no words match the given filters") + return + } + if err != nil { + slog.Error("words", "error", err) + writeError(w, 500, "internal", "internal error") + return + } + writeJSON(w, 200, map[string]any{ + "lang": lang, + "count": len(words), + "words": words, + }) + } +} + func handleDefine(dict *dictionary.Dictionary) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { word := r.URL.Query().Get("word") @@ -312,11 +358,17 @@ func handleDefine(dict *dictionary.Dictionary) http.HandlerFunc { writeError(w, 500, "internal", "internal error") return } - writeJSON(w, 200, map[string]any{ + resp := map[string]any{ "word": word, "lang": lang, "definitions": defs, - }) + } + if lang == "en" { + if v, err := dict.WordVariant(word, lang); err == nil && v != "" { + resp["variant"] = v + } + } + writeJSON(w, 200, resp) } } diff --git a/cmd/server/server_test.go b/cmd/server/server_test.go index dc88eb9..d72df35 100644 --- a/cmd/server/server_test.go +++ b/cmd/server/server_test.go @@ -49,6 +49,7 @@ func setupTestServer(t *testing.T) (*dictionary.Dictionary, *http.ServeMux) { mux.HandleFunc("GET /define", handleDefine(dict)) mux.HandleFunc("GET /synonyms", handleSynonyms(dict)) mux.HandleFunc("GET /translate", handleTranslate(dict)) + mux.HandleFunc("GET /words", handleWords(dict)) mux.HandleFunc("GET /health", handleHealth(dict, ":memory:")) t.Cleanup(func() { db.Close() }) @@ -201,6 +202,55 @@ func TestTranslateEndpoint(t *testing.T) { } } +func TestWordsEndpoint(t *testing.T) { + _, mux := setupTestServer(t) + srv := httptest.NewServer(mux) + defer srv.Close() + + // All English words + resp, _ := http.Get(srv.URL + "/words?lang=en") + if resp.StatusCode != 200 { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + var result struct { + Lang string `json:"lang"` + Count int `json:"count"` + Words []string `json:"words"` + } + json.NewDecoder(resp.Body).Decode(&result) + resp.Body.Close() + if result.Lang != "en" { + t.Errorf("lang = %q, want en", result.Lang) + } + if result.Count != len(result.Words) { + t.Errorf("count %d != len(words) %d", result.Count, len(result.Words)) + } + if result.Count != 2 { // happy, cat + t.Errorf("count = %d, want 2", result.Count) + } + + // With length filter — no match + resp, _ = http.Get(srv.URL + "/words?lang=en&min=100") + if resp.StatusCode != 404 { + t.Errorf("no match: status = %d, want 404", resp.StatusCode) + } + resp.Body.Close() + + // Missing lang + resp, _ = http.Get(srv.URL + "/words") + if resp.StatusCode != 400 { + t.Errorf("missing lang: status = %d, want 400", resp.StatusCode) + } + resp.Body.Close() + + // Bad lang + resp, _ = http.Get(srv.URL + "/words?lang=xx") + if resp.StatusCode != 400 { + t.Errorf("bad lang: status = %d, want 400", resp.StatusCode) + } + resp.Body.Close() +} + func TestHealthEndpoint(t *testing.T) { _, mux := setupTestServer(t) srv := httptest.NewServer(mux) diff --git a/internal/dictionary/db.go b/internal/dictionary/db.go index dbb5c98..050eaff 100644 --- a/internal/dictionary/db.go +++ b/internal/dictionary/db.go @@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS words ( pos TEXT, frequency INTEGER DEFAULT 0, difficulty REAL DEFAULT NULL, + variant TEXT, UNIQUE(word, lang) ); @@ -159,6 +160,7 @@ func BootstrapSchema(db *sql.DB) error { migrations := []string{ "ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL", "ALTER TABLE pronunciations ADD COLUMN cmu_tail TEXT", + "ALTER TABLE words ADD COLUMN variant TEXT", } for _, m := range migrations { if _, err := db.Exec(m); err != nil { @@ -205,6 +207,7 @@ func PopulateCMUTails(db *sql.DB) (int, error) { if err != nil { return 0, fmt.Errorf("dictionary: populate cmu tails: %w", err) } + defer rows.Close() type entry struct { id int @@ -214,12 +217,13 @@ func PopulateCMUTails(db *sql.DB) (int, error) { for rows.Next() { var e entry if err := rows.Scan(&e.id, &e.value); err != nil { - rows.Close() return 0, fmt.Errorf("dictionary: populate cmu tails scan: %w", err) } entries = append(entries, e) } - rows.Close() + if err := rows.Err(); err != nil { + return 0, fmt.Errorf("dictionary: populate cmu tails iterate: %w", err) + } if len(entries) == 0 { return 0, nil diff --git a/internal/dictionary/dict.go b/internal/dictionary/dict.go index 6774fa6..6523d91 100644 --- a/internal/dictionary/dict.go +++ b/internal/dictionary/dict.go @@ -28,6 +28,7 @@ type Options struct { MinFrequency int // 0 = no filter MinDifficulty float64 // 0.0 = no filter MaxDifficulty float64 // 0.0 = no filter + Variant string // "", "us", "gb" — empty means no filter } type Definition struct { diff --git a/internal/dictionary/dict_test.go b/internal/dictionary/dict_test.go index a8fd710..ceb2943 100644 --- a/internal/dictionary/dict_test.go +++ b/internal/dictionary/dict_test.go @@ -44,10 +44,13 @@ func seedTestData(t *testing.T, db *sql.DB) { // Definitions mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling pleasure', 'wordnet', 10)") mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling joy or contentment', 'wiktionary', 20)") + mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (4, 'noun', 'um animal felino', 'dicionario', 10)") + mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (4, 'noun', 'um felino doméstico', 'wiktionary', 20)") // Synonyms mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'glad', 'wordnet')") mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'joyful', 'wordnet')") + mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (4, 'bichano', 'wiktionary')") // Translations mustExec(t, db, "INSERT INTO translations (word_id, translation, target_lang, source) VALUES (5, 'chat', 'fr', 'wiktionary')") @@ -69,6 +72,8 @@ func TestIsValidWord(t *testing.T) { {"nonexistent", "en", false}, {"chat", "fr", true}, {"chat", "en", false}, + {"gato", "pt-PT", true}, + {"gato", "en", false}, } for _, tt := range tests { got, err := d.IsValidWord(tt.word, tt.lang) @@ -143,6 +148,18 @@ func TestDefine(t *testing.T) { t.Errorf("second def source = %q, want wiktionary", defs[1].Source) } + // pt-PT definitions + defs, err = d.Define("gato", "pt-PT") + if err != nil { + t.Fatalf("Define(gato): %v", err) + } + if len(defs) != 2 { + t.Fatalf("Define(gato) returned %d defs, want 2", len(defs)) + } + if defs[0].Source != "dicionario" { + t.Errorf("first pt-PT def source = %q, want dicionario", defs[0].Source) + } + // No definitions defs, err = d.Define("run", "en") if err != nil { @@ -167,6 +184,15 @@ func TestSynonyms(t *testing.T) { t.Fatalf("Synonyms returned %d, want 2", len(syns)) } + // pt-PT synonyms + syns, err = d.Synonyms("gato", "pt-PT") + if err != nil { + t.Fatalf("Synonyms(gato): %v", err) + } + if len(syns) != 1 || syns[0] != "bichano" { + t.Errorf("Synonyms(gato) = %v, want [bichano]", syns) + } + // No synonyms syns, err = d.Synonyms("run", "en") if err != nil { @@ -294,6 +320,124 @@ func TestMeta(t *testing.T) { } } +func TestWords(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + + // Add some 5-letter words with varying frequency + mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('crane', 'en', 'noun', 200)") + mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('house', 'en', 'noun', 800)") + mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('light', 'en', 'noun', 500)") + mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('quick', 'en', 'adjective', 100)") + + d := NewFromDB(db) + + // All 5-letter English words + words, err := d.Words("en", Options{MinLength: 5, MaxLength: 5}) + if err != nil { + t.Fatalf("Words: %v", err) + } + if len(words) != 5 { // crane, happy, house, light, quick + t.Errorf("Words returned %d, want 5", len(words)) + } + + // Filter by frequency + words, err = d.Words("en", Options{MinLength: 5, MaxLength: 5, MinFrequency: 500}) + if err != nil { + t.Fatalf("Words with freq filter: %v", err) + } + if len(words) != 2 { // house(800), light(500) + t.Errorf("Words with freq filter returned %d, want 2", len(words)) + } + + // Filter by POS + words, err = d.Words("en", Options{MinLength: 5, MaxLength: 5, POS: "noun"}) + if err != nil { + t.Fatalf("Words with POS filter: %v", err) + } + if len(words) != 3 { // crane, house, light + t.Errorf("Words with POS filter returned %d, want 3", len(words)) + } + + // No match + _, err = d.Words("en", Options{MinLength: 100}) + if err != ErrNoMatch { + t.Errorf("expected ErrNoMatch, got %v", err) + } + + // Results are ordered alphabetically + words, err = d.Words("en", Options{MinLength: 5, MaxLength: 5, POS: "noun"}) + if err != nil { + t.Fatal(err) + } + for i := 1; i < len(words); i++ { + if words[i] < words[i-1] { + t.Errorf("words not sorted: %q before %q", words[i-1], words[i]) + } + } +} + +func TestPopulateCMUTails(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + + // Insert CMU pronunciation entries without tails + mustExec(t, db, "INSERT INTO pronunciations (word_id, format, value, source) VALUES (5, 'cmu', 'K AE1 T', 'cmudict')") + mustExec(t, db, "INSERT INTO pronunciations (word_id, format, value, source) VALUES (1, 'cmu', 'HH AE1 P IY0', 'cmudict')") + // IPA entry should be ignored + mustExec(t, db, "INSERT INTO pronunciations (word_id, format, value, source) VALUES (2, 'ipa', '/ɹʌn/', 'wiktionary')") + + n, err := PopulateCMUTails(db) + if err != nil { + t.Fatalf("PopulateCMUTails: %v", err) + } + if n != 2 { + t.Errorf("PopulateCMUTails returned %d, want 2", n) + } + + // Verify tails were set correctly + var tail string + db.QueryRow("SELECT cmu_tail FROM pronunciations WHERE value = 'K AE1 T'").Scan(&tail) + if tail != "AE1 T" { + t.Errorf("cmu_tail for 'K AE1 T' = %q, want 'AE1 T'", tail) + } + + db.QueryRow("SELECT cmu_tail FROM pronunciations WHERE value = 'HH AE1 P IY0'").Scan(&tail) + if tail != "AE1 P IY0" { + t.Errorf("cmu_tail for 'HH AE1 P IY0' = %q, want 'AE1 P IY0'", tail) + } + + // Running again should be a no-op + n, err = PopulateCMUTails(db) + if err != nil { + t.Fatalf("PopulateCMUTails second run: %v", err) + } + if n != 0 { + t.Errorf("PopulateCMUTails second run returned %d, want 0", n) + } +} + +func TestCmuTail(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"K AE1 T", "AE1 T"}, + {"HH AE1 P IY0", "AE1 P IY0"}, + {"IH0 F EH1 M ER0 AH0 L", "EH1 M ER0 AH0 L"}, + {"AH0", "AH0"}, // no stressed vowel, falls back to any vowel + {"K T S", ""}, // no vowels at all + } + for _, tt := range tests { + got := cmuTail(tt.input) + if got != tt.want { + t.Errorf("cmuTail(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + func TestWordCount(t *testing.T) { db := setupTestDB(t) defer db.Close() @@ -310,4 +454,15 @@ func TestWordCount(t *testing.T) { if counts["fr"] != 1 { t.Errorf("fr word count = %d, want 1", counts["fr"]) } + if counts["pt-PT"] != 1 { + t.Errorf("pt-PT word count = %d, want 1", counts["pt-PT"]) + } + + total, err := d.TotalWordCount() + if err != nil { + t.Fatal(err) + } + if total != 5 { + t.Errorf("total word count = %d, want 5", total) + } } diff --git a/internal/dictionary/query.go b/internal/dictionary/query.go index 1d96f33..f2169c9 100644 --- a/internal/dictionary/query.go +++ b/internal/dictionary/query.go @@ -18,6 +18,22 @@ func (d *Dictionary) IsValidWord(word, lang string) (bool, error) { return exists, nil } +// WordVariant returns the regional variant tag for a word ("us", "gb", or ""). +func (d *Dictionary) WordVariant(word, lang string) (string, error) { + var variant sql.NullString + err := d.db.QueryRow( + "SELECT variant FROM words WHERE word = ? AND lang = ?", + strings.ToLower(word), lang, + ).Scan(&variant) + if err == sql.ErrNoRows { + return "", nil + } + if err != nil { + return "", fmt.Errorf("dictionary: word variant: %w", err) + } + return variant.String, nil +} + type RandomResult struct { Word string `json:"word"` Difficulty *float64 `json:"difficulty,omitempty"` @@ -51,6 +67,10 @@ func (d *Dictionary) RandomWord(lang string, opts Options) (RandomResult, error) query += " AND difficulty <= ?" args = append(args, opts.MaxDifficulty) } + if opts.Variant != "" { + query += " AND variant = ?" + args = append(args, opts.Variant) + } query += " ORDER BY RANDOM() LIMIT 1" @@ -69,6 +89,64 @@ func (d *Dictionary) RandomWord(lang string, opts Options) (RandomResult, error) return result, nil } +func (d *Dictionary) Words(lang string, opts Options) ([]string, error) { + query := "SELECT DISTINCT word FROM words WHERE lang = ?" + args := []any{lang} + + if opts.POS != "" { + query += " AND pos = ?" + args = append(args, opts.POS) + } + if opts.MinLength > 0 { + query += " AND LENGTH(word) >= ?" + args = append(args, opts.MinLength) + } + if opts.MaxLength > 0 { + query += " AND LENGTH(word) <= ?" + args = append(args, opts.MaxLength) + } + if opts.MinFrequency > 0 { + query += " AND frequency >= ?" + args = append(args, opts.MinFrequency) + } + if opts.MinDifficulty > 0 { + query += " AND difficulty >= ?" + args = append(args, opts.MinDifficulty) + } + if opts.MaxDifficulty > 0 { + query += " AND difficulty <= ?" + args = append(args, opts.MaxDifficulty) + } + if opts.Variant != "" { + query += " AND variant = ?" + args = append(args, opts.Variant) + } + + query += " ORDER BY word LIMIT 20000" + + rows, err := d.db.Query(query, args...) + if err != nil { + return nil, fmt.Errorf("dictionary: words: %w", err) + } + defer rows.Close() + + var words []string + for rows.Next() { + var w string + if err := rows.Scan(&w); err != nil { + return nil, fmt.Errorf("dictionary: words scan: %w", err) + } + words = append(words, w) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("dictionary: words: %w", err) + } + if len(words) == 0 { + return nil, ErrNoMatch + } + return words, nil +} + func (d *Dictionary) Define(word, lang string) ([]Definition, error) { rows, err := d.db.Query(` SELECT d.pos, d.gloss, d.source, d.priority @@ -187,6 +265,15 @@ func (d *Dictionary) WordCount() (map[string]int, error) { return counts, rows.Err() } +func (d *Dictionary) TotalWordCount() (int, error) { + var count int + err := d.db.QueryRow("SELECT COUNT(*) FROM words").Scan(&count) + if err != nil { + return 0, fmt.Errorf("dictionary: total word count: %w", err) + } + return count, nil +} + // Frequency returns the frequency score for a word in a language. // Returns 0 if the word is not found or has no frequency data. // Higher values indicate more common words. diff --git a/internal/loader/scowl.go b/internal/loader/scowl.go index 7e92b83..02dbebd 100644 --- a/internal/loader/scowl.go +++ b/internal/loader/scowl.go @@ -31,35 +31,61 @@ var scowlFrequency = map[int]int{ } type scowlFile struct { - path string - size int + path string + size int + variant string // "us", "gb", or "" for common English } func (SCOWLLoader) Load(db *sql.DB, dataDir string) error { - pattern := filepath.Join(dataDir, "scowl", "final", "english-words.*") - files, err := filepath.Glob(pattern) - if err != nil { - return fmt.Errorf("scowl: glob: %w", err) + type prefixVariant struct { + prefix string + variant string } - if len(files) == 0 { - return fmt.Errorf("scowl: no files matching %s", pattern) + prefixes := []prefixVariant{ + {"english-words", ""}, + {"american-words", "us"}, + {"british-words", "gb"}, } - // Filter to sizes <= 70 and track the tier + var allFiles []string + for _, pv := range prefixes { + pattern := filepath.Join(dataDir, "scowl", "final", pv.prefix+".*") + matches, err := filepath.Glob(pattern) + if err != nil { + return fmt.Errorf("scowl: glob: %w", err) + } + allFiles = append(allFiles, matches...) + } + if len(allFiles) == 0 { + return fmt.Errorf("scowl: no word files found in %s", filepath.Join(dataDir, "scowl", "final")) + } + + // Build a map from prefix to variant for quick lookup + prefixToVariant := make(map[string]string) + for _, pv := range prefixes { + prefixToVariant[pv.prefix] = pv.variant + } + + // Filter to sizes <= 70 and track the tier + variant var selected []scowlFile - for _, f := range files { + for _, f := range allFiles { base := filepath.Base(f) - // Files are like english-words.10, english-words.20, etc. + // Files are like english-words.10, american-words.35, etc. parts := strings.SplitN(base, ".", 2) if len(parts) != 2 { continue } + prefix := parts[0] + variant, ok := prefixToVariant[prefix] + if !ok { + continue + } var size int if _, err := fmt.Sscanf(parts[1], "%d", &size); err != nil { continue } if size <= 70 { - selected = append(selected, scowlFile{path: f, size: size}) + selected = append(selected, scowlFile{path: f, size: size, variant: variant}) } } @@ -69,8 +95,15 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error { } defer tx.Rollback() - stmt, err := tx.Prepare(`INSERT INTO words (word, lang, frequency) VALUES (?, 'en', ?) - ON CONFLICT(word, lang) DO UPDATE SET frequency = MAX(frequency, excluded.frequency)`) + stmt, err := tx.Prepare(`INSERT INTO words (word, lang, frequency, variant) VALUES (?, 'en', ?, ?) + ON CONFLICT(word, lang) DO UPDATE SET + frequency = MAX(frequency, excluded.frequency), + variant = CASE + WHEN words.variant IS NULL THEN excluded.variant + WHEN excluded.variant IS NULL THEN words.variant + WHEN words.variant = excluded.variant THEN words.variant + ELSE NULL + END`) if err != nil { return fmt.Errorf("scowl: prepare: %w", err) } @@ -79,7 +112,7 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error { var count int for _, sf := range selected { freq := scowlFrequency[sf.size] - n, err := loadSCOWLFile(stmt, sf.path, freq) + n, err := loadSCOWLFile(stmt, sf.path, freq, sf.variant) if err != nil { return fmt.Errorf("scowl: load %s: %w", sf.path, err) } @@ -93,13 +126,19 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error { return nil } -func loadSCOWLFile(stmt *sql.Stmt, path string, freq int) (int, error) { +func loadSCOWLFile(stmt *sql.Stmt, path string, freq int, variant string) (int, error) { f, err := os.Open(path) if err != nil { return 0, err } defer f.Close() + // Convert empty variant to SQL NULL + var variantParam any + if variant != "" { + variantParam = variant + } + var count int scanner := bufio.NewScanner(f) for scanner.Scan() { @@ -111,7 +150,7 @@ func loadSCOWLFile(stmt *sql.Stmt, path string, freq int) (int, error) { if containsAny(word, " ", "'", "-") || containsDigit(word) { continue } - if _, err := stmt.Exec(word, freq); err != nil { + if _, err := stmt.Exec(word, freq, variantParam); err != nil { return 0, err } count++