Add regional variant tagging (us/gb) for English words

- SCOWL loader now loads american-words and british-words alongside english-words, tagging each with variant "us", "gb", or NULL (common) - Words appearing in both American and British lists get NULL variant - Add variant column to words table with schema migration - API: /define returns variant field for English, /random and /words accept ?variant=us|gb filter - Add total word count to server status line - Fix rows.Close leak in PopulateCMUTails, add rows.Err() check - Add tests for PopulateCMUTails, cmuTail, TotalWordCount - Expand test seed data and assertions to cover pt-PT definitions, synonyms, and validity checks - Update README with variant docs, /difficulty, /words, /rhyme endpoints, and current database statistics Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-04 15:21:27 -07:00
parent 28ec11ba4e
commit 704c06f41d
8 changed files with 488 additions and 71 deletions
--- a/README.md
+++ b/README.md
@@ -64,7 +64,7 @@ curl 'localhost:7777/valid?word=running&lang=en'

 ### `GET /random`

-Return a random word. Optional filters: `pos` (`noun`, `verb`, `adjective`, `adverb`), `min`/`max` (word length), `min_freq` (frequency score), `min_difficulty`/`max_difficulty` (0.0–1.0).
+Return a random word. Optional filters: `pos` (`noun`, `verb`, `adjective`, `adverb`), `min`/`max` (word length), `min_freq` (frequency score), `min_difficulty`/`max_difficulty` (0.0–1.0), `variant` (`us` or `gb`, English only).

 ```bash
 curl 'localhost:7777/random?lang=en&pos=noun&min=5&max=8&max_difficulty=0.5'
@@ -73,16 +73,17 @@ curl 'localhost:7777/random?lang=en&pos=noun&min=5&max=8&max_difficulty=0.5'

 ### `GET /define`

-Return all definitions ordered by source priority. Curated sources (WordNet, WOLF, Dicionário Aberto) appear before Wiktionary supplemental definitions.
+Return all definitions ordered by source priority. Curated sources (WordNet, WOLF, Dicionário Aberto) appear before Wiktionary supplemental definitions. For English words, the response includes a `variant` field (`"us"`, `"gb"`, or omitted for common English) indicating regional spelling.

 ```bash
-curl 'localhost:7777/define?word=ephemeral&lang=en'
+curl 'localhost:7777/define?word=ardor&lang=en'
 {
-  "word": "ephemeral",
+  "word": "ardor",
  "lang": "en",
+  "variant": "us",
  "definitions": [
-    {"pos":"adjective","gloss":"lasting for a very short time","source":"wordnet","priority":10},
-    {"pos":"adjective","gloss":"lasting only for a day","source":"wiktionary","priority":20}
+    {"pos":"noun","gloss":"a feeling of strong eagerness","source":"wordnet","priority":10},
+    {"pos":"noun","gloss":"feelings of great warmth and intensity","source":"wordnet","priority":10}
  ]
 }
 ```
@@ -176,6 +177,33 @@ curl 'localhost:7777/frequency/batch?words=house,cat,ephemeral&lang=en'
 {"lang":"en","frequencies":{"cat":800,"ephemeral":50,"house":800}}
 ```

+### `GET /difficulty`
+
+Return the difficulty score for a word (0.0 = easiest, 1.0 = hardest).
+
+```bash
+curl 'localhost:7777/difficulty?word=ephemeral&lang=en'
+{"word":"ephemeral","lang":"en","difficulty":0.72}
+```
+
+### `GET /words`
+
+Return a list of words matching filters. Same filter options as `/random` plus `variant`. Returns up to 20,000 results.
+
+```bash
+curl 'localhost:7777/words?lang=en&min=5&max=5&variant=gb&min_freq=500'
+{"lang":"en","count":42,"words":["colour","fibre","honour",...]}
+```
+
+### `GET /rhyme`
+
+Return English rhyming words based on CMU phoneme tail matching. Optional `limit` parameter (default 10).
+
+```bash
+curl 'localhost:7777/rhyme?word=cat&limit=5'
+{"word":"cat","rhymes":["bat","flat","hat","mat","sat"]}
+```
+
 ### `GET /health`

 Health check with database statistics. Always unauthenticated — safe for monitoring tools.
@@ -185,9 +213,9 @@ curl localhost:7777/health
 {
  "status": "ok",
  "db_path": "./dict.db",
-  "word_counts": {"en":214832,"fr":82104,"pt-PT":71088,"zh":120000},
-  "def_counts":  {"en":380000,"fr":200000,"pt-PT":120000,"zh":80000},
-  "imported_at": "2026-03-28T14:22:00Z",
+  "word_counts": {"en":136615,"fr":56096,"pt-PT":136300,"zh":120883},
+  "def_counts":  {"en":361640,"fr":137738,"pt-PT":84577,"zh":199929},
+  "imported_at": "2026-04-04T21:31:17Z",
  "schema_version": "2"
 }
 ```
@@ -248,7 +276,7 @@ go run ./cmd/dictimport [flags]

 | Loader | Language | What it provides |
 |---|---|---|
-| `scowl` | en | Word list with frequency tiers |
+| `scowl` | en | Word list with frequency tiers and regional variant tagging (us/gb) |
 | `affix-en` | en | Inflected form expansion |
 | `wordnet` | en | Definitions, synonyms, antonyms, synset IDs |
 | `subtlex` | en | SUBTLEX-US subtitle frequency data |
@@ -267,6 +295,7 @@ go run ./cmd/dictimport [flags]
 | `wiktionary-pt` | pt-PT | Supplemental definitions, synonyms, antonyms, translations, IPA, etymology (filtered to European Portuguese entries) |
 | `cedict` | zh | Words, definitions, translations via CC-CEDICT |
 | `wiktionary-zh` | zh | Supplemental Chinese data |
+| `prune-orphans` | all | Removes ghost words with no definitions, frequency, or references |
 | `difficulty` | all | Composite difficulty scoring — runs last |

 ---
@@ -276,7 +305,7 @@ go run ./cmd/dictimport [flags]
 | Source | Language | What it provides |
 |---|---|---|
 | [SCOWL](https://wordlist.aspell.net/) | en | Word list + frequency tiers |
-| [WordNet 3.1](https://wordnet.princeton.edu/) | en | Definitions, synonyms, antonyms, synset IDs |
+| [WordNet 3.0](https://wordnet.princeton.edu/) | en | Definitions, synonyms, antonyms, synset IDs |
 | [SUBTLEX-US](http://www.ugent.be/pp/experimentele-psychologie/en/research/documents/subtlexus) | en | Subtitle-based word frequency |
 | [CMU Pronouncing Dictionary](http://www.speech.cs.cmu.edu/cgi-bin/cmudict) | en | Phoneme pronunciation |
 | [Hunspell/LibreOffice](https://github.com/LibreOffice/dictionaries) | fr, pt-PT | Word lists + affix rules |
@@ -310,7 +339,7 @@ Schema version 2. Full schema in `internal/dictionary/db.go`.

 | Table | Purpose |
 |---|---|
-| `words` | Core word inventory: word, lang, pos, frequency, difficulty |
+| `words` | Core word inventory: word, lang, pos, frequency, difficulty, variant |
 | `definitions` | Definitions with source and priority |
 | `synonyms` | Synonym relationships |
 | `antonyms` | Antonym relationships |
@@ -393,15 +422,15 @@ data/                 Source data files (gitignored)

 ## Expected Database Size

-| Language | Words | Definitions | Size |
-|---|---|---|---|
-| en | ~220k+ | ~380k | ~180 MB |
-| fr | ~90k+ | ~200k | ~80 MB |
-| pt-PT | ~75k+ | ~120k | ~50 MB |
-| zh | varies | ~80k | ~20 MB |
-| **Total** | | | **~330 MB** |
+| Language | Words | Definitions |
+|---|---|---|
+| en | ~137k | ~362k |
+| fr | ~56k | ~138k |
+| pt-PT | ~136k | ~85k |
+| zh | ~121k | ~200k |
+| **Total** | **~450k** | **~785k** |

-Word counts exceed base form counts due to affix expansion of inflected forms.
+English words include ~3,800 US-only and ~3,900 GB-only regional variants. Word counts include inflected forms from affix expansion, with ghost words pruned automatically.

 ---

--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -103,9 +103,15 @@ func main() {
 	mux.HandleFunc("GET /pronunciation", handlePronunciation(dict))
 	mux.HandleFunc("GET /etymology", handleEtymology(dict))
 	mux.HandleFunc("GET /difficulty", handleDifficulty(dict))
+	mux.HandleFunc("GET /words", handleWords(dict))
 	mux.HandleFunc("GET /rhyme", handleRhyme(dict))
 	mux.HandleFunc("GET /health", handleHealth(dict, *dbPath))

+	// Cache total word count for status line
+	if wc, err := dict.TotalWordCount(); err == nil {
+		totalWords.Store(uint64(wc))
+	}
+
 	serverStart = time.Now()
 	addr := fmt.Sprintf("%s:%d", *host, *port)

@@ -154,6 +160,7 @@ func (r *statusRecorder) WriteHeader(code int) {
 var (
 	queryCount  atomic.Uint64
 	errorCount  atomic.Uint64
+	totalWords  atomic.Uint64
 	serverStart time.Time
 	lastPath    atomic.Value // stores string
 )
@@ -172,8 +179,9 @@ func loggingMiddleware(next http.Handler) http.Handler {
 		rps := float64(n) / time.Since(serverStart).Seconds()
 		errs := errorCount.Load()
 		last, _ := lastPath.Load().(string)
-		fmt.Fprintf(os.Stdout, "\r\033[2KQueries: %d | Errors: %d | Req/s: %.1f | Uptime: %s | Last: %s",
-			n, errs, rps, uptime, last)
+		words := totalWords.Load()
+		fmt.Fprintf(os.Stdout, "\r\033[2KWords: %dk | Queries: %d | Errors: %d | Req/s: %.1f | Uptime: %s | Last: %s",
+			words/1000, n, errs, rps, uptime, last)
 		slog.Debug("request",
 			"method", r.Method,
 			"path", r.URL.Path,
@@ -201,6 +209,39 @@ func authMiddleware(token string, next http.Handler) http.Handler {
 	})
 }

+func parseOptions(r *http.Request) dictionary.Options {
+	opts := dictionary.Options{
+		POS:     r.URL.Query().Get("pos"),
+		Variant: r.URL.Query().Get("variant"),
+	}
+	if v := r.URL.Query().Get("min"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil {
+			opts.MinLength = n
+		}
+	}
+	if v := r.URL.Query().Get("max"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil {
+			opts.MaxLength = n
+		}
+	}
+	if v := r.URL.Query().Get("min_freq"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil {
+			opts.MinFrequency = n
+		}
+	}
+	if v := r.URL.Query().Get("min_difficulty"); v != "" {
+		if f, err := strconv.ParseFloat(v, 64); err == nil {
+			opts.MinDifficulty = f
+		}
+	}
+	if v := r.URL.Query().Get("max_difficulty"); v != "" {
+		if f, err := strconv.ParseFloat(v, 64); err == nil {
+			opts.MaxDifficulty = f
+		}
+	}
+	return opts
+}
+
 func writeJSON(w http.ResponseWriter, status int, v any) {
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(status)
@@ -246,34 +287,7 @@ func handleRandom(dict *dictionary.Dictionary) http.HandlerFunc {
 			return
 		}

-		opts := dictionary.Options{
-			POS: r.URL.Query().Get("pos"),
-		}
-		if v := r.URL.Query().Get("min"); v != "" {
-			if n, err := strconv.Atoi(v); err == nil {
-				opts.MinLength = n
-			}
-		}
-		if v := r.URL.Query().Get("max"); v != "" {
-			if n, err := strconv.Atoi(v); err == nil {
-				opts.MaxLength = n
-			}
-		}
-		if v := r.URL.Query().Get("min_freq"); v != "" {
-			if n, err := strconv.Atoi(v); err == nil {
-				opts.MinFrequency = n
-			}
-		}
-		if v := r.URL.Query().Get("min_difficulty"); v != "" {
-			if f, err := strconv.ParseFloat(v, 64); err == nil {
-				opts.MinDifficulty = f
-			}
-		}
-		if v := r.URL.Query().Get("max_difficulty"); v != "" {
-			if f, err := strconv.ParseFloat(v, 64); err == nil {
-				opts.MaxDifficulty = f
-			}
-		}
+		opts := parseOptions(r)

 		result, err := dict.RandomWord(lang, opts)
 		if err == dictionary.ErrNoMatch {
@@ -293,6 +307,38 @@ func handleRandom(dict *dictionary.Dictionary) http.HandlerFunc {
 	}
 }

+func handleWords(dict *dictionary.Dictionary) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		lang := r.URL.Query().Get("lang")
+		if lang == "" {
+			writeError(w, 400, "bad_request", "lang is required")
+			return
+		}
+		if !dictionary.ValidLang(lang) {
+			writeError(w, 400, "bad_request", "unsupported language: "+lang)
+			return
+		}
+
+		opts := parseOptions(r)
+
+		words, err := dict.Words(lang, opts)
+		if err == dictionary.ErrNoMatch {
+			writeError(w, 404, "no_match", "no words match the given filters")
+			return
+		}
+		if err != nil {
+			slog.Error("words", "error", err)
+			writeError(w, 500, "internal", "internal error")
+			return
+		}
+		writeJSON(w, 200, map[string]any{
+			"lang":  lang,
+			"count": len(words),
+			"words": words,
+		})
+	}
+}
+
 func handleDefine(dict *dictionary.Dictionary) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		word := r.URL.Query().Get("word")
@@ -312,11 +358,17 @@ func handleDefine(dict *dictionary.Dictionary) http.HandlerFunc {
 			writeError(w, 500, "internal", "internal error")
 			return
 		}
-		writeJSON(w, 200, map[string]any{
+		resp := map[string]any{
 			"word":        word,
 			"lang":        lang,
 			"definitions": defs,
-		})
+		}
+		if lang == "en" {
+			if v, err := dict.WordVariant(word, lang); err == nil && v != "" {
+				resp["variant"] = v
+			}
+		}
+		writeJSON(w, 200, resp)
 	}
 }

--- a/cmd/server/server_test.go
+++ b/cmd/server/server_test.go
@@ -49,6 +49,7 @@ func setupTestServer(t *testing.T) (*dictionary.Dictionary, *http.ServeMux) {
 	mux.HandleFunc("GET /define", handleDefine(dict))
 	mux.HandleFunc("GET /synonyms", handleSynonyms(dict))
 	mux.HandleFunc("GET /translate", handleTranslate(dict))
+	mux.HandleFunc("GET /words", handleWords(dict))
 	mux.HandleFunc("GET /health", handleHealth(dict, ":memory:"))

 	t.Cleanup(func() { db.Close() })
@@ -201,6 +202,55 @@ func TestTranslateEndpoint(t *testing.T) {
 	}
 }

+func TestWordsEndpoint(t *testing.T) {
+	_, mux := setupTestServer(t)
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	// All English words
+	resp, _ := http.Get(srv.URL + "/words?lang=en")
+	if resp.StatusCode != 200 {
+		t.Fatalf("status = %d, want 200", resp.StatusCode)
+	}
+	var result struct {
+		Lang  string   `json:"lang"`
+		Count int      `json:"count"`
+		Words []string `json:"words"`
+	}
+	json.NewDecoder(resp.Body).Decode(&result)
+	resp.Body.Close()
+	if result.Lang != "en" {
+		t.Errorf("lang = %q, want en", result.Lang)
+	}
+	if result.Count != len(result.Words) {
+		t.Errorf("count %d != len(words) %d", result.Count, len(result.Words))
+	}
+	if result.Count != 2 { // happy, cat
+		t.Errorf("count = %d, want 2", result.Count)
+	}
+
+	// With length filter — no match
+	resp, _ = http.Get(srv.URL + "/words?lang=en&min=100")
+	if resp.StatusCode != 404 {
+		t.Errorf("no match: status = %d, want 404", resp.StatusCode)
+	}
+	resp.Body.Close()
+
+	// Missing lang
+	resp, _ = http.Get(srv.URL + "/words")
+	if resp.StatusCode != 400 {
+		t.Errorf("missing lang: status = %d, want 400", resp.StatusCode)
+	}
+	resp.Body.Close()
+
+	// Bad lang
+	resp, _ = http.Get(srv.URL + "/words?lang=xx")
+	if resp.StatusCode != 400 {
+		t.Errorf("bad lang: status = %d, want 400", resp.StatusCode)
+	}
+	resp.Body.Close()
+}
+
 func TestHealthEndpoint(t *testing.T) {
 	_, mux := setupTestServer(t)
 	srv := httptest.NewServer(mux)
--- a/internal/dictionary/db.go
+++ b/internal/dictionary/db.go
@@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS words (
    pos       TEXT,
    frequency  INTEGER DEFAULT 0,
    difficulty REAL DEFAULT NULL,
+    variant    TEXT,
    UNIQUE(word, lang)
 );

@@ -159,6 +160,7 @@ func BootstrapSchema(db *sql.DB) error {
 	migrations := []string{
 		"ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL",
 		"ALTER TABLE pronunciations ADD COLUMN cmu_tail TEXT",
+		"ALTER TABLE words ADD COLUMN variant TEXT",
 	}
 	for _, m := range migrations {
 		if _, err := db.Exec(m); err != nil {
@@ -205,6 +207,7 @@ func PopulateCMUTails(db *sql.DB) (int, error) {
 	if err != nil {
 		return 0, fmt.Errorf("dictionary: populate cmu tails: %w", err)
 	}
+	defer rows.Close()

 	type entry struct {
 		id    int
@@ -214,12 +217,13 @@ func PopulateCMUTails(db *sql.DB) (int, error) {
 	for rows.Next() {
 		var e entry
 		if err := rows.Scan(&e.id, &e.value); err != nil {
-			rows.Close()
 			return 0, fmt.Errorf("dictionary: populate cmu tails scan: %w", err)
 		}
 		entries = append(entries, e)
 	}
-	rows.Close()
+	if err := rows.Err(); err != nil {
+		return 0, fmt.Errorf("dictionary: populate cmu tails iterate: %w", err)
+	}

 	if len(entries) == 0 {
 		return 0, nil
--- a/internal/dictionary/dict.go
+++ b/internal/dictionary/dict.go
@@ -28,6 +28,7 @@ type Options struct {
 	MinFrequency  int     // 0 = no filter
 	MinDifficulty float64 // 0.0 = no filter
 	MaxDifficulty float64 // 0.0 = no filter
+	Variant       string  // "", "us", "gb" — empty means no filter
 }

 type Definition struct {
--- a/internal/dictionary/dict_test.go
+++ b/internal/dictionary/dict_test.go
@@ -44,10 +44,13 @@ func seedTestData(t *testing.T, db *sql.DB) {
 	// Definitions
 	mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling pleasure', 'wordnet', 10)")
 	mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling joy or contentment', 'wiktionary', 20)")
+	mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (4, 'noun', 'um animal felino', 'dicionario', 10)")
+	mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (4, 'noun', 'um felino doméstico', 'wiktionary', 20)")

 	// Synonyms
 	mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'glad', 'wordnet')")
 	mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'joyful', 'wordnet')")
+	mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (4, 'bichano', 'wiktionary')")

 	// Translations
 	mustExec(t, db, "INSERT INTO translations (word_id, translation, target_lang, source) VALUES (5, 'chat', 'fr', 'wiktionary')")
@@ -69,6 +72,8 @@ func TestIsValidWord(t *testing.T) {
 		{"nonexistent", "en", false},
 		{"chat", "fr", true},
 		{"chat", "en", false},
+		{"gato", "pt-PT", true},
+		{"gato", "en", false},
 	}
 	for _, tt := range tests {
 		got, err := d.IsValidWord(tt.word, tt.lang)
@@ -143,6 +148,18 @@ func TestDefine(t *testing.T) {
 		t.Errorf("second def source = %q, want wiktionary", defs[1].Source)
 	}

+	// pt-PT definitions
+	defs, err = d.Define("gato", "pt-PT")
+	if err != nil {
+		t.Fatalf("Define(gato): %v", err)
+	}
+	if len(defs) != 2 {
+		t.Fatalf("Define(gato) returned %d defs, want 2", len(defs))
+	}
+	if defs[0].Source != "dicionario" {
+		t.Errorf("first pt-PT def source = %q, want dicionario", defs[0].Source)
+	}
+
 	// No definitions
 	defs, err = d.Define("run", "en")
 	if err != nil {
@@ -167,6 +184,15 @@ func TestSynonyms(t *testing.T) {
 		t.Fatalf("Synonyms returned %d, want 2", len(syns))
 	}

+	// pt-PT synonyms
+	syns, err = d.Synonyms("gato", "pt-PT")
+	if err != nil {
+		t.Fatalf("Synonyms(gato): %v", err)
+	}
+	if len(syns) != 1 || syns[0] != "bichano" {
+		t.Errorf("Synonyms(gato) = %v, want [bichano]", syns)
+	}
+
 	// No synonyms
 	syns, err = d.Synonyms("run", "en")
 	if err != nil {
@@ -294,6 +320,124 @@ func TestMeta(t *testing.T) {
 	}
 }

+func TestWords(t *testing.T) {
+	db := setupTestDB(t)
+	defer db.Close()
+	seedTestData(t, db)
+
+	// Add some 5-letter words with varying frequency
+	mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('crane', 'en', 'noun', 200)")
+	mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('house', 'en', 'noun', 800)")
+	mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('light', 'en', 'noun', 500)")
+	mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('quick', 'en', 'adjective', 100)")
+
+	d := NewFromDB(db)
+
+	// All 5-letter English words
+	words, err := d.Words("en", Options{MinLength: 5, MaxLength: 5})
+	if err != nil {
+		t.Fatalf("Words: %v", err)
+	}
+	if len(words) != 5 { // crane, happy, house, light, quick
+		t.Errorf("Words returned %d, want 5", len(words))
+	}
+
+	// Filter by frequency
+	words, err = d.Words("en", Options{MinLength: 5, MaxLength: 5, MinFrequency: 500})
+	if err != nil {
+		t.Fatalf("Words with freq filter: %v", err)
+	}
+	if len(words) != 2 { // house(800), light(500)
+		t.Errorf("Words with freq filter returned %d, want 2", len(words))
+	}
+
+	// Filter by POS
+	words, err = d.Words("en", Options{MinLength: 5, MaxLength: 5, POS: "noun"})
+	if err != nil {
+		t.Fatalf("Words with POS filter: %v", err)
+	}
+	if len(words) != 3 { // crane, house, light
+		t.Errorf("Words with POS filter returned %d, want 3", len(words))
+	}
+
+	// No match
+	_, err = d.Words("en", Options{MinLength: 100})
+	if err != ErrNoMatch {
+		t.Errorf("expected ErrNoMatch, got %v", err)
+	}
+
+	// Results are ordered alphabetically
+	words, err = d.Words("en", Options{MinLength: 5, MaxLength: 5, POS: "noun"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 1; i < len(words); i++ {
+		if words[i] < words[i-1] {
+			t.Errorf("words not sorted: %q before %q", words[i-1], words[i])
+		}
+	}
+}
+
+func TestPopulateCMUTails(t *testing.T) {
+	db := setupTestDB(t)
+	defer db.Close()
+	seedTestData(t, db)
+
+	// Insert CMU pronunciation entries without tails
+	mustExec(t, db, "INSERT INTO pronunciations (word_id, format, value, source) VALUES (5, 'cmu', 'K AE1 T', 'cmudict')")
+	mustExec(t, db, "INSERT INTO pronunciations (word_id, format, value, source) VALUES (1, 'cmu', 'HH AE1 P IY0', 'cmudict')")
+	// IPA entry should be ignored
+	mustExec(t, db, "INSERT INTO pronunciations (word_id, format, value, source) VALUES (2, 'ipa', '/ɹʌn/', 'wiktionary')")
+
+	n, err := PopulateCMUTails(db)
+	if err != nil {
+		t.Fatalf("PopulateCMUTails: %v", err)
+	}
+	if n != 2 {
+		t.Errorf("PopulateCMUTails returned %d, want 2", n)
+	}
+
+	// Verify tails were set correctly
+	var tail string
+	db.QueryRow("SELECT cmu_tail FROM pronunciations WHERE value = 'K AE1 T'").Scan(&tail)
+	if tail != "AE1 T" {
+		t.Errorf("cmu_tail for 'K AE1 T' = %q, want 'AE1 T'", tail)
+	}
+
+	db.QueryRow("SELECT cmu_tail FROM pronunciations WHERE value = 'HH AE1 P IY0'").Scan(&tail)
+	if tail != "AE1 P IY0" {
+		t.Errorf("cmu_tail for 'HH AE1 P IY0' = %q, want 'AE1 P IY0'", tail)
+	}
+
+	// Running again should be a no-op
+	n, err = PopulateCMUTails(db)
+	if err != nil {
+		t.Fatalf("PopulateCMUTails second run: %v", err)
+	}
+	if n != 0 {
+		t.Errorf("PopulateCMUTails second run returned %d, want 0", n)
+	}
+}
+
+func TestCmuTail(t *testing.T) {
+	tests := []struct {
+		input string
+		want  string
+	}{
+		{"K AE1 T", "AE1 T"},
+		{"HH AE1 P IY0", "AE1 P IY0"},
+		{"IH0 F EH1 M ER0 AH0 L", "EH1 M ER0 AH0 L"},
+		{"AH0", "AH0"},       // no stressed vowel, falls back to any vowel
+		{"K T S", ""},          // no vowels at all
+	}
+	for _, tt := range tests {
+		got := cmuTail(tt.input)
+		if got != tt.want {
+			t.Errorf("cmuTail(%q) = %q, want %q", tt.input, got, tt.want)
+		}
+	}
+}
+
 func TestWordCount(t *testing.T) {
 	db := setupTestDB(t)
 	defer db.Close()
@@ -310,4 +454,15 @@ func TestWordCount(t *testing.T) {
 	if counts["fr"] != 1 {
 		t.Errorf("fr word count = %d, want 1", counts["fr"])
 	}
+	if counts["pt-PT"] != 1 {
+		t.Errorf("pt-PT word count = %d, want 1", counts["pt-PT"])
+	}
+
+	total, err := d.TotalWordCount()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if total != 5 {
+		t.Errorf("total word count = %d, want 5", total)
+	}
 }
--- a/internal/dictionary/query.go
+++ b/internal/dictionary/query.go
@@ -18,6 +18,22 @@ func (d *Dictionary) IsValidWord(word, lang string) (bool, error) {
 	return exists, nil
 }

+// WordVariant returns the regional variant tag for a word ("us", "gb", or "").
+func (d *Dictionary) WordVariant(word, lang string) (string, error) {
+	var variant sql.NullString
+	err := d.db.QueryRow(
+		"SELECT variant FROM words WHERE word = ? AND lang = ?",
+		strings.ToLower(word), lang,
+	).Scan(&variant)
+	if err == sql.ErrNoRows {
+		return "", nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("dictionary: word variant: %w", err)
+	}
+	return variant.String, nil
+}
+
 type RandomResult struct {
 	Word       string   `json:"word"`
 	Difficulty *float64 `json:"difficulty,omitempty"`
@@ -51,6 +67,10 @@ func (d *Dictionary) RandomWord(lang string, opts Options) (RandomResult, error)
 		query += " AND difficulty <= ?"
 		args = append(args, opts.MaxDifficulty)
 	}
+	if opts.Variant != "" {
+		query += " AND variant = ?"
+		args = append(args, opts.Variant)
+	}

 	query += " ORDER BY RANDOM() LIMIT 1"

@@ -69,6 +89,64 @@ func (d *Dictionary) RandomWord(lang string, opts Options) (RandomResult, error)
 	return result, nil
 }

+func (d *Dictionary) Words(lang string, opts Options) ([]string, error) {
+	query := "SELECT DISTINCT word FROM words WHERE lang = ?"
+	args := []any{lang}
+
+	if opts.POS != "" {
+		query += " AND pos = ?"
+		args = append(args, opts.POS)
+	}
+	if opts.MinLength > 0 {
+		query += " AND LENGTH(word) >= ?"
+		args = append(args, opts.MinLength)
+	}
+	if opts.MaxLength > 0 {
+		query += " AND LENGTH(word) <= ?"
+		args = append(args, opts.MaxLength)
+	}
+	if opts.MinFrequency > 0 {
+		query += " AND frequency >= ?"
+		args = append(args, opts.MinFrequency)
+	}
+	if opts.MinDifficulty > 0 {
+		query += " AND difficulty >= ?"
+		args = append(args, opts.MinDifficulty)
+	}
+	if opts.MaxDifficulty > 0 {
+		query += " AND difficulty <= ?"
+		args = append(args, opts.MaxDifficulty)
+	}
+	if opts.Variant != "" {
+		query += " AND variant = ?"
+		args = append(args, opts.Variant)
+	}
+
+	query += " ORDER BY word LIMIT 20000"
+
+	rows, err := d.db.Query(query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("dictionary: words: %w", err)
+	}
+	defer rows.Close()
+
+	var words []string
+	for rows.Next() {
+		var w string
+		if err := rows.Scan(&w); err != nil {
+			return nil, fmt.Errorf("dictionary: words scan: %w", err)
+		}
+		words = append(words, w)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("dictionary: words: %w", err)
+	}
+	if len(words) == 0 {
+		return nil, ErrNoMatch
+	}
+	return words, nil
+}
+
 func (d *Dictionary) Define(word, lang string) ([]Definition, error) {
 	rows, err := d.db.Query(`
 		SELECT d.pos, d.gloss, d.source, d.priority
@@ -187,6 +265,15 @@ func (d *Dictionary) WordCount() (map[string]int, error) {
 	return counts, rows.Err()
 }

+func (d *Dictionary) TotalWordCount() (int, error) {
+	var count int
+	err := d.db.QueryRow("SELECT COUNT(*) FROM words").Scan(&count)
+	if err != nil {
+		return 0, fmt.Errorf("dictionary: total word count: %w", err)
+	}
+	return count, nil
+}
+
 // Frequency returns the frequency score for a word in a language.
 // Returns 0 if the word is not found or has no frequency data.
 // Higher values indicate more common words.
--- a/internal/loader/scowl.go
+++ b/internal/loader/scowl.go
@@ -33,33 +33,59 @@ var scowlFrequency = map[int]int{
 type scowlFile struct {
 	path    string
 	size    int
+	variant string // "us", "gb", or "" for common English
 }

 func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
-	pattern := filepath.Join(dataDir, "scowl", "final", "english-words.*")
-	files, err := filepath.Glob(pattern)
+	type prefixVariant struct {
+		prefix  string
+		variant string
+	}
+	prefixes := []prefixVariant{
+		{"english-words", ""},
+		{"american-words", "us"},
+		{"british-words", "gb"},
+	}
+
+	var allFiles []string
+	for _, pv := range prefixes {
+		pattern := filepath.Join(dataDir, "scowl", "final", pv.prefix+".*")
+		matches, err := filepath.Glob(pattern)
 		if err != nil {
 			return fmt.Errorf("scowl: glob: %w", err)
 		}
-	if len(files) == 0 {
-		return fmt.Errorf("scowl: no files matching %s", pattern)
+		allFiles = append(allFiles, matches...)
+	}
+	if len(allFiles) == 0 {
+		return fmt.Errorf("scowl: no word files found in %s", filepath.Join(dataDir, "scowl", "final"))
 	}

-	// Filter to sizes <= 70 and track the tier
+	// Build a map from prefix to variant for quick lookup
+	prefixToVariant := make(map[string]string)
+	for _, pv := range prefixes {
+		prefixToVariant[pv.prefix] = pv.variant
+	}
+
+	// Filter to sizes <= 70 and track the tier + variant
 	var selected []scowlFile
-	for _, f := range files {
+	for _, f := range allFiles {
 		base := filepath.Base(f)
-		// Files are like english-words.10, english-words.20, etc.
+		// Files are like english-words.10, american-words.35, etc.
 		parts := strings.SplitN(base, ".", 2)
 		if len(parts) != 2 {
 			continue
 		}
+		prefix := parts[0]
+		variant, ok := prefixToVariant[prefix]
+		if !ok {
+			continue
+		}
 		var size int
 		if _, err := fmt.Sscanf(parts[1], "%d", &size); err != nil {
 			continue
 		}
 		if size <= 70 {
-			selected = append(selected, scowlFile{path: f, size: size})
+			selected = append(selected, scowlFile{path: f, size: size, variant: variant})
 		}
 	}

@@ -69,8 +95,15 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
 	}
 	defer tx.Rollback()

-	stmt, err := tx.Prepare(`INSERT INTO words (word, lang, frequency) VALUES (?, 'en', ?)
-		ON CONFLICT(word, lang) DO UPDATE SET frequency = MAX(frequency, excluded.frequency)`)
+	stmt, err := tx.Prepare(`INSERT INTO words (word, lang, frequency, variant) VALUES (?, 'en', ?, ?)
+		ON CONFLICT(word, lang) DO UPDATE SET
+			frequency = MAX(frequency, excluded.frequency),
+			variant = CASE
+				WHEN words.variant IS NULL THEN excluded.variant
+				WHEN excluded.variant IS NULL THEN words.variant
+				WHEN words.variant = excluded.variant THEN words.variant
+				ELSE NULL
+			END`)
 	if err != nil {
 		return fmt.Errorf("scowl: prepare: %w", err)
 	}
@@ -79,7 +112,7 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
 	var count int
 	for _, sf := range selected {
 		freq := scowlFrequency[sf.size]
-		n, err := loadSCOWLFile(stmt, sf.path, freq)
+		n, err := loadSCOWLFile(stmt, sf.path, freq, sf.variant)
 		if err != nil {
 			return fmt.Errorf("scowl: load %s: %w", sf.path, err)
 		}
@@ -93,13 +126,19 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
 	return nil
 }

-func loadSCOWLFile(stmt *sql.Stmt, path string, freq int) (int, error) {
+func loadSCOWLFile(stmt *sql.Stmt, path string, freq int, variant string) (int, error) {
 	f, err := os.Open(path)
 	if err != nil {
 		return 0, err
 	}
 	defer f.Close()

+	// Convert empty variant to SQL NULL
+	var variantParam any
+	if variant != "" {
+		variantParam = variant
+	}
+
 	var count int
 	scanner := bufio.NewScanner(f)
 	for scanner.Scan() {
@@ -111,7 +150,7 @@ func loadSCOWLFile(stmt *sql.Stmt, path string, freq int) (int, error) {
 		if containsAny(word, " ", "'", "-") || containsDigit(word) {
 			continue
 		}
-		if _, err := stmt.Exec(word, freq); err != nil {
+		if _, err := stmt.Exec(word, freq, variantParam); err != nil {
 			return 0, err
 		}
 		count++