drwily/dreamdict
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add regional variant tagging (us/gb) for English words

Browse Source 
- SCOWL loader now loads american-words and british-words alongside
  english-words, tagging each with variant "us", "gb", or NULL (common)
- Words appearing in both American and British lists get NULL variant
- Add variant column to words table with schema migration
- API: /define returns variant field for English, /random and /words
  accept ?variant=us|gb filter
- Add total word count to server status line
- Fix rows.Close leak in PopulateCMUTails, add rows.Err() check
- Add tests for PopulateCMUTails, cmuTail, TotalWordCount
- Expand test seed data and assertions to cover pt-PT definitions,
  synonyms, and validity checks
- Update README with variant docs, /difficulty, /words, /rhyme
  endpoints, and current database statistics

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
prosolis
2026-04-04 15:21:27 -07:00
parent 28ec11ba4e
commit 704c06f41d
8 changed files with 488 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/dictionary/db.go
View File

@@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS words (
pos TEXT,
frequency INTEGER DEFAULT 0,
difficulty REAL DEFAULT NULL,
variant TEXT,
UNIQUE(word, lang)
);
@@ -159,6 +160,7 @@ func BootstrapSchema(db *sql.DB) error {
migrations := []string{
"ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL",
"ALTER TABLE pronunciations ADD COLUMN cmu_tail TEXT",
"ALTER TABLE words ADD COLUMN variant TEXT",
}
for _, m := range migrations {
if _, err := db.Exec(m); err != nil {
@@ -205,6 +207,7 @@ func PopulateCMUTails(db *sql.DB) (int, error) {
if err != nil {
return 0, fmt.Errorf("dictionary: populate cmu tails: %w", err)
}
defer rows.Close()
type entry struct {
id int
@@ -214,12 +217,13 @@ func PopulateCMUTails(db *sql.DB) (int, error) {
for rows.Next() {
var e entry
if err := rows.Scan(&e.id, &e.value); err != nil {
rows.Close()
return 0, fmt.Errorf("dictionary: populate cmu tails scan: %w", err)
}
entries = append(entries, e)
}
rows.Close()
if err := rows.Err(); err != nil {
return 0, fmt.Errorf("dictionary: populate cmu tails iterate: %w", err)
}
if len(entries) == 0 {
return 0, nil

1
internal/dictionary/dict.go
View File

@@ -28,6 +28,7 @@ type Options struct {
MinFrequency int // 0 = no filter
MinDifficulty float64 // 0.0 = no filter
MaxDifficulty float64 // 0.0 = no filter
Variant string // "", "us", "gb" — empty means no filter
}
type Definition struct {

155
internal/dictionary/dict_test.go
View File

@@ -44,10 +44,13 @@ func seedTestData(t *testing.T, db *sql.DB) {
// Definitions
mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling pleasure', 'wordnet', 10)")
mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling joy or contentment', 'wiktionary', 20)")
mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (4, 'noun', 'um animal felino', 'dicionario', 10)")
mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (4, 'noun', 'um felino doméstico', 'wiktionary', 20)")
// Synonyms
mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'glad', 'wordnet')")
mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'joyful', 'wordnet')")
mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (4, 'bichano', 'wiktionary')")
// Translations
mustExec(t, db, "INSERT INTO translations (word_id, translation, target_lang, source) VALUES (5, 'chat', 'fr', 'wiktionary')")
@@ -69,6 +72,8 @@ func TestIsValidWord(t *testing.T) {
{"nonexistent", "en", false},
{"chat", "fr", true},
{"chat", "en", false},
{"gato", "pt-PT", true},
{"gato", "en", false},
}
for _, tt := range tests {
got, err := d.IsValidWord(tt.word, tt.lang)
@@ -143,6 +148,18 @@ func TestDefine(t *testing.T) {
t.Errorf("second def source = %q, want wiktionary", defs[1].Source)
}
// pt-PT definitions
defs, err = d.Define("gato", "pt-PT")
if err != nil {
t.Fatalf("Define(gato): %v", err)
}
if len(defs) != 2 {
t.Fatalf("Define(gato) returned %d defs, want 2", len(defs))
}
if defs[0].Source != "dicionario" {
t.Errorf("first pt-PT def source = %q, want dicionario", defs[0].Source)
}
// No definitions
defs, err = d.Define("run", "en")
if err != nil {
@@ -167,6 +184,15 @@ func TestSynonyms(t *testing.T) {
t.Fatalf("Synonyms returned %d, want 2", len(syns))
}
// pt-PT synonyms
syns, err = d.Synonyms("gato", "pt-PT")
if err != nil {
t.Fatalf("Synonyms(gato): %v", err)
}
if len(syns) != 1 || syns[0] != "bichano" {
t.Errorf("Synonyms(gato) = %v, want [bichano]", syns)
}
// No synonyms
syns, err = d.Synonyms("run", "en")
if err != nil {
@@ -294,6 +320,124 @@ func TestMeta(t *testing.T) {
}
}
func TestWords(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
seedTestData(t, db)
// Add some 5-letter words with varying frequency
mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('crane', 'en', 'noun', 200)")
mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('house', 'en', 'noun', 800)")
mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('light', 'en', 'noun', 500)")
mustExec(t, db, "INSERT INTO words (word, lang, pos, frequency) VALUES ('quick', 'en', 'adjective', 100)")
d := NewFromDB(db)
// All 5-letter English words
words, err := d.Words("en", Options{MinLength: 5, MaxLength: 5})
if err != nil {
t.Fatalf("Words: %v", err)
}
if len(words) != 5 { // crane, happy, house, light, quick
t.Errorf("Words returned %d, want 5", len(words))
}
// Filter by frequency
words, err = d.Words("en", Options{MinLength: 5, MaxLength: 5, MinFrequency: 500})
if err != nil {
t.Fatalf("Words with freq filter: %v", err)
}
if len(words) != 2 { // house(800), light(500)
t.Errorf("Words with freq filter returned %d, want 2", len(words))
}
// Filter by POS
words, err = d.Words("en", Options{MinLength: 5, MaxLength: 5, POS: "noun"})
if err != nil {
t.Fatalf("Words with POS filter: %v", err)
}
if len(words) != 3 { // crane, house, light
t.Errorf("Words with POS filter returned %d, want 3", len(words))
}
// No match
_, err = d.Words("en", Options{MinLength: 100})
if err != ErrNoMatch {
t.Errorf("expected ErrNoMatch, got %v", err)
}
// Results are ordered alphabetically
words, err = d.Words("en", Options{MinLength: 5, MaxLength: 5, POS: "noun"})
if err != nil {
t.Fatal(err)
}
for i := 1; i < len(words); i++ {
if words[i] < words[i-1] {
t.Errorf("words not sorted: %q before %q", words[i-1], words[i])
}
}
}
func TestPopulateCMUTails(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
seedTestData(t, db)
// Insert CMU pronunciation entries without tails
mustExec(t, db, "INSERT INTO pronunciations (word_id, format, value, source) VALUES (5, 'cmu', 'K AE1 T', 'cmudict')")
mustExec(t, db, "INSERT INTO pronunciations (word_id, format, value, source) VALUES (1, 'cmu', 'HH AE1 P IY0', 'cmudict')")
// IPA entry should be ignored
mustExec(t, db, "INSERT INTO pronunciations (word_id, format, value, source) VALUES (2, 'ipa', '/ɹʌn/', 'wiktionary')")
n, err := PopulateCMUTails(db)
if err != nil {
t.Fatalf("PopulateCMUTails: %v", err)
}
if n != 2 {
t.Errorf("PopulateCMUTails returned %d, want 2", n)
}
// Verify tails were set correctly
var tail string
db.QueryRow("SELECT cmu_tail FROM pronunciations WHERE value = 'K AE1 T'").Scan(&tail)
if tail != "AE1 T" {
t.Errorf("cmu_tail for 'K AE1 T' = %q, want 'AE1 T'", tail)
}
db.QueryRow("SELECT cmu_tail FROM pronunciations WHERE value = 'HH AE1 P IY0'").Scan(&tail)
if tail != "AE1 P IY0" {
t.Errorf("cmu_tail for 'HH AE1 P IY0' = %q, want 'AE1 P IY0'", tail)
}
// Running again should be a no-op
n, err = PopulateCMUTails(db)
if err != nil {
t.Fatalf("PopulateCMUTails second run: %v", err)
}
if n != 0 {
t.Errorf("PopulateCMUTails second run returned %d, want 0", n)
}
}
func TestCmuTail(t *testing.T) {
tests := []struct {
input string
want string
}{
{"K AE1 T", "AE1 T"},
{"HH AE1 P IY0", "AE1 P IY0"},
{"IH0 F EH1 M ER0 AH0 L", "EH1 M ER0 AH0 L"},
{"AH0", "AH0"}, // no stressed vowel, falls back to any vowel
{"K T S", ""}, // no vowels at all
}
for _, tt := range tests {
got := cmuTail(tt.input)
if got != tt.want {
t.Errorf("cmuTail(%q) = %q, want %q", tt.input, got, tt.want)
}
}
}
func TestWordCount(t *testing.T) {
db := setupTestDB(t)
defer db.Close()
@@ -310,4 +454,15 @@ func TestWordCount(t *testing.T) {
if counts["fr"] != 1 {
t.Errorf("fr word count = %d, want 1", counts["fr"])
}
if counts["pt-PT"] != 1 {
t.Errorf("pt-PT word count = %d, want 1", counts["pt-PT"])
}
total, err := d.TotalWordCount()
if err != nil {
t.Fatal(err)
}
if total != 5 {
t.Errorf("total word count = %d, want 5", total)
}
}

87
internal/dictionary/query.go
View File

@@ -18,6 +18,22 @@ func (d *Dictionary) IsValidWord(word, lang string) (bool, error) {
return exists, nil
}
// WordVariant returns the regional variant tag for a word ("us", "gb", or "").
func (d *Dictionary) WordVariant(word, lang string) (string, error) {
var variant sql.NullString
err := d.db.QueryRow(
"SELECT variant FROM words WHERE word = ? AND lang = ?",
strings.ToLower(word), lang,
).Scan(&variant)
if err == sql.ErrNoRows {
return "", nil
}
if err != nil {
return "", fmt.Errorf("dictionary: word variant: %w", err)
}
return variant.String, nil
}
type RandomResult struct {
Word string `json:"word"`
Difficulty *float64 `json:"difficulty,omitempty"`
@@ -51,6 +67,10 @@ func (d *Dictionary) RandomWord(lang string, opts Options) (RandomResult, error)
query += " AND difficulty <= ?"
args = append(args, opts.MaxDifficulty)
}
if opts.Variant != "" {
query += " AND variant = ?"
args = append(args, opts.Variant)
}
query += " ORDER BY RANDOM() LIMIT 1"
@@ -69,6 +89,64 @@ func (d *Dictionary) RandomWord(lang string, opts Options) (RandomResult, error)
return result, nil
}
func (d *Dictionary) Words(lang string, opts Options) ([]string, error) {
query := "SELECT DISTINCT word FROM words WHERE lang = ?"
args := []any{lang}
if opts.POS != "" {
query += " AND pos = ?"
args = append(args, opts.POS)
}
if opts.MinLength > 0 {
query += " AND LENGTH(word) >= ?"
args = append(args, opts.MinLength)
}
if opts.MaxLength > 0 {
query += " AND LENGTH(word) <= ?"
args = append(args, opts.MaxLength)
}
if opts.MinFrequency > 0 {
query += " AND frequency >= ?"
args = append(args, opts.MinFrequency)
}
if opts.MinDifficulty > 0 {
query += " AND difficulty >= ?"
args = append(args, opts.MinDifficulty)
}
if opts.MaxDifficulty > 0 {
query += " AND difficulty <= ?"
args = append(args, opts.MaxDifficulty)
}
if opts.Variant != "" {
query += " AND variant = ?"
args = append(args, opts.Variant)
}
query += " ORDER BY word LIMIT 20000"
rows, err := d.db.Query(query, args...)
if err != nil {
return nil, fmt.Errorf("dictionary: words: %w", err)
}
defer rows.Close()
var words []string
for rows.Next() {
var w string
if err := rows.Scan(&w); err != nil {
return nil, fmt.Errorf("dictionary: words scan: %w", err)
}
words = append(words, w)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("dictionary: words: %w", err)
}
if len(words) == 0 {
return nil, ErrNoMatch
}
return words, nil
}
func (d *Dictionary) Define(word, lang string) ([]Definition, error) {
rows, err := d.db.Query(`
SELECT d.pos, d.gloss, d.source, d.priority
@@ -187,6 +265,15 @@ func (d *Dictionary) WordCount() (map[string]int, error) {
return counts, rows.Err()
}
func (d *Dictionary) TotalWordCount() (int, error) {
var count int
err := d.db.QueryRow("SELECT COUNT(*) FROM words").Scan(&count)
if err != nil {
return 0, fmt.Errorf("dictionary: total word count: %w", err)
}
return count, nil
}
// Frequency returns the frequency score for a word in a language.
// Returns 0 if the word is not found or has no frequency data.
// Higher values indicate more common words.

73
internal/loader/scowl.go
View File

@@ -31,35 +31,61 @@ var scowlFrequency = map[int]int{
}
type scowlFile struct {
path string
size int
path string
size int
variant string // "us", "gb", or "" for common English
}
func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
pattern := filepath.Join(dataDir, "scowl", "final", "english-words.*")
files, err := filepath.Glob(pattern)
if err != nil {
return fmt.Errorf("scowl: glob: %w", err)
type prefixVariant struct {
prefix string
variant string
}
if len(files) == 0 {
return fmt.Errorf("scowl: no files matching %s", pattern)
prefixes := []prefixVariant{
{"english-words", ""},
{"american-words", "us"},
{"british-words", "gb"},
}
// Filter to sizes <= 70 and track the tier
var allFiles []string
for _, pv := range prefixes {
pattern := filepath.Join(dataDir, "scowl", "final", pv.prefix+".*")
matches, err := filepath.Glob(pattern)
if err != nil {
return fmt.Errorf("scowl: glob: %w", err)
}
allFiles = append(allFiles, matches...)
}
if len(allFiles) == 0 {
return fmt.Errorf("scowl: no word files found in %s", filepath.Join(dataDir, "scowl", "final"))
}
// Build a map from prefix to variant for quick lookup
prefixToVariant := make(map[string]string)
for _, pv := range prefixes {
prefixToVariant[pv.prefix] = pv.variant
}
// Filter to sizes <= 70 and track the tier + variant
var selected []scowlFile
for _, f := range files {
for _, f := range allFiles {
base := filepath.Base(f)
// Files are like english-words.10, english-words.20, etc.
// Files are like english-words.10, american-words.35, etc.
parts := strings.SplitN(base, ".", 2)
if len(parts) != 2 {
continue
}
prefix := parts[0]
variant, ok := prefixToVariant[prefix]
if !ok {
continue
}
var size int
if _, err := fmt.Sscanf(parts[1], "%d", &size); err != nil {
continue
}
if size <= 70 {
selected = append(selected, scowlFile{path: f, size: size})
selected = append(selected, scowlFile{path: f, size: size, variant: variant})
}
}
@@ -69,8 +95,15 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
}
defer tx.Rollback()
stmt, err := tx.Prepare(`INSERT INTO words (word, lang, frequency) VALUES (?, 'en', ?)
ON CONFLICT(word, lang) DO UPDATE SET frequency = MAX(frequency, excluded.frequency)`)
stmt, err := tx.Prepare(`INSERT INTO words (word, lang, frequency, variant) VALUES (?, 'en', ?, ?)
ON CONFLICT(word, lang) DO UPDATE SET
frequency = MAX(frequency, excluded.frequency),
variant = CASE
WHEN words.variant IS NULL THEN excluded.variant
WHEN excluded.variant IS NULL THEN words.variant
WHEN words.variant = excluded.variant THEN words.variant
ELSE NULL
END`)
if err != nil {
return fmt.Errorf("scowl: prepare: %w", err)
}
@@ -79,7 +112,7 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
var count int
for _, sf := range selected {
freq := scowlFrequency[sf.size]
n, err := loadSCOWLFile(stmt, sf.path, freq)
n, err := loadSCOWLFile(stmt, sf.path, freq, sf.variant)
if err != nil {
return fmt.Errorf("scowl: load %s: %w", sf.path, err)
}
@@ -93,13 +126,19 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
return nil
}
func loadSCOWLFile(stmt *sql.Stmt, path string, freq int) (int, error) {
func loadSCOWLFile(stmt *sql.Stmt, path string, freq int, variant string) (int, error) {
f, err := os.Open(path)
if err != nil {
return 0, err
}
defer f.Close()
// Convert empty variant to SQL NULL
var variantParam any
if variant != "" {
variantParam = variant
}
var count int
scanner := bufio.NewScanner(f)
for scanner.Scan() {
@@ -111,7 +150,7 @@ func loadSCOWLFile(stmt *sql.Stmt, path string, freq int) (int, error) {
if containsAny(word, " ", "'", "-") || containsDigit(word) {
continue
}
if _, err := stmt.Exec(word, freq); err != nil {
if _, err := stmt.Exec(word, freq, variantParam); err != nil {
return 0, err
}
count++