drwily/dreamdict
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Phase 2+3 features: antonyms, backing, pronunciation, etymology, difficulty, affix expansion

Browse Source 
New endpoints: /antonyms, /backing, /pronunciation, /etymology with difficulty
scoring on /random. Cross-language synset backing links French/Portuguese words
to English equivalents via WordNet 3.0 synset IDs (matching WOLF and OMW offsets).

New loaders: Hunspell affix expansion (fr, pt-PT), English programmatic inflector,
CMU Pronouncing Dictionary, SUBTLEX-US frequency, CETEMPúblico frequency,
Open Multilingual Wordnet (Portuguese), and SQL-based difficulty scoring.

Schema v2 adds tables: antonyms, synsets, word_synsets, pronunciations, etymology
with migration support for existing databases.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
prosolis
2026-04-02 01:01:06 -07:00
parent d5e4ce0d36
commit 207086da08
22 changed files with 3102 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
cmd/dictimport/main.go
View File

@@ -25,7 +25,13 @@ func main() {
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})))
db, err := sql.Open("sqlite", *dbPath+"?_pragma=journal_mode(wal)&_pragma=foreign_keys(on)")
db, err := sql.Open("sqlite", *dbPath+
"?_pragma=journal_mode(wal)"+
"&_pragma=foreign_keys(on)"+
"&_pragma=synchronous(off)"+
"&_pragma=cache_size(-64000)"+
"&_pragma=mmap_size(268435456)"+
"&_pragma=temp_store(memory)")
if err != nil {
slog.Error("open db", "error", err)
os.Exit(1)
@@ -35,7 +41,7 @@ func main() {
if *clean {
slog.Info("dropping all tables")
for _, table := range []string{"translations", "synonyms", "definitions", "words", "meta"} {
for _, table := range []string{"etymology", "pronunciations", "word_synsets", "synsets", "antonyms", "translations", "synonyms", "definitions", "words", "meta"} {
if _, err := db.Exec("DROP TABLE IF EXISTS " + table); err != nil {
slog.Error("drop table", "table", table, "error", err)
os.Exit(1)
@@ -65,7 +71,10 @@ func main() {
if langSet["en"] {
loaders = append(loaders,
loader.SCOWLLoader{},
loader.EnglishAffixLoader{},
loader.WordNetLoader{},
loader.SubtlexLoader{},
loader.CMUDictLoader{},
loader.WiktionaryLoader{Lang: "en", FileName: "kaikki-en.jsonl"},
)
}
@@ -73,6 +82,7 @@ func main() {
if langSet["fr"] {
loaders = append(loaders,
loader.HunspellLoader{Lang: "fr", FileName: "fr_FR/fr.dic"},
loader.AffixLoader{Lang: "fr", DicFile: "fr_FR/fr.dic", AffFile: "fr_FR/fr.aff"},
loader.LexiqueLoader{},
loader.WOLFLoader{},
loader.WiktionaryLoader{Lang: "fr", FileName: "kaikki-fr.jsonl"},
@@ -82,7 +92,10 @@ func main() {
if langSet["pt-PT"] {
loaders = append(loaders,
loader.HunspellLoader{Lang: "pt-PT", FileName: "pt_PT/pt_PT.dic"},
loader.AffixLoader{Lang: "pt-PT", DicFile: "pt_PT/pt_PT.dic", AffFile: "pt_PT/pt_PT.aff"},
loader.DicionarioLoader{},
loader.CETEMPublicoLoader{},
loader.OMWLoader{},
loader.WiktionaryLoader{Lang: "pt-PT", FileName: "kaikki-pt.jsonl"},
)
}
@@ -94,6 +107,9 @@ func main() {
)
}
// Difficulty scorer runs last, after all word data is loaded
loaders = append(loaders, loader.DifficultyScorer{})
start := time.Now()
if err := loader.Run(db, *dataDir, loaders, skipSet); err != nil {
slog.Error("import failed", "error", err)
@@ -104,7 +120,7 @@ func main() {
now := time.Now().UTC().Format(time.RFC3339)
for _, kv := range [][2]string{
{"imported_at", now},
{"schema_version", "1"},
{"schema_version", "2"},
} {
if _, err := db.Exec(
"INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", kv[0], kv[1],

108
cmd/server/bench_test.go Normal file
View File

@@ -0,0 +1,108 @@
package main
import (
"database/sql"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"dreamdict/internal/dictionary"
_ "modernc.org/sqlite"
)
func setupBenchServer(b *testing.B) *http.ServeMux {
b.Helper()
db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(on)")
if err != nil {
b.Fatal(err)
}
db.SetMaxOpenConns(1)
if err := dictionary.BootstrapSchema(db); err != nil {
b.Fatal(err)
}
exec := func(q string, args ...any) {
if _, err := db.Exec(q, args...); err != nil {
b.Fatalf("seed: %s: %v", q, err)
}
}
exec("INSERT INTO meta (key, value) VALUES ('schema_version', '1')")
exec("INSERT INTO meta (key, value) VALUES ('imported_at', '2025-01-01T00:00:00Z')")
poses := []string{"noun", "verb", "adjective", "adverb"}
for i := 1; i <= 500; i++ {
pos := poses[i%len(poses)]
word := fmt.Sprintf("word%04d", i)
exec("INSERT INTO words (id, word, lang, pos, frequency) VALUES (?, ?, 'en', ?, ?)",
i, word, pos, i%100)
exec("INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (?, ?, ?, 'wordnet', 10)",
i, pos, fmt.Sprintf("def of %s", word))
exec("INSERT INTO synonyms (word_id, synonym, source) VALUES (?, ?, 'wordnet')",
i, fmt.Sprintf("syn_%s", word))
exec("INSERT INTO translations (word_id, translation, target_lang, source) VALUES (?, ?, 'fr', 'wiktionary')",
i, fmt.Sprintf("fr_%s", word))
}
dict := dictionary.NewFromDB(db)
b.Cleanup(func() { db.Close() })
mux := http.NewServeMux()
mux.HandleFunc("GET /valid", handleValid(dict))
mux.HandleFunc("GET /random", handleRandom(dict))
mux.HandleFunc("GET /define", handleDefine(dict))
mux.HandleFunc("GET /synonyms", handleSynonyms(dict))
mux.HandleFunc("GET /translate", handleTranslate(dict))
mux.HandleFunc("GET /health", handleHealth(dict, ":memory:"))
return mux
}
func benchRequest(b *testing.B, mux *http.ServeMux, path string) {
b.Helper()
req := httptest.NewRequest("GET", path, nil)
b.ResetTimer()
for b.Loop() {
w := httptest.NewRecorder()
mux.ServeHTTP(w, req)
if w.Code != 200 {
b.Fatalf("status %d for %s", w.Code, path)
}
}
}
func BenchmarkHTTPValid(b *testing.B) {
mux := setupBenchServer(b)
benchRequest(b, mux, "/valid?word=word0003&lang=en")
}
func BenchmarkHTTPRandom(b *testing.B) {
mux := setupBenchServer(b)
benchRequest(b, mux, "/random?lang=en")
}
func BenchmarkHTTPRandomFiltered(b *testing.B) {
mux := setupBenchServer(b)
benchRequest(b, mux, "/random?lang=en&pos=noun&min=5&max=10")
}
func BenchmarkHTTPDefine(b *testing.B) {
mux := setupBenchServer(b)
benchRequest(b, mux, "/define?word=word0003&lang=en")
}
func BenchmarkHTTPSynonyms(b *testing.B) {
mux := setupBenchServer(b)
benchRequest(b, mux, "/synonyms?word=word0003&lang=en")
}
func BenchmarkHTTPTranslate(b *testing.B) {
mux := setupBenchServer(b)
benchRequest(b, mux, "/translate?word=word0003&from=en&to=fr")
}
func BenchmarkHTTPHealth(b *testing.B) {
mux := setupBenchServer(b)
benchRequest(b, mux, "/health")
}

147
cmd/server/main.go
View File

@@ -80,9 +80,13 @@ func main() {
mux.HandleFunc("GET /random", handleRandom(dict))
mux.HandleFunc("GET /define", handleDefine(dict))
mux.HandleFunc("GET /synonyms", handleSynonyms(dict))
mux.HandleFunc("GET /antonyms", handleAntonyms(dict))
mux.HandleFunc("GET /translate", handleTranslate(dict))
mux.HandleFunc("GET /backing", handleBacking(dict))
mux.HandleFunc("GET /frequency", handleFrequency(dict))
mux.HandleFunc("GET /frequency/batch", handleFrequencyBatch(dict))
mux.HandleFunc("GET /pronunciation", handlePronunciation(dict))
mux.HandleFunc("GET /etymology", handleEtymology(dict))
mux.HandleFunc("GET /health", handleHealth(dict, *dbPath))
serverStart = time.Now()
@@ -220,8 +224,18 @@ func handleRandom(dict *dictionary.Dictionary) http.HandlerFunc {
opts.MinFrequency = n
}
}
if v := r.URL.Query().Get("min_difficulty"); v != "" {
if f, err := strconv.ParseFloat(v, 64); err == nil {
opts.MinDifficulty = f
}
}
if v := r.URL.Query().Get("max_difficulty"); v != "" {
if f, err := strconv.ParseFloat(v, 64); err == nil {
opts.MaxDifficulty = f
}
}
word, err := dict.RandomWord(lang, opts)
result, err := dict.RandomWord(lang, opts)
if err == dictionary.ErrNoMatch {
writeError(w, 404, "no_match", "no words match the given filters")
return
@@ -231,7 +245,11 @@ func handleRandom(dict *dictionary.Dictionary) http.HandlerFunc {
writeError(w, 500, "internal", "internal error")
return
}
writeJSON(w, 200, map[string]string{"word": word})
resp := map[string]any{"word": result.Word}
if result.Difficulty != nil {
resp["difficulty"] = *result.Difficulty
}
writeJSON(w, 200, resp)
}
}
@@ -381,8 +399,125 @@ func handleFrequencyBatch(dict *dictionary.Dictionary) http.HandlerFunc {
}
}
func handleHealth(dict *dictionary.Dictionary, dbPath string) http.HandlerFunc {
func handleAntonyms(dict *dictionary.Dictionary) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
word := r.URL.Query().Get("word")
lang := r.URL.Query().Get("lang")
if word == "" || lang == "" {
writeError(w, 400, "bad_request", "word and lang are required")
return
}
if !dictionary.ValidLang(lang) {
writeError(w, 400, "bad_request", "unsupported language: "+lang)
return
}
ants, err := dict.Antonyms(word, lang)
if err != nil {
slog.Error("antonyms", "error", err)
writeError(w, 500, "internal", "internal error")
return
}
writeJSON(w, 200, map[string]any{
"word": word,
"lang": lang,
"antonyms": ants,
})
}
}
func handleBacking(dict *dictionary.Dictionary) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
word := r.URL.Query().Get("word")
lang := r.URL.Query().Get("lang")
if word == "" || lang == "" {
writeError(w, 400, "bad_request", "word and lang are required")
return
}
if !dictionary.ValidLang(lang) {
writeError(w, 400, "bad_request", "unsupported language: "+lang)
return
}
equivs, err := dict.EnglishBacking(word, lang)
if err != nil {
slog.Error("backing", "error", err)
writeError(w, 500, "internal", "internal error")
return
}
writeJSON(w, 200, map[string]any{
"word": word,
"lang": lang,
"equivalents": equivs,
})
}
}
func handlePronunciation(dict *dictionary.Dictionary) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
word := r.URL.Query().Get("word")
lang := r.URL.Query().Get("lang")
if word == "" || lang == "" {
writeError(w, 400, "bad_request", "word and lang are required")
return
}
if !dictionary.ValidLang(lang) {
writeError(w, 400, "bad_request", "unsupported language: "+lang)
return
}
prons, err := dict.Pronunciation(word, lang)
if err != nil {
slog.Error("pronunciation", "error", err)
writeError(w, 500, "internal", "internal error")
return
}
writeJSON(w, 200, map[string]any{
"word": word,
"lang": lang,
"pronunciations": prons,
})
}
}
func handleEtymology(dict *dictionary.Dictionary) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
word := r.URL.Query().Get("word")
lang := r.URL.Query().Get("lang")
if word == "" || lang == "" {
writeError(w, 400, "bad_request", "word and lang are required")
return
}
if !dictionary.ValidLang(lang) {
writeError(w, 400, "bad_request", "unsupported language: "+lang)
return
}
etym, err := dict.Etymology(word, lang)
if err != nil {
slog.Error("etymology", "error", err)
writeError(w, 500, "internal", "internal error")
return
}
writeJSON(w, 200, map[string]any{
"word": word,
"lang": lang,
"etymology": etym,
})
}
}
func handleHealth(dict *dictionary.Dictionary, dbPath string) http.HandlerFunc {
// The database is read-only at runtime, so cache the health response
// after the first (slow) computation.
var cached atomic.Value
return func(w http.ResponseWriter, r *http.Request) {
if v := cached.Load(); v != nil {
writeJSON(w, 200, v)
return
}
wordCounts, err := dict.WordCount()
if err != nil {
slog.Error("health: word count", "error", err)
@@ -409,13 +544,15 @@ func handleHealth(dict *dictionary.Dictionary, dbPath string) http.HandlerFunc {
return
}
writeJSON(w, 200, map[string]any{
resp := map[string]any{
"status": "ok",
"db_path": dbPath,
"word_counts": wordCounts,
"def_counts": defCounts,
"imported_at": importedAt,
"schema_version": schemaVersion,
})
}
cached.Store(resp)
writeJSON(w, 200, resp)
}
}

226
cmd/server/server_test.go Normal file
View File

@@ -0,0 +1,226 @@
package main
import (
"database/sql"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"dreamdict/internal/dictionary"
_ "modernc.org/sqlite"
)
func mustExec(t *testing.T, db *sql.DB, query string, args ...any) {
t.Helper()
if _, err := db.Exec(query, args...); err != nil {
t.Fatalf("mustExec: %s: %v", query, err)
}
}
func setupTestServer(t *testing.T) (*dictionary.Dictionary, *http.ServeMux) {
t.Helper()
db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(on)")
if err != nil {
t.Fatal(err)
}
db.SetMaxOpenConns(1)
if err := dictionary.BootstrapSchema(db); err != nil {
t.Fatal(err)
}
// Seed data
mustExec(t, db, "INSERT INTO meta (key, value) VALUES ('schema_version', '1')")
mustExec(t, db, "INSERT INTO meta (key, value) VALUES ('imported_at', '2025-01-01T00:00:00Z')")
mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (1, 'happy', 'en', 'adjective', 0)")
mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (2, 'cat', 'en', 'noun', 0)")
mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (3, 'chat', 'fr', 'noun', 5000)")
mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling pleasure', 'wordnet', 10)")
mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling joy', 'wiktionary', 20)")
mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'glad', 'wordnet')")
mustExec(t, db, "INSERT INTO translations (word_id, translation, target_lang, source) VALUES (2, 'chat', 'fr', 'wiktionary')")
dict := dictionary.NewFromDB(db)
mux := http.NewServeMux()
mux.HandleFunc("GET /valid", handleValid(dict))
mux.HandleFunc("GET /random", handleRandom(dict))
mux.HandleFunc("GET /define", handleDefine(dict))
mux.HandleFunc("GET /synonyms", handleSynonyms(dict))
mux.HandleFunc("GET /translate", handleTranslate(dict))
mux.HandleFunc("GET /health", handleHealth(dict, ":memory:"))
t.Cleanup(func() { db.Close() })
return dict, mux
}
func TestValidEndpoint(t *testing.T) {
_, mux := setupTestServer(t)
srv := httptest.NewServer(mux)
defer srv.Close()
// Valid word
resp, _ := http.Get(srv.URL + "/valid?word=happy&lang=en")
if resp.StatusCode != 200 {
t.Fatalf("status = %d, want 200", resp.StatusCode)
}
var result map[string]bool
json.NewDecoder(resp.Body).Decode(&result)
resp.Body.Close()
if !result["valid"] {
t.Error("expected valid=true")
}
// Invalid word
resp, _ = http.Get(srv.URL + "/valid?word=zzzzz&lang=en")
json.NewDecoder(resp.Body).Decode(&result)
resp.Body.Close()
if result["valid"] {
t.Error("expected valid=false")
}
// Missing params
resp, _ = http.Get(srv.URL + "/valid?word=happy")
if resp.StatusCode != 400 {
t.Errorf("missing lang: status = %d, want 400", resp.StatusCode)
}
resp.Body.Close()
}
func TestRandomEndpoint(t *testing.T) {
_, mux := setupTestServer(t)
srv := httptest.NewServer(mux)
defer srv.Close()
resp, _ := http.Get(srv.URL + "/random?lang=en")
if resp.StatusCode != 200 {
t.Fatalf("status = %d, want 200", resp.StatusCode)
}
var result map[string]string
json.NewDecoder(resp.Body).Decode(&result)
resp.Body.Close()
if result["word"] == "" {
t.Error("expected non-empty word")
}
// No match
resp, _ = http.Get(srv.URL + "/random?lang=en&min=100")
if resp.StatusCode != 404 {
t.Errorf("no match: status = %d, want 404", resp.StatusCode)
}
resp.Body.Close()
}
func TestDefineEndpoint(t *testing.T) {
_, mux := setupTestServer(t)
srv := httptest.NewServer(mux)
defer srv.Close()
resp, _ := http.Get(srv.URL + "/define?word=happy&lang=en")
if resp.StatusCode != 200 {
t.Fatalf("status = %d, want 200", resp.StatusCode)
}
var result struct {
Word string `json:"word"`
Lang string `json:"lang"`
Definitions []dictionary.Definition `json:"definitions"`
}
json.NewDecoder(resp.Body).Decode(&result)
resp.Body.Close()
if len(result.Definitions) != 2 {
t.Errorf("got %d definitions, want 2", len(result.Definitions))
}
if result.Definitions[0].Priority > result.Definitions[1].Priority {
t.Error("definitions not ordered by priority")
}
// Word with no definitions
resp, _ = http.Get(srv.URL + "/define?word=cat&lang=en")
if resp.StatusCode != 200 {
t.Fatalf("no defs: status = %d, want 200", resp.StatusCode)
}
json.NewDecoder(resp.Body).Decode(&result)
resp.Body.Close()
if len(result.Definitions) != 0 {
t.Errorf("got %d definitions, want 0", len(result.Definitions))
}
}
func TestSynonymsEndpoint(t *testing.T) {
_, mux := setupTestServer(t)
srv := httptest.NewServer(mux)
defer srv.Close()
resp, _ := http.Get(srv.URL + "/synonyms?word=happy&lang=en")
if resp.StatusCode != 200 {
t.Fatalf("status = %d, want 200", resp.StatusCode)
}
var result struct {
Synonyms []string `json:"synonyms"`
}
json.NewDecoder(resp.Body).Decode(&result)
resp.Body.Close()
if len(result.Synonyms) != 1 {
t.Errorf("got %d synonyms, want 1", len(result.Synonyms))
}
// No synonyms
resp, _ = http.Get(srv.URL + "/synonyms?word=cat&lang=en")
json.NewDecoder(resp.Body).Decode(&result)
resp.Body.Close()
if len(result.Synonyms) != 0 {
t.Errorf("got %d synonyms, want 0", len(result.Synonyms))
}
}
func TestTranslateEndpoint(t *testing.T) {
_, mux := setupTestServer(t)
srv := httptest.NewServer(mux)
defer srv.Close()
resp, _ := http.Get(srv.URL + "/translate?word=cat&from=en&to=fr")
if resp.StatusCode != 200 {
t.Fatalf("status = %d, want 200", resp.StatusCode)
}
var result struct {
Translations []string `json:"translations"`
}
json.NewDecoder(resp.Body).Decode(&result)
resp.Body.Close()
if len(result.Translations) != 1 || result.Translations[0] != "chat" {
t.Errorf("got %v, want [chat]", result.Translations)
}
// No translations
resp, _ = http.Get(srv.URL + "/translate?word=happy&from=en&to=fr")
json.NewDecoder(resp.Body).Decode(&result)
resp.Body.Close()
if len(result.Translations) != 0 {
t.Errorf("got %d, want 0", len(result.Translations))
}
}
func TestHealthEndpoint(t *testing.T) {
_, mux := setupTestServer(t)
srv := httptest.NewServer(mux)
defer srv.Close()
resp, _ := http.Get(srv.URL + "/health")
if resp.StatusCode != 200 {
t.Fatalf("status = %d, want 200", resp.StatusCode)
}
var result map[string]any
json.NewDecoder(resp.Body).Decode(&result)
resp.Body.Close()
if result["status"] != "ok" {
t.Errorf("status = %v, want ok", result["status"])
}
counts, ok := result["word_counts"].(map[string]any)
if !ok {
t.Fatal("word_counts not a map")
}
if counts["en"].(float64) != 2 {
t.Errorf("en word count = %v, want 2", counts["en"])
}
}