From 5b1eea67f21b9d4c455e507d2bb159d60ee660cb Mon Sep 17 00:00:00 2001 From: prosolis <5590409+prosolis@users.noreply.github.com> Date: Mon, 30 Mar 2026 17:20:14 -0700 Subject: [PATCH] Add Mandarin (zh) language support Add CC-CEDICT loader and Wiktionary Chinese as data sources for Mandarin. Also fix all Wiktionary download sections to retain compressed archives and save checksums for integrity verification. Co-Authored-By: Claude Opus 4.6 --- cmd/dictimport/main.go | 136 ++++++++++++++++++++++++++++++++++ internal/dictionary/dict.go | 2 +- internal/loader/cedict.go | 132 +++++++++++++++++++++++++++++++++ internal/loader/wiktionary.go | 1 + scripts/download-dict-data.sh | 59 +++++++++++++-- 5 files changed, 323 insertions(+), 7 deletions(-) create mode 100644 cmd/dictimport/main.go create mode 100644 internal/loader/cedict.go diff --git a/cmd/dictimport/main.go b/cmd/dictimport/main.go new file mode 100644 index 0000000..58b85a9 --- /dev/null +++ b/cmd/dictimport/main.go @@ -0,0 +1,136 @@ +package main + +import ( + "database/sql" + "flag" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "dreamdict/internal/dictionary" + "dreamdict/internal/loader" + + _ "modernc.org/sqlite" +) + +func main() { + dbPath := flag.String("db", "./dict.db", "Path to output dict.db") + dataDir := flag.String("data", "./data", "Path to data directory") + langs := flag.String("langs", "en,fr,pt-PT,zh", "Comma-separated languages to import") + clean := flag.Bool("clean", false, "Drop and recreate all tables before import") + skipFlag := flag.String("skip", "", "Comma-separated loader names to skip") + flag.Parse() + + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))) + + db, err := sql.Open("sqlite", *dbPath+"?_pragma=journal_mode(wal)&_pragma=foreign_keys(on)") + if err != nil { + slog.Error("open db", "error", err) + os.Exit(1) + } + defer db.Close() + db.SetMaxOpenConns(1) + + if *clean { + slog.Info("dropping all tables") + for _, table := range []string{"translations", "synonyms", "definitions", "words", "meta"} { + if _, err := db.Exec("DROP TABLE IF EXISTS " + table); err != nil { + slog.Error("drop table", "table", table, "error", err) + os.Exit(1) + } + } + } + + if err := dictionary.BootstrapSchema(db); err != nil { + slog.Error("bootstrap schema", "error", err) + os.Exit(1) + } + + langSet := make(map[string]bool) + for _, l := range strings.Split(*langs, ",") { + langSet[strings.TrimSpace(l)] = true + } + + skipSet := make(map[string]bool) + if *skipFlag != "" { + for _, s := range strings.Split(*skipFlag, ",") { + skipSet[strings.TrimSpace(s)] = true + } + } + + var loaders []loader.Loader + + if langSet["en"] { + loaders = append(loaders, + loader.SCOWLLoader{}, + loader.WordNetLoader{}, + loader.WiktionaryLoader{Lang: "en", FileName: "kaikki-en.jsonl"}, + ) + } + + if langSet["fr"] { + loaders = append(loaders, + loader.HunspellLoader{Lang: "fr", FileName: "fr_FR/fr.dic"}, + loader.LexiqueLoader{}, + loader.WOLFLoader{}, + loader.WiktionaryLoader{Lang: "fr", FileName: "kaikki-fr.jsonl"}, + ) + } + + if langSet["pt-PT"] { + loaders = append(loaders, + loader.HunspellLoader{Lang: "pt-PT", FileName: "pt_PT/pt_PT.dic"}, + loader.DicionarioLoader{}, + loader.WiktionaryLoader{Lang: "pt-PT", FileName: "kaikki-pt.jsonl"}, + ) + } + + if langSet["zh"] { + loaders = append(loaders, + loader.CEDICTLoader{}, + loader.WiktionaryLoader{Lang: "zh", FileName: "kaikki-zh.jsonl"}, + ) + } + + start := time.Now() + if err := loader.Run(db, *dataDir, loaders, skipSet); err != nil { + slog.Error("import failed", "error", err) + os.Exit(1) + } + + // Write meta + now := time.Now().UTC().Format(time.RFC3339) + for _, kv := range [][2]string{ + {"imported_at", now}, + {"schema_version", "1"}, + } { + if _, err := db.Exec( + "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", kv[0], kv[1], + ); err != nil { + slog.Error("write meta", "error", err) + os.Exit(1) + } + } + + // Print summary + dict := dictionary.NewFromDB(db) + wordCounts, _ := dict.WordCount() + defCounts, _ := dict.DefCount() + + fmt.Println("\n=== Import Summary ===") + fmt.Printf("Elapsed: %s\n", time.Since(start).Round(time.Millisecond)) + fmt.Println("Word counts:") + for _, lang := range []string{"en", "fr", "pt-PT", "zh"} { + if c, ok := wordCounts[lang]; ok { + fmt.Printf(" %s: %d\n", lang, c) + } + } + fmt.Println("Definition counts:") + for _, lang := range []string{"en", "fr", "pt-PT", "zh"} { + if c, ok := defCounts[lang]; ok { + fmt.Printf(" %s: %d\n", lang, c) + } + } +} diff --git a/internal/dictionary/dict.go b/internal/dictionary/dict.go index da7990f..d533a19 100644 --- a/internal/dictionary/dict.go +++ b/internal/dictionary/dict.go @@ -10,7 +10,7 @@ var ( ErrNotSeeded = errors.New("dictionary: database not seeded, run: go run ./cmd/dictimport") ) -var supportedLangs = []string{"en", "fr", "pt-PT"} +var supportedLangs = []string{"en", "fr", "pt-PT", "zh"} func ValidLang(lang string) bool { for _, l := range supportedLangs { diff --git a/internal/loader/cedict.go b/internal/loader/cedict.go new file mode 100644 index 0000000..b78a931 --- /dev/null +++ b/internal/loader/cedict.go @@ -0,0 +1,132 @@ +package loader + +import ( + "bufio" + "database/sql" + "fmt" + "log/slog" + "os" + "strings" +) + +// CEDICTLoader imports words and definitions from the CC-CEDICT dictionary. +// CC-CEDICT line format: Traditional Simplified [pin1 yin1] /def1/def2/ +type CEDICTLoader struct{} + +func (CEDICTLoader) Name() string { return "cedict" } + +func (CEDICTLoader) Load(db *sql.DB, dataDir string) error { + path := dataDir + "/cedict_ts.u8" + return loadCEDICT(db, path) +} + +var cedictPOSPrefixes = map[string]string{ + "to ": "verb", + "(tw)": "", +} + +func loadCEDICT(db *sql.DB, path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("cedict: open: %w", err) + } + defer f.Close() + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("cedict: begin tx: %w", err) + } + defer tx.Rollback() + + stmtWord, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang) VALUES (?, 'zh')") + if err != nil { + return fmt.Errorf("cedict: prepare word: %w", err) + } + defer stmtWord.Close() + + stmtDef, err := tx.Prepare(`INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority) + SELECT id, ?, ?, 'cedict', 15 FROM words WHERE word = ? AND lang = 'zh'`) + if err != nil { + return fmt.Errorf("cedict: prepare def: %w", err) + } + defer stmtDef.Close() + + stmtTrans, err := tx.Prepare(`INSERT OR IGNORE INTO translations (word_id, translation, target_lang, source) + SELECT id, ?, 'en', 'cedict' FROM words WHERE word = ? AND lang = 'zh'`) + if err != nil { + return fmt.Errorf("cedict: prepare trans: %w", err) + } + defer stmtTrans.Close() + + scanner := bufio.NewScanner(f) + var wordCount, defCount, transCount int + + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "#") || line == "" { + continue + } + + // Format: Traditional Simplified [pinyin] /def1/def2/ + bracketStart := strings.Index(line, "[") + slashStart := strings.Index(line, "/") + if bracketStart < 0 || slashStart < 0 { + continue + } + + parts := strings.SplitN(line[:bracketStart], " ", 3) + if len(parts) < 2 { + continue + } + simplified := parts[1] + + if simplified == "" || containsDigit(simplified) { + continue + } + + // Insert the simplified form as the word + if _, err := stmtWord.Exec(simplified); err != nil { + return fmt.Errorf("cedict: insert word: %w", err) + } + wordCount++ + + // Parse definitions between slashes + defPart := strings.TrimRight(line[slashStart:], "/") + defs := strings.Split(defPart, "/") + for _, d := range defs { + d = strings.TrimSpace(d) + if d == "" || len(d) > 500 { + continue + } + + // Guess POS from definition text + pos := "" + lower := strings.ToLower(d) + if strings.HasPrefix(lower, "to ") { + pos = "verb" + } + + // Store as definition + if _, err := stmtDef.Exec(pos, d, simplified); err != nil { + return fmt.Errorf("cedict: insert def: %w", err) + } + defCount++ + + // English definitions also serve as translations + if _, err := stmtTrans.Exec(strings.ToLower(d), simplified); err != nil { + return fmt.Errorf("cedict: insert trans: %w", err) + } + transCount++ + } + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("cedict: scan: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("cedict: commit: %w", err) + } + + slog.Info("cedict loaded", "words", wordCount, "definitions", defCount, "translations", transCount) + return nil +} diff --git a/internal/loader/wiktionary.go b/internal/loader/wiktionary.go index ef529b2..98c6f7d 100644 --- a/internal/loader/wiktionary.go +++ b/internal/loader/wiktionary.go @@ -52,6 +52,7 @@ var wiktLangCodeMap = map[string]string{ "fr": "fr", "pt": "pt-PT", "en": "en", + "zh": "zh", } const ( diff --git a/scripts/download-dict-data.sh b/scripts/download-dict-data.sh index 071bcb2..98cb6c3 100755 --- a/scripts/download-dict-data.sh +++ b/scripts/download-dict-data.sh @@ -184,9 +184,11 @@ if [[ -f "$DATA_DIR/kaikki-en.jsonl" && "$FORCE" != "true" ]]; then fi fi if [[ ! -f "$DATA_DIR/kaikki-en.jsonl" ]]; then - download "https://kaikki.org/dictionary/English/kaikki.org-dictionary-English.jsonl.gz" "$DATA_DIR/kaikki-en.jsonl.gz" + KAIKKI_EN_GZ="$DATA_DIR/kaikki-en.jsonl.gz" + download "https://kaikki.org/dictionary/English/kaikki.org-dictionary-English.jsonl.gz" "$KAIKKI_EN_GZ" echo " [decompress] kaikki-en.jsonl.gz" - gunzip -f "$DATA_DIR/kaikki-en.jsonl.gz" + gunzip -kf "$KAIKKI_EN_GZ" + save_checksum "kaikki-en" "$KAIKKI_EN_GZ" fi # ============================================================ @@ -264,9 +266,11 @@ if [[ -f "$DATA_DIR/kaikki-fr.jsonl" && "$FORCE" != "true" ]]; then fi fi if [[ ! -f "$DATA_DIR/kaikki-fr.jsonl" ]]; then - download "https://kaikki.org/dictionary/French/kaikki.org-dictionary-French.jsonl.gz" "$DATA_DIR/kaikki-fr.jsonl.gz" + KAIKKI_FR_GZ="$DATA_DIR/kaikki-fr.jsonl.gz" + download "https://kaikki.org/dictionary/French/kaikki.org-dictionary-French.jsonl.gz" "$KAIKKI_FR_GZ" echo " [decompress] kaikki-fr.jsonl.gz" - gunzip -f "$DATA_DIR/kaikki-fr.jsonl.gz" + gunzip -kf "$KAIKKI_FR_GZ" + save_checksum "kaikki-fr" "$KAIKKI_FR_GZ" fi # ============================================================ @@ -322,9 +326,52 @@ if [[ -f "$DATA_DIR/kaikki-pt.jsonl" && "$FORCE" != "true" ]]; then fi fi if [[ ! -f "$DATA_DIR/kaikki-pt.jsonl" ]]; then - download "https://kaikki.org/dictionary/Portuguese/kaikki.org-dictionary-Portuguese.jsonl.gz" "$DATA_DIR/kaikki-pt.jsonl.gz" + KAIKKI_PT_GZ="$DATA_DIR/kaikki-pt.jsonl.gz" + download "https://kaikki.org/dictionary/Portuguese/kaikki.org-dictionary-Portuguese.jsonl.gz" "$KAIKKI_PT_GZ" echo " [decompress] kaikki-pt.jsonl.gz" - gunzip -f "$DATA_DIR/kaikki-pt.jsonl.gz" + gunzip -kf "$KAIKKI_PT_GZ" + save_checksum "kaikki-pt" "$KAIKKI_PT_GZ" +fi + +# ============================================================ +# Mandarin — CC-CEDICT +# ============================================================ +echo "=== CC-CEDICT (Mandarin word list + definitions) ===" +if [[ -f "$DATA_DIR/cedict_ts.u8" && "$FORCE" != "true" ]]; then + # Expect at least 3 MB + if check_min_size "$DATA_DIR/cedict_ts.u8" 3000000; then + echo " [skip] cedict_ts.u8 already exists (size ok)" + else + echo " [warn] cedict_ts.u8 looks truncated, re-downloading" + rm -f "$DATA_DIR/cedict_ts.u8" + fi +fi +if [[ ! -f "$DATA_DIR/cedict_ts.u8" ]]; then + CEDICT_GZ="$DATA_DIR/cedict_ts.u8.gz" + download "https://www.mdbg.net/chinese/export/cedict/cedict_1_0_ts_utf-8_mdbg.txt.gz" "$CEDICT_GZ" + echo " [decompress] cedict_ts.u8.gz" + gunzip -kf "$CEDICT_GZ" + save_checksum "cedict" "$CEDICT_GZ" +fi + +# ============================================================ +# Mandarin — Wiktionary (kaikki) +# ============================================================ +echo "=== Wiktionary Chinese (compressed) ===" +if [[ -f "$DATA_DIR/kaikki-zh.jsonl" && "$FORCE" != "true" ]]; then + if check_min_size "$DATA_DIR/kaikki-zh.jsonl" 5000000; then + echo " [skip] kaikki-zh.jsonl already exists (size ok)" + else + echo " [warn] kaikki-zh.jsonl looks truncated, re-downloading" + rm -f "$DATA_DIR/kaikki-zh.jsonl" + fi +fi +if [[ ! -f "$DATA_DIR/kaikki-zh.jsonl" ]]; then + KAIKKI_ZH_GZ="$DATA_DIR/kaikki-zh.jsonl.gz" + download "https://kaikki.org/dictionary/Chinese/kaikki.org-dictionary-Chinese.jsonl.gz" "$KAIKKI_ZH_GZ" + echo " [decompress] kaikki-zh.jsonl.gz" + gunzip -kf "$KAIKKI_ZH_GZ" + save_checksum "kaikki-zh" "$KAIKKI_ZH_GZ" fi echo ""