Add Mandarin (zh) language support

Add CC-CEDICT loader and Wiktionary Chinese as data sources for
Mandarin. Also fix all Wiktionary download sections to retain
compressed archives and save checksums for integrity verification.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

cmd/dictimport/main.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"database/sql"
+	"flag"
+	"fmt"
+	"log/slog"
+	"os"
+	"strings"
+	"time"
+
+	"dreamdict/internal/dictionary"
+	"dreamdict/internal/loader"
+
+	_ "modernc.org/sqlite"
+)
+
+func main() {
+	dbPath := flag.String("db", "./dict.db", "Path to output dict.db")
+	dataDir := flag.String("data", "./data", "Path to data directory")
+	langs := flag.String("langs", "en,fr,pt-PT,zh", "Comma-separated languages to import")
+	clean := flag.Bool("clean", false, "Drop and recreate all tables before import")
+	skipFlag := flag.String("skip", "", "Comma-separated loader names to skip")
+	flag.Parse()
+
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})))
+
+	db, err := sql.Open("sqlite", *dbPath+"?_pragma=journal_mode(wal)&_pragma=foreign_keys(on)")
+	if err != nil {
+		slog.Error("open db", "error", err)
+		os.Exit(1)
+	}
+	defer db.Close()
+	db.SetMaxOpenConns(1)
+
+	if *clean {
+		slog.Info("dropping all tables")
+		for _, table := range []string{"translations", "synonyms", "definitions", "words", "meta"} {
+			if _, err := db.Exec("DROP TABLE IF EXISTS " + table); err != nil {
+				slog.Error("drop table", "table", table, "error", err)
+				os.Exit(1)
+			}
+		}
+	}
+
+	if err := dictionary.BootstrapSchema(db); err != nil {
+		slog.Error("bootstrap schema", "error", err)
+		os.Exit(1)
+	}
+
+	langSet := make(map[string]bool)
+	for _, l := range strings.Split(*langs, ",") {
+		langSet[strings.TrimSpace(l)] = true
+	}
+
+	skipSet := make(map[string]bool)
+	if *skipFlag != "" {
+		for _, s := range strings.Split(*skipFlag, ",") {
+			skipSet[strings.TrimSpace(s)] = true
+		}
+	}
+
+	var loaders []loader.Loader
+
+	if langSet["en"] {
+		loaders = append(loaders,
+			loader.SCOWLLoader{},
+			loader.WordNetLoader{},
+			loader.WiktionaryLoader{Lang: "en", FileName: "kaikki-en.jsonl"},
+		)
+	}
+
+	if langSet["fr"] {
+		loaders = append(loaders,
+			loader.HunspellLoader{Lang: "fr", FileName: "fr_FR/fr.dic"},
+			loader.LexiqueLoader{},
+			loader.WOLFLoader{},
+			loader.WiktionaryLoader{Lang: "fr", FileName: "kaikki-fr.jsonl"},
+		)
+	}
+
+	if langSet["pt-PT"] {
+		loaders = append(loaders,
+			loader.HunspellLoader{Lang: "pt-PT", FileName: "pt_PT/pt_PT.dic"},
+			loader.DicionarioLoader{},
+			loader.WiktionaryLoader{Lang: "pt-PT", FileName: "kaikki-pt.jsonl"},
+		)
+	}
+
+	if langSet["zh"] {
+		loaders = append(loaders,
+			loader.CEDICTLoader{},
+			loader.WiktionaryLoader{Lang: "zh", FileName: "kaikki-zh.jsonl"},
+		)
+	}
+
+	start := time.Now()
+	if err := loader.Run(db, *dataDir, loaders, skipSet); err != nil {
+		slog.Error("import failed", "error", err)
+		os.Exit(1)
+	}
+
+	// Write meta
+	now := time.Now().UTC().Format(time.RFC3339)
+	for _, kv := range [][2]string{
+		{"imported_at", now},
+		{"schema_version", "1"},
+	} {
+		if _, err := db.Exec(
+			"INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", kv[0], kv[1],
+		); err != nil {
+			slog.Error("write meta", "error", err)
+			os.Exit(1)
+		}
+	}
+
+	// Print summary
+	dict := dictionary.NewFromDB(db)
+	wordCounts, _ := dict.WordCount()
+	defCounts, _ := dict.DefCount()
+
+	fmt.Println("\n=== Import Summary ===")
+	fmt.Printf("Elapsed: %s\n", time.Since(start).Round(time.Millisecond))
+	fmt.Println("Word counts:")
+	for _, lang := range []string{"en", "fr", "pt-PT", "zh"} {
+		if c, ok := wordCounts[lang]; ok {
+			fmt.Printf("  %s: %d\n", lang, c)
+		}
+	}
+	fmt.Println("Definition counts:")
+	for _, lang := range []string{"en", "fr", "pt-PT", "zh"} {
+		if c, ok := defCounts[lang]; ok {
+			fmt.Printf("  %s: %d\n", lang, c)
+		}
+	}
+}

internal/dictionary/dict.go

@@ -10,7 +10,7 @@ var (
 	ErrNotSeeded = errors.New("dictionary: database not seeded, run: go run ./cmd/dictimport")
 )
 
-var supportedLangs = []string{"en", "fr", "pt-PT"}
+var supportedLangs = []string{"en", "fr", "pt-PT", "zh"}
 
 func ValidLang(lang string) bool {
 	for _, l := range supportedLangs {

internal/loader/cedict.go

@@ -0,0 +1,132 @@
+package loader
+
+import (
+	"bufio"
+	"database/sql"
+	"fmt"
+	"log/slog"
+	"os"
+	"strings"
+)
+
+// CEDICTLoader imports words and definitions from the CC-CEDICT dictionary.
+// CC-CEDICT line format: Traditional Simplified [pin1 yin1] /def1/def2/
+type CEDICTLoader struct{}
+
+func (CEDICTLoader) Name() string { return "cedict" }
+
+func (CEDICTLoader) Load(db *sql.DB, dataDir string) error {
+	path := dataDir + "/cedict_ts.u8"
+	return loadCEDICT(db, path)
+}
+
+var cedictPOSPrefixes = map[string]string{
+	"to ":  "verb",
+	"(tw)": "",
+}
+
+func loadCEDICT(db *sql.DB, path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("cedict: open: %w", err)
+	}
+	defer f.Close()
+
+	tx, err := db.Begin()
+	if err != nil {
+		return fmt.Errorf("cedict: begin tx: %w", err)
+	}
+	defer tx.Rollback()
+
+	stmtWord, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang) VALUES (?, 'zh')")
+	if err != nil {
+		return fmt.Errorf("cedict: prepare word: %w", err)
+	}
+	defer stmtWord.Close()
+
+	stmtDef, err := tx.Prepare(`INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority)
+		SELECT id, ?, ?, 'cedict', 15 FROM words WHERE word = ? AND lang = 'zh'`)
+	if err != nil {
+		return fmt.Errorf("cedict: prepare def: %w", err)
+	}
+	defer stmtDef.Close()
+
+	stmtTrans, err := tx.Prepare(`INSERT OR IGNORE INTO translations (word_id, translation, target_lang, source)
+		SELECT id, ?, 'en', 'cedict' FROM words WHERE word = ? AND lang = 'zh'`)
+	if err != nil {
+		return fmt.Errorf("cedict: prepare trans: %w", err)
+	}
+	defer stmtTrans.Close()
+
+	scanner := bufio.NewScanner(f)
+	var wordCount, defCount, transCount int
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "#") || line == "" {
+			continue
+		}
+
+		// Format: Traditional Simplified [pinyin] /def1/def2/
+		bracketStart := strings.Index(line, "[")
+		slashStart := strings.Index(line, "/")
+		if bracketStart < 0 || slashStart < 0 {
+			continue
+		}
+
+		parts := strings.SplitN(line[:bracketStart], " ", 3)
+		if len(parts) < 2 {
+			continue
+		}
+		simplified := parts[1]
+
+		if simplified == "" || containsDigit(simplified) {
+			continue
+		}
+
+		// Insert the simplified form as the word
+		if _, err := stmtWord.Exec(simplified); err != nil {
+			return fmt.Errorf("cedict: insert word: %w", err)
+		}
+		wordCount++
+
+		// Parse definitions between slashes
+		defPart := strings.TrimRight(line[slashStart:], "/")
+		defs := strings.Split(defPart, "/")
+		for _, d := range defs {
+			d = strings.TrimSpace(d)
+			if d == "" || len(d) > 500 {
+				continue
+			}
+
+			// Guess POS from definition text
+			pos := ""
+			lower := strings.ToLower(d)
+			if strings.HasPrefix(lower, "to ") {
+				pos = "verb"
+			}
+
+			// Store as definition
+			if _, err := stmtDef.Exec(pos, d, simplified); err != nil {
+				return fmt.Errorf("cedict: insert def: %w", err)
+			}
+			defCount++
+
+			// English definitions also serve as translations
+			if _, err := stmtTrans.Exec(strings.ToLower(d), simplified); err != nil {
+				return fmt.Errorf("cedict: insert trans: %w", err)
+			}
+			transCount++
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("cedict: scan: %w", err)
+	}
+
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("cedict: commit: %w", err)
+	}
+
+	slog.Info("cedict loaded", "words", wordCount, "definitions", defCount, "translations", transCount)
+	return nil
+}

internal/loader/wiktionary.go

@@ -52,6 +52,7 @@ var wiktLangCodeMap = map[string]string{
 	"fr": "fr",
 	"pt": "pt-PT",
 	"en": "en",
+	"zh": "zh",
 }
 
 const (

scripts/download-dict-data.sh

@@ -184,9 +184,11 @@ if [[ -f "$DATA_DIR/kaikki-en.jsonl" && "$FORCE" != "true" ]]; then
     fi
 fi
 if [[ ! -f "$DATA_DIR/kaikki-en.jsonl" ]]; then
-    download "https://kaikki.org/dictionary/English/kaikki.org-dictionary-English.jsonl.gz" "$DATA_DIR/kaikki-en.jsonl.gz"
+    KAIKKI_EN_GZ="$DATA_DIR/kaikki-en.jsonl.gz"
+    download "https://kaikki.org/dictionary/English/kaikki.org-dictionary-English.jsonl.gz" "$KAIKKI_EN_GZ"
     echo "  [decompress] kaikki-en.jsonl.gz"
-    gunzip -f "$DATA_DIR/kaikki-en.jsonl.gz"
+    gunzip -kf "$KAIKKI_EN_GZ"
+    save_checksum "kaikki-en" "$KAIKKI_EN_GZ"
 fi
 
 # ============================================================
@@ -264,9 +266,11 @@ if [[ -f "$DATA_DIR/kaikki-fr.jsonl" && "$FORCE" != "true" ]]; then
     fi
 fi
 if [[ ! -f "$DATA_DIR/kaikki-fr.jsonl" ]]; then
-    download "https://kaikki.org/dictionary/French/kaikki.org-dictionary-French.jsonl.gz" "$DATA_DIR/kaikki-fr.jsonl.gz"
+    KAIKKI_FR_GZ="$DATA_DIR/kaikki-fr.jsonl.gz"
+    download "https://kaikki.org/dictionary/French/kaikki.org-dictionary-French.jsonl.gz" "$KAIKKI_FR_GZ"
     echo "  [decompress] kaikki-fr.jsonl.gz"
-    gunzip -f "$DATA_DIR/kaikki-fr.jsonl.gz"
+    gunzip -kf "$KAIKKI_FR_GZ"
+    save_checksum "kaikki-fr" "$KAIKKI_FR_GZ"
 fi
 
 # ============================================================
@@ -322,9 +326,52 @@ if [[ -f "$DATA_DIR/kaikki-pt.jsonl" && "$FORCE" != "true" ]]; then
     fi
 fi
 if [[ ! -f "$DATA_DIR/kaikki-pt.jsonl" ]]; then
-    download "https://kaikki.org/dictionary/Portuguese/kaikki.org-dictionary-Portuguese.jsonl.gz" "$DATA_DIR/kaikki-pt.jsonl.gz"
+    KAIKKI_PT_GZ="$DATA_DIR/kaikki-pt.jsonl.gz"
+    download "https://kaikki.org/dictionary/Portuguese/kaikki.org-dictionary-Portuguese.jsonl.gz" "$KAIKKI_PT_GZ"
     echo "  [decompress] kaikki-pt.jsonl.gz"
-    gunzip -f "$DATA_DIR/kaikki-pt.jsonl.gz"
+    gunzip -kf "$KAIKKI_PT_GZ"
+    save_checksum "kaikki-pt" "$KAIKKI_PT_GZ"
+fi
+
+# ============================================================
+# Mandarin — CC-CEDICT
+# ============================================================
+echo "=== CC-CEDICT (Mandarin word list + definitions) ==="
+if [[ -f "$DATA_DIR/cedict_ts.u8" && "$FORCE" != "true" ]]; then
+    # Expect at least 3 MB
+    if check_min_size "$DATA_DIR/cedict_ts.u8" 3000000; then
+        echo "  [skip] cedict_ts.u8 already exists (size ok)"
+    else
+        echo "  [warn] cedict_ts.u8 looks truncated, re-downloading"
+        rm -f "$DATA_DIR/cedict_ts.u8"
+    fi
+fi
+if [[ ! -f "$DATA_DIR/cedict_ts.u8" ]]; then
+    CEDICT_GZ="$DATA_DIR/cedict_ts.u8.gz"
+    download "https://www.mdbg.net/chinese/export/cedict/cedict_1_0_ts_utf-8_mdbg.txt.gz" "$CEDICT_GZ"
+    echo "  [decompress] cedict_ts.u8.gz"
+    gunzip -kf "$CEDICT_GZ"
+    save_checksum "cedict" "$CEDICT_GZ"
+fi
+
+# ============================================================
+# Mandarin — Wiktionary (kaikki)
+# ============================================================
+echo "=== Wiktionary Chinese (compressed) ==="
+if [[ -f "$DATA_DIR/kaikki-zh.jsonl" && "$FORCE" != "true" ]]; then
+    if check_min_size "$DATA_DIR/kaikki-zh.jsonl" 5000000; then
+        echo "  [skip] kaikki-zh.jsonl already exists (size ok)"
+    else
+        echo "  [warn] kaikki-zh.jsonl looks truncated, re-downloading"
+        rm -f "$DATA_DIR/kaikki-zh.jsonl"
+    fi
+fi
+if [[ ! -f "$DATA_DIR/kaikki-zh.jsonl" ]]; then
+    KAIKKI_ZH_GZ="$DATA_DIR/kaikki-zh.jsonl.gz"
+    download "https://kaikki.org/dictionary/Chinese/kaikki.org-dictionary-Chinese.jsonl.gz" "$KAIKKI_ZH_GZ"
+    echo "  [decompress] kaikki-zh.jsonl.gz"
+    gunzip -kf "$KAIKKI_ZH_GZ"
+    save_checksum "kaikki-zh" "$KAIKKI_ZH_GZ"
 fi
 
 echo ""