diff --git a/internal/dictionary/db.go b/internal/dictionary/db.go index ff5da72..dbb5c98 100644 --- a/internal/dictionary/db.go +++ b/internal/dictionary/db.go @@ -58,6 +58,7 @@ CREATE TABLE IF NOT EXISTS translations ( ); CREATE INDEX IF NOT EXISTS idx_translations_word_id ON translations(word_id); +CREATE INDEX IF NOT EXISTS idx_translations_reverse ON translations(translation, target_lang); CREATE TABLE IF NOT EXISTS meta ( key TEXT PRIMARY KEY, diff --git a/internal/dictionary/query.go b/internal/dictionary/query.go index 3ad4fe2..1d96f33 100644 --- a/internal/dictionary/query.go +++ b/internal/dictionary/query.go @@ -128,12 +128,25 @@ func (d *Dictionary) Synonyms(word, lang string) ([]string, error) { } func (d *Dictionary) Translate(word, fromLang, toLang string) ([]string, error) { + // Forward: find translations stored for this word in fromLang → toLang + // Reverse: find words in toLang that have this word as a translation to fromLang + // This makes translations bidirectional: if English kaikki has "house" → "casa" (pt-PT), + // querying "casa" from pt-PT to en will find "house" via the reverse lookup. rows, err := d.db.Query(` - SELECT DISTINCT t.translation - FROM translations t - JOIN words w ON w.id = t.word_id - WHERE w.word = ? AND w.lang = ? AND t.target_lang = ? - ORDER BY t.translation`, + SELECT DISTINCT result FROM ( + SELECT t.translation AS result + FROM translations t + JOIN words w ON w.id = t.word_id + WHERE w.word = ? AND w.lang = ? AND t.target_lang = ? + UNION + SELECT w.word AS result + FROM translations t + JOIN words w ON w.id = t.word_id + WHERE t.translation = ? AND t.target_lang = ? AND w.lang = ? + ) ORDER BY result`, + // Forward: word=word, lang=fromLang, target_lang=toLang + strings.ToLower(word), fromLang, toLang, + // Reverse: translation=word, target_lang=fromLang (stored target matches our source), lang=toLang strings.ToLower(word), fromLang, toLang, ) if err != nil { diff --git a/internal/loader/affix.go b/internal/loader/affix.go index b3149ae..2c44b2a 100644 --- a/internal/loader/affix.go +++ b/internal/loader/affix.go @@ -205,9 +205,9 @@ func parseDicFile(path string) ([]dicEntry, error) { } // maxFormsPerWord caps the number of inflected forms generated per base word. -// Portuguese and French .aff files can produce hundreds of cross-product forms -// per word; most are valid but the combinatorial explosion bloats the DB. -const maxFormsPerWord = 30 +// Portuguese verbs can have 60+ valid conjugations (gerunds, subjunctives, etc.), +// so the cap must be generous enough to avoid cutting off common forms. +const maxFormsPerWord = 80 func expandWord(word, flags string, rules map[string]*affixRuleSet) []string { seen := map[string]bool{word: true} // base form already in DB diff --git a/internal/loader/cetempublico.go b/internal/loader/cetempublico.go index a97f43e..82b85d6 100644 --- a/internal/loader/cetempublico.go +++ b/internal/loader/cetempublico.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strconv" "strings" + "unicode/utf8" ) // CETEMPublicoLoader loads frequency data for European Portuguese words. @@ -24,6 +25,8 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { filepath.Join(dataDir, "cetempublico-freq.tsv"), filepath.Join(dataDir, "pt-freq.tsv"), filepath.Join(dataDir, "pt_PT-freq.tsv"), + filepath.Join(dataDir, "pt_50k.txt"), // hermitdave/FrequencyWords + filepath.Join(dataDir, "pt_full.txt"), // hermitdave/FrequencyWords } var path string @@ -37,12 +40,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { slog.Warn("cetempublico: no data file found, skipping", "searched", candidates) return nil } - - f, err := os.Open(path) - if err != nil { - return fmt.Errorf("cetempublico: open: %w", err) - } - defer f.Close() + slog.Info("cetempublico: using file", "path", path) tx, err := db.Begin() if err != nil { @@ -50,43 +48,78 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { } defer tx.Rollback() - stmt, err := tx.Prepare(` + stmtUpdate, err := tx.Prepare(` UPDATE words SET frequency = ? WHERE word = ? AND lang = 'pt-PT' AND frequency = 0`) if err != nil { - return fmt.Errorf("cetempublico: prepare: %w", err) + return fmt.Errorf("cetempublico: prepare update: %w", err) } - defer stmt.Close() + defer stmtUpdate.Close() - scanner := bufio.NewScanner(f) - // Skip header if present + stmtInsert, err := tx.Prepare(` + INSERT OR IGNORE INTO words (word, lang, frequency) VALUES (?, 'pt-PT', ?)`) + if err != nil { + return fmt.Errorf("cetempublico: prepare insert: %w", err) + } + defer stmtInsert.Close() + + // Read entire file to detect encoding before parsing. + // The file is typically <20 MB so this is fine. + rawBytes, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("cetempublico: read file: %w", err) + } + + isLatin1 := !utf8.Valid(rawBytes) + if isLatin1 { + slog.Info("cetempublico: detected ISO-8859-1 encoding, converting") + rawBytes = []byte(latin1ToUTF8(rawBytes)) + } + + scanner := bufio.NewScanner(strings.NewReader(string(rawBytes))) + + // Peek first line to detect header vs data + var firstLine string if scanner.Scan() { - first := scanner.Text() - fields := strings.Split(first, "\t") - // If first line looks like data (second field is numeric), process it + firstLine = scanner.Text() + fields := strings.Fields(firstLine) if len(fields) >= 2 { - if _, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64); err != nil { - // It's a header, skip it - } else { - // It's data, we need to process it — but we already consumed it - // Re-process below + if _, err := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64); err != nil { + firstLine = "" // It's a header, skip it } } } - var count int + var count, skipped, inserted int processLine := func(line string) error { + // Support TSV and space-separated, with either column order: + // "word\tcount", "word count", or "count\tword", "count word" fields := strings.Split(line, "\t") + if len(fields) < 2 { + fields = strings.Fields(line) + } if len(fields) < 2 { return nil } - word := strings.ToLower(strings.TrimSpace(fields[0])) + // Detect column order: if first field is numeric, it's count-first + var word string + var freqStr string + if _, err := strconv.ParseFloat(strings.TrimSpace(fields[0]), 64); err == nil { + freqStr = strings.TrimSpace(fields[0]) + word = strings.ToLower(strings.TrimSpace(fields[1])) + } else { + word = strings.ToLower(strings.TrimSpace(fields[0])) + freqStr = strings.TrimSpace(fields[1]) + } + if word == "" || containsDigit(word) || containsSpace(word) || containsNonLatin(word) { + skipped++ return nil } - freqVal, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64) + freqVal, err := strconv.ParseFloat(freqStr, 64) if err != nil || freqVal <= 0 { + skipped++ return nil } @@ -106,16 +139,33 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { freq = 1 } - res, err := stmt.Exec(freq, word) + res, err := stmtUpdate.Exec(freq, word) if err != nil { return fmt.Errorf("cetempublico: update: %w", err) } if n, _ := res.RowsAffected(); n > 0 { count++ + } else { + // Word not in DB yet — insert it with frequency + if _, err := stmtInsert.Exec(word, freq); err != nil { + return fmt.Errorf("cetempublico: insert: %w", err) + } + inserted++ + count++ + } + if count+skipped <= 5 { + slog.Info("cetempublico: sample", "word", word, "freq", freq, "total", count, "inserted", inserted) } return nil } + // Process first line if it was data (not a header) + if firstLine != "" { + if err := processLine(firstLine); err != nil { + return err + } + } + for scanner.Scan() { if err := processLine(scanner.Text()); err != nil { return err @@ -129,6 +179,17 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { if err := tx.Commit(); err != nil { return fmt.Errorf("cetempublico: commit: %w", err) } - slog.Info("cetempublico loaded", "updated_words", count) + slog.Info("cetempublico loaded", "updated_words", count, "inserted_new", inserted, "skipped", skipped) return nil } + +// latin1ToUTF8 converts ISO-8859-1 bytes to a UTF-8 string. +// Each byte in ISO-8859-1 maps directly to the same Unicode code point. +func latin1ToUTF8(b []byte) string { + var buf strings.Builder + buf.Grow(len(b) + len(b)/4) // accented chars expand + for _, c := range b { + buf.WriteRune(rune(c)) + } + return buf.String() +} diff --git a/internal/loader/wiktionary.go b/internal/loader/wiktionary.go index 8bfcd62..8577b40 100644 --- a/internal/loader/wiktionary.go +++ b/internal/loader/wiktionary.go @@ -27,9 +27,15 @@ type wiktEntry struct { Word string `json:"word"` POS string `json:"pos"` LangCode string `json:"lang_code"` - Senses []struct { - Glosses []string `json:"glosses"` - Tags []string `json:"tags"` + Senses []struct { + Glosses []string `json:"glosses"` + Tags []string `json:"tags"` + Synonyms []struct { + Word string `json:"word"` + } `json:"synonyms"` + Antonyms []struct { + Word string `json:"word"` + } `json:"antonyms"` } `json:"senses"` Synonyms []struct { Word string `json:"word"` @@ -188,7 +194,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error { pos := wiktPOSMap[entry.POS] - // Process senses + // Process senses (definitions + sense-level synonyms/antonyms) for _, sense := range entry.Senses { if hasTag(sense.Tags, "form-of", "alt-of") { continue @@ -208,9 +214,33 @@ func loadWiktionary(db *sql.DB, path, lang string) error { } defCount++ } + + // Sense-level synonyms + for _, syn := range sense.Synonyms { + synWord := strings.ToLower(syn.Word) + if synWord == "" || containsDigit(synWord) || containsSpace(synWord) { + continue + } + if _, err := batch.stmtSyn.Exec(synWord, word, lang); err != nil { + return fmt.Errorf("wiktionary: insert syn: %w", err) + } + synCount++ + } + + // Sense-level antonyms + for _, ant := range sense.Antonyms { + antWord := strings.ToLower(ant.Word) + if antWord == "" || containsDigit(antWord) || containsSpace(antWord) { + continue + } + if _, err := batch.stmtAnt.Exec(antWord, word, lang); err != nil { + return fmt.Errorf("wiktionary: insert ant: %w", err) + } + antCount++ + } } - // Process synonyms + // Process top-level synonyms for _, syn := range entry.Synonyms { synWord := strings.ToLower(syn.Word) if synWord == "" || containsDigit(synWord) || containsSpace(synWord) { @@ -222,7 +252,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error { synCount++ } - // Process antonyms + // Process top-level antonyms for _, ant := range entry.Antonyms { antWord := strings.ToLower(ant.Word) if antWord == "" || containsDigit(antWord) || containsSpace(antWord) { diff --git a/scripts/download-dict-data.sh b/scripts/download-dict-data.sh index b1e01f2..81b49c2 100755 --- a/scripts/download-dict-data.sh +++ b/scripts/download-dict-data.sh @@ -412,17 +412,21 @@ fi # pt-PT — CETEMPúblico frequency (or fallback frequency list) # ============================================================ echo "=== Portuguese frequency data ===" -PT_FREQ_FILE="$DATA_DIR/cetempublico-freq.tsv" +PT_FREQ_FILE="$DATA_DIR/pt_50k.txt" if [[ -f "$PT_FREQ_FILE" && "$FORCE" != "true" ]]; then - echo " [skip] cetempublico-freq.tsv already exists" -elif [[ -f "$DATA_DIR/pt-freq.tsv" && "$FORCE" != "true" ]]; then - echo " [skip] pt-freq.tsv already exists" -else - echo " [info] CETEMPúblico frequency data requires manual download from Linguateca." - echo " Visit: https://www.linguateca.pt/acesso/corpus.php?corpus=CETEMPUBLICO" - echo " Generate a word frequency list and save as: $PT_FREQ_FILE" - echo " Format: wordfrequency (one per line)" - echo " The import will proceed without frequency data if this file is absent." + if check_min_size "$PT_FREQ_FILE" 100000; then + echo " [skip] pt_50k.txt already exists (size ok)" + else + echo " [warn] pt_50k.txt looks truncated, re-downloading" + rm -f "$PT_FREQ_FILE" + fi +fi +if [[ ! -f "$PT_FREQ_FILE" ]]; then + # OpenSubtitles-derived frequency list (hermitdave/FrequencyWords) + download "https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2016/pt/pt_50k.txt" "$PT_FREQ_FILE" 2>/dev/null || { + echo " [warn] Portuguese frequency download failed." + echo " The import will proceed without frequency data." + } fi # ============================================================