Fix Portuguese coverage gaps: synonyms, translations, frequency, gerunds

- Extract sense-level synonyms/antonyms from Wiktionary (not just top-level),
  dramatically improving Portuguese synonym coverage
- Add bidirectional translation lookup so pt→en queries find en→pt entries
  stored by English Wiktionary loader, with reverse index for performance
- Fix frequency loader: auto-detect column order (count\tword vs word\tcount),
  auto-detect ISO-8859-1 encoding and convert to UTF-8, insert missing words
  instead of only updating existing ones
- Switch WordNet download from 3.1 to 3.0 to match WOLF/OMW synset offsets
- Add OpenSubtitles-derived Portuguese frequency list (hermitdave/FrequencyWords)
- Increase affix expansion cap from 30→80 forms per word for Portuguese gerunds

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

internal/dictionary/db.go

@@ -58,6 +58,7 @@ CREATE TABLE IF NOT EXISTS translations (
 );
 
 CREATE INDEX IF NOT EXISTS idx_translations_word_id ON translations(word_id);
+CREATE INDEX IF NOT EXISTS idx_translations_reverse ON translations(translation, target_lang);
 
 CREATE TABLE IF NOT EXISTS meta (
     key   TEXT PRIMARY KEY,

internal/dictionary/query.go

@@ -128,12 +128,25 @@ func (d *Dictionary) Synonyms(word, lang string) ([]string, error) {
 }
 
 func (d *Dictionary) Translate(word, fromLang, toLang string) ([]string, error) {
+	// Forward: find translations stored for this word in fromLang → toLang
+	// Reverse: find words in toLang that have this word as a translation to fromLang
+	// This makes translations bidirectional: if English kaikki has "house" → "casa" (pt-PT),
+	// querying "casa" from pt-PT to en will find "house" via the reverse lookup.
 	rows, err := d.db.Query(`
-		SELECT DISTINCT t.translation
-		FROM translations t
-		JOIN words w ON w.id = t.word_id
-		WHERE w.word = ? AND w.lang = ? AND t.target_lang = ?
-		ORDER BY t.translation`,
+		SELECT DISTINCT result FROM (
+			SELECT t.translation AS result
+			FROM translations t
+			JOIN words w ON w.id = t.word_id
+			WHERE w.word = ? AND w.lang = ? AND t.target_lang = ?
+			UNION
+			SELECT w.word AS result
+			FROM translations t
+			JOIN words w ON w.id = t.word_id
+			WHERE t.translation = ? AND t.target_lang = ? AND w.lang = ?
+		) ORDER BY result`,
+		// Forward:  word=word, lang=fromLang, target_lang=toLang
+		strings.ToLower(word), fromLang, toLang,
+		// Reverse:  translation=word, target_lang=fromLang (stored target matches our source), lang=toLang
 		strings.ToLower(word), fromLang, toLang,
 	)
 	if err != nil {

internal/loader/affix.go

@@ -205,9 +205,9 @@ func parseDicFile(path string) ([]dicEntry, error) {
 }
 
 // maxFormsPerWord caps the number of inflected forms generated per base word.
-// Portuguese and French .aff files can produce hundreds of cross-product forms
-// per word; most are valid but the combinatorial explosion bloats the DB.
-const maxFormsPerWord = 30
+// Portuguese verbs can have 60+ valid conjugations (gerunds, subjunctives, etc.),
+// so the cap must be generous enough to avoid cutting off common forms.
+const maxFormsPerWord = 80
 
 func expandWord(word, flags string, rules map[string]*affixRuleSet) []string {
 	seen := map[string]bool{word: true} // base form already in DB

internal/loader/cetempublico.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"unicode/utf8"
 )
 
 // CETEMPublicoLoader loads frequency data for European Portuguese words.
@@ -24,6 +25,8 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 		filepath.Join(dataDir, "cetempublico-freq.tsv"),
 		filepath.Join(dataDir, "pt-freq.tsv"),
 		filepath.Join(dataDir, "pt_PT-freq.tsv"),
+		filepath.Join(dataDir, "pt_50k.txt"),      // hermitdave/FrequencyWords
+		filepath.Join(dataDir, "pt_full.txt"),      // hermitdave/FrequencyWords
 	}
 
 	var path string
@@ -37,12 +40,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 		slog.Warn("cetempublico: no data file found, skipping", "searched", candidates)
 		return nil
 	}
-
-	f, err := os.Open(path)
-	if err != nil {
-		return fmt.Errorf("cetempublico: open: %w", err)
-	}
-	defer f.Close()
+	slog.Info("cetempublico: using file", "path", path)
 
 	tx, err := db.Begin()
 	if err != nil {
@@ -50,43 +48,78 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 	}
 	defer tx.Rollback()
 
-	stmt, err := tx.Prepare(`
+	stmtUpdate, err := tx.Prepare(`
 		UPDATE words SET frequency = ? WHERE word = ? AND lang = 'pt-PT' AND frequency = 0`)
 	if err != nil {
-		return fmt.Errorf("cetempublico: prepare: %w", err)
+		return fmt.Errorf("cetempublico: prepare update: %w", err)
 	}
-	defer stmt.Close()
+	defer stmtUpdate.Close()
 
-	scanner := bufio.NewScanner(f)
-	// Skip header if present
+	stmtInsert, err := tx.Prepare(`
+		INSERT OR IGNORE INTO words (word, lang, frequency) VALUES (?, 'pt-PT', ?)`)
+	if err != nil {
+		return fmt.Errorf("cetempublico: prepare insert: %w", err)
+	}
+	defer stmtInsert.Close()
+
+	// Read entire file to detect encoding before parsing.
+	// The file is typically <20 MB so this is fine.
+	rawBytes, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("cetempublico: read file: %w", err)
+	}
+
+	isLatin1 := !utf8.Valid(rawBytes)
+	if isLatin1 {
+		slog.Info("cetempublico: detected ISO-8859-1 encoding, converting")
+		rawBytes = []byte(latin1ToUTF8(rawBytes))
+	}
+
+	scanner := bufio.NewScanner(strings.NewReader(string(rawBytes)))
+
+	// Peek first line to detect header vs data
+	var firstLine string
 	if scanner.Scan() {
-		first := scanner.Text()
-		fields := strings.Split(first, "\t")
-		// If first line looks like data (second field is numeric), process it
+		firstLine = scanner.Text()
+		fields := strings.Fields(firstLine)
 		if len(fields) >= 2 {
-			if _, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64); err != nil {
-				// It's a header, skip it
-			} else {
-				// It's data, we need to process it — but we already consumed it
-				// Re-process below
+			if _, err := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64); err != nil {
+				firstLine = "" // It's a header, skip it
 			}
 		}
 	}
 
-	var count int
+	var count, skipped, inserted int
 	processLine := func(line string) error {
+		// Support TSV and space-separated, with either column order:
+		//   "word\tcount", "word count", or "count\tword", "count word"
 		fields := strings.Split(line, "\t")
+		if len(fields) < 2 {
+			fields = strings.Fields(line)
+		}
 		if len(fields) < 2 {
 			return nil
 		}
 
-		word := strings.ToLower(strings.TrimSpace(fields[0]))
+		// Detect column order: if first field is numeric, it's count-first
+		var word string
+		var freqStr string
+		if _, err := strconv.ParseFloat(strings.TrimSpace(fields[0]), 64); err == nil {
+			freqStr = strings.TrimSpace(fields[0])
+			word = strings.ToLower(strings.TrimSpace(fields[1]))
+		} else {
+			word = strings.ToLower(strings.TrimSpace(fields[0]))
+			freqStr = strings.TrimSpace(fields[1])
+		}
+
 		if word == "" || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
+			skipped++
 			return nil
 		}
 
-		freqVal, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64)
+		freqVal, err := strconv.ParseFloat(freqStr, 64)
 		if err != nil || freqVal <= 0 {
+			skipped++
 			return nil
 		}
 
@@ -106,16 +139,33 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 			freq = 1
 		}
 
-		res, err := stmt.Exec(freq, word)
+		res, err := stmtUpdate.Exec(freq, word)
 		if err != nil {
 			return fmt.Errorf("cetempublico: update: %w", err)
 		}
 		if n, _ := res.RowsAffected(); n > 0 {
 			count++
+		} else {
+			// Word not in DB yet — insert it with frequency
+			if _, err := stmtInsert.Exec(word, freq); err != nil {
+				return fmt.Errorf("cetempublico: insert: %w", err)
+			}
+			inserted++
+			count++
+		}
+		if count+skipped <= 5 {
+			slog.Info("cetempublico: sample", "word", word, "freq", freq, "total", count, "inserted", inserted)
 		}
 		return nil
 	}
 
+	// Process first line if it was data (not a header)
+	if firstLine != "" {
+		if err := processLine(firstLine); err != nil {
+			return err
+		}
+	}
+
 	for scanner.Scan() {
 		if err := processLine(scanner.Text()); err != nil {
 			return err
@@ -129,6 +179,17 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 	if err := tx.Commit(); err != nil {
 		return fmt.Errorf("cetempublico: commit: %w", err)
 	}
-	slog.Info("cetempublico loaded", "updated_words", count)
+	slog.Info("cetempublico loaded", "updated_words", count, "inserted_new", inserted, "skipped", skipped)
 	return nil
 }
+
+// latin1ToUTF8 converts ISO-8859-1 bytes to a UTF-8 string.
+// Each byte in ISO-8859-1 maps directly to the same Unicode code point.
+func latin1ToUTF8(b []byte) string {
+	var buf strings.Builder
+	buf.Grow(len(b) + len(b)/4) // accented chars expand
+	for _, c := range b {
+		buf.WriteRune(rune(c))
+	}
+	return buf.String()
+}

internal/loader/wiktionary.go

@@ -27,9 +27,15 @@ type wiktEntry struct {
 	Word     string `json:"word"`
 	POS      string `json:"pos"`
 	LangCode string `json:"lang_code"`
-	Senses   []struct {
-		Glosses []string `json:"glosses"`
-		Tags    []string `json:"tags"`
+	Senses []struct {
+		Glosses  []string `json:"glosses"`
+		Tags     []string `json:"tags"`
+		Synonyms []struct {
+			Word string `json:"word"`
+		} `json:"synonyms"`
+		Antonyms []struct {
+			Word string `json:"word"`
+		} `json:"antonyms"`
 	} `json:"senses"`
 	Synonyms []struct {
 		Word string `json:"word"`
@@ -188,7 +194,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
 
 		pos := wiktPOSMap[entry.POS]
 
-		// Process senses
+		// Process senses (definitions + sense-level synonyms/antonyms)
 		for _, sense := range entry.Senses {
 			if hasTag(sense.Tags, "form-of", "alt-of") {
 				continue
@@ -208,9 +214,33 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
 				}
 				defCount++
 			}
+
+			// Sense-level synonyms
+			for _, syn := range sense.Synonyms {
+				synWord := strings.ToLower(syn.Word)
+				if synWord == "" || containsDigit(synWord) || containsSpace(synWord) {
+					continue
+				}
+				if _, err := batch.stmtSyn.Exec(synWord, word, lang); err != nil {
+					return fmt.Errorf("wiktionary: insert syn: %w", err)
+				}
+				synCount++
+			}
+
+			// Sense-level antonyms
+			for _, ant := range sense.Antonyms {
+				antWord := strings.ToLower(ant.Word)
+				if antWord == "" || containsDigit(antWord) || containsSpace(antWord) {
+					continue
+				}
+				if _, err := batch.stmtAnt.Exec(antWord, word, lang); err != nil {
+					return fmt.Errorf("wiktionary: insert ant: %w", err)
+				}
+				antCount++
+			}
 		}
 
-		// Process synonyms
+		// Process top-level synonyms
 		for _, syn := range entry.Synonyms {
 			synWord := strings.ToLower(syn.Word)
 			if synWord == "" || containsDigit(synWord) || containsSpace(synWord) {
@@ -222,7 +252,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
 			synCount++
 		}
 
-		// Process antonyms
+		// Process top-level antonyms
 		for _, ant := range entry.Antonyms {
 			antWord := strings.ToLower(ant.Word)
 			if antWord == "" || containsDigit(antWord) || containsSpace(antWord) {

scripts/download-dict-data.sh

@@ -412,17 +412,21 @@ fi
 # pt-PT — CETEMPúblico frequency (or fallback frequency list)
 # ============================================================
 echo "=== Portuguese frequency data ==="
-PT_FREQ_FILE="$DATA_DIR/cetempublico-freq.tsv"
+PT_FREQ_FILE="$DATA_DIR/pt_50k.txt"
 if [[ -f "$PT_FREQ_FILE" && "$FORCE" != "true" ]]; then
-    echo "  [skip] cetempublico-freq.tsv already exists"
-elif [[ -f "$DATA_DIR/pt-freq.tsv" && "$FORCE" != "true" ]]; then
-    echo "  [skip] pt-freq.tsv already exists"
-else
-    echo "  [info] CETEMPúblico frequency data requires manual download from Linguateca."
-    echo "  Visit: https://www.linguateca.pt/acesso/corpus.php?corpus=CETEMPUBLICO"
-    echo "  Generate a word frequency list and save as: $PT_FREQ_FILE"
-    echo "  Format: word<TAB>frequency (one per line)"
-    echo "  The import will proceed without frequency data if this file is absent."
+    if check_min_size "$PT_FREQ_FILE" 100000; then
+        echo "  [skip] pt_50k.txt already exists (size ok)"
+    else
+        echo "  [warn] pt_50k.txt looks truncated, re-downloading"
+        rm -f "$PT_FREQ_FILE"
+    fi
+fi
+if [[ ! -f "$PT_FREQ_FILE" ]]; then
+    # OpenSubtitles-derived frequency list (hermitdave/FrequencyWords)
+    download "https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2016/pt/pt_50k.txt" "$PT_FREQ_FILE" 2>/dev/null || {
+        echo "  [warn] Portuguese frequency download failed."
+        echo "  The import will proceed without frequency data."
+    }
 fi
 
 # ============================================================