Fix Portuguese coverage gaps: synonyms, translations, frequency, gerunds

- Extract sense-level synonyms/antonyms from Wiktionary (not just top-level),
  dramatically improving Portuguese synonym coverage
- Add bidirectional translation lookup so pt→en queries find en→pt entries
  stored by English Wiktionary loader, with reverse index for performance
- Fix frequency loader: auto-detect column order (count\tword vs word\tcount),
  auto-detect ISO-8859-1 encoding and convert to UTF-8, insert missing words
  instead of only updating existing ones
- Switch WordNet download from 3.1 to 3.0 to match WOLF/OMW synset offsets
- Add OpenSubtitles-derived Portuguese frequency list (hermitdave/FrequencyWords)
- Increase affix expansion cap from 30→80 forms per word for Portuguese gerunds

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

internal/loader/affix.go

@@ -205,9 +205,9 @@ func parseDicFile(path string) ([]dicEntry, error) {
 }
 
 // maxFormsPerWord caps the number of inflected forms generated per base word.
-// Portuguese and French .aff files can produce hundreds of cross-product forms
-// per word; most are valid but the combinatorial explosion bloats the DB.
-const maxFormsPerWord = 30
+// Portuguese verbs can have 60+ valid conjugations (gerunds, subjunctives, etc.),
+// so the cap must be generous enough to avoid cutting off common forms.
+const maxFormsPerWord = 80
 
 func expandWord(word, flags string, rules map[string]*affixRuleSet) []string {
 	seen := map[string]bool{word: true} // base form already in DB

internal/loader/cetempublico.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"unicode/utf8"
 )
 
 // CETEMPublicoLoader loads frequency data for European Portuguese words.
@@ -24,6 +25,8 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 		filepath.Join(dataDir, "cetempublico-freq.tsv"),
 		filepath.Join(dataDir, "pt-freq.tsv"),
 		filepath.Join(dataDir, "pt_PT-freq.tsv"),
+		filepath.Join(dataDir, "pt_50k.txt"),      // hermitdave/FrequencyWords
+		filepath.Join(dataDir, "pt_full.txt"),      // hermitdave/FrequencyWords
 	}
 
 	var path string
@@ -37,12 +40,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 		slog.Warn("cetempublico: no data file found, skipping", "searched", candidates)
 		return nil
 	}
-
-	f, err := os.Open(path)
-	if err != nil {
-		return fmt.Errorf("cetempublico: open: %w", err)
-	}
-	defer f.Close()
+	slog.Info("cetempublico: using file", "path", path)
 
 	tx, err := db.Begin()
 	if err != nil {
@@ -50,43 +48,78 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 	}
 	defer tx.Rollback()
 
-	stmt, err := tx.Prepare(`
+	stmtUpdate, err := tx.Prepare(`
 		UPDATE words SET frequency = ? WHERE word = ? AND lang = 'pt-PT' AND frequency = 0`)
 	if err != nil {
-		return fmt.Errorf("cetempublico: prepare: %w", err)
+		return fmt.Errorf("cetempublico: prepare update: %w", err)
 	}
-	defer stmt.Close()
+	defer stmtUpdate.Close()
 
-	scanner := bufio.NewScanner(f)
-	// Skip header if present
+	stmtInsert, err := tx.Prepare(`
+		INSERT OR IGNORE INTO words (word, lang, frequency) VALUES (?, 'pt-PT', ?)`)
+	if err != nil {
+		return fmt.Errorf("cetempublico: prepare insert: %w", err)
+	}
+	defer stmtInsert.Close()
+
+	// Read entire file to detect encoding before parsing.
+	// The file is typically <20 MB so this is fine.
+	rawBytes, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("cetempublico: read file: %w", err)
+	}
+
+	isLatin1 := !utf8.Valid(rawBytes)
+	if isLatin1 {
+		slog.Info("cetempublico: detected ISO-8859-1 encoding, converting")
+		rawBytes = []byte(latin1ToUTF8(rawBytes))
+	}
+
+	scanner := bufio.NewScanner(strings.NewReader(string(rawBytes)))
+
+	// Peek first line to detect header vs data
+	var firstLine string
 	if scanner.Scan() {
-		first := scanner.Text()
-		fields := strings.Split(first, "\t")
-		// If first line looks like data (second field is numeric), process it
+		firstLine = scanner.Text()
+		fields := strings.Fields(firstLine)
 		if len(fields) >= 2 {
-			if _, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64); err != nil {
-				// It's a header, skip it
-			} else {
-				// It's data, we need to process it — but we already consumed it
-				// Re-process below
+			if _, err := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64); err != nil {
+				firstLine = "" // It's a header, skip it
 			}
 		}
 	}
 
-	var count int
+	var count, skipped, inserted int
 	processLine := func(line string) error {
+		// Support TSV and space-separated, with either column order:
+		//   "word\tcount", "word count", or "count\tword", "count word"
 		fields := strings.Split(line, "\t")
+		if len(fields) < 2 {
+			fields = strings.Fields(line)
+		}
 		if len(fields) < 2 {
 			return nil
 		}
 
-		word := strings.ToLower(strings.TrimSpace(fields[0]))
+		// Detect column order: if first field is numeric, it's count-first
+		var word string
+		var freqStr string
+		if _, err := strconv.ParseFloat(strings.TrimSpace(fields[0]), 64); err == nil {
+			freqStr = strings.TrimSpace(fields[0])
+			word = strings.ToLower(strings.TrimSpace(fields[1]))
+		} else {
+			word = strings.ToLower(strings.TrimSpace(fields[0]))
+			freqStr = strings.TrimSpace(fields[1])
+		}
+
 		if word == "" || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
+			skipped++
 			return nil
 		}
 
-		freqVal, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64)
+		freqVal, err := strconv.ParseFloat(freqStr, 64)
 		if err != nil || freqVal <= 0 {
+			skipped++
 			return nil
 		}
 
@@ -106,16 +139,33 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 			freq = 1
 		}
 
-		res, err := stmt.Exec(freq, word)
+		res, err := stmtUpdate.Exec(freq, word)
 		if err != nil {
 			return fmt.Errorf("cetempublico: update: %w", err)
 		}
 		if n, _ := res.RowsAffected(); n > 0 {
 			count++
+		} else {
+			// Word not in DB yet — insert it with frequency
+			if _, err := stmtInsert.Exec(word, freq); err != nil {
+				return fmt.Errorf("cetempublico: insert: %w", err)
+			}
+			inserted++
+			count++
+		}
+		if count+skipped <= 5 {
+			slog.Info("cetempublico: sample", "word", word, "freq", freq, "total", count, "inserted", inserted)
 		}
 		return nil
 	}
 
+	// Process first line if it was data (not a header)
+	if firstLine != "" {
+		if err := processLine(firstLine); err != nil {
+			return err
+		}
+	}
+
 	for scanner.Scan() {
 		if err := processLine(scanner.Text()); err != nil {
 			return err
@@ -129,6 +179,17 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 	if err := tx.Commit(); err != nil {
 		return fmt.Errorf("cetempublico: commit: %w", err)
 	}
-	slog.Info("cetempublico loaded", "updated_words", count)
+	slog.Info("cetempublico loaded", "updated_words", count, "inserted_new", inserted, "skipped", skipped)
 	return nil
 }
+
+// latin1ToUTF8 converts ISO-8859-1 bytes to a UTF-8 string.
+// Each byte in ISO-8859-1 maps directly to the same Unicode code point.
+func latin1ToUTF8(b []byte) string {
+	var buf strings.Builder
+	buf.Grow(len(b) + len(b)/4) // accented chars expand
+	for _, c := range b {
+		buf.WriteRune(rune(c))
+	}
+	return buf.String()
+}

internal/loader/wiktionary.go

@@ -27,9 +27,15 @@ type wiktEntry struct {
 	Word     string `json:"word"`
 	POS      string `json:"pos"`
 	LangCode string `json:"lang_code"`
-	Senses   []struct {
-		Glosses []string `json:"glosses"`
-		Tags    []string `json:"tags"`
+	Senses []struct {
+		Glosses  []string `json:"glosses"`
+		Tags     []string `json:"tags"`
+		Synonyms []struct {
+			Word string `json:"word"`
+		} `json:"synonyms"`
+		Antonyms []struct {
+			Word string `json:"word"`
+		} `json:"antonyms"`
 	} `json:"senses"`
 	Synonyms []struct {
 		Word string `json:"word"`
@@ -188,7 +194,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
 
 		pos := wiktPOSMap[entry.POS]
 
-		// Process senses
+		// Process senses (definitions + sense-level synonyms/antonyms)
 		for _, sense := range entry.Senses {
 			if hasTag(sense.Tags, "form-of", "alt-of") {
 				continue
@@ -208,9 +214,33 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
 				}
 				defCount++
 			}
+
+			// Sense-level synonyms
+			for _, syn := range sense.Synonyms {
+				synWord := strings.ToLower(syn.Word)
+				if synWord == "" || containsDigit(synWord) || containsSpace(synWord) {
+					continue
+				}
+				if _, err := batch.stmtSyn.Exec(synWord, word, lang); err != nil {
+					return fmt.Errorf("wiktionary: insert syn: %w", err)
+				}
+				synCount++
+			}
+
+			// Sense-level antonyms
+			for _, ant := range sense.Antonyms {
+				antWord := strings.ToLower(ant.Word)
+				if antWord == "" || containsDigit(antWord) || containsSpace(antWord) {
+					continue
+				}
+				if _, err := batch.stmtAnt.Exec(antWord, word, lang); err != nil {
+					return fmt.Errorf("wiktionary: insert ant: %w", err)
+				}
+				antCount++
+			}
 		}
 
-		// Process synonyms
+		// Process top-level synonyms
 		for _, syn := range entry.Synonyms {
 			synWord := strings.ToLower(syn.Word)
 			if synWord == "" || containsDigit(synWord) || containsSpace(synWord) {
@@ -222,7 +252,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
 			synCount++
 		}
 
-		// Process antonyms
+		// Process top-level antonyms
 		for _, ant := range entry.Antonyms {
 			antWord := strings.ToLower(ant.Word)
 			if antWord == "" || containsDigit(antWord) || containsSpace(antWord) {