Fix Portuguese coverage gaps: synonyms, translations, frequency, gerunds

- Extract sense-level synonyms/antonyms from Wiktionary (not just top-level),
  dramatically improving Portuguese synonym coverage
- Add bidirectional translation lookup so pt→en queries find en→pt entries
  stored by English Wiktionary loader, with reverse index for performance
- Fix frequency loader: auto-detect column order (count\tword vs word\tcount),
  auto-detect ISO-8859-1 encoding and convert to UTF-8, insert missing words
  instead of only updating existing ones
- Switch WordNet download from 3.1 to 3.0 to match WOLF/OMW synset offsets
- Add OpenSubtitles-derived Portuguese frequency list (hermitdave/FrequencyWords)
- Increase affix expansion cap from 30→80 forms per word for Portuguese gerunds

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

internal/loader/cetempublico.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"unicode/utf8"
 )
 
 // CETEMPublicoLoader loads frequency data for European Portuguese words.
@@ -24,6 +25,8 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 		filepath.Join(dataDir, "cetempublico-freq.tsv"),
 		filepath.Join(dataDir, "pt-freq.tsv"),
 		filepath.Join(dataDir, "pt_PT-freq.tsv"),
+		filepath.Join(dataDir, "pt_50k.txt"),      // hermitdave/FrequencyWords
+		filepath.Join(dataDir, "pt_full.txt"),      // hermitdave/FrequencyWords
 	}
 
 	var path string
@@ -37,12 +40,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 		slog.Warn("cetempublico: no data file found, skipping", "searched", candidates)
 		return nil
 	}
-
-	f, err := os.Open(path)
-	if err != nil {
-		return fmt.Errorf("cetempublico: open: %w", err)
-	}
-	defer f.Close()
+	slog.Info("cetempublico: using file", "path", path)
 
 	tx, err := db.Begin()
 	if err != nil {
@@ -50,43 +48,78 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 	}
 	defer tx.Rollback()
 
-	stmt, err := tx.Prepare(`
+	stmtUpdate, err := tx.Prepare(`
 		UPDATE words SET frequency = ? WHERE word = ? AND lang = 'pt-PT' AND frequency = 0`)
 	if err != nil {
-		return fmt.Errorf("cetempublico: prepare: %w", err)
+		return fmt.Errorf("cetempublico: prepare update: %w", err)
 	}
-	defer stmt.Close()
+	defer stmtUpdate.Close()
 
-	scanner := bufio.NewScanner(f)
-	// Skip header if present
+	stmtInsert, err := tx.Prepare(`
+		INSERT OR IGNORE INTO words (word, lang, frequency) VALUES (?, 'pt-PT', ?)`)
+	if err != nil {
+		return fmt.Errorf("cetempublico: prepare insert: %w", err)
+	}
+	defer stmtInsert.Close()
+
+	// Read entire file to detect encoding before parsing.
+	// The file is typically <20 MB so this is fine.
+	rawBytes, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("cetempublico: read file: %w", err)
+	}
+
+	isLatin1 := !utf8.Valid(rawBytes)
+	if isLatin1 {
+		slog.Info("cetempublico: detected ISO-8859-1 encoding, converting")
+		rawBytes = []byte(latin1ToUTF8(rawBytes))
+	}
+
+	scanner := bufio.NewScanner(strings.NewReader(string(rawBytes)))
+
+	// Peek first line to detect header vs data
+	var firstLine string
 	if scanner.Scan() {
-		first := scanner.Text()
-		fields := strings.Split(first, "\t")
-		// If first line looks like data (second field is numeric), process it
+		firstLine = scanner.Text()
+		fields := strings.Fields(firstLine)
 		if len(fields) >= 2 {
-			if _, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64); err != nil {
-				// It's a header, skip it
-			} else {
-				// It's data, we need to process it — but we already consumed it
-				// Re-process below
+			if _, err := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64); err != nil {
+				firstLine = "" // It's a header, skip it
 			}
 		}
 	}
 
-	var count int
+	var count, skipped, inserted int
 	processLine := func(line string) error {
+		// Support TSV and space-separated, with either column order:
+		//   "word\tcount", "word count", or "count\tword", "count word"
 		fields := strings.Split(line, "\t")
+		if len(fields) < 2 {
+			fields = strings.Fields(line)
+		}
 		if len(fields) < 2 {
 			return nil
 		}
 
-		word := strings.ToLower(strings.TrimSpace(fields[0]))
+		// Detect column order: if first field is numeric, it's count-first
+		var word string
+		var freqStr string
+		if _, err := strconv.ParseFloat(strings.TrimSpace(fields[0]), 64); err == nil {
+			freqStr = strings.TrimSpace(fields[0])
+			word = strings.ToLower(strings.TrimSpace(fields[1]))
+		} else {
+			word = strings.ToLower(strings.TrimSpace(fields[0]))
+			freqStr = strings.TrimSpace(fields[1])
+		}
+
 		if word == "" || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
+			skipped++
 			return nil
 		}
 
-		freqVal, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64)
+		freqVal, err := strconv.ParseFloat(freqStr, 64)
 		if err != nil || freqVal <= 0 {
+			skipped++
 			return nil
 		}
 
@@ -106,16 +139,33 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 			freq = 1
 		}
 
-		res, err := stmt.Exec(freq, word)
+		res, err := stmtUpdate.Exec(freq, word)
 		if err != nil {
 			return fmt.Errorf("cetempublico: update: %w", err)
 		}
 		if n, _ := res.RowsAffected(); n > 0 {
 			count++
+		} else {
+			// Word not in DB yet — insert it with frequency
+			if _, err := stmtInsert.Exec(word, freq); err != nil {
+				return fmt.Errorf("cetempublico: insert: %w", err)
+			}
+			inserted++
+			count++
+		}
+		if count+skipped <= 5 {
+			slog.Info("cetempublico: sample", "word", word, "freq", freq, "total", count, "inserted", inserted)
 		}
 		return nil
 	}
 
+	// Process first line if it was data (not a header)
+	if firstLine != "" {
+		if err := processLine(firstLine); err != nil {
+			return err
+		}
+	}
+
 	for scanner.Scan() {
 		if err := processLine(scanner.Text()); err != nil {
 			return err
@@ -129,6 +179,17 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 	if err := tx.Commit(); err != nil {
 		return fmt.Errorf("cetempublico: commit: %w", err)
 	}
-	slog.Info("cetempublico loaded", "updated_words", count)
+	slog.Info("cetempublico loaded", "updated_words", count, "inserted_new", inserted, "skipped", skipped)
 	return nil
 }
+
+// latin1ToUTF8 converts ISO-8859-1 bytes to a UTF-8 string.
+// Each byte in ISO-8859-1 maps directly to the same Unicode code point.
+func latin1ToUTF8(b []byte) string {
+	var buf strings.Builder
+	buf.Grow(len(b) + len(b)/4) // accented chars expand
+	for _, c := range b {
+		buf.WriteRune(rune(c))
+	}
+	return buf.String()
+}