Add loader tests and fix header detection and punctuation filtering

Tests: 23 new tests across cetempublico, affix, and wiktionary loaders
covering column order detection, ISO-8859-1 encoding, junk filtering,
frequency scaling, affix expansion, and sense-level synonym extraction.

Fixes: header detection now checks either field for numeric value (was
only checking last field, breaking count-first format). Added hasLetter
filter to reject pure punctuation entries like "." and ",".

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

internal/loader/affix_test.go

@@ -0,0 +1,169 @@
+package loader
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestExpandWord_SuffixRule(t *testing.T) {
+	rules := map[string]*affixRuleSet{
+		"S": {
+			ruleType: "SFX",
+			rules: []affixRule{
+				{ruleType: "SFX", strip: "", affix: "s", matchers: nil},
+			},
+		},
+	}
+
+	forms := expandWord("cat", "S", rules)
+	if len(forms) != 1 || forms[0] != "cats" {
+		t.Errorf("expected [cats], got %v", forms)
+	}
+}
+
+func TestExpandWord_StripAndReplace(t *testing.T) {
+	rules := map[string]*affixRuleSet{
+		"D": {
+			ruleType: "SFX",
+			rules: []affixRule{
+				{ruleType: "SFX", strip: "e", affix: "ing", matchers: parseConditionMatchers("e")},
+			},
+		},
+	}
+
+	forms := expandWord("make", "D", rules)
+	if len(forms) != 1 || forms[0] != "making" {
+		t.Errorf("expected [making], got %v", forms)
+	}
+
+	// Word not matching condition should produce no forms
+	forms2 := expandWord("run", "D", rules)
+	if len(forms2) != 0 {
+		t.Errorf("expected no forms for 'run' with 'e' condition, got %v", forms2)
+	}
+}
+
+func TestExpandWord_PrefixRule(t *testing.T) {
+	rules := map[string]*affixRuleSet{
+		"U": {
+			ruleType: "PFX",
+			rules: []affixRule{
+				{ruleType: "PFX", strip: "", affix: "un", matchers: nil},
+			},
+		},
+	}
+
+	forms := expandWord("happy", "U", rules)
+	if len(forms) != 1 || forms[0] != "unhappy" {
+		t.Errorf("expected [unhappy], got %v", forms)
+	}
+}
+
+func TestExpandWord_MaxCap(t *testing.T) {
+	// Create a rule that generates many forms
+	rules := map[string]*affixRuleSet{
+		"X": {
+			ruleType: "SFX",
+			rules:    make([]affixRule, 100),
+		},
+	}
+	for i := range rules["X"].rules {
+		rules["X"].rules[i] = affixRule{ruleType: "SFX", strip: "", affix: string(rune('a' + i%26)) + string(rune('0'+i/26))}
+	}
+
+	forms := expandWord("test", "X", rules)
+	if len(forms) > maxFormsPerWord {
+		t.Errorf("expected at most %d forms, got %d", maxFormsPerWord, len(forms))
+	}
+}
+
+func TestParseAffFile(t *testing.T) {
+	dir := t.TempDir()
+	affContent := `SET UTF-8
+
+SFX S Y 1
+SFX S 0 s .
+
+PFX U Y 1
+PFX U 0 un .
+`
+	affPath := filepath.Join(dir, "test.aff")
+	os.WriteFile(affPath, []byte(affContent), 0644)
+
+	rules, err := parseAffFile(affPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, ok := rules["S"]; !ok {
+		t.Error("expected SFX rule 'S'")
+	}
+	if _, ok := rules["U"]; !ok {
+		t.Error("expected PFX rule 'U'")
+	}
+}
+
+func TestParseDicFile(t *testing.T) {
+	dir := t.TempDir()
+	dicContent := "3\ncat/S\ndog/S\nhappy/U\n"
+	dicPath := filepath.Join(dir, "test.dic")
+	os.WriteFile(dicPath, []byte(dicContent), 0644)
+
+	entries, err := parseDicFile(dicPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(entries) != 3 {
+		t.Fatalf("expected 3 entries, got %d", len(entries))
+	}
+	if entries[0].word != "cat" || entries[0].flags != "S" {
+		t.Errorf("expected cat/S, got %s/%s", entries[0].word, entries[0].flags)
+	}
+}
+
+func TestAffixLoader_NoFileSkips(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	loader := AffixLoader{Lang: "en", DicFile: "nonexistent.dic", AffFile: "nonexistent.aff"}
+	if err := loader.Load(db, t.TempDir()); err != nil {
+		t.Errorf("expected graceful skip, got: %v", err)
+	}
+}
+
+func TestAffixLoader_EndToEnd(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	dir := t.TempDir()
+
+	affContent := `SET UTF-8
+
+SFX S Y 1
+SFX S 0 s .
+`
+	os.WriteFile(filepath.Join(dir, "test.aff"), []byte(affContent), 0644)
+
+	dicContent := "2\ncat/S\ndog/S\n"
+	os.WriteFile(filepath.Join(dir, "test.dic"), []byte(dicContent), 0644)
+
+	loader := AffixLoader{Lang: "en", DicFile: "test.dic", AffFile: "test.aff"}
+	if err := loader.Load(db, dir); err != nil {
+		t.Fatal(err)
+	}
+
+	var count int
+	db.QueryRow("SELECT COUNT(*) FROM words WHERE lang = 'en'").Scan(&count)
+	if count != 2 {
+		t.Errorf("expected 2 expanded forms (cats, dogs), got %d", count)
+	}
+
+	// Verify specific forms exist
+	var exists bool
+	db.QueryRow("SELECT EXISTS(SELECT 1 FROM words WHERE word = 'cats' AND lang = 'en')").Scan(&exists)
+	if !exists {
+		t.Error("expected 'cats' to exist")
+	}
+}

internal/loader/cetempublico.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"unicode"
 	"unicode/utf8"
 )
 
@@ -74,14 +75,17 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 
 	scanner := bufio.NewScanner(strings.NewReader(string(rawBytes)))
 
-	// Peek first line to detect header vs data
+	// Peek first line to detect header vs data.
+	// A data line has at least one numeric field (either "word count" or "count word").
 	var firstLine string
 	if scanner.Scan() {
 		firstLine = scanner.Text()
 		fields := strings.Fields(firstLine)
 		if len(fields) >= 2 {
-			if _, err := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64); err != nil {
+			_, errFirst := strconv.ParseFloat(strings.TrimSpace(fields[0]), 64)
-				firstLine = "" // It's a header, skip it
+			_, errLast := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64)
+			if errFirst != nil && errLast != nil {
+				firstLine = "" // Neither field is numeric — it's a header, skip it
 			}
 		}
 	}
@@ -109,7 +113,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 			freqStr = strings.TrimSpace(fields[1])
 		}
 
-		if word == "" || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
+		if word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
 			return nil
 		}
 
@@ -175,6 +179,15 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 	return nil
 }
 
+func hasLetter(s string) bool {
+	for _, r := range s {
+		if unicode.IsLetter(r) {
+			return true
+		}
+	}
+	return false
+}
+
 // latin1ToUTF8 converts ISO-8859-1 bytes to a UTF-8 string.
 // Each byte in ISO-8859-1 maps directly to the same Unicode code point.
 func latin1ToUTF8(b []byte) string {

internal/loader/cetempublico_test.go

@@ -0,0 +1,159 @@
+package loader
+
+import (
+	"database/sql"
+	"os"
+	"path/filepath"
+	"testing"
+
+	_ "modernc.org/sqlite"
+)
+
+func seedWord(t *testing.T, db *sql.DB, word, lang string) {
+	t.Helper()
+	if _, err := db.Exec("INSERT INTO words (word, lang) VALUES (?, ?)", word, lang); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func queryFreq(t *testing.T, db *sql.DB, word string) int {
+	t.Helper()
+	var freq int
+	if err := db.QueryRow("SELECT frequency FROM words WHERE word = ?", word).Scan(&freq); err != nil {
+		t.Fatalf("query frequency for %q: %v", word, err)
+	}
+	return freq
+}
+
+func TestCETEMPublico_WordSpaceFormat(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	seedWord(t, db, "casa", "pt-PT")
+	seedWord(t, db, "homem", "pt-PT")
+
+	dir := t.TempDir()
+	os.WriteFile(filepath.Join(dir, "pt_50k.txt"), []byte("casa 50000\nhomem 30000\n"), 0644)
+
+	if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil {
+		t.Fatal(err)
+	}
+
+	if f := queryFreq(t, db, "casa"); f == 0 {
+		t.Error("expected non-zero frequency for 'casa'")
+	}
+	if f := queryFreq(t, db, "homem"); f == 0 {
+		t.Error("expected non-zero frequency for 'homem'")
+	}
+}
+
+func TestCETEMPublico_ReversedColumns(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	seedWord(t, db, "grande", "pt-PT")
+
+	dir := t.TempDir()
+	// CETEMPúblico format: count<tab>word
+	os.WriteFile(filepath.Join(dir, "cetempublico-freq.tsv"), []byte("500000\tgrande\n"), 0644)
+
+	if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil {
+		t.Fatal(err)
+	}
+
+	if f := queryFreq(t, db, "grande"); f == 0 {
+		t.Error("expected non-zero frequency for 'grande' with reversed columns")
+	}
+}
+
+func TestCETEMPublico_InsertsMissingWords(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	dir := t.TempDir()
+	os.WriteFile(filepath.Join(dir, "pt_50k.txt"), []byte("que 5000000\n"), 0644)
+
+	if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil {
+		t.Fatal(err)
+	}
+
+	// "que" was not seeded — loader should have inserted it
+	if f := queryFreq(t, db, "que"); f == 0 {
+		t.Error("expected non-zero frequency for auto-inserted word 'que'")
+	}
+}
+
+func TestCETEMPublico_ISO88591(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	seedWord(t, db, "não", "pt-PT")
+
+	dir := t.TempDir()
+	// ISO-8859-1: "5000\tnão\n" where ã = 0xE3 (single byte)
+	os.WriteFile(filepath.Join(dir, "cetempublico-freq.tsv"),
+		[]byte{0x35, 0x30, 0x30, 0x30, 0x09, 0x6E, 0xE3, 0x6F, 0x0A}, 0644)
+
+	if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil {
+		t.Fatal(err)
+	}
+
+	if f := queryFreq(t, db, "não"); f == 0 {
+		t.Error("expected non-zero frequency for ISO-8859-1 encoded 'não'")
+	}
+}
+
+func TestCETEMPublico_FiltersJunk(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	dir := t.TempDir()
+	os.WriteFile(filepath.Join(dir, "pt_50k.txt"),
+		[]byte("word1 5000\n. 90000\n, 80000\nboa 3000\n"), 0644)
+
+	if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil {
+		t.Fatal(err)
+	}
+
+	var count int
+	db.QueryRow("SELECT COUNT(*) FROM words WHERE lang = 'pt-PT'").Scan(&count)
+	if count != 1 {
+		t.Errorf("expected 1 word (boa only), got %d", count)
+	}
+}
+
+func TestCETEMPublico_NoFileSkips(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	if err := (CETEMPublicoLoader{}).Load(db, t.TempDir()); err != nil {
+		t.Errorf("expected graceful skip with no file, got: %v", err)
+	}
+}
+
+func TestCETEMPublico_FrequencyScaling(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	seedWord(t, db, "raro", "pt-PT")
+	seedWord(t, db, "comum", "pt-PT")
+
+	dir := t.TempDir()
+	// "comum" has much higher frequency than "raro"
+	os.WriteFile(filepath.Join(dir, "pt_50k.txt"),
+		[]byte("comum 5000000\nraro 50\n"), 0644)
+
+	if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil {
+		t.Fatal(err)
+	}
+
+	comum := queryFreq(t, db, "comum")
+	raro := queryFreq(t, db, "raro")
+
+	if comum <= raro {
+		t.Errorf("expected 'comum' (%d) > 'raro' (%d)", comum, raro)
+	}
+	if comum > 10000 {
+		t.Errorf("frequency should cap at 10000, got %d", comum)
+	}
+}

internal/loader/wiktionary_test.go

@@ -166,6 +166,66 @@ func TestHasTag(t *testing.T) {
 	}
 }
 
+func TestSenseLevelSynonyms(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('happy', 'en')"); err != nil {
+		t.Fatal(err)
+	}
+
+	dir := t.TempDir()
+	// Synonyms nested inside senses, not top-level
+	jsonl := `{"word":"happy","pos":"adj","lang_code":"en","senses":[{"glosses":["feeling joy"],"tags":[],"synonyms":[{"word":"joyful"},{"word":"glad"}],"antonyms":[{"word":"sad"}]}],"synonyms":[],"antonyms":[],"translations":[]}` + "\n"
+	path := writeJSONL(t, dir, "test.jsonl", jsonl)
+
+	if err := loadWiktionary(db, path, "en"); err != nil {
+		t.Fatal(err)
+	}
+
+	var synCount int
+	if err := db.QueryRow("SELECT COUNT(*) FROM synonyms").Scan(&synCount); err != nil {
+		t.Fatal(err)
+	}
+	if synCount != 2 {
+		t.Errorf("expected 2 sense-level synonyms, got %d", synCount)
+	}
+
+	var antCount int
+	if err := db.QueryRow("SELECT COUNT(*) FROM antonyms").Scan(&antCount); err != nil {
+		t.Fatal(err)
+	}
+	if antCount != 1 {
+		t.Errorf("expected 1 sense-level antonym, got %d", antCount)
+	}
+}
+
+func TestTopAndSenseLevelSynonymsMerge(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('big', 'en')"); err != nil {
+		t.Fatal(err)
+	}
+
+	dir := t.TempDir()
+	// Both top-level and sense-level synonyms
+	jsonl := `{"word":"big","pos":"adj","lang_code":"en","senses":[{"glosses":["of great size"],"tags":[],"synonyms":[{"word":"huge"}]}],"synonyms":[{"word":"large"}],"antonyms":[],"translations":[]}` + "\n"
+	path := writeJSONL(t, dir, "test.jsonl", jsonl)
+
+	if err := loadWiktionary(db, path, "en"); err != nil {
+		t.Fatal(err)
+	}
+
+	var synCount int
+	if err := db.QueryRow("SELECT COUNT(*) FROM synonyms").Scan(&synCount); err != nil {
+		t.Fatal(err)
+	}
+	if synCount != 2 {
+		t.Errorf("expected 2 synonyms (1 top-level + 1 sense-level), got %d", synCount)
+	}
+}
+
 func TestHasBrazilTag(t *testing.T) {
 	tests := []struct {
 		tags []string