From 2045cc2adaf509e0dddcbf0cbc6485c1dd112bb4 Mon Sep 17 00:00:00 2001 From: prosolis <5590409+prosolis@users.noreply.github.com> Date: Fri, 3 Apr 2026 09:15:13 -0700 Subject: [PATCH] Add loader tests and fix header detection and punctuation filtering Tests: 23 new tests across cetempublico, affix, and wiktionary loaders covering column order detection, ISO-8859-1 encoding, junk filtering, frequency scaling, affix expansion, and sense-level synonym extraction. Fixes: header detection now checks either field for numeric value (was only checking last field, breaking count-first format). Added hasLetter filter to reject pure punctuation entries like "." and ",". Co-Authored-By: Claude Opus 4.6 --- internal/loader/affix_test.go | 169 +++++++++++++++++++++++++++ internal/loader/cetempublico.go | 21 +++- internal/loader/cetempublico_test.go | 159 +++++++++++++++++++++++++ internal/loader/wiktionary_test.go | 60 ++++++++++ 4 files changed, 405 insertions(+), 4 deletions(-) create mode 100644 internal/loader/affix_test.go create mode 100644 internal/loader/cetempublico_test.go diff --git a/internal/loader/affix_test.go b/internal/loader/affix_test.go new file mode 100644 index 0000000..6c3b2cc --- /dev/null +++ b/internal/loader/affix_test.go @@ -0,0 +1,169 @@ +package loader + +import ( + "os" + "path/filepath" + "testing" +) + +func TestExpandWord_SuffixRule(t *testing.T) { + rules := map[string]*affixRuleSet{ + "S": { + ruleType: "SFX", + rules: []affixRule{ + {ruleType: "SFX", strip: "", affix: "s", matchers: nil}, + }, + }, + } + + forms := expandWord("cat", "S", rules) + if len(forms) != 1 || forms[0] != "cats" { + t.Errorf("expected [cats], got %v", forms) + } +} + +func TestExpandWord_StripAndReplace(t *testing.T) { + rules := map[string]*affixRuleSet{ + "D": { + ruleType: "SFX", + rules: []affixRule{ + {ruleType: "SFX", strip: "e", affix: "ing", matchers: parseConditionMatchers("e")}, + }, + }, + } + + forms := expandWord("make", "D", rules) + if len(forms) != 1 || forms[0] != "making" { + t.Errorf("expected [making], got %v", forms) + } + + // Word not matching condition should produce no forms + forms2 := expandWord("run", "D", rules) + if len(forms2) != 0 { + t.Errorf("expected no forms for 'run' with 'e' condition, got %v", forms2) + } +} + +func TestExpandWord_PrefixRule(t *testing.T) { + rules := map[string]*affixRuleSet{ + "U": { + ruleType: "PFX", + rules: []affixRule{ + {ruleType: "PFX", strip: "", affix: "un", matchers: nil}, + }, + }, + } + + forms := expandWord("happy", "U", rules) + if len(forms) != 1 || forms[0] != "unhappy" { + t.Errorf("expected [unhappy], got %v", forms) + } +} + +func TestExpandWord_MaxCap(t *testing.T) { + // Create a rule that generates many forms + rules := map[string]*affixRuleSet{ + "X": { + ruleType: "SFX", + rules: make([]affixRule, 100), + }, + } + for i := range rules["X"].rules { + rules["X"].rules[i] = affixRule{ruleType: "SFX", strip: "", affix: string(rune('a' + i%26)) + string(rune('0'+i/26))} + } + + forms := expandWord("test", "X", rules) + if len(forms) > maxFormsPerWord { + t.Errorf("expected at most %d forms, got %d", maxFormsPerWord, len(forms)) + } +} + +func TestParseAffFile(t *testing.T) { + dir := t.TempDir() + affContent := `SET UTF-8 + +SFX S Y 1 +SFX S 0 s . + +PFX U Y 1 +PFX U 0 un . +` + affPath := filepath.Join(dir, "test.aff") + os.WriteFile(affPath, []byte(affContent), 0644) + + rules, err := parseAffFile(affPath) + if err != nil { + t.Fatal(err) + } + + if _, ok := rules["S"]; !ok { + t.Error("expected SFX rule 'S'") + } + if _, ok := rules["U"]; !ok { + t.Error("expected PFX rule 'U'") + } +} + +func TestParseDicFile(t *testing.T) { + dir := t.TempDir() + dicContent := "3\ncat/S\ndog/S\nhappy/U\n" + dicPath := filepath.Join(dir, "test.dic") + os.WriteFile(dicPath, []byte(dicContent), 0644) + + entries, err := parseDicFile(dicPath) + if err != nil { + t.Fatal(err) + } + + if len(entries) != 3 { + t.Fatalf("expected 3 entries, got %d", len(entries)) + } + if entries[0].word != "cat" || entries[0].flags != "S" { + t.Errorf("expected cat/S, got %s/%s", entries[0].word, entries[0].flags) + } +} + +func TestAffixLoader_NoFileSkips(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + loader := AffixLoader{Lang: "en", DicFile: "nonexistent.dic", AffFile: "nonexistent.aff"} + if err := loader.Load(db, t.TempDir()); err != nil { + t.Errorf("expected graceful skip, got: %v", err) + } +} + +func TestAffixLoader_EndToEnd(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + dir := t.TempDir() + + affContent := `SET UTF-8 + +SFX S Y 1 +SFX S 0 s . +` + os.WriteFile(filepath.Join(dir, "test.aff"), []byte(affContent), 0644) + + dicContent := "2\ncat/S\ndog/S\n" + os.WriteFile(filepath.Join(dir, "test.dic"), []byte(dicContent), 0644) + + loader := AffixLoader{Lang: "en", DicFile: "test.dic", AffFile: "test.aff"} + if err := loader.Load(db, dir); err != nil { + t.Fatal(err) + } + + var count int + db.QueryRow("SELECT COUNT(*) FROM words WHERE lang = 'en'").Scan(&count) + if count != 2 { + t.Errorf("expected 2 expanded forms (cats, dogs), got %d", count) + } + + // Verify specific forms exist + var exists bool + db.QueryRow("SELECT EXISTS(SELECT 1 FROM words WHERE word = 'cats' AND lang = 'en')").Scan(&exists) + if !exists { + t.Error("expected 'cats' to exist") + } +} diff --git a/internal/loader/cetempublico.go b/internal/loader/cetempublico.go index 167c8fb..b6d62c1 100644 --- a/internal/loader/cetempublico.go +++ b/internal/loader/cetempublico.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strconv" "strings" + "unicode" "unicode/utf8" ) @@ -74,14 +75,17 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { scanner := bufio.NewScanner(strings.NewReader(string(rawBytes))) - // Peek first line to detect header vs data + // Peek first line to detect header vs data. + // A data line has at least one numeric field (either "word count" or "count word"). var firstLine string if scanner.Scan() { firstLine = scanner.Text() fields := strings.Fields(firstLine) if len(fields) >= 2 { - if _, err := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64); err != nil { - firstLine = "" // It's a header, skip it + _, errFirst := strconv.ParseFloat(strings.TrimSpace(fields[0]), 64) + _, errLast := strconv.ParseFloat(strings.TrimSpace(fields[len(fields)-1]), 64) + if errFirst != nil && errLast != nil { + firstLine = "" // Neither field is numeric — it's a header, skip it } } } @@ -109,7 +113,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { freqStr = strings.TrimSpace(fields[1]) } - if word == "" || containsDigit(word) || containsSpace(word) || containsNonLatin(word) { + if word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) { return nil } @@ -175,6 +179,15 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { return nil } +func hasLetter(s string) bool { + for _, r := range s { + if unicode.IsLetter(r) { + return true + } + } + return false +} + // latin1ToUTF8 converts ISO-8859-1 bytes to a UTF-8 string. // Each byte in ISO-8859-1 maps directly to the same Unicode code point. func latin1ToUTF8(b []byte) string { diff --git a/internal/loader/cetempublico_test.go b/internal/loader/cetempublico_test.go new file mode 100644 index 0000000..5199ecb --- /dev/null +++ b/internal/loader/cetempublico_test.go @@ -0,0 +1,159 @@ +package loader + +import ( + "database/sql" + "os" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func seedWord(t *testing.T, db *sql.DB, word, lang string) { + t.Helper() + if _, err := db.Exec("INSERT INTO words (word, lang) VALUES (?, ?)", word, lang); err != nil { + t.Fatal(err) + } +} + +func queryFreq(t *testing.T, db *sql.DB, word string) int { + t.Helper() + var freq int + if err := db.QueryRow("SELECT frequency FROM words WHERE word = ?", word).Scan(&freq); err != nil { + t.Fatalf("query frequency for %q: %v", word, err) + } + return freq +} + +func TestCETEMPublico_WordSpaceFormat(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + seedWord(t, db, "casa", "pt-PT") + seedWord(t, db, "homem", "pt-PT") + + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "pt_50k.txt"), []byte("casa 50000\nhomem 30000\n"), 0644) + + if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil { + t.Fatal(err) + } + + if f := queryFreq(t, db, "casa"); f == 0 { + t.Error("expected non-zero frequency for 'casa'") + } + if f := queryFreq(t, db, "homem"); f == 0 { + t.Error("expected non-zero frequency for 'homem'") + } +} + +func TestCETEMPublico_ReversedColumns(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + seedWord(t, db, "grande", "pt-PT") + + dir := t.TempDir() + // CETEMPúblico format: countword + os.WriteFile(filepath.Join(dir, "cetempublico-freq.tsv"), []byte("500000\tgrande\n"), 0644) + + if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil { + t.Fatal(err) + } + + if f := queryFreq(t, db, "grande"); f == 0 { + t.Error("expected non-zero frequency for 'grande' with reversed columns") + } +} + +func TestCETEMPublico_InsertsMissingWords(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "pt_50k.txt"), []byte("que 5000000\n"), 0644) + + if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil { + t.Fatal(err) + } + + // "que" was not seeded — loader should have inserted it + if f := queryFreq(t, db, "que"); f == 0 { + t.Error("expected non-zero frequency for auto-inserted word 'que'") + } +} + +func TestCETEMPublico_ISO88591(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + seedWord(t, db, "não", "pt-PT") + + dir := t.TempDir() + // ISO-8859-1: "5000\tnão\n" where ã = 0xE3 (single byte) + os.WriteFile(filepath.Join(dir, "cetempublico-freq.tsv"), + []byte{0x35, 0x30, 0x30, 0x30, 0x09, 0x6E, 0xE3, 0x6F, 0x0A}, 0644) + + if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil { + t.Fatal(err) + } + + if f := queryFreq(t, db, "não"); f == 0 { + t.Error("expected non-zero frequency for ISO-8859-1 encoded 'não'") + } +} + +func TestCETEMPublico_FiltersJunk(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "pt_50k.txt"), + []byte("word1 5000\n. 90000\n, 80000\nboa 3000\n"), 0644) + + if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil { + t.Fatal(err) + } + + var count int + db.QueryRow("SELECT COUNT(*) FROM words WHERE lang = 'pt-PT'").Scan(&count) + if count != 1 { + t.Errorf("expected 1 word (boa only), got %d", count) + } +} + +func TestCETEMPublico_NoFileSkips(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + if err := (CETEMPublicoLoader{}).Load(db, t.TempDir()); err != nil { + t.Errorf("expected graceful skip with no file, got: %v", err) + } +} + +func TestCETEMPublico_FrequencyScaling(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + seedWord(t, db, "raro", "pt-PT") + seedWord(t, db, "comum", "pt-PT") + + dir := t.TempDir() + // "comum" has much higher frequency than "raro" + os.WriteFile(filepath.Join(dir, "pt_50k.txt"), + []byte("comum 5000000\nraro 50\n"), 0644) + + if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil { + t.Fatal(err) + } + + comum := queryFreq(t, db, "comum") + raro := queryFreq(t, db, "raro") + + if comum <= raro { + t.Errorf("expected 'comum' (%d) > 'raro' (%d)", comum, raro) + } + if comum > 10000 { + t.Errorf("frequency should cap at 10000, got %d", comum) + } +} diff --git a/internal/loader/wiktionary_test.go b/internal/loader/wiktionary_test.go index f51793a..218b10f 100644 --- a/internal/loader/wiktionary_test.go +++ b/internal/loader/wiktionary_test.go @@ -166,6 +166,66 @@ func TestHasTag(t *testing.T) { } } +func TestSenseLevelSynonyms(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('happy', 'en')"); err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + // Synonyms nested inside senses, not top-level + jsonl := `{"word":"happy","pos":"adj","lang_code":"en","senses":[{"glosses":["feeling joy"],"tags":[],"synonyms":[{"word":"joyful"},{"word":"glad"}],"antonyms":[{"word":"sad"}]}],"synonyms":[],"antonyms":[],"translations":[]}` + "\n" + path := writeJSONL(t, dir, "test.jsonl", jsonl) + + if err := loadWiktionary(db, path, "en"); err != nil { + t.Fatal(err) + } + + var synCount int + if err := db.QueryRow("SELECT COUNT(*) FROM synonyms").Scan(&synCount); err != nil { + t.Fatal(err) + } + if synCount != 2 { + t.Errorf("expected 2 sense-level synonyms, got %d", synCount) + } + + var antCount int + if err := db.QueryRow("SELECT COUNT(*) FROM antonyms").Scan(&antCount); err != nil { + t.Fatal(err) + } + if antCount != 1 { + t.Errorf("expected 1 sense-level antonym, got %d", antCount) + } +} + +func TestTopAndSenseLevelSynonymsMerge(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('big', 'en')"); err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + // Both top-level and sense-level synonyms + jsonl := `{"word":"big","pos":"adj","lang_code":"en","senses":[{"glosses":["of great size"],"tags":[],"synonyms":[{"word":"huge"}]}],"synonyms":[{"word":"large"}],"antonyms":[],"translations":[]}` + "\n" + path := writeJSONL(t, dir, "test.jsonl", jsonl) + + if err := loadWiktionary(db, path, "en"); err != nil { + t.Fatal(err) + } + + var synCount int + if err := db.QueryRow("SELECT COUNT(*) FROM synonyms").Scan(&synCount); err != nil { + t.Fatal(err) + } + if synCount != 2 { + t.Errorf("expected 2 synonyms (1 top-level + 1 sense-level), got %d", synCount) + } +} + func TestHasBrazilTag(t *testing.T) { tests := []struct { tags []string