diff --git a/cmd/dictimport/main.go b/cmd/dictimport/main.go index 9a36e49..46e52af 100644 --- a/cmd/dictimport/main.go +++ b/cmd/dictimport/main.go @@ -107,8 +107,8 @@ func main() { ) } - // Difficulty scorer runs last, after all word data is loaded - loaders = append(loaders, loader.DifficultyScorer{}) + // Prune ghost words from affix expansion, then score difficulty + loaders = append(loaders, loader.PruneOrphanWords{}, loader.DifficultyScorer{}) start := time.Now() if err := loader.Run(db, *dataDir, loaders, skipSet); err != nil { diff --git a/internal/loader/affix.go b/internal/loader/affix.go index 2c44b2a..de0f2d8 100644 --- a/internal/loader/affix.go +++ b/internal/loader/affix.go @@ -48,7 +48,7 @@ func (a AffixLoader) Load(db *sql.DB, dataDir string) error { seen := make(map[string]struct{}) for _, entry := range dicEntries { for _, form := range expandWord(entry.word, entry.flags, rules) { - if containsDigit(form) || containsSpace(form) { + if !hasLetter(form) || containsDigit(form) || containsSpace(form) { continue } if _, dup := seen[form]; dup { diff --git a/internal/loader/cetempublico.go b/internal/loader/cetempublico.go index b6d62c1..80de0d2 100644 --- a/internal/loader/cetempublico.go +++ b/internal/loader/cetempublico.go @@ -9,7 +9,6 @@ import ( "path/filepath" "strconv" "strings" - "unicode" "unicode/utf8" ) @@ -54,12 +53,6 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { } defer stmtUpdate.Close() - stmtInsert, err := tx.Prepare(` - INSERT OR IGNORE INTO words (word, lang, frequency) VALUES (?, 'pt-PT', ?)`) - if err != nil { - return fmt.Errorf("cetempublico: prepare insert: %w", err) - } - defer stmtInsert.Close() // Read entire file to detect encoding before parsing. // The file is typically <20 MB so this is fine. @@ -90,7 +83,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { } } - var count, inserted int + var count int processLine := func(line string) error { // Support TSV and space-separated, with either column order: // "word\tcount", "word count", or "count\tword", "count word" @@ -144,13 +137,6 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { } if n, _ := res.RowsAffected(); n > 0 { count++ - } else { - // Word not in DB yet — insert it with frequency - if _, err := stmtInsert.Exec(word, freq); err != nil { - return fmt.Errorf("cetempublico: insert: %w", err) - } - inserted++ - count++ } return nil } @@ -175,19 +161,10 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { if err := tx.Commit(); err != nil { return fmt.Errorf("cetempublico: commit: %w", err) } - slog.Info("cetempublico loaded", "updated_words", count, "inserted_new", inserted) + slog.Info("cetempublico loaded", "updated_words", count) return nil } -func hasLetter(s string) bool { - for _, r := range s { - if unicode.IsLetter(r) { - return true - } - } - return false -} - // latin1ToUTF8 converts ISO-8859-1 bytes to a UTF-8 string. // Each byte in ISO-8859-1 maps directly to the same Unicode code point. func latin1ToUTF8(b []byte) string { diff --git a/internal/loader/cetempublico_test.go b/internal/loader/cetempublico_test.go index 5199ecb..5956a92 100644 --- a/internal/loader/cetempublico_test.go +++ b/internal/loader/cetempublico_test.go @@ -66,20 +66,30 @@ func TestCETEMPublico_ReversedColumns(t *testing.T) { } } -func TestCETEMPublico_InsertsMissingWords(t *testing.T) { +func TestCETEMPublico_SkipsUnknownWords(t *testing.T) { db := setupWiktTestDB(t) defer db.Close() + // Seed one word, leave "desconhecido" unknown + seedWord(t, db, "que", "pt-PT") + dir := t.TempDir() - os.WriteFile(filepath.Join(dir, "pt_50k.txt"), []byte("que 5000000\n"), 0644) + os.WriteFile(filepath.Join(dir, "pt_50k.txt"), []byte("que 5000000\ndesconhecido 1000\n"), 0644) if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil { t.Fatal(err) } - // "que" was not seeded — loader should have inserted it + // "que" was seeded — should get frequency if f := queryFreq(t, db, "que"); f == 0 { - t.Error("expected non-zero frequency for auto-inserted word 'que'") + t.Error("expected non-zero frequency for known word 'que'") + } + + // "desconhecido" was not seeded — should NOT be inserted + var count int + db.QueryRow("SELECT COUNT(*) FROM words WHERE word = 'desconhecido'").Scan(&count) + if count != 0 { + t.Error("expected unknown word 'desconhecido' to be skipped, not inserted") } } @@ -107,6 +117,11 @@ func TestCETEMPublico_FiltersJunk(t *testing.T) { db := setupWiktTestDB(t) defer db.Close() + // Seed all candidate words — only "boa" should get frequency + seedWord(t, db, "boa", "pt-PT") + seedWord(t, db, "word1", "pt-PT") // contains digit — filtered + // "." and "," are not valid words so we don't seed them + dir := t.TempDir() os.WriteFile(filepath.Join(dir, "pt_50k.txt"), []byte("word1 5000\n. 90000\n, 80000\nboa 3000\n"), 0644) @@ -115,10 +130,11 @@ func TestCETEMPublico_FiltersJunk(t *testing.T) { t.Fatal(err) } + // Only "boa" should get frequency — word1 has digit, . and , aren't words var count int - db.QueryRow("SELECT COUNT(*) FROM words WHERE lang = 'pt-PT'").Scan(&count) + db.QueryRow("SELECT COUNT(*) FROM words WHERE lang = 'pt-PT' AND frequency > 0").Scan(&count) if count != 1 { - t.Errorf("expected 1 word (boa only), got %d", count) + t.Errorf("expected 1 word with frequency (boa only), got %d", count) } } diff --git a/internal/loader/dicionario.go b/internal/loader/dicionario.go index 93fb7ee..a4d908a 100644 --- a/internal/loader/dicionario.go +++ b/internal/loader/dicionario.go @@ -72,7 +72,7 @@ func (DicionarioLoader) Load(db *sql.DB, dataDir string) error { } word := strings.ToLower(strings.TrimSpace(entry.Form.Orth)) - if word == "" || containsDigit(word) || containsSpace(word) { + if word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) { continue } diff --git a/internal/loader/hunspell.go b/internal/loader/hunspell.go index 8ecc588..ab8311e 100644 --- a/internal/loader/hunspell.go +++ b/internal/loader/hunspell.go @@ -57,7 +57,7 @@ func loadHunspell(db *sql.DB, path, lang string) error { word := strings.SplitN(line, "/", 2)[0] word = strings.ToLower(word) - if containsDigit(word) || containsSpace(word) || containsNonLatin(word) { + if !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) { continue } diff --git a/internal/loader/prune.go b/internal/loader/prune.go new file mode 100644 index 0000000..f99ee0f --- /dev/null +++ b/internal/loader/prune.go @@ -0,0 +1,36 @@ +package loader + +import ( + "database/sql" + "fmt" + "log/slog" +) + +// PruneOrphanWords removes words that have no evidence of being real: +// no definitions, no synonyms, no antonyms, no translations, +// no pronunciations, no etymology, no synset membership, and zero frequency. +// These are typically mechanical affix expansion artifacts. +type PruneOrphanWords struct{} + +func (PruneOrphanWords) Name() string { return "prune-orphans" } + +func (PruneOrphanWords) Load(db *sql.DB, _ string) error { + res, err := db.Exec(` + DELETE FROM words + WHERE frequency = 0 + AND NOT EXISTS (SELECT 1 FROM definitions d WHERE d.word_id = words.id) + AND NOT EXISTS (SELECT 1 FROM synonyms s WHERE s.word_id = words.id) + AND NOT EXISTS (SELECT 1 FROM antonyms a WHERE a.word_id = words.id) + AND NOT EXISTS (SELECT 1 FROM translations t WHERE t.word_id = words.id) + AND NOT EXISTS (SELECT 1 FROM pronunciations p WHERE p.word_id = words.id) + AND NOT EXISTS (SELECT 1 FROM etymology e WHERE e.word_id = words.id) + AND NOT EXISTS (SELECT 1 FROM word_synsets ws WHERE ws.word_id = words.id) + `) + if err != nil { + return fmt.Errorf("prune orphans: %w", err) + } + + n, _ := res.RowsAffected() + slog.Info("pruned orphan words", "deleted", n) + return nil +} diff --git a/internal/loader/prune_test.go b/internal/loader/prune_test.go new file mode 100644 index 0000000..dc869a6 --- /dev/null +++ b/internal/loader/prune_test.go @@ -0,0 +1,50 @@ +package loader + +import ( + "testing" +) + +func TestPruneOrphanWords(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + // Insert a real word with a definition + db.Exec("INSERT INTO words (word, lang, frequency) VALUES ('cat', 'en', 500)") + db.Exec("INSERT INTO definitions (word_id, pos, gloss, source) SELECT id, 'noun', 'a feline', 'test' FROM words WHERE word='cat'") + + // Insert a word with only frequency (no definitions) — should survive + db.Exec("INSERT INTO words (word, lang, frequency) VALUES ('the', 'en', 9999)") + + // Insert a ghost word — no definitions, no frequency, no anything + db.Exec("INSERT INTO words (word, lang, frequency) VALUES ('cheeseburgered', 'en', 0)") + + // Insert a word with synonyms but no definitions — should survive + db.Exec("INSERT INTO words (word, lang, frequency) VALUES ('glad', 'en', 0)") + db.Exec("INSERT INTO synonyms (word_id, synonym, source) SELECT id, 'happy', 'test' FROM words WHERE word='glad'") + + pruner := PruneOrphanWords{} + if err := pruner.Load(db, ""); err != nil { + t.Fatal(err) + } + + var count int + db.QueryRow("SELECT COUNT(*) FROM words").Scan(&count) + if count != 3 { + t.Errorf("expected 3 surviving words (cat, the, glad), got %d", count) + } + + // Verify the ghost was deleted + var exists bool + db.QueryRow("SELECT EXISTS(SELECT 1 FROM words WHERE word='cheeseburgered')").Scan(&exists) + if exists { + t.Error("expected 'cheeseburgered' to be pruned") + } + + // Verify real words survived + for _, word := range []string{"cat", "the", "glad"} { + db.QueryRow("SELECT EXISTS(SELECT 1 FROM words WHERE word=?)", word).Scan(&exists) + if !exists { + t.Errorf("expected '%s' to survive pruning", word) + } + } +} diff --git a/internal/loader/scowl.go b/internal/loader/scowl.go index 37a92ad..847edea 100644 --- a/internal/loader/scowl.go +++ b/internal/loader/scowl.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "unicode" ) type SCOWLLoader struct{} @@ -148,3 +149,12 @@ func containsNonLatin(s string) bool { } return false } + +func hasLetter(s string) bool { + for _, r := range s { + if unicode.IsLetter(r) { + return true + } + } + return false +}