Prune ghost words from affix expansion and stop frequency loader creating words

- Add PruneOrphanWords post-import step that deletes words with no
  definitions, synonyms, translations, pronunciations, etymology,
  synsets, or frequency data
- Remove INSERT path from CETEMPublico loader — frequency data should
  annotate existing words, not create new ones (was inflating pt-PT
  from 136K real words to 718K)
- Add hasLetter filter to hunspell, affix, and dicionario loaders to
  block punctuation entries like ".", ",", "(" from entering the DB
- Move hasLetter helper to scowl.go alongside other shared filters

Impact: en 655K→129K, pt-PT 1.5M→136K, fr 80K→56K (on reimport)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

cmd/dictimport/main.go

@@ -107,8 +107,8 @@ func main() {
 		)
 	}
 
-	// Difficulty scorer runs last, after all word data is loaded
-	loaders = append(loaders, loader.DifficultyScorer{})
+	// Prune ghost words from affix expansion, then score difficulty
+	loaders = append(loaders, loader.PruneOrphanWords{}, loader.DifficultyScorer{})
 
 	start := time.Now()
 	if err := loader.Run(db, *dataDir, loaders, skipSet); err != nil {

internal/loader/affix.go

@@ -48,7 +48,7 @@ func (a AffixLoader) Load(db *sql.DB, dataDir string) error {
 	seen := make(map[string]struct{})
 	for _, entry := range dicEntries {
 		for _, form := range expandWord(entry.word, entry.flags, rules) {
-			if containsDigit(form) || containsSpace(form) {
+			if !hasLetter(form) || containsDigit(form) || containsSpace(form) {
 				continue
 			}
 			if _, dup := seen[form]; dup {

internal/loader/cetempublico.go

@@ -9,7 +9,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"unicode"
 	"unicode/utf8"
 )
 
@@ -54,12 +53,6 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 	}
 	defer stmtUpdate.Close()
 
-	stmtInsert, err := tx.Prepare(`
-		INSERT OR IGNORE INTO words (word, lang, frequency) VALUES (?, 'pt-PT', ?)`)
-	if err != nil {
-		return fmt.Errorf("cetempublico: prepare insert: %w", err)
-	}
-	defer stmtInsert.Close()
 
 	// Read entire file to detect encoding before parsing.
 	// The file is typically <20 MB so this is fine.
@@ -90,7 +83,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 		}
 	}
 
-	var count, inserted int
+	var count int
 	processLine := func(line string) error {
 		// Support TSV and space-separated, with either column order:
 		//   "word\tcount", "word count", or "count\tword", "count word"
@@ -144,13 +137,6 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 		}
 		if n, _ := res.RowsAffected(); n > 0 {
 			count++
-		} else {
-			// Word not in DB yet — insert it with frequency
-			if _, err := stmtInsert.Exec(word, freq); err != nil {
-				return fmt.Errorf("cetempublico: insert: %w", err)
-			}
-			inserted++
-			count++
 		}
 		return nil
 	}
@@ -175,19 +161,10 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
 	if err := tx.Commit(); err != nil {
 		return fmt.Errorf("cetempublico: commit: %w", err)
 	}
-	slog.Info("cetempublico loaded", "updated_words", count, "inserted_new", inserted)
+	slog.Info("cetempublico loaded", "updated_words", count)
 	return nil
 }
 
-func hasLetter(s string) bool {
-	for _, r := range s {
-		if unicode.IsLetter(r) {
-			return true
-		}
-	}
-	return false
-}
-
 // latin1ToUTF8 converts ISO-8859-1 bytes to a UTF-8 string.
 // Each byte in ISO-8859-1 maps directly to the same Unicode code point.
 func latin1ToUTF8(b []byte) string {

internal/loader/cetempublico_test.go

@@ -66,20 +66,30 @@ func TestCETEMPublico_ReversedColumns(t *testing.T) {
 	}
 }
 
-func TestCETEMPublico_InsertsMissingWords(t *testing.T) {
+func TestCETEMPublico_SkipsUnknownWords(t *testing.T) {
 	db := setupWiktTestDB(t)
 	defer db.Close()
 
+	// Seed one word, leave "desconhecido" unknown
+	seedWord(t, db, "que", "pt-PT")
+
 	dir := t.TempDir()
-	os.WriteFile(filepath.Join(dir, "pt_50k.txt"), []byte("que 5000000\n"), 0644)
+	os.WriteFile(filepath.Join(dir, "pt_50k.txt"), []byte("que 5000000\ndesconhecido 1000\n"), 0644)
 
 	if err := (CETEMPublicoLoader{}).Load(db, dir); err != nil {
 		t.Fatal(err)
 	}
 
-	// "que" was not seeded — loader should have inserted it
+	// "que" was seeded — should get frequency
 	if f := queryFreq(t, db, "que"); f == 0 {
-		t.Error("expected non-zero frequency for auto-inserted word 'que'")
+		t.Error("expected non-zero frequency for known word 'que'")
+	}
+
+	// "desconhecido" was not seeded — should NOT be inserted
+	var count int
+	db.QueryRow("SELECT COUNT(*) FROM words WHERE word = 'desconhecido'").Scan(&count)
+	if count != 0 {
+		t.Error("expected unknown word 'desconhecido' to be skipped, not inserted")
 	}
 }
 
@@ -107,6 +117,11 @@ func TestCETEMPublico_FiltersJunk(t *testing.T) {
 	db := setupWiktTestDB(t)
 	defer db.Close()
 
+	// Seed all candidate words — only "boa" should get frequency
+	seedWord(t, db, "boa", "pt-PT")
+	seedWord(t, db, "word1", "pt-PT") // contains digit — filtered
+	// "." and "," are not valid words so we don't seed them
+
 	dir := t.TempDir()
 	os.WriteFile(filepath.Join(dir, "pt_50k.txt"),
 		[]byte("word1 5000\n. 90000\n, 80000\nboa 3000\n"), 0644)
@@ -115,10 +130,11 @@ func TestCETEMPublico_FiltersJunk(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// Only "boa" should get frequency — word1 has digit, . and , aren't words
 	var count int
-	db.QueryRow("SELECT COUNT(*) FROM words WHERE lang = 'pt-PT'").Scan(&count)
+	db.QueryRow("SELECT COUNT(*) FROM words WHERE lang = 'pt-PT' AND frequency > 0").Scan(&count)
 	if count != 1 {
-		t.Errorf("expected 1 word (boa only), got %d", count)
+		t.Errorf("expected 1 word with frequency (boa only), got %d", count)
 	}
 }
 

internal/loader/dicionario.go

@@ -72,7 +72,7 @@ func (DicionarioLoader) Load(db *sql.DB, dataDir string) error {
 		}
 
 		word := strings.ToLower(strings.TrimSpace(entry.Form.Orth))
-		if word == "" || containsDigit(word) || containsSpace(word) {
+		if word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) {
 			continue
 		}
 

internal/loader/hunspell.go

@@ -57,7 +57,7 @@ func loadHunspell(db *sql.DB, path, lang string) error {
 		word := strings.SplitN(line, "/", 2)[0]
 		word = strings.ToLower(word)
 
-		if containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
+		if !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
 			continue
 		}
 

internal/loader/prune.go

@@ -0,0 +1,36 @@
+package loader
+
+import (
+	"database/sql"
+	"fmt"
+	"log/slog"
+)
+
+// PruneOrphanWords removes words that have no evidence of being real:
+// no definitions, no synonyms, no antonyms, no translations,
+// no pronunciations, no etymology, no synset membership, and zero frequency.
+// These are typically mechanical affix expansion artifacts.
+type PruneOrphanWords struct{}
+
+func (PruneOrphanWords) Name() string { return "prune-orphans" }
+
+func (PruneOrphanWords) Load(db *sql.DB, _ string) error {
+	res, err := db.Exec(`
+		DELETE FROM words
+		WHERE frequency = 0
+		  AND NOT EXISTS (SELECT 1 FROM definitions   d  WHERE d.word_id  = words.id)
+		  AND NOT EXISTS (SELECT 1 FROM synonyms      s  WHERE s.word_id  = words.id)
+		  AND NOT EXISTS (SELECT 1 FROM antonyms      a  WHERE a.word_id  = words.id)
+		  AND NOT EXISTS (SELECT 1 FROM translations  t  WHERE t.word_id  = words.id)
+		  AND NOT EXISTS (SELECT 1 FROM pronunciations p WHERE p.word_id  = words.id)
+		  AND NOT EXISTS (SELECT 1 FROM etymology     e  WHERE e.word_id  = words.id)
+		  AND NOT EXISTS (SELECT 1 FROM word_synsets  ws WHERE ws.word_id = words.id)
+	`)
+	if err != nil {
+		return fmt.Errorf("prune orphans: %w", err)
+	}
+
+	n, _ := res.RowsAffected()
+	slog.Info("pruned orphan words", "deleted", n)
+	return nil
+}

internal/loader/prune_test.go

@@ -0,0 +1,50 @@
+package loader
+
+import (
+	"testing"
+)
+
+func TestPruneOrphanWords(t *testing.T) {
+	db := setupWiktTestDB(t)
+	defer db.Close()
+
+	// Insert a real word with a definition
+	db.Exec("INSERT INTO words (word, lang, frequency) VALUES ('cat', 'en', 500)")
+	db.Exec("INSERT INTO definitions (word_id, pos, gloss, source) SELECT id, 'noun', 'a feline', 'test' FROM words WHERE word='cat'")
+
+	// Insert a word with only frequency (no definitions) — should survive
+	db.Exec("INSERT INTO words (word, lang, frequency) VALUES ('the', 'en', 9999)")
+
+	// Insert a ghost word — no definitions, no frequency, no anything
+	db.Exec("INSERT INTO words (word, lang, frequency) VALUES ('cheeseburgered', 'en', 0)")
+
+	// Insert a word with synonyms but no definitions — should survive
+	db.Exec("INSERT INTO words (word, lang, frequency) VALUES ('glad', 'en', 0)")
+	db.Exec("INSERT INTO synonyms (word_id, synonym, source) SELECT id, 'happy', 'test' FROM words WHERE word='glad'")
+
+	pruner := PruneOrphanWords{}
+	if err := pruner.Load(db, ""); err != nil {
+		t.Fatal(err)
+	}
+
+	var count int
+	db.QueryRow("SELECT COUNT(*) FROM words").Scan(&count)
+	if count != 3 {
+		t.Errorf("expected 3 surviving words (cat, the, glad), got %d", count)
+	}
+
+	// Verify the ghost was deleted
+	var exists bool
+	db.QueryRow("SELECT EXISTS(SELECT 1 FROM words WHERE word='cheeseburgered')").Scan(&exists)
+	if exists {
+		t.Error("expected 'cheeseburgered' to be pruned")
+	}
+
+	// Verify real words survived
+	for _, word := range []string{"cat", "the", "glad"} {
+		db.QueryRow("SELECT EXISTS(SELECT 1 FROM words WHERE word=?)", word).Scan(&exists)
+		if !exists {
+			t.Errorf("expected '%s' to survive pruning", word)
+		}
+	}
+}

internal/loader/scowl.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"unicode"
 )
 
 type SCOWLLoader struct{}
@@ -148,3 +149,12 @@ func containsNonLatin(s string) bool {
 	}
 	return false
 }
+
+func hasLetter(s string) bool {
+	for _, r := range s {
+		if unicode.IsLetter(r) {
+			return true
+		}
+	}
+	return false
+}