Prune ghost words from affix expansion and stop frequency loader creating words

- Add PruneOrphanWords post-import step that deletes words with no
  definitions, synonyms, translations, pronunciations, etymology,
  synsets, or frequency data
- Remove INSERT path from CETEMPublico loader — frequency data should
  annotate existing words, not create new ones (was inflating pt-PT
  from 136K real words to 718K)
- Add hasLetter filter to hunspell, affix, and dicionario loaders to
  block punctuation entries like ".", ",", "(" from entering the DB
- Move hasLetter helper to scowl.go alongside other shared filters

Impact: en 655K→129K, pt-PT 1.5M→136K, fr 80K→56K (on reimport)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
prosolis
2026-04-03 19:55:37 -07:00
parent 2045cc2ada
commit aff583dc28
9 changed files with 125 additions and 36 deletions

36
internal/loader/prune.go Normal file
View File

@@ -0,0 +1,36 @@
package loader
import (
"database/sql"
"fmt"
"log/slog"
)
// PruneOrphanWords removes words that have no evidence of being real:
// no definitions, no synonyms, no antonyms, no translations,
// no pronunciations, no etymology, no synset membership, and zero frequency.
// These are typically mechanical affix expansion artifacts.
type PruneOrphanWords struct{}
func (PruneOrphanWords) Name() string { return "prune-orphans" }
func (PruneOrphanWords) Load(db *sql.DB, _ string) error {
res, err := db.Exec(`
DELETE FROM words
WHERE frequency = 0
AND NOT EXISTS (SELECT 1 FROM definitions d WHERE d.word_id = words.id)
AND NOT EXISTS (SELECT 1 FROM synonyms s WHERE s.word_id = words.id)
AND NOT EXISTS (SELECT 1 FROM antonyms a WHERE a.word_id = words.id)
AND NOT EXISTS (SELECT 1 FROM translations t WHERE t.word_id = words.id)
AND NOT EXISTS (SELECT 1 FROM pronunciations p WHERE p.word_id = words.id)
AND NOT EXISTS (SELECT 1 FROM etymology e WHERE e.word_id = words.id)
AND NOT EXISTS (SELECT 1 FROM word_synsets ws WHERE ws.word_id = words.id)
`)
if err != nil {
return fmt.Errorf("prune orphans: %w", err)
}
n, _ := res.RowsAffected()
slog.Info("pruned orphan words", "deleted", n)
return nil
}