Prune ghost words from affix expansion and stop frequency loader creating words
- Add PruneOrphanWords post-import step that deletes words with no
definitions, synonyms, translations, pronunciations, etymology,
synsets, or frequency data
- Remove INSERT path from CETEMPublico loader — frequency data should
annotate existing words, not create new ones (was inflating pt-PT
from 136K real words to 718K)
- Add hasLetter filter to hunspell, affix, and dicionario loaders to
block punctuation entries like ".", ",", "(" from entering the DB
- Move hasLetter helper to scowl.go alongside other shared filters
Impact: en 655K→129K, pt-PT 1.5M→136K, fr 80K→56K (on reimport)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
36
internal/loader/prune.go
Normal file
36
internal/loader/prune.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// PruneOrphanWords removes words that have no evidence of being real:
|
||||
// no definitions, no synonyms, no antonyms, no translations,
|
||||
// no pronunciations, no etymology, no synset membership, and zero frequency.
|
||||
// These are typically mechanical affix expansion artifacts.
|
||||
type PruneOrphanWords struct{}
|
||||
|
||||
func (PruneOrphanWords) Name() string { return "prune-orphans" }
|
||||
|
||||
func (PruneOrphanWords) Load(db *sql.DB, _ string) error {
|
||||
res, err := db.Exec(`
|
||||
DELETE FROM words
|
||||
WHERE frequency = 0
|
||||
AND NOT EXISTS (SELECT 1 FROM definitions d WHERE d.word_id = words.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM synonyms s WHERE s.word_id = words.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM antonyms a WHERE a.word_id = words.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM translations t WHERE t.word_id = words.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM pronunciations p WHERE p.word_id = words.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM etymology e WHERE e.word_id = words.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM word_synsets ws WHERE ws.word_id = words.id)
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prune orphans: %w", err)
|
||||
}
|
||||
|
||||
n, _ := res.RowsAffected()
|
||||
slog.Info("pruned orphan words", "deleted", n)
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user