aff583dc28
- Add PruneOrphanWords post-import step that deletes words with no
definitions, synonyms, translations, pronunciations, etymology,
synsets, or frequency data
- Remove INSERT path from CETEMPublico loader — frequency data should
annotate existing words, not create new ones (was inflating pt-PT
from 136K real words to 718K)
- Add hasLetter filter to hunspell, affix, and dicionario loaders to
block punctuation entries like ".", ",", "(" from entering the DB
- Move hasLetter helper to scowl.go alongside other shared filters
Impact: en 655K→129K, pt-PT 1.5M→136K, fr 80K→56K (on reimport)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
37 lines
1.2 KiB
Go
37 lines
1.2 KiB
Go
package loader
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"log/slog"
|
|
)
|
|
|
|
// PruneOrphanWords removes words that have no evidence of being real:
|
|
// no definitions, no synonyms, no antonyms, no translations,
|
|
// no pronunciations, no etymology, no synset membership, and zero frequency.
|
|
// These are typically mechanical affix expansion artifacts.
|
|
type PruneOrphanWords struct{}
|
|
|
|
func (PruneOrphanWords) Name() string { return "prune-orphans" }
|
|
|
|
func (PruneOrphanWords) Load(db *sql.DB, _ string) error {
|
|
res, err := db.Exec(`
|
|
DELETE FROM words
|
|
WHERE frequency = 0
|
|
AND NOT EXISTS (SELECT 1 FROM definitions d WHERE d.word_id = words.id)
|
|
AND NOT EXISTS (SELECT 1 FROM synonyms s WHERE s.word_id = words.id)
|
|
AND NOT EXISTS (SELECT 1 FROM antonyms a WHERE a.word_id = words.id)
|
|
AND NOT EXISTS (SELECT 1 FROM translations t WHERE t.word_id = words.id)
|
|
AND NOT EXISTS (SELECT 1 FROM pronunciations p WHERE p.word_id = words.id)
|
|
AND NOT EXISTS (SELECT 1 FROM etymology e WHERE e.word_id = words.id)
|
|
AND NOT EXISTS (SELECT 1 FROM word_synsets ws WHERE ws.word_id = words.id)
|
|
`)
|
|
if err != nil {
|
|
return fmt.Errorf("prune orphans: %w", err)
|
|
}
|
|
|
|
n, _ := res.RowsAffected()
|
|
slog.Info("pruned orphan words", "deleted", n)
|
|
return nil
|
|
}
|