aff583dc28
- Add PruneOrphanWords post-import step that deletes words with no
definitions, synonyms, translations, pronunciations, etymology,
synsets, or frequency data
- Remove INSERT path from CETEMPublico loader — frequency data should
annotate existing words, not create new ones (was inflating pt-PT
from 136K real words to 718K)
- Add hasLetter filter to hunspell, affix, and dicionario loaders to
block punctuation entries like ".", ",", "(" from entering the DB
- Move hasLetter helper to scowl.go alongside other shared filters
Impact: en 655K→129K, pt-PT 1.5M→136K, fr 80K→56K (on reimport)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
79 lines
1.7 KiB
Go
79 lines
1.7 KiB
Go
package loader
|
|
|
|
import (
|
|
"bufio"
|
|
"database/sql"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"strings"
|
|
)
|
|
|
|
type HunspellLoader struct {
|
|
Lang string // "fr" or "pt-PT"
|
|
FileName string // e.g. "fr_FR/fr.dic" or "pt_PT/pt_PT.dic"
|
|
}
|
|
|
|
func (h HunspellLoader) Name() string { return "hunspell-" + h.Lang }
|
|
|
|
func (h HunspellLoader) Load(db *sql.DB, dataDir string) error {
|
|
path := dataDir + "/" + h.FileName
|
|
return loadHunspell(db, path, h.Lang)
|
|
}
|
|
|
|
func loadHunspell(db *sql.DB, path, lang string) error {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return fmt.Errorf("hunspell: open %s: %w", path, err)
|
|
}
|
|
defer f.Close()
|
|
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
return fmt.Errorf("hunspell: begin tx: %w", err)
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
stmt, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang) VALUES (?, ?)")
|
|
if err != nil {
|
|
return fmt.Errorf("hunspell: prepare: %w", err)
|
|
}
|
|
defer stmt.Close()
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
|
|
// Skip first line (word count)
|
|
if scanner.Scan() {
|
|
// discard
|
|
}
|
|
|
|
var count int
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line == "" {
|
|
continue
|
|
}
|
|
// Split on "/" to get base word
|
|
word := strings.SplitN(line, "/", 2)[0]
|
|
word = strings.ToLower(word)
|
|
|
|
if !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
|
|
continue
|
|
}
|
|
|
|
if _, err := stmt.Exec(word, lang); err != nil {
|
|
return fmt.Errorf("hunspell: insert: %w", err)
|
|
}
|
|
count++
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return fmt.Errorf("hunspell: scan: %w", err)
|
|
}
|
|
|
|
if err := tx.Commit(); err != nil {
|
|
return fmt.Errorf("hunspell: commit: %w", err)
|
|
}
|
|
slog.Info("hunspell loaded", "lang", lang, "words", count)
|
|
return nil
|
|
}
|