diff --git a/internal/loader/affix.go b/internal/loader/affix.go index de0f2d8..a718302 100644 --- a/internal/loader/affix.go +++ b/internal/loader/affix.go @@ -48,7 +48,7 @@ func (a AffixLoader) Load(db *sql.DB, dataDir string) error { seen := make(map[string]struct{}) for _, entry := range dicEntries { for _, form := range expandWord(entry.word, entry.flags, rules) { - if !hasLetter(form) || containsDigit(form) || containsSpace(form) { + if isJunkWord(form) { continue } if _, dup := seen[form]; dup { @@ -194,7 +194,7 @@ func parseDicFile(path string) ([]dicEntry, error) { flags = parts[1] } - if containsDigit(word) || containsNonLatin(word) { + if isJunkWord(word) { continue } diff --git a/internal/loader/cetempublico.go b/internal/loader/cetempublico.go index 80de0d2..4baa34d 100644 --- a/internal/loader/cetempublico.go +++ b/internal/loader/cetempublico.go @@ -106,7 +106,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error { freqStr = strings.TrimSpace(fields[1]) } - if word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) { + if isJunkWord(word) { return nil } diff --git a/internal/loader/cmudict.go b/internal/loader/cmudict.go index 46f5e62..41e6edb 100644 --- a/internal/loader/cmudict.go +++ b/internal/loader/cmudict.go @@ -89,7 +89,7 @@ func (CMUDictLoader) Load(db *sql.DB, dataDir string) error { continue } - if containsDigit(word) || containsSpace(word) { + if isJunkWord(word) { continue } diff --git a/internal/loader/dicionario.go b/internal/loader/dicionario.go index a4d908a..6adbfbc 100644 --- a/internal/loader/dicionario.go +++ b/internal/loader/dicionario.go @@ -72,7 +72,7 @@ func (DicionarioLoader) Load(db *sql.DB, dataDir string) error { } word := strings.ToLower(strings.TrimSpace(entry.Form.Orth)) - if word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) { + if isJunkWord(word) { continue } diff --git a/internal/loader/hunspell.go b/internal/loader/hunspell.go index ab8311e..f04b223 100644 --- a/internal/loader/hunspell.go +++ b/internal/loader/hunspell.go @@ -57,7 +57,7 @@ func loadHunspell(db *sql.DB, path, lang string) error { word := strings.SplitN(line, "/", 2)[0] word = strings.ToLower(word) - if !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) { + if isJunkWord(word) { continue } diff --git a/internal/loader/omw.go b/internal/loader/omw.go index e4791b6..3ffe58b 100644 --- a/internal/loader/omw.go +++ b/internal/loader/omw.go @@ -103,7 +103,7 @@ func (OMWLoader) Load(db *sql.DB, dataDir string) error { continue } - if word == "" || containsDigit(word) || containsSpace(word) { + if isJunkWord(word) { continue } diff --git a/internal/loader/scowl.go b/internal/loader/scowl.go index 847edea..7e92b83 100644 --- a/internal/loader/scowl.go +++ b/internal/loader/scowl.go @@ -158,3 +158,8 @@ func hasLetter(s string) bool { } return false } + +// isJunkWord returns true if the word should be rejected from the dictionary. +func isJunkWord(word string) bool { + return word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) +} diff --git a/internal/loader/subtlex.go b/internal/loader/subtlex.go index 90fc0ed..21276fd 100644 --- a/internal/loader/subtlex.go +++ b/internal/loader/subtlex.go @@ -80,7 +80,7 @@ func (SubtlexLoader) Load(db *sql.DB, dataDir string) error { } word := strings.ToLower(strings.TrimSpace(fields[0])) - if word == "" || containsDigit(word) || containsSpace(word) { + if isJunkWord(word) { continue } diff --git a/internal/loader/wiktionary.go b/internal/loader/wiktionary.go index 8577b40..4c9adcd 100644 --- a/internal/loader/wiktionary.go +++ b/internal/loader/wiktionary.go @@ -188,7 +188,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error { } word := strings.ToLower(entry.Word) - if containsDigit(word) || containsSpace(word) { + if isJunkWord(word) { continue } diff --git a/internal/loader/wolf.go b/internal/loader/wolf.go index 7dbb7d3..c78aa8a 100644 --- a/internal/loader/wolf.go +++ b/internal/loader/wolf.go @@ -129,7 +129,7 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error { var words []string for _, lit := range synset.Literals { w := strings.ToLower(strings.TrimSpace(lit.Value)) - if w != "" && !containsDigit(w) && !containsSpace(w) { + if !isJunkWord(w) { words = append(words, w) } }