drwily/dreamdict
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Consolidate word filtering into shared isJunkWord helper

Browse Source 
Replace copy-pasted hasLetter/containsDigit/containsSpace/containsNonLatin
checks across 9 loaders with a single isJunkWord() call. SCOWL and CEDICT
keep their own filtering for valid reasons (SCOWL rejects hyphens/apostrophes,
CEDICT handles non-Latin Chinese characters).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
prosolis
2026-04-03 20:01:02 -07:00
parent aff583dc28
commit 28ec11ba4e
10 changed files with 15 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/loader/affix.go
View File

@@ -48,7 +48,7 @@ func (a AffixLoader) Load(db *sql.DB, dataDir string) error {
seen := make(map[string]struct{}) seen := make(map[string]struct{})
for _, entry := range dicEntries { for _, entry := range dicEntries {
for _, form := range expandWord(entry.word, entry.flags, rules) { for _, form := range expandWord(entry.word, entry.flags, rules) {
if !hasLetter(form) || containsDigit(form) || containsSpace(form) { if isJunkWord(form) {
continue continue
} }
if _, dup := seen[form]; dup { if _, dup := seen[form]; dup {
@@ -194,7 +194,7 @@ func parseDicFile(path string) ([]dicEntry, error) {
flags = parts[1] flags = parts[1]
} }
if containsDigit(word) || containsNonLatin(word) { if isJunkWord(word) {
continue continue
} }

2
internal/loader/cetempublico.go
View File

@@ -106,7 +106,7 @@ func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
freqStr = strings.TrimSpace(fields[1]) freqStr = strings.TrimSpace(fields[1])
} }
if word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) { if isJunkWord(word) {
return nil return nil
} }

2
internal/loader/cmudict.go
View File

@@ -89,7 +89,7 @@ func (CMUDictLoader) Load(db *sql.DB, dataDir string) error {
continue continue
} }
if containsDigit(word) || containsSpace(word) { if isJunkWord(word) {
continue continue
} }

2
internal/loader/dicionario.go
View File

@@ -72,7 +72,7 @@ func (DicionarioLoader) Load(db *sql.DB, dataDir string) error {
} }
word := strings.ToLower(strings.TrimSpace(entry.Form.Orth)) word := strings.ToLower(strings.TrimSpace(entry.Form.Orth))
if word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) { if isJunkWord(word) {
continue continue
} }

2
internal/loader/hunspell.go
View File

@@ -57,7 +57,7 @@ func loadHunspell(db *sql.DB, path, lang string) error {
word := strings.SplitN(line, "/", 2)[0] word := strings.SplitN(line, "/", 2)[0]
word = strings.ToLower(word) word = strings.ToLower(word)
if !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word) { if isJunkWord(word) {
continue continue
} }

2
internal/loader/omw.go
View File

@@ -103,7 +103,7 @@ func (OMWLoader) Load(db *sql.DB, dataDir string) error {
continue continue
} }
if word == "" || containsDigit(word) || containsSpace(word) { if isJunkWord(word) {
continue continue
} }

5
internal/loader/scowl.go
View File

@@ -158,3 +158,8 @@ func hasLetter(s string) bool {
} }
return false return false
} }
// isJunkWord returns true if the word should be rejected from the dictionary.
func isJunkWord(word string) bool {
return word == "" || !hasLetter(word) || containsDigit(word) || containsSpace(word) || containsNonLatin(word)
}

2
internal/loader/subtlex.go
View File

@@ -80,7 +80,7 @@ func (SubtlexLoader) Load(db *sql.DB, dataDir string) error {
} }
word := strings.ToLower(strings.TrimSpace(fields[0])) word := strings.ToLower(strings.TrimSpace(fields[0]))
if word == "" || containsDigit(word) || containsSpace(word) { if isJunkWord(word) {
continue continue
} }

2
internal/loader/wiktionary.go
View File

@@ -188,7 +188,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
} }
word := strings.ToLower(entry.Word) word := strings.ToLower(entry.Word)
if containsDigit(word) || containsSpace(word) { if isJunkWord(word) {
continue continue
} }

2
internal/loader/wolf.go
View File

@@ -129,7 +129,7 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error {
var words []string var words []string
for _, lit := range synset.Literals { for _, lit := range synset.Literals {
w := strings.ToLower(strings.TrimSpace(lit.Value)) w := strings.ToLower(strings.TrimSpace(lit.Value))
if w != "" && !containsDigit(w) && !containsSpace(w) { if !isJunkWord(w) {
words = append(words, w) words = append(words, w)
} }
} }