Consolidate word filtering into shared isJunkWord helper
Replace copy-pasted hasLetter/containsDigit/containsSpace/containsNonLatin checks across 9 loaders with a single isJunkWord() call. SCOWL and CEDICT keep their own filtering for valid reasons (SCOWL rejects hyphens/apostrophes, CEDICT handles non-Latin Chinese characters). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -48,7 +48,7 @@ func (a AffixLoader) Load(db *sql.DB, dataDir string) error {
|
||||
seen := make(map[string]struct{})
|
||||
for _, entry := range dicEntries {
|
||||
for _, form := range expandWord(entry.word, entry.flags, rules) {
|
||||
if !hasLetter(form) || containsDigit(form) || containsSpace(form) {
|
||||
if isJunkWord(form) {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[form]; dup {
|
||||
@@ -194,7 +194,7 @@ func parseDicFile(path string) ([]dicEntry, error) {
|
||||
flags = parts[1]
|
||||
}
|
||||
|
||||
if containsDigit(word) || containsNonLatin(word) {
|
||||
if isJunkWord(word) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user