Consolidate word filtering into shared isJunkWord helper
Replace copy-pasted hasLetter/containsDigit/containsSpace/containsNonLatin checks across 9 loaders with a single isJunkWord() call. SCOWL and CEDICT keep their own filtering for valid reasons (SCOWL rejects hyphens/apostrophes, CEDICT handles non-Latin Chinese characters). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -129,7 +129,7 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error {
|
||||
var words []string
|
||||
for _, lit := range synset.Literals {
|
||||
w := strings.ToLower(strings.TrimSpace(lit.Value))
|
||||
if w != "" && !containsDigit(w) && !containsSpace(w) {
|
||||
if !isJunkWord(w) {
|
||||
words = append(words, w)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user