Fix Portuguese coverage gaps: synonyms, translations, frequency, gerunds
- Extract sense-level synonyms/antonyms from Wiktionary (not just top-level), dramatically improving Portuguese synonym coverage - Add bidirectional translation lookup so pt→en queries find en→pt entries stored by English Wiktionary loader, with reverse index for performance - Fix frequency loader: auto-detect column order (count\tword vs word\tcount), auto-detect ISO-8859-1 encoding and convert to UTF-8, insert missing words instead of only updating existing ones - Switch WordNet download from 3.1 to 3.0 to match WOLF/OMW synset offsets - Add OpenSubtitles-derived Portuguese frequency list (hermitdave/FrequencyWords) - Increase affix expansion cap from 30→80 forms per word for Portuguese gerunds Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -27,9 +27,15 @@ type wiktEntry struct {
|
||||
Word string `json:"word"`
|
||||
POS string `json:"pos"`
|
||||
LangCode string `json:"lang_code"`
|
||||
Senses []struct {
|
||||
Glosses []string `json:"glosses"`
|
||||
Tags []string `json:"tags"`
|
||||
Senses []struct {
|
||||
Glosses []string `json:"glosses"`
|
||||
Tags []string `json:"tags"`
|
||||
Synonyms []struct {
|
||||
Word string `json:"word"`
|
||||
} `json:"synonyms"`
|
||||
Antonyms []struct {
|
||||
Word string `json:"word"`
|
||||
} `json:"antonyms"`
|
||||
} `json:"senses"`
|
||||
Synonyms []struct {
|
||||
Word string `json:"word"`
|
||||
@@ -188,7 +194,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
|
||||
|
||||
pos := wiktPOSMap[entry.POS]
|
||||
|
||||
// Process senses
|
||||
// Process senses (definitions + sense-level synonyms/antonyms)
|
||||
for _, sense := range entry.Senses {
|
||||
if hasTag(sense.Tags, "form-of", "alt-of") {
|
||||
continue
|
||||
@@ -208,9 +214,33 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
|
||||
}
|
||||
defCount++
|
||||
}
|
||||
|
||||
// Sense-level synonyms
|
||||
for _, syn := range sense.Synonyms {
|
||||
synWord := strings.ToLower(syn.Word)
|
||||
if synWord == "" || containsDigit(synWord) || containsSpace(synWord) {
|
||||
continue
|
||||
}
|
||||
if _, err := batch.stmtSyn.Exec(synWord, word, lang); err != nil {
|
||||
return fmt.Errorf("wiktionary: insert syn: %w", err)
|
||||
}
|
||||
synCount++
|
||||
}
|
||||
|
||||
// Sense-level antonyms
|
||||
for _, ant := range sense.Antonyms {
|
||||
antWord := strings.ToLower(ant.Word)
|
||||
if antWord == "" || containsDigit(antWord) || containsSpace(antWord) {
|
||||
continue
|
||||
}
|
||||
if _, err := batch.stmtAnt.Exec(antWord, word, lang); err != nil {
|
||||
return fmt.Errorf("wiktionary: insert ant: %w", err)
|
||||
}
|
||||
antCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Process synonyms
|
||||
// Process top-level synonyms
|
||||
for _, syn := range entry.Synonyms {
|
||||
synWord := strings.ToLower(syn.Word)
|
||||
if synWord == "" || containsDigit(synWord) || containsSpace(synWord) {
|
||||
@@ -222,7 +252,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
|
||||
synCount++
|
||||
}
|
||||
|
||||
// Process antonyms
|
||||
// Process top-level antonyms
|
||||
for _, ant := range entry.Antonyms {
|
||||
antWord := strings.ToLower(ant.Word)
|
||||
if antWord == "" || containsDigit(antWord) || containsSpace(antWord) {
|
||||
|
||||
Reference in New Issue
Block a user