Add Phase 2+3 features: antonyms, backing, pronunciation, etymology, difficulty, affix expansion
New endpoints: /antonyms, /backing, /pronunciation, /etymology with difficulty scoring on /random. Cross-language synset backing links French/Portuguese words to English equivalents via WordNet 3.0 synset IDs (matching WOLF and OMW offsets). New loaders: Hunspell affix expansion (fr, pt-PT), English programmatic inflector, CMU Pronouncing Dictionary, SUBTLEX-US frequency, CETEMPúblico frequency, Open Multilingual Wordnet (Portuguese), and SQL-based difficulty scoring. Schema v2 adds tables: antonyms, synsets, word_synsets, pronunciations, etymology with migration support for existing databases. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
prosolis
@@ -15,6 +15,24 @@ type WordNetLoader struct{}
|
||||
|
||||
func (WordNetLoader) Name() string { return "wordnet" }
|
||||
|
||||
// ssTypeMap maps WordNet ss_type codes to POS and synset pos suffix.
|
||||
var ssTypeMap = map[string]string{
|
||||
"n": "noun",
|
||||
"v": "verb",
|
||||
"a": "adjective",
|
||||
"s": "adjective", // satellite adjective
|
||||
"r": "adverb",
|
||||
}
|
||||
|
||||
// ssTypeSuffix maps WordNet ss_type to synset ID suffix character.
|
||||
var ssTypeSuffix = map[string]string{
|
||||
"n": "n",
|
||||
"v": "v",
|
||||
"a": "a",
|
||||
"s": "a", // satellite adjectives share 'a' namespace
|
||||
"r": "r",
|
||||
}
|
||||
|
||||
func (WordNetLoader) Load(db *sql.DB, dataDir string) error {
|
||||
posFiles := map[string]string{
|
||||
"data.noun": "noun",
|
||||
@@ -45,111 +63,219 @@ func (WordNetLoader) Load(db *sql.DB, dataDir string) error {
|
||||
}
|
||||
defer stmtSyn.Close()
|
||||
|
||||
var defCount, synCount int
|
||||
stmtAnt, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO antonyms (word_id, antonym, source)
|
||||
SELECT id, ?, 'wordnet' FROM words WHERE word = ? AND lang = 'en'`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: prepare ant: %w", err)
|
||||
}
|
||||
defer stmtAnt.Close()
|
||||
|
||||
stmtSynset, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO synsets (synset_id, pos) VALUES (?, ?)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: prepare synset: %w", err)
|
||||
}
|
||||
defer stmtSynset.Close()
|
||||
|
||||
stmtWordSynset, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO word_synsets (word_id, synset_id, source)
|
||||
SELECT w.id, s.id, 'wordnet'
|
||||
FROM words w, synsets s
|
||||
WHERE w.word = ? AND w.lang = 'en' AND s.synset_id = ?`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: prepare word_synset: %w", err)
|
||||
}
|
||||
defer stmtWordSynset.Close()
|
||||
|
||||
var defCount, synCount, antCount, synsetCount int
|
||||
|
||||
for file, pos := range posFiles {
|
||||
path := filepath.Join(dataDir, "wordnet", "dict", file)
|
||||
d, s, err := loadWordNetFile(stmtDef, stmtSyn, path, pos)
|
||||
d, s, a, sc, err := loadWordNetFile(stmtDef, stmtSyn, stmtAnt, stmtSynset, stmtWordSynset, path, pos)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: %s: %w", file, err)
|
||||
}
|
||||
defCount += d
|
||||
synCount += s
|
||||
antCount += a
|
||||
synsetCount += sc
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("wordnet: commit: %w", err)
|
||||
}
|
||||
slog.Info("wordnet loaded", "definitions", defCount, "synonyms", synCount)
|
||||
slog.Info("wordnet loaded", "definitions", defCount, "synonyms", synCount, "antonyms", antCount, "synsets", synsetCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadWordNetFile(stmtDef, stmtSyn *sql.Stmt, path, pos string) (int, int, error) {
|
||||
func loadWordNetFile(stmtDef, stmtSyn, stmtAnt, stmtSynset, stmtWordSynset *sql.Stmt, path, pos string) (int, int, int, int, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var defCount, synCount int
|
||||
// Single pass: insert defs/syns/synsets as we go, collect lightweight
|
||||
// antonym pointers + synset->words map for deferred antonym resolution.
|
||||
type deferredAnt struct {
|
||||
srcWord string
|
||||
targetSynsetKey string
|
||||
tgtWordIdx int
|
||||
}
|
||||
|
||||
synsetWords := make(map[string][]string) // only stores word lists, not full parse data
|
||||
var pendingAnts []deferredAnt
|
||||
var defCount, synCount, antCount, synsetCount int
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
buf := make([]byte, 0, 64*1024)
|
||||
scanner.Buffer(buf, 1024*1024)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
// Skip header lines (start with two spaces)
|
||||
if strings.HasPrefix(line, " ") {
|
||||
continue
|
||||
}
|
||||
|
||||
words, gloss := parseWordNetLine(line)
|
||||
if gloss == "" || len(words) == 0 {
|
||||
parsed := parseWordNetLine(line)
|
||||
if parsed.gloss == "" || len(parsed.words) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Insert definitions for each word
|
||||
for _, w := range words {
|
||||
if _, err := stmtDef.Exec(pos, gloss, w); err != nil {
|
||||
return 0, 0, err
|
||||
// Build synset->words map (needed for antonym resolution)
|
||||
if parsed.synsetID != "" {
|
||||
synsetWords[parsed.synsetID] = parsed.words
|
||||
|
||||
if _, err := stmtSynset.Exec(parsed.synsetID, pos); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
synsetCount++
|
||||
|
||||
for _, w := range parsed.words {
|
||||
if _, err := stmtWordSynset.Exec(w, parsed.synsetID); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Insert definitions
|
||||
for _, w := range parsed.words {
|
||||
if _, err := stmtDef.Exec(pos, parsed.gloss, w); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
defCount++
|
||||
}
|
||||
|
||||
// Insert synonym pairs (skip words with underscores)
|
||||
// Insert synonym pairs
|
||||
var cleanWords []string
|
||||
for _, w := range words {
|
||||
for _, w := range parsed.words {
|
||||
if !strings.Contains(w, "_") {
|
||||
cleanWords = append(cleanWords, w)
|
||||
}
|
||||
}
|
||||
for i, w1 := range cleanWords {
|
||||
for j, w2 := range cleanWords {
|
||||
if i == j {
|
||||
continue
|
||||
if i != j {
|
||||
if _, err := stmtSyn.Exec(w2, w1); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
synCount++
|
||||
}
|
||||
if _, err := stmtSyn.Exec(w2, w1); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
synCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Collect antonym pointers for deferred resolution
|
||||
for _, ap := range parsed.antonymPtrs {
|
||||
if ap.srcWordIdx < 1 || ap.srcWordIdx > len(parsed.words) {
|
||||
continue
|
||||
}
|
||||
srcWord := strings.ToLower(parsed.words[ap.srcWordIdx-1])
|
||||
if strings.Contains(srcWord, "_") {
|
||||
continue
|
||||
}
|
||||
pendingAnts = append(pendingAnts, deferredAnt{
|
||||
srcWord: srcWord,
|
||||
targetSynsetKey: ap.targetSynsetKey,
|
||||
tgtWordIdx: ap.tgtWordIdx,
|
||||
})
|
||||
}
|
||||
}
|
||||
return defCount, synCount, scanner.Err()
|
||||
if err := scanner.Err(); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
|
||||
// Resolve deferred antonyms now that all synsets are mapped
|
||||
for _, da := range pendingAnts {
|
||||
tgtWords, ok := synsetWords[da.targetSynsetKey]
|
||||
if !ok || da.tgtWordIdx < 1 || da.tgtWordIdx > len(tgtWords) {
|
||||
continue
|
||||
}
|
||||
tgtWord := strings.ToLower(tgtWords[da.tgtWordIdx-1])
|
||||
if strings.Contains(tgtWord, "_") || tgtWord == da.srcWord {
|
||||
continue
|
||||
}
|
||||
if _, err := stmtAnt.Exec(tgtWord, da.srcWord); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
if _, err := stmtAnt.Exec(da.srcWord, tgtWord); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
antCount += 2
|
||||
}
|
||||
|
||||
return defCount, synCount, antCount, synsetCount, nil
|
||||
}
|
||||
|
||||
func parseWordNetLine(line string) ([]string, string) {
|
||||
type wordNetParsed struct {
|
||||
words []string
|
||||
gloss string
|
||||
synsetID string
|
||||
// antonymPtrs: each entry is {targetSynsetKey, sourceWordIdx, targetWordIdx}
|
||||
antonymPtrs []antonymPtr
|
||||
}
|
||||
|
||||
type antonymPtr struct {
|
||||
targetSynsetKey string // "offset-pos" e.g. "00123456-a"
|
||||
srcWordIdx int // 1-based word index in source synset
|
||||
tgtWordIdx int // 1-based word index in target synset
|
||||
}
|
||||
|
||||
func parseWordNetLine(line string) wordNetParsed {
|
||||
// Format: synset_offset lex_filenum ss_type w_cnt word lex_id [word lex_id...] p_cnt [ptr...] | gloss
|
||||
glossIdx := strings.Index(line, "| ")
|
||||
if glossIdx == -1 {
|
||||
return nil, ""
|
||||
return wordNetParsed{}
|
||||
}
|
||||
|
||||
gloss := strings.TrimSpace(line[glossIdx+2:])
|
||||
// Take gloss before first ";" (drops usage examples)
|
||||
if semiIdx := strings.Index(gloss, ";"); semiIdx != -1 {
|
||||
gloss = strings.TrimSpace(gloss[:semiIdx])
|
||||
}
|
||||
if gloss == "" {
|
||||
return nil, ""
|
||||
return wordNetParsed{}
|
||||
}
|
||||
|
||||
dataPart := line[:glossIdx]
|
||||
fields := strings.Fields(dataPart)
|
||||
if len(fields) < 6 {
|
||||
return nil, ""
|
||||
return wordNetParsed{}
|
||||
}
|
||||
|
||||
synsetOffset := fields[0]
|
||||
ssType := fields[2]
|
||||
posSuffix := ssTypeSuffix[ssType]
|
||||
synsetID := ""
|
||||
if posSuffix != "" {
|
||||
synsetID = synsetOffset + "-" + posSuffix
|
||||
}
|
||||
|
||||
// fields[3] = w_cnt (hex)
|
||||
wc, err := strconv.ParseInt(fields[3], 16, 0)
|
||||
if err != nil {
|
||||
slog.Debug("wordnet: bad word count", "field", fields[3], "error", err)
|
||||
return nil, ""
|
||||
return wordNetParsed{}
|
||||
}
|
||||
wordCount := int(wc)
|
||||
if wordCount <= 0 || wordCount > 100 {
|
||||
slog.Debug("wordnet: unreasonable word count", "count", wordCount)
|
||||
return nil, ""
|
||||
return wordNetParsed{}
|
||||
}
|
||||
|
||||
var words []string
|
||||
@@ -157,5 +283,41 @@ func parseWordNetLine(line string) ([]string, string) {
|
||||
w := strings.ToLower(fields[4+i*2])
|
||||
words = append(words, w)
|
||||
}
|
||||
return words, gloss
|
||||
|
||||
// Parse pointers
|
||||
ptrStart := 4 + wordCount*2
|
||||
var antPtrs []antonymPtr
|
||||
if ptrStart < len(fields) {
|
||||
pc, err := strconv.Atoi(fields[ptrStart])
|
||||
if err == nil {
|
||||
for i := 0; i < pc; i++ {
|
||||
base := ptrStart + 1 + i*4
|
||||
if base+3 >= len(fields) {
|
||||
break
|
||||
}
|
||||
if fields[base] != "!" {
|
||||
continue
|
||||
}
|
||||
tgtOffset := fields[base+1]
|
||||
tgtPosChar := fields[base+2]
|
||||
srcTgt := fields[base+3]
|
||||
if len(srcTgt) != 4 {
|
||||
continue
|
||||
}
|
||||
srcIdx, e1 := strconv.ParseInt(srcTgt[0:2], 16, 0)
|
||||
tgtIdx, e2 := strconv.ParseInt(srcTgt[2:4], 16, 0)
|
||||
if e1 != nil || e2 != nil {
|
||||
continue
|
||||
}
|
||||
tgtKey := tgtOffset + "-" + tgtPosChar
|
||||
antPtrs = append(antPtrs, antonymPtr{
|
||||
targetSynsetKey: tgtKey,
|
||||
srcWordIdx: int(srcIdx),
|
||||
tgtWordIdx: int(tgtIdx),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return wordNetParsed{words: words, gloss: gloss, synsetID: synsetID, antonymPtrs: antPtrs}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user