Add Phase 2+3 features: antonyms, backing, pronunciation, etymology, difficulty, affix expansion
New endpoints: /antonyms, /backing, /pronunciation, /etymology with difficulty scoring on /random. Cross-language synset backing links French/Portuguese words to English equivalents via WordNet 3.0 synset IDs (matching WOLF and OMW offsets). New loaders: Hunspell affix expansion (fr, pt-PT), English programmatic inflector, CMU Pronouncing Dictionary, SUBTLEX-US frequency, CETEMPúblico frequency, Open Multilingual Wordnet (Portuguese), and SQL-based difficulty scoring. Schema v2 adds tables: antonyms, synsets, word_synsets, pronunciations, etymology with migration support for existing databases. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
prosolis
151
internal/dictionary/bench_test.go
Normal file
151
internal/dictionary/bench_test.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package dictionary
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func setupBenchDB(b *testing.B) (*Dictionary, *sql.DB) {
|
||||
b.Helper()
|
||||
db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(on)")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
if err := BootstrapSchema(db); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
// Seed a realistic-ish dataset: 1000 words with definitions, synonyms, translations
|
||||
exec := func(q string, args ...any) {
|
||||
if _, err := db.Exec(q, args...); err != nil {
|
||||
b.Fatalf("seed: %s: %v", q, err)
|
||||
}
|
||||
}
|
||||
|
||||
exec("INSERT INTO meta (key, value) VALUES ('schema_version', '1')")
|
||||
exec("INSERT INTO meta (key, value) VALUES ('imported_at', '2025-01-01T00:00:00Z')")
|
||||
|
||||
poses := []string{"noun", "verb", "adjective", "adverb"}
|
||||
langs := []string{"en", "fr", "pt-PT"}
|
||||
|
||||
for i := 1; i <= 1000; i++ {
|
||||
lang := langs[i%len(langs)]
|
||||
pos := poses[i%len(poses)]
|
||||
word := fmt.Sprintf("word%04d", i)
|
||||
exec("INSERT INTO words (id, word, lang, pos, frequency) VALUES (?, ?, ?, ?, ?)",
|
||||
i, word, lang, pos, i%100)
|
||||
|
||||
// 2 definitions per word
|
||||
exec("INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (?, ?, ?, 'wordnet', 10)",
|
||||
i, pos, fmt.Sprintf("definition A of %s", word))
|
||||
exec("INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (?, ?, ?, 'wiktionary', 20)",
|
||||
i, pos, fmt.Sprintf("definition B of %s", word))
|
||||
|
||||
// 2 synonyms
|
||||
exec("INSERT INTO synonyms (word_id, synonym, source) VALUES (?, ?, 'wordnet')",
|
||||
i, fmt.Sprintf("syn1_%s", word))
|
||||
exec("INSERT INTO synonyms (word_id, synonym, source) VALUES (?, ?, 'wordnet')",
|
||||
i, fmt.Sprintf("syn2_%s", word))
|
||||
|
||||
// Translation for en words
|
||||
if lang == "en" {
|
||||
exec("INSERT INTO translations (word_id, translation, target_lang, source) VALUES (?, ?, 'fr', 'wiktionary')",
|
||||
i, fmt.Sprintf("fr_%s", word))
|
||||
}
|
||||
}
|
||||
|
||||
return NewFromDB(db), db
|
||||
}
|
||||
|
||||
func BenchmarkIsValidWord(b *testing.B) {
|
||||
d, db := setupBenchDB(b)
|
||||
defer db.Close()
|
||||
|
||||
b.Run("hit", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
d.IsValidWord("word0003", "en")
|
||||
}
|
||||
})
|
||||
b.Run("miss", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
d.IsValidWord("nonexistent", "en")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkRandomWord(b *testing.B) {
|
||||
d, db := setupBenchDB(b)
|
||||
defer db.Close()
|
||||
|
||||
b.Run("no_filter", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
d.RandomWord("en", Options{})
|
||||
}
|
||||
})
|
||||
b.Run("with_filters", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
d.RandomWord("en", Options{POS: "noun", MinLength: 4, MaxLength: 10})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkDefine(b *testing.B) {
|
||||
d, db := setupBenchDB(b)
|
||||
defer db.Close()
|
||||
|
||||
b.Run("with_defs", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
d.Define("word0003", "en")
|
||||
}
|
||||
})
|
||||
b.Run("no_defs", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
d.Define("nonexistent", "en")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkSynonyms(b *testing.B) {
|
||||
d, db := setupBenchDB(b)
|
||||
defer db.Close()
|
||||
|
||||
b.Run("with_syns", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
d.Synonyms("word0003", "en")
|
||||
}
|
||||
})
|
||||
b.Run("no_syns", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
d.Synonyms("nonexistent", "en")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkTranslate(b *testing.B) {
|
||||
d, db := setupBenchDB(b)
|
||||
defer db.Close()
|
||||
|
||||
b.Run("with_trans", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
d.Translate("word0003", "en", "fr")
|
||||
}
|
||||
})
|
||||
b.Run("no_trans", func(b *testing.B) {
|
||||
for b.Loop() {
|
||||
d.Translate("nonexistent", "en", "fr")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkWordCount(b *testing.B) {
|
||||
d, db := setupBenchDB(b)
|
||||
defer db.Close()
|
||||
|
||||
for b.Loop() {
|
||||
d.WordCount()
|
||||
}
|
||||
}
|
||||
@@ -3,11 +3,12 @@ package dictionary
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const schemaVersion = "1"
|
||||
const schemaVersion = "2"
|
||||
|
||||
const schemaSQL = `
|
||||
CREATE TABLE IF NOT EXISTS words (
|
||||
@@ -15,7 +16,8 @@ CREATE TABLE IF NOT EXISTS words (
|
||||
word TEXT NOT NULL,
|
||||
lang TEXT NOT NULL,
|
||||
pos TEXT,
|
||||
frequency INTEGER DEFAULT 0,
|
||||
frequency INTEGER DEFAULT 0,
|
||||
difficulty REAL DEFAULT NULL,
|
||||
UNIQUE(word, lang)
|
||||
);
|
||||
|
||||
@@ -61,6 +63,53 @@ CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS antonyms (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
|
||||
antonym TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
UNIQUE(word_id, antonym)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_antonyms_word_id ON antonyms(word_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS synsets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
synset_id TEXT NOT NULL UNIQUE,
|
||||
pos TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS word_synsets (
|
||||
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
|
||||
synset_id INTEGER NOT NULL REFERENCES synsets(id) ON DELETE CASCADE,
|
||||
source TEXT NOT NULL,
|
||||
PRIMARY KEY (word_id, synset_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_word_synsets_word_id ON word_synsets(word_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_word_synsets_synset_id ON word_synsets(synset_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pronunciations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
|
||||
format TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
UNIQUE(word_id, format, source)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pronunciations_word_id ON pronunciations(word_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS etymology (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
|
||||
text TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
UNIQUE(word_id, source)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_etymology_word_id ON etymology(word_id);
|
||||
`
|
||||
|
||||
func openDB(dbPath string) (*sql.DB, error) {
|
||||
@@ -77,11 +126,19 @@ func openDB(dbPath string) (*sql.DB, error) {
|
||||
}
|
||||
|
||||
func openReadOnlyDB(dbPath string) (*sql.DB, error) {
|
||||
db, err := sql.Open("sqlite", dbPath+"?mode=ro&_pragma=journal_mode(wal)&_pragma=foreign_keys(on)")
|
||||
db, err := sql.Open("sqlite", dbPath+
|
||||
"?mode=ro"+
|
||||
"&_pragma=journal_mode(wal)"+
|
||||
"&_pragma=foreign_keys(on)"+
|
||||
"&_pragma=cache_size(-64000)"+
|
||||
"&_pragma=mmap_size(268435456)"+
|
||||
"&_pragma=temp_store(memory)")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: open db (ro): %w", err)
|
||||
}
|
||||
// No MaxOpenConns limit — WAL mode supports concurrent reads
|
||||
// Pin to one connection so pragmas are consistent. WAL mode still
|
||||
// allows concurrent reads from the same connection in Go's model.
|
||||
db.SetMaxOpenConns(1)
|
||||
if err := db.Ping(); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("dictionary: ping db: %w", err)
|
||||
@@ -94,5 +151,25 @@ func BootstrapSchema(db *sql.DB) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("dictionary: bootstrap schema: %w", err)
|
||||
}
|
||||
|
||||
// Schema migrations for databases created before v2.
|
||||
// ALTER TABLE ADD COLUMN is a no-op if the column already exists in SQLite
|
||||
// when we catch the "duplicate column" error.
|
||||
migrations := []string{
|
||||
"ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL",
|
||||
}
|
||||
for _, m := range migrations {
|
||||
if _, err := db.Exec(m); err != nil {
|
||||
// Ignore "duplicate column name" — means migration already applied
|
||||
if !isDuplicateColumn(err) {
|
||||
return fmt.Errorf("dictionary: migration: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isDuplicateColumn(err error) bool {
|
||||
return err != nil && strings.Contains(err.Error(), "duplicate column")
|
||||
}
|
||||
|
||||
@@ -22,10 +22,12 @@ func ValidLang(lang string) bool {
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
MinLength int
|
||||
MaxLength int
|
||||
POS string // "", "noun", "verb", "adjective", "adverb"
|
||||
MinFrequency int // 0 = no filter
|
||||
MinLength int
|
||||
MaxLength int
|
||||
POS string // "", "noun", "verb", "adjective", "adverb"
|
||||
MinFrequency int // 0 = no filter
|
||||
MinDifficulty float64 // 0.0 = no filter
|
||||
MaxDifficulty float64 // 0.0 = no filter
|
||||
}
|
||||
|
||||
type Definition struct {
|
||||
@@ -35,6 +37,18 @@ type Definition struct {
|
||||
Priority int `json:"priority"`
|
||||
}
|
||||
|
||||
type EnglishEquivalent struct {
|
||||
Word string `json:"word"`
|
||||
Definition string `json:"definition"`
|
||||
Synset string `json:"synset"`
|
||||
}
|
||||
|
||||
type Pronunciation struct {
|
||||
Format string `json:"format"`
|
||||
Value string `json:"value"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
type Dictionary struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
@@ -89,30 +89,30 @@ func TestRandomWord(t *testing.T) {
|
||||
d := NewFromDB(db)
|
||||
|
||||
// Basic random word
|
||||
word, err := d.RandomWord("en", Options{})
|
||||
result, err := d.RandomWord("en", Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("RandomWord: %v", err)
|
||||
}
|
||||
if word == "" {
|
||||
if result.Word == "" {
|
||||
t.Error("RandomWord returned empty string")
|
||||
}
|
||||
|
||||
// With POS filter
|
||||
word, err = d.RandomWord("en", Options{POS: "adjective"})
|
||||
result, err = d.RandomWord("en", Options{POS: "adjective"})
|
||||
if err != nil {
|
||||
t.Fatalf("RandomWord with POS: %v", err)
|
||||
}
|
||||
if word != "happy" {
|
||||
t.Errorf("RandomWord(adjective) = %q, want happy", word)
|
||||
if result.Word != "happy" {
|
||||
t.Errorf("RandomWord(adjective) = %q, want happy", result.Word)
|
||||
}
|
||||
|
||||
// With length filters
|
||||
word, err = d.RandomWord("en", Options{MinLength: 4, MaxLength: 5})
|
||||
result, err = d.RandomWord("en", Options{MinLength: 4, MaxLength: 5})
|
||||
if err != nil {
|
||||
t.Fatalf("RandomWord with length: %v", err)
|
||||
}
|
||||
if len(word) < 4 || len(word) > 5 {
|
||||
t.Errorf("RandomWord length %d not in [4,5]", len(word))
|
||||
if len(result.Word) < 4 || len(result.Word) > 5 {
|
||||
t.Errorf("RandomWord length %d not in [4,5]", len(result.Word))
|
||||
}
|
||||
|
||||
// No match
|
||||
|
||||
@@ -18,8 +18,13 @@ func (d *Dictionary) IsValidWord(word, lang string) (bool, error) {
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (d *Dictionary) RandomWord(lang string, opts Options) (string, error) {
|
||||
query := "SELECT word FROM words WHERE lang = ?"
|
||||
type RandomResult struct {
|
||||
Word string `json:"word"`
|
||||
Difficulty *float64 `json:"difficulty,omitempty"`
|
||||
}
|
||||
|
||||
func (d *Dictionary) RandomWord(lang string, opts Options) (RandomResult, error) {
|
||||
query := "SELECT word, difficulty FROM words WHERE lang = ?"
|
||||
args := []any{lang}
|
||||
|
||||
if opts.POS != "" {
|
||||
@@ -38,18 +43,30 @@ func (d *Dictionary) RandomWord(lang string, opts Options) (string, error) {
|
||||
query += " AND frequency >= ?"
|
||||
args = append(args, opts.MinFrequency)
|
||||
}
|
||||
if opts.MinDifficulty > 0 {
|
||||
query += " AND difficulty >= ?"
|
||||
args = append(args, opts.MinDifficulty)
|
||||
}
|
||||
if opts.MaxDifficulty > 0 {
|
||||
query += " AND difficulty <= ?"
|
||||
args = append(args, opts.MaxDifficulty)
|
||||
}
|
||||
|
||||
query += " ORDER BY RANDOM() LIMIT 1"
|
||||
|
||||
var word string
|
||||
err := d.db.QueryRow(query, args...).Scan(&word)
|
||||
var result RandomResult
|
||||
var diff sql.NullFloat64
|
||||
err := d.db.QueryRow(query, args...).Scan(&result.Word, &diff)
|
||||
if err == sql.ErrNoRows {
|
||||
return "", ErrNoMatch
|
||||
return RandomResult{}, ErrNoMatch
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("dictionary: random word: %w", err)
|
||||
return RandomResult{}, fmt.Errorf("dictionary: random word: %w", err)
|
||||
}
|
||||
return word, nil
|
||||
if diff.Valid {
|
||||
result.Difficulty = &diff.Float64
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (d *Dictionary) Define(word, lang string) ([]Definition, error) {
|
||||
@@ -175,6 +192,44 @@ func (d *Dictionary) Frequency(word, lang string) (int, error) {
|
||||
return freq, nil
|
||||
}
|
||||
|
||||
// FrequencyBatch returns frequency scores for multiple words in a language.
|
||||
// Words not found or without frequency data are returned with a score of 0.
|
||||
func (d *Dictionary) FrequencyBatch(words []string, lang string) (map[string]int, error) {
|
||||
if len(words) == 0 {
|
||||
return map[string]int{}, nil
|
||||
}
|
||||
|
||||
placeholders := make([]string, len(words))
|
||||
args := make([]any, 0, len(words)+1)
|
||||
args = append(args, lang)
|
||||
for i, w := range words {
|
||||
placeholders[i] = "?"
|
||||
args = append(args, strings.ToLower(w))
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(
|
||||
"SELECT word, COALESCE(frequency, 0) FROM words WHERE lang = ? AND word IN (%s)",
|
||||
strings.Join(placeholders, ","),
|
||||
)
|
||||
|
||||
rows, err := d.db.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: frequency batch: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
result := make(map[string]int, len(words))
|
||||
for rows.Next() {
|
||||
var word string
|
||||
var freq int
|
||||
if err := rows.Scan(&word, &freq); err != nil {
|
||||
return nil, fmt.Errorf("dictionary: frequency batch scan: %w", err)
|
||||
}
|
||||
result[word] = freq
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (d *Dictionary) Meta(key string) (string, error) {
|
||||
var val string
|
||||
err := d.db.QueryRow("SELECT value FROM meta WHERE key = ?", key).Scan(&val)
|
||||
@@ -187,6 +242,143 @@ func (d *Dictionary) Meta(key string) (string, error) {
|
||||
return val, nil
|
||||
}
|
||||
|
||||
func (d *Dictionary) Antonyms(word, lang string) ([]string, error) {
|
||||
rows, err := d.db.Query(`
|
||||
SELECT DISTINCT a.antonym
|
||||
FROM antonyms a
|
||||
JOIN words w ON w.id = a.word_id
|
||||
WHERE w.word = ? AND w.lang = ?
|
||||
ORDER BY a.antonym`,
|
||||
strings.ToLower(word), lang,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: antonyms: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var ants []string
|
||||
for rows.Next() {
|
||||
var a string
|
||||
if err := rows.Scan(&a); err != nil {
|
||||
return nil, fmt.Errorf("dictionary: antonyms scan: %w", err)
|
||||
}
|
||||
ants = append(ants, a)
|
||||
}
|
||||
if ants == nil {
|
||||
ants = []string{}
|
||||
}
|
||||
return ants, rows.Err()
|
||||
}
|
||||
|
||||
// EnglishBacking returns English words that share a synset with the given
|
||||
// non-English word, providing semantic equivalents with WordNet definitions.
|
||||
func (d *Dictionary) EnglishBacking(word, lang string) ([]EnglishEquivalent, error) {
|
||||
// Find synsets for this word, then find English words in those synsets,
|
||||
// along with their WordNet definitions.
|
||||
rows, err := d.db.Query(`
|
||||
SELECT DISTINCT ew.word,
|
||||
COALESCE(
|
||||
(SELECT d.gloss FROM definitions d
|
||||
WHERE d.word_id = ew.id AND d.source = 'wordnet' AND d.pos = s.pos
|
||||
ORDER BY d.priority ASC LIMIT 1),
|
||||
(SELECT d.gloss FROM definitions d
|
||||
WHERE d.word_id = ew.id AND d.source = 'wordnet'
|
||||
ORDER BY d.priority ASC LIMIT 1),
|
||||
''
|
||||
) AS gloss,
|
||||
s.synset_id
|
||||
FROM words w
|
||||
JOIN word_synsets ws ON ws.word_id = w.id
|
||||
JOIN synsets s ON s.id = ws.synset_id
|
||||
JOIN word_synsets ews ON ews.synset_id = s.id
|
||||
JOIN words ew ON ew.id = ews.word_id AND ew.lang = 'en'
|
||||
WHERE w.word = ? AND w.lang = ?
|
||||
ORDER BY s.synset_id, ew.word`,
|
||||
strings.ToLower(word), lang,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: english backing: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []EnglishEquivalent
|
||||
seen := make(map[string]bool)
|
||||
for rows.Next() {
|
||||
var eq EnglishEquivalent
|
||||
if err := rows.Scan(&eq.Word, &eq.Definition, &eq.Synset); err != nil {
|
||||
return nil, fmt.Errorf("dictionary: english backing scan: %w", err)
|
||||
}
|
||||
key := eq.Word + "|" + eq.Synset
|
||||
if !seen[key] {
|
||||
seen[key] = true
|
||||
results = append(results, eq)
|
||||
}
|
||||
}
|
||||
if results == nil {
|
||||
results = []EnglishEquivalent{}
|
||||
}
|
||||
|
||||
// Fallback to translations table if no synset matches
|
||||
if len(results) == 0 {
|
||||
trans, err := d.Translate(word, lang, "en")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, t := range trans {
|
||||
results = append(results, EnglishEquivalent{Word: t})
|
||||
}
|
||||
}
|
||||
|
||||
return results, rows.Err()
|
||||
}
|
||||
|
||||
func (d *Dictionary) Pronunciation(word, lang string) ([]Pronunciation, error) {
|
||||
rows, err := d.db.Query(`
|
||||
SELECT p.format, p.value, p.source
|
||||
FROM pronunciations p
|
||||
JOIN words w ON w.id = p.word_id
|
||||
WHERE w.word = ? AND w.lang = ?
|
||||
ORDER BY p.source, p.format`,
|
||||
strings.ToLower(word), lang,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: pronunciation: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var prons []Pronunciation
|
||||
for rows.Next() {
|
||||
var p Pronunciation
|
||||
if err := rows.Scan(&p.Format, &p.Value, &p.Source); err != nil {
|
||||
return nil, fmt.Errorf("dictionary: pronunciation scan: %w", err)
|
||||
}
|
||||
prons = append(prons, p)
|
||||
}
|
||||
if prons == nil {
|
||||
prons = []Pronunciation{}
|
||||
}
|
||||
return prons, rows.Err()
|
||||
}
|
||||
|
||||
func (d *Dictionary) Etymology(word, lang string) (string, error) {
|
||||
var text string
|
||||
err := d.db.QueryRow(`
|
||||
SELECT e.text
|
||||
FROM etymology e
|
||||
JOIN words w ON w.id = e.word_id
|
||||
WHERE w.word = ? AND w.lang = ?
|
||||
LIMIT 1`,
|
||||
strings.ToLower(word), lang,
|
||||
).Scan(&text)
|
||||
if err == sql.ErrNoRows {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("dictionary: etymology: %w", err)
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (d *Dictionary) DefCount() (map[string]int, error) {
|
||||
rows, err := d.db.Query(`
|
||||
SELECT w.lang, COUNT(*)
|
||||
|
||||
537
internal/loader/affix.go
Normal file
537
internal/loader/affix.go
Normal file
@@ -0,0 +1,537 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// AffixLoader expands inflected forms from Hunspell .aff/.dic pairs
|
||||
// and inserts them into the words table alongside base forms.
|
||||
type AffixLoader struct {
|
||||
Lang string // "en", "fr", "pt-PT"
|
||||
DicFile string // path relative to dataDir, e.g. "fr_FR/fr.dic"
|
||||
AffFile string // path relative to dataDir, e.g. "fr_FR/fr.aff"
|
||||
}
|
||||
|
||||
func (a AffixLoader) Name() string { return "affix-" + a.Lang }
|
||||
|
||||
func (a AffixLoader) Load(db *sql.DB, dataDir string) error {
|
||||
affPath := filepath.Join(dataDir, a.AffFile)
|
||||
dicPath := filepath.Join(dataDir, a.DicFile)
|
||||
|
||||
if _, err := os.Stat(affPath); os.IsNotExist(err) {
|
||||
slog.Warn("affix: .aff file not found, skipping", "path", affPath)
|
||||
return nil
|
||||
}
|
||||
if _, err := os.Stat(dicPath); os.IsNotExist(err) {
|
||||
slog.Warn("affix: .dic file not found, skipping", "path", dicPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
rules, err := parseAffFile(affPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("affix: parse aff: %w", err)
|
||||
}
|
||||
|
||||
dicEntries, err := parseDicFile(dicPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("affix: parse dic: %w", err)
|
||||
}
|
||||
|
||||
var forms []string
|
||||
seen := make(map[string]struct{})
|
||||
for _, entry := range dicEntries {
|
||||
for _, form := range expandWord(entry.word, entry.flags, rules) {
|
||||
if containsDigit(form) || containsSpace(form) {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[form]; dup {
|
||||
continue
|
||||
}
|
||||
seen[form] = struct{}{}
|
||||
forms = append(forms, form)
|
||||
}
|
||||
}
|
||||
|
||||
if err := bulkInsertWords(db, forms, a.Lang, 250); err != nil {
|
||||
return fmt.Errorf("affix: %w", err)
|
||||
}
|
||||
slog.Info("affix loaded", "lang", a.Lang, "expanded_forms", len(forms))
|
||||
return nil
|
||||
}
|
||||
|
||||
type affixRule struct {
|
||||
ruleType string // "SFX" or "PFX"
|
||||
strip string
|
||||
affix string
|
||||
matchers []condMatcher // pre-parsed condition
|
||||
}
|
||||
|
||||
type affixRuleSet struct {
|
||||
rules []affixRule
|
||||
combinable bool
|
||||
ruleType string
|
||||
}
|
||||
|
||||
type dicEntry struct {
|
||||
word string
|
||||
flags string
|
||||
}
|
||||
|
||||
func parseAffFile(path string) (map[string]*affixRuleSet, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
rules := make(map[string]*affixRuleSet)
|
||||
scanner := bufio.NewScanner(f)
|
||||
buf := make([]byte, 0, 64*1024)
|
||||
scanner.Buffer(buf, 1024*1024)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 4 {
|
||||
continue
|
||||
}
|
||||
|
||||
ruleType := fields[0]
|
||||
if ruleType != "SFX" && ruleType != "PFX" {
|
||||
continue
|
||||
}
|
||||
|
||||
flag := fields[1]
|
||||
|
||||
// Header line: SFX flag combinable count
|
||||
if _, err := strconv.Atoi(fields[len(fields)-1]); err == nil && len(fields) == 4 {
|
||||
combinable := fields[2] == "Y"
|
||||
if _, exists := rules[flag]; !exists {
|
||||
rules[flag] = &affixRuleSet{
|
||||
combinable: combinable,
|
||||
ruleType: ruleType,
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Rule line: SFX flag strip affix [condition]
|
||||
if len(fields) < 4 {
|
||||
continue
|
||||
}
|
||||
|
||||
strip := fields[2]
|
||||
if strip == "0" {
|
||||
strip = ""
|
||||
}
|
||||
affix := fields[3]
|
||||
if affix == "0" {
|
||||
affix = ""
|
||||
}
|
||||
|
||||
// Strip any continuation flags from affix (e.g., "ing/S" -> "ing")
|
||||
if slashIdx := strings.Index(affix, "/"); slashIdx != -1 {
|
||||
affix = affix[:slashIdx]
|
||||
}
|
||||
|
||||
condition := "."
|
||||
if len(fields) >= 5 {
|
||||
condition = fields[4]
|
||||
}
|
||||
|
||||
rs, exists := rules[flag]
|
||||
if !exists {
|
||||
rs = &affixRuleSet{ruleType: ruleType}
|
||||
rules[flag] = rs
|
||||
}
|
||||
|
||||
rs.rules = append(rs.rules, affixRule{
|
||||
ruleType: ruleType,
|
||||
strip: strip,
|
||||
affix: affix,
|
||||
matchers: parseConditionMatchers(condition),
|
||||
})
|
||||
}
|
||||
|
||||
return rules, scanner.Err()
|
||||
}
|
||||
|
||||
func parseDicFile(path string) ([]dicEntry, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
// Skip first line (word count)
|
||||
if scanner.Scan() {
|
||||
// discard
|
||||
}
|
||||
|
||||
var entries []dicEntry
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
parts := strings.SplitN(line, "/", 2)
|
||||
word := strings.ToLower(parts[0])
|
||||
flags := ""
|
||||
if len(parts) > 1 {
|
||||
flags = parts[1]
|
||||
}
|
||||
|
||||
if containsDigit(word) || containsNonLatin(word) {
|
||||
continue
|
||||
}
|
||||
|
||||
entries = append(entries, dicEntry{word: word, flags: flags})
|
||||
}
|
||||
|
||||
return entries, scanner.Err()
|
||||
}
|
||||
|
||||
// maxFormsPerWord caps the number of inflected forms generated per base word.
|
||||
// Portuguese and French .aff files can produce hundreds of cross-product forms
|
||||
// per word; most are valid but the combinatorial explosion bloats the DB.
|
||||
const maxFormsPerWord = 30
|
||||
|
||||
func expandWord(word, flags string, rules map[string]*affixRuleSet) []string {
|
||||
seen := map[string]bool{word: true} // base form already in DB
|
||||
var result []string
|
||||
|
||||
// Apply single-flag suffix/prefix rules
|
||||
for _, r := range flags {
|
||||
rs, ok := rules[string(r)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, rule := range rs.rules {
|
||||
if len(result) >= maxFormsPerWord {
|
||||
return result
|
||||
}
|
||||
form := applyRule(word, rule)
|
||||
if form != "" && !seen[form] {
|
||||
seen[form] = true
|
||||
result = append(result, form)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func applyRule(word string, rule affixRule) string {
|
||||
if !matchCondition(word, rule.matchers, rule.ruleType) {
|
||||
return ""
|
||||
}
|
||||
|
||||
switch rule.ruleType {
|
||||
case "SFX":
|
||||
base := word
|
||||
if rule.strip != "" {
|
||||
if !strings.HasSuffix(base, rule.strip) {
|
||||
return ""
|
||||
}
|
||||
base = base[:len(base)-len(rule.strip)]
|
||||
}
|
||||
return base + rule.affix
|
||||
|
||||
case "PFX":
|
||||
base := word
|
||||
if rule.strip != "" {
|
||||
if !strings.HasPrefix(base, rule.strip) {
|
||||
return ""
|
||||
}
|
||||
base = base[len(rule.strip):]
|
||||
}
|
||||
return rule.affix + base
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func matchCondition(word string, matchers []condMatcher, ruleType string) bool {
|
||||
if len(matchers) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
wordRunes := []rune(word)
|
||||
if len(wordRunes) < len(matchers) {
|
||||
return false
|
||||
}
|
||||
|
||||
if ruleType == "SFX" {
|
||||
start := len(wordRunes) - len(matchers)
|
||||
for i, m := range matchers {
|
||||
if !m.matches(wordRunes[start+i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for i, m := range matchers {
|
||||
if !m.matches(wordRunes[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type condMatcher struct {
|
||||
chars []rune
|
||||
negate bool
|
||||
}
|
||||
|
||||
func (m condMatcher) matches(r rune) bool {
|
||||
if len(m.chars) == 0 {
|
||||
return true // "." wildcard
|
||||
}
|
||||
for _, c := range m.chars {
|
||||
if c == r {
|
||||
return !m.negate
|
||||
}
|
||||
}
|
||||
return m.negate
|
||||
}
|
||||
|
||||
func parseConditionMatchers(condition string) []condMatcher {
|
||||
var matchers []condMatcher
|
||||
runes := []rune(condition)
|
||||
i := 0
|
||||
for i < len(runes) {
|
||||
if runes[i] == '[' {
|
||||
i++
|
||||
negate := false
|
||||
if i < len(runes) && runes[i] == '^' {
|
||||
negate = true
|
||||
i++
|
||||
}
|
||||
var chars []rune
|
||||
for i < len(runes) && runes[i] != ']' {
|
||||
chars = append(chars, runes[i])
|
||||
i++
|
||||
}
|
||||
if i < len(runes) {
|
||||
i++ // skip ']'
|
||||
}
|
||||
matchers = append(matchers, condMatcher{chars: chars, negate: negate})
|
||||
} else if runes[i] == '.' {
|
||||
matchers = append(matchers, condMatcher{}) // wildcard
|
||||
i++
|
||||
} else {
|
||||
matchers = append(matchers, condMatcher{chars: []rune{runes[i]}})
|
||||
i++
|
||||
}
|
||||
}
|
||||
return matchers
|
||||
}
|
||||
|
||||
// EnglishAffixLoader generates common English inflected forms from base words
|
||||
// already in the database. SCOWL doesn't ship .aff rules, so this uses
|
||||
// programmatic English morphology rules instead.
|
||||
type EnglishAffixLoader struct{}
|
||||
|
||||
func (EnglishAffixLoader) Name() string { return "affix-en" }
|
||||
|
||||
func (EnglishAffixLoader) Load(db *sql.DB, dataDir string) error {
|
||||
// Read all base words first, then close the query before writing.
|
||||
rows, err := db.Query("SELECT word FROM words WHERE lang = 'en'")
|
||||
if err != nil {
|
||||
return fmt.Errorf("affix-en: query words: %w", err)
|
||||
}
|
||||
|
||||
var baseWords []string
|
||||
for rows.Next() {
|
||||
var w string
|
||||
if err := rows.Scan(&w); err != nil {
|
||||
rows.Close()
|
||||
return fmt.Errorf("affix-en: scan: %w", err)
|
||||
}
|
||||
baseWords = append(baseWords, w)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("affix-en: rows err: %w", err)
|
||||
}
|
||||
|
||||
// Generate all forms in memory first — avoids per-word DB round trips
|
||||
// during generation. Deduplicate against base words.
|
||||
baseSet := make(map[string]struct{}, len(baseWords))
|
||||
for _, w := range baseWords {
|
||||
baseSet[w] = struct{}{}
|
||||
}
|
||||
|
||||
var newForms []string
|
||||
seen := make(map[string]struct{}, len(baseWords))
|
||||
for _, w := range baseWords {
|
||||
for _, form := range englishInflections(w) {
|
||||
if _, isBase := baseSet[form]; isBase {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[form]; dup {
|
||||
continue
|
||||
}
|
||||
seen[form] = struct{}{}
|
||||
newForms = append(newForms, form)
|
||||
}
|
||||
}
|
||||
|
||||
// Multi-row batch insert — 250 rows per statement keeps us under
|
||||
// SQLite's default 500-variable limit (250 * 2 params = 500).
|
||||
if err := bulkInsertWords(db, newForms, "en", 250); err != nil {
|
||||
return fmt.Errorf("affix-en: %w", err)
|
||||
}
|
||||
slog.Info("affix-en loaded", "base_words", len(baseWords), "new_forms", len(newForms))
|
||||
return nil
|
||||
}
|
||||
|
||||
// bulkInsertWords inserts words in multi-row batches within a single transaction.
|
||||
// batchSize controls how many rows per INSERT statement (keep ≤250 for SQLite's
|
||||
// default 500-variable limit since each row uses 2 params).
|
||||
func bulkInsertWords(db *sql.DB, words []string, lang string, batchSize int) error {
|
||||
if len(words) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("bulk insert: begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
for i := 0; i < len(words); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(words) {
|
||||
end = len(words)
|
||||
}
|
||||
batch := words[i:end]
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("INSERT OR IGNORE INTO words (word, lang) VALUES ")
|
||||
args := make([]any, 0, len(batch)*2)
|
||||
for j, w := range batch {
|
||||
if j > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString("(?,?)")
|
||||
args = append(args, w, lang)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(b.String(), args...); err != nil {
|
||||
return fmt.Errorf("bulk insert: exec: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// englishInflections generates common English inflected forms from a base word.
|
||||
// Skips words that are too short, too long, or already look like inflected forms.
|
||||
func englishInflections(word string) []string {
|
||||
if len(word) < 3 || len(word) > 15 {
|
||||
return nil
|
||||
}
|
||||
// Skip words that already look inflected — avoids "runnings", "happinesses", etc.
|
||||
if strings.HasSuffix(word, "ing") || strings.HasSuffix(word, "ness") ||
|
||||
strings.HasSuffix(word, "ment") || strings.HasSuffix(word, "tion") ||
|
||||
strings.HasSuffix(word, "sion") || strings.HasSuffix(word, "ally") ||
|
||||
strings.HasSuffix(word, "ised") || strings.HasSuffix(word, "ized") {
|
||||
return nil
|
||||
}
|
||||
|
||||
var forms []string
|
||||
add := func(s string) {
|
||||
if s != word && s != "" {
|
||||
forms = append(forms, s)
|
||||
}
|
||||
}
|
||||
|
||||
last := word[len(word)-1]
|
||||
|
||||
isVowel := func(b byte) bool {
|
||||
return b == 'a' || b == 'e' || b == 'i' || b == 'o' || b == 'u'
|
||||
}
|
||||
|
||||
// Plural / verb -s forms
|
||||
switch {
|
||||
case strings.HasSuffix(word, "s") || strings.HasSuffix(word, "x") ||
|
||||
strings.HasSuffix(word, "z") || strings.HasSuffix(word, "ch") ||
|
||||
strings.HasSuffix(word, "sh"):
|
||||
add(word + "es")
|
||||
case strings.HasSuffix(word, "y") && len(word) >= 2 && !isVowel(word[len(word)-2]):
|
||||
add(word[:len(word)-1] + "ies")
|
||||
default:
|
||||
add(word + "s")
|
||||
}
|
||||
|
||||
// Past tense / past participle -ed, present participle -ing
|
||||
switch {
|
||||
case last == 'e':
|
||||
add(word + "d") // bake -> baked
|
||||
add(word[:len(word)-1] + "ing") // bake -> baking
|
||||
case strings.HasSuffix(word, "y") && len(word) >= 2 && !isVowel(word[len(word)-2]):
|
||||
add(word[:len(word)-1] + "ied") // carry -> carried
|
||||
add(word + "ing") // carry -> carrying
|
||||
case len(word) >= 3 && !isVowel(last) && isVowel(word[len(word)-2]) && !isVowel(word[len(word)-3]) &&
|
||||
last != 'w' && last != 'x' && last != 'y':
|
||||
// CVC pattern: double consonant (run -> running, stop -> stopped)
|
||||
add(word + string(last) + "ed")
|
||||
add(word + string(last) + "ing")
|
||||
// Also add without doubling (some words don't double)
|
||||
add(word + "ed")
|
||||
add(word + "ing")
|
||||
default:
|
||||
add(word + "ed")
|
||||
add(word + "ing")
|
||||
}
|
||||
|
||||
// Comparative / superlative for short adjectives
|
||||
if len(word) <= 7 {
|
||||
switch {
|
||||
case last == 'e':
|
||||
add(word + "r")
|
||||
add(word + "st")
|
||||
case strings.HasSuffix(word, "y") && len(word) >= 2 && !isVowel(word[len(word)-2]):
|
||||
add(word[:len(word)-1] + "ier")
|
||||
add(word[:len(word)-1] + "iest")
|
||||
default:
|
||||
add(word + "er")
|
||||
add(word + "est")
|
||||
}
|
||||
}
|
||||
|
||||
// -ly adverb form
|
||||
switch {
|
||||
case strings.HasSuffix(word, "le"):
|
||||
add(word[:len(word)-2] + "ly")
|
||||
case strings.HasSuffix(word, "y") && len(word) >= 2:
|
||||
add(word[:len(word)-1] + "ily")
|
||||
case strings.HasSuffix(word, "ic"):
|
||||
add(word + "ally")
|
||||
default:
|
||||
add(word + "ly")
|
||||
}
|
||||
|
||||
// -ness
|
||||
if strings.HasSuffix(word, "y") && len(word) >= 2 && !isVowel(word[len(word)-2]) {
|
||||
add(word[:len(word)-1] + "iness")
|
||||
} else {
|
||||
add(word + "ness")
|
||||
}
|
||||
|
||||
return forms
|
||||
}
|
||||
134
internal/loader/cetempublico.go
Normal file
134
internal/loader/cetempublico.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CETEMPublicoLoader loads frequency data for European Portuguese words.
|
||||
// Accepts a simple word-frequency TSV file derived from the CETEMPúblico corpus
|
||||
// or any frequency list in "word<tab>count" or "word<tab>frequency_per_million" format.
|
||||
// Falls back to Wiktionary frequency tags if the primary source is unavailable.
|
||||
type CETEMPublicoLoader struct{}
|
||||
|
||||
func (CETEMPublicoLoader) Name() string { return "cetempublico" }
|
||||
|
||||
func (CETEMPublicoLoader) Load(db *sql.DB, dataDir string) error {
|
||||
candidates := []string{
|
||||
filepath.Join(dataDir, "cetempublico-freq.tsv"),
|
||||
filepath.Join(dataDir, "pt-freq.tsv"),
|
||||
filepath.Join(dataDir, "pt_PT-freq.tsv"),
|
||||
}
|
||||
|
||||
var path string
|
||||
for _, c := range candidates {
|
||||
if _, err := os.Stat(c); err == nil {
|
||||
path = c
|
||||
break
|
||||
}
|
||||
}
|
||||
if path == "" {
|
||||
slog.Warn("cetempublico: no data file found, skipping", "searched", candidates)
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cetempublico: open: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cetempublico: begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare(`
|
||||
UPDATE words SET frequency = ? WHERE word = ? AND lang = 'pt-PT' AND frequency = 0`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cetempublico: prepare: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
// Skip header if present
|
||||
if scanner.Scan() {
|
||||
first := scanner.Text()
|
||||
fields := strings.Split(first, "\t")
|
||||
// If first line looks like data (second field is numeric), process it
|
||||
if len(fields) >= 2 {
|
||||
if _, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64); err != nil {
|
||||
// It's a header, skip it
|
||||
} else {
|
||||
// It's data, we need to process it — but we already consumed it
|
||||
// Re-process below
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var count int
|
||||
processLine := func(line string) error {
|
||||
fields := strings.Split(line, "\t")
|
||||
if len(fields) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
word := strings.ToLower(strings.TrimSpace(fields[0]))
|
||||
if word == "" || containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
|
||||
return nil
|
||||
}
|
||||
|
||||
freqVal, err := strconv.ParseFloat(strings.TrimSpace(fields[1]), 64)
|
||||
if err != nil || freqVal <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// If values are raw counts (>100), convert to a 0-10000 scale
|
||||
// If already per-million, multiply by 100
|
||||
freq := int(freqVal)
|
||||
if freqVal > 10000 {
|
||||
// Assume raw counts — log-scale normalization
|
||||
freq = int(freqVal / 10)
|
||||
if freq > 10000 {
|
||||
freq = 10000
|
||||
}
|
||||
} else if freqVal < 100 {
|
||||
freq = int(freqVal * 100)
|
||||
}
|
||||
if freq <= 0 {
|
||||
freq = 1
|
||||
}
|
||||
|
||||
res, err := stmt.Exec(freq, word)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cetempublico: update: %w", err)
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n > 0 {
|
||||
count++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
for scanner.Scan() {
|
||||
if err := processLine(scanner.Text()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return fmt.Errorf("cetempublico: scan: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("cetempublico: commit: %w", err)
|
||||
}
|
||||
slog.Info("cetempublico loaded", "updated_words", count)
|
||||
return nil
|
||||
}
|
||||
114
internal/loader/cmudict.go
Normal file
114
internal/loader/cmudict.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CMUDictLoader loads pronunciation data from the CMU Pronouncing Dictionary.
|
||||
// Format: WORD P1 P2 P3 (two-space separated word and phonemes)
|
||||
// Coverage: English only (~134,000 entries).
|
||||
type CMUDictLoader struct{}
|
||||
|
||||
func (CMUDictLoader) Name() string { return "cmudict" }
|
||||
|
||||
func (CMUDictLoader) Load(db *sql.DB, dataDir string) error {
|
||||
candidates := []string{
|
||||
filepath.Join(dataDir, "cmudict-0.7b"),
|
||||
filepath.Join(dataDir, "cmudict.dict"),
|
||||
filepath.Join(dataDir, "cmudict"),
|
||||
}
|
||||
|
||||
var path string
|
||||
for _, c := range candidates {
|
||||
if _, err := os.Stat(c); err == nil {
|
||||
path = c
|
||||
break
|
||||
}
|
||||
}
|
||||
if path == "" {
|
||||
slog.Warn("cmudict: no data file found, skipping", "searched", candidates)
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cmudict: open: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cmudict: begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO pronunciations (word_id, format, value, source)
|
||||
SELECT id, 'cmu', ?, 'cmudict' FROM words WHERE word = ? AND lang = 'en'`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cmudict: prepare: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
var count int
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if line == "" || strings.HasPrefix(line, ";;;") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Format: "WORD PH1 PH2 PH3" (two spaces between word and phonemes)
|
||||
// Some entries have variant markers like "WORD(2) PH1 PH2"
|
||||
parts := strings.SplitN(line, " ", 2)
|
||||
if len(parts) != 2 {
|
||||
// Try single space (some versions)
|
||||
idx := strings.IndexByte(line, ' ')
|
||||
if idx == -1 {
|
||||
continue
|
||||
}
|
||||
parts = []string{line[:idx], strings.TrimSpace(line[idx+1:])}
|
||||
}
|
||||
|
||||
word := strings.ToLower(strings.TrimSpace(parts[0]))
|
||||
phonemes := strings.TrimSpace(parts[1])
|
||||
|
||||
if word == "" || phonemes == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip variant entries like "WORD(2)" — keep only primary pronunciation
|
||||
if strings.Contains(word, "(") {
|
||||
continue
|
||||
}
|
||||
|
||||
if containsDigit(word) || containsSpace(word) {
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := stmt.Exec(phonemes, word)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cmudict: insert: %w", err)
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n > 0 {
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return fmt.Errorf("cmudict: scan: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("cmudict: commit: %w", err)
|
||||
}
|
||||
slog.Info("cmudict loaded", "pronunciations", count)
|
||||
return nil
|
||||
}
|
||||
72
internal/loader/difficulty.go
Normal file
72
internal/loader/difficulty.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// DifficultyScorer computes a composite difficulty score for all words
|
||||
// in the database. It runs after all other loaders have completed.
|
||||
// Score range: 0.0 (easiest) to 1.0 (hardest).
|
||||
//
|
||||
// Formula:
|
||||
// difficulty = normalize(1/frequency) * 0.5
|
||||
// + normalize(length) * 0.3
|
||||
// + normalize(syllable_count_approx) * 0.2
|
||||
//
|
||||
// Computed entirely in SQL for performance — no row-by-row round trips.
|
||||
type DifficultyScorer struct{}
|
||||
|
||||
func (DifficultyScorer) Name() string { return "difficulty" }
|
||||
|
||||
func (DifficultyScorer) Load(db *sql.DB, _ string) error {
|
||||
langs := []string{"en", "fr", "pt-PT", "zh"}
|
||||
|
||||
var totalUpdated int64
|
||||
for _, lang := range langs {
|
||||
n, err := scoreLang(db, lang)
|
||||
if err != nil {
|
||||
return fmt.Errorf("difficulty: %s: %w", lang, err)
|
||||
}
|
||||
totalUpdated += n
|
||||
}
|
||||
|
||||
slog.Info("difficulty scored", "total_words", totalUpdated)
|
||||
return nil
|
||||
}
|
||||
|
||||
func scoreLang(db *sql.DB, lang string) (int64, error) {
|
||||
// Single UPDATE using SQLite math to compute difficulty in-place.
|
||||
// log() isn't available in base SQLite, so we approximate the frequency
|
||||
// component using a reciprocal: 1.0 / (1.0 + frequency/max_freq).
|
||||
// This gives a 0–1 range where 0 = most common, ~1 = rarest.
|
||||
//
|
||||
// Length and a rough syllable proxy (length/3) are normalized against
|
||||
// per-language maximums via a subquery.
|
||||
const q = `
|
||||
UPDATE words SET difficulty = ROUND(
|
||||
CASE
|
||||
WHEN stats.max_freq > 0 AND frequency > 0
|
||||
THEN (1.0 - CAST(frequency AS REAL) / stats.max_freq) * 0.5
|
||||
ELSE 0.5
|
||||
END
|
||||
+ (CAST(LENGTH(word) AS REAL) / stats.max_len) * 0.3
|
||||
+ MIN(1.0, CAST(LENGTH(word) AS REAL) / 3.0 / (stats.max_len / 3.0)) * 0.2
|
||||
, 3)
|
||||
FROM (
|
||||
SELECT
|
||||
MAX(frequency) AS max_freq,
|
||||
MAX(LENGTH(word)) AS max_len
|
||||
FROM words WHERE lang = ?1
|
||||
) AS stats
|
||||
WHERE words.lang = ?1
|
||||
`
|
||||
|
||||
res, err := db.Exec(q, lang)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("update difficulty: %w", err)
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
return n, nil
|
||||
}
|
||||
159
internal/loader/omw.go
Normal file
159
internal/loader/omw.go
Normal file
@@ -0,0 +1,159 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// OMWLoader loads Open Multilingual Wordnet data for Portuguese,
|
||||
// mapping pt-PT words to Princeton WordNet synset IDs.
|
||||
type OMWLoader struct{}
|
||||
|
||||
func (OMWLoader) Name() string { return "omw-pt" }
|
||||
|
||||
func (OMWLoader) Load(db *sql.DB, dataDir string) error {
|
||||
// OMW Portuguese data comes as a tab-separated file.
|
||||
// Format varies by release but typically:
|
||||
// synset_id<tab>relation<tab>word
|
||||
// where synset_id is like "eng-30-00001740-n" and relation is "lemma"
|
||||
//
|
||||
// Also supports the WN-LMF XML format and the simpler tab format from
|
||||
// https://github.com/omwn/omw-data
|
||||
|
||||
// Try multiple possible file locations
|
||||
candidates := []string{
|
||||
filepath.Join(dataDir, "omw", "wn-data-por.tab"),
|
||||
filepath.Join(dataDir, "omw", "wn-por.tab"),
|
||||
filepath.Join(dataDir, "omw-pt.tab"),
|
||||
}
|
||||
|
||||
var path string
|
||||
for _, c := range candidates {
|
||||
if _, err := os.Stat(c); err == nil {
|
||||
path = c
|
||||
break
|
||||
}
|
||||
}
|
||||
if path == "" {
|
||||
slog.Warn("omw-pt: no data file found, skipping", "searched", candidates)
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("omw: begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmtSynset, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO synsets (synset_id, pos) VALUES (?, ?)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("omw: prepare synset: %w", err)
|
||||
}
|
||||
defer stmtSynset.Close()
|
||||
|
||||
stmtWordSynset, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO word_synsets (word_id, synset_id, source)
|
||||
SELECT w.id, s.id, 'omw'
|
||||
FROM words w, synsets s
|
||||
WHERE w.word = ? AND w.lang = 'pt-PT' AND s.synset_id = ?`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("omw: prepare word_synset: %w", err)
|
||||
}
|
||||
defer stmtWordSynset.Close()
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("omw: open: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
posMap := map[string]string{
|
||||
"n": "noun",
|
||||
"v": "verb",
|
||||
"a": "adjective",
|
||||
"r": "adverb",
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
var synsetCount, linkCount int
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, "#") || line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
fields := strings.Split(line, "\t")
|
||||
if len(fields) < 3 {
|
||||
continue
|
||||
}
|
||||
|
||||
synsetRaw := fields[0]
|
||||
relation := fields[1]
|
||||
word := strings.ToLower(strings.TrimSpace(fields[2]))
|
||||
|
||||
// Only process lemma relations
|
||||
if relation != "lemma" {
|
||||
continue
|
||||
}
|
||||
|
||||
if word == "" || containsDigit(word) || containsSpace(word) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Normalize synset ID: "eng-30-00001740-n" -> "00001740-n"
|
||||
synsetID := normalizeOMWSynsetID(synsetRaw)
|
||||
if synsetID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract POS from synset ID
|
||||
parts := strings.Split(synsetID, "-")
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
pos := posMap[parts[1]]
|
||||
if pos == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := stmtSynset.Exec(synsetID, pos); err != nil {
|
||||
return fmt.Errorf("omw: insert synset: %w", err)
|
||||
}
|
||||
synsetCount++
|
||||
|
||||
if _, err := stmtWordSynset.Exec(word, synsetID); err != nil {
|
||||
return fmt.Errorf("omw: insert word_synset: %w", err)
|
||||
}
|
||||
linkCount++
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return fmt.Errorf("omw: scan: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("omw: commit: %w", err)
|
||||
}
|
||||
slog.Info("omw-pt loaded", "synsets", synsetCount, "links", linkCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeOMWSynsetID(raw string) string {
|
||||
// Format: "eng-30-00001740-n" -> "00001740-n"
|
||||
parts := strings.Split(raw, "-")
|
||||
if len(parts) >= 4 && parts[0] == "eng" {
|
||||
return parts[2] + "-" + parts[3]
|
||||
}
|
||||
// Already normalized
|
||||
if len(parts) == 2 && len(parts[0]) == 8 {
|
||||
return raw
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -68,7 +68,8 @@ func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang, frequency) VALUES (?, 'en', ?)")
|
||||
stmt, err := tx.Prepare(`INSERT INTO words (word, lang, frequency) VALUES (?, 'en', ?)
|
||||
ON CONFLICT(word, lang) DO UPDATE SET frequency = MAX(frequency, excluded.frequency)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scowl: prepare: %w", err)
|
||||
}
|
||||
|
||||
129
internal/loader/subtlex.go
Normal file
129
internal/loader/subtlex.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SubtlexLoader loads SUBTLEX-US frequency data for English words.
|
||||
// Source: subtitle frequency corpus (Brysbaert & New, 2009).
|
||||
// File: SUBTLEX-US.tsv (tab-separated, exported from the original XLSX).
|
||||
type SubtlexLoader struct{}
|
||||
|
||||
func (SubtlexLoader) Name() string { return "subtlex" }
|
||||
|
||||
func (SubtlexLoader) Load(db *sql.DB, dataDir string) error {
|
||||
candidates := []string{
|
||||
filepath.Join(dataDir, "SUBTLEX-US.tsv"),
|
||||
filepath.Join(dataDir, "subtlex-us.tsv"),
|
||||
filepath.Join(dataDir, "SUBTLEX-US.txt"),
|
||||
filepath.Join(dataDir, "SUBTLEX-US.csv"),
|
||||
filepath.Join(dataDir, "SUBTLEXus74286wordstextversion.tsv"),
|
||||
}
|
||||
|
||||
var path string
|
||||
for _, c := range candidates {
|
||||
if _, err := os.Stat(c); err == nil {
|
||||
path = c
|
||||
break
|
||||
}
|
||||
}
|
||||
// Last resort: glob for anything with "subtlex" in the name
|
||||
if path == "" {
|
||||
matches, _ := filepath.Glob(filepath.Join(dataDir, "*[Ss][Uu][Bb][Tt][Ll][Ee][Xx]*"))
|
||||
if len(matches) > 0 {
|
||||
path = matches[0]
|
||||
}
|
||||
}
|
||||
if path == "" {
|
||||
slog.Warn("subtlex: no data file found, skipping", "searched", candidates)
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("subtlex: open: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("subtlex: begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare(`
|
||||
UPDATE words SET frequency = ? WHERE word = ? AND lang = 'en' AND frequency = 0`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("subtlex: prepare: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
// Skip header
|
||||
if scanner.Scan() {
|
||||
// Determine column layout from header
|
||||
}
|
||||
|
||||
var count int
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
fields := strings.Split(line, "\t")
|
||||
if len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
word := strings.ToLower(strings.TrimSpace(fields[0]))
|
||||
if word == "" || containsDigit(word) || containsSpace(word) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Frequency per million — try common column positions
|
||||
// SUBTLEX-US format varies, but frequency per million is typically col 5 or 6
|
||||
var freqPerMillion float64
|
||||
for _, idx := range []int{5, 4, 3, 1} {
|
||||
if idx < len(fields) {
|
||||
if f, err := strconv.ParseFloat(strings.TrimSpace(fields[idx]), 64); err == nil && f > 0 {
|
||||
freqPerMillion = f
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if freqPerMillion <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Convert to integer score (multiply by 100, cap at 10000)
|
||||
freq := int(freqPerMillion * 100)
|
||||
if freq > 10000 {
|
||||
freq = 10000
|
||||
}
|
||||
if freq <= 0 {
|
||||
freq = 1
|
||||
}
|
||||
|
||||
res, err := stmt.Exec(freq, word)
|
||||
if err != nil {
|
||||
return fmt.Errorf("subtlex: update: %w", err)
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n > 0 {
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return fmt.Errorf("subtlex: scan: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("subtlex: commit: %w", err)
|
||||
}
|
||||
slog.Info("subtlex loaded", "updated_words", count)
|
||||
return nil
|
||||
}
|
||||
@@ -34,10 +34,17 @@ type wiktEntry struct {
|
||||
Synonyms []struct {
|
||||
Word string `json:"word"`
|
||||
} `json:"synonyms"`
|
||||
Antonyms []struct {
|
||||
Word string `json:"word"`
|
||||
} `json:"antonyms"`
|
||||
Translations []struct {
|
||||
Code string `json:"code"`
|
||||
Word string `json:"word"`
|
||||
} `json:"translations"`
|
||||
Sounds []struct {
|
||||
IPA string `json:"ipa"`
|
||||
} `json:"sounds"`
|
||||
EtymologyText string `json:"etymology_text"`
|
||||
}
|
||||
|
||||
var wiktPOSMap = map[string]string{
|
||||
@@ -59,6 +66,9 @@ const (
|
||||
defSQL = `INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority) SELECT id, ?, ?, 'wiktionary', 20 FROM words WHERE word = ? AND lang = ?`
|
||||
synSQL = `INSERT OR IGNORE INTO synonyms (word_id, synonym, source) SELECT id, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?`
|
||||
transSQL = `INSERT OR IGNORE INTO translations (word_id, translation, target_lang, source) SELECT id, ?, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?`
|
||||
antSQL = `INSERT OR IGNORE INTO antonyms (word_id, antonym, source) SELECT id, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?`
|
||||
pronSQL = `INSERT OR IGNORE INTO pronunciations (word_id, format, value, source) SELECT id, 'ipa', ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?`
|
||||
etymSQL = `INSERT OR IGNORE INTO etymology (word_id, text, source) SELECT id, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?`
|
||||
)
|
||||
|
||||
// txBatch owns a transaction and its prepared statements.
|
||||
@@ -68,6 +78,9 @@ type txBatch struct {
|
||||
stmtDef *sql.Stmt
|
||||
stmtSyn *sql.Stmt
|
||||
stmtTrans *sql.Stmt
|
||||
stmtAnt *sql.Stmt
|
||||
stmtPron *sql.Stmt
|
||||
stmtEtym *sql.Stmt
|
||||
}
|
||||
|
||||
func newTxBatch(db *sql.DB) (*txBatch, error) {
|
||||
@@ -77,41 +90,67 @@ func newTxBatch(db *sql.DB) (*txBatch, error) {
|
||||
}
|
||||
b := &txBatch{tx: tx}
|
||||
|
||||
b.stmtDef, err = tx.Prepare(defSQL)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
prepareOrRollback := func(sql string) (*sql.Stmt, error) {
|
||||
s, e := tx.Prepare(sql)
|
||||
if e != nil {
|
||||
b.closeStmts()
|
||||
tx.Rollback()
|
||||
}
|
||||
return s, e
|
||||
}
|
||||
|
||||
if b.stmtDef, err = prepareOrRollback(defSQL); err != nil {
|
||||
return nil, fmt.Errorf("prepare def: %w", err)
|
||||
}
|
||||
b.stmtSyn, err = tx.Prepare(synSQL)
|
||||
if err != nil {
|
||||
b.stmtDef.Close()
|
||||
tx.Rollback()
|
||||
if b.stmtSyn, err = prepareOrRollback(synSQL); err != nil {
|
||||
return nil, fmt.Errorf("prepare syn: %w", err)
|
||||
}
|
||||
b.stmtTrans, err = tx.Prepare(transSQL)
|
||||
if err != nil {
|
||||
b.stmtDef.Close()
|
||||
b.stmtSyn.Close()
|
||||
tx.Rollback()
|
||||
if b.stmtTrans, err = prepareOrRollback(transSQL); err != nil {
|
||||
return nil, fmt.Errorf("prepare trans: %w", err)
|
||||
}
|
||||
if b.stmtAnt, err = prepareOrRollback(antSQL); err != nil {
|
||||
return nil, fmt.Errorf("prepare ant: %w", err)
|
||||
}
|
||||
if b.stmtPron, err = prepareOrRollback(pronSQL); err != nil {
|
||||
return nil, fmt.Errorf("prepare pron: %w", err)
|
||||
}
|
||||
if b.stmtEtym, err = prepareOrRollback(etymSQL); err != nil {
|
||||
return nil, fmt.Errorf("prepare etym: %w", err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (b *txBatch) closeStmts() {
|
||||
if b.stmtDef != nil {
|
||||
b.stmtDef.Close()
|
||||
}
|
||||
if b.stmtSyn != nil {
|
||||
b.stmtSyn.Close()
|
||||
}
|
||||
if b.stmtTrans != nil {
|
||||
b.stmtTrans.Close()
|
||||
}
|
||||
if b.stmtAnt != nil {
|
||||
b.stmtAnt.Close()
|
||||
}
|
||||
if b.stmtPron != nil {
|
||||
b.stmtPron.Close()
|
||||
}
|
||||
if b.stmtEtym != nil {
|
||||
b.stmtEtym.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (b *txBatch) Close() {
|
||||
if b == nil {
|
||||
return
|
||||
}
|
||||
b.stmtDef.Close()
|
||||
b.stmtSyn.Close()
|
||||
b.stmtTrans.Close()
|
||||
b.closeStmts()
|
||||
b.tx.Rollback() // no-op after commit
|
||||
}
|
||||
|
||||
func (b *txBatch) Commit() error {
|
||||
b.stmtDef.Close()
|
||||
b.stmtSyn.Close()
|
||||
b.stmtTrans.Close()
|
||||
b.closeStmts()
|
||||
return b.tx.Commit()
|
||||
}
|
||||
|
||||
@@ -131,8 +170,8 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 0, 10*1024*1024), 10*1024*1024)
|
||||
|
||||
var defCount, synCount, transCount, lineCount int
|
||||
commitInterval := 10000
|
||||
var defCount, synCount, transCount, antCount, pronCount, etymCount, lineCount int
|
||||
commitInterval := 50000
|
||||
|
||||
for scanner.Scan() {
|
||||
lineCount++
|
||||
@@ -183,6 +222,40 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
|
||||
synCount++
|
||||
}
|
||||
|
||||
// Process antonyms
|
||||
for _, ant := range entry.Antonyms {
|
||||
antWord := strings.ToLower(ant.Word)
|
||||
if antWord == "" || containsDigit(antWord) || containsSpace(antWord) {
|
||||
continue
|
||||
}
|
||||
if _, err := batch.stmtAnt.Exec(antWord, word, lang); err != nil {
|
||||
return fmt.Errorf("wiktionary: insert ant: %w", err)
|
||||
}
|
||||
antCount++
|
||||
}
|
||||
|
||||
// Process pronunciations (IPA)
|
||||
for _, sound := range entry.Sounds {
|
||||
ipa := strings.TrimSpace(sound.IPA)
|
||||
if ipa == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := batch.stmtPron.Exec(ipa, word, lang); err != nil {
|
||||
return fmt.Errorf("wiktionary: insert pron: %w", err)
|
||||
}
|
||||
pronCount++
|
||||
break // only first IPA per entry
|
||||
}
|
||||
|
||||
// Process etymology
|
||||
etymText := strings.TrimSpace(entry.EtymologyText)
|
||||
if etymText != "" && len(etymText) > 10 && !strings.HasPrefix(etymText, "See ") {
|
||||
if _, err := batch.stmtEtym.Exec(etymText, word, lang); err != nil {
|
||||
return fmt.Errorf("wiktionary: insert etym: %w", err)
|
||||
}
|
||||
etymCount++
|
||||
}
|
||||
|
||||
// Process translations
|
||||
for _, tr := range entry.Translations {
|
||||
targetLang, ok := wiktLangCodeMap[tr.Code]
|
||||
@@ -218,7 +291,7 @@ func loadWiktionary(db *sql.DB, path, lang string) error {
|
||||
return fmt.Errorf("wiktionary: final commit: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("wiktionary loaded", "lang", lang, "definitions", defCount, "synonyms", synCount, "translations", transCount)
|
||||
slog.Info("wiktionary loaded", "lang", lang, "definitions", defCount, "synonyms", synCount, "translations", transCount, "antonyms", antCount, "pronunciations", pronCount, "etymologies", etymCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -58,14 +58,32 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error {
|
||||
}
|
||||
defer stmtSyn.Close()
|
||||
|
||||
stmtSynset, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO synsets (synset_id, pos) VALUES (?, ?)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wolf: prepare synset: %w", err)
|
||||
}
|
||||
defer stmtSynset.Close()
|
||||
|
||||
stmtWordSynset, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO word_synsets (word_id, synset_id, source)
|
||||
SELECT w.id, s.id, 'wolf'
|
||||
FROM words w, synsets s
|
||||
WHERE w.word = ? AND w.lang = 'fr' AND s.synset_id = ?`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wolf: prepare word_synset: %w", err)
|
||||
}
|
||||
defer stmtWordSynset.Close()
|
||||
|
||||
posMap := map[string]string{
|
||||
"n": "noun",
|
||||
"v": "verb",
|
||||
"a": "adjective",
|
||||
"r": "adverb",
|
||||
"b": "adverb", // WOLF uses "b" for adverb
|
||||
}
|
||||
|
||||
var defCount, synCount, decodeErrors int
|
||||
var defCount, synCount, synsetCount, decodeErrors int
|
||||
|
||||
decoder := xml.NewDecoder(reader)
|
||||
for {
|
||||
@@ -83,6 +101,8 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error {
|
||||
}
|
||||
|
||||
var synset struct {
|
||||
ID string `xml:"ID"`
|
||||
IDAlt string `xml:"id,attr"`
|
||||
POS string `xml:"POS"`
|
||||
Literals []struct {
|
||||
Value string `xml:",chardata"`
|
||||
@@ -95,9 +115,17 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error {
|
||||
continue
|
||||
}
|
||||
|
||||
pos := posMap[strings.ToLower(synset.POS)]
|
||||
pos := posMap[strings.ToLower(strings.TrimSpace(synset.POS))]
|
||||
gloss := strings.TrimSpace(synset.DEF)
|
||||
|
||||
// WOLF synset IDs are Princeton WordNet IDs (e.g., "eng-30-00914031-a")
|
||||
// Normalize to our format: "00914031-a"
|
||||
rawID := synset.ID
|
||||
if rawID == "" {
|
||||
rawID = synset.IDAlt
|
||||
}
|
||||
wolfSynsetID := normalizeWOLFSynsetID(rawID)
|
||||
|
||||
var words []string
|
||||
for _, lit := range synset.Literals {
|
||||
w := strings.ToLower(strings.TrimSpace(lit.Value))
|
||||
@@ -106,6 +134,19 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Insert synset and link words
|
||||
if wolfSynsetID != "" && pos != "" {
|
||||
if _, err := stmtSynset.Exec(wolfSynsetID, pos); err != nil {
|
||||
return fmt.Errorf("wolf: insert synset: %w", err)
|
||||
}
|
||||
synsetCount++
|
||||
for _, w := range words {
|
||||
if _, err := stmtWordSynset.Exec(w, wolfSynsetID); err != nil {
|
||||
return fmt.Errorf("wolf: insert word_synset: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if gloss != "" {
|
||||
for _, w := range words {
|
||||
if _, err := stmtDef.Exec(pos, gloss, w); err != nil {
|
||||
@@ -135,6 +176,35 @@ func (WOLFLoader) Load(db *sql.DB, dataDir string) error {
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("wolf: commit: %w", err)
|
||||
}
|
||||
slog.Info("wolf loaded", "definitions", defCount, "synonyms", synCount)
|
||||
slog.Info("wolf loaded", "definitions", defCount, "synonyms", synCount, "synsets", synsetCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeWOLFSynsetID extracts the Princeton synset ID from WOLF format.
|
||||
// WOLF uses "eng-30-XXXXXXXX-P" format; we want "XXXXXXXX-P".
|
||||
// WOLF uses "b" for adverb POS but WordNet uses "r", so we remap.
|
||||
func normalizeWOLFSynsetID(wolfID string) string {
|
||||
wolfID = strings.TrimSpace(wolfID)
|
||||
// Common format: "eng-30-00914031-a"
|
||||
parts := strings.Split(wolfID, "-")
|
||||
if len(parts) >= 4 && parts[0] == "eng" {
|
||||
posSuffix := wolfPosSuffix(parts[3])
|
||||
return parts[2] + "-" + posSuffix
|
||||
}
|
||||
// Fallback: if it's already in our format
|
||||
if len(parts) == 2 && len(parts[0]) == 8 {
|
||||
return parts[0] + "-" + wolfPosSuffix(parts[1])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// wolfPosSuffix maps WOLF POS codes to WordNet synset ID suffixes.
|
||||
func wolfPosSuffix(pos string) string {
|
||||
pos = strings.TrimSpace(pos)
|
||||
switch pos {
|
||||
case "b":
|
||||
return "r" // WOLF "b" (adverb) → WordNet "r"
|
||||
default:
|
||||
return pos
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,24 @@ type WordNetLoader struct{}
|
||||
|
||||
func (WordNetLoader) Name() string { return "wordnet" }
|
||||
|
||||
// ssTypeMap maps WordNet ss_type codes to POS and synset pos suffix.
|
||||
var ssTypeMap = map[string]string{
|
||||
"n": "noun",
|
||||
"v": "verb",
|
||||
"a": "adjective",
|
||||
"s": "adjective", // satellite adjective
|
||||
"r": "adverb",
|
||||
}
|
||||
|
||||
// ssTypeSuffix maps WordNet ss_type to synset ID suffix character.
|
||||
var ssTypeSuffix = map[string]string{
|
||||
"n": "n",
|
||||
"v": "v",
|
||||
"a": "a",
|
||||
"s": "a", // satellite adjectives share 'a' namespace
|
||||
"r": "r",
|
||||
}
|
||||
|
||||
func (WordNetLoader) Load(db *sql.DB, dataDir string) error {
|
||||
posFiles := map[string]string{
|
||||
"data.noun": "noun",
|
||||
@@ -45,111 +63,219 @@ func (WordNetLoader) Load(db *sql.DB, dataDir string) error {
|
||||
}
|
||||
defer stmtSyn.Close()
|
||||
|
||||
var defCount, synCount int
|
||||
stmtAnt, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO antonyms (word_id, antonym, source)
|
||||
SELECT id, ?, 'wordnet' FROM words WHERE word = ? AND lang = 'en'`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: prepare ant: %w", err)
|
||||
}
|
||||
defer stmtAnt.Close()
|
||||
|
||||
stmtSynset, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO synsets (synset_id, pos) VALUES (?, ?)`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: prepare synset: %w", err)
|
||||
}
|
||||
defer stmtSynset.Close()
|
||||
|
||||
stmtWordSynset, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO word_synsets (word_id, synset_id, source)
|
||||
SELECT w.id, s.id, 'wordnet'
|
||||
FROM words w, synsets s
|
||||
WHERE w.word = ? AND w.lang = 'en' AND s.synset_id = ?`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: prepare word_synset: %w", err)
|
||||
}
|
||||
defer stmtWordSynset.Close()
|
||||
|
||||
var defCount, synCount, antCount, synsetCount int
|
||||
|
||||
for file, pos := range posFiles {
|
||||
path := filepath.Join(dataDir, "wordnet", "dict", file)
|
||||
d, s, err := loadWordNetFile(stmtDef, stmtSyn, path, pos)
|
||||
d, s, a, sc, err := loadWordNetFile(stmtDef, stmtSyn, stmtAnt, stmtSynset, stmtWordSynset, path, pos)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: %s: %w", file, err)
|
||||
}
|
||||
defCount += d
|
||||
synCount += s
|
||||
antCount += a
|
||||
synsetCount += sc
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("wordnet: commit: %w", err)
|
||||
}
|
||||
slog.Info("wordnet loaded", "definitions", defCount, "synonyms", synCount)
|
||||
slog.Info("wordnet loaded", "definitions", defCount, "synonyms", synCount, "antonyms", antCount, "synsets", synsetCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadWordNetFile(stmtDef, stmtSyn *sql.Stmt, path, pos string) (int, int, error) {
|
||||
func loadWordNetFile(stmtDef, stmtSyn, stmtAnt, stmtSynset, stmtWordSynset *sql.Stmt, path, pos string) (int, int, int, int, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var defCount, synCount int
|
||||
// Single pass: insert defs/syns/synsets as we go, collect lightweight
|
||||
// antonym pointers + synset->words map for deferred antonym resolution.
|
||||
type deferredAnt struct {
|
||||
srcWord string
|
||||
targetSynsetKey string
|
||||
tgtWordIdx int
|
||||
}
|
||||
|
||||
synsetWords := make(map[string][]string) // only stores word lists, not full parse data
|
||||
var pendingAnts []deferredAnt
|
||||
var defCount, synCount, antCount, synsetCount int
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
buf := make([]byte, 0, 64*1024)
|
||||
scanner.Buffer(buf, 1024*1024)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
// Skip header lines (start with two spaces)
|
||||
if strings.HasPrefix(line, " ") {
|
||||
continue
|
||||
}
|
||||
|
||||
words, gloss := parseWordNetLine(line)
|
||||
if gloss == "" || len(words) == 0 {
|
||||
parsed := parseWordNetLine(line)
|
||||
if parsed.gloss == "" || len(parsed.words) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Insert definitions for each word
|
||||
for _, w := range words {
|
||||
if _, err := stmtDef.Exec(pos, gloss, w); err != nil {
|
||||
return 0, 0, err
|
||||
// Build synset->words map (needed for antonym resolution)
|
||||
if parsed.synsetID != "" {
|
||||
synsetWords[parsed.synsetID] = parsed.words
|
||||
|
||||
if _, err := stmtSynset.Exec(parsed.synsetID, pos); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
synsetCount++
|
||||
|
||||
for _, w := range parsed.words {
|
||||
if _, err := stmtWordSynset.Exec(w, parsed.synsetID); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Insert definitions
|
||||
for _, w := range parsed.words {
|
||||
if _, err := stmtDef.Exec(pos, parsed.gloss, w); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
defCount++
|
||||
}
|
||||
|
||||
// Insert synonym pairs (skip words with underscores)
|
||||
// Insert synonym pairs
|
||||
var cleanWords []string
|
||||
for _, w := range words {
|
||||
for _, w := range parsed.words {
|
||||
if !strings.Contains(w, "_") {
|
||||
cleanWords = append(cleanWords, w)
|
||||
}
|
||||
}
|
||||
for i, w1 := range cleanWords {
|
||||
for j, w2 := range cleanWords {
|
||||
if i == j {
|
||||
continue
|
||||
if i != j {
|
||||
if _, err := stmtSyn.Exec(w2, w1); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
synCount++
|
||||
}
|
||||
if _, err := stmtSyn.Exec(w2, w1); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
synCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Collect antonym pointers for deferred resolution
|
||||
for _, ap := range parsed.antonymPtrs {
|
||||
if ap.srcWordIdx < 1 || ap.srcWordIdx > len(parsed.words) {
|
||||
continue
|
||||
}
|
||||
srcWord := strings.ToLower(parsed.words[ap.srcWordIdx-1])
|
||||
if strings.Contains(srcWord, "_") {
|
||||
continue
|
||||
}
|
||||
pendingAnts = append(pendingAnts, deferredAnt{
|
||||
srcWord: srcWord,
|
||||
targetSynsetKey: ap.targetSynsetKey,
|
||||
tgtWordIdx: ap.tgtWordIdx,
|
||||
})
|
||||
}
|
||||
}
|
||||
return defCount, synCount, scanner.Err()
|
||||
if err := scanner.Err(); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
|
||||
// Resolve deferred antonyms now that all synsets are mapped
|
||||
for _, da := range pendingAnts {
|
||||
tgtWords, ok := synsetWords[da.targetSynsetKey]
|
||||
if !ok || da.tgtWordIdx < 1 || da.tgtWordIdx > len(tgtWords) {
|
||||
continue
|
||||
}
|
||||
tgtWord := strings.ToLower(tgtWords[da.tgtWordIdx-1])
|
||||
if strings.Contains(tgtWord, "_") || tgtWord == da.srcWord {
|
||||
continue
|
||||
}
|
||||
if _, err := stmtAnt.Exec(tgtWord, da.srcWord); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
if _, err := stmtAnt.Exec(da.srcWord, tgtWord); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
antCount += 2
|
||||
}
|
||||
|
||||
return defCount, synCount, antCount, synsetCount, nil
|
||||
}
|
||||
|
||||
func parseWordNetLine(line string) ([]string, string) {
|
||||
type wordNetParsed struct {
|
||||
words []string
|
||||
gloss string
|
||||
synsetID string
|
||||
// antonymPtrs: each entry is {targetSynsetKey, sourceWordIdx, targetWordIdx}
|
||||
antonymPtrs []antonymPtr
|
||||
}
|
||||
|
||||
type antonymPtr struct {
|
||||
targetSynsetKey string // "offset-pos" e.g. "00123456-a"
|
||||
srcWordIdx int // 1-based word index in source synset
|
||||
tgtWordIdx int // 1-based word index in target synset
|
||||
}
|
||||
|
||||
func parseWordNetLine(line string) wordNetParsed {
|
||||
// Format: synset_offset lex_filenum ss_type w_cnt word lex_id [word lex_id...] p_cnt [ptr...] | gloss
|
||||
glossIdx := strings.Index(line, "| ")
|
||||
if glossIdx == -1 {
|
||||
return nil, ""
|
||||
return wordNetParsed{}
|
||||
}
|
||||
|
||||
gloss := strings.TrimSpace(line[glossIdx+2:])
|
||||
// Take gloss before first ";" (drops usage examples)
|
||||
if semiIdx := strings.Index(gloss, ";"); semiIdx != -1 {
|
||||
gloss = strings.TrimSpace(gloss[:semiIdx])
|
||||
}
|
||||
if gloss == "" {
|
||||
return nil, ""
|
||||
return wordNetParsed{}
|
||||
}
|
||||
|
||||
dataPart := line[:glossIdx]
|
||||
fields := strings.Fields(dataPart)
|
||||
if len(fields) < 6 {
|
||||
return nil, ""
|
||||
return wordNetParsed{}
|
||||
}
|
||||
|
||||
synsetOffset := fields[0]
|
||||
ssType := fields[2]
|
||||
posSuffix := ssTypeSuffix[ssType]
|
||||
synsetID := ""
|
||||
if posSuffix != "" {
|
||||
synsetID = synsetOffset + "-" + posSuffix
|
||||
}
|
||||
|
||||
// fields[3] = w_cnt (hex)
|
||||
wc, err := strconv.ParseInt(fields[3], 16, 0)
|
||||
if err != nil {
|
||||
slog.Debug("wordnet: bad word count", "field", fields[3], "error", err)
|
||||
return nil, ""
|
||||
return wordNetParsed{}
|
||||
}
|
||||
wordCount := int(wc)
|
||||
if wordCount <= 0 || wordCount > 100 {
|
||||
slog.Debug("wordnet: unreasonable word count", "count", wordCount)
|
||||
return nil, ""
|
||||
return wordNetParsed{}
|
||||
}
|
||||
|
||||
var words []string
|
||||
@@ -157,5 +283,41 @@ func parseWordNetLine(line string) ([]string, string) {
|
||||
w := strings.ToLower(fields[4+i*2])
|
||||
words = append(words, w)
|
||||
}
|
||||
return words, gloss
|
||||
|
||||
// Parse pointers
|
||||
ptrStart := 4 + wordCount*2
|
||||
var antPtrs []antonymPtr
|
||||
if ptrStart < len(fields) {
|
||||
pc, err := strconv.Atoi(fields[ptrStart])
|
||||
if err == nil {
|
||||
for i := 0; i < pc; i++ {
|
||||
base := ptrStart + 1 + i*4
|
||||
if base+3 >= len(fields) {
|
||||
break
|
||||
}
|
||||
if fields[base] != "!" {
|
||||
continue
|
||||
}
|
||||
tgtOffset := fields[base+1]
|
||||
tgtPosChar := fields[base+2]
|
||||
srcTgt := fields[base+3]
|
||||
if len(srcTgt) != 4 {
|
||||
continue
|
||||
}
|
||||
srcIdx, e1 := strconv.ParseInt(srcTgt[0:2], 16, 0)
|
||||
tgtIdx, e2 := strconv.ParseInt(srcTgt[2:4], 16, 0)
|
||||
if e1 != nil || e2 != nil {
|
||||
continue
|
||||
}
|
||||
tgtKey := tgtOffset + "-" + tgtPosChar
|
||||
antPtrs = append(antPtrs, antonymPtr{
|
||||
targetSynsetKey: tgtKey,
|
||||
srcWordIdx: int(srcIdx),
|
||||
tgtWordIdx: int(tgtIdx),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return wordNetParsed{words: words, gloss: gloss, synsetID: synsetID, antonymPtrs: antPtrs}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user