Initial commit: DreamDict HTTP dictionary service
Go-based dictionary backend providing word validation, definitions, synonyms, and cross-language translation for English, French, and Portuguese. Designed as a self-hosted replacement for Wordnik, backing GogoBee. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
prosolis
98
internal/dictionary/db.go
Normal file
98
internal/dictionary/db.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package dictionary
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const schemaVersion = "1"
|
||||
|
||||
const schemaSQL = `
|
||||
CREATE TABLE IF NOT EXISTS words (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word TEXT NOT NULL,
|
||||
lang TEXT NOT NULL,
|
||||
pos TEXT,
|
||||
frequency INTEGER DEFAULT 0,
|
||||
UNIQUE(word, lang)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_words_lang ON words(lang);
|
||||
CREATE INDEX IF NOT EXISTS idx_words_lang_pos ON words(lang, pos);
|
||||
CREATE INDEX IF NOT EXISTS idx_words_word ON words(word);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS definitions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
|
||||
pos TEXT,
|
||||
gloss TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
priority INTEGER NOT NULL DEFAULT 99,
|
||||
UNIQUE(word_id, gloss)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_definitions_word_id ON definitions(word_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_definitions_word_id_priority ON definitions(word_id, priority);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS synonyms (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
|
||||
synonym TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
UNIQUE(word_id, synonym)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_synonyms_word_id ON synonyms(word_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS translations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE,
|
||||
translation TEXT NOT NULL,
|
||||
target_lang TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
UNIQUE(word_id, translation, target_lang)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_translations_word_id ON translations(word_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
`
|
||||
|
||||
func openDB(dbPath string) (*sql.DB, error) {
|
||||
db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(wal)&_pragma=foreign_keys(on)")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: open db: %w", err)
|
||||
}
|
||||
db.SetMaxOpenConns(1) // SQLite write serialization for import
|
||||
if err := db.Ping(); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("dictionary: ping db: %w", err)
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func openReadOnlyDB(dbPath string) (*sql.DB, error) {
|
||||
db, err := sql.Open("sqlite", dbPath+"?mode=ro&_pragma=journal_mode(wal)&_pragma=foreign_keys(on)")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: open db (ro): %w", err)
|
||||
}
|
||||
// No MaxOpenConns limit — WAL mode supports concurrent reads
|
||||
if err := db.Ping(); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("dictionary: ping db: %w", err)
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func BootstrapSchema(db *sql.DB) error {
|
||||
_, err := db.Exec(schemaSQL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dictionary: bootstrap schema: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
93
internal/dictionary/dict.go
Normal file
93
internal/dictionary/dict.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package dictionary
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNoMatch = errors.New("dictionary: no word matches filters")
|
||||
ErrNotSeeded = errors.New("dictionary: database not seeded, run: go run ./cmd/dictimport")
|
||||
)
|
||||
|
||||
var supportedLangs = []string{"en", "fr", "pt-PT"}
|
||||
|
||||
func ValidLang(lang string) bool {
|
||||
for _, l := range supportedLangs {
|
||||
if l == lang {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
MinLength int
|
||||
MaxLength int
|
||||
POS string // "", "noun", "verb", "adjective", "adverb"
|
||||
MinFrequency int // 0 = no filter
|
||||
}
|
||||
|
||||
type Definition struct {
|
||||
POS string `json:"pos"`
|
||||
Gloss string `json:"gloss"`
|
||||
Source string `json:"source"`
|
||||
Priority int `json:"priority"`
|
||||
}
|
||||
|
||||
type Dictionary struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// New opens the database read-write (for import CLI).
|
||||
func New(dbPath string) (*Dictionary, error) {
|
||||
db, err := openDB(dbPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := &Dictionary{db: db}
|
||||
if err := d.checkSeeded(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// NewReadOnly opens the database read-only with concurrent read support (for server).
|
||||
func NewReadOnly(dbPath string) (*Dictionary, error) {
|
||||
db, err := openReadOnlyDB(dbPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := &Dictionary{db: db}
|
||||
if err := d.checkSeeded(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// NewFromDB creates a Dictionary from an existing *sql.DB (useful for testing).
|
||||
func NewFromDB(db *sql.DB) *Dictionary {
|
||||
return &Dictionary{db: db}
|
||||
}
|
||||
|
||||
func (d *Dictionary) Close() error {
|
||||
return d.db.Close()
|
||||
}
|
||||
|
||||
func (d *Dictionary) checkSeeded() error {
|
||||
var count int
|
||||
err := d.db.QueryRow("SELECT COUNT(*) FROM meta").Scan(&count)
|
||||
if err != nil {
|
||||
return ErrNotSeeded
|
||||
}
|
||||
if count == 0 {
|
||||
return ErrNotSeeded
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Dictionary) SupportedLangs() []string {
|
||||
return supportedLangs
|
||||
}
|
||||
313
internal/dictionary/dict_test.go
Normal file
313
internal/dictionary/dict_test.go
Normal file
@@ -0,0 +1,313 @@
|
||||
package dictionary
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func setupTestDB(t *testing.T) *sql.DB {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(on)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
if err := BootstrapSchema(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func mustExec(t *testing.T, db *sql.DB, query string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := db.Exec(query, args...); err != nil {
|
||||
t.Fatalf("mustExec: %s: %v", query, err)
|
||||
}
|
||||
}
|
||||
|
||||
func seedTestData(t *testing.T, db *sql.DB) {
|
||||
t.Helper()
|
||||
|
||||
// Meta
|
||||
mustExec(t, db, "INSERT INTO meta (key, value) VALUES ('schema_version', '1')")
|
||||
mustExec(t, db, "INSERT INTO meta (key, value) VALUES ('imported_at', '2025-01-01T00:00:00Z')")
|
||||
|
||||
// Words
|
||||
mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (1, 'happy', 'en', 'adjective', 0)")
|
||||
mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (2, 'run', 'en', 'verb', 0)")
|
||||
mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (3, 'chat', 'fr', 'noun', 5000)")
|
||||
mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (4, 'gato', 'pt-PT', 'noun', 0)")
|
||||
mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (5, 'cat', 'en', 'noun', 0)")
|
||||
|
||||
// Definitions
|
||||
mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling pleasure', 'wordnet', 10)")
|
||||
mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling joy or contentment', 'wiktionary', 20)")
|
||||
|
||||
// Synonyms
|
||||
mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'glad', 'wordnet')")
|
||||
mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'joyful', 'wordnet')")
|
||||
|
||||
// Translations
|
||||
mustExec(t, db, "INSERT INTO translations (word_id, translation, target_lang, source) VALUES (5, 'chat', 'fr', 'wiktionary')")
|
||||
mustExec(t, db, "INSERT INTO translations (word_id, translation, target_lang, source) VALUES (5, 'gato', 'pt-PT', 'wiktionary')")
|
||||
}
|
||||
|
||||
func TestIsValidWord(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
d := NewFromDB(db)
|
||||
|
||||
tests := []struct {
|
||||
word, lang string
|
||||
want bool
|
||||
}{
|
||||
{"happy", "en", true},
|
||||
{"Happy", "en", true}, // case-insensitive
|
||||
{"nonexistent", "en", false},
|
||||
{"chat", "fr", true},
|
||||
{"chat", "en", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got, err := d.IsValidWord(tt.word, tt.lang)
|
||||
if err != nil {
|
||||
t.Errorf("IsValidWord(%q, %q) error: %v", tt.word, tt.lang, err)
|
||||
continue
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("IsValidWord(%q, %q) = %v, want %v", tt.word, tt.lang, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRandomWord(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
d := NewFromDB(db)
|
||||
|
||||
// Basic random word
|
||||
word, err := d.RandomWord("en", Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("RandomWord: %v", err)
|
||||
}
|
||||
if word == "" {
|
||||
t.Error("RandomWord returned empty string")
|
||||
}
|
||||
|
||||
// With POS filter
|
||||
word, err = d.RandomWord("en", Options{POS: "adjective"})
|
||||
if err != nil {
|
||||
t.Fatalf("RandomWord with POS: %v", err)
|
||||
}
|
||||
if word != "happy" {
|
||||
t.Errorf("RandomWord(adjective) = %q, want happy", word)
|
||||
}
|
||||
|
||||
// With length filters
|
||||
word, err = d.RandomWord("en", Options{MinLength: 4, MaxLength: 5})
|
||||
if err != nil {
|
||||
t.Fatalf("RandomWord with length: %v", err)
|
||||
}
|
||||
if len(word) < 4 || len(word) > 5 {
|
||||
t.Errorf("RandomWord length %d not in [4,5]", len(word))
|
||||
}
|
||||
|
||||
// No match
|
||||
_, err = d.RandomWord("en", Options{MinLength: 100})
|
||||
if err != ErrNoMatch {
|
||||
t.Errorf("expected ErrNoMatch, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefine(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
d := NewFromDB(db)
|
||||
|
||||
defs, err := d.Define("happy", "en")
|
||||
if err != nil {
|
||||
t.Fatalf("Define: %v", err)
|
||||
}
|
||||
if len(defs) != 2 {
|
||||
t.Fatalf("Define returned %d defs, want 2", len(defs))
|
||||
}
|
||||
// Priority ordering: wordnet (10) before wiktionary (20)
|
||||
if defs[0].Source != "wordnet" {
|
||||
t.Errorf("first def source = %q, want wordnet", defs[0].Source)
|
||||
}
|
||||
if defs[1].Source != "wiktionary" {
|
||||
t.Errorf("second def source = %q, want wiktionary", defs[1].Source)
|
||||
}
|
||||
|
||||
// No definitions
|
||||
defs, err = d.Define("run", "en")
|
||||
if err != nil {
|
||||
t.Fatalf("Define(run): %v", err)
|
||||
}
|
||||
if len(defs) != 0 {
|
||||
t.Errorf("Define(run) returned %d defs, want 0", len(defs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSynonyms(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
d := NewFromDB(db)
|
||||
|
||||
syns, err := d.Synonyms("happy", "en")
|
||||
if err != nil {
|
||||
t.Fatalf("Synonyms: %v", err)
|
||||
}
|
||||
if len(syns) != 2 {
|
||||
t.Fatalf("Synonyms returned %d, want 2", len(syns))
|
||||
}
|
||||
|
||||
// No synonyms
|
||||
syns, err = d.Synonyms("run", "en")
|
||||
if err != nil {
|
||||
t.Fatalf("Synonyms(run): %v", err)
|
||||
}
|
||||
if len(syns) != 0 {
|
||||
t.Errorf("Synonyms(run) returned %d, want 0", len(syns))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslate(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
d := NewFromDB(db)
|
||||
|
||||
trans, err := d.Translate("cat", "en", "fr")
|
||||
if err != nil {
|
||||
t.Fatalf("Translate: %v", err)
|
||||
}
|
||||
if len(trans) != 1 || trans[0] != "chat" {
|
||||
t.Errorf("Translate(cat, en, fr) = %v, want [chat]", trans)
|
||||
}
|
||||
|
||||
// No translations
|
||||
trans, err = d.Translate("happy", "en", "fr")
|
||||
if err != nil {
|
||||
t.Fatalf("Translate(happy): %v", err)
|
||||
}
|
||||
if len(trans) != 0 {
|
||||
t.Errorf("Translate(happy) returned %d, want 0", len(trans))
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrNotSeeded(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
// Don't seed — meta table is empty
|
||||
d := NewFromDB(db)
|
||||
err := d.checkSeeded()
|
||||
if err != ErrNotSeeded {
|
||||
t.Errorf("expected ErrNotSeeded, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlossDeduplication(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
|
||||
// Insert duplicate gloss — should be ignored by UNIQUE constraint
|
||||
_, err := db.Exec("INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling pleasure', 'wiktionary', 20)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := NewFromDB(db)
|
||||
defs, err := d.Define("happy", "en")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(defs) != 2 {
|
||||
t.Errorf("expected 2 defs after dedup, got %d", len(defs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSynonymDeduplication(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
|
||||
_, err := db.Exec("INSERT OR IGNORE INTO synonyms (word_id, synonym, source) VALUES (1, 'glad', 'wiktionary')")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := NewFromDB(db)
|
||||
syns, err := d.Synonyms("happy", "en")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(syns) != 2 {
|
||||
t.Errorf("expected 2 synonyms after dedup, got %d", len(syns))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslationDeduplication(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
|
||||
_, err := db.Exec("INSERT OR IGNORE INTO translations (word_id, translation, target_lang, source) VALUES (5, 'chat', 'fr', 'other')")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := NewFromDB(db)
|
||||
trans, err := d.Translate("cat", "en", "fr")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(trans) != 1 {
|
||||
t.Errorf("expected 1 translation after dedup, got %d", len(trans))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeta(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
d := NewFromDB(db)
|
||||
|
||||
val, err := d.Meta("schema_version")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if val != "1" {
|
||||
t.Errorf("Meta(schema_version) = %q, want 1", val)
|
||||
}
|
||||
|
||||
// Missing key returns empty string, no error
|
||||
val, err = d.Meta("nonexistent")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if val != "" {
|
||||
t.Errorf("Meta(nonexistent) = %q, want empty", val)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWordCount(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
seedTestData(t, db)
|
||||
d := NewFromDB(db)
|
||||
|
||||
counts, err := d.WordCount()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if counts["en"] != 3 {
|
||||
t.Errorf("en word count = %d, want 3", counts["en"])
|
||||
}
|
||||
if counts["fr"] != 1 {
|
||||
t.Errorf("fr word count = %d, want 1", counts["fr"])
|
||||
}
|
||||
}
|
||||
193
internal/dictionary/query.go
Normal file
193
internal/dictionary/query.go
Normal file
@@ -0,0 +1,193 @@
|
||||
package dictionary
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (d *Dictionary) IsValidWord(word, lang string) (bool, error) {
|
||||
var exists bool
|
||||
err := d.db.QueryRow(
|
||||
"SELECT EXISTS(SELECT 1 FROM words WHERE word = ? AND lang = ?)",
|
||||
strings.ToLower(word), lang,
|
||||
).Scan(&exists)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("dictionary: is valid word: %w", err)
|
||||
}
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
func (d *Dictionary) RandomWord(lang string, opts Options) (string, error) {
|
||||
query := "SELECT word FROM words WHERE lang = ?"
|
||||
args := []any{lang}
|
||||
|
||||
if opts.POS != "" {
|
||||
query += " AND pos = ?"
|
||||
args = append(args, opts.POS)
|
||||
}
|
||||
if opts.MinLength > 0 {
|
||||
query += " AND LENGTH(word) >= ?"
|
||||
args = append(args, opts.MinLength)
|
||||
}
|
||||
if opts.MaxLength > 0 {
|
||||
query += " AND LENGTH(word) <= ?"
|
||||
args = append(args, opts.MaxLength)
|
||||
}
|
||||
if opts.MinFrequency > 0 {
|
||||
query += " AND frequency >= ?"
|
||||
args = append(args, opts.MinFrequency)
|
||||
}
|
||||
|
||||
query += " ORDER BY RANDOM() LIMIT 1"
|
||||
|
||||
var word string
|
||||
err := d.db.QueryRow(query, args...).Scan(&word)
|
||||
if err == sql.ErrNoRows {
|
||||
return "", ErrNoMatch
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("dictionary: random word: %w", err)
|
||||
}
|
||||
return word, nil
|
||||
}
|
||||
|
||||
func (d *Dictionary) Define(word, lang string) ([]Definition, error) {
|
||||
rows, err := d.db.Query(`
|
||||
SELECT d.pos, d.gloss, d.source, d.priority
|
||||
FROM definitions d
|
||||
JOIN words w ON w.id = d.word_id
|
||||
WHERE w.word = ? AND w.lang = ?
|
||||
ORDER BY d.priority ASC, d.id ASC`,
|
||||
strings.ToLower(word), lang,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: define: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var defs []Definition
|
||||
for rows.Next() {
|
||||
var def Definition
|
||||
var pos sql.NullString
|
||||
if err := rows.Scan(&pos, &def.Gloss, &def.Source, &def.Priority); err != nil {
|
||||
return nil, fmt.Errorf("dictionary: define scan: %w", err)
|
||||
}
|
||||
def.POS = pos.String
|
||||
defs = append(defs, def)
|
||||
}
|
||||
if defs == nil {
|
||||
defs = []Definition{}
|
||||
}
|
||||
return defs, rows.Err()
|
||||
}
|
||||
|
||||
func (d *Dictionary) Synonyms(word, lang string) ([]string, error) {
|
||||
rows, err := d.db.Query(`
|
||||
SELECT DISTINCT s.synonym
|
||||
FROM synonyms s
|
||||
JOIN words w ON w.id = s.word_id
|
||||
WHERE w.word = ? AND w.lang = ?
|
||||
ORDER BY s.synonym`,
|
||||
strings.ToLower(word), lang,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: synonyms: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var syns []string
|
||||
for rows.Next() {
|
||||
var syn string
|
||||
if err := rows.Scan(&syn); err != nil {
|
||||
return nil, fmt.Errorf("dictionary: synonyms scan: %w", err)
|
||||
}
|
||||
syns = append(syns, syn)
|
||||
}
|
||||
if syns == nil {
|
||||
syns = []string{}
|
||||
}
|
||||
return syns, rows.Err()
|
||||
}
|
||||
|
||||
func (d *Dictionary) Translate(word, fromLang, toLang string) ([]string, error) {
|
||||
rows, err := d.db.Query(`
|
||||
SELECT DISTINCT t.translation
|
||||
FROM translations t
|
||||
JOIN words w ON w.id = t.word_id
|
||||
WHERE w.word = ? AND w.lang = ? AND t.target_lang = ?
|
||||
ORDER BY t.translation`,
|
||||
strings.ToLower(word), fromLang, toLang,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: translate: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var trans []string
|
||||
for rows.Next() {
|
||||
var t string
|
||||
if err := rows.Scan(&t); err != nil {
|
||||
return nil, fmt.Errorf("dictionary: translate scan: %w", err)
|
||||
}
|
||||
trans = append(trans, t)
|
||||
}
|
||||
if trans == nil {
|
||||
trans = []string{}
|
||||
}
|
||||
return trans, rows.Err()
|
||||
}
|
||||
|
||||
func (d *Dictionary) WordCount() (map[string]int, error) {
|
||||
rows, err := d.db.Query("SELECT lang, COUNT(*) FROM words GROUP BY lang")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: word count: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
counts := make(map[string]int)
|
||||
for rows.Next() {
|
||||
var lang string
|
||||
var count int
|
||||
if err := rows.Scan(&lang, &count); err != nil {
|
||||
return nil, fmt.Errorf("dictionary: word count scan: %w", err)
|
||||
}
|
||||
counts[lang] = count
|
||||
}
|
||||
return counts, rows.Err()
|
||||
}
|
||||
|
||||
func (d *Dictionary) Meta(key string) (string, error) {
|
||||
var val string
|
||||
err := d.db.QueryRow("SELECT value FROM meta WHERE key = ?", key).Scan(&val)
|
||||
if err == sql.ErrNoRows {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("dictionary: meta %s: %w", key, err)
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
func (d *Dictionary) DefCount() (map[string]int, error) {
|
||||
rows, err := d.db.Query(`
|
||||
SELECT w.lang, COUNT(*)
|
||||
FROM definitions d
|
||||
JOIN words w ON w.id = d.word_id
|
||||
GROUP BY w.lang`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: def count: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
counts := make(map[string]int)
|
||||
for rows.Next() {
|
||||
var lang string
|
||||
var count int
|
||||
if err := rows.Scan(&lang, &count); err != nil {
|
||||
return nil, fmt.Errorf("dictionary: def count scan: %w", err)
|
||||
}
|
||||
counts[lang] = count
|
||||
}
|
||||
return counts, rows.Err()
|
||||
}
|
||||
118
internal/loader/dicionario.go
Normal file
118
internal/loader/dicionario.go
Normal file
@@ -0,0 +1,118 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type DicionarioLoader struct{}
|
||||
|
||||
func (DicionarioLoader) Name() string { return "dicionario" }
|
||||
|
||||
func (DicionarioLoader) Load(db *sql.DB, dataDir string) error {
|
||||
path := filepath.Join(dataDir, "dicionario-aberto.xml")
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dicionario: open: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("dicionario: begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmtDef, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority)
|
||||
SELECT id, ?, ?, 'dicionario-aberto', 10 FROM words WHERE word = ? AND lang = 'pt-PT'`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dicionario: prepare: %w", err)
|
||||
}
|
||||
defer stmtDef.Close()
|
||||
|
||||
var defCount, decodeErrors int
|
||||
|
||||
decoder := xml.NewDecoder(f)
|
||||
for {
|
||||
tok, err := decoder.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("dicionario: xml decode: %w", err)
|
||||
}
|
||||
|
||||
se, ok := tok.(xml.StartElement)
|
||||
if !ok || se.Name.Local != "entry" {
|
||||
continue
|
||||
}
|
||||
|
||||
var entry struct {
|
||||
Form struct {
|
||||
Orth string `xml:"orth"`
|
||||
} `xml:"form"`
|
||||
Senses []struct {
|
||||
GramGrp struct {
|
||||
POS string `xml:"pos"`
|
||||
} `xml:"gramGrp"`
|
||||
Def string `xml:"def"`
|
||||
} `xml:"sense"`
|
||||
}
|
||||
if err := decoder.DecodeElement(&entry, &se); err != nil {
|
||||
decodeErrors++
|
||||
slog.Debug("dicionario: skipping entry", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
word := strings.ToLower(strings.TrimSpace(entry.Form.Orth))
|
||||
if word == "" || containsDigit(word) || containsSpace(word) {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, sense := range entry.Senses {
|
||||
gloss := strings.TrimSpace(sense.Def)
|
||||
if gloss == "" {
|
||||
continue
|
||||
}
|
||||
pos := mapPtPOS(strings.TrimSpace(sense.GramGrp.POS))
|
||||
if _, err := stmtDef.Exec(pos, gloss, word); err != nil {
|
||||
return fmt.Errorf("dicionario: insert: %w", err)
|
||||
}
|
||||
defCount++
|
||||
}
|
||||
}
|
||||
|
||||
if decodeErrors > 0 {
|
||||
slog.Warn("dicionario: skipped malformed entries", "count", decodeErrors)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("dicionario: commit: %w", err)
|
||||
}
|
||||
slog.Info("dicionario loaded", "definitions", defCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
func mapPtPOS(pos string) string {
|
||||
pos = strings.ToLower(pos)
|
||||
switch {
|
||||
case strings.Contains(pos, "noun"), strings.HasPrefix(pos, "n."), strings.HasPrefix(pos, "s."),
|
||||
strings.HasPrefix(pos, "m."), strings.HasPrefix(pos, "f."):
|
||||
return "noun"
|
||||
case strings.Contains(pos, "verb"), strings.HasPrefix(pos, "v."):
|
||||
return "verb"
|
||||
case strings.Contains(pos, "adj"):
|
||||
return "adjective"
|
||||
case strings.Contains(pos, "adv"):
|
||||
return "adverb"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
78
internal/loader/hunspell.go
Normal file
78
internal/loader/hunspell.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type HunspellLoader struct {
|
||||
Lang string // "fr" or "pt-PT"
|
||||
FileName string // e.g. "fr_FR/fr.dic" or "pt_PT/pt_PT.dic"
|
||||
}
|
||||
|
||||
func (h HunspellLoader) Name() string { return "hunspell-" + h.Lang }
|
||||
|
||||
func (h HunspellLoader) Load(db *sql.DB, dataDir string) error {
|
||||
path := dataDir + "/" + h.FileName
|
||||
return loadHunspell(db, path, h.Lang)
|
||||
}
|
||||
|
||||
func loadHunspell(db *sql.DB, path, lang string) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hunspell: open %s: %w", path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("hunspell: begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang) VALUES (?, ?)")
|
||||
if err != nil {
|
||||
return fmt.Errorf("hunspell: prepare: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
|
||||
// Skip first line (word count)
|
||||
if scanner.Scan() {
|
||||
// discard
|
||||
}
|
||||
|
||||
var count int
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
// Split on "/" to get base word
|
||||
word := strings.SplitN(line, "/", 2)[0]
|
||||
word = strings.ToLower(word)
|
||||
|
||||
if containsDigit(word) || containsSpace(word) || containsNonLatin(word) {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := stmt.Exec(word, lang); err != nil {
|
||||
return fmt.Errorf("hunspell: insert: %w", err)
|
||||
}
|
||||
count++
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return fmt.Errorf("hunspell: scan: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("hunspell: commit: %w", err)
|
||||
}
|
||||
slog.Info("hunspell loaded", "lang", lang, "words", count)
|
||||
return nil
|
||||
}
|
||||
90
internal/loader/lexique.go
Normal file
90
internal/loader/lexique.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type LexiqueLoader struct{}
|
||||
|
||||
func (LexiqueLoader) Name() string { return "lexique" }
|
||||
|
||||
func (LexiqueLoader) Load(db *sql.DB, dataDir string) error {
|
||||
path := filepath.Join(dataDir, "Lexique383.tsv")
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("lexique: open: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
posMap := map[string]string{
|
||||
"NOM": "noun",
|
||||
"VER": "verb",
|
||||
"ADJ": "adjective",
|
||||
"ADV": "adverb",
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("lexique: begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare(`
|
||||
UPDATE words SET pos = ?, frequency = ?
|
||||
WHERE word = ? AND lang = 'fr' AND pos IS NULL`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("lexique: prepare: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
// Skip header
|
||||
if scanner.Scan() {
|
||||
// discard header line
|
||||
}
|
||||
|
||||
var count int
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
fields := strings.Split(line, "\t")
|
||||
if len(fields) < 9 {
|
||||
continue
|
||||
}
|
||||
|
||||
word := strings.ToLower(fields[0])
|
||||
cgram := fields[3]
|
||||
freqStr := fields[8]
|
||||
|
||||
pos, ok := posMap[cgram]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
freq := 0.0
|
||||
if f, err := strconv.ParseFloat(freqStr, 64); err == nil {
|
||||
freq = f
|
||||
}
|
||||
freqInt := int(freq * 1000)
|
||||
|
||||
if _, err := stmt.Exec(pos, freqInt, word); err != nil {
|
||||
return fmt.Errorf("lexique: update: %w", err)
|
||||
}
|
||||
count++
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return fmt.Errorf("lexique: scan: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("lexique: commit: %w", err)
|
||||
}
|
||||
slog.Info("lexique loaded", "enriched", count)
|
||||
return nil
|
||||
}
|
||||
29
internal/loader/loader.go
Normal file
29
internal/loader/loader.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Loader interface {
|
||||
Name() string
|
||||
Load(db *sql.DB, dataDir string) error
|
||||
}
|
||||
|
||||
func Run(db *sql.DB, dataDir string, loaders []Loader, skip map[string]bool) error {
|
||||
for _, l := range loaders {
|
||||
if skip[l.Name()] {
|
||||
slog.Info("skipping loader", "name", l.Name())
|
||||
continue
|
||||
}
|
||||
slog.Info("running loader", "name", l.Name())
|
||||
start := time.Now()
|
||||
if err := l.Load(db, dataDir); err != nil {
|
||||
return fmt.Errorf("loader %s: %w", l.Name(), err)
|
||||
}
|
||||
slog.Info("loader complete", "name", l.Name(), "elapsed", time.Since(start).Round(time.Millisecond))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
128
internal/loader/scowl.go
Normal file
128
internal/loader/scowl.go
Normal file
@@ -0,0 +1,128 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type SCOWLLoader struct{}
|
||||
|
||||
func (SCOWLLoader) Name() string { return "scowl" }
|
||||
|
||||
func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
|
||||
pattern := filepath.Join(dataDir, "scowl", "final", "english-words.*")
|
||||
files, err := filepath.Glob(pattern)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scowl: glob: %w", err)
|
||||
}
|
||||
if len(files) == 0 {
|
||||
return fmt.Errorf("scowl: no files matching %s", pattern)
|
||||
}
|
||||
|
||||
// Filter to sizes <= 70
|
||||
var selected []string
|
||||
for _, f := range files {
|
||||
base := filepath.Base(f)
|
||||
// Files are like english-words.10, english-words.20, etc.
|
||||
parts := strings.SplitN(base, ".", 2)
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
var size int
|
||||
if _, err := fmt.Sscanf(parts[1], "%d", &size); err != nil {
|
||||
continue
|
||||
}
|
||||
if size <= 70 {
|
||||
selected = append(selected, f)
|
||||
}
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("scowl: begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang) VALUES (?, 'en')")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scowl: prepare: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
var count int
|
||||
for _, f := range selected {
|
||||
n, err := loadSCOWLFile(stmt, f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scowl: load %s: %w", f, err)
|
||||
}
|
||||
count += n
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("scowl: commit: %w", err)
|
||||
}
|
||||
slog.Info("scowl loaded", "words", count, "files", len(selected))
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadSCOWLFile(stmt *sql.Stmt, path string) (int, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var count int
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
word := strings.ToLower(line)
|
||||
if containsAny(word, " ", "'", "-") || containsDigit(word) {
|
||||
continue
|
||||
}
|
||||
if _, err := stmt.Exec(word); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
count++
|
||||
}
|
||||
return count, scanner.Err()
|
||||
}
|
||||
|
||||
func containsAny(s string, subs ...string) bool {
|
||||
for _, sub := range subs {
|
||||
if strings.Contains(s, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsDigit(s string) bool {
|
||||
for _, r := range s {
|
||||
if r >= '0' && r <= '9' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsSpace(s string) bool {
|
||||
return strings.ContainsRune(s, ' ')
|
||||
}
|
||||
|
||||
func containsNonLatin(s string) bool {
|
||||
for _, r := range s {
|
||||
if r > 0x024F && r != '\'' && r != '-' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
242
internal/loader/wiktionary.go
Normal file
242
internal/loader/wiktionary.go
Normal file
@@ -0,0 +1,242 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type WiktionaryLoader struct {
|
||||
Lang string // "en", "fr", "pt-PT"
|
||||
FileName string // e.g. "kaikki-en.jsonl"
|
||||
}
|
||||
|
||||
func (w WiktionaryLoader) Name() string { return "wiktionary-" + w.Lang }
|
||||
|
||||
func (w WiktionaryLoader) Load(db *sql.DB, dataDir string) error {
|
||||
path := filepath.Join(dataDir, w.FileName)
|
||||
return loadWiktionary(db, path, w.Lang)
|
||||
}
|
||||
|
||||
type wiktEntry struct {
|
||||
Word string `json:"word"`
|
||||
POS string `json:"pos"`
|
||||
LangCode string `json:"lang_code"`
|
||||
Senses []struct {
|
||||
Glosses []string `json:"glosses"`
|
||||
Tags []string `json:"tags"`
|
||||
} `json:"senses"`
|
||||
Synonyms []struct {
|
||||
Word string `json:"word"`
|
||||
} `json:"synonyms"`
|
||||
Translations []struct {
|
||||
Code string `json:"code"`
|
||||
Word string `json:"word"`
|
||||
} `json:"translations"`
|
||||
}
|
||||
|
||||
var wiktPOSMap = map[string]string{
|
||||
"noun": "noun",
|
||||
"verb": "verb",
|
||||
"adj": "adjective",
|
||||
"adv": "adverb",
|
||||
"name": "noun",
|
||||
}
|
||||
|
||||
var wiktLangCodeMap = map[string]string{
|
||||
"fr": "fr",
|
||||
"pt": "pt-PT",
|
||||
"en": "en",
|
||||
}
|
||||
|
||||
const (
|
||||
defSQL = `INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority) SELECT id, ?, ?, 'wiktionary', 20 FROM words WHERE word = ? AND lang = ?`
|
||||
synSQL = `INSERT OR IGNORE INTO synonyms (word_id, synonym, source) SELECT id, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?`
|
||||
transSQL = `INSERT OR IGNORE INTO translations (word_id, translation, target_lang, source) SELECT id, ?, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?`
|
||||
)
|
||||
|
||||
// txBatch owns a transaction and its prepared statements.
|
||||
// Close() is always safe to call — it rolls back (no-op after commit) and closes stmts.
|
||||
type txBatch struct {
|
||||
tx *sql.Tx
|
||||
stmtDef *sql.Stmt
|
||||
stmtSyn *sql.Stmt
|
||||
stmtTrans *sql.Stmt
|
||||
}
|
||||
|
||||
func newTxBatch(db *sql.DB) (*txBatch, error) {
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
b := &txBatch{tx: tx}
|
||||
|
||||
b.stmtDef, err = tx.Prepare(defSQL)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
return nil, fmt.Errorf("prepare def: %w", err)
|
||||
}
|
||||
b.stmtSyn, err = tx.Prepare(synSQL)
|
||||
if err != nil {
|
||||
b.stmtDef.Close()
|
||||
tx.Rollback()
|
||||
return nil, fmt.Errorf("prepare syn: %w", err)
|
||||
}
|
||||
b.stmtTrans, err = tx.Prepare(transSQL)
|
||||
if err != nil {
|
||||
b.stmtDef.Close()
|
||||
b.stmtSyn.Close()
|
||||
tx.Rollback()
|
||||
return nil, fmt.Errorf("prepare trans: %w", err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (b *txBatch) Close() {
|
||||
if b == nil {
|
||||
return
|
||||
}
|
||||
b.stmtDef.Close()
|
||||
b.stmtSyn.Close()
|
||||
b.stmtTrans.Close()
|
||||
b.tx.Rollback() // no-op after commit
|
||||
}
|
||||
|
||||
func (b *txBatch) Commit() error {
|
||||
b.stmtDef.Close()
|
||||
b.stmtSyn.Close()
|
||||
b.stmtTrans.Close()
|
||||
return b.tx.Commit()
|
||||
}
|
||||
|
||||
func loadWiktionary(db *sql.DB, path, lang string) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wiktionary: open: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
batch, err := newTxBatch(db)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wiktionary: %w", err)
|
||||
}
|
||||
defer batch.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 0, 10*1024*1024), 10*1024*1024)
|
||||
|
||||
var defCount, synCount, transCount, lineCount int
|
||||
commitInterval := 10000
|
||||
|
||||
for scanner.Scan() {
|
||||
lineCount++
|
||||
|
||||
var entry wiktEntry
|
||||
if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
word := strings.ToLower(entry.Word)
|
||||
if containsDigit(word) || containsSpace(word) {
|
||||
continue
|
||||
}
|
||||
|
||||
pos := wiktPOSMap[entry.POS]
|
||||
|
||||
// Process senses
|
||||
for _, sense := range entry.Senses {
|
||||
if hasTag(sense.Tags, "form-of", "alt-of") {
|
||||
continue
|
||||
}
|
||||
// pt-PT Brazil filtering
|
||||
if lang == "pt-PT" && hasBrazilTag(sense.Tags) {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, gloss := range sense.Glosses {
|
||||
gloss = strings.TrimSpace(gloss)
|
||||
if gloss == "" || strings.HasPrefix(gloss, "(") || len(gloss) > 500 {
|
||||
continue
|
||||
}
|
||||
if _, err := batch.stmtDef.Exec(pos, gloss, word, lang); err != nil {
|
||||
return fmt.Errorf("wiktionary: insert def: %w", err)
|
||||
}
|
||||
defCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Process synonyms
|
||||
for _, syn := range entry.Synonyms {
|
||||
synWord := strings.ToLower(syn.Word)
|
||||
if synWord == "" || containsDigit(synWord) || containsSpace(synWord) {
|
||||
continue
|
||||
}
|
||||
if _, err := batch.stmtSyn.Exec(synWord, word, lang); err != nil {
|
||||
return fmt.Errorf("wiktionary: insert syn: %w", err)
|
||||
}
|
||||
synCount++
|
||||
}
|
||||
|
||||
// Process translations
|
||||
for _, tr := range entry.Translations {
|
||||
targetLang, ok := wiktLangCodeMap[tr.Code]
|
||||
if !ok || targetLang == lang {
|
||||
continue
|
||||
}
|
||||
trWord := strings.ToLower(tr.Word)
|
||||
if trWord == "" || containsDigit(trWord) || containsSpace(trWord) {
|
||||
continue
|
||||
}
|
||||
if _, err := batch.stmtTrans.Exec(trWord, targetLang, word, lang); err != nil {
|
||||
return fmt.Errorf("wiktionary: insert trans: %w", err)
|
||||
}
|
||||
transCount++
|
||||
}
|
||||
|
||||
// Periodic commit — close current batch and start a new one
|
||||
if lineCount%commitInterval == 0 {
|
||||
if err := batch.Commit(); err != nil {
|
||||
return fmt.Errorf("wiktionary: commit: %w", err)
|
||||
}
|
||||
batch, err = newTxBatch(db)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wiktionary: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return fmt.Errorf("wiktionary: scan: %w", err)
|
||||
}
|
||||
|
||||
if err := batch.Commit(); err != nil {
|
||||
return fmt.Errorf("wiktionary: final commit: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("wiktionary loaded", "lang", lang, "definitions", defCount, "synonyms", synCount, "translations", transCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasTag(tags []string, targets ...string) bool {
|
||||
for _, tag := range tags {
|
||||
for _, t := range targets {
|
||||
if tag == t {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasBrazilTag(tags []string) bool {
|
||||
for _, tag := range tags {
|
||||
if tag == "Brazil" || tag == "Brazilian Portuguese" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
187
internal/loader/wiktionary_test.go
Normal file
187
internal/loader/wiktionary_test.go
Normal file
@@ -0,0 +1,187 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"dreamdict/internal/dictionary"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func setupWiktTestDB(t *testing.T) *sql.DB {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(on)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.SetMaxOpenConns(1)
|
||||
if err := dictionary.BootstrapSchema(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func writeJSONL(t *testing.T, dir, name, content string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestFormOfFiltering(t *testing.T) {
|
||||
db := setupWiktTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
// Seed a word so definitions can reference it
|
||||
if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('run', 'en')"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
jsonl := `{"word":"run","pos":"verb","lang_code":"en","senses":[{"glosses":["to move quickly"],"tags":[]},{"glosses":["past tense of run"],"tags":["form-of"]},{"glosses":["alternative form"],"tags":["alt-of"]}],"synonyms":[],"translations":[]}` + "\n"
|
||||
path := writeJSONL(t, dir, "test.jsonl", jsonl)
|
||||
|
||||
if err := loadWiktionary(db, path, "en"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Only the first sense should be loaded — form-of and alt-of should be filtered
|
||||
var count int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM definitions").Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Errorf("expected 1 definition (form-of filtered), got %d", count)
|
||||
}
|
||||
|
||||
// Verify the correct gloss was kept
|
||||
var gloss string
|
||||
if err := db.QueryRow("SELECT gloss FROM definitions").Scan(&gloss); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gloss != "to move quickly" {
|
||||
t.Errorf("expected 'to move quickly', got %q", gloss)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrazilFiltering(t *testing.T) {
|
||||
db := setupWiktTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('autocarro', 'pt-PT')"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
jsonl := `{"word":"autocarro","pos":"noun","lang_code":"pt","senses":[{"glosses":["a bus (Portugal)"],"tags":["Portugal"]},{"glosses":["a bus (Brazil)"],"tags":["Brazil"]},{"glosses":["a vehicle for transport"],"tags":[]}],"synonyms":[],"translations":[]}` + "\n"
|
||||
path := writeJSONL(t, dir, "test.jsonl", jsonl)
|
||||
|
||||
if err := loadWiktionary(db, path, "pt-PT"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Brazil-tagged sense should be filtered; Portugal-tagged and untagged should remain
|
||||
var count int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM definitions").Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Errorf("expected 2 definitions (Brazil filtered), got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrazilianPortugueseTagFiltering(t *testing.T) {
|
||||
db := setupWiktTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('ônibus', 'pt-PT')"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
jsonl := `{"word":"ônibus","pos":"noun","lang_code":"pt","senses":[{"glosses":["a bus"],"tags":["Brazilian Portuguese"]}],"synonyms":[],"translations":[]}` + "\n"
|
||||
path := writeJSONL(t, dir, "test.jsonl", jsonl)
|
||||
|
||||
if err := loadWiktionary(db, path, "pt-PT"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var count int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM definitions").Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("expected 0 definitions (Brazilian Portuguese filtered), got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlossFiltering(t *testing.T) {
|
||||
db := setupWiktTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('test', 'en')"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
jsonl := `{"word":"test","pos":"noun","lang_code":"en","senses":[{"glosses":["a valid definition"],"tags":[]},{"glosses":["(archaic form)"],"tags":[]},{"glosses":[""],"tags":[]}],"synonyms":[],"translations":[]}` + "\n"
|
||||
path := writeJSONL(t, dir, "test.jsonl", jsonl)
|
||||
|
||||
if err := loadWiktionary(db, path, "en"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Only "a valid definition" should pass — parenthetical and empty filtered
|
||||
var count int
|
||||
if err := db.QueryRow("SELECT COUNT(*) FROM definitions").Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Errorf("expected 1 definition (gloss filtering), got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasTag(t *testing.T) {
|
||||
tests := []struct {
|
||||
tags []string
|
||||
targets []string
|
||||
want bool
|
||||
}{
|
||||
{[]string{"form-of", "verb"}, []string{"form-of"}, true},
|
||||
{[]string{"verb"}, []string{"form-of", "alt-of"}, false},
|
||||
{[]string{"alt-of"}, []string{"form-of", "alt-of"}, true},
|
||||
{nil, []string{"form-of"}, false},
|
||||
{[]string{}, []string{"form-of"}, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := hasTag(tt.tags, tt.targets...)
|
||||
if got != tt.want {
|
||||
t.Errorf("hasTag(%v, %v) = %v, want %v", tt.tags, tt.targets, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasBrazilTag(t *testing.T) {
|
||||
tests := []struct {
|
||||
tags []string
|
||||
want bool
|
||||
}{
|
||||
{[]string{"Brazil"}, true},
|
||||
{[]string{"Brazilian Portuguese"}, true},
|
||||
{[]string{"Portugal"}, false},
|
||||
{[]string{"European Portuguese"}, false},
|
||||
{nil, false},
|
||||
{[]string{}, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := hasBrazilTag(tt.tags)
|
||||
if got != tt.want {
|
||||
t.Errorf("hasBrazilTag(%v) = %v, want %v", tt.tags, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
140
internal/loader/wolf.go
Normal file
140
internal/loader/wolf.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"compress/bzip2"
|
||||
"database/sql"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type WOLFLoader struct{}
|
||||
|
||||
func (WOLFLoader) Name() string { return "wolf" }
|
||||
|
||||
func (WOLFLoader) Load(db *sql.DB, dataDir string) error {
|
||||
// Try decompressed first, then compressed
|
||||
path := filepath.Join(dataDir, "wolf.xml")
|
||||
var reader io.Reader
|
||||
var closer io.Closer
|
||||
|
||||
if f, err := os.Open(path); err == nil {
|
||||
reader = f
|
||||
closer = f
|
||||
} else {
|
||||
bzPath := filepath.Join(dataDir, "wolf-1.0b4.xml.bz2")
|
||||
f, err := os.Open(bzPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wolf: open: %w (also tried %s)", err, path)
|
||||
}
|
||||
reader = bzip2.NewReader(f)
|
||||
closer = f
|
||||
}
|
||||
defer closer.Close()
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("wolf: begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmtDef, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority)
|
||||
SELECT id, ?, ?, 'wolf', 10 FROM words WHERE word = ? AND lang = 'fr'`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wolf: prepare def: %w", err)
|
||||
}
|
||||
defer stmtDef.Close()
|
||||
|
||||
stmtSyn, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO synonyms (word_id, synonym, source)
|
||||
SELECT id, ?, 'wolf' FROM words WHERE word = ? AND lang = 'fr'`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wolf: prepare syn: %w", err)
|
||||
}
|
||||
defer stmtSyn.Close()
|
||||
|
||||
posMap := map[string]string{
|
||||
"n": "noun",
|
||||
"v": "verb",
|
||||
"a": "adjective",
|
||||
"r": "adverb",
|
||||
}
|
||||
|
||||
var defCount, synCount, decodeErrors int
|
||||
|
||||
decoder := xml.NewDecoder(reader)
|
||||
for {
|
||||
tok, err := decoder.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("wolf: xml decode: %w", err)
|
||||
}
|
||||
|
||||
se, ok := tok.(xml.StartElement)
|
||||
if !ok || se.Name.Local != "SYNSET" {
|
||||
continue
|
||||
}
|
||||
|
||||
var synset struct {
|
||||
POS string `xml:"POS"`
|
||||
Literals []struct {
|
||||
Value string `xml:",chardata"`
|
||||
} `xml:"SYNONYM>LITERAL"`
|
||||
DEF string `xml:"DEF"`
|
||||
}
|
||||
if err := decoder.DecodeElement(&synset, &se); err != nil {
|
||||
decodeErrors++
|
||||
slog.Debug("wolf: skipping synset", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
pos := posMap[strings.ToLower(synset.POS)]
|
||||
gloss := strings.TrimSpace(synset.DEF)
|
||||
|
||||
var words []string
|
||||
for _, lit := range synset.Literals {
|
||||
w := strings.ToLower(strings.TrimSpace(lit.Value))
|
||||
if w != "" && !containsDigit(w) && !containsSpace(w) {
|
||||
words = append(words, w)
|
||||
}
|
||||
}
|
||||
|
||||
if gloss != "" {
|
||||
for _, w := range words {
|
||||
if _, err := stmtDef.Exec(pos, gloss, w); err != nil {
|
||||
return fmt.Errorf("wolf: insert def: %w", err)
|
||||
}
|
||||
defCount++
|
||||
}
|
||||
}
|
||||
|
||||
for i, w1 := range words {
|
||||
for j, w2 := range words {
|
||||
if i == j {
|
||||
continue
|
||||
}
|
||||
if _, err := stmtSyn.Exec(w2, w1); err != nil {
|
||||
return fmt.Errorf("wolf: insert syn: %w", err)
|
||||
}
|
||||
synCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if decodeErrors > 0 {
|
||||
slog.Warn("wolf: skipped malformed entries", "count", decodeErrors)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("wolf: commit: %w", err)
|
||||
}
|
||||
slog.Info("wolf loaded", "definitions", defCount, "synonyms", synCount)
|
||||
return nil
|
||||
}
|
||||
161
internal/loader/wordnet.go
Normal file
161
internal/loader/wordnet.go
Normal file
@@ -0,0 +1,161 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type WordNetLoader struct{}
|
||||
|
||||
func (WordNetLoader) Name() string { return "wordnet" }
|
||||
|
||||
func (WordNetLoader) Load(db *sql.DB, dataDir string) error {
|
||||
posFiles := map[string]string{
|
||||
"data.noun": "noun",
|
||||
"data.verb": "verb",
|
||||
"data.adj": "adjective",
|
||||
"data.adv": "adverb",
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmtDef, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority)
|
||||
SELECT id, ?, ?, 'wordnet', 10 FROM words WHERE word = ? AND lang = 'en'`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: prepare def: %w", err)
|
||||
}
|
||||
defer stmtDef.Close()
|
||||
|
||||
stmtSyn, err := tx.Prepare(`
|
||||
INSERT OR IGNORE INTO synonyms (word_id, synonym, source)
|
||||
SELECT id, ?, 'wordnet' FROM words WHERE word = ? AND lang = 'en'`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: prepare syn: %w", err)
|
||||
}
|
||||
defer stmtSyn.Close()
|
||||
|
||||
var defCount, synCount int
|
||||
|
||||
for file, pos := range posFiles {
|
||||
path := filepath.Join(dataDir, "wordnet", "dict", file)
|
||||
d, s, err := loadWordNetFile(stmtDef, stmtSyn, path, pos)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wordnet: %s: %w", file, err)
|
||||
}
|
||||
defCount += d
|
||||
synCount += s
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("wordnet: commit: %w", err)
|
||||
}
|
||||
slog.Info("wordnet loaded", "definitions", defCount, "synonyms", synCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadWordNetFile(stmtDef, stmtSyn *sql.Stmt, path, pos string) (int, int, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var defCount, synCount int
|
||||
scanner := bufio.NewScanner(f)
|
||||
buf := make([]byte, 0, 64*1024)
|
||||
scanner.Buffer(buf, 1024*1024)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
// Skip header lines (start with two spaces)
|
||||
if strings.HasPrefix(line, " ") {
|
||||
continue
|
||||
}
|
||||
|
||||
words, gloss := parseWordNetLine(line)
|
||||
if gloss == "" || len(words) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Insert definitions for each word
|
||||
for _, w := range words {
|
||||
if _, err := stmtDef.Exec(pos, gloss, w); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
defCount++
|
||||
}
|
||||
|
||||
// Insert synonym pairs (skip words with underscores)
|
||||
var cleanWords []string
|
||||
for _, w := range words {
|
||||
if !strings.Contains(w, "_") {
|
||||
cleanWords = append(cleanWords, w)
|
||||
}
|
||||
}
|
||||
for i, w1 := range cleanWords {
|
||||
for j, w2 := range cleanWords {
|
||||
if i == j {
|
||||
continue
|
||||
}
|
||||
if _, err := stmtSyn.Exec(w2, w1); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
synCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
return defCount, synCount, scanner.Err()
|
||||
}
|
||||
|
||||
func parseWordNetLine(line string) ([]string, string) {
|
||||
// Format: synset_offset lex_filenum ss_type w_cnt word lex_id [word lex_id...] p_cnt [ptr...] | gloss
|
||||
glossIdx := strings.Index(line, "| ")
|
||||
if glossIdx == -1 {
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
gloss := strings.TrimSpace(line[glossIdx+2:])
|
||||
// Take gloss before first ";" (drops usage examples)
|
||||
if semiIdx := strings.Index(gloss, ";"); semiIdx != -1 {
|
||||
gloss = strings.TrimSpace(gloss[:semiIdx])
|
||||
}
|
||||
if gloss == "" {
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
dataPart := line[:glossIdx]
|
||||
fields := strings.Fields(dataPart)
|
||||
if len(fields) < 6 {
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
// fields[3] = w_cnt (hex)
|
||||
wc, err := strconv.ParseInt(fields[3], 16, 0)
|
||||
if err != nil {
|
||||
slog.Debug("wordnet: bad word count", "field", fields[3], "error", err)
|
||||
return nil, ""
|
||||
}
|
||||
wordCount := int(wc)
|
||||
if wordCount <= 0 || wordCount > 100 {
|
||||
slog.Debug("wordnet: unreasonable word count", "count", wordCount)
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
var words []string
|
||||
for i := 0; i < wordCount && 4+i*2 < len(fields); i++ {
|
||||
w := strings.ToLower(fields[4+i*2])
|
||||
words = append(words, w)
|
||||
}
|
||||
return words, gloss
|
||||
}
|
||||
Reference in New Issue
Block a user