eeea8d4d34
Go-based dictionary backend providing word validation, definitions, synonyms, and cross-language translation for English, French, and Portuguese. Designed as a self-hosted replacement for Wordnik, backing GogoBee. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
91 lines
1.7 KiB
Go
91 lines
1.7 KiB
Go
package loader
|
|
|
|
import (
|
|
"bufio"
|
|
"database/sql"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
type LexiqueLoader struct{}
|
|
|
|
func (LexiqueLoader) Name() string { return "lexique" }
|
|
|
|
func (LexiqueLoader) Load(db *sql.DB, dataDir string) error {
|
|
path := filepath.Join(dataDir, "Lexique383.tsv")
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return fmt.Errorf("lexique: open: %w", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
posMap := map[string]string{
|
|
"NOM": "noun",
|
|
"VER": "verb",
|
|
"ADJ": "adjective",
|
|
"ADV": "adverb",
|
|
}
|
|
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
return fmt.Errorf("lexique: begin tx: %w", err)
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
stmt, err := tx.Prepare(`
|
|
UPDATE words SET pos = ?, frequency = ?
|
|
WHERE word = ? AND lang = 'fr' AND pos IS NULL`)
|
|
if err != nil {
|
|
return fmt.Errorf("lexique: prepare: %w", err)
|
|
}
|
|
defer stmt.Close()
|
|
|
|
scanner := bufio.NewScanner(f)
|
|
// Skip header
|
|
if scanner.Scan() {
|
|
// discard header line
|
|
}
|
|
|
|
var count int
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
fields := strings.Split(line, "\t")
|
|
if len(fields) < 9 {
|
|
continue
|
|
}
|
|
|
|
word := strings.ToLower(fields[0])
|
|
cgram := fields[3]
|
|
freqStr := fields[8]
|
|
|
|
pos, ok := posMap[cgram]
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
freq := 0.0
|
|
if f, err := strconv.ParseFloat(freqStr, 64); err == nil {
|
|
freq = f
|
|
}
|
|
freqInt := int(freq * 1000)
|
|
|
|
if _, err := stmt.Exec(pos, freqInt, word); err != nil {
|
|
return fmt.Errorf("lexique: update: %w", err)
|
|
}
|
|
count++
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return fmt.Errorf("lexique: scan: %w", err)
|
|
}
|
|
|
|
if err := tx.Commit(); err != nil {
|
|
return fmt.Errorf("lexique: commit: %w", err)
|
|
}
|
|
slog.Info("lexique loaded", "enriched", count)
|
|
return nil
|
|
}
|