eeea8d4d34
Go-based dictionary backend providing word validation, definitions, synonyms, and cross-language translation for English, French, and Portuguese. Designed as a self-hosted replacement for Wordnik, backing GogoBee. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
129 lines
2.5 KiB
Go
129 lines
2.5 KiB
Go
package loader
|
|
|
|
import (
|
|
"bufio"
|
|
"database/sql"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
)
|
|
|
|
type SCOWLLoader struct{}
|
|
|
|
func (SCOWLLoader) Name() string { return "scowl" }
|
|
|
|
func (SCOWLLoader) Load(db *sql.DB, dataDir string) error {
|
|
pattern := filepath.Join(dataDir, "scowl", "final", "english-words.*")
|
|
files, err := filepath.Glob(pattern)
|
|
if err != nil {
|
|
return fmt.Errorf("scowl: glob: %w", err)
|
|
}
|
|
if len(files) == 0 {
|
|
return fmt.Errorf("scowl: no files matching %s", pattern)
|
|
}
|
|
|
|
// Filter to sizes <= 70
|
|
var selected []string
|
|
for _, f := range files {
|
|
base := filepath.Base(f)
|
|
// Files are like english-words.10, english-words.20, etc.
|
|
parts := strings.SplitN(base, ".", 2)
|
|
if len(parts) != 2 {
|
|
continue
|
|
}
|
|
var size int
|
|
if _, err := fmt.Sscanf(parts[1], "%d", &size); err != nil {
|
|
continue
|
|
}
|
|
if size <= 70 {
|
|
selected = append(selected, f)
|
|
}
|
|
}
|
|
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
return fmt.Errorf("scowl: begin tx: %w", err)
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
stmt, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang) VALUES (?, 'en')")
|
|
if err != nil {
|
|
return fmt.Errorf("scowl: prepare: %w", err)
|
|
}
|
|
defer stmt.Close()
|
|
|
|
var count int
|
|
for _, f := range selected {
|
|
n, err := loadSCOWLFile(stmt, f)
|
|
if err != nil {
|
|
return fmt.Errorf("scowl: load %s: %w", f, err)
|
|
}
|
|
count += n
|
|
}
|
|
|
|
if err := tx.Commit(); err != nil {
|
|
return fmt.Errorf("scowl: commit: %w", err)
|
|
}
|
|
slog.Info("scowl loaded", "words", count, "files", len(selected))
|
|
return nil
|
|
}
|
|
|
|
func loadSCOWLFile(stmt *sql.Stmt, path string) (int, error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer f.Close()
|
|
|
|
var count int
|
|
scanner := bufio.NewScanner(f)
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
word := strings.ToLower(line)
|
|
if containsAny(word, " ", "'", "-") || containsDigit(word) {
|
|
continue
|
|
}
|
|
if _, err := stmt.Exec(word); err != nil {
|
|
return 0, err
|
|
}
|
|
count++
|
|
}
|
|
return count, scanner.Err()
|
|
}
|
|
|
|
func containsAny(s string, subs ...string) bool {
|
|
for _, sub := range subs {
|
|
if strings.Contains(s, sub) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func containsDigit(s string) bool {
|
|
for _, r := range s {
|
|
if r >= '0' && r <= '9' {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func containsSpace(s string) bool {
|
|
return strings.ContainsRune(s, ' ')
|
|
}
|
|
|
|
func containsNonLatin(s string) bool {
|
|
for _, r := range s {
|
|
if r > 0x024F && r != '\'' && r != '-' {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|