From eeea8d4d348ae14e35b5568d05596806f47710f7 Mon Sep 17 00:00:00 2001 From: prosolis <5590409+prosolis@users.noreply.github.com> Date: Sat, 28 Mar 2026 16:57:40 -0700 Subject: [PATCH] Initial commit: DreamDict HTTP dictionary service Go-based dictionary backend providing word validation, definitions, synonyms, and cross-language translation for English, French, and Portuguese. Designed as a self-hosted replacement for Wordnik, backing GogoBee. Co-Authored-By: Claude Opus 4.6 --- .gitignore | 5 + README.md | 192 ++++++++ data/.gitkeep | 0 deploy/dreamdict.service | 25 + dreamdict-blueprint.md | 763 +++++++++++++++++++++++++++++ go.mod | 17 + go.sum | 51 ++ internal/dictionary/db.go | 98 ++++ internal/dictionary/dict.go | 93 ++++ internal/dictionary/dict_test.go | 313 ++++++++++++ internal/dictionary/query.go | 193 ++++++++ internal/loader/dicionario.go | 118 +++++ internal/loader/hunspell.go | 78 +++ internal/loader/lexique.go | 90 ++++ internal/loader/loader.go | 29 ++ internal/loader/scowl.go | 128 +++++ internal/loader/wiktionary.go | 242 +++++++++ internal/loader/wiktionary_test.go | 187 +++++++ internal/loader/wolf.go | 140 ++++++ internal/loader/wordnet.go | 161 ++++++ scripts/download-dict-data.sh | 333 +++++++++++++ 21 files changed, 3256 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 data/.gitkeep create mode 100644 deploy/dreamdict.service create mode 100644 dreamdict-blueprint.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/dictionary/db.go create mode 100644 internal/dictionary/dict.go create mode 100644 internal/dictionary/dict_test.go create mode 100644 internal/dictionary/query.go create mode 100644 internal/loader/dicionario.go create mode 100644 internal/loader/hunspell.go create mode 100644 internal/loader/lexique.go create mode 100644 internal/loader/loader.go create mode 100644 internal/loader/scowl.go create mode 100644 internal/loader/wiktionary.go create mode 100644 internal/loader/wiktionary_test.go create mode 100644 internal/loader/wolf.go create mode 100644 internal/loader/wordnet.go create mode 100755 scripts/download-dict-data.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..064178d --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +dict.db +data/ +!data/.gitkeep +dictimport +server diff --git a/README.md b/README.md new file mode 100644 index 0000000..c81783b --- /dev/null +++ b/README.md @@ -0,0 +1,192 @@ +# DreamDict + +A self-contained HTTP dictionary service providing word validation, random word generation, definitions, synonyms, and cross-language translation for English (`en`), French (`fr`), and European Portuguese (`pt-PT`). + +Runs as a plain Go binary managed by systemd. Binds to localhost only — no external network exposure. Designed as a dictionary backend for [GogoBee](https://github.com/reala-misaki/gogobee), replacing Wordnik. + +## Requirements + +- Go 1.25+ +- No CGO required (uses [modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite)) + +## Quick Start + +```bash +# Download all dictionary source data (~2-3 GB) +./scripts/download-dict-data.sh + +# Build the database (~310 MB) +go run ./cmd/dictimport --data ./data --db ./dict.db + +# Start the server +go run ./cmd/server --db ./dict.db +``` + +The server listens on `127.0.0.1:7777` by default. + +## API + +All endpoints are read-only `GET` requests returning JSON. + +### `GET /valid?word=...&lang=...` + +Check if a word exists in a language's word list. + +``` +$ curl 'localhost:7777/valid?word=ephemeral&lang=en' +{"valid":true} +``` + +### `GET /random?lang=...` + +Return a random word. Optional filters: `pos` (`noun`, `verb`, `adjective`, `adverb`), `min`/`max` (word length), `min_freq` (frequency score, meaningful for `fr`). + +``` +$ curl 'localhost:7777/random?lang=en&pos=noun&min=5&max=8' +{"word":"lantern"} +``` + +### `GET /define?word=...&lang=...` + +Return all definitions, ordered by source priority. + +``` +$ curl 'localhost:7777/define?word=ephemeral&lang=en' +{"word":"ephemeral","lang":"en","definitions":[ + {"pos":"adjective","gloss":"lasting for a very short time","source":"wordnet","priority":10}, + {"pos":"adjective","gloss":"lasting only for a day","source":"wiktionary","priority":20} +]} +``` + +### `GET /synonyms?word=...&lang=...` + +Return deduplicated synonyms. + +``` +$ curl 'localhost:7777/synonyms?word=happy&lang=en' +{"word":"happy","lang":"en","synonyms":["content","glad","joyful","pleased"]} +``` + +### `GET /translate?word=...&from=...&to=...` + +Return known translations between supported languages. + +``` +$ curl 'localhost:7777/translate?word=cat&from=en&to=fr' +{"word":"cat","from":"en","to":"fr","translations":["chat","minet"]} +``` + +### `GET /health` + +Health check with database stats. + +``` +$ curl localhost:7777/health +{"status":"ok","db_path":"./dict.db","word_counts":{"en":214832,"fr":82104,"pt-PT":71088},...} +``` + +### Error Responses + +```json +{"error": "bad_request", "message": "word and lang are required"} +{"error": "no_match", "message": "no words match the given filters"} +``` + +## Server Configuration + +| Flag | Env Var | Default | Description | +|---|---|---|---| +| `--port` | `DREAMDICT_PORT` | `7777` | Listen port | +| `--host` | `DREAMDICT_HOST` | `127.0.0.1` | Listen host | +| `--db` | `DREAMDICT_DB` | `./dict.db` | Path to SQLite database | +| `--log-level` | | `info` | `debug`, `info`, `warn`, `error` | + +Flags take precedence over environment variables. + +## Import CLI + +```bash +go run ./cmd/dictimport [flags] +``` + +| Flag | Default | Description | +|---|---|---| +| `--db` | `./dict.db` | Output database path | +| `--data` | `./data` | Source data directory | +| `--langs` | `en,fr,pt-PT` | Languages to import | +| `--clean` | `false` | Drop and recreate all tables first | +| `--skip` | | Comma-separated loader names to skip | + +Loader names: `scowl`, `wordnet`, `wiktionary-en`, `hunspell-fr`, `lexique`, `wolf`, `wiktionary-fr`, `hunspell-pt`, `dicionario`, `wiktionary-pt`. + +## Data Sources + +| Source | Language | What it provides | +|---|---|---| +| [SCOWL](https://wordlist.aspell.net/) | en | Word list | +| [WordNet](https://wordnet.princeton.edu/) | en | Definitions + synonyms | +| [Hunspell/LibreOffice](https://github.com/LibreOffice/dictionaries) | fr, pt-PT | Word lists | +| [Lexique](http://www.lexique.org/) | fr | POS + frequency enrichment | +| [WOLF](https://github.com/nicolashernandez/WOLF) | fr | Definitions + synonyms | +| [Dicionario Aberto](https://github.com/jorgecardoso/dicionario-aberto) | pt-PT | Definitions | +| [Wiktionary (kaikki.org)](https://kaikki.org/) | all | Supplemental definitions, synonyms, translations | + +Run `./scripts/download-dict-data.sh` to fetch everything. Pass `--force` to re-download existing files. + +## Deployment + +A systemd unit file is provided at `deploy/dreamdict.service`. + +```bash +# Create system user +useradd --system --no-create-home --shell /usr/sbin/nologin dreamdict + +# Build and install +go build -o /usr/local/bin/dreamdict ./cmd/server +mkdir -p /opt/dreamdict /etc/dreamdict +go run ./cmd/dictimport --db /opt/dreamdict/dict.db --data ./data +chown -R dreamdict:dreamdict /opt/dreamdict + +# Install service +cp deploy/dreamdict.service /etc/systemd/system/ +echo 'DREAMDICT_PORT=7777' > /etc/dreamdict/dreamdict.env +echo 'DREAMDICT_DB=/opt/dreamdict/dict.db' >> /etc/dreamdict/dreamdict.env +systemctl daemon-reload +systemctl enable --now dreamdict +``` + +### Updating the Database + +```bash +systemctl stop dreamdict +go run ./cmd/dictimport --db /opt/dreamdict/dict.db --clean +chown dreamdict:dreamdict /opt/dreamdict/dict.db +systemctl start dreamdict +``` + +## Testing + +```bash +go test ./... +``` + +## Project Structure + +``` +cmd/server/ HTTP server +cmd/dictimport/ One-time data import CLI +internal/dictionary/ Core package: DB schema, types, queries +internal/loader/ Data source parsers (one per source) +scripts/ Download script for source data +deploy/ systemd unit file +data/ Source data files (gitignored) +``` + +## Expected Database Size + +| Language | Words | Definitions | Size | +|---|---|---|---| +| en | ~220k | ~380k | ~180 MB | +| fr | ~90k | ~200k | ~80 MB | +| pt-PT | ~75k | ~120k | ~50 MB | +| **Total** | | | **~310 MB** | diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/deploy/dreamdict.service b/deploy/dreamdict.service new file mode 100644 index 0000000..f370910 --- /dev/null +++ b/deploy/dreamdict.service @@ -0,0 +1,25 @@ +[Unit] +Description=DreamDict dictionary service +After=network.target + +[Service] +Type=simple +User=dreamdict +Group=dreamdict +EnvironmentFile=/etc/dreamdict/dreamdict.env +ExecStart=/usr/local/bin/dreamdict server \ + --host 127.0.0.1 \ + --port 7777 \ + --db /opt/dreamdict/dict.db +Restart=on-failure +RestartSec=5s + +# Hardening +NoNewPrivileges=true +ProtectSystem=strict +ReadWritePaths=/opt/dreamdict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/dreamdict-blueprint.md b/dreamdict-blueprint.md new file mode 100644 index 0000000..1274cfb --- /dev/null +++ b/dreamdict-blueprint.md @@ -0,0 +1,763 @@ +# DreamDict — Standalone Dictionary Service: Implementation Blueprint + +## Overview + +`dreamdict` is a self-contained HTTP dictionary service providing word validation, random +word generation, definitions, synonyms, and cross-language translation for English (`en`), +French (`fr`), and European Portuguese (`pt-PT`). + +It runs as a plain Go binary managed by systemd on the same host as GogoBee and Ollama. +GogoBee calls it over localhost HTTP. No Docker, no Traefik, no external network exposure. + +If GogoBee later containerizes, only the service URL in GogoBee's config changes. The +service itself is unmodified. + +--- + +## Motivation + +GogoBee is approaching 50,000 lines of code across 47+ modules. A dictionary package with +loaders, a SQLite dependency, and an import CLI would compound that bloat. Dictionary logic +has no business living inside a Matrix bot. `dreamdict` is a clean extraction: self-contained +infrastructure with a stable HTTP API, independently deployable, independently updatable, +with no knowledge of Matrix, rooms, or bot commands. + +--- + +## Repository + +Separate repository: `dreamdict` (not a GogoBee internal package). + +``` +dreamdict/ +├── cmd/ +│ ├── server/ +│ │ └── main.go # HTTP server binary +│ └── dictimport/ +│ └── main.go # One-time data import CLI +├── internal/ +│ ├── dictionary/ +│ │ ├── dict.go # Core package: public API +│ │ ├── db.go # SQLite connection + schema bootstrap +│ │ └── query.go # IsValidWord, RandomWord, Define, Synonyms, Translate +│ └── loader/ +│ ├── loader.go # Shared interface + orchestrator +│ ├── scowl.go # English word list +│ ├── hunspell.go # French + pt-PT word lists (shared, parameterized) +│ ├── wordnet.go # English definitions + synonyms +│ ├── lexique.go # French POS + frequency enrichment +│ ├── wolf.go # French definitions + synonyms +│ ├── dicionario.go # pt-PT definitions +│ └── wiktionary.go # All langs: supplemental definitions + translations +├── data/ +│ └── .gitkeep # Source data files (not committed) +├── dict.db # Generated SQLite database (not committed) +├── scripts/ +│ └── download-dict-data.sh +├── deploy/ +│ └── dreamdict.service # systemd unit file +├── go.mod +└── README.md +``` + +--- + +## HTTP API + +The server listens on `127.0.0.1:7777` by default (configurable via `--port` flag or +`DREAMDICT_PORT` env var). All responses are JSON. All endpoints are read-only GET requests. + +### `GET /valid` + +Check if a word exists in a language's word list. + +**Query params:** `word` (required), `lang` (required: `en`, `fr`, `pt-PT`) + +**Response:** +```json +{"valid": true} +``` + +--- + +### `GET /random` + +Return a random word for a language, with optional filters. + +**Query params:** +- `lang` (required) +- `pos` (optional: `noun`, `verb`, `adjective`, `adverb`) +- `min` (optional: minimum word length, integer) +- `max` (optional: maximum word length, integer) +- `min_freq` (optional: minimum frequency score, integer; only meaningful for `fr`) + +**Response:** +```json +{"word": "ephemeral"} +``` + +**Error (no words match filters):** +```json +{"error": "no_match", "message": "no words match the given filters"} +``` + +--- + +### `GET /define` + +Return all definitions for a word, ordered by source priority. + +**Query params:** `word` (required), `lang` (required) + +**Response:** +```json +{ + "word": "ephemeral", + "lang": "en", + "definitions": [ + {"pos": "adjective", "gloss": "lasting for a very short time", "source": "wordnet", "priority": 10}, + {"pos": "adjective", "gloss": "lasting only for a day", "source": "wiktionary", "priority": 20} + ] +} +``` + +**Word exists but has no definitions:** +```json +{"word": "...", "lang": "...", "definitions": []} +``` + +--- + +### `GET /synonyms` + +Return deduplicated synonyms for a word. + +**Query params:** `word` (required), `lang` (required) + +**Response:** +```json +{"word": "happy", "lang": "en", "synonyms": ["glad", "joyful", "content", "pleased"]} +``` + +--- + +### `GET /translate` + +Return known equivalents of a word in another language. + +**Query params:** `word` (required), `from` (required), `to` (required) + +**Response:** +```json +{"word": "cat", "from": "en", "to": "fr", "translations": ["chat", "minet"]} +``` + +--- + +### `GET /health` + +Health check and database stats. + +**Response:** +```json +{ + "status": "ok", + "db_path": "/opt/dreamdict/dict.db", + "word_counts": {"en": 214832, "fr": 82104, "pt-PT": 71088}, + "def_counts": {"en": 380000, "fr": 200000, "pt-PT": 120000}, + "imported_at": "2025-03-28T14:22:00Z", + "schema_version": "1" +} +``` + +--- + +## Core Package (`internal/dictionary`) + +### `dict.go` + +```go +package dictionary + +type Options struct { + MinLength int + MaxLength int + POS string // "", "noun", "verb", "adjective", "adverb" + MinFrequency int // 0 = no filter +} + +type Definition struct { + POS string + Gloss string + Source string + Priority int +} + +type Dictionary struct { /* unexported */ } + +func New(dbPath string) (*Dictionary, error) +func (d *Dictionary) Close() error +func (d *Dictionary) IsValidWord(word, lang string) (bool, error) +func (d *Dictionary) RandomWord(lang string, opts Options) (string, error) +func (d *Dictionary) Define(word, lang string) ([]Definition, error) +func (d *Dictionary) Synonyms(word, lang string) ([]string, error) +func (d *Dictionary) Translate(word, fromLang, toLang string) ([]string, error) +func (d *Dictionary) WordCount() (map[string]int, error) +func (d *Dictionary) SupportedLangs() ([]string, error) + +func ValidLang(lang string) bool // "en", "fr", "pt-PT" + +var ( + ErrNoMatch = errors.New("dictionary: no word matches filters") + ErrNotSeeded = errors.New("dictionary: database not seeded, run: go run ./cmd/dictimport") +) +``` + +--- + +## Database Schema + +Use `modernc.org/sqlite` (pure Go, no CGO). + +```sql +CREATE TABLE IF NOT EXISTS words ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word TEXT NOT NULL, + lang TEXT NOT NULL, + pos TEXT, + frequency INTEGER DEFAULT 0, + UNIQUE(word, lang) +); + +CREATE INDEX IF NOT EXISTS idx_words_lang ON words(lang); +CREATE INDEX IF NOT EXISTS idx_words_lang_pos ON words(lang, pos); +CREATE INDEX IF NOT EXISTS idx_words_word ON words(word); + +CREATE TABLE IF NOT EXISTS definitions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + pos TEXT, + gloss TEXT NOT NULL, + source TEXT NOT NULL, + priority INTEGER NOT NULL DEFAULT 99, + UNIQUE(word_id, gloss) +); + +CREATE INDEX IF NOT EXISTS idx_definitions_word_id ON definitions(word_id); +CREATE INDEX IF NOT EXISTS idx_definitions_word_id_priority ON definitions(word_id, priority); + +CREATE TABLE IF NOT EXISTS synonyms ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + synonym TEXT NOT NULL, + source TEXT NOT NULL, + UNIQUE(word_id, synonym) +); + +CREATE INDEX IF NOT EXISTS idx_synonyms_word_id ON synonyms(word_id); + +CREATE TABLE IF NOT EXISTS translations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + translation TEXT NOT NULL, + target_lang TEXT NOT NULL, + source TEXT NOT NULL, + UNIQUE(word_id, translation, target_lang) +); + +CREATE INDEX IF NOT EXISTS idx_translations_word_id ON translations(word_id); + +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +``` + +### Definition Priority + +| Source | Priority | +|---|---| +| `wordnet` | 10 | +| `wolf` | 10 | +| `dicionario-aberto` | 10 | +| `wiktionary` | 20 | + +--- + +## Server Binary (`cmd/server/main.go`) + +### Flags / Env Vars + +``` +--port PORT Listen port (default 7777) DREAMDICT_PORT +--db PATH Path to dict.db DREAMDICT_DB +--host HOST Listen host (default 127.0.0.1) DREAMDICT_HOST +``` + +Default host is `127.0.0.1` — the service binds localhost only. It is never exposed to the +external network. If GogoBee containerizes in future, change host to `0.0.0.0` and control +access at the Docker network level. + +### Startup Behaviour + +- Open the SQLite database on startup; log and exit if unavailable or not seeded +- Register HTTP handlers for all six endpoints +- Log `dreamdict listening on 127.0.0.1:7777` on successful start +- Handle `SIGTERM` / `SIGINT` for graceful shutdown (close DB, drain in-flight requests) + +### HTTP Handler Pattern + +All handlers follow the same structure: +1. Parse and validate query params — return `400` with `{"error": "...", "message": "..."}` on bad input +2. Call the appropriate `dictionary.*` method +3. Return `200` with JSON response, or `404` if the word is not found, or `500` on + unexpected errors +4. Log errors at ERROR level; log each request at DEBUG level (controllable via `--log-level`) + +Use only standard library `net/http` — no framework needed for six endpoints. + +--- + +## Import CLI (`cmd/dictimport/main.go`) + +### Flags + +``` +--db Path to output dict.db (default: ./dict.db) +--data Path to data directory (default: ./data/) +--langs Comma-separated langs (default: en,fr,pt-PT) +--clean Drop and recreate all tables before import +--skip Comma-separated loader names to skip +``` + +### Execution Order + +**English:** +1. `scowl` — populates words +2. `wordnet` — definitions + synonyms +3. `wiktionary-en` — supplemental definitions, synonyms, translations + +**French:** +1. `hunspell-fr` — populates words +2. `lexique` — enriches POS + frequency +3. `wolf` — definitions + synonyms +4. `wiktionary-fr` — supplemental definitions, synonyms, translations + +**pt-PT:** +1. `hunspell-pt` — populates words +2. `dicionario` — definitions +3. `wiktionary-pt` — supplemental definitions, synonyms, translations + +Use `INSERT OR IGNORE` throughout. Wrap each loader in a single transaction; commit on +completion. Print per-loader row counts and elapsed time. Print final summary. + +--- + +## Data Sources & Loader Specs + +--- + +### English — SCOWL (`loader/scowl.go`) + +**Source:** https://wordlist.aspell.net/ +**Files:** All `final/english-words.NN` where NN ≤ 70 + +One word per line. Skip blank lines and `#` comments. Lowercase. Skip words with +spaces, digits, hyphens, or apostrophes. `INSERT OR IGNORE INTO words (word, lang) VALUES (?, 'en')`. + +**Expected:** ~170,000–220,000 words. + +--- + +### French + pt-PT — Hunspell (`loader/hunspell.go`) + +**Source:** https://github.com/LibreOffice/dictionaries +**Files:** `fr_FR/fr_FR.dic`, `pt_PT/pt_PT.dic` + +Skip line 1 (word count). Split each line on `/`, take left side as base word. Lowercase. +Skip words with digits, spaces, or non-Latin characters. + +Same function handles both languages: `func Load(path string, lang string, db *sql.DB) error` + +**Expected:** ~60,000–90,000 words per language. + +--- + +### English Definitions + Synonyms — WordNet (`loader/wordnet.go`) + +**Source:** https://wordnet.princeton.edu/download/current-version +**Files:** `dict/data.noun`, `dict/data.verb`, `dict/data.adj`, `dict/data.adv` + +Skip lines starting with two spaces (headers). Parse `ss_type` for POS. Extract all words +in synset. Take gloss text before the first `;` (drops usage examples). Insert definitions +at `priority=10`. Insert intra-synset word pairs as synonyms. Skip words containing `_` +for synonym insertion. + +--- + +### French Enrichment — Lexique (`loader/lexique.go`) + +**Source:** http://www.lexique.org/?page_id=319 +**File:** `Lexique383.tsv` + +Columns: 0=`ortho`, 3=`cgram`, 8=`freqfilms2`. POS map: `NOM→noun`, `VER→verb`, +`ADJ→adjective`, `ADV→adverb`. Frequency = `freqfilms2 × 1000` truncated to int. +On conflict with existing Hunspell row: `UPDATE ... SET pos=?, frequency=? WHERE pos IS NULL`. + +--- + +### French Definitions + Synonyms — WOLF (`loader/wolf.go`) + +**Source:** https://github.com/nicolashernandez/WOLF +**File:** `wolf-1.0b4.xml.bz2` → decompress to `wolf.xml` + +Stream-parse XML with `encoding/xml`. For each ``: collect `` values, +map ``, insert definitions at `priority=10`, insert intra-synset synonym pairs. + +--- + +### pt-PT Definitions — Dicionário Aberto (`loader/dicionario.go`) + +**Source:** https://github.com/jorgecardoso/dicionario-aberto +**File:** `dicionario-aberto.xml` + +**⚠️ Inspect the file before writing the parser:** +```bash +head -100 data/dicionario-aberto.xml +``` +Print distinct `` values before writing the POS map. Element names in the live file +may differ from older documentation. Stream-parse XML. Insert definitions at `priority=10`. + +--- + +### All Languages — Wiktionary (`loader/wiktionary.go`) + +**Source:** kaikki.org pre-parsed Wiktionary JSONL dumps + +| Target file | Lang | URL | +|---|---|---| +| `kaikki-en.jsonl` | `en` | https://kaikki.org/dictionary/English/kaikki.org-dictionary-English.json | +| `kaikki-fr.jsonl` | `fr` | https://kaikki.org/dictionary/French/kaikki.org-dictionary-French.json | +| `kaikki-pt.jsonl` | `pt-PT` | https://kaikki.org/dictionary/Portuguese/kaikki.org-dictionary-Portuguese.json | + +**JSON structure per line:** +```json +{ + "word": "example", + "pos": "noun", + "lang_code": "en", + "senses": [ + {"glosses": ["a representative instance"], "tags": []}, + {"glosses": ["past tense of exemplify"], "tags": ["form-of"]} + ], + "synonyms": [{"word": "instance"}], + "translations": [ + {"code": "fr", "word": "exemple"}, + {"code": "pt", "word": "exemplo"} + ] +} +``` + +**POS mapping:** `noun→noun`, `verb→verb`, `adj→adjective`, `adv→adverb`, `name→noun`. + +**Import logic:** +- Use `bufio.Scanner` with 10MB buffer (default 64KB truncates long lines silently) +- Wrap in single transaction; commit every 10,000 rows +- Verify `lang_code` matches expected language; skip if not +- Lowercase word; skip if contains spaces or digits +- Skip senses tagged `form-of` or `alt-of` +- Skip glosses that are empty, start with `(`, or exceed 500 characters +- Insert definitions at `priority=20`; `UNIQUE(word_id, gloss)` handles deduplication silently +- Insert synonyms; `UNIQUE(word_id, synonym)` handles deduplication silently +- **Translations:** for each entry in `translations[]`, if `code` maps to a supported lang + and `code != source lang`, insert into `translations` table + - Code mapping: `fr→fr`, `pt→pt-PT`, `en→en` + - Skip translations where `word` contains spaces or digits + +**pt-PT Brazil filtering:** Reject senses where `tags` contains `"Brazil"` or +`"Brazilian Portuguese"`. Accept untagged senses and senses tagged `"Portugal"` or +`"European Portuguese"`. + +**Expected additional definitions after deduplication:** +- en: ~200,000–400,000 +- fr: ~80,000–150,000 +- pt-PT: ~40,000–80,000 + +--- + +## Deployment (systemd) + +### File Locations + +``` +/usr/local/bin/dreamdict # compiled binary +/opt/dreamdict/dict.db # SQLite database +/opt/dreamdict/data/ # source data files (can be deleted after import) +/etc/dreamdict/dreamdict.env # environment config +/var/log/dreamdict/ # logs (or use journald) +``` + +### `deploy/dreamdict.service` + +```ini +[Unit] +Description=DreamDict dictionary service +After=network.target +# If GogoBee ever runs as a service too: +# Before=dreamdict.service + +[Service] +Type=simple +User=dreamdict +Group=dreamdict +EnvironmentFile=/etc/dreamdict/dreamdict.env +ExecStart=/usr/local/bin/dreamdict server \ + --host 127.0.0.1 \ + --port 7777 \ + --db /opt/dreamdict/dict.db +Restart=on-failure +RestartSec=5s + +# Hardening +NoNewPrivileges=true +ProtectSystem=strict +ReadWritePaths=/opt/dreamdict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +### `/etc/dreamdict/dreamdict.env` + +``` +DREAMDICT_PORT=7777 +DREAMDICT_DB=/opt/dreamdict/dict.db +``` + +### Install Steps + +```bash +# Create user +useradd --system --no-create-home --shell /usr/sbin/nologin dreamdict + +# Create dirs +mkdir -p /opt/dreamdict /etc/dreamdict /var/log/dreamdict +chown dreamdict:dreamdict /opt/dreamdict /var/log/dreamdict + +# Build and install binary +go build -o /usr/local/bin/dreamdict ./cmd/server +chmod 755 /usr/local/bin/dreamdict + +# Run import (as yourself, not dreamdict user) +go run ./cmd/dictimport --db /opt/dreamdict/dict.db --data /opt/dreamdict/data/ +chown dreamdict:dreamdict /opt/dreamdict/dict.db + +# Install and start service +cp deploy/dreamdict.service /etc/systemd/system/ +systemctl daemon-reload +systemctl enable --now dreamdict + +# Verify +systemctl status dreamdict +curl http://127.0.0.1:7777/health +``` + +### Updating the Database + +```bash +# Stop service, rebuild database, restart +systemctl stop dreamdict +go run ./cmd/dictimport --db /opt/dreamdict/dict.db --clean +chown dreamdict:dreamdict /opt/dreamdict/dict.db +systemctl start dreamdict +``` + +--- + +## GogoBee Integration + +GogoBee gets a single `DreamDictClient` struct. Zero dictionary logic inside the bot. + +### Client Package + +Add to GogoBee: `internal/dreamclient/client.go` + +```go +package dreamclient + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" +) + +type Client struct { + baseURL string + httpClient *http.Client +} + +func New(baseURL string) *Client { + return &Client{ + baseURL: baseURL, + httpClient: &http.Client{Timeout: 3 * time.Second}, + } +} + +func (c *Client) IsValidWord(word, lang string) (bool, error) +func (c *Client) RandomWord(lang string, pos string, min, max int) (string, error) +func (c *Client) Define(word, lang string) ([]Definition, error) +func (c *Client) Synonyms(word, lang string) ([]string, error) +func (c *Client) Translate(word, fromLang, toLang string) ([]string, error) +func (c *Client) Health() (*HealthResponse, error) +``` + +### Configuration + +In GogoBee's config: +```yaml +dictionary: + url: "http://127.0.0.1:7777" +``` + +When GogoBee eventually containerizes, this becomes the container network address. +No other changes required. + +### Replacing Wordnik Calls + +Run `grep -r -i wordnik .` in the GogoBee repo to find all call sites. Replace: + +| Wordnik call | GogoBee replacement | +|---|---| +| `GET /word/{word}/valid` | `dictClient.IsValidWord(word, lang)` | +| `GET /words/randomWord` | `dictClient.RandomWord(lang, pos, min, max)` | +| `GET /word/{word}/definitions` | `dictClient.Define(word, lang)` | +| `GET /word/{word}/relatedWords` | `dictClient.Synonyms(word, lang)` | + +Calls previously hardcoded to English: keep `lang = "en"`. + +### Startup Behaviour + +On GogoBee startup, call `/health` to verify dreamdict is reachable. Log a warning (not +a fatal error) if unavailable — dictionary-dependent commands should fail gracefully with +a user-facing message rather than taking the whole bot down. + +--- + +## Data File Checklist + +| File | Lang | Source | +|---|---|---| +| `scowl/final/english-words.*` | en | wordlist.aspell.net | +| `wordnet/dict/data.{noun,verb,adj,adv}` | en | wordnet.princeton.edu | +| `kaikki-en.jsonl` | en | kaikki.org | +| `fr_FR.dic` | fr | github.com/LibreOffice/dictionaries | +| `Lexique383.tsv` | fr | lexique.org | +| `wolf.xml` | fr | github.com/nicolashernandez/WOLF | +| `kaikki-fr.jsonl` | fr | kaikki.org | +| `pt_PT.dic` | pt-PT | github.com/LibreOffice/dictionaries | +| `dicionario-aberto.xml` | pt-PT | github.com/jorgecardoso/dicionario-aberto | +| `kaikki-pt.jsonl` | pt-PT | kaikki.org | + +`scripts/download-dict-data.sh` automates all downloads. Idempotent — skips existing files +unless `--force` passed. Handles tar extraction (SCOWL), bz2 decompression (WOLF), and +git clones where no stable direct URL exists. + +--- + +## `go.mod` Dependencies + +``` +modernc.org/sqlite # Pure-Go SQLite, no CGO +``` + +Standard library covers everything else: `net/http`, `encoding/xml`, `encoding/json`, +`bufio`, `compress/bzip2`, `database/sql`, `flag`, `log/slog`. + +--- + +## Testing + +### Service Tests (`internal/dictionary/dict_test.go`) + +Use `:memory:` SQLite. Cover: + +1. `IsValidWord` — known word true, unknown false, case-insensitive +2. `RandomWord` — matches length + POS filters; returns `ErrNoMatch` on empty set +3. `Define` — priority ordering (wordnet before wiktionary); empty slice for no definitions +4. `Synonyms` — deduplicated; empty slice when none +5. `Translate` — returns correct target-lang translations; empty slice when none +6. `ErrNotSeeded` — when `meta` table has no rows +7. Gloss deduplication — identical gloss twice → one row +8. Synonym deduplication — identical synonym twice → one row +9. Translation deduplication — identical translation twice → one row +10. `form-of` filtering — senses tagged `form-of` not inserted +11. pt-PT Brazil filtering — `Brazil`-tagged senses rejected; untagged accepted + +### HTTP Handler Tests (`cmd/server/`) + +Use `httptest.NewServer`. Cover: + +1. `/valid` — 200 true/false, 400 on missing params +2. `/random` — 200 with word, 404 on no match +3. `/define` — 200 with ordered definitions, 200 with empty array on no defs +4. `/synonyms` — 200 with list, 200 with empty array +5. `/translate` — 200 with list, 200 with empty array +6. `/health` — 200 with counts + +--- + +## Expected Database Size + +| Language | Words | Definitions | Approx size | +|---|---|---|---| +| en | ~220,000 | ~380,000 | ~180MB | +| fr | ~90,000 | ~200,000 | ~80MB | +| pt-PT | ~75,000 | ~120,000 | ~50MB | +| **Total** | | | **~310MB** | + +Add to `.gitignore`: `dict.db`, `data/` + +--- + +## Phase 2: Affix Expansion + +**Implement shortly after initial launch, not at initial launch.** + +The base-forms-only approach in the initial import means `IsValidWord("running", "en")` +returns false because only `"run"` is stored. For Hangman this doesn't matter — DreamDict +generates the puzzle word and it's always a base form. But for any command where a player +submits a word for validation (UNO, word games, future puzzle modes), this is a real +correctness bug, not a theoretical one. + +**Recommended approach:** affix expansion at import time using `go-hunspell` +(`github.com/trustmaster/go-hunspell`). Parse the `.aff` rules for each language and +generate all valid inflected forms during `dictimport`, storing them in the `words` table +alongside base forms. This keeps `IsValidWord` as a simple indexed lookup with no runtime +stemming logic. + +**Do not implement stemming as an alternative.** Stemming at query time is more complex, +language-specific, and redundant if you expand affixes at import time. Pick one direction — +affix expansion at import is the cleaner path for this architecture. + +**Sequencing:** get the service running and validate data quality across all three languages +first. Affix expansion is a `dictimport` change only — the HTTP API, schema, and GogoBee +client are untouched. + +--- + +## Deferred: Traefik Route for Second Consumer + +When a second consumer for DreamDict exists, adding a Traefik route on parodia.dev's +internal network is trivial — one label, one DNS entry, twenty minutes of work. Do it +then, not now. The service architecture requires no changes at that point. + +--- + +## Not Worth Doing: HTTP Caching Headers + +The consumers are on localhost, there is no intermediary cache, and SQLite responses are +already faster than any cache lookup. Drop this entirely. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1d069be --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module dreamdict + +go 1.25.0 + +require modernc.org/sqlite v1.48.0 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.42.0 // indirect + modernc.org/libc v1.70.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8411195 --- /dev/null +++ b/go.sum @@ -0,0 +1,51 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= +modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.32.0 h1:hjG66bI/kqIPX1b2yT6fr/jt+QedtP2fqojG2VrFuVw= +modernc.org/ccgo/v4 v4.32.0/go.mod h1:6F08EBCx5uQc38kMGl+0Nm0oWczoo1c7cgpzEry7Uc0= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.70.0 h1:U58NawXqXbgpZ/dcdS9kMshu08aiA6b7gusEusqzNkw= +modernc.org/libc v1.70.0/go.mod h1:OVmxFGP1CI/Z4L3E0Q3Mf1PDE0BucwMkcXjjLntvHJo= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.48.0 h1:ElZyLop3Q2mHYk5IFPPXADejZrlHu7APbpB0sF78bq4= +modernc.org/sqlite v1.48.0/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/dictionary/db.go b/internal/dictionary/db.go new file mode 100644 index 0000000..8df7347 --- /dev/null +++ b/internal/dictionary/db.go @@ -0,0 +1,98 @@ +package dictionary + +import ( + "database/sql" + "fmt" + + _ "modernc.org/sqlite" +) + +const schemaVersion = "1" + +const schemaSQL = ` +CREATE TABLE IF NOT EXISTS words ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word TEXT NOT NULL, + lang TEXT NOT NULL, + pos TEXT, + frequency INTEGER DEFAULT 0, + UNIQUE(word, lang) +); + +CREATE INDEX IF NOT EXISTS idx_words_lang ON words(lang); +CREATE INDEX IF NOT EXISTS idx_words_lang_pos ON words(lang, pos); +CREATE INDEX IF NOT EXISTS idx_words_word ON words(word); + +CREATE TABLE IF NOT EXISTS definitions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + pos TEXT, + gloss TEXT NOT NULL, + source TEXT NOT NULL, + priority INTEGER NOT NULL DEFAULT 99, + UNIQUE(word_id, gloss) +); + +CREATE INDEX IF NOT EXISTS idx_definitions_word_id ON definitions(word_id); +CREATE INDEX IF NOT EXISTS idx_definitions_word_id_priority ON definitions(word_id, priority); + +CREATE TABLE IF NOT EXISTS synonyms ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + synonym TEXT NOT NULL, + source TEXT NOT NULL, + UNIQUE(word_id, synonym) +); + +CREATE INDEX IF NOT EXISTS idx_synonyms_word_id ON synonyms(word_id); + +CREATE TABLE IF NOT EXISTS translations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + word_id INTEGER NOT NULL REFERENCES words(id) ON DELETE CASCADE, + translation TEXT NOT NULL, + target_lang TEXT NOT NULL, + source TEXT NOT NULL, + UNIQUE(word_id, translation, target_lang) +); + +CREATE INDEX IF NOT EXISTS idx_translations_word_id ON translations(word_id); + +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +` + +func openDB(dbPath string) (*sql.DB, error) { + db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(wal)&_pragma=foreign_keys(on)") + if err != nil { + return nil, fmt.Errorf("dictionary: open db: %w", err) + } + db.SetMaxOpenConns(1) // SQLite write serialization for import + if err := db.Ping(); err != nil { + db.Close() + return nil, fmt.Errorf("dictionary: ping db: %w", err) + } + return db, nil +} + +func openReadOnlyDB(dbPath string) (*sql.DB, error) { + db, err := sql.Open("sqlite", dbPath+"?mode=ro&_pragma=journal_mode(wal)&_pragma=foreign_keys(on)") + if err != nil { + return nil, fmt.Errorf("dictionary: open db (ro): %w", err) + } + // No MaxOpenConns limit — WAL mode supports concurrent reads + if err := db.Ping(); err != nil { + db.Close() + return nil, fmt.Errorf("dictionary: ping db: %w", err) + } + return db, nil +} + +func BootstrapSchema(db *sql.DB) error { + _, err := db.Exec(schemaSQL) + if err != nil { + return fmt.Errorf("dictionary: bootstrap schema: %w", err) + } + return nil +} diff --git a/internal/dictionary/dict.go b/internal/dictionary/dict.go new file mode 100644 index 0000000..da7990f --- /dev/null +++ b/internal/dictionary/dict.go @@ -0,0 +1,93 @@ +package dictionary + +import ( + "database/sql" + "errors" +) + +var ( + ErrNoMatch = errors.New("dictionary: no word matches filters") + ErrNotSeeded = errors.New("dictionary: database not seeded, run: go run ./cmd/dictimport") +) + +var supportedLangs = []string{"en", "fr", "pt-PT"} + +func ValidLang(lang string) bool { + for _, l := range supportedLangs { + if l == lang { + return true + } + } + return false +} + +type Options struct { + MinLength int + MaxLength int + POS string // "", "noun", "verb", "adjective", "adverb" + MinFrequency int // 0 = no filter +} + +type Definition struct { + POS string `json:"pos"` + Gloss string `json:"gloss"` + Source string `json:"source"` + Priority int `json:"priority"` +} + +type Dictionary struct { + db *sql.DB +} + +// New opens the database read-write (for import CLI). +func New(dbPath string) (*Dictionary, error) { + db, err := openDB(dbPath) + if err != nil { + return nil, err + } + d := &Dictionary{db: db} + if err := d.checkSeeded(); err != nil { + db.Close() + return nil, err + } + return d, nil +} + +// NewReadOnly opens the database read-only with concurrent read support (for server). +func NewReadOnly(dbPath string) (*Dictionary, error) { + db, err := openReadOnlyDB(dbPath) + if err != nil { + return nil, err + } + d := &Dictionary{db: db} + if err := d.checkSeeded(); err != nil { + db.Close() + return nil, err + } + return d, nil +} + +// NewFromDB creates a Dictionary from an existing *sql.DB (useful for testing). +func NewFromDB(db *sql.DB) *Dictionary { + return &Dictionary{db: db} +} + +func (d *Dictionary) Close() error { + return d.db.Close() +} + +func (d *Dictionary) checkSeeded() error { + var count int + err := d.db.QueryRow("SELECT COUNT(*) FROM meta").Scan(&count) + if err != nil { + return ErrNotSeeded + } + if count == 0 { + return ErrNotSeeded + } + return nil +} + +func (d *Dictionary) SupportedLangs() []string { + return supportedLangs +} diff --git a/internal/dictionary/dict_test.go b/internal/dictionary/dict_test.go new file mode 100644 index 0000000..e313ef4 --- /dev/null +++ b/internal/dictionary/dict_test.go @@ -0,0 +1,313 @@ +package dictionary + +import ( + "database/sql" + "testing" + + _ "modernc.org/sqlite" +) + +func setupTestDB(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(on)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + if err := BootstrapSchema(db); err != nil { + t.Fatal(err) + } + return db +} + +func mustExec(t *testing.T, db *sql.DB, query string, args ...any) { + t.Helper() + if _, err := db.Exec(query, args...); err != nil { + t.Fatalf("mustExec: %s: %v", query, err) + } +} + +func seedTestData(t *testing.T, db *sql.DB) { + t.Helper() + + // Meta + mustExec(t, db, "INSERT INTO meta (key, value) VALUES ('schema_version', '1')") + mustExec(t, db, "INSERT INTO meta (key, value) VALUES ('imported_at', '2025-01-01T00:00:00Z')") + + // Words + mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (1, 'happy', 'en', 'adjective', 0)") + mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (2, 'run', 'en', 'verb', 0)") + mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (3, 'chat', 'fr', 'noun', 5000)") + mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (4, 'gato', 'pt-PT', 'noun', 0)") + mustExec(t, db, "INSERT INTO words (id, word, lang, pos, frequency) VALUES (5, 'cat', 'en', 'noun', 0)") + + // Definitions + mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling pleasure', 'wordnet', 10)") + mustExec(t, db, "INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling joy or contentment', 'wiktionary', 20)") + + // Synonyms + mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'glad', 'wordnet')") + mustExec(t, db, "INSERT INTO synonyms (word_id, synonym, source) VALUES (1, 'joyful', 'wordnet')") + + // Translations + mustExec(t, db, "INSERT INTO translations (word_id, translation, target_lang, source) VALUES (5, 'chat', 'fr', 'wiktionary')") + mustExec(t, db, "INSERT INTO translations (word_id, translation, target_lang, source) VALUES (5, 'gato', 'pt-PT', 'wiktionary')") +} + +func TestIsValidWord(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + d := NewFromDB(db) + + tests := []struct { + word, lang string + want bool + }{ + {"happy", "en", true}, + {"Happy", "en", true}, // case-insensitive + {"nonexistent", "en", false}, + {"chat", "fr", true}, + {"chat", "en", false}, + } + for _, tt := range tests { + got, err := d.IsValidWord(tt.word, tt.lang) + if err != nil { + t.Errorf("IsValidWord(%q, %q) error: %v", tt.word, tt.lang, err) + continue + } + if got != tt.want { + t.Errorf("IsValidWord(%q, %q) = %v, want %v", tt.word, tt.lang, got, tt.want) + } + } +} + +func TestRandomWord(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + d := NewFromDB(db) + + // Basic random word + word, err := d.RandomWord("en", Options{}) + if err != nil { + t.Fatalf("RandomWord: %v", err) + } + if word == "" { + t.Error("RandomWord returned empty string") + } + + // With POS filter + word, err = d.RandomWord("en", Options{POS: "adjective"}) + if err != nil { + t.Fatalf("RandomWord with POS: %v", err) + } + if word != "happy" { + t.Errorf("RandomWord(adjective) = %q, want happy", word) + } + + // With length filters + word, err = d.RandomWord("en", Options{MinLength: 4, MaxLength: 5}) + if err != nil { + t.Fatalf("RandomWord with length: %v", err) + } + if len(word) < 4 || len(word) > 5 { + t.Errorf("RandomWord length %d not in [4,5]", len(word)) + } + + // No match + _, err = d.RandomWord("en", Options{MinLength: 100}) + if err != ErrNoMatch { + t.Errorf("expected ErrNoMatch, got %v", err) + } +} + +func TestDefine(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + d := NewFromDB(db) + + defs, err := d.Define("happy", "en") + if err != nil { + t.Fatalf("Define: %v", err) + } + if len(defs) != 2 { + t.Fatalf("Define returned %d defs, want 2", len(defs)) + } + // Priority ordering: wordnet (10) before wiktionary (20) + if defs[0].Source != "wordnet" { + t.Errorf("first def source = %q, want wordnet", defs[0].Source) + } + if defs[1].Source != "wiktionary" { + t.Errorf("second def source = %q, want wiktionary", defs[1].Source) + } + + // No definitions + defs, err = d.Define("run", "en") + if err != nil { + t.Fatalf("Define(run): %v", err) + } + if len(defs) != 0 { + t.Errorf("Define(run) returned %d defs, want 0", len(defs)) + } +} + +func TestSynonyms(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + d := NewFromDB(db) + + syns, err := d.Synonyms("happy", "en") + if err != nil { + t.Fatalf("Synonyms: %v", err) + } + if len(syns) != 2 { + t.Fatalf("Synonyms returned %d, want 2", len(syns)) + } + + // No synonyms + syns, err = d.Synonyms("run", "en") + if err != nil { + t.Fatalf("Synonyms(run): %v", err) + } + if len(syns) != 0 { + t.Errorf("Synonyms(run) returned %d, want 0", len(syns)) + } +} + +func TestTranslate(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + d := NewFromDB(db) + + trans, err := d.Translate("cat", "en", "fr") + if err != nil { + t.Fatalf("Translate: %v", err) + } + if len(trans) != 1 || trans[0] != "chat" { + t.Errorf("Translate(cat, en, fr) = %v, want [chat]", trans) + } + + // No translations + trans, err = d.Translate("happy", "en", "fr") + if err != nil { + t.Fatalf("Translate(happy): %v", err) + } + if len(trans) != 0 { + t.Errorf("Translate(happy) returned %d, want 0", len(trans)) + } +} + +func TestErrNotSeeded(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + // Don't seed — meta table is empty + d := NewFromDB(db) + err := d.checkSeeded() + if err != ErrNotSeeded { + t.Errorf("expected ErrNotSeeded, got %v", err) + } +} + +func TestGlossDeduplication(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + + // Insert duplicate gloss — should be ignored by UNIQUE constraint + _, err := db.Exec("INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority) VALUES (1, 'adjective', 'feeling pleasure', 'wiktionary', 20)") + if err != nil { + t.Fatal(err) + } + d := NewFromDB(db) + defs, err := d.Define("happy", "en") + if err != nil { + t.Fatal(err) + } + if len(defs) != 2 { + t.Errorf("expected 2 defs after dedup, got %d", len(defs)) + } +} + +func TestSynonymDeduplication(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + + _, err := db.Exec("INSERT OR IGNORE INTO synonyms (word_id, synonym, source) VALUES (1, 'glad', 'wiktionary')") + if err != nil { + t.Fatal(err) + } + d := NewFromDB(db) + syns, err := d.Synonyms("happy", "en") + if err != nil { + t.Fatal(err) + } + if len(syns) != 2 { + t.Errorf("expected 2 synonyms after dedup, got %d", len(syns)) + } +} + +func TestTranslationDeduplication(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + + _, err := db.Exec("INSERT OR IGNORE INTO translations (word_id, translation, target_lang, source) VALUES (5, 'chat', 'fr', 'other')") + if err != nil { + t.Fatal(err) + } + d := NewFromDB(db) + trans, err := d.Translate("cat", "en", "fr") + if err != nil { + t.Fatal(err) + } + if len(trans) != 1 { + t.Errorf("expected 1 translation after dedup, got %d", len(trans)) + } +} + +func TestMeta(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + d := NewFromDB(db) + + val, err := d.Meta("schema_version") + if err != nil { + t.Fatal(err) + } + if val != "1" { + t.Errorf("Meta(schema_version) = %q, want 1", val) + } + + // Missing key returns empty string, no error + val, err = d.Meta("nonexistent") + if err != nil { + t.Fatal(err) + } + if val != "" { + t.Errorf("Meta(nonexistent) = %q, want empty", val) + } +} + +func TestWordCount(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + d := NewFromDB(db) + + counts, err := d.WordCount() + if err != nil { + t.Fatal(err) + } + if counts["en"] != 3 { + t.Errorf("en word count = %d, want 3", counts["en"]) + } + if counts["fr"] != 1 { + t.Errorf("fr word count = %d, want 1", counts["fr"]) + } +} diff --git a/internal/dictionary/query.go b/internal/dictionary/query.go new file mode 100644 index 0000000..9aae969 --- /dev/null +++ b/internal/dictionary/query.go @@ -0,0 +1,193 @@ +package dictionary + +import ( + "database/sql" + "fmt" + "strings" +) + +func (d *Dictionary) IsValidWord(word, lang string) (bool, error) { + var exists bool + err := d.db.QueryRow( + "SELECT EXISTS(SELECT 1 FROM words WHERE word = ? AND lang = ?)", + strings.ToLower(word), lang, + ).Scan(&exists) + if err != nil { + return false, fmt.Errorf("dictionary: is valid word: %w", err) + } + return exists, nil +} + +func (d *Dictionary) RandomWord(lang string, opts Options) (string, error) { + query := "SELECT word FROM words WHERE lang = ?" + args := []any{lang} + + if opts.POS != "" { + query += " AND pos = ?" + args = append(args, opts.POS) + } + if opts.MinLength > 0 { + query += " AND LENGTH(word) >= ?" + args = append(args, opts.MinLength) + } + if opts.MaxLength > 0 { + query += " AND LENGTH(word) <= ?" + args = append(args, opts.MaxLength) + } + if opts.MinFrequency > 0 { + query += " AND frequency >= ?" + args = append(args, opts.MinFrequency) + } + + query += " ORDER BY RANDOM() LIMIT 1" + + var word string + err := d.db.QueryRow(query, args...).Scan(&word) + if err == sql.ErrNoRows { + return "", ErrNoMatch + } + if err != nil { + return "", fmt.Errorf("dictionary: random word: %w", err) + } + return word, nil +} + +func (d *Dictionary) Define(word, lang string) ([]Definition, error) { + rows, err := d.db.Query(` + SELECT d.pos, d.gloss, d.source, d.priority + FROM definitions d + JOIN words w ON w.id = d.word_id + WHERE w.word = ? AND w.lang = ? + ORDER BY d.priority ASC, d.id ASC`, + strings.ToLower(word), lang, + ) + if err != nil { + return nil, fmt.Errorf("dictionary: define: %w", err) + } + defer rows.Close() + + var defs []Definition + for rows.Next() { + var def Definition + var pos sql.NullString + if err := rows.Scan(&pos, &def.Gloss, &def.Source, &def.Priority); err != nil { + return nil, fmt.Errorf("dictionary: define scan: %w", err) + } + def.POS = pos.String + defs = append(defs, def) + } + if defs == nil { + defs = []Definition{} + } + return defs, rows.Err() +} + +func (d *Dictionary) Synonyms(word, lang string) ([]string, error) { + rows, err := d.db.Query(` + SELECT DISTINCT s.synonym + FROM synonyms s + JOIN words w ON w.id = s.word_id + WHERE w.word = ? AND w.lang = ? + ORDER BY s.synonym`, + strings.ToLower(word), lang, + ) + if err != nil { + return nil, fmt.Errorf("dictionary: synonyms: %w", err) + } + defer rows.Close() + + var syns []string + for rows.Next() { + var syn string + if err := rows.Scan(&syn); err != nil { + return nil, fmt.Errorf("dictionary: synonyms scan: %w", err) + } + syns = append(syns, syn) + } + if syns == nil { + syns = []string{} + } + return syns, rows.Err() +} + +func (d *Dictionary) Translate(word, fromLang, toLang string) ([]string, error) { + rows, err := d.db.Query(` + SELECT DISTINCT t.translation + FROM translations t + JOIN words w ON w.id = t.word_id + WHERE w.word = ? AND w.lang = ? AND t.target_lang = ? + ORDER BY t.translation`, + strings.ToLower(word), fromLang, toLang, + ) + if err != nil { + return nil, fmt.Errorf("dictionary: translate: %w", err) + } + defer rows.Close() + + var trans []string + for rows.Next() { + var t string + if err := rows.Scan(&t); err != nil { + return nil, fmt.Errorf("dictionary: translate scan: %w", err) + } + trans = append(trans, t) + } + if trans == nil { + trans = []string{} + } + return trans, rows.Err() +} + +func (d *Dictionary) WordCount() (map[string]int, error) { + rows, err := d.db.Query("SELECT lang, COUNT(*) FROM words GROUP BY lang") + if err != nil { + return nil, fmt.Errorf("dictionary: word count: %w", err) + } + defer rows.Close() + + counts := make(map[string]int) + for rows.Next() { + var lang string + var count int + if err := rows.Scan(&lang, &count); err != nil { + return nil, fmt.Errorf("dictionary: word count scan: %w", err) + } + counts[lang] = count + } + return counts, rows.Err() +} + +func (d *Dictionary) Meta(key string) (string, error) { + var val string + err := d.db.QueryRow("SELECT value FROM meta WHERE key = ?", key).Scan(&val) + if err == sql.ErrNoRows { + return "", nil + } + if err != nil { + return "", fmt.Errorf("dictionary: meta %s: %w", key, err) + } + return val, nil +} + +func (d *Dictionary) DefCount() (map[string]int, error) { + rows, err := d.db.Query(` + SELECT w.lang, COUNT(*) + FROM definitions d + JOIN words w ON w.id = d.word_id + GROUP BY w.lang`) + if err != nil { + return nil, fmt.Errorf("dictionary: def count: %w", err) + } + defer rows.Close() + + counts := make(map[string]int) + for rows.Next() { + var lang string + var count int + if err := rows.Scan(&lang, &count); err != nil { + return nil, fmt.Errorf("dictionary: def count scan: %w", err) + } + counts[lang] = count + } + return counts, rows.Err() +} diff --git a/internal/loader/dicionario.go b/internal/loader/dicionario.go new file mode 100644 index 0000000..93fb7ee --- /dev/null +++ b/internal/loader/dicionario.go @@ -0,0 +1,118 @@ +package loader + +import ( + "database/sql" + "encoding/xml" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" +) + +type DicionarioLoader struct{} + +func (DicionarioLoader) Name() string { return "dicionario" } + +func (DicionarioLoader) Load(db *sql.DB, dataDir string) error { + path := filepath.Join(dataDir, "dicionario-aberto.xml") + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("dicionario: open: %w", err) + } + defer f.Close() + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("dicionario: begin tx: %w", err) + } + defer tx.Rollback() + + stmtDef, err := tx.Prepare(` + INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority) + SELECT id, ?, ?, 'dicionario-aberto', 10 FROM words WHERE word = ? AND lang = 'pt-PT'`) + if err != nil { + return fmt.Errorf("dicionario: prepare: %w", err) + } + defer stmtDef.Close() + + var defCount, decodeErrors int + + decoder := xml.NewDecoder(f) + for { + tok, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("dicionario: xml decode: %w", err) + } + + se, ok := tok.(xml.StartElement) + if !ok || se.Name.Local != "entry" { + continue + } + + var entry struct { + Form struct { + Orth string `xml:"orth"` + } `xml:"form"` + Senses []struct { + GramGrp struct { + POS string `xml:"pos"` + } `xml:"gramGrp"` + Def string `xml:"def"` + } `xml:"sense"` + } + if err := decoder.DecodeElement(&entry, &se); err != nil { + decodeErrors++ + slog.Debug("dicionario: skipping entry", "error", err) + continue + } + + word := strings.ToLower(strings.TrimSpace(entry.Form.Orth)) + if word == "" || containsDigit(word) || containsSpace(word) { + continue + } + + for _, sense := range entry.Senses { + gloss := strings.TrimSpace(sense.Def) + if gloss == "" { + continue + } + pos := mapPtPOS(strings.TrimSpace(sense.GramGrp.POS)) + if _, err := stmtDef.Exec(pos, gloss, word); err != nil { + return fmt.Errorf("dicionario: insert: %w", err) + } + defCount++ + } + } + + if decodeErrors > 0 { + slog.Warn("dicionario: skipped malformed entries", "count", decodeErrors) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("dicionario: commit: %w", err) + } + slog.Info("dicionario loaded", "definitions", defCount) + return nil +} + +func mapPtPOS(pos string) string { + pos = strings.ToLower(pos) + switch { + case strings.Contains(pos, "noun"), strings.HasPrefix(pos, "n."), strings.HasPrefix(pos, "s."), + strings.HasPrefix(pos, "m."), strings.HasPrefix(pos, "f."): + return "noun" + case strings.Contains(pos, "verb"), strings.HasPrefix(pos, "v."): + return "verb" + case strings.Contains(pos, "adj"): + return "adjective" + case strings.Contains(pos, "adv"): + return "adverb" + default: + return "" + } +} diff --git a/internal/loader/hunspell.go b/internal/loader/hunspell.go new file mode 100644 index 0000000..8ecc588 --- /dev/null +++ b/internal/loader/hunspell.go @@ -0,0 +1,78 @@ +package loader + +import ( + "bufio" + "database/sql" + "fmt" + "log/slog" + "os" + "strings" +) + +type HunspellLoader struct { + Lang string // "fr" or "pt-PT" + FileName string // e.g. "fr_FR/fr.dic" or "pt_PT/pt_PT.dic" +} + +func (h HunspellLoader) Name() string { return "hunspell-" + h.Lang } + +func (h HunspellLoader) Load(db *sql.DB, dataDir string) error { + path := dataDir + "/" + h.FileName + return loadHunspell(db, path, h.Lang) +} + +func loadHunspell(db *sql.DB, path, lang string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("hunspell: open %s: %w", path, err) + } + defer f.Close() + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("hunspell: begin tx: %w", err) + } + defer tx.Rollback() + + stmt, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang) VALUES (?, ?)") + if err != nil { + return fmt.Errorf("hunspell: prepare: %w", err) + } + defer stmt.Close() + + scanner := bufio.NewScanner(f) + + // Skip first line (word count) + if scanner.Scan() { + // discard + } + + var count int + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + // Split on "/" to get base word + word := strings.SplitN(line, "/", 2)[0] + word = strings.ToLower(word) + + if containsDigit(word) || containsSpace(word) || containsNonLatin(word) { + continue + } + + if _, err := stmt.Exec(word, lang); err != nil { + return fmt.Errorf("hunspell: insert: %w", err) + } + count++ + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("hunspell: scan: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("hunspell: commit: %w", err) + } + slog.Info("hunspell loaded", "lang", lang, "words", count) + return nil +} diff --git a/internal/loader/lexique.go b/internal/loader/lexique.go new file mode 100644 index 0000000..32a027c --- /dev/null +++ b/internal/loader/lexique.go @@ -0,0 +1,90 @@ +package loader + +import ( + "bufio" + "database/sql" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" +) + +type LexiqueLoader struct{} + +func (LexiqueLoader) Name() string { return "lexique" } + +func (LexiqueLoader) Load(db *sql.DB, dataDir string) error { + path := filepath.Join(dataDir, "Lexique383.tsv") + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("lexique: open: %w", err) + } + defer f.Close() + + posMap := map[string]string{ + "NOM": "noun", + "VER": "verb", + "ADJ": "adjective", + "ADV": "adverb", + } + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("lexique: begin tx: %w", err) + } + defer tx.Rollback() + + stmt, err := tx.Prepare(` + UPDATE words SET pos = ?, frequency = ? + WHERE word = ? AND lang = 'fr' AND pos IS NULL`) + if err != nil { + return fmt.Errorf("lexique: prepare: %w", err) + } + defer stmt.Close() + + scanner := bufio.NewScanner(f) + // Skip header + if scanner.Scan() { + // discard header line + } + + var count int + for scanner.Scan() { + line := scanner.Text() + fields := strings.Split(line, "\t") + if len(fields) < 9 { + continue + } + + word := strings.ToLower(fields[0]) + cgram := fields[3] + freqStr := fields[8] + + pos, ok := posMap[cgram] + if !ok { + continue + } + + freq := 0.0 + if f, err := strconv.ParseFloat(freqStr, 64); err == nil { + freq = f + } + freqInt := int(freq * 1000) + + if _, err := stmt.Exec(pos, freqInt, word); err != nil { + return fmt.Errorf("lexique: update: %w", err) + } + count++ + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("lexique: scan: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("lexique: commit: %w", err) + } + slog.Info("lexique loaded", "enriched", count) + return nil +} diff --git a/internal/loader/loader.go b/internal/loader/loader.go new file mode 100644 index 0000000..1a63c1f --- /dev/null +++ b/internal/loader/loader.go @@ -0,0 +1,29 @@ +package loader + +import ( + "database/sql" + "fmt" + "log/slog" + "time" +) + +type Loader interface { + Name() string + Load(db *sql.DB, dataDir string) error +} + +func Run(db *sql.DB, dataDir string, loaders []Loader, skip map[string]bool) error { + for _, l := range loaders { + if skip[l.Name()] { + slog.Info("skipping loader", "name", l.Name()) + continue + } + slog.Info("running loader", "name", l.Name()) + start := time.Now() + if err := l.Load(db, dataDir); err != nil { + return fmt.Errorf("loader %s: %w", l.Name(), err) + } + slog.Info("loader complete", "name", l.Name(), "elapsed", time.Since(start).Round(time.Millisecond)) + } + return nil +} diff --git a/internal/loader/scowl.go b/internal/loader/scowl.go new file mode 100644 index 0000000..a0d0f1c --- /dev/null +++ b/internal/loader/scowl.go @@ -0,0 +1,128 @@ +package loader + +import ( + "bufio" + "database/sql" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" +) + +type SCOWLLoader struct{} + +func (SCOWLLoader) Name() string { return "scowl" } + +func (SCOWLLoader) Load(db *sql.DB, dataDir string) error { + pattern := filepath.Join(dataDir, "scowl", "final", "english-words.*") + files, err := filepath.Glob(pattern) + if err != nil { + return fmt.Errorf("scowl: glob: %w", err) + } + if len(files) == 0 { + return fmt.Errorf("scowl: no files matching %s", pattern) + } + + // Filter to sizes <= 70 + var selected []string + for _, f := range files { + base := filepath.Base(f) + // Files are like english-words.10, english-words.20, etc. + parts := strings.SplitN(base, ".", 2) + if len(parts) != 2 { + continue + } + var size int + if _, err := fmt.Sscanf(parts[1], "%d", &size); err != nil { + continue + } + if size <= 70 { + selected = append(selected, f) + } + } + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("scowl: begin tx: %w", err) + } + defer tx.Rollback() + + stmt, err := tx.Prepare("INSERT OR IGNORE INTO words (word, lang) VALUES (?, 'en')") + if err != nil { + return fmt.Errorf("scowl: prepare: %w", err) + } + defer stmt.Close() + + var count int + for _, f := range selected { + n, err := loadSCOWLFile(stmt, f) + if err != nil { + return fmt.Errorf("scowl: load %s: %w", f, err) + } + count += n + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("scowl: commit: %w", err) + } + slog.Info("scowl loaded", "words", count, "files", len(selected)) + return nil +} + +func loadSCOWLFile(stmt *sql.Stmt, path string) (int, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + + var count int + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + word := strings.ToLower(line) + if containsAny(word, " ", "'", "-") || containsDigit(word) { + continue + } + if _, err := stmt.Exec(word); err != nil { + return 0, err + } + count++ + } + return count, scanner.Err() +} + +func containsAny(s string, subs ...string) bool { + for _, sub := range subs { + if strings.Contains(s, sub) { + return true + } + } + return false +} + +func containsDigit(s string) bool { + for _, r := range s { + if r >= '0' && r <= '9' { + return true + } + } + return false +} + +func containsSpace(s string) bool { + return strings.ContainsRune(s, ' ') +} + +func containsNonLatin(s string) bool { + for _, r := range s { + if r > 0x024F && r != '\'' && r != '-' { + return true + } + } + return false +} diff --git a/internal/loader/wiktionary.go b/internal/loader/wiktionary.go new file mode 100644 index 0000000..ef529b2 --- /dev/null +++ b/internal/loader/wiktionary.go @@ -0,0 +1,242 @@ +package loader + +import ( + "bufio" + "database/sql" + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" +) + +type WiktionaryLoader struct { + Lang string // "en", "fr", "pt-PT" + FileName string // e.g. "kaikki-en.jsonl" +} + +func (w WiktionaryLoader) Name() string { return "wiktionary-" + w.Lang } + +func (w WiktionaryLoader) Load(db *sql.DB, dataDir string) error { + path := filepath.Join(dataDir, w.FileName) + return loadWiktionary(db, path, w.Lang) +} + +type wiktEntry struct { + Word string `json:"word"` + POS string `json:"pos"` + LangCode string `json:"lang_code"` + Senses []struct { + Glosses []string `json:"glosses"` + Tags []string `json:"tags"` + } `json:"senses"` + Synonyms []struct { + Word string `json:"word"` + } `json:"synonyms"` + Translations []struct { + Code string `json:"code"` + Word string `json:"word"` + } `json:"translations"` +} + +var wiktPOSMap = map[string]string{ + "noun": "noun", + "verb": "verb", + "adj": "adjective", + "adv": "adverb", + "name": "noun", +} + +var wiktLangCodeMap = map[string]string{ + "fr": "fr", + "pt": "pt-PT", + "en": "en", +} + +const ( + defSQL = `INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority) SELECT id, ?, ?, 'wiktionary', 20 FROM words WHERE word = ? AND lang = ?` + synSQL = `INSERT OR IGNORE INTO synonyms (word_id, synonym, source) SELECT id, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?` + transSQL = `INSERT OR IGNORE INTO translations (word_id, translation, target_lang, source) SELECT id, ?, ?, 'wiktionary' FROM words WHERE word = ? AND lang = ?` +) + +// txBatch owns a transaction and its prepared statements. +// Close() is always safe to call — it rolls back (no-op after commit) and closes stmts. +type txBatch struct { + tx *sql.Tx + stmtDef *sql.Stmt + stmtSyn *sql.Stmt + stmtTrans *sql.Stmt +} + +func newTxBatch(db *sql.DB) (*txBatch, error) { + tx, err := db.Begin() + if err != nil { + return nil, fmt.Errorf("begin tx: %w", err) + } + b := &txBatch{tx: tx} + + b.stmtDef, err = tx.Prepare(defSQL) + if err != nil { + tx.Rollback() + return nil, fmt.Errorf("prepare def: %w", err) + } + b.stmtSyn, err = tx.Prepare(synSQL) + if err != nil { + b.stmtDef.Close() + tx.Rollback() + return nil, fmt.Errorf("prepare syn: %w", err) + } + b.stmtTrans, err = tx.Prepare(transSQL) + if err != nil { + b.stmtDef.Close() + b.stmtSyn.Close() + tx.Rollback() + return nil, fmt.Errorf("prepare trans: %w", err) + } + return b, nil +} + +func (b *txBatch) Close() { + if b == nil { + return + } + b.stmtDef.Close() + b.stmtSyn.Close() + b.stmtTrans.Close() + b.tx.Rollback() // no-op after commit +} + +func (b *txBatch) Commit() error { + b.stmtDef.Close() + b.stmtSyn.Close() + b.stmtTrans.Close() + return b.tx.Commit() +} + +func loadWiktionary(db *sql.DB, path, lang string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("wiktionary: open: %w", err) + } + defer f.Close() + + batch, err := newTxBatch(db) + if err != nil { + return fmt.Errorf("wiktionary: %w", err) + } + defer batch.Close() + + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 0, 10*1024*1024), 10*1024*1024) + + var defCount, synCount, transCount, lineCount int + commitInterval := 10000 + + for scanner.Scan() { + lineCount++ + + var entry wiktEntry + if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil { + continue + } + + word := strings.ToLower(entry.Word) + if containsDigit(word) || containsSpace(word) { + continue + } + + pos := wiktPOSMap[entry.POS] + + // Process senses + for _, sense := range entry.Senses { + if hasTag(sense.Tags, "form-of", "alt-of") { + continue + } + // pt-PT Brazil filtering + if lang == "pt-PT" && hasBrazilTag(sense.Tags) { + continue + } + + for _, gloss := range sense.Glosses { + gloss = strings.TrimSpace(gloss) + if gloss == "" || strings.HasPrefix(gloss, "(") || len(gloss) > 500 { + continue + } + if _, err := batch.stmtDef.Exec(pos, gloss, word, lang); err != nil { + return fmt.Errorf("wiktionary: insert def: %w", err) + } + defCount++ + } + } + + // Process synonyms + for _, syn := range entry.Synonyms { + synWord := strings.ToLower(syn.Word) + if synWord == "" || containsDigit(synWord) || containsSpace(synWord) { + continue + } + if _, err := batch.stmtSyn.Exec(synWord, word, lang); err != nil { + return fmt.Errorf("wiktionary: insert syn: %w", err) + } + synCount++ + } + + // Process translations + for _, tr := range entry.Translations { + targetLang, ok := wiktLangCodeMap[tr.Code] + if !ok || targetLang == lang { + continue + } + trWord := strings.ToLower(tr.Word) + if trWord == "" || containsDigit(trWord) || containsSpace(trWord) { + continue + } + if _, err := batch.stmtTrans.Exec(trWord, targetLang, word, lang); err != nil { + return fmt.Errorf("wiktionary: insert trans: %w", err) + } + transCount++ + } + + // Periodic commit — close current batch and start a new one + if lineCount%commitInterval == 0 { + if err := batch.Commit(); err != nil { + return fmt.Errorf("wiktionary: commit: %w", err) + } + batch, err = newTxBatch(db) + if err != nil { + return fmt.Errorf("wiktionary: %w", err) + } + } + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("wiktionary: scan: %w", err) + } + + if err := batch.Commit(); err != nil { + return fmt.Errorf("wiktionary: final commit: %w", err) + } + + slog.Info("wiktionary loaded", "lang", lang, "definitions", defCount, "synonyms", synCount, "translations", transCount) + return nil +} + +func hasTag(tags []string, targets ...string) bool { + for _, tag := range tags { + for _, t := range targets { + if tag == t { + return true + } + } + } + return false +} + +func hasBrazilTag(tags []string) bool { + for _, tag := range tags { + if tag == "Brazil" || tag == "Brazilian Portuguese" { + return true + } + } + return false +} diff --git a/internal/loader/wiktionary_test.go b/internal/loader/wiktionary_test.go new file mode 100644 index 0000000..f51793a --- /dev/null +++ b/internal/loader/wiktionary_test.go @@ -0,0 +1,187 @@ +package loader + +import ( + "database/sql" + "os" + "path/filepath" + "testing" + + "dreamdict/internal/dictionary" + + _ "modernc.org/sqlite" +) + +func setupWiktTestDB(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(on)") + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + if err := dictionary.BootstrapSchema(db); err != nil { + t.Fatal(err) + } + return db +} + +func writeJSONL(t *testing.T, dir, name, content string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + return path +} + +func TestFormOfFiltering(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + // Seed a word so definitions can reference it + if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('run', 'en')"); err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + jsonl := `{"word":"run","pos":"verb","lang_code":"en","senses":[{"glosses":["to move quickly"],"tags":[]},{"glosses":["past tense of run"],"tags":["form-of"]},{"glosses":["alternative form"],"tags":["alt-of"]}],"synonyms":[],"translations":[]}` + "\n" + path := writeJSONL(t, dir, "test.jsonl", jsonl) + + if err := loadWiktionary(db, path, "en"); err != nil { + t.Fatal(err) + } + + // Only the first sense should be loaded — form-of and alt-of should be filtered + var count int + if err := db.QueryRow("SELECT COUNT(*) FROM definitions").Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Errorf("expected 1 definition (form-of filtered), got %d", count) + } + + // Verify the correct gloss was kept + var gloss string + if err := db.QueryRow("SELECT gloss FROM definitions").Scan(&gloss); err != nil { + t.Fatal(err) + } + if gloss != "to move quickly" { + t.Errorf("expected 'to move quickly', got %q", gloss) + } +} + +func TestBrazilFiltering(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('autocarro', 'pt-PT')"); err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + jsonl := `{"word":"autocarro","pos":"noun","lang_code":"pt","senses":[{"glosses":["a bus (Portugal)"],"tags":["Portugal"]},{"glosses":["a bus (Brazil)"],"tags":["Brazil"]},{"glosses":["a vehicle for transport"],"tags":[]}],"synonyms":[],"translations":[]}` + "\n" + path := writeJSONL(t, dir, "test.jsonl", jsonl) + + if err := loadWiktionary(db, path, "pt-PT"); err != nil { + t.Fatal(err) + } + + // Brazil-tagged sense should be filtered; Portugal-tagged and untagged should remain + var count int + if err := db.QueryRow("SELECT COUNT(*) FROM definitions").Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Errorf("expected 2 definitions (Brazil filtered), got %d", count) + } +} + +func TestBrazilianPortugueseTagFiltering(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('ônibus', 'pt-PT')"); err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + jsonl := `{"word":"ônibus","pos":"noun","lang_code":"pt","senses":[{"glosses":["a bus"],"tags":["Brazilian Portuguese"]}],"synonyms":[],"translations":[]}` + "\n" + path := writeJSONL(t, dir, "test.jsonl", jsonl) + + if err := loadWiktionary(db, path, "pt-PT"); err != nil { + t.Fatal(err) + } + + var count int + if err := db.QueryRow("SELECT COUNT(*) FROM definitions").Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Errorf("expected 0 definitions (Brazilian Portuguese filtered), got %d", count) + } +} + +func TestGlossFiltering(t *testing.T) { + db := setupWiktTestDB(t) + defer db.Close() + + if _, err := db.Exec("INSERT INTO words (word, lang) VALUES ('test', 'en')"); err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + jsonl := `{"word":"test","pos":"noun","lang_code":"en","senses":[{"glosses":["a valid definition"],"tags":[]},{"glosses":["(archaic form)"],"tags":[]},{"glosses":[""],"tags":[]}],"synonyms":[],"translations":[]}` + "\n" + path := writeJSONL(t, dir, "test.jsonl", jsonl) + + if err := loadWiktionary(db, path, "en"); err != nil { + t.Fatal(err) + } + + // Only "a valid definition" should pass — parenthetical and empty filtered + var count int + if err := db.QueryRow("SELECT COUNT(*) FROM definitions").Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Errorf("expected 1 definition (gloss filtering), got %d", count) + } +} + +func TestHasTag(t *testing.T) { + tests := []struct { + tags []string + targets []string + want bool + }{ + {[]string{"form-of", "verb"}, []string{"form-of"}, true}, + {[]string{"verb"}, []string{"form-of", "alt-of"}, false}, + {[]string{"alt-of"}, []string{"form-of", "alt-of"}, true}, + {nil, []string{"form-of"}, false}, + {[]string{}, []string{"form-of"}, false}, + } + for _, tt := range tests { + got := hasTag(tt.tags, tt.targets...) + if got != tt.want { + t.Errorf("hasTag(%v, %v) = %v, want %v", tt.tags, tt.targets, got, tt.want) + } + } +} + +func TestHasBrazilTag(t *testing.T) { + tests := []struct { + tags []string + want bool + }{ + {[]string{"Brazil"}, true}, + {[]string{"Brazilian Portuguese"}, true}, + {[]string{"Portugal"}, false}, + {[]string{"European Portuguese"}, false}, + {nil, false}, + {[]string{}, false}, + } + for _, tt := range tests { + got := hasBrazilTag(tt.tags) + if got != tt.want { + t.Errorf("hasBrazilTag(%v) = %v, want %v", tt.tags, got, tt.want) + } + } +} diff --git a/internal/loader/wolf.go b/internal/loader/wolf.go new file mode 100644 index 0000000..ef13564 --- /dev/null +++ b/internal/loader/wolf.go @@ -0,0 +1,140 @@ +package loader + +import ( + "compress/bzip2" + "database/sql" + "encoding/xml" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" +) + +type WOLFLoader struct{} + +func (WOLFLoader) Name() string { return "wolf" } + +func (WOLFLoader) Load(db *sql.DB, dataDir string) error { + // Try decompressed first, then compressed + path := filepath.Join(dataDir, "wolf.xml") + var reader io.Reader + var closer io.Closer + + if f, err := os.Open(path); err == nil { + reader = f + closer = f + } else { + bzPath := filepath.Join(dataDir, "wolf-1.0b4.xml.bz2") + f, err := os.Open(bzPath) + if err != nil { + return fmt.Errorf("wolf: open: %w (also tried %s)", err, path) + } + reader = bzip2.NewReader(f) + closer = f + } + defer closer.Close() + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("wolf: begin tx: %w", err) + } + defer tx.Rollback() + + stmtDef, err := tx.Prepare(` + INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority) + SELECT id, ?, ?, 'wolf', 10 FROM words WHERE word = ? AND lang = 'fr'`) + if err != nil { + return fmt.Errorf("wolf: prepare def: %w", err) + } + defer stmtDef.Close() + + stmtSyn, err := tx.Prepare(` + INSERT OR IGNORE INTO synonyms (word_id, synonym, source) + SELECT id, ?, 'wolf' FROM words WHERE word = ? AND lang = 'fr'`) + if err != nil { + return fmt.Errorf("wolf: prepare syn: %w", err) + } + defer stmtSyn.Close() + + posMap := map[string]string{ + "n": "noun", + "v": "verb", + "a": "adjective", + "r": "adverb", + } + + var defCount, synCount, decodeErrors int + + decoder := xml.NewDecoder(reader) + for { + tok, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("wolf: xml decode: %w", err) + } + + se, ok := tok.(xml.StartElement) + if !ok || se.Name.Local != "SYNSET" { + continue + } + + var synset struct { + POS string `xml:"POS"` + Literals []struct { + Value string `xml:",chardata"` + } `xml:"SYNONYM>LITERAL"` + DEF string `xml:"DEF"` + } + if err := decoder.DecodeElement(&synset, &se); err != nil { + decodeErrors++ + slog.Debug("wolf: skipping synset", "error", err) + continue + } + + pos := posMap[strings.ToLower(synset.POS)] + gloss := strings.TrimSpace(synset.DEF) + + var words []string + for _, lit := range synset.Literals { + w := strings.ToLower(strings.TrimSpace(lit.Value)) + if w != "" && !containsDigit(w) && !containsSpace(w) { + words = append(words, w) + } + } + + if gloss != "" { + for _, w := range words { + if _, err := stmtDef.Exec(pos, gloss, w); err != nil { + return fmt.Errorf("wolf: insert def: %w", err) + } + defCount++ + } + } + + for i, w1 := range words { + for j, w2 := range words { + if i == j { + continue + } + if _, err := stmtSyn.Exec(w2, w1); err != nil { + return fmt.Errorf("wolf: insert syn: %w", err) + } + synCount++ + } + } + } + + if decodeErrors > 0 { + slog.Warn("wolf: skipped malformed entries", "count", decodeErrors) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("wolf: commit: %w", err) + } + slog.Info("wolf loaded", "definitions", defCount, "synonyms", synCount) + return nil +} diff --git a/internal/loader/wordnet.go b/internal/loader/wordnet.go new file mode 100644 index 0000000..07158b8 --- /dev/null +++ b/internal/loader/wordnet.go @@ -0,0 +1,161 @@ +package loader + +import ( + "bufio" + "database/sql" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" +) + +type WordNetLoader struct{} + +func (WordNetLoader) Name() string { return "wordnet" } + +func (WordNetLoader) Load(db *sql.DB, dataDir string) error { + posFiles := map[string]string{ + "data.noun": "noun", + "data.verb": "verb", + "data.adj": "adjective", + "data.adv": "adverb", + } + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("wordnet: begin tx: %w", err) + } + defer tx.Rollback() + + stmtDef, err := tx.Prepare(` + INSERT OR IGNORE INTO definitions (word_id, pos, gloss, source, priority) + SELECT id, ?, ?, 'wordnet', 10 FROM words WHERE word = ? AND lang = 'en'`) + if err != nil { + return fmt.Errorf("wordnet: prepare def: %w", err) + } + defer stmtDef.Close() + + stmtSyn, err := tx.Prepare(` + INSERT OR IGNORE INTO synonyms (word_id, synonym, source) + SELECT id, ?, 'wordnet' FROM words WHERE word = ? AND lang = 'en'`) + if err != nil { + return fmt.Errorf("wordnet: prepare syn: %w", err) + } + defer stmtSyn.Close() + + var defCount, synCount int + + for file, pos := range posFiles { + path := filepath.Join(dataDir, "wordnet", "dict", file) + d, s, err := loadWordNetFile(stmtDef, stmtSyn, path, pos) + if err != nil { + return fmt.Errorf("wordnet: %s: %w", file, err) + } + defCount += d + synCount += s + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("wordnet: commit: %w", err) + } + slog.Info("wordnet loaded", "definitions", defCount, "synonyms", synCount) + return nil +} + +func loadWordNetFile(stmtDef, stmtSyn *sql.Stmt, path, pos string) (int, int, error) { + f, err := os.Open(path) + if err != nil { + return 0, 0, err + } + defer f.Close() + + var defCount, synCount int + scanner := bufio.NewScanner(f) + buf := make([]byte, 0, 64*1024) + scanner.Buffer(buf, 1024*1024) + + for scanner.Scan() { + line := scanner.Text() + // Skip header lines (start with two spaces) + if strings.HasPrefix(line, " ") { + continue + } + + words, gloss := parseWordNetLine(line) + if gloss == "" || len(words) == 0 { + continue + } + + // Insert definitions for each word + for _, w := range words { + if _, err := stmtDef.Exec(pos, gloss, w); err != nil { + return 0, 0, err + } + defCount++ + } + + // Insert synonym pairs (skip words with underscores) + var cleanWords []string + for _, w := range words { + if !strings.Contains(w, "_") { + cleanWords = append(cleanWords, w) + } + } + for i, w1 := range cleanWords { + for j, w2 := range cleanWords { + if i == j { + continue + } + if _, err := stmtSyn.Exec(w2, w1); err != nil { + return 0, 0, err + } + synCount++ + } + } + } + return defCount, synCount, scanner.Err() +} + +func parseWordNetLine(line string) ([]string, string) { + // Format: synset_offset lex_filenum ss_type w_cnt word lex_id [word lex_id...] p_cnt [ptr...] | gloss + glossIdx := strings.Index(line, "| ") + if glossIdx == -1 { + return nil, "" + } + + gloss := strings.TrimSpace(line[glossIdx+2:]) + // Take gloss before first ";" (drops usage examples) + if semiIdx := strings.Index(gloss, ";"); semiIdx != -1 { + gloss = strings.TrimSpace(gloss[:semiIdx]) + } + if gloss == "" { + return nil, "" + } + + dataPart := line[:glossIdx] + fields := strings.Fields(dataPart) + if len(fields) < 6 { + return nil, "" + } + + // fields[3] = w_cnt (hex) + wc, err := strconv.ParseInt(fields[3], 16, 0) + if err != nil { + slog.Debug("wordnet: bad word count", "field", fields[3], "error", err) + return nil, "" + } + wordCount := int(wc) + if wordCount <= 0 || wordCount > 100 { + slog.Debug("wordnet: unreasonable word count", "count", wordCount) + return nil, "" + } + + var words []string + for i := 0; i < wordCount && 4+i*2 < len(fields); i++ { + w := strings.ToLower(fields[4+i*2]) + words = append(words, w) + } + return words, gloss +} diff --git a/scripts/download-dict-data.sh b/scripts/download-dict-data.sh new file mode 100755 index 0000000..071bcb2 --- /dev/null +++ b/scripts/download-dict-data.sh @@ -0,0 +1,333 @@ +#!/usr/bin/env bash +set -euo pipefail + +DATA_DIR="${1:-./data}" +FORCE="${FORCE:-false}" + +if [[ "${1:-}" == "--force" ]] || [[ "${2:-}" == "--force" ]]; then + FORCE=true + if [[ "${1:-}" == "--force" ]]; then + DATA_DIR="./data" + fi +fi + +mkdir -p "$DATA_DIR" + +# ---- Integrity helpers ---- + +# verify_archive — test that a compressed archive is intact. +# Supports .tar.gz, .gz, .bz2. Returns 0 on success. +verify_archive() { + local file="$1" + case "$file" in + *.tar.gz|*.tgz) tar -tzf "$file" >/dev/null 2>&1 ;; + *.gz) gunzip -t "$file" 2>/dev/null ;; + *.bz2) bunzip2 -t "$file" 2>/dev/null ;; + *) return 0 ;; # unknown format, skip check + esac +} + +# verify_checksum +# Returns 0 if the file's SHA-256 matches. +verify_checksum() { + local file="$1" expected="$2" + local actual + actual=$(sha256sum "$file" | awk '{print $1}') + if [[ "$actual" != "$expected" ]]; then + echo " [FAIL] checksum mismatch for $file" + echo " expected: $expected" + echo " got: $actual" + return 1 + fi + echo " [ok] checksum verified: $file" + return 0 +} + +# check_min_size +# Returns 0 if file exists and is at least min_bytes. +check_min_size() { + local file="$1" min="$2" + if [[ ! -f "$file" ]]; then return 1; fi + local size + size=$(stat --printf='%s' "$file" 2>/dev/null || stat -f '%z' "$file" 2>/dev/null || echo 0) + (( size >= min )) +} + +# Known SHA-256 checksums for pinned-version downloads. +# These are verified once and recorded; update when bumping versions. +CHECKSUM_FILE="$DATA_DIR/.checksums" +touch "$CHECKSUM_FILE" + +# save_checksum