Files
dreamdict/internal/dictionary/query.go
prosolis 207086da08 Add Phase 2+3 features: antonyms, backing, pronunciation, etymology, difficulty, affix expansion
New endpoints: /antonyms, /backing, /pronunciation, /etymology with difficulty
scoring on /random. Cross-language synset backing links French/Portuguese words
to English equivalents via WordNet 3.0 synset IDs (matching WOLF and OMW offsets).

New loaders: Hunspell affix expansion (fr, pt-PT), English programmatic inflector,
CMU Pronouncing Dictionary, SUBTLEX-US frequency, CETEMPúblico frequency,
Open Multilingual Wordnet (Portuguese), and SQL-based difficulty scoring.

Schema v2 adds tables: antonyms, synsets, word_synsets, pronunciations, etymology
with migration support for existing databases.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-02 01:01:06 -07:00

404 lines
10 KiB
Go

package dictionary
import (
"database/sql"
"fmt"
"strings"
)
func (d *Dictionary) IsValidWord(word, lang string) (bool, error) {
var exists bool
err := d.db.QueryRow(
"SELECT EXISTS(SELECT 1 FROM words WHERE word = ? AND lang = ?)",
strings.ToLower(word), lang,
).Scan(&exists)
if err != nil {
return false, fmt.Errorf("dictionary: is valid word: %w", err)
}
return exists, nil
}
type RandomResult struct {
Word string `json:"word"`
Difficulty *float64 `json:"difficulty,omitempty"`
}
func (d *Dictionary) RandomWord(lang string, opts Options) (RandomResult, error) {
query := "SELECT word, difficulty FROM words WHERE lang = ?"
args := []any{lang}
if opts.POS != "" {
query += " AND pos = ?"
args = append(args, opts.POS)
}
if opts.MinLength > 0 {
query += " AND LENGTH(word) >= ?"
args = append(args, opts.MinLength)
}
if opts.MaxLength > 0 {
query += " AND LENGTH(word) <= ?"
args = append(args, opts.MaxLength)
}
if opts.MinFrequency > 0 {
query += " AND frequency >= ?"
args = append(args, opts.MinFrequency)
}
if opts.MinDifficulty > 0 {
query += " AND difficulty >= ?"
args = append(args, opts.MinDifficulty)
}
if opts.MaxDifficulty > 0 {
query += " AND difficulty <= ?"
args = append(args, opts.MaxDifficulty)
}
query += " ORDER BY RANDOM() LIMIT 1"
var result RandomResult
var diff sql.NullFloat64
err := d.db.QueryRow(query, args...).Scan(&result.Word, &diff)
if err == sql.ErrNoRows {
return RandomResult{}, ErrNoMatch
}
if err != nil {
return RandomResult{}, fmt.Errorf("dictionary: random word: %w", err)
}
if diff.Valid {
result.Difficulty = &diff.Float64
}
return result, nil
}
func (d *Dictionary) Define(word, lang string) ([]Definition, error) {
rows, err := d.db.Query(`
SELECT d.pos, d.gloss, d.source, d.priority
FROM definitions d
JOIN words w ON w.id = d.word_id
WHERE w.word = ? AND w.lang = ?
ORDER BY d.priority ASC, d.id ASC`,
strings.ToLower(word), lang,
)
if err != nil {
return nil, fmt.Errorf("dictionary: define: %w", err)
}
defer rows.Close()
var defs []Definition
for rows.Next() {
var def Definition
var pos sql.NullString
if err := rows.Scan(&pos, &def.Gloss, &def.Source, &def.Priority); err != nil {
return nil, fmt.Errorf("dictionary: define scan: %w", err)
}
def.POS = pos.String
defs = append(defs, def)
}
if defs == nil {
defs = []Definition{}
}
return defs, rows.Err()
}
func (d *Dictionary) Synonyms(word, lang string) ([]string, error) {
rows, err := d.db.Query(`
SELECT DISTINCT s.synonym
FROM synonyms s
JOIN words w ON w.id = s.word_id
WHERE w.word = ? AND w.lang = ?
ORDER BY s.synonym`,
strings.ToLower(word), lang,
)
if err != nil {
return nil, fmt.Errorf("dictionary: synonyms: %w", err)
}
defer rows.Close()
var syns []string
for rows.Next() {
var syn string
if err := rows.Scan(&syn); err != nil {
return nil, fmt.Errorf("dictionary: synonyms scan: %w", err)
}
syns = append(syns, syn)
}
if syns == nil {
syns = []string{}
}
return syns, rows.Err()
}
func (d *Dictionary) Translate(word, fromLang, toLang string) ([]string, error) {
rows, err := d.db.Query(`
SELECT DISTINCT t.translation
FROM translations t
JOIN words w ON w.id = t.word_id
WHERE w.word = ? AND w.lang = ? AND t.target_lang = ?
ORDER BY t.translation`,
strings.ToLower(word), fromLang, toLang,
)
if err != nil {
return nil, fmt.Errorf("dictionary: translate: %w", err)
}
defer rows.Close()
var trans []string
for rows.Next() {
var t string
if err := rows.Scan(&t); err != nil {
return nil, fmt.Errorf("dictionary: translate scan: %w", err)
}
trans = append(trans, t)
}
if trans == nil {
trans = []string{}
}
return trans, rows.Err()
}
func (d *Dictionary) WordCount() (map[string]int, error) {
rows, err := d.db.Query("SELECT lang, COUNT(*) FROM words GROUP BY lang")
if err != nil {
return nil, fmt.Errorf("dictionary: word count: %w", err)
}
defer rows.Close()
counts := make(map[string]int)
for rows.Next() {
var lang string
var count int
if err := rows.Scan(&lang, &count); err != nil {
return nil, fmt.Errorf("dictionary: word count scan: %w", err)
}
counts[lang] = count
}
return counts, rows.Err()
}
// Frequency returns the frequency score for a word in a language.
// Returns 0 if the word is not found or has no frequency data.
// Higher values indicate more common words.
func (d *Dictionary) Frequency(word, lang string) (int, error) {
var freq int
err := d.db.QueryRow(
"SELECT COALESCE(frequency, 0) FROM words WHERE word = ? AND lang = ?",
strings.ToLower(word), lang,
).Scan(&freq)
if err == sql.ErrNoRows {
return 0, nil
}
if err != nil {
return 0, fmt.Errorf("dictionary: frequency: %w", err)
}
return freq, nil
}
// FrequencyBatch returns frequency scores for multiple words in a language.
// Words not found or without frequency data are returned with a score of 0.
func (d *Dictionary) FrequencyBatch(words []string, lang string) (map[string]int, error) {
if len(words) == 0 {
return map[string]int{}, nil
}
placeholders := make([]string, len(words))
args := make([]any, 0, len(words)+1)
args = append(args, lang)
for i, w := range words {
placeholders[i] = "?"
args = append(args, strings.ToLower(w))
}
query := fmt.Sprintf(
"SELECT word, COALESCE(frequency, 0) FROM words WHERE lang = ? AND word IN (%s)",
strings.Join(placeholders, ","),
)
rows, err := d.db.Query(query, args...)
if err != nil {
return nil, fmt.Errorf("dictionary: frequency batch: %w", err)
}
defer rows.Close()
result := make(map[string]int, len(words))
for rows.Next() {
var word string
var freq int
if err := rows.Scan(&word, &freq); err != nil {
return nil, fmt.Errorf("dictionary: frequency batch scan: %w", err)
}
result[word] = freq
}
return result, rows.Err()
}
func (d *Dictionary) Meta(key string) (string, error) {
var val string
err := d.db.QueryRow("SELECT value FROM meta WHERE key = ?", key).Scan(&val)
if err == sql.ErrNoRows {
return "", nil
}
if err != nil {
return "", fmt.Errorf("dictionary: meta %s: %w", key, err)
}
return val, nil
}
func (d *Dictionary) Antonyms(word, lang string) ([]string, error) {
rows, err := d.db.Query(`
SELECT DISTINCT a.antonym
FROM antonyms a
JOIN words w ON w.id = a.word_id
WHERE w.word = ? AND w.lang = ?
ORDER BY a.antonym`,
strings.ToLower(word), lang,
)
if err != nil {
return nil, fmt.Errorf("dictionary: antonyms: %w", err)
}
defer rows.Close()
var ants []string
for rows.Next() {
var a string
if err := rows.Scan(&a); err != nil {
return nil, fmt.Errorf("dictionary: antonyms scan: %w", err)
}
ants = append(ants, a)
}
if ants == nil {
ants = []string{}
}
return ants, rows.Err()
}
// EnglishBacking returns English words that share a synset with the given
// non-English word, providing semantic equivalents with WordNet definitions.
func (d *Dictionary) EnglishBacking(word, lang string) ([]EnglishEquivalent, error) {
// Find synsets for this word, then find English words in those synsets,
// along with their WordNet definitions.
rows, err := d.db.Query(`
SELECT DISTINCT ew.word,
COALESCE(
(SELECT d.gloss FROM definitions d
WHERE d.word_id = ew.id AND d.source = 'wordnet' AND d.pos = s.pos
ORDER BY d.priority ASC LIMIT 1),
(SELECT d.gloss FROM definitions d
WHERE d.word_id = ew.id AND d.source = 'wordnet'
ORDER BY d.priority ASC LIMIT 1),
''
) AS gloss,
s.synset_id
FROM words w
JOIN word_synsets ws ON ws.word_id = w.id
JOIN synsets s ON s.id = ws.synset_id
JOIN word_synsets ews ON ews.synset_id = s.id
JOIN words ew ON ew.id = ews.word_id AND ew.lang = 'en'
WHERE w.word = ? AND w.lang = ?
ORDER BY s.synset_id, ew.word`,
strings.ToLower(word), lang,
)
if err != nil {
return nil, fmt.Errorf("dictionary: english backing: %w", err)
}
defer rows.Close()
var results []EnglishEquivalent
seen := make(map[string]bool)
for rows.Next() {
var eq EnglishEquivalent
if err := rows.Scan(&eq.Word, &eq.Definition, &eq.Synset); err != nil {
return nil, fmt.Errorf("dictionary: english backing scan: %w", err)
}
key := eq.Word + "|" + eq.Synset
if !seen[key] {
seen[key] = true
results = append(results, eq)
}
}
if results == nil {
results = []EnglishEquivalent{}
}
// Fallback to translations table if no synset matches
if len(results) == 0 {
trans, err := d.Translate(word, lang, "en")
if err != nil {
return nil, err
}
for _, t := range trans {
results = append(results, EnglishEquivalent{Word: t})
}
}
return results, rows.Err()
}
func (d *Dictionary) Pronunciation(word, lang string) ([]Pronunciation, error) {
rows, err := d.db.Query(`
SELECT p.format, p.value, p.source
FROM pronunciations p
JOIN words w ON w.id = p.word_id
WHERE w.word = ? AND w.lang = ?
ORDER BY p.source, p.format`,
strings.ToLower(word), lang,
)
if err != nil {
return nil, fmt.Errorf("dictionary: pronunciation: %w", err)
}
defer rows.Close()
var prons []Pronunciation
for rows.Next() {
var p Pronunciation
if err := rows.Scan(&p.Format, &p.Value, &p.Source); err != nil {
return nil, fmt.Errorf("dictionary: pronunciation scan: %w", err)
}
prons = append(prons, p)
}
if prons == nil {
prons = []Pronunciation{}
}
return prons, rows.Err()
}
func (d *Dictionary) Etymology(word, lang string) (string, error) {
var text string
err := d.db.QueryRow(`
SELECT e.text
FROM etymology e
JOIN words w ON w.id = e.word_id
WHERE w.word = ? AND w.lang = ?
LIMIT 1`,
strings.ToLower(word), lang,
).Scan(&text)
if err == sql.ErrNoRows {
return "", nil
}
if err != nil {
return "", fmt.Errorf("dictionary: etymology: %w", err)
}
return text, nil
}
func (d *Dictionary) DefCount() (map[string]int, error) {
rows, err := d.db.Query(`
SELECT w.lang, COUNT(*)
FROM definitions d
JOIN words w ON w.id = d.word_id
GROUP BY w.lang`)
if err != nil {
return nil, fmt.Errorf("dictionary: def count: %w", err)
}
defer rows.Close()
counts := make(map[string]int)
for rows.Next() {
var lang string
var count int
if err := rows.Scan(&lang, &count); err != nil {
return nil, fmt.Errorf("dictionary: def count scan: %w", err)
}
counts[lang] = count
}
return counts, rows.Err()
}