Add Phase 2+3 features: antonyms, backing, pronunciation, etymology, difficulty, affix expansion

New endpoints: /antonyms, /backing, /pronunciation, /etymology with difficulty
scoring on /random. Cross-language synset backing links French/Portuguese words
to English equivalents via WordNet 3.0 synset IDs (matching WOLF and OMW offsets).

New loaders: Hunspell affix expansion (fr, pt-PT), English programmatic inflector,
CMU Pronouncing Dictionary, SUBTLEX-US frequency, CETEMPúblico frequency,
Open Multilingual Wordnet (Portuguese), and SQL-based difficulty scoring.

Schema v2 adds tables: antonyms, synsets, word_synsets, pronunciations, etymology
with migration support for existing databases.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
prosolis
2026-04-02 01:01:06 -07:00
parent d5e4ce0d36
commit 207086da08
22 changed files with 3102 additions and 120 deletions

View File

@@ -0,0 +1,72 @@
package loader
import (
"database/sql"
"fmt"
"log/slog"
)
// DifficultyScorer computes a composite difficulty score for all words
// in the database. It runs after all other loaders have completed.
// Score range: 0.0 (easiest) to 1.0 (hardest).
//
// Formula:
// difficulty = normalize(1/frequency) * 0.5
// + normalize(length) * 0.3
// + normalize(syllable_count_approx) * 0.2
//
// Computed entirely in SQL for performance — no row-by-row round trips.
type DifficultyScorer struct{}
func (DifficultyScorer) Name() string { return "difficulty" }
func (DifficultyScorer) Load(db *sql.DB, _ string) error {
langs := []string{"en", "fr", "pt-PT", "zh"}
var totalUpdated int64
for _, lang := range langs {
n, err := scoreLang(db, lang)
if err != nil {
return fmt.Errorf("difficulty: %s: %w", lang, err)
}
totalUpdated += n
}
slog.Info("difficulty scored", "total_words", totalUpdated)
return nil
}
func scoreLang(db *sql.DB, lang string) (int64, error) {
// Single UPDATE using SQLite math to compute difficulty in-place.
// log() isn't available in base SQLite, so we approximate the frequency
// component using a reciprocal: 1.0 / (1.0 + frequency/max_freq).
// This gives a 01 range where 0 = most common, ~1 = rarest.
//
// Length and a rough syllable proxy (length/3) are normalized against
// per-language maximums via a subquery.
const q = `
UPDATE words SET difficulty = ROUND(
CASE
WHEN stats.max_freq > 0 AND frequency > 0
THEN (1.0 - CAST(frequency AS REAL) / stats.max_freq) * 0.5
ELSE 0.5
END
+ (CAST(LENGTH(word) AS REAL) / stats.max_len) * 0.3
+ MIN(1.0, CAST(LENGTH(word) AS REAL) / 3.0 / (stats.max_len / 3.0)) * 0.2
, 3)
FROM (
SELECT
MAX(frequency) AS max_freq,
MAX(LENGTH(word)) AS max_len
FROM words WHERE lang = ?1
) AS stats
WHERE words.lang = ?1
`
res, err := db.Exec(q, lang)
if err != nil {
return 0, fmt.Errorf("update difficulty: %w", err)
}
n, _ := res.RowsAffected()
return n, nil
}