Add Phase 2+3 features: antonyms, backing, pronunciation, etymology, difficulty, affix expansion
New endpoints: /antonyms, /backing, /pronunciation, /etymology with difficulty scoring on /random. Cross-language synset backing links French/Portuguese words to English equivalents via WordNet 3.0 synset IDs (matching WOLF and OMW offsets). New loaders: Hunspell affix expansion (fr, pt-PT), English programmatic inflector, CMU Pronouncing Dictionary, SUBTLEX-US frequency, CETEMPúblico frequency, Open Multilingual Wordnet (Portuguese), and SQL-based difficulty scoring. Schema v2 adds tables: antonyms, synsets, word_synsets, pronunciations, etymology with migration support for existing databases. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
72
internal/loader/difficulty.go
Normal file
72
internal/loader/difficulty.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package loader
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// DifficultyScorer computes a composite difficulty score for all words
|
||||
// in the database. It runs after all other loaders have completed.
|
||||
// Score range: 0.0 (easiest) to 1.0 (hardest).
|
||||
//
|
||||
// Formula:
|
||||
// difficulty = normalize(1/frequency) * 0.5
|
||||
// + normalize(length) * 0.3
|
||||
// + normalize(syllable_count_approx) * 0.2
|
||||
//
|
||||
// Computed entirely in SQL for performance — no row-by-row round trips.
|
||||
type DifficultyScorer struct{}
|
||||
|
||||
func (DifficultyScorer) Name() string { return "difficulty" }
|
||||
|
||||
func (DifficultyScorer) Load(db *sql.DB, _ string) error {
|
||||
langs := []string{"en", "fr", "pt-PT", "zh"}
|
||||
|
||||
var totalUpdated int64
|
||||
for _, lang := range langs {
|
||||
n, err := scoreLang(db, lang)
|
||||
if err != nil {
|
||||
return fmt.Errorf("difficulty: %s: %w", lang, err)
|
||||
}
|
||||
totalUpdated += n
|
||||
}
|
||||
|
||||
slog.Info("difficulty scored", "total_words", totalUpdated)
|
||||
return nil
|
||||
}
|
||||
|
||||
func scoreLang(db *sql.DB, lang string) (int64, error) {
|
||||
// Single UPDATE using SQLite math to compute difficulty in-place.
|
||||
// log() isn't available in base SQLite, so we approximate the frequency
|
||||
// component using a reciprocal: 1.0 / (1.0 + frequency/max_freq).
|
||||
// This gives a 0–1 range where 0 = most common, ~1 = rarest.
|
||||
//
|
||||
// Length and a rough syllable proxy (length/3) are normalized against
|
||||
// per-language maximums via a subquery.
|
||||
const q = `
|
||||
UPDATE words SET difficulty = ROUND(
|
||||
CASE
|
||||
WHEN stats.max_freq > 0 AND frequency > 0
|
||||
THEN (1.0 - CAST(frequency AS REAL) / stats.max_freq) * 0.5
|
||||
ELSE 0.5
|
||||
END
|
||||
+ (CAST(LENGTH(word) AS REAL) / stats.max_len) * 0.3
|
||||
+ MIN(1.0, CAST(LENGTH(word) AS REAL) / 3.0 / (stats.max_len / 3.0)) * 0.2
|
||||
, 3)
|
||||
FROM (
|
||||
SELECT
|
||||
MAX(frequency) AS max_freq,
|
||||
MAX(LENGTH(word)) AS max_len
|
||||
FROM words WHERE lang = ?1
|
||||
) AS stats
|
||||
WHERE words.lang = ?1
|
||||
`
|
||||
|
||||
res, err := db.Exec(q, lang)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("update difficulty: %w", err)
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
return n, nil
|
||||
}
|
||||
Reference in New Issue
Block a user