207086da08
New endpoints: /antonyms, /backing, /pronunciation, /etymology with difficulty scoring on /random. Cross-language synset backing links French/Portuguese words to English equivalents via WordNet 3.0 synset IDs (matching WOLF and OMW offsets). New loaders: Hunspell affix expansion (fr, pt-PT), English programmatic inflector, CMU Pronouncing Dictionary, SUBTLEX-US frequency, CETEMPúblico frequency, Open Multilingual Wordnet (Portuguese), and SQL-based difficulty scoring. Schema v2 adds tables: antonyms, synsets, word_synsets, pronunciations, etymology with migration support for existing databases. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
73 lines
2.0 KiB
Go
73 lines
2.0 KiB
Go
package loader
|
||
|
||
import (
|
||
"database/sql"
|
||
"fmt"
|
||
"log/slog"
|
||
)
|
||
|
||
// DifficultyScorer computes a composite difficulty score for all words
|
||
// in the database. It runs after all other loaders have completed.
|
||
// Score range: 0.0 (easiest) to 1.0 (hardest).
|
||
//
|
||
// Formula:
|
||
// difficulty = normalize(1/frequency) * 0.5
|
||
// + normalize(length) * 0.3
|
||
// + normalize(syllable_count_approx) * 0.2
|
||
//
|
||
// Computed entirely in SQL for performance — no row-by-row round trips.
|
||
type DifficultyScorer struct{}
|
||
|
||
func (DifficultyScorer) Name() string { return "difficulty" }
|
||
|
||
func (DifficultyScorer) Load(db *sql.DB, _ string) error {
|
||
langs := []string{"en", "fr", "pt-PT", "zh"}
|
||
|
||
var totalUpdated int64
|
||
for _, lang := range langs {
|
||
n, err := scoreLang(db, lang)
|
||
if err != nil {
|
||
return fmt.Errorf("difficulty: %s: %w", lang, err)
|
||
}
|
||
totalUpdated += n
|
||
}
|
||
|
||
slog.Info("difficulty scored", "total_words", totalUpdated)
|
||
return nil
|
||
}
|
||
|
||
func scoreLang(db *sql.DB, lang string) (int64, error) {
|
||
// Single UPDATE using SQLite math to compute difficulty in-place.
|
||
// log() isn't available in base SQLite, so we approximate the frequency
|
||
// component using a reciprocal: 1.0 / (1.0 + frequency/max_freq).
|
||
// This gives a 0–1 range where 0 = most common, ~1 = rarest.
|
||
//
|
||
// Length and a rough syllable proxy (length/3) are normalized against
|
||
// per-language maximums via a subquery.
|
||
const q = `
|
||
UPDATE words SET difficulty = ROUND(
|
||
CASE
|
||
WHEN stats.max_freq > 0 AND frequency > 0
|
||
THEN (1.0 - CAST(frequency AS REAL) / stats.max_freq) * 0.5
|
||
ELSE 0.5
|
||
END
|
||
+ (CAST(LENGTH(word) AS REAL) / stats.max_len) * 0.3
|
||
+ MIN(1.0, CAST(LENGTH(word) AS REAL) / 3.0 / (stats.max_len / 3.0)) * 0.2
|
||
, 3)
|
||
FROM (
|
||
SELECT
|
||
MAX(frequency) AS max_freq,
|
||
MAX(LENGTH(word)) AS max_len
|
||
FROM words WHERE lang = ?1
|
||
) AS stats
|
||
WHERE words.lang = ?1
|
||
`
|
||
|
||
res, err := db.Exec(q, lang)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("update difficulty: %w", err)
|
||
}
|
||
n, _ := res.RowsAffected()
|
||
return n, nil
|
||
}
|