Add /difficulty and /rhyme endpoints, CMU tail migration for rhyme matching

New /difficulty endpoint returns word difficulty scores (0.0-1.0).
New /rhyme endpoint uses CMU phoneme tail matching to find rhyming words,
sorted by frequency. Adds cmu_tail column migration with batch population
at startup, and index for efficient reverse lookups.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
prosolis
2026-04-02 06:45:29 -07:00
parent 207086da08
commit 25ed087c3e
3 changed files with 245 additions and 0 deletions

View File

@@ -157,6 +157,7 @@ func BootstrapSchema(db *sql.DB) error {
// when we catch the "duplicate column" error.
migrations := []string{
"ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL",
"ALTER TABLE pronunciations ADD COLUMN cmu_tail TEXT",
}
for _, m := range migrations {
if _, err := db.Exec(m); err != nil {
@@ -170,6 +171,109 @@ func BootstrapSchema(db *sql.DB) error {
return nil
}
// Migrate opens the database read-write, applies any pending schema migrations,
// and populates derived data (e.g. CMU rhyme tails). Returns the number of
// CMU tails populated (0 if already up to date).
func Migrate(dbPath string) (int, error) {
db, err := openDB(dbPath)
if err != nil {
return 0, err
}
defer db.Close()
if err := BootstrapSchema(db); err != nil {
return 0, err
}
return PopulateCMUTails(db)
}
func isDuplicateColumn(err error) bool {
return err != nil && strings.Contains(err.Error(), "duplicate column")
}
// PopulateCMUTails computes the rhyme tail for all CMU pronunciation entries
// that don't already have one. The tail is the phoneme sequence from the last
// stressed vowel onward (e.g. "K AE1 T" → "AE1 T").
func PopulateCMUTails(db *sql.DB) (int, error) {
// Create index if it doesn't exist.
db.Exec("CREATE INDEX IF NOT EXISTS idx_pronunciations_cmu_tail ON pronunciations(cmu_tail)")
rows, err := db.Query(
`SELECT id, value FROM pronunciations WHERE format = 'cmu' AND (cmu_tail IS NULL OR cmu_tail = '')`)
if err != nil {
return 0, fmt.Errorf("dictionary: populate cmu tails: %w", err)
}
type entry struct {
id int
value string
}
var entries []entry
for rows.Next() {
var e entry
if err := rows.Scan(&e.id, &e.value); err != nil {
rows.Close()
return 0, fmt.Errorf("dictionary: populate cmu tails scan: %w", err)
}
entries = append(entries, e)
}
rows.Close()
if len(entries) == 0 {
return 0, nil
}
tx, err := db.Begin()
if err != nil {
return 0, fmt.Errorf("dictionary: populate cmu tails begin: %w", err)
}
defer tx.Rollback()
stmt, err := tx.Prepare("UPDATE pronunciations SET cmu_tail = ? WHERE id = ?")
if err != nil {
return 0, fmt.Errorf("dictionary: populate cmu tails prepare: %w", err)
}
defer stmt.Close()
count := 0
for _, e := range entries {
tail := cmuTail(e.value)
if tail == "" {
continue
}
if _, err := stmt.Exec(tail, e.id); err != nil {
return 0, fmt.Errorf("dictionary: populate cmu tails update: %w", err)
}
count++
}
if err := tx.Commit(); err != nil {
return 0, fmt.Errorf("dictionary: populate cmu tails commit: %w", err)
}
return count, nil
}
// cmuTail extracts the phoneme sequence from the last stressed vowel onward.
// e.g. "K AE1 T" → "AE1 T", "IH0 F EH1 M ER0 AH0 L" → "EH1 M ER0 AH0 L"
func cmuTail(phonemes string) string {
parts := strings.Fields(phonemes)
lastStressed := -1
for i, p := range parts {
if len(p) >= 2 && p[len(p)-1] == '1' {
lastStressed = i
}
}
// Fall back to last vowel with any stress marker.
if lastStressed == -1 {
for i, p := range parts {
if len(p) >= 2 && (p[len(p)-1] == '0' || p[len(p)-1] == '2') {
lastStressed = i
}
}
}
if lastStressed == -1 {
return ""
}
return strings.Join(parts[lastStressed:], " ")
}

View File

@@ -379,6 +379,82 @@ func (d *Dictionary) Etymology(word, lang string) (string, error) {
return text, nil
}
// Difficulty returns the difficulty score for a word in a language.
// Returns -1 if the word is not found.
func (d *Dictionary) Difficulty(word, lang string) (float64, error) {
var diff sql.NullFloat64
err := d.db.QueryRow(
"SELECT difficulty FROM words WHERE word = ? AND lang = ?",
strings.ToLower(word), lang,
).Scan(&diff)
if err == sql.ErrNoRows {
return -1, nil
}
if err != nil {
return 0, fmt.Errorf("dictionary: difficulty: %w", err)
}
if !diff.Valid {
return -1, nil
}
return diff.Float64, nil
}
// Rhymes returns words that rhyme with the given word by matching CMU phoneme tails.
// English only. Returns up to `limit` results sorted by frequency descending.
func (d *Dictionary) Rhymes(word string, limit int) ([]string, error) {
if limit <= 0 {
limit = 10
}
if limit > 50 {
limit = 50
}
// Get the CMU tail for the input word.
var tail sql.NullString
err := d.db.QueryRow(`
SELECT p.cmu_tail
FROM pronunciations p
JOIN words w ON w.id = p.word_id
WHERE w.word = ? AND w.lang = 'en' AND p.format = 'cmu' AND p.cmu_tail IS NOT NULL
LIMIT 1`,
strings.ToLower(word),
).Scan(&tail)
if err == sql.ErrNoRows || !tail.Valid || tail.String == "" {
return []string{}, nil
}
if err != nil {
return nil, fmt.Errorf("dictionary: rhymes: %w", err)
}
// Find other words with the same tail.
rows, err := d.db.Query(`
SELECT DISTINCT w.word
FROM pronunciations p
JOIN words w ON w.id = p.word_id
WHERE p.cmu_tail = ? AND p.format = 'cmu' AND w.lang = 'en' AND w.word != ?
ORDER BY w.frequency DESC
LIMIT ?`,
tail.String, strings.ToLower(word), limit,
)
if err != nil {
return nil, fmt.Errorf("dictionary: rhymes: %w", err)
}
defer rows.Close()
var rhymes []string
for rows.Next() {
var r string
if err := rows.Scan(&r); err != nil {
return nil, fmt.Errorf("dictionary: rhymes scan: %w", err)
}
rhymes = append(rhymes, r)
}
if rhymes == nil {
rhymes = []string{}
}
return rhymes, rows.Err()
}
func (d *Dictionary) DefCount() (map[string]int, error) {
rows, err := d.db.Query(`
SELECT w.lang, COUNT(*)