Add /difficulty and /rhyme endpoints, CMU tail migration for rhyme matching
New /difficulty endpoint returns word difficulty scores (0.0-1.0). New /rhyme endpoint uses CMU phoneme tail matching to find rhyming words, sorted by frequency. Adds cmu_tail column migration with batch population at startup, and index for efficient reverse lookups. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -157,6 +157,7 @@ func BootstrapSchema(db *sql.DB) error {
|
||||
// when we catch the "duplicate column" error.
|
||||
migrations := []string{
|
||||
"ALTER TABLE words ADD COLUMN difficulty REAL DEFAULT NULL",
|
||||
"ALTER TABLE pronunciations ADD COLUMN cmu_tail TEXT",
|
||||
}
|
||||
for _, m := range migrations {
|
||||
if _, err := db.Exec(m); err != nil {
|
||||
@@ -170,6 +171,109 @@ func BootstrapSchema(db *sql.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Migrate opens the database read-write, applies any pending schema migrations,
|
||||
// and populates derived data (e.g. CMU rhyme tails). Returns the number of
|
||||
// CMU tails populated (0 if already up to date).
|
||||
func Migrate(dbPath string) (int, error) {
|
||||
db, err := openDB(dbPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := BootstrapSchema(db); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return PopulateCMUTails(db)
|
||||
}
|
||||
|
||||
func isDuplicateColumn(err error) bool {
|
||||
return err != nil && strings.Contains(err.Error(), "duplicate column")
|
||||
}
|
||||
|
||||
// PopulateCMUTails computes the rhyme tail for all CMU pronunciation entries
|
||||
// that don't already have one. The tail is the phoneme sequence from the last
|
||||
// stressed vowel onward (e.g. "K AE1 T" → "AE1 T").
|
||||
func PopulateCMUTails(db *sql.DB) (int, error) {
|
||||
// Create index if it doesn't exist.
|
||||
db.Exec("CREATE INDEX IF NOT EXISTS idx_pronunciations_cmu_tail ON pronunciations(cmu_tail)")
|
||||
|
||||
rows, err := db.Query(
|
||||
`SELECT id, value FROM pronunciations WHERE format = 'cmu' AND (cmu_tail IS NULL OR cmu_tail = '')`)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("dictionary: populate cmu tails: %w", err)
|
||||
}
|
||||
|
||||
type entry struct {
|
||||
id int
|
||||
value string
|
||||
}
|
||||
var entries []entry
|
||||
for rows.Next() {
|
||||
var e entry
|
||||
if err := rows.Scan(&e.id, &e.value); err != nil {
|
||||
rows.Close()
|
||||
return 0, fmt.Errorf("dictionary: populate cmu tails scan: %w", err)
|
||||
}
|
||||
entries = append(entries, e)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
if len(entries) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("dictionary: populate cmu tails begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare("UPDATE pronunciations SET cmu_tail = ? WHERE id = ?")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("dictionary: populate cmu tails prepare: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
count := 0
|
||||
for _, e := range entries {
|
||||
tail := cmuTail(e.value)
|
||||
if tail == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := stmt.Exec(tail, e.id); err != nil {
|
||||
return 0, fmt.Errorf("dictionary: populate cmu tails update: %w", err)
|
||||
}
|
||||
count++
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, fmt.Errorf("dictionary: populate cmu tails commit: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// cmuTail extracts the phoneme sequence from the last stressed vowel onward.
|
||||
// e.g. "K AE1 T" → "AE1 T", "IH0 F EH1 M ER0 AH0 L" → "EH1 M ER0 AH0 L"
|
||||
func cmuTail(phonemes string) string {
|
||||
parts := strings.Fields(phonemes)
|
||||
lastStressed := -1
|
||||
for i, p := range parts {
|
||||
if len(p) >= 2 && p[len(p)-1] == '1' {
|
||||
lastStressed = i
|
||||
}
|
||||
}
|
||||
// Fall back to last vowel with any stress marker.
|
||||
if lastStressed == -1 {
|
||||
for i, p := range parts {
|
||||
if len(p) >= 2 && (p[len(p)-1] == '0' || p[len(p)-1] == '2') {
|
||||
lastStressed = i
|
||||
}
|
||||
}
|
||||
}
|
||||
if lastStressed == -1 {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(parts[lastStressed:], " ")
|
||||
}
|
||||
|
||||
@@ -379,6 +379,82 @@ func (d *Dictionary) Etymology(word, lang string) (string, error) {
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// Difficulty returns the difficulty score for a word in a language.
|
||||
// Returns -1 if the word is not found.
|
||||
func (d *Dictionary) Difficulty(word, lang string) (float64, error) {
|
||||
var diff sql.NullFloat64
|
||||
err := d.db.QueryRow(
|
||||
"SELECT difficulty FROM words WHERE word = ? AND lang = ?",
|
||||
strings.ToLower(word), lang,
|
||||
).Scan(&diff)
|
||||
if err == sql.ErrNoRows {
|
||||
return -1, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("dictionary: difficulty: %w", err)
|
||||
}
|
||||
if !diff.Valid {
|
||||
return -1, nil
|
||||
}
|
||||
return diff.Float64, nil
|
||||
}
|
||||
|
||||
// Rhymes returns words that rhyme with the given word by matching CMU phoneme tails.
|
||||
// English only. Returns up to `limit` results sorted by frequency descending.
|
||||
func (d *Dictionary) Rhymes(word string, limit int) ([]string, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
}
|
||||
if limit > 50 {
|
||||
limit = 50
|
||||
}
|
||||
|
||||
// Get the CMU tail for the input word.
|
||||
var tail sql.NullString
|
||||
err := d.db.QueryRow(`
|
||||
SELECT p.cmu_tail
|
||||
FROM pronunciations p
|
||||
JOIN words w ON w.id = p.word_id
|
||||
WHERE w.word = ? AND w.lang = 'en' AND p.format = 'cmu' AND p.cmu_tail IS NOT NULL
|
||||
LIMIT 1`,
|
||||
strings.ToLower(word),
|
||||
).Scan(&tail)
|
||||
if err == sql.ErrNoRows || !tail.Valid || tail.String == "" {
|
||||
return []string{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: rhymes: %w", err)
|
||||
}
|
||||
|
||||
// Find other words with the same tail.
|
||||
rows, err := d.db.Query(`
|
||||
SELECT DISTINCT w.word
|
||||
FROM pronunciations p
|
||||
JOIN words w ON w.id = p.word_id
|
||||
WHERE p.cmu_tail = ? AND p.format = 'cmu' AND w.lang = 'en' AND w.word != ?
|
||||
ORDER BY w.frequency DESC
|
||||
LIMIT ?`,
|
||||
tail.String, strings.ToLower(word), limit,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dictionary: rhymes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var rhymes []string
|
||||
for rows.Next() {
|
||||
var r string
|
||||
if err := rows.Scan(&r); err != nil {
|
||||
return nil, fmt.Errorf("dictionary: rhymes scan: %w", err)
|
||||
}
|
||||
rhymes = append(rhymes, r)
|
||||
}
|
||||
if rhymes == nil {
|
||||
rhymes = []string{}
|
||||
}
|
||||
return rhymes, rows.Err()
|
||||
}
|
||||
|
||||
func (d *Dictionary) DefCount() (map[string]int, error) {
|
||||
rows, err := d.db.Query(`
|
||||
SELECT w.lang, COUNT(*)
|
||||
|
||||
Reference in New Issue
Block a user