Editor: document tone, right-click word lookup, expanded stats
Four enhancements to make the editor fit real school usage:
- Per-document tone (academic/professional/casual/humorous/creative/
persuasive/general): new documents.tone column (migration 0002), threaded
through the docs API, a bilingual ToneSelect dropdown on the title row, and
injected into the grammar-checkpoint LLM prompt so advice fits the register.
The voice pass stays tone-agnostic.
- Right-click word lookup: a new offline `lexicon` package serves definitions
(Wordset, modern ESL-friendly glosses) and synonyms (WordNet synsets first,
then frequency+stopword-ranked Moby for breadth) from gzipped embedded data,
behind /api/word/{word} with light morphology. The WordCard popover shows the
definition and tappable synonym pills that swap the word in place.
- Expanded writing stats: clicking the word count opens a StatsPanel with page
count, sentences, paragraphs, reading time, average word length, word variety,
and Flesch-Kincaid reading level — all computed client-side.
Claude-Session: https://claude.ai/code/session_016Yr6jELuRc7hyzYLccQKZd
This commit is contained in:
205
internal/lexicon/lexicon.go
Normal file
205
internal/lexicon/lexicon.go
Normal file
@@ -0,0 +1,205 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Meaning is one sense of a word: its part of speech, the gloss, and an optional
|
||||
// usage example. The frontend renders a few of these in the word popover.
|
||||
type Meaning struct {
|
||||
PartOfSpeech string `json:"part_of_speech"`
|
||||
Definition string `json:"definition"`
|
||||
Example string `json:"example,omitempty"`
|
||||
}
|
||||
|
||||
// Result is the full lookup for one word. Either list may be empty (the word
|
||||
// isn't a headword in that dataset); the frontend handles a partial or empty
|
||||
// result gracefully.
|
||||
type Result struct {
|
||||
Word string `json:"word"`
|
||||
Definitions []Meaning `json:"definitions"`
|
||||
Synonyms []string `json:"synonyms"`
|
||||
}
|
||||
|
||||
// maxSynonyms caps how many synonyms we hand the popover, even though the dataset
|
||||
// stores up to ~50 per word — a long flat wall of words overwhelms more than it
|
||||
// helps, especially for an ESL reader scanning for the right fit.
|
||||
const maxSynonyms = 16
|
||||
|
||||
// maxDefinitions caps the senses shown so the popover stays a glance, not an
|
||||
// essay.
|
||||
const maxDefinitions = 4
|
||||
|
||||
// Lexicon holds the lazily-loaded datasets. The maps are populated once, on the
|
||||
// first Lookup, behind a sync.Once so startup stays instant and a load error is
|
||||
// remembered rather than retried on every request.
|
||||
type Lexicon struct {
|
||||
once sync.Once
|
||||
loadErr error
|
||||
defs map[string][][]string // word → [[pos, def, example], …]
|
||||
synonyms map[string][]string // word → [synonym, …]
|
||||
}
|
||||
|
||||
// New returns a Lexicon. The datasets aren't read until the first Lookup.
|
||||
func New() *Lexicon { return &Lexicon{} }
|
||||
|
||||
func (l *Lexicon) load() {
|
||||
l.once.Do(func() {
|
||||
if err := gunzipJSON(definitionsGz, &l.defs); err != nil {
|
||||
l.loadErr = fmt.Errorf("load definitions: %w", err)
|
||||
return
|
||||
}
|
||||
if err := gunzipJSON(synonymsGz, &l.synonyms); err != nil {
|
||||
l.loadErr = fmt.Errorf("load synonyms: %w", err)
|
||||
return
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Lookup returns the definition senses and synonyms for word. It first tries the
|
||||
// word as written (lowercased), then a few simple morphological reductions
|
||||
// (plurals, -ed/-ing/-ly) so "running" or "happily" still resolve. A word found
|
||||
// in neither dataset yields a Result with empty lists (not an error).
|
||||
func (l *Lexicon) Lookup(word string) (Result, error) {
|
||||
l.load()
|
||||
if l.loadErr != nil {
|
||||
return Result{}, l.loadErr
|
||||
}
|
||||
|
||||
norm := strings.ToLower(strings.TrimSpace(word))
|
||||
res := Result{Word: word, Definitions: []Meaning{}, Synonyms: []string{}}
|
||||
if norm == "" {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
if raw := lookupDefs(l.defs, norm); raw != nil {
|
||||
for _, m := range raw {
|
||||
res.Definitions = append(res.Definitions, toMeaning(m))
|
||||
if len(res.Definitions) >= maxDefinitions {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if syns := lookupSyns(l.synonyms, norm); syns != nil {
|
||||
if len(syns) > maxSynonyms {
|
||||
syns = syns[:maxSynonyms]
|
||||
}
|
||||
res.Synonyms = append(res.Synonyms, syns...)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// lookupDefs / lookupSyns walk the candidate forms of a word and return the
|
||||
// first dataset hit. They're separate (rather than a generic helper) only
|
||||
// because the two maps have different value types.
|
||||
func lookupDefs(m map[string][][]string, word string) [][]string {
|
||||
for _, c := range candidates(word) {
|
||||
if v, ok := m[c]; ok {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func lookupSyns(m map[string][]string, word string) []string {
|
||||
for _, c := range candidates(word) {
|
||||
if v, ok := m[c]; ok {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// candidates returns the lemma forms to try, in priority order: the word itself,
|
||||
// then conservative de-inflections. This is deliberately lightweight — a full
|
||||
// stemmer would over-reduce ("business" → "busy") and surface wrong entries; a
|
||||
// handful of common English suffix rules covers the everyday cases without a
|
||||
// dependency. Duplicates are fine (map lookup is cheap); order is what matters.
|
||||
func candidates(word string) []string {
|
||||
out := []string{word}
|
||||
add := func(s string) {
|
||||
if len(s) >= 2 && s != word {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.HasSuffix(word, "ies"): // studies → study
|
||||
add(word[:len(word)-3] + "y")
|
||||
case strings.HasSuffix(word, "es"): // boxes → box, wishes → wish
|
||||
add(word[:len(word)-2])
|
||||
add(word[:len(word)-1])
|
||||
case strings.HasSuffix(word, "s"): // cats → cat
|
||||
add(word[:len(word)-1])
|
||||
}
|
||||
|
||||
if strings.HasSuffix(word, "ing") { // running → run, making → make
|
||||
stem := word[:len(word)-3]
|
||||
add(stem)
|
||||
add(stem + "e")
|
||||
add(undouble(stem))
|
||||
}
|
||||
if strings.HasSuffix(word, "ed") { // hoped → hope, stopped → stop
|
||||
stem := word[:len(word)-2]
|
||||
add(stem)
|
||||
add(word[:len(word)-1])
|
||||
add(undouble(stem))
|
||||
}
|
||||
if strings.HasSuffix(word, "ly") { // happily handled above via ies path? no — quickly → quick
|
||||
add(word[:len(word)-2])
|
||||
}
|
||||
if strings.HasSuffix(word, "ily") { // happily → happy
|
||||
add(word[:len(word)-3] + "y")
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// undouble collapses a doubled final consonant (stopp → stop, runn → run) so the
|
||||
// -ed/-ing stems of doubled-consonant verbs resolve to their base form.
|
||||
func undouble(stem string) string {
|
||||
n := len(stem)
|
||||
if n >= 2 && stem[n-1] == stem[n-2] {
|
||||
return stem[:n-1]
|
||||
}
|
||||
return stem
|
||||
}
|
||||
|
||||
// toMeaning maps a compact [pos, def, example] triple from the dataset onto the
|
||||
// JSON-friendly Meaning. The dataset always stores three elements, but we guard
|
||||
// the length so a malformed row can't panic.
|
||||
func toMeaning(m []string) Meaning {
|
||||
var out Meaning
|
||||
if len(m) > 0 {
|
||||
out.PartOfSpeech = m[0]
|
||||
}
|
||||
if len(m) > 1 {
|
||||
out.Definition = m[1]
|
||||
}
|
||||
if len(m) > 2 {
|
||||
out.Example = m[2]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// gunzipJSON decompresses gz and decodes the JSON into v.
|
||||
func gunzipJSON(gz []byte, v any) error {
|
||||
r, err := gzip.NewReader(bytes.NewReader(gz))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer r.Close()
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(data, v)
|
||||
}
|
||||
Reference in New Issue
Block a user