The garden learns which language a card is in, and read-aloud stops guessing
Phase 28 (c), the last of the phase. A word met inside a Portuguese document is a Portuguese card: migration 0018 mirrors documents.doc_lang onto vocab_words, set server-side from the ownership lookup capture was already making. Every card still reviews — filtering the queue to the half she is learning would drop the words she actually met. Read-aloud was the larger surprise. detectLang routed Han/kana to Chinese and everything else to en-US, so the zh pair was accidentally right and every Latin pair wrong. doc_lang now reaches the client read-only on the document JSON, and docLang(text, verdict) answers for a passage taken out of it — with the script test still winning, because quoted Chinese must never be spelled out one "Chinese letter" at a time. Claude-Session: https://claude.ai/code/session_01GJHNvirh7Hzhc9RL3HAvz7
This commit is contained in:
+14
-3
File diff suppressed because one or more lines are too long
@@ -613,6 +613,31 @@ ALTER TABLE users ADD COLUMN direction TEXT NOT NULL DEFAULT 'learning_en'
|
|||||||
stmt: `
|
stmt: `
|
||||||
ALTER TABLE documents ADD COLUMN doc_lang TEXT NOT NULL DEFAULT ''
|
ALTER TABLE documents ADD COLUMN doc_lang TEXT NOT NULL DEFAULT ''
|
||||||
CHECK(doc_lang IN ('', 'en', 'pair'));
|
CHECK(doc_lang IN ('', 'en', 'pair'));
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Which language a garden card is in — the same '' | 'en' | 'pair'
|
||||||
|
// vocabulary as documents.doc_lang, and set from it: a word is captured
|
||||||
|
// (or a phrase planted) out of a document, so the document's verdict is
|
||||||
|
// the card's language. A card with no document keeps '', which reads as
|
||||||
|
// English like every other empty here.
|
||||||
|
//
|
||||||
|
// The garden needed this the moment a document could be written in her
|
||||||
|
// own language. Before Phase 28 every card was English by construction;
|
||||||
|
// now a Portuguese lookup lands beside an English one with nothing to
|
||||||
|
// tell them apart, and two surfaces get it wrong without the tag — the
|
||||||
|
// review card's read-aloud (which would say a Portuguese word in a US
|
||||||
|
// English voice) and the panel, where a mixed garden is illegible.
|
||||||
|
//
|
||||||
|
// Every card is reviewed regardless. Filtering the queue to the half she
|
||||||
|
// is learning was the alternative and is wrong for the writer this is
|
||||||
|
// for: the words she met while writing Portuguese are still words she
|
||||||
|
// met, and a garden that quietly drops them is a garden that stops being
|
||||||
|
// a record of her reading.
|
||||||
|
name: "0018_vocab_lang",
|
||||||
|
stmt: `
|
||||||
|
ALTER TABLE vocab_words ADD COLUMN lang TEXT NOT NULL DEFAULT ''
|
||||||
|
CHECK(lang IN ('', 'en', 'pair'));
|
||||||
`,
|
`,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
+27
-22
@@ -34,15 +34,20 @@ type User struct {
|
|||||||
// (source of truth for the editor); `ContentText` is the flattened plain text
|
// (source of truth for the editor); `ContentText` is the flattened plain text
|
||||||
// kept in sync on every save and fed to the LLM.
|
// kept in sync on every save and fed to the LLM.
|
||||||
type Document struct {
|
type Document struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
UserID string `json:"user_id"`
|
UserID string `json:"user_id"`
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
Content string `json:"content"` // Tiptap JSON
|
Content string `json:"content"` // Tiptap JSON
|
||||||
ContentText string `json:"content_text"` // plain text for the LLM
|
ContentText string `json:"content_text"` // plain text for the LLM
|
||||||
Tone string `json:"tone"` // target writing tone; steers LLM advice
|
Tone string `json:"tone"` // target writing tone; steers LLM advice
|
||||||
WordCount int `json:"word_count"`
|
WordCount int `json:"word_count"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
// DocLang is which language this document is written in — '' | 'en' | 'pair'
|
||||||
UpdatedAt time.Time `json:"updated_at"`
|
// (migration 0017), written by the checkpoint pass and never by the client.
|
||||||
|
// It reaches the client read-only, for the one decision the client has to
|
||||||
|
// make on its own: which voice reads a selection aloud. '' means English.
|
||||||
|
DocLang string `json:"doc_lang"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
|
||||||
// PreserveHistory opts this document out of auto-snapshot pruning so its
|
// PreserveHistory opts this document out of auto-snapshot pruning so its
|
||||||
// full writing trail survives as authorship evidence (see the passport).
|
// full writing trail survives as authorship evidence (see the passport).
|
||||||
@@ -110,15 +115,15 @@ const (
|
|||||||
// coordinates at render time (spec Note #6). `Replacement` is empty for `voice`
|
// coordinates at render time (spec Note #6). `Replacement` is empty for `voice`
|
||||||
// flags — those are awareness-only, with no correction to apply.
|
// flags — those are awareness-only, with no correction to apply.
|
||||||
type Suggestion struct {
|
type Suggestion struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
DocID string `json:"doc_id"`
|
DocID string `json:"doc_id"`
|
||||||
FromPos int `json:"from_pos"`
|
FromPos int `json:"from_pos"`
|
||||||
ToPos int `json:"to_pos"`
|
ToPos int `json:"to_pos"`
|
||||||
Original string `json:"original"`
|
Original string `json:"original"`
|
||||||
Replacement string `json:"replacement"`
|
Replacement string `json:"replacement"`
|
||||||
Explanation string `json:"explanation"`
|
Explanation string `json:"explanation"`
|
||||||
Type string `json:"type"` // grammar | phrasing | idiom | clarity | translate | voice | collocation
|
Type string `json:"type"` // grammar | phrasing | idiom | clarity | translate | voice | collocation
|
||||||
Status string `json:"status"` // pending | accepted | rejected
|
Status string `json:"status"` // pending | accepted | rejected
|
||||||
// Source names the engine that proposed the edit, not its family: an offline
|
// Source names the engine that proposed the edit, not its family: an offline
|
||||||
// rule and the model can both propose a collocation, and the writer is never
|
// rule and the model can both propose a collocation, and the writer is never
|
||||||
// told which one spoke. It exists so each pass can replace its own rows.
|
// told which one spoke. It exists so each pass can replace its own rows.
|
||||||
@@ -128,10 +133,10 @@ type Suggestion struct {
|
|||||||
|
|
||||||
// Suggestion type and status values, mirrored from the schema CHECK constraints.
|
// Suggestion type and status values, mirrored from the schema CHECK constraints.
|
||||||
const (
|
const (
|
||||||
SuggestionTypeGrammar = "grammar"
|
SuggestionTypeGrammar = "grammar"
|
||||||
SuggestionTypePhrasing = "phrasing"
|
SuggestionTypePhrasing = "phrasing"
|
||||||
SuggestionTypeIdiom = "idiom"
|
SuggestionTypeIdiom = "idiom"
|
||||||
SuggestionTypeClarity = "clarity"
|
SuggestionTypeClarity = "clarity"
|
||||||
// A span she wrote in her own language, rendered into English. Not a
|
// A span she wrote in her own language, rendered into English. Not a
|
||||||
// correction — nothing was wrong with it — which is why it is its own type
|
// correction — nothing was wrong with it — which is why it is its own type
|
||||||
// rather than a clarity fix: the card is the pair model's flagship moment
|
// rather than a clarity fix: the card is the pair model's flagship moment
|
||||||
|
|||||||
@@ -225,13 +225,14 @@ func (h *Handler) fetch(userID, id string) (db.Document, error) {
|
|||||||
var doc db.Document
|
var doc db.Document
|
||||||
err := h.DB.QueryRow(
|
err := h.DB.QueryRow(
|
||||||
`SELECT id, user_id, title, content, content_text, tone, word_count,
|
`SELECT id, user_id, title, content, content_text, tone, word_count,
|
||||||
created_at, updated_at, preserve_history
|
created_at, updated_at, preserve_history, doc_lang
|
||||||
FROM documents
|
FROM documents
|
||||||
WHERE id = ? AND user_id = ?`,
|
WHERE id = ? AND user_id = ?`,
|
||||||
id, userID,
|
id, userID,
|
||||||
).Scan(
|
).Scan(
|
||||||
&doc.ID, &doc.UserID, &doc.Title, &doc.Content, &doc.ContentText,
|
&doc.ID, &doc.UserID, &doc.Title, &doc.Content, &doc.ContentText,
|
||||||
&doc.Tone, &doc.WordCount, &doc.CreatedAt, &doc.UpdatedAt, &doc.PreserveHistory,
|
&doc.Tone, &doc.WordCount, &doc.CreatedAt, &doc.UpdatedAt, &doc.PreserveHistory,
|
||||||
|
&doc.DocLang,
|
||||||
)
|
)
|
||||||
return doc, err
|
return doc, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -825,13 +825,13 @@ func (h *Handler) setStatus(w http.ResponseWriter, r *http.Request, status strin
|
|||||||
// hands over a reusable chunk, which is the only thing worth reviewing in a week.
|
// hands over a reusable chunk, which is the only thing worth reviewing in a week.
|
||||||
func (h *Handler) plant(id, userID string) {
|
func (h *Handler) plant(id, userID string) {
|
||||||
var s db.Suggestion
|
var s db.Suggestion
|
||||||
var contentText string
|
var contentText, docLang string
|
||||||
err := h.DB.QueryRow(
|
err := h.DB.QueryRow(
|
||||||
`SELECT s.type, s.original, s.replacement, s.explanation, s.doc_id, d.content_text
|
`SELECT s.type, s.original, s.replacement, s.explanation, s.doc_id, d.content_text, d.doc_lang
|
||||||
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||||
WHERE s.id = ? AND d.user_id = ?`,
|
WHERE s.id = ? AND d.user_id = ?`,
|
||||||
id, userID,
|
id, userID,
|
||||||
).Scan(&s.Type, &s.Original, &s.Replacement, &s.Explanation, &s.DocID, &contentText)
|
).Scan(&s.Type, &s.Original, &s.Replacement, &s.Explanation, &s.DocID, &contentText, &docLang)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, sql.ErrNoRows) {
|
if !errors.Is(err, sql.ErrNoRows) {
|
||||||
log.Printf("suggestions: could not read %s for planting: %v", id, err)
|
log.Printf("suggestions: could not read %s for planting: %v", id, err)
|
||||||
@@ -850,6 +850,9 @@ func (h *Handler) plant(id, userID string) {
|
|||||||
Meaning: s.Explanation,
|
Meaning: s.Explanation,
|
||||||
Example: correctedSentence(contentText, s.Original, s.Replacement),
|
Example: correctedSentence(contentText, s.Original, s.Replacement),
|
||||||
DocID: &docID,
|
DocID: &docID,
|
||||||
|
// The chunk is her own sentence, corrected — so it is in the document's
|
||||||
|
// language, whatever the collocation pass was framed in.
|
||||||
|
Lang: docLang,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
log.Printf("suggestions: could not plant %s: %v", id, err)
|
log.Printf("suggestions: could not plant %s: %v", id, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -184,3 +184,60 @@ func TestCorrectedSentence(t *testing.T) {
|
|||||||
t.Errorf("unterminated doc: got %q", got)
|
t.Errorf("unterminated doc: got %q", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestPlantedPhraseCarriesTheDocumentLanguage: a chunk planted out of a document
|
||||||
|
// written in her own language is a card in that language. The collocation pass
|
||||||
|
// itself deliberately did not flip in Phase 28 — its prompt is per-language
|
||||||
|
// knowledge, not framing — but the phrase it hands over is still lifted from her
|
||||||
|
// prose, so the card has to know what language that prose was in or the garden
|
||||||
|
// will read it aloud in the wrong voice.
|
||||||
|
func TestPlantedPhraseCarriesTheDocumentLanguage(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
if _, err := h.DB.Exec(`UPDATE documents SET doc_lang = 'pair' WHERE id = ?`, docID); err != nil {
|
||||||
|
t.Fatalf("set doc_lang: %v", err)
|
||||||
|
}
|
||||||
|
id := seedSuggestion(t, h, docID,
|
||||||
|
"Ontem foi difícil. Tive de tomar uma decisão sobre o trabalho.",
|
||||||
|
db.SuggestionTypeCollocation, "tomar uma decisão", "tomar uma decisão",
|
||||||
|
"Em português diz-se “tomar” uma decisão.")
|
||||||
|
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("accept: got %d, want 204", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
var lang string
|
||||||
|
if err := h.DB.QueryRow(
|
||||||
|
`SELECT lang FROM vocab_words WHERE user_id = ? AND word = ?`,
|
||||||
|
db.LocalUserID, "tomar uma decisão",
|
||||||
|
).Scan(&lang); err != nil {
|
||||||
|
t.Fatalf("read planted card: %v", err)
|
||||||
|
}
|
||||||
|
if lang != "pair" {
|
||||||
|
t.Fatalf("planted card lang = %q, want pair", lang)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPlantedPhraseOnAnEnglishDocumentIsUnchanged is the other half, and the one
|
||||||
|
// every account is on today: an English document plants an English card, and the
|
||||||
|
// empty backfill and 'en' both mean that.
|
||||||
|
func TestPlantedPhraseOnAnEnglishDocumentIsUnchanged(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
id := seedSuggestion(t, h, docID, "I had to do a decision about the job.",
|
||||||
|
db.SuggestionTypeCollocation, "do a decision", "make a decision",
|
||||||
|
"English pairs “make” with “decision”.")
|
||||||
|
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("accept: got %d, want 204", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
var lang string
|
||||||
|
if err := h.DB.QueryRow(
|
||||||
|
`SELECT lang FROM vocab_words WHERE user_id = ? AND word = ?`,
|
||||||
|
db.LocalUserID, "make a decision",
|
||||||
|
).Scan(&lang); err != nil {
|
||||||
|
t.Fatalf("read planted card: %v", err)
|
||||||
|
}
|
||||||
|
if lang == "pair" {
|
||||||
|
t.Fatalf("planted card lang = %q on an English document, want en or empty", lang)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+32
-16
@@ -19,13 +19,18 @@ import (
|
|||||||
// Word is one entry in the vocabulary garden: the looked-up word with its gloss,
|
// Word is one entry in the vocabulary garden: the looked-up word with its gloss,
|
||||||
// phonetic, and the sentence it was met in, plus its spaced-repetition state.
|
// phonetic, and the sentence it was met in, plus its spaced-repetition state.
|
||||||
type Word struct {
|
type Word struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
Word string `json:"word"`
|
Word string `json:"word"`
|
||||||
Gloss string `json:"gloss"`
|
Gloss string `json:"gloss"`
|
||||||
Definition string `json:"definition"` // English fallback meaning when there's no Chinese gloss
|
Definition string `json:"definition"` // English fallback meaning when there's no Chinese gloss
|
||||||
Phonetic string `json:"phonetic"`
|
Phonetic string `json:"phonetic"`
|
||||||
Example string `json:"example"`
|
Example string `json:"example"`
|
||||||
DocID *string `json:"doc_id"`
|
DocID *string `json:"doc_id"`
|
||||||
|
// Lang is '' | 'en' | 'pair' — the language of the document the word was met
|
||||||
|
// in (see migration 0018). '' reads as English, like everywhere else this
|
||||||
|
// vocabulary appears. The client needs it to pick a read-aloud voice: "comum"
|
||||||
|
// is unguessable from its letters, so the card has to carry the answer.
|
||||||
|
Lang string `json:"lang"`
|
||||||
DueAt time.Time `json:"due_at"`
|
DueAt time.Time `json:"due_at"`
|
||||||
IntervalDays int `json:"interval_days"`
|
IntervalDays int `json:"interval_days"`
|
||||||
Ease float64 `json:"ease"`
|
Ease float64 `json:"ease"`
|
||||||
@@ -54,7 +59,7 @@ func (h *Handler) Routes() chi.Router {
|
|||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
const vocabColumns = `id, word, gloss, definition, phonetic, example, doc_id,
|
const vocabColumns = `id, word, gloss, definition, phonetic, example, doc_id, lang,
|
||||||
due_at, interval_days, ease, reps, lapses, last_reviewed, created_at`
|
due_at, interval_days, ease, reps, lapses, last_reviewed, created_at`
|
||||||
|
|
||||||
func scanWord(s interface {
|
func scanWord(s interface {
|
||||||
@@ -62,7 +67,7 @@ func scanWord(s interface {
|
|||||||
}) (Word, error) {
|
}) (Word, error) {
|
||||||
var w Word
|
var w Word
|
||||||
err := s.Scan(
|
err := s.Scan(
|
||||||
&w.ID, &w.Word, &w.Gloss, &w.Definition, &w.Phonetic, &w.Example, &w.DocID,
|
&w.ID, &w.Word, &w.Gloss, &w.Definition, &w.Phonetic, &w.Example, &w.DocID, &w.Lang,
|
||||||
&w.DueAt, &w.IntervalDays, &w.Ease, &w.Reps, &w.Lapses, &w.LastReviewed, &w.CreatedAt,
|
&w.DueAt, &w.IntervalDays, &w.Ease, &w.Reps, &w.Lapses, &w.LastReviewed, &w.CreatedAt,
|
||||||
)
|
)
|
||||||
return w, err
|
return w, err
|
||||||
@@ -165,15 +170,22 @@ func (h *Handler) capture(w http.ResponseWriter, r *http.Request) {
|
|||||||
// would hit the foreign key and leak a raw "FOREIGN KEY constraint" 500
|
// would hit the foreign key and leak a raw "FOREIGN KEY constraint" 500
|
||||||
// instead of a clean 400 (and, once auth lands, would let a word be attached
|
// instead of a clean 400 (and, once auth lands, would let a word be attached
|
||||||
// to another user's document).
|
// to another user's document).
|
||||||
|
//
|
||||||
|
// The same row-scoped lookup answers what language the card is in: a word is
|
||||||
|
// met inside a document, so the document's verdict is the word's language.
|
||||||
|
// Asking the document rather than trusting a `lang` in the request body is
|
||||||
|
// the same choice `runPass` makes — the client never gets to name a language
|
||||||
|
// the server can already read. A word with no document is '', which reads as
|
||||||
|
// English.
|
||||||
|
lang := ""
|
||||||
if req.DocID != nil {
|
if req.DocID != nil {
|
||||||
if strings.TrimSpace(*req.DocID) == "" {
|
if strings.TrimSpace(*req.DocID) == "" {
|
||||||
req.DocID = nil
|
req.DocID = nil
|
||||||
} else {
|
} else {
|
||||||
var ok int
|
|
||||||
err := h.DB.QueryRow(
|
err := h.DB.QueryRow(
|
||||||
`SELECT 1 FROM documents WHERE id = ? AND user_id = ?`,
|
`SELECT doc_lang FROM documents WHERE id = ? AND user_id = ?`,
|
||||||
*req.DocID, userID,
|
*req.DocID, userID,
|
||||||
).Scan(&ok)
|
).Scan(&lang)
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
httputil.ErrorJSON(w, http.StatusBadRequest, "unknown doc_id")
|
httputil.ErrorJSON(w, http.StatusBadRequest, "unknown doc_id")
|
||||||
return
|
return
|
||||||
@@ -189,15 +201,19 @@ func (h *Handler) capture(w http.ResponseWriter, r *http.Request) {
|
|||||||
// schedule (due_at/reps/interval/ease) alone so re-looking-up a word never
|
// schedule (due_at/reps/interval/ease) alone so re-looking-up a word never
|
||||||
// resets its progress.
|
// resets its progress.
|
||||||
_, err := h.DB.Exec(
|
_, err := h.DB.Exec(
|
||||||
`INSERT INTO vocab_words (user_id, word, gloss, definition, phonetic, example, doc_id, due_at, interval_days)
|
`INSERT INTO vocab_words (user_id, word, gloss, definition, phonetic, example, doc_id, lang, due_at, interval_days)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now', '+1 day'), 1)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now', '+1 day'), 1)
|
||||||
ON CONFLICT(user_id, word) DO UPDATE SET
|
ON CONFLICT(user_id, word) DO UPDATE SET
|
||||||
gloss = excluded.gloss,
|
gloss = excluded.gloss,
|
||||||
definition = excluded.definition,
|
definition = excluded.definition,
|
||||||
phonetic = excluded.phonetic,
|
phonetic = excluded.phonetic,
|
||||||
example = CASE WHEN excluded.example != '' THEN excluded.example ELSE vocab_words.example END,
|
example = CASE WHEN excluded.example != '' THEN excluded.example ELSE vocab_words.example END,
|
||||||
doc_id = COALESCE(excluded.doc_id, vocab_words.doc_id)`,
|
doc_id = COALESCE(excluded.doc_id, vocab_words.doc_id),
|
||||||
userID, word, req.Gloss, req.Definition, req.Phonetic, req.Example, req.DocID,
|
-- lang travels with doc_id, and for the same reason: it is the new
|
||||||
|
-- context or it is nothing. A lookup made outside any document must
|
||||||
|
-- not relabel a card that was captured inside one.
|
||||||
|
lang = CASE WHEN excluded.doc_id IS NOT NULL THEN excluded.lang ELSE vocab_words.lang END`,
|
||||||
|
userID, word, req.Gloss, req.Definition, req.Phonetic, req.Example, req.DocID, lang,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
httputil.ServerError(w, err)
|
httputil.ServerError(w, err)
|
||||||
|
|||||||
@@ -253,3 +253,89 @@ func TestDocLinkSurvivesDocDelete(t *testing.T) {
|
|||||||
t.Fatalf("doc_id should be nulled after doc delete, got %v", *all[0].DocID)
|
t.Fatalf("doc_id should be nulled after doc delete, got %v", *all[0].DocID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCaptureTakesLanguageFromItsDocument is the garden's half of Phase 28: a
|
||||||
|
// word met inside a document written in her own language is a card in that
|
||||||
|
// language, and the server reads that off the document rather than being told.
|
||||||
|
//
|
||||||
|
// The three cases are the three the client can actually produce: a lookup inside
|
||||||
|
// a flipped document, a lookup inside an English one, and a lookup with no
|
||||||
|
// document at all (the search box) — the last of which is '', which reads as
|
||||||
|
// English everywhere this value is used.
|
||||||
|
func TestCaptureTakesLanguageFromItsDocument(t *testing.T) {
|
||||||
|
srv, database := newTestServer(t)
|
||||||
|
|
||||||
|
seed := func(lang string) string {
|
||||||
|
t.Helper()
|
||||||
|
var id string
|
||||||
|
if err := database.QueryRow(
|
||||||
|
`INSERT INTO documents (user_id, content_text, doc_lang) VALUES (?, 'hi', ?) RETURNING id`,
|
||||||
|
db.LocalUserID, lang,
|
||||||
|
).Scan(&id); err != nil {
|
||||||
|
t.Fatalf("seed doc: %v", err)
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
pairDoc, enDoc := seed("pair"), seed("en")
|
||||||
|
|
||||||
|
capture := func(word, body string) Word {
|
||||||
|
t.Helper()
|
||||||
|
rec := do(t, srv, http.MethodPost, "/vocab", body)
|
||||||
|
if rec.Code != http.StatusCreated {
|
||||||
|
t.Fatalf("capture %s: code=%d body=%s", word, rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
var w Word
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &w); err != nil {
|
||||||
|
t.Fatalf("decode %s: %v", word, err)
|
||||||
|
}
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := capture("comum", `{"word":"comum","doc_id":"`+pairDoc+`"}`); got.Lang != "pair" {
|
||||||
|
t.Fatalf("word from a flipped document: lang=%q, want pair", got.Lang)
|
||||||
|
}
|
||||||
|
if got := capture("reception", `{"word":"reception","doc_id":"`+enDoc+`"}`); got.Lang != "en" {
|
||||||
|
t.Fatalf("word from an English document: lang=%q, want en", got.Lang)
|
||||||
|
}
|
||||||
|
if got := capture("orphan", `{"word":"orphan"}`); got.Lang != "" {
|
||||||
|
t.Fatalf("word with no document: lang=%q, want empty", got.Lang)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A client that names a language is ignored: the document is the authority,
|
||||||
|
// the same way runPass never lets the request pick its own target.
|
||||||
|
if got := capture("comum", `{"word":"comum","lang":"en","doc_id":"`+pairDoc+`"}`); got.Lang != "pair" {
|
||||||
|
t.Fatalf("client-supplied lang should not win: lang=%q, want pair", got.Lang)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRecaptureOutsideADocumentKeepsItsLanguage pins the one asymmetry in the
|
||||||
|
// upsert. Re-looking-up a word refreshes its context, but a lookup made with no
|
||||||
|
// document carries no verdict — and relabelling a Portuguese card English
|
||||||
|
// because she checked the word again from the search box would silently move it
|
||||||
|
// to the wrong voice. lang travels with doc_id, or it doesn't travel.
|
||||||
|
func TestRecaptureOutsideADocumentKeepsItsLanguage(t *testing.T) {
|
||||||
|
srv, database := newTestServer(t)
|
||||||
|
var docID string
|
||||||
|
if err := database.QueryRow(
|
||||||
|
`INSERT INTO documents (user_id, content_text, doc_lang) VALUES (?, 'olá', 'pair') RETURNING id`,
|
||||||
|
db.LocalUserID,
|
||||||
|
).Scan(&docID); err != nil {
|
||||||
|
t.Fatalf("seed doc: %v", err)
|
||||||
|
}
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/vocab",
|
||||||
|
`{"word":"saudade","gloss":"","doc_id":"`+docID+`"}`); rec.Code != http.StatusCreated {
|
||||||
|
t.Fatalf("capture: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
rec := do(t, srv, http.MethodPost, "/vocab", `{"word":"saudade","gloss":"longing"}`)
|
||||||
|
if rec.Code != http.StatusCreated {
|
||||||
|
t.Fatalf("recapture: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
var w Word
|
||||||
|
_ = json.Unmarshal(rec.Body.Bytes(), &w)
|
||||||
|
if w.Lang != "pair" {
|
||||||
|
t.Fatalf("recapture outside a document: lang=%q, want pair held", w.Lang)
|
||||||
|
}
|
||||||
|
if w.Gloss != "longing" {
|
||||||
|
t.Fatalf("recapture should still refresh the gloss, got %q", w.Gloss)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -28,6 +28,10 @@ type Phrase struct {
|
|||||||
Meaning string // why it's better — the suggestion's explanation
|
Meaning string // why it's better — the suggestion's explanation
|
||||||
Example string // the sentence she met it in, already corrected
|
Example string // the sentence she met it in, already corrected
|
||||||
DocID *string // where, so "where did I see this?" stays one tap
|
DocID *string // where, so "where did I see this?" stays one tap
|
||||||
|
// Lang is the document's verdict ('' | 'en' | 'pair'), because the chunk is
|
||||||
|
// lifted out of her own prose and is therefore in whatever language that
|
||||||
|
// prose is. See migration 0018.
|
||||||
|
Lang string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Phrase-card caps. A collocation is a short chunk; anything longer is a
|
// Phrase-card caps. A collocation is a short chunk; anything longer is a
|
||||||
@@ -84,13 +88,14 @@ func Plant(ex Execer, userID string, p Phrase) (bool, error) {
|
|||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
res, err := ex.Exec(
|
res, err := ex.Exec(
|
||||||
`INSERT INTO vocab_words (user_id, word, gloss, definition, phonetic, example, doc_id, due_at, interval_days)
|
`INSERT INTO vocab_words (user_id, word, gloss, definition, phonetic, example, doc_id, lang, due_at, interval_days)
|
||||||
VALUES (?, ?, '', ?, '', ?, ?, datetime('now', '+1 day'), 1)
|
VALUES (?, ?, '', ?, '', ?, ?, ?, datetime('now', '+1 day'), 1)
|
||||||
ON CONFLICT(user_id, word) DO NOTHING`,
|
ON CONFLICT(user_id, word) DO NOTHING`,
|
||||||
userID, key,
|
userID, key,
|
||||||
clamp(strings.TrimSpace(p.Meaning), maxDefinitionLen),
|
clamp(strings.TrimSpace(p.Meaning), maxDefinitionLen),
|
||||||
clamp(strings.TrimSpace(p.Example), maxExampleLen),
|
clamp(strings.TrimSpace(p.Example), maxExampleLen),
|
||||||
p.DocID,
|
p.DocID,
|
||||||
|
p.Lang,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
|
|||||||
@@ -615,6 +615,7 @@ export default function App() {
|
|||||||
<EditorCore
|
<EditorCore
|
||||||
key={`${currentDoc.id}:${editorEpoch}`}
|
key={`${currentDoc.id}:${editorEpoch}`}
|
||||||
docId={currentDoc.id}
|
docId={currentDoc.id}
|
||||||
|
docLang={currentDoc.doc_lang}
|
||||||
initialContent={currentDoc.content}
|
initialContent={currentDoc.content}
|
||||||
onChange={handleEditorChange}
|
onChange={handleEditorChange}
|
||||||
segmenter={segmenter}
|
segmenter={segmenter}
|
||||||
|
|||||||
@@ -44,8 +44,18 @@ export interface Document {
|
|||||||
// When true, this document's automatic snapshots are never pruned, so its
|
// When true, this document's automatic snapshots are never pruned, so its
|
||||||
// full writing trail survives as authorship evidence (see the passport).
|
// full writing trail survives as authorship evidence (see the passport).
|
||||||
preserve_history: boolean
|
preserve_history: boolean
|
||||||
|
// Which language this document is written in, as decided server-side by the
|
||||||
|
// checkpoint pass: '' | 'en' | 'pair' ('' reads as English). Read-only — the
|
||||||
|
// editor never sends it. It is here so read-aloud can use the right voice on a
|
||||||
|
// document written in her own language.
|
||||||
|
doc_lang: DocLang
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The document-language verdict, shared by documents and garden cards. 'pair'
|
||||||
|
// names the writer's own language rather than a language code, so changing her
|
||||||
|
// pair re-reads her documents instead of stranding a stale name on them.
|
||||||
|
export type DocLang = '' | 'en' | 'pair'
|
||||||
|
|
||||||
// Fields the editor sends on auto-save. All optional so a rename can send title
|
// Fields the editor sends on auto-save. All optional so a rename can send title
|
||||||
// alone; the editor sends the full set.
|
// alone; the editor sends the full set.
|
||||||
export interface DocUpdate {
|
export interface DocUpdate {
|
||||||
@@ -125,6 +135,9 @@ export interface VocabWord {
|
|||||||
phonetic: string
|
phonetic: string
|
||||||
example: string
|
example: string
|
||||||
doc_id: string | null
|
doc_id: string | null
|
||||||
|
// The language of the document this word was met in — the card's own language
|
||||||
|
// (migration 0018). Read-aloud needs it: "comum" is unguessable from letters.
|
||||||
|
lang: DocLang
|
||||||
due_at: string
|
due_at: string
|
||||||
interval_days: number
|
interval_days: number
|
||||||
ease: number
|
ease: number
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||||
import { nativeLang, speak, stopSpeech } from './speech'
|
import { docLang, nativeLang, speak, stopSpeech } from './speech'
|
||||||
import { resetPackForTests, setPackLang } from '../i18n'
|
import { resetPackForTests, setPackLang } from '../i18n'
|
||||||
|
|
||||||
// Read-aloud has two jobs beyond "make a sound": ask for the right pace, and ask
|
// Read-aloud has two jobs beyond "make a sound": ask for the right pace, and ask
|
||||||
@@ -85,3 +85,33 @@ describe('nativeLang', () => {
|
|||||||
expect(bodies.at(-1)).toMatchObject({ text: 'chat', lang: 'fr-FR' })
|
expect(bodies.at(-1)).toMatchObject({ text: 'chat', lang: 'fr-FR' })
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('docLang', () => {
|
||||||
|
// The document's verdict is the answer for a passage lifted out of it, because
|
||||||
|
// for a Latin pair there is no other answer available: an English sentence and
|
||||||
|
// a Portuguese one are the same letters.
|
||||||
|
it('reads a flipped document in her own language', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(docLang('Ontem foi difícil.', 'pair')).toBe('pt-PT')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reads an English document in English, and treats the backfill as English', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(docLang('Yesterday was hard.', 'en')).toBe('en-US')
|
||||||
|
expect(docLang('Yesterday was hard.', '')).toBe('en-US')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('lets the script win over the verdict, so quoted Chinese is never spelled out', () => {
|
||||||
|
// An English document quoting Chinese is 'en' by verdict, and the English
|
||||||
|
// voice reads Han characters one "Chinese letter" at a time — the one
|
||||||
|
// failure worse than silence.
|
||||||
|
setPackLang('zh')
|
||||||
|
expect(docLang('你好世界', 'en')).toBe('zh-CN')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('is what the editor selection and the garden card ask for', () => {
|
||||||
|
setPackLang('fr')
|
||||||
|
speak('Le chat dort.', docLang('Le chat dort.', 'pair'))
|
||||||
|
expect(bodies.at(-1)).toMatchObject({ text: 'Le chat dort.', lang: 'fr-FR' })
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|||||||
@@ -90,6 +90,24 @@ export function nativeLang(): string {
|
|||||||
return pack().locale
|
return pack().locale
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// docLang turns a document-language verdict ('' | 'en' | 'pair', decided
|
||||||
|
// server-side — see internal/suggestions/doclang.go) into a locale for a passage
|
||||||
|
// taken out of that document. It is what the editor's read-aloud and the garden's
|
||||||
|
// review card ask instead of guessing.
|
||||||
|
//
|
||||||
|
// The script test still wins, and that is not redundant with the verdict. A
|
||||||
|
// Chinese sentence quoted inside an English document is 'en' by verdict and
|
||||||
|
// still has to be read by the Chinese voice: the English voice spells Han
|
||||||
|
// characters out one "Chinese letter" at a time, which is the one failure loud
|
||||||
|
// enough to be worse than no audio. In the other direction there is nothing to
|
||||||
|
// test — an English sentence inside Portuguese prose looks exactly like the
|
||||||
|
// Portuguese around it — so the document's verdict is the only answer available,
|
||||||
|
// and it is the answer this phase decided on.
|
||||||
|
export function docLang(text: string, verdict: string): string {
|
||||||
|
if (CJK.test(text)) return 'zh-CN'
|
||||||
|
return verdict === 'pair' ? pack().locale : 'en-US'
|
||||||
|
}
|
||||||
|
|
||||||
// speak reads `text` aloud, cancelling anything already in flight so rapid taps
|
// speak reads `text` aloud, cancelling anything already in flight so rapid taps
|
||||||
// don't queue up. `lang` defaults to a guess from the text (Chinese vs English)
|
// don't queue up. `lang` defaults to a guess from the text (Chinese vs English)
|
||||||
// so callers can just pass the selection; pass an explicit locale to override.
|
// so callers can just pass the selection; pass an explicit locale to override.
|
||||||
|
|||||||
@@ -33,8 +33,8 @@ import { Composition } from './Composition'
|
|||||||
import { RewritePreview, type RewriteStatus } from './RewritePreview'
|
import { RewritePreview, type RewriteStatus } from './RewritePreview'
|
||||||
import { planBatch } from './acceptBatch'
|
import { planBatch } from './acceptBatch'
|
||||||
import { entryId, idAfterRemoval, stepId, type Direction, type Span } from './triage'
|
import { entryId, idAfterRemoval, stepId, type Direction, type Span } from './triage'
|
||||||
import { api, type Suggestion, type SuggestionType, type WordInfo } from '../../api/client'
|
import { api, type DocLang, type Suggestion, type SuggestionType, type WordInfo } from '../../api/client'
|
||||||
import { speak, speechSupported } from '../../audio/speech'
|
import { docLang as docLocale, speak, speechSupported } from '../../audio/speech'
|
||||||
import type { SpellChecker } from '../../hooks/useSpellChecker'
|
import type { SpellChecker } from '../../hooks/useSpellChecker'
|
||||||
import { fromIME } from '../../lib/ime'
|
import { fromIME } from '../../lib/ime'
|
||||||
import type { Segmenter } from '../../lib/segment'
|
import type { Segmenter } from '../../lib/segment'
|
||||||
@@ -65,6 +65,13 @@ export interface EditorChange {
|
|||||||
interface Props {
|
interface Props {
|
||||||
// Changing docId reloads the editor with that document's content.
|
// Changing docId reloads the editor with that document's content.
|
||||||
docId: string
|
docId: string
|
||||||
|
// Which language this document is written in, as the server decided it
|
||||||
|
// ('' | 'en' | 'pair'). Read-aloud is the only thing here that reads it: a
|
||||||
|
// Portuguese selection has to be read by the Portuguese voice, and nothing in
|
||||||
|
// the letters says so. Updated by the server on save, so it trails a language
|
||||||
|
// flip by one auto-save — a passage read in the old voice once is the whole
|
||||||
|
// cost of not blocking the editor on a check.
|
||||||
|
docLang: DocLang
|
||||||
initialContent: string
|
initialContent: string
|
||||||
onChange: (change: EditorChange) => void
|
onChange: (change: EditorChange) => void
|
||||||
// LLM suggestions to highlight; accept/dismiss notify the parent for the API
|
// LLM suggestions to highlight; accept/dismiss notify the parent for the API
|
||||||
@@ -255,6 +262,7 @@ interface RewriteState {
|
|||||||
// decoration layer. Hovering a highlight opens its SuggestionCard.
|
// decoration layer. Hovering a highlight opens its SuggestionCard.
|
||||||
export function EditorCore({
|
export function EditorCore({
|
||||||
docId,
|
docId,
|
||||||
|
docLang,
|
||||||
initialContent,
|
initialContent,
|
||||||
onChange,
|
onChange,
|
||||||
suggestions,
|
suggestions,
|
||||||
@@ -1532,8 +1540,10 @@ export function EditorCore({
|
|||||||
<SelectionBubble
|
<SelectionBubble
|
||||||
style={{ top: selection.top, left: selection.left, transform: 'translateY(calc(-100% - 8px))' }}
|
style={{ top: selection.top, left: selection.left, transform: 'translateY(calc(-100% - 8px))' }}
|
||||||
onRewrite={handleRewrite}
|
onRewrite={handleRewrite}
|
||||||
onSpeak={speechSupported() ? () => speak(selection.text) : null}
|
onSpeak={speechSupported() ? () => speak(selection.text, docLocale(selection.text, docLang)) : null}
|
||||||
onSpeakSlow={speechSupported() ? () => speak(selection.text, undefined, true) : null}
|
onSpeakSlow={
|
||||||
|
speechSupported() ? () => speak(selection.text, docLocale(selection.text, docLang), true) : null
|
||||||
|
}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
{rewrite && (
|
{rewrite && (
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { useCallback, useEffect, useMemo, useState } from 'react'
|
import { useCallback, useEffect, useMemo, useState } from 'react'
|
||||||
import { api, type VocabGrade, type VocabWord } from '../../api/client'
|
import { api, type VocabGrade, type VocabWord } from '../../api/client'
|
||||||
import { speak, speechSupported, stopSpeech } from '../../audio/speech'
|
import { docLang as docLocale, speak, speechSupported, stopSpeech } from '../../audio/speech'
|
||||||
import { useFocusTrap } from '../../hooks/useFocusTrap'
|
import { useFocusTrap } from '../../hooks/useFocusTrap'
|
||||||
import { usePack, type Line } from '../../i18n'
|
import { usePack, type Line } from '../../i18n'
|
||||||
import { JournalView } from './JournalView'
|
import { JournalView } from './JournalView'
|
||||||
@@ -314,7 +314,21 @@ function GardenView({
|
|||||||
{blossom(w.reps)}
|
{blossom(w.reps)}
|
||||||
</span>
|
</span>
|
||||||
<span className="flex min-w-0 flex-1 flex-col">
|
<span className="flex min-w-0 flex-1 flex-col">
|
||||||
<span className="truncate text-sm font-bold text-plum">{w.word}</span>
|
<span className="flex min-w-0 items-center gap-1.5">
|
||||||
|
<span className="truncate text-sm font-bold text-plum">{w.word}</span>
|
||||||
|
{/* Only a card in her own language is marked. An English
|
||||||
|
garden with a badge on every blossom would be a
|
||||||
|
garden with no badges at all; the marker exists so
|
||||||
|
the rare Portuguese word is legible among them. */}
|
||||||
|
{w.lang === 'pair' && (
|
||||||
|
<span
|
||||||
|
className="shrink-0 rounded-full px-1.5 py-0.5 text-[9px] font-bold lowercase"
|
||||||
|
style={{ background: 'var(--color-surface-alt)', color: 'var(--color-muted)' }}
|
||||||
|
>
|
||||||
|
{t.nativeName}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
{(w.gloss || w.definition) && (
|
{(w.gloss || w.definition) && (
|
||||||
<span
|
<span
|
||||||
className="truncate text-xs"
|
className="truncate text-xs"
|
||||||
@@ -354,7 +368,7 @@ function GardenView({
|
|||||||
{speechSupported() && (
|
{speechSupported() && (
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => speak(w.word)}
|
onClick={() => speak(w.word, docLocale(w.word, w.lang))}
|
||||||
className="rounded-full px-2.5 py-1 text-xs font-semibold"
|
className="rounded-full px-2.5 py-1 text-xs font-semibold"
|
||||||
style={{ background: 'var(--color-surface-alt)' }}
|
style={{ background: 'var(--color-surface-alt)' }}
|
||||||
>
|
>
|
||||||
@@ -477,7 +491,7 @@ function ReviewSession({
|
|||||||
<>
|
<>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => speak(card.word)}
|
onClick={() => speak(card.word, docLocale(card.word, card.lang))}
|
||||||
aria-label={`Pronounce ${card.word}`}
|
aria-label={`Pronounce ${card.word}`}
|
||||||
className="flex h-6 w-6 items-center justify-center rounded-full text-xs"
|
className="flex h-6 w-6 items-center justify-center rounded-full text-xs"
|
||||||
style={{ background: 'var(--color-surface)' }}
|
style={{ background: 'var(--color-surface)' }}
|
||||||
@@ -488,7 +502,7 @@ function ReviewSession({
|
|||||||
worth hearing stretched out. */}
|
worth hearing stretched out. */}
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => speak(card.word, undefined, true)}
|
onClick={() => speak(card.word, docLocale(card.word, card.lang), true)}
|
||||||
aria-label={`Pronounce ${card.word} slowly`}
|
aria-label={`Pronounce ${card.word} slowly`}
|
||||||
title={t.garden.readSlowly}
|
title={t.garden.readSlowly}
|
||||||
className="flex h-6 w-6 items-center justify-center rounded-full text-xs"
|
className="flex h-6 w-6 items-center justify-center rounded-full text-xs"
|
||||||
|
|||||||
Reference in New Issue
Block a user