The zh pair's other direction, and a rule pack that mostly says no
`pair_lang` had always been answering a second question nobody asked: it says which two languages, and every surface built on it assumed English was the one being learned. That is why hanzi is never tokenized, never spell-checked, never glossed — correct for a Mandarin native practising English, backwards for an English native practising Mandarin. `users.direction` (migration 0016) separates the two questions; a `zh-learner` pair code would have been cheaper and would have made two directions of one pair look like two unrelated languages to every query. Segmentation is what replaces `wordAt` where there are no spaces: a shortest-path walk over log-probabilities, 232 ms and 14 MB for 188,522 words. The browser gets the word list because segmentation runs on hover; the server keeps the whole dictionary. Their coverage gates come out opposite on purpose — the client list is frequency-gated because the segmentation is measurably identical without the tail, and the dictionary is gated by nothing, because its only power is to explain and the word a learner stops on is the rare one. The 错别字 pack is 24 confusable pairs behind two mechanical gates. One admits a pair only if the wrong form is not a dictionary word and the right form is, which is why it refuses 自已 for 自己 — a real error whose wrong form is a headword. The other asks the segmenter whether the two characters already belong to two different words, without which 自己经常, 睡觉的时候 and 不知到底 would all be corrupted silently into text still made of real characters. Not deployed (this carries a migration), not seen in a browser, and no account has ever been in the learner direction. The IME composition guards were in scope and are not done — see BUILD_PLAN Phase 26.
This commit is contained in:
prosolis
@@ -24,7 +24,7 @@ func patchMe(t *testing.T, users *UserStore, id, body string) *httptest.Response
|
||||
func TestSetPairLang(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPairLang("bob", "pt-PT"); err != nil {
|
||||
if err := users.SetPair("bob", "pt-PT", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set pt-PT: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "pt-PT" {
|
||||
@@ -34,14 +34,14 @@ func TestSetPairLang(t *testing.T) {
|
||||
// Every pair with a langpack, not just the first one: this list and the
|
||||
// frontend's PACKS are two copies of the same fact, and the day they
|
||||
// disagree is the day she can pick a pair the app cannot render.
|
||||
if err := users.SetPairLang("bob", "fr"); err != nil {
|
||||
if err := users.SetPair("bob", "fr", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set fr: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "fr" {
|
||||
t.Fatalf("pair_lang = %q, want fr", u.PairLang)
|
||||
}
|
||||
|
||||
if err := users.SetPairLang("bob", "es"); err != nil {
|
||||
if err := users.SetPair("bob", "es", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set es: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "es" {
|
||||
@@ -50,7 +50,7 @@ func TestSetPairLang(t *testing.T) {
|
||||
|
||||
// And back — a writer who tries a pair and doesn't like it must be able to
|
||||
// return, which is the whole reason the picker exists.
|
||||
if err := users.SetPairLang("bob", "zh"); err != nil {
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set zh: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "zh" {
|
||||
@@ -70,7 +70,7 @@ func TestSetPairLangRejectsUnshippedPairs(t *testing.T) {
|
||||
// regional code Petal has not decided about is refused rather than rounded
|
||||
// to the nearest pack it happens to have.
|
||||
for _, lang := range []string{"es-ES", "pt-BR", "fr-CA", "de", "klingon", "", " "} {
|
||||
if err := users.SetPairLang("bob", lang); err == nil {
|
||||
if err := users.SetPair("bob", lang, DirectionLearningEn); err == nil {
|
||||
t.Fatalf("stored unshipped pair %q", lang)
|
||||
}
|
||||
}
|
||||
@@ -81,7 +81,7 @@ func TestSetPairLangRejectsUnshippedPairs(t *testing.T) {
|
||||
|
||||
func TestSetPairLangUnknownUser(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
if err := users.SetPairLang("nobody", "pt-PT"); err == nil {
|
||||
if err := users.SetPair("nobody", "pt-PT", DirectionLearningEn); err == nil {
|
||||
t.Fatal("set a pair language on an account that does not exist")
|
||||
}
|
||||
}
|
||||
@@ -108,9 +108,9 @@ func TestUpdateMeHandlerRejects(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
for name, body := range map[string]string{
|
||||
"unshipped pair": `{"pair_lang":"es-ES"}`,
|
||||
"missing field": `{}`,
|
||||
"not json": `pt-PT`,
|
||||
"unshipped pair": `{"pair_lang":"es-ES"}`,
|
||||
"unknown direction": `{"direction":"learning_klingon"}`,
|
||||
"not json": `pt-PT`,
|
||||
} {
|
||||
if w := patchMe(t, users, "bob", body); w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: status = %d, want 400", name, w.Code)
|
||||
@@ -126,3 +126,116 @@ func TestUpdateMeHandlerRejects(t *testing.T) {
|
||||
t.Fatalf("unknown user: status = %d, want 401", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// An empty body used to be a 400, back when pair_lang was the only field and a
|
||||
// request that named none of it could only be a client bug. With two optional
|
||||
// fields it is an ordinary PATCH that changes nothing, and it has to be: the
|
||||
// picker sends one field without knowing the other, and "omitted" has to mean
|
||||
// "leave it alone" for that to be safe.
|
||||
func TestUpdateMeHandlerEmptyBodyChangesNothing(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningPair); err != nil {
|
||||
t.Fatalf("set up: %v", err)
|
||||
}
|
||||
w := patchMe(t, users, "bob", `{}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
u, _ := users.Get("bob")
|
||||
if u.PairLang != "zh" || u.Direction != DirectionLearningPair {
|
||||
t.Fatalf("empty PATCH moved the account to %q/%q", u.PairLang, u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The direction axis: an account can be turned around and turned back, and the
|
||||
// default every existing row already carries is the one it had before the column
|
||||
// existed.
|
||||
func TestDirectionRoundTrip(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("a fresh account starts at %q, want %q", u.Direction, DirectionLearningEn)
|
||||
}
|
||||
|
||||
w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("turn around: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
var got db.User
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
// The response carries the direction, not just the pair — the client reads
|
||||
// its whole state back from here rather than assuming the write took.
|
||||
if got.Direction != DirectionLearningPair || got.PairLang != "zh" {
|
||||
t.Fatalf("response = %+v, want bob learning zh", got)
|
||||
}
|
||||
|
||||
if w := patchMe(t, users, "bob", `{"direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||
t.Fatalf("turn back: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("direction = %q after turning back", u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The refusal this axis exists to make: a pair with no word list cannot be
|
||||
// learned toward, however good its langpack is. fr, es and pt-PT all have copy,
|
||||
// voices and spelling dictionaries — and nothing that could segment a sentence
|
||||
// or read from that language into English, which is what a learner needs.
|
||||
func TestLearnerDirectionRefusedForPairsWithoutData(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
for _, lang := range []string{"pt-PT", "fr", "es"} {
|
||||
if err := users.SetPair("bob", lang, DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set %s: %v", lang, err)
|
||||
}
|
||||
w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: status = %d, want 400", lang, w.Code)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("%s: a refused write still moved direction to %q", lang, u.Direction)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The two-field combination the handler validates as one decision. An account
|
||||
// already learning Chinese that asks only to change pair is asking for a state
|
||||
// neither field names on its own — French with segmentation — and it must not
|
||||
// arrive by leaving one field out.
|
||||
func TestPairChangeCannotStrandTheLearnerDirection(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningPair); err != nil {
|
||||
t.Fatalf("set up: %v", err)
|
||||
}
|
||||
if w := patchMe(t, users, "bob", `{"pair_lang":"fr"}`); w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", w.Code)
|
||||
}
|
||||
u, _ := users.Get("bob")
|
||||
if u.PairLang != "zh" || u.Direction != DirectionLearningPair {
|
||||
t.Fatalf("refused write left the account at %q/%q", u.PairLang, u.Direction)
|
||||
}
|
||||
|
||||
// Naming both at once is how that move is actually made, and it works.
|
||||
if w := patchMe(t, users, "bob", `{"pair_lang":"fr","direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||
t.Fatalf("both fields: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "fr" || u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("account = %q/%q, want fr/learning_en", u.PairLang, u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The CHECK constraint is the last line, below the handler and below SetPair:
|
||||
// a direction that reaches the column by any other route is still refused.
|
||||
func TestDirectionCheckConstraint(t *testing.T) {
|
||||
_, users, database := newStores(t)
|
||||
if _, err := database.Exec(`UPDATE users SET direction = 'sideways' WHERE id = 'bob'`); err == nil {
|
||||
t.Fatal("the users.direction CHECK accepted 'sideways'")
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("direction = %q after a refused UPDATE", u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
+97
-11
@@ -49,9 +49,9 @@ func (u *UserStore) Upsert(sub, email, displayName string) error {
|
||||
func (u *UserStore) Get(id string) (db.User, error) {
|
||||
var user db.User
|
||||
err := u.db.QueryRow(
|
||||
`SELECT id, email, COALESCE(display_name, ''), created_at, pair_lang
|
||||
`SELECT id, email, COALESCE(display_name, ''), created_at, pair_lang, direction
|
||||
FROM users WHERE id = ?`, id,
|
||||
).Scan(&user.ID, &user.Email, &user.DisplayName, &user.CreatedAt, &user.PairLang)
|
||||
).Scan(&user.ID, &user.Email, &user.DisplayName, &user.CreatedAt, &user.PairLang, &user.Direction)
|
||||
return user, err
|
||||
}
|
||||
|
||||
@@ -93,12 +93,63 @@ func pairIsShipped(lang string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// SetPairLang moves an account to another (English + X) pair.
|
||||
func (u *UserStore) SetPairLang(id, lang string) error {
|
||||
// The two directions a pair can be travelled in. `DirectionLearningEn` is the
|
||||
// original assumption made explicit: the writer is native in X and practising
|
||||
// English. `DirectionLearningPair` is the other way round.
|
||||
const (
|
||||
DirectionLearningEn = "learning_en"
|
||||
DirectionLearningPair = "learning_pair"
|
||||
)
|
||||
|
||||
// The pairs whose *learner* direction Petal can actually serve, which is a
|
||||
// narrower thing than a shipped pair and narrower again than a langpack.
|
||||
//
|
||||
// Turning a pair around needs data no langpack carries: a word list to segment
|
||||
// with, and a dictionary that reads from the pair language into English. Chinese
|
||||
// has both as of Phase 26 (CC-CEDICT + jieba); French, Spanish and Portuguese
|
||||
// have neither yet, and — unlike a missing pack, which leaves a writer looking
|
||||
// at copy she cannot read — a missing word list would leave her looking at an
|
||||
// editor that silently does nothing when she hovers. Both are bad; only one is
|
||||
// legible as a bug. So the server refuses, for the same reason and by the same
|
||||
// mechanism as `shippedPairs`.
|
||||
//
|
||||
// This list is expected to grow one pair at a time and never to be inferred:
|
||||
// segmentation is a property of a writing system, and there is no rule that
|
||||
// derives "has a word list" from a language code.
|
||||
var learnerPairs = []string{"zh"}
|
||||
|
||||
// SupportsLearnerDirection reports whether a pair can be turned around.
|
||||
func SupportsLearnerDirection(lang string) bool {
|
||||
for _, p := range learnerPairs {
|
||||
if p == lang {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func directionIsKnown(d string) bool {
|
||||
return d == DirectionLearningEn || d == DirectionLearningPair
|
||||
}
|
||||
|
||||
// SetPair moves an account to another (English + X) pair, in a given direction.
|
||||
//
|
||||
// The two are written together because they constrain each other: a direction is
|
||||
// only meaningful for a pair that can be travelled in it, and validating them a
|
||||
// field at a time would let a two-step change pass through a state that neither
|
||||
// step is allowed to leave behind.
|
||||
func (u *UserStore) SetPair(id, lang, direction string) error {
|
||||
if !pairIsShipped(lang) {
|
||||
return errors.New("auth: unshipped pair language " + lang)
|
||||
}
|
||||
res, err := u.db.Exec(`UPDATE users SET pair_lang = ? WHERE id = ?`, lang, id)
|
||||
if !directionIsKnown(direction) {
|
||||
return errors.New("auth: unknown direction " + direction)
|
||||
}
|
||||
if direction == DirectionLearningPair && !SupportsLearnerDirection(lang) {
|
||||
return errors.New("auth: no learner direction for " + lang)
|
||||
}
|
||||
res, err := u.db.Exec(
|
||||
`UPDATE users SET pair_lang = ?, direction = ? WHERE id = ?`, lang, direction, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -108,8 +159,8 @@ func (u *UserStore) SetPairLang(id, lang string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateMeHandler changes the caller's own settings — today, the one setting
|
||||
// there is: which language Petal speaks alongside her English.
|
||||
// UpdateMeHandler changes the caller's own settings: which language Petal
|
||||
// speaks alongside her English, and which of the two she is learning.
|
||||
//
|
||||
// It answers with the whole updated user rather than an empty 204 so the client
|
||||
// has one shape to trust: /api/me and this return the same thing, and the app
|
||||
@@ -119,24 +170,59 @@ func (u *UserStore) SetPairLang(id, lang string) error {
|
||||
// dictionary, her read-aloud voice, which word-lookup provider answers, and the
|
||||
// language the prompts ask the model to explain in. All of those read
|
||||
// `users.pair_lang` at use time, so all of them follow from this one write.
|
||||
//
|
||||
// Both fields are optional and each defaults to what the account already has, so
|
||||
// the picker can send one without knowing the other. That matters for the
|
||||
// combination this endpoint exists to prevent: a client that sent only
|
||||
// `pair_lang: "fr"` while the account sat on `learning_pair` would otherwise ask
|
||||
// for French-with-segmentation, which does not exist. Here it is one decision
|
||||
// with one validation, and the answer carries whatever actually landed.
|
||||
func (u *UserStore) UpdateMeHandler() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
PairLang string `json:"pair_lang"`
|
||||
PairLang *string `json:"pair_lang"`
|
||||
Direction *string `json:"direction"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
httputil.BadRequest(w, "invalid request body")
|
||||
return
|
||||
}
|
||||
lang := strings.TrimSpace(body.PairLang)
|
||||
id := UserID(r.Context())
|
||||
current, err := u.Get(id)
|
||||
if err != nil {
|
||||
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||
return
|
||||
}
|
||||
|
||||
lang, direction := current.PairLang, current.Direction
|
||||
if body.PairLang != nil {
|
||||
lang = strings.TrimSpace(*body.PairLang)
|
||||
}
|
||||
if body.Direction != nil {
|
||||
direction = strings.TrimSpace(*body.Direction)
|
||||
}
|
||||
|
||||
if !pairIsShipped(lang) {
|
||||
// Name the ones that work. A writer who lands here has picked from a
|
||||
// stale client, and "not a language" tells her nothing.
|
||||
httputil.BadRequest(w, "unsupported language pair — Petal speaks "+strings.Join(shippedPairs, ", "))
|
||||
return
|
||||
}
|
||||
id := UserID(r.Context())
|
||||
if err := u.SetPairLang(id, lang); err != nil {
|
||||
if !directionIsKnown(direction) {
|
||||
httputil.BadRequest(w, "unknown direction — expected "+DirectionLearningEn+" or "+DirectionLearningPair)
|
||||
return
|
||||
}
|
||||
if direction == DirectionLearningPair && !SupportsLearnerDirection(lang) {
|
||||
// Refused rather than quietly downgraded to learning_en. A silent
|
||||
// downgrade would leave the writer looking at an editor that behaves
|
||||
// like the one she just tried to leave, with nothing to read as an
|
||||
// explanation — and the caller cannot tell the two outcomes apart
|
||||
// without diffing the response it was given.
|
||||
httputil.BadRequest(w, "Petal can only be learned toward "+strings.Join(learnerPairs, ", ")+" so far")
|
||||
return
|
||||
}
|
||||
|
||||
if err := u.SetPair(id, lang, direction); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||
return
|
||||
|
||||
@@ -561,6 +561,34 @@ DROP TABLE suggestions;
|
||||
ALTER TABLE suggestions_new RENAME TO suggestions;
|
||||
CREATE INDEX idx_suggestions_doc_id ON suggestions(doc_id);
|
||||
CREATE INDEX idx_suggestions_resolved ON suggestions(status, resolved_at);
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Which half of the pair is being learned.
|
||||
//
|
||||
// `pair_lang` (0010) has always answered "which two languages", and every
|
||||
// surface built on it assumed the answer to a second question nobody had
|
||||
// asked: that English is the language being *learned*. That assumption is
|
||||
// load-bearing in a dozen places — CJK is deliberately never tokenized,
|
||||
// never spell-checked, never glossed; the prompts explain English in her
|
||||
// language; the vocabulary garden captures English words. All correct for
|
||||
// a Mandarin native practising English, and all backwards for an English
|
||||
// native practising Mandarin.
|
||||
//
|
||||
// A second pair code ('zh-learner') was the cheaper option and is the
|
||||
// wrong shape: it would make the two directions of one pair look like two
|
||||
// unrelated languages to every query, and it would have to be repeated for
|
||||
// fr, es and pt-PT before any of them could turn around. A column keeps
|
||||
// the two questions separate, which is what they are.
|
||||
//
|
||||
// 'learning_en' is the default and is what every existing row means — the
|
||||
// backfill is the DEFAULT itself, and it is right rather than merely
|
||||
// convenient: all three accounts today are Mandarin natives writing
|
||||
// English.
|
||||
name: "0016_user_direction",
|
||||
stmt: `
|
||||
ALTER TABLE users ADD COLUMN direction TEXT NOT NULL DEFAULT 'learning_en'
|
||||
CHECK(direction IN ('learning_en','learning_pair'));
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -15,6 +15,19 @@ type User struct {
|
||||
// today, "pt-PT"/"fr"/"es" once the langpacks land. It selects the UI copy
|
||||
// and dictionary set, not the language they may type in.
|
||||
PairLang string `json:"pair_lang"`
|
||||
|
||||
// Direction says which half of the pair is being *learned*. Every pair until
|
||||
// now assumed one answer: the writer is native in X and practising English,
|
||||
// so hanzi is never tokenized and English is what gets underlined. Turn it
|
||||
// around — a native English speaker learning Chinese — and the same pair
|
||||
// wants the opposite of nearly every default.
|
||||
//
|
||||
// It is a separate column from PairLang rather than a second pair code
|
||||
// ("zh-learner") because it is a genuinely separate question: the pair says
|
||||
// *which two languages*, this says *which way round*. Keeping them apart is
|
||||
// what lets fr, es and pt-PT inherit the learner direction later without a
|
||||
// second langpack each.
|
||||
Direction string `json:"direction"`
|
||||
}
|
||||
|
||||
// Document is a single piece of writing. `Content` is the Tiptap JSON document
|
||||
|
||||
@@ -36,3 +36,13 @@ var glossGz []byte
|
||||
//
|
||||
//go:embed data/phonetic.json.gz
|
||||
var phoneticGz []byte
|
||||
|
||||
// hanziGz is the gzipped Chinese→English map: simplified headword → [[pinyin,
|
||||
// senses], …]. Built from CC-CEDICT (scripts/build_cedict.py), unfiltered — the
|
||||
// word a learner stops on is the one they do not know, so this is the one
|
||||
// dataset here with no frequency gate. Loaded on its own sync.Once (see
|
||||
// hanzi.go), not with the four above, because only a learner-direction account
|
||||
// ever asks for it.
|
||||
//
|
||||
//go:embed data/hanzi.json.gz
|
||||
var hanziGz []byte
|
||||
|
||||
Binary file not shown.
@@ -42,6 +42,36 @@ func (h *Handler) GlossRoutes() chi.Router {
|
||||
return r
|
||||
}
|
||||
|
||||
// HanziRoutes returns the router mounted at /api/hanzi — a Chinese word to its
|
||||
// pinyin and English senses, for a writer going the other way through the zh
|
||||
// pair (`users.direction = 'learning_pair'`).
|
||||
//
|
||||
// It does not go through [Handler.providerFor], and that is not an oversight.
|
||||
// providerFor picks a dictionary by the writer's *pair*, to answer "what does
|
||||
// this English word mean in her language" — a question whose answer differs per
|
||||
// pair. This endpoint asks the opposite question of exactly one language, and
|
||||
// [auth.SupportsLearnerDirection] already guarantees that language is Chinese.
|
||||
// Routing it through the pair would add a database read per hover to choose
|
||||
// between one option and itself.
|
||||
func (h *Handler) HanziRoutes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/{word}", h.hanzi)
|
||||
return r
|
||||
}
|
||||
|
||||
// hanzi answers a Chinese word lookup. Like the other two, a miss is a 200 with
|
||||
// empty lists — a hover that lands on a word the dictionary has never heard of
|
||||
// is an ordinary thing to happen while reading, and the tooltip simply doesn't
|
||||
// open.
|
||||
func (h *Handler) hanzi(w http.ResponseWriter, r *http.Request) {
|
||||
res, err := h.Set.Hanzi(pathWord(r))
|
||||
if err != nil {
|
||||
writeLookupErr(w, err)
|
||||
return
|
||||
}
|
||||
writeLookup(w, res)
|
||||
}
|
||||
|
||||
// providerFor returns the provider for the caller's language pair.
|
||||
//
|
||||
// The pair language is read here rather than threaded down because a word
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// The Chinese half of the lexicon: a word written in hanzi to its pinyin and
|
||||
// English senses. This is the mirror image of `gloss` — that one reads English
|
||||
// and answers in Chinese, for a Mandarin native practising English; this one
|
||||
// reads Chinese and answers in English, for the other direction of the same
|
||||
// pair (`users.direction = 'learning_pair'`).
|
||||
//
|
||||
// It is deliberately not folded into [Lexicon.load]. That method reads four
|
||||
// datasets on the first lookup of any kind, and this one is 3.1 MB gzipped that
|
||||
// only a learner-direction account will ever ask for — every other writer would
|
||||
// pay the decompression and the resident memory for a map they never touch. Its
|
||||
// own sync.Once means the cost lands on the first Chinese hover and nowhere
|
||||
// else.
|
||||
|
||||
// HanziReading is one pronunciation of a word and the senses it carries in that
|
||||
// pronunciation. A word usually has one; the ones that have two are why this is
|
||||
// a list rather than a pair of strings. 得 is dé, "to obtain", *and* de, the
|
||||
// particle that makes 说得很好 mean "speaks well" — a learner shown only the
|
||||
// first has been told something false about the sentence in front of them.
|
||||
type HanziReading struct {
|
||||
Pinyin string `json:"pinyin"`
|
||||
Senses string `json:"senses"`
|
||||
}
|
||||
|
||||
// HanziChar is one character of a word that the dictionary could not answer as
|
||||
// a whole. See [Lexicon.Hanzi].
|
||||
type HanziChar struct {
|
||||
Char string `json:"char"`
|
||||
Pinyin string `json:"pinyin"`
|
||||
Senses string `json:"senses"`
|
||||
}
|
||||
|
||||
// HanziResult is what a Chinese word lookup answers. Readings is empty for a
|
||||
// word the dictionary does not have, in which case Chars may carry the
|
||||
// character-by-character reading instead.
|
||||
type HanziResult struct {
|
||||
Word string `json:"word"`
|
||||
Readings []HanziReading `json:"readings"`
|
||||
Chars []HanziChar `json:"chars"`
|
||||
}
|
||||
|
||||
type hanziStore struct {
|
||||
once sync.Once
|
||||
err error
|
||||
// word → [[pinyin, senses], …], exactly as scripts/build_cedict.py writes it.
|
||||
entries map[string][][]string
|
||||
}
|
||||
|
||||
var hanzi hanziStore
|
||||
|
||||
func (h *hanziStore) load() {
|
||||
h.once.Do(func() {
|
||||
if err := gunzipJSON(hanziGz, &h.entries); err != nil {
|
||||
h.err = fmt.Errorf("load hanzi: %w", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// maxHanziChars caps the per-character fallback. A run longer than this is
|
||||
// almost certainly a phrase the segmenter split badly rather than a word, and
|
||||
// spelling out eight characters one at a time is a wall, not a hint.
|
||||
const maxHanziChars = 6
|
||||
|
||||
// Hanzi returns the pinyin and English senses of a Chinese word.
|
||||
//
|
||||
// There is no de-inflection walk here, and its absence is a fact about the
|
||||
// language rather than an omission: Chinese words do not inflect, so the
|
||||
// candidate forms [lookupGloss] tries for "running" → "run" have no analogue.
|
||||
// A lookup either hits the headword or it does not.
|
||||
//
|
||||
// What it does instead is fall back to the characters. The segmentation word
|
||||
// list is a superset of this dictionary — every glossable word can be
|
||||
// segmented, but jieba knows ordinary compounds CC-CEDICT has no entry for — so
|
||||
// a hover really can land on a word with nothing to say about it. Chinese
|
||||
// compounds are usually transparent from their parts (电脑 is "electric brain"),
|
||||
// which makes the character reading a genuinely useful second answer rather
|
||||
// than a consolation prize. It is returned as its own field so the surface can
|
||||
// say which of the two it is showing; a caller that only wants whole words can
|
||||
// ignore it.
|
||||
func (l *Lexicon) Hanzi(word string) (HanziResult, error) {
|
||||
hanzi.load()
|
||||
if hanzi.err != nil {
|
||||
return HanziResult{}, hanzi.err
|
||||
}
|
||||
|
||||
norm := strings.TrimSpace(word)
|
||||
res := HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}
|
||||
if norm == "" {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
if rows, ok := hanzi.entries[norm]; ok {
|
||||
res.Readings = toReadings(rows)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
chars := []rune(norm)
|
||||
if len(chars) < 2 || len(chars) > maxHanziChars {
|
||||
// A single character that missed has no parts to fall back to, and a long
|
||||
// run is not a word. Either way the honest answer is nothing.
|
||||
return res, nil
|
||||
}
|
||||
for _, r := range chars {
|
||||
if !unicode.Is(unicode.Han, r) {
|
||||
// Mixed input (a stray letter or digit inside the run) is not something
|
||||
// the character reading can explain, and guessing at the hanzi parts of
|
||||
// it would be worse than silence.
|
||||
return HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}, nil
|
||||
}
|
||||
rows, ok := hanzi.entries[string(r)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
first := toReadings(rows)
|
||||
if len(first) == 0 {
|
||||
continue
|
||||
}
|
||||
res.Chars = append(res.Chars, HanziChar{
|
||||
Char: string(r),
|
||||
Pinyin: first[0].Pinyin,
|
||||
Senses: first[0].Senses,
|
||||
})
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func toReadings(rows [][]string) []HanziReading {
|
||||
out := make([]HanziReading, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if len(row) < 2 {
|
||||
continue
|
||||
}
|
||||
out = append(out, HanziReading{Pinyin: row[0], Senses: row[1]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
// The Chinese direction of the lexicon, against the real embedded asset — not a
|
||||
// fixture. The dataset is built by scripts/build_cedict.py, which asserts its
|
||||
// own invariants at build time; what these assert is that the *lookup* over it
|
||||
// behaves, including on the entries the build script goes out of its way to keep.
|
||||
|
||||
func TestHanziLookup(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
res, err := l.Hanzi("公园")
|
||||
if err != nil {
|
||||
t.Fatalf("lookup 公园: %v", err)
|
||||
}
|
||||
if len(res.Readings) == 0 {
|
||||
t.Fatal("公园 has no readings")
|
||||
}
|
||||
// Tone marks, not the numbered pinyin CC-CEDICT stores. The number is the
|
||||
// storage format; the marks are what a learner reads.
|
||||
if got := res.Readings[0].Pinyin; got != "gōngyuán" {
|
||||
t.Errorf("公园 pinyin = %q, want gōngyuán", got)
|
||||
}
|
||||
if !strings.Contains(res.Readings[0].Senses, "park") {
|
||||
t.Errorf("公园 senses = %q, want something about a park", res.Readings[0].Senses)
|
||||
}
|
||||
// A word answered whole says nothing about its characters — the fallback is
|
||||
// the other branch, and sending both would double the payload of the common
|
||||
// case to no purpose.
|
||||
if len(res.Chars) != 0 {
|
||||
t.Errorf("a whole-word hit also returned %d characters", len(res.Chars))
|
||||
}
|
||||
}
|
||||
|
||||
// 得 is the reason readings are a list. Answered with only dé "to obtain", a
|
||||
// learner hovering it in 说得很好 has been told something false about the
|
||||
// sentence they are looking at.
|
||||
func TestHanziParticleCarriesItsGrammaticalReading(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
for _, particle := range []string{"的", "地", "得"} {
|
||||
res, err := l.Hanzi(particle)
|
||||
if err != nil {
|
||||
t.Fatalf("lookup %s: %v", particle, err)
|
||||
}
|
||||
var found bool
|
||||
for _, r := range res.Readings {
|
||||
if r.Pinyin == "de" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s never reads as neutral \"de\": %+v", particle, res.Readings)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The fallback the segmentation gap makes necessary: jieba knows ordinary
|
||||
// compounds CC-CEDICT has no headword for, so a hover can land on a real word
|
||||
// with no entry. Chinese compounds are usually transparent from their parts, so
|
||||
// the characters are a real second answer.
|
||||
func TestHanziFallsBackToCharacters(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
// Constructed rather than borrowed from the corpus: a word that CC-CEDICT
|
||||
// *does* carry would test the other branch, and which compounds it happens to
|
||||
// omit is not something a test should pin.
|
||||
const made = "猫书"
|
||||
if _, ok := hanzi.entries[made]; ok {
|
||||
t.Skipf("%s has become a real headword; pick another compound", made)
|
||||
}
|
||||
res, err := l.Hanzi(made)
|
||||
if err != nil {
|
||||
t.Fatalf("lookup %s: %v", made, err)
|
||||
}
|
||||
if len(res.Readings) != 0 {
|
||||
t.Fatalf("%s answered as a whole word: %+v", made, res.Readings)
|
||||
}
|
||||
if len(res.Chars) != 2 {
|
||||
t.Fatalf("character fallback gave %d entries, want 2: %+v", len(res.Chars), res.Chars)
|
||||
}
|
||||
if res.Chars[0].Char != "猫" || !strings.Contains(res.Chars[0].Senses, "cat") {
|
||||
t.Errorf("first character = %+v, want 猫 ~ cat", res.Chars[0])
|
||||
}
|
||||
if res.Chars[0].Pinyin != "māo" {
|
||||
t.Errorf("猫 pinyin = %q, want māo", res.Chars[0].Pinyin)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHanziMisses(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
for name, word := range map[string]string{
|
||||
// A single character with no entry has no parts to fall back to.
|
||||
"lone unknown character": "龥",
|
||||
"empty": "",
|
||||
"whitespace": " ",
|
||||
// Not Chinese at all: the English tokenizer owns these, and answering
|
||||
// would mean guessing.
|
||||
"english": "hello",
|
||||
"mixed": "猫cat",
|
||||
// Longer than a word: a bad segmentation, not something to spell out
|
||||
// character by character.
|
||||
"a whole clause": "我今天早上去公园跑步了",
|
||||
} {
|
||||
res, err := l.Hanzi(word)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if len(res.Readings) != 0 || len(res.Chars) != 0 {
|
||||
t.Errorf("%s (%q) answered with %+v / %+v", name, word, res.Readings, res.Chars)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHanziEndpoint(t *testing.T) {
|
||||
h := NewHandler(nil, NewSet(nil))
|
||||
r := chi.NewRouter()
|
||||
r.Mount("/hanzi", h.HanziRoutes())
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/"+"跑步", nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d", w.Code)
|
||||
}
|
||||
var got HanziResult
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.Word != "跑步" || len(got.Readings) == 0 || got.Readings[0].Pinyin != "pǎobù" {
|
||||
t.Fatalf("response = %+v", got)
|
||||
}
|
||||
|
||||
// A miss is a 200 with empty lists, like the other two lookups — the tooltip
|
||||
// quietly doesn't open rather than showing an error over her writing.
|
||||
w = httptest.NewRecorder()
|
||||
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/zzz", nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("miss: status = %d, want 200", w.Code)
|
||||
}
|
||||
}
|
||||
@@ -109,3 +109,13 @@ func (g glossless) Lookup(word string) (Result, error) {
|
||||
func (g glossless) Gloss(word string) (GlossResult, error) {
|
||||
return GlossResult{Word: word}, nil
|
||||
}
|
||||
|
||||
// Hanzi answers a Chinese-word lookup from the embedded CC-CEDICT map.
|
||||
//
|
||||
// It is on the Set rather than on [Provider] because it is not the same
|
||||
// question the other two ask. Lookup and Gloss vary by pair — which is why they
|
||||
// are behind an interface with two implementations — while this one is asked of
|
||||
// Chinese or not at all: the learner direction exists for exactly one pair (see
|
||||
// auth.learnerPairs), and DreamDict's own CC-CEDICT would be a second copy of
|
||||
// the same dictionary, chosen by a rule with one branch.
|
||||
func (s *Set) Hanzi(word string) (HanziResult, error) { return s.embedded.Hanzi(word) }
|
||||
|
||||
@@ -201,3 +201,36 @@ func TestOfflineCardWinsSpanCollision(t *testing.T) {
|
||||
t.Errorf("the exact offline card should own the span, got %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestOfflineHanziFindingStaysMechanics: a 错别字 the Chinese rule pack found —
|
||||
// both halves written in hanzi — files as an ordinary mechanics row.
|
||||
//
|
||||
// The check is worth its own test because there is a rule one layer over that
|
||||
// would plausibly claim it. `isTranslation` re-labels an edit whose original
|
||||
// reads as the writer's language and whose replacement reads as English, which
|
||||
// is exactly how a zh-pair writer's quoted Chinese becomes a 'translate' card.
|
||||
// A wrong-character fix looks like the first half of that and nothing like the
|
||||
// second: 己经 → 已经 never leaves Chinese. It must stay a tidy-up in her own
|
||||
// sentence, on the same rail as a doubled word, with no rendering-into-English
|
||||
// implied anywhere.
|
||||
func TestOfflineHanziFindingStaysMechanics(t *testing.T) {
|
||||
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":1,"to":3,"original":"己经","replacement":"已经","explanation":"已经 (already) takes 已","type":"mechanics"}
|
||||
]`)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want the one finding, got %+v", got)
|
||||
}
|
||||
if got[0].Type != db.SuggestionTypeMechanics {
|
||||
t.Errorf("hanzi fix filed as %q, want %q", got[0].Type, db.SuggestionTypeMechanics)
|
||||
}
|
||||
if got[0].Source != db.SuggestionSourceLocal {
|
||||
t.Errorf("source = %q, want %q", got[0].Source, db.SuggestionSourceLocal)
|
||||
}
|
||||
// The characters survive the round trip intact — a mangled span here would
|
||||
// replace the wrong characters in her document.
|
||||
if got[0].Original != "己经" || got[0].Replacement != "已经" {
|
||||
t.Errorf("round-tripped as %q → %q", got[0].Original, got[0].Replacement)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user