Merge: the zh learner direction, the es pair, IME composition guards
This commit is contained in:
prosolis
+99
-3
File diff suppressed because one or more lines are too long
@@ -207,6 +207,9 @@ func main() {
|
||||
lex := lexicon.NewHandler(database.DB, lexSet)
|
||||
pr.Mount("/word", lex.Routes())
|
||||
pr.Mount("/gloss", lex.GlossRoutes())
|
||||
// The same lookup pointing the other way: a Chinese word to its pinyin
|
||||
// and English senses, for an account whose direction is learning_pair.
|
||||
pr.Mount("/hanzi", lex.HanziRoutes())
|
||||
|
||||
// Vocabulary garden: words the writer looks up are captured here and
|
||||
// surfaced for gentle spaced-repetition review.
|
||||
|
||||
@@ -59,6 +59,9 @@ TTS_VOICE_EN=en_US-amy-medium
|
||||
TTS_VOICE_ZH=zh_CN-huayan-medium
|
||||
TTS_VOICE_PT=pt_PT-tugão-medium
|
||||
TTS_VOICE_FR=fr_FR-siwis-medium
|
||||
# Mexican, not peninsular — the es pack is written in neutral Latin American
|
||||
# Spanish, and es_ES-davefx-medium would read it in the accent it avoids.
|
||||
TTS_VOICE_ES=es_MX-ald-medium
|
||||
TTS_AUDIO_FORMAT=mp3
|
||||
TTS_TIMEOUT=15s
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@ services:
|
||||
# hyphen, and there is one Portuguese voice loaded either way.
|
||||
TTS_ENDPOINT_PT: http://piper-pt:5000
|
||||
TTS_ENDPOINT_FR: http://piper-fr:5000
|
||||
TTS_ENDPOINT_ES: http://piper-es:5000
|
||||
# The sidecars run piper-tts 1.6.0, which serves synthesis on
|
||||
# /synthesize; millenia's older server keeps the default "/".
|
||||
TTS_PATH: /synthesize
|
||||
@@ -173,6 +174,29 @@ services:
|
||||
networks:
|
||||
- internal
|
||||
|
||||
# Spanish, for the es pair — and the Portuguese trap rather than the French
|
||||
# one. Piper's catalogue has nine Spanish voices, six of them es_ES, and the
|
||||
# obvious pick (es_ES-davefx-medium, which the build plan itself named) is
|
||||
# peninsular. The es pack is written in neutral Latin American Spanish, so a
|
||||
# Castilian voice would read it aloud in the accent the copy was written to
|
||||
# avoid — the same wrong-country default that pt-PT hit through packaging,
|
||||
# arriving here through the voice list. Only two Latin American voices exist,
|
||||
# es_AR-daniela-high and es_MX; Mexican is the neutral broadcast standard and
|
||||
# ald-medium matches the register of the other four. ASCII, so the
|
||||
# percent-encoded download fallback added for tugão never has to fire.
|
||||
piper-es:
|
||||
build:
|
||||
context: deploy/piper
|
||||
image: petal-piper:local
|
||||
container_name: petal-piper-es
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PIPER_VOICE: ${TTS_VOICE_ES:-es_MX-ald-medium}
|
||||
volumes:
|
||||
- piper-voices:/voices
|
||||
networks:
|
||||
- internal
|
||||
|
||||
networks:
|
||||
# Created and owned by the host's Traefik stack.
|
||||
traefik:
|
||||
|
||||
+134
-11
@@ -24,7 +24,7 @@ func patchMe(t *testing.T, users *UserStore, id, body string) *httptest.Response
|
||||
func TestSetPairLang(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPairLang("bob", "pt-PT"); err != nil {
|
||||
if err := users.SetPair("bob", "pt-PT", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set pt-PT: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "pt-PT" {
|
||||
@@ -34,16 +34,23 @@ func TestSetPairLang(t *testing.T) {
|
||||
// Every pair with a langpack, not just the first one: this list and the
|
||||
// frontend's PACKS are two copies of the same fact, and the day they
|
||||
// disagree is the day she can pick a pair the app cannot render.
|
||||
if err := users.SetPairLang("bob", "fr"); err != nil {
|
||||
if err := users.SetPair("bob", "fr", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set fr: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "fr" {
|
||||
t.Fatalf("pair_lang = %q, want fr", u.PairLang)
|
||||
}
|
||||
|
||||
if err := users.SetPair("bob", "es", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set es: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "es" {
|
||||
t.Fatalf("pair_lang = %q, want es", u.PairLang)
|
||||
}
|
||||
|
||||
// And back — a writer who tries a pair and doesn't like it must be able to
|
||||
// return, which is the whole reason the picker exists.
|
||||
if err := users.SetPairLang("bob", "zh"); err != nil {
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set zh: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "zh" {
|
||||
@@ -56,11 +63,14 @@ func TestSetPairLang(t *testing.T) {
|
||||
func TestSetPairLangRejectsUnshippedPairs(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
// "es" is the real case here — the pair whose pack has not been written yet.
|
||||
// "pt-BR" is the near-miss that matters most: a Brazilian code must not be
|
||||
// quietly served European copy and a European voice.
|
||||
for _, lang := range []string{"es", "pt-BR", "fr-CA", "klingon", "", " "} {
|
||||
if err := users.SetPairLang("bob", lang); err == nil {
|
||||
// The near-misses are the ones that matter, and there are two of them now.
|
||||
// "pt-BR" must not be quietly served European copy and a European voice;
|
||||
// "es-ES" is the same mistake pointing the other way, because the es pack is
|
||||
// deliberately Latin American and reads itself aloud in a Mexican voice. A
|
||||
// regional code Petal has not decided about is refused rather than rounded
|
||||
// to the nearest pack it happens to have.
|
||||
for _, lang := range []string{"es-ES", "pt-BR", "fr-CA", "de", "klingon", "", " "} {
|
||||
if err := users.SetPair("bob", lang, DirectionLearningEn); err == nil {
|
||||
t.Fatalf("stored unshipped pair %q", lang)
|
||||
}
|
||||
}
|
||||
@@ -71,7 +81,7 @@ func TestSetPairLangRejectsUnshippedPairs(t *testing.T) {
|
||||
|
||||
func TestSetPairLangUnknownUser(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
if err := users.SetPairLang("nobody", "pt-PT"); err == nil {
|
||||
if err := users.SetPair("nobody", "pt-PT", DirectionLearningEn); err == nil {
|
||||
t.Fatal("set a pair language on an account that does not exist")
|
||||
}
|
||||
}
|
||||
@@ -98,8 +108,8 @@ func TestUpdateMeHandlerRejects(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
for name, body := range map[string]string{
|
||||
"unshipped pair": `{"pair_lang":"es"}`,
|
||||
"missing field": `{}`,
|
||||
"unshipped pair": `{"pair_lang":"es-ES"}`,
|
||||
"unknown direction": `{"direction":"learning_klingon"}`,
|
||||
"not json": `pt-PT`,
|
||||
} {
|
||||
if w := patchMe(t, users, "bob", body); w.Code != http.StatusBadRequest {
|
||||
@@ -116,3 +126,116 @@ func TestUpdateMeHandlerRejects(t *testing.T) {
|
||||
t.Fatalf("unknown user: status = %d, want 401", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// An empty body used to be a 400, back when pair_lang was the only field and a
|
||||
// request that named none of it could only be a client bug. With two optional
|
||||
// fields it is an ordinary PATCH that changes nothing, and it has to be: the
|
||||
// picker sends one field without knowing the other, and "omitted" has to mean
|
||||
// "leave it alone" for that to be safe.
|
||||
func TestUpdateMeHandlerEmptyBodyChangesNothing(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningPair); err != nil {
|
||||
t.Fatalf("set up: %v", err)
|
||||
}
|
||||
w := patchMe(t, users, "bob", `{}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
u, _ := users.Get("bob")
|
||||
if u.PairLang != "zh" || u.Direction != DirectionLearningPair {
|
||||
t.Fatalf("empty PATCH moved the account to %q/%q", u.PairLang, u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The direction axis: an account can be turned around and turned back, and the
|
||||
// default every existing row already carries is the one it had before the column
|
||||
// existed.
|
||||
func TestDirectionRoundTrip(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("a fresh account starts at %q, want %q", u.Direction, DirectionLearningEn)
|
||||
}
|
||||
|
||||
w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("turn around: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
var got db.User
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
// The response carries the direction, not just the pair — the client reads
|
||||
// its whole state back from here rather than assuming the write took.
|
||||
if got.Direction != DirectionLearningPair || got.PairLang != "zh" {
|
||||
t.Fatalf("response = %+v, want bob learning zh", got)
|
||||
}
|
||||
|
||||
if w := patchMe(t, users, "bob", `{"direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||
t.Fatalf("turn back: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("direction = %q after turning back", u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The refusal this axis exists to make: a pair with no word list cannot be
|
||||
// learned toward, however good its langpack is. fr, es and pt-PT all have copy,
|
||||
// voices and spelling dictionaries — and nothing that could segment a sentence
|
||||
// or read from that language into English, which is what a learner needs.
|
||||
func TestLearnerDirectionRefusedForPairsWithoutData(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
for _, lang := range []string{"pt-PT", "fr", "es"} {
|
||||
if err := users.SetPair("bob", lang, DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set %s: %v", lang, err)
|
||||
}
|
||||
w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: status = %d, want 400", lang, w.Code)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("%s: a refused write still moved direction to %q", lang, u.Direction)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The two-field combination the handler validates as one decision. An account
|
||||
// already learning Chinese that asks only to change pair is asking for a state
|
||||
// neither field names on its own — French with segmentation — and it must not
|
||||
// arrive by leaving one field out.
|
||||
func TestPairChangeCannotStrandTheLearnerDirection(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningPair); err != nil {
|
||||
t.Fatalf("set up: %v", err)
|
||||
}
|
||||
if w := patchMe(t, users, "bob", `{"pair_lang":"fr"}`); w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", w.Code)
|
||||
}
|
||||
u, _ := users.Get("bob")
|
||||
if u.PairLang != "zh" || u.Direction != DirectionLearningPair {
|
||||
t.Fatalf("refused write left the account at %q/%q", u.PairLang, u.Direction)
|
||||
}
|
||||
|
||||
// Naming both at once is how that move is actually made, and it works.
|
||||
if w := patchMe(t, users, "bob", `{"pair_lang":"fr","direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||
t.Fatalf("both fields: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "fr" || u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("account = %q/%q, want fr/learning_en", u.PairLang, u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The CHECK constraint is the last line, below the handler and below SetPair:
|
||||
// a direction that reaches the column by any other route is still refused.
|
||||
func TestDirectionCheckConstraint(t *testing.T) {
|
||||
_, users, database := newStores(t)
|
||||
if _, err := database.Exec(`UPDATE users SET direction = 'sideways' WHERE id = 'bob'`); err == nil {
|
||||
t.Fatal("the users.direction CHECK accepted 'sideways'")
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("direction = %q after a refused UPDATE", u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
+112
-13
@@ -49,9 +49,9 @@ func (u *UserStore) Upsert(sub, email, displayName string) error {
|
||||
func (u *UserStore) Get(id string) (db.User, error) {
|
||||
var user db.User
|
||||
err := u.db.QueryRow(
|
||||
`SELECT id, email, COALESCE(display_name, ''), created_at, pair_lang
|
||||
`SELECT id, email, COALESCE(display_name, ''), created_at, pair_lang, direction
|
||||
FROM users WHERE id = ?`, id,
|
||||
).Scan(&user.ID, &user.Email, &user.DisplayName, &user.CreatedAt, &user.PairLang)
|
||||
).Scan(&user.ID, &user.Email, &user.DisplayName, &user.CreatedAt, &user.PairLang, &user.Direction)
|
||||
return user, err
|
||||
}
|
||||
|
||||
@@ -76,8 +76,13 @@ func (u *UserStore) MeHandler() http.HandlerFunc {
|
||||
// this one names the pairs Petal can render itself in, which requires a langpack
|
||||
// on the frontend. Accepting a code with no pack would leave her looking at
|
||||
// Chinese with no way back except another guess, so the server refuses it. es
|
||||
// joins this list on the day its pack lands, not before.
|
||||
var shippedPairs = []string{"zh", "pt-PT", "fr"}
|
||||
// joined on the day its pack landed, not before.
|
||||
//
|
||||
// These four are now every pair PairLang names on the frontend, which makes the
|
||||
// two lists look redundant. They are not: the next pair will exist in the type
|
||||
// and in the prompts long before it has copy, and this list is the one that
|
||||
// says a writer may actually be sent there.
|
||||
var shippedPairs = []string{"zh", "pt-PT", "fr", "es"}
|
||||
|
||||
func pairIsShipped(lang string) bool {
|
||||
for _, p := range shippedPairs {
|
||||
@@ -88,12 +93,63 @@ func pairIsShipped(lang string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// SetPairLang moves an account to another (English + X) pair.
|
||||
func (u *UserStore) SetPairLang(id, lang string) error {
|
||||
// The two directions a pair can be travelled in. `DirectionLearningEn` is the
|
||||
// original assumption made explicit: the writer is native in X and practising
|
||||
// English. `DirectionLearningPair` is the other way round.
|
||||
const (
|
||||
DirectionLearningEn = "learning_en"
|
||||
DirectionLearningPair = "learning_pair"
|
||||
)
|
||||
|
||||
// The pairs whose *learner* direction Petal can actually serve, which is a
|
||||
// narrower thing than a shipped pair and narrower again than a langpack.
|
||||
//
|
||||
// Turning a pair around needs data no langpack carries: a word list to segment
|
||||
// with, and a dictionary that reads from the pair language into English. Chinese
|
||||
// has both as of Phase 26 (CC-CEDICT + jieba); French, Spanish and Portuguese
|
||||
// have neither yet, and — unlike a missing pack, which leaves a writer looking
|
||||
// at copy she cannot read — a missing word list would leave her looking at an
|
||||
// editor that silently does nothing when she hovers. Both are bad; only one is
|
||||
// legible as a bug. So the server refuses, for the same reason and by the same
|
||||
// mechanism as `shippedPairs`.
|
||||
//
|
||||
// This list is expected to grow one pair at a time and never to be inferred:
|
||||
// segmentation is a property of a writing system, and there is no rule that
|
||||
// derives "has a word list" from a language code.
|
||||
var learnerPairs = []string{"zh"}
|
||||
|
||||
// SupportsLearnerDirection reports whether a pair can be turned around.
|
||||
func SupportsLearnerDirection(lang string) bool {
|
||||
for _, p := range learnerPairs {
|
||||
if p == lang {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func directionIsKnown(d string) bool {
|
||||
return d == DirectionLearningEn || d == DirectionLearningPair
|
||||
}
|
||||
|
||||
// SetPair moves an account to another (English + X) pair, in a given direction.
|
||||
//
|
||||
// The two are written together because they constrain each other: a direction is
|
||||
// only meaningful for a pair that can be travelled in it, and validating them a
|
||||
// field at a time would let a two-step change pass through a state that neither
|
||||
// step is allowed to leave behind.
|
||||
func (u *UserStore) SetPair(id, lang, direction string) error {
|
||||
if !pairIsShipped(lang) {
|
||||
return errors.New("auth: unshipped pair language " + lang)
|
||||
}
|
||||
res, err := u.db.Exec(`UPDATE users SET pair_lang = ? WHERE id = ?`, lang, id)
|
||||
if !directionIsKnown(direction) {
|
||||
return errors.New("auth: unknown direction " + direction)
|
||||
}
|
||||
if direction == DirectionLearningPair && !SupportsLearnerDirection(lang) {
|
||||
return errors.New("auth: no learner direction for " + lang)
|
||||
}
|
||||
res, err := u.db.Exec(
|
||||
`UPDATE users SET pair_lang = ?, direction = ? WHERE id = ?`, lang, direction, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -103,8 +159,8 @@ func (u *UserStore) SetPairLang(id, lang string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateMeHandler changes the caller's own settings — today, the one setting
|
||||
// there is: which language Petal speaks alongside her English.
|
||||
// UpdateMeHandler changes the caller's own settings: which language Petal
|
||||
// speaks alongside her English, and which of the two she is learning.
|
||||
//
|
||||
// It answers with the whole updated user rather than an empty 204 so the client
|
||||
// has one shape to trust: /api/me and this return the same thing, and the app
|
||||
@@ -114,24 +170,67 @@ func (u *UserStore) SetPairLang(id, lang string) error {
|
||||
// dictionary, her read-aloud voice, which word-lookup provider answers, and the
|
||||
// language the prompts ask the model to explain in. All of those read
|
||||
// `users.pair_lang` at use time, so all of them follow from this one write.
|
||||
//
|
||||
// Both fields are optional and each defaults to what the account already has, so
|
||||
// the picker can send one without knowing the other. That matters for the
|
||||
// combination this endpoint exists to prevent: a client that sent only
|
||||
// `pair_lang: "fr"` while the account sat on `learning_pair` would otherwise ask
|
||||
// for French-with-segmentation, which does not exist. Here it is one decision
|
||||
// with one validation, and the answer carries whatever actually landed.
|
||||
func (u *UserStore) UpdateMeHandler() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
PairLang string `json:"pair_lang"`
|
||||
PairLang *string `json:"pair_lang"`
|
||||
Direction *string `json:"direction"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
httputil.BadRequest(w, "invalid request body")
|
||||
return
|
||||
}
|
||||
lang := strings.TrimSpace(body.PairLang)
|
||||
id := UserID(r.Context())
|
||||
current, err := u.Get(id)
|
||||
if err != nil {
|
||||
// Only a missing row means "not signed in". A dictionary-file or
|
||||
// SQLite fault answered as 401 would trip the client's session
|
||||
// interceptor and throw a writer out of an app she is still signed
|
||||
// in to — the same distinction SetPair's error branch makes below.
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||
return
|
||||
}
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
lang, direction := current.PairLang, current.Direction
|
||||
if body.PairLang != nil {
|
||||
lang = strings.TrimSpace(*body.PairLang)
|
||||
}
|
||||
if body.Direction != nil {
|
||||
direction = strings.TrimSpace(*body.Direction)
|
||||
}
|
||||
|
||||
if !pairIsShipped(lang) {
|
||||
// Name the ones that work. A writer who lands here has picked from a
|
||||
// stale client, and "not a language" tells her nothing.
|
||||
httputil.BadRequest(w, "unsupported language pair — Petal speaks "+strings.Join(shippedPairs, ", "))
|
||||
return
|
||||
}
|
||||
id := UserID(r.Context())
|
||||
if err := u.SetPairLang(id, lang); err != nil {
|
||||
if !directionIsKnown(direction) {
|
||||
httputil.BadRequest(w, "unknown direction — expected "+DirectionLearningEn+" or "+DirectionLearningPair)
|
||||
return
|
||||
}
|
||||
if direction == DirectionLearningPair && !SupportsLearnerDirection(lang) {
|
||||
// Refused rather than quietly downgraded to learning_en. A silent
|
||||
// downgrade would leave the writer looking at an editor that behaves
|
||||
// like the one she just tried to leave, with nothing to read as an
|
||||
// explanation — and the caller cannot tell the two outcomes apart
|
||||
// without diffing the response it was given.
|
||||
httputil.BadRequest(w, "Petal can only be learned toward "+strings.Join(learnerPairs, ", ")+" so far")
|
||||
return
|
||||
}
|
||||
|
||||
if err := u.SetPair(id, lang, direction); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||
return
|
||||
|
||||
@@ -16,6 +16,8 @@ func TestTTSVoicesDiscovery(t *testing.T) {
|
||||
"TTS_VOICE_PT=pt_PT-tugão-medium",
|
||||
"TTS_ENDPOINT_FR=http://piper-fr:5000",
|
||||
"TTS_VOICE_FR=fr_FR-siwis-medium",
|
||||
"TTS_ENDPOINT_ES=http://piper-es:5000",
|
||||
"TTS_VOICE_ES=es_MX-ald-medium",
|
||||
// Noise that must not become a language.
|
||||
"TTS_PATH=/synthesize",
|
||||
"PATH=/usr/bin",
|
||||
@@ -30,6 +32,10 @@ func TestTTSVoicesDiscovery(t *testing.T) {
|
||||
// Phase 24's whole TTS change: a fourth language costs two lines here
|
||||
// and a compose service, and no Go at all.
|
||||
"fr": {"http://piper-fr:5000", "fr_FR-siwis-medium"},
|
||||
// And a fifth cost exactly the same, which is the claim actually being
|
||||
// tested. The voice is Mexican on purpose: the es pack is Latin
|
||||
// American, and es_ES-davefx-medium would read it in the wrong accent.
|
||||
"es": {"http://piper-es:5000", "es_MX-ald-medium"},
|
||||
}
|
||||
if len(voices) != len(want) {
|
||||
t.Fatalf("discovered %v, want %v", voices, want)
|
||||
|
||||
@@ -561,6 +561,34 @@ DROP TABLE suggestions;
|
||||
ALTER TABLE suggestions_new RENAME TO suggestions;
|
||||
CREATE INDEX idx_suggestions_doc_id ON suggestions(doc_id);
|
||||
CREATE INDEX idx_suggestions_resolved ON suggestions(status, resolved_at);
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Which half of the pair is being learned.
|
||||
//
|
||||
// `pair_lang` (0010) has always answered "which two languages", and every
|
||||
// surface built on it assumed the answer to a second question nobody had
|
||||
// asked: that English is the language being *learned*. That assumption is
|
||||
// load-bearing in a dozen places — CJK is deliberately never tokenized,
|
||||
// never spell-checked, never glossed; the prompts explain English in her
|
||||
// language; the vocabulary garden captures English words. All correct for
|
||||
// a Mandarin native practising English, and all backwards for an English
|
||||
// native practising Mandarin.
|
||||
//
|
||||
// A second pair code ('zh-learner') was the cheaper option and is the
|
||||
// wrong shape: it would make the two directions of one pair look like two
|
||||
// unrelated languages to every query, and it would have to be repeated for
|
||||
// fr, es and pt-PT before any of them could turn around. A column keeps
|
||||
// the two questions separate, which is what they are.
|
||||
//
|
||||
// 'learning_en' is the default and is what every existing row means — the
|
||||
// backfill is the DEFAULT itself, and it is right rather than merely
|
||||
// convenient: all three accounts today are Mandarin natives writing
|
||||
// English.
|
||||
name: "0016_user_direction",
|
||||
stmt: `
|
||||
ALTER TABLE users ADD COLUMN direction TEXT NOT NULL DEFAULT 'learning_en'
|
||||
CHECK(direction IN ('learning_en','learning_pair'));
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -15,6 +15,19 @@ type User struct {
|
||||
// today, "pt-PT"/"fr"/"es" once the langpacks land. It selects the UI copy
|
||||
// and dictionary set, not the language they may type in.
|
||||
PairLang string `json:"pair_lang"`
|
||||
|
||||
// Direction says which half of the pair is being *learned*. Every pair until
|
||||
// now assumed one answer: the writer is native in X and practising English,
|
||||
// so hanzi is never tokenized and English is what gets underlined. Turn it
|
||||
// around — a native English speaker learning Chinese — and the same pair
|
||||
// wants the opposite of nearly every default.
|
||||
//
|
||||
// It is a separate column from PairLang rather than a second pair code
|
||||
// ("zh-learner") because it is a genuinely separate question: the pair says
|
||||
// *which two languages*, this says *which way round*. Keeping them apart is
|
||||
// what lets fr, es and pt-PT inherit the learner direction later without a
|
||||
// second langpack each.
|
||||
Direction string `json:"direction"`
|
||||
}
|
||||
|
||||
// Document is a single piece of writing. `Content` is the Tiptap JSON document
|
||||
|
||||
@@ -36,3 +36,13 @@ var glossGz []byte
|
||||
//
|
||||
//go:embed data/phonetic.json.gz
|
||||
var phoneticGz []byte
|
||||
|
||||
// hanziGz is the gzipped Chinese→English map: simplified headword → [[pinyin,
|
||||
// senses], …]. Built from CC-CEDICT (scripts/build_cedict.py), unfiltered — the
|
||||
// word a learner stops on is the one they do not know, so this is the one
|
||||
// dataset here with no frequency gate. Loaded on its own sync.Once (see
|
||||
// hanzi.go), not with the four above, because only a learner-direction account
|
||||
// ever asks for it.
|
||||
//
|
||||
//go:embed data/hanzi.json.gz
|
||||
var hanziGz []byte
|
||||
|
||||
Binary file not shown.
@@ -42,6 +42,36 @@ func (h *Handler) GlossRoutes() chi.Router {
|
||||
return r
|
||||
}
|
||||
|
||||
// HanziRoutes returns the router mounted at /api/hanzi — a Chinese word to its
|
||||
// pinyin and English senses, for a writer going the other way through the zh
|
||||
// pair (`users.direction = 'learning_pair'`).
|
||||
//
|
||||
// It does not go through [Handler.providerFor], and that is not an oversight.
|
||||
// providerFor picks a dictionary by the writer's *pair*, to answer "what does
|
||||
// this English word mean in her language" — a question whose answer differs per
|
||||
// pair. This endpoint asks the opposite question of exactly one language, and
|
||||
// [auth.SupportsLearnerDirection] already guarantees that language is Chinese.
|
||||
// Routing it through the pair would add a database read per hover to choose
|
||||
// between one option and itself.
|
||||
func (h *Handler) HanziRoutes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/{word}", h.hanzi)
|
||||
return r
|
||||
}
|
||||
|
||||
// hanzi answers a Chinese word lookup. Like the other two, a miss is a 200 with
|
||||
// empty lists — a hover that lands on a word the dictionary has never heard of
|
||||
// is an ordinary thing to happen while reading, and the tooltip simply doesn't
|
||||
// open.
|
||||
func (h *Handler) hanzi(w http.ResponseWriter, r *http.Request) {
|
||||
res, err := h.Set.Hanzi(pathWord(r))
|
||||
if err != nil {
|
||||
writeLookupErr(w, err)
|
||||
return
|
||||
}
|
||||
writeLookup(w, res)
|
||||
}
|
||||
|
||||
// providerFor returns the provider for the caller's language pair.
|
||||
//
|
||||
// The pair language is read here rather than threaded down because a word
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// The Chinese half of the lexicon: a word written in hanzi to its pinyin and
|
||||
// English senses. This is the mirror image of `gloss` — that one reads English
|
||||
// and answers in Chinese, for a Mandarin native practising English; this one
|
||||
// reads Chinese and answers in English, for the other direction of the same
|
||||
// pair (`users.direction = 'learning_pair'`).
|
||||
//
|
||||
// It is deliberately not folded into [Lexicon.load]. That method reads four
|
||||
// datasets on the first lookup of any kind, and this one is 3.1 MB gzipped that
|
||||
// only a learner-direction account will ever ask for — every other writer would
|
||||
// pay the decompression and the resident memory for a map they never touch. Its
|
||||
// own sync.Once means the cost lands on the first Chinese hover and nowhere
|
||||
// else.
|
||||
|
||||
// HanziReading is one pronunciation of a word and the senses it carries in that
|
||||
// pronunciation. A word usually has one; the ones that have two are why this is
|
||||
// a list rather than a pair of strings. 得 is dé, "to obtain", *and* de, the
|
||||
// particle that makes 说得很好 mean "speaks well" — a learner shown only the
|
||||
// first has been told something false about the sentence in front of them.
|
||||
type HanziReading struct {
|
||||
Pinyin string `json:"pinyin"`
|
||||
Senses string `json:"senses"`
|
||||
}
|
||||
|
||||
// HanziChar is one character of a word that the dictionary could not answer as
|
||||
// a whole. See [Lexicon.Hanzi].
|
||||
type HanziChar struct {
|
||||
Char string `json:"char"`
|
||||
Pinyin string `json:"pinyin"`
|
||||
Senses string `json:"senses"`
|
||||
}
|
||||
|
||||
// HanziResult is what a Chinese word lookup answers. Readings is empty for a
|
||||
// word the dictionary does not have, in which case Chars may carry the
|
||||
// character-by-character reading instead.
|
||||
type HanziResult struct {
|
||||
Word string `json:"word"`
|
||||
Readings []HanziReading `json:"readings"`
|
||||
Chars []HanziChar `json:"chars"`
|
||||
}
|
||||
|
||||
type hanziStore struct {
|
||||
once sync.Once
|
||||
err error
|
||||
// word → [[pinyin, senses], …], exactly as scripts/build_cedict.py writes it.
|
||||
entries map[string][][]string
|
||||
}
|
||||
|
||||
var hanzi hanziStore
|
||||
|
||||
func (h *hanziStore) load() {
|
||||
h.once.Do(func() {
|
||||
if err := gunzipJSON(hanziGz, &h.entries); err != nil {
|
||||
h.err = fmt.Errorf("load hanzi: %w", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// maxHanziChars caps the per-character fallback. A run longer than this is
|
||||
// almost certainly a phrase the segmenter split badly rather than a word, and
|
||||
// spelling out eight characters one at a time is a wall, not a hint.
|
||||
const maxHanziChars = 6
|
||||
|
||||
// Hanzi returns the pinyin and English senses of a Chinese word.
|
||||
//
|
||||
// There is no de-inflection walk here, and its absence is a fact about the
|
||||
// language rather than an omission: Chinese words do not inflect, so the
|
||||
// candidate forms [lookupGloss] tries for "running" → "run" have no analogue.
|
||||
// A lookup either hits the headword or it does not.
|
||||
//
|
||||
// What it does instead is fall back to the characters. The segmentation word
|
||||
// list is a superset of this dictionary — every glossable word can be
|
||||
// segmented, but jieba knows ordinary compounds CC-CEDICT has no entry for — so
|
||||
// a hover really can land on a word with nothing to say about it. Chinese
|
||||
// compounds are usually transparent from their parts (电脑 is "electric brain"),
|
||||
// which makes the character reading a genuinely useful second answer rather
|
||||
// than a consolation prize. It is returned as its own field so the surface can
|
||||
// say which of the two it is showing; a caller that only wants whole words can
|
||||
// ignore it.
|
||||
func (l *Lexicon) Hanzi(word string) (HanziResult, error) {
|
||||
hanzi.load()
|
||||
if hanzi.err != nil {
|
||||
return HanziResult{}, hanzi.err
|
||||
}
|
||||
|
||||
norm := strings.TrimSpace(word)
|
||||
res := HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}
|
||||
if norm == "" {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
if rows, ok := hanzi.entries[norm]; ok {
|
||||
res.Readings = toReadings(rows)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
chars := []rune(norm)
|
||||
if len(chars) < 2 || len(chars) > maxHanziChars {
|
||||
// A single character that missed has no parts to fall back to, and a long
|
||||
// run is not a word. Either way the honest answer is nothing.
|
||||
return res, nil
|
||||
}
|
||||
for _, r := range chars {
|
||||
if !unicode.Is(unicode.Han, r) {
|
||||
// Mixed input (a stray letter or digit inside the run) is not something
|
||||
// the character reading can explain, and guessing at the hanzi parts of
|
||||
// it would be worse than silence.
|
||||
return HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}, nil
|
||||
}
|
||||
rows, ok := hanzi.entries[string(r)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
first := toReadings(rows)
|
||||
if len(first) == 0 {
|
||||
continue
|
||||
}
|
||||
res.Chars = append(res.Chars, HanziChar{
|
||||
Char: string(r),
|
||||
Pinyin: first[0].Pinyin,
|
||||
Senses: first[0].Senses,
|
||||
})
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func toReadings(rows [][]string) []HanziReading {
|
||||
out := make([]HanziReading, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if len(row) < 2 {
|
||||
continue
|
||||
}
|
||||
out = append(out, HanziReading{Pinyin: row[0], Senses: row[1]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
// The Chinese direction of the lexicon, against the real embedded asset — not a
|
||||
// fixture. The dataset is built by scripts/build_cedict.py, which asserts its
|
||||
// own invariants at build time; what these assert is that the *lookup* over it
|
||||
// behaves, including on the entries the build script goes out of its way to keep.
|
||||
|
||||
func TestHanziLookup(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
res, err := l.Hanzi("公园")
|
||||
if err != nil {
|
||||
t.Fatalf("lookup 公园: %v", err)
|
||||
}
|
||||
if len(res.Readings) == 0 {
|
||||
t.Fatal("公园 has no readings")
|
||||
}
|
||||
// Tone marks, not the numbered pinyin CC-CEDICT stores. The number is the
|
||||
// storage format; the marks are what a learner reads.
|
||||
if got := res.Readings[0].Pinyin; got != "gōngyuán" {
|
||||
t.Errorf("公园 pinyin = %q, want gōngyuán", got)
|
||||
}
|
||||
if !strings.Contains(res.Readings[0].Senses, "park") {
|
||||
t.Errorf("公园 senses = %q, want something about a park", res.Readings[0].Senses)
|
||||
}
|
||||
// A word answered whole says nothing about its characters — the fallback is
|
||||
// the other branch, and sending both would double the payload of the common
|
||||
// case to no purpose.
|
||||
if len(res.Chars) != 0 {
|
||||
t.Errorf("a whole-word hit also returned %d characters", len(res.Chars))
|
||||
}
|
||||
}
|
||||
|
||||
// 得 is the reason readings are a list. Answered with only dé "to obtain", a
|
||||
// learner hovering it in 说得很好 has been told something false about the
|
||||
// sentence they are looking at.
|
||||
func TestHanziParticleCarriesItsGrammaticalReading(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
for _, particle := range []string{"的", "地", "得"} {
|
||||
res, err := l.Hanzi(particle)
|
||||
if err != nil {
|
||||
t.Fatalf("lookup %s: %v", particle, err)
|
||||
}
|
||||
var found bool
|
||||
for _, r := range res.Readings {
|
||||
if r.Pinyin == "de" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s never reads as neutral \"de\": %+v", particle, res.Readings)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The fallback the segmentation gap makes necessary: jieba knows ordinary
|
||||
// compounds CC-CEDICT has no headword for, so a hover can land on a real word
|
||||
// with no entry. Chinese compounds are usually transparent from their parts, so
|
||||
// the characters are a real second answer.
|
||||
func TestHanziFallsBackToCharacters(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
// Constructed rather than borrowed from the corpus: a word that CC-CEDICT
|
||||
// *does* carry would test the other branch, and which compounds it happens to
|
||||
// omit is not something a test should pin.
|
||||
const made = "猫书"
|
||||
if _, ok := hanzi.entries[made]; ok {
|
||||
t.Skipf("%s has become a real headword; pick another compound", made)
|
||||
}
|
||||
res, err := l.Hanzi(made)
|
||||
if err != nil {
|
||||
t.Fatalf("lookup %s: %v", made, err)
|
||||
}
|
||||
if len(res.Readings) != 0 {
|
||||
t.Fatalf("%s answered as a whole word: %+v", made, res.Readings)
|
||||
}
|
||||
if len(res.Chars) != 2 {
|
||||
t.Fatalf("character fallback gave %d entries, want 2: %+v", len(res.Chars), res.Chars)
|
||||
}
|
||||
if res.Chars[0].Char != "猫" || !strings.Contains(res.Chars[0].Senses, "cat") {
|
||||
t.Errorf("first character = %+v, want 猫 ~ cat", res.Chars[0])
|
||||
}
|
||||
if res.Chars[0].Pinyin != "māo" {
|
||||
t.Errorf("猫 pinyin = %q, want māo", res.Chars[0].Pinyin)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHanziMisses(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
for name, word := range map[string]string{
|
||||
// A single character with no entry has no parts to fall back to.
|
||||
"lone unknown character": "龥",
|
||||
"empty": "",
|
||||
"whitespace": " ",
|
||||
// Not Chinese at all: the English tokenizer owns these, and answering
|
||||
// would mean guessing.
|
||||
"english": "hello",
|
||||
"mixed": "猫cat",
|
||||
// Longer than a word: a bad segmentation, not something to spell out
|
||||
// character by character.
|
||||
"a whole clause": "我今天早上去公园跑步了",
|
||||
} {
|
||||
res, err := l.Hanzi(word)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if len(res.Readings) != 0 || len(res.Chars) != 0 {
|
||||
t.Errorf("%s (%q) answered with %+v / %+v", name, word, res.Readings, res.Chars)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHanziEndpoint(t *testing.T) {
|
||||
h := NewHandler(nil, NewSet(nil))
|
||||
r := chi.NewRouter()
|
||||
r.Mount("/hanzi", h.HanziRoutes())
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/"+"跑步", nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d", w.Code)
|
||||
}
|
||||
var got HanziResult
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.Word != "跑步" || len(got.Readings) == 0 || got.Readings[0].Pinyin != "pǎobù" {
|
||||
t.Fatalf("response = %+v", got)
|
||||
}
|
||||
|
||||
// A miss is a 200 with empty lists, like the other two lookups — the tooltip
|
||||
// quietly doesn't open rather than showing an error over her writing.
|
||||
w = httptest.NewRecorder()
|
||||
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/zzz", nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("miss: status = %d, want 200", w.Code)
|
||||
}
|
||||
}
|
||||
@@ -109,3 +109,13 @@ func (g glossless) Lookup(word string) (Result, error) {
|
||||
func (g glossless) Gloss(word string) (GlossResult, error) {
|
||||
return GlossResult{Word: word}, nil
|
||||
}
|
||||
|
||||
// Hanzi answers a Chinese-word lookup from the embedded CC-CEDICT map.
|
||||
//
|
||||
// It is on the Set rather than on [Provider] because it is not the same
|
||||
// question the other two ask. Lookup and Gloss vary by pair — which is why they
|
||||
// are behind an interface with two implementations — while this one is asked of
|
||||
// Chinese or not at all: the learner direction exists for exactly one pair (see
|
||||
// auth.learnerPairs), and DreamDict's own CC-CEDICT would be a second copy of
|
||||
// the same dictionary, chosen by a rule with one branch.
|
||||
func (s *Set) Hanzi(word string) (HanziResult, error) { return s.embedded.Hanzi(word) }
|
||||
|
||||
@@ -201,3 +201,36 @@ func TestOfflineCardWinsSpanCollision(t *testing.T) {
|
||||
t.Errorf("the exact offline card should own the span, got %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestOfflineHanziFindingStaysMechanics: a 错别字 the Chinese rule pack found —
|
||||
// both halves written in hanzi — files as an ordinary mechanics row.
|
||||
//
|
||||
// The check is worth its own test because there is a rule one layer over that
|
||||
// would plausibly claim it. `isTranslation` re-labels an edit whose original
|
||||
// reads as the writer's language and whose replacement reads as English, which
|
||||
// is exactly how a zh-pair writer's quoted Chinese becomes a 'translate' card.
|
||||
// A wrong-character fix looks like the first half of that and nothing like the
|
||||
// second: 己经 → 已经 never leaves Chinese. It must stay a tidy-up in her own
|
||||
// sentence, on the same rail as a doubled word, with no rendering-into-English
|
||||
// implied anywhere.
|
||||
func TestOfflineHanziFindingStaysMechanics(t *testing.T) {
|
||||
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":1,"to":3,"original":"己经","replacement":"已经","explanation":"已经 (already) takes 已","type":"mechanics"}
|
||||
]`)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want the one finding, got %+v", got)
|
||||
}
|
||||
if got[0].Type != db.SuggestionTypeMechanics {
|
||||
t.Errorf("hanzi fix filed as %q, want %q", got[0].Type, db.SuggestionTypeMechanics)
|
||||
}
|
||||
if got[0].Source != db.SuggestionSourceLocal {
|
||||
t.Errorf("source = %q, want %q", got[0].Source, db.SuggestionSourceLocal)
|
||||
}
|
||||
// The characters survive the round trip intact — a mangled span here would
|
||||
// replace the wrong characters in her document.
|
||||
if got[0].Original != "己经" || got[0].Replacement != "已经" {
|
||||
t.Errorf("round-tripped as %q → %q", got[0].Original, got[0].Replacement)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the two Chinese assets the learner direction of the zh pair needs.
|
||||
|
||||
Why two, and why they are split the way they are
|
||||
------------------------------------------------
|
||||
Every other pair Petal ships needs one asset: a word list the browser loads so
|
||||
it can underline. Chinese needs two, because the browser and the server want
|
||||
different halves of the same dictionary and for different reasons.
|
||||
|
||||
* **The browser needs a word list, and it needs it offline.** Chinese is
|
||||
written without spaces, so there is no such thing as "the word under the
|
||||
cursor" until something segments the sentence. Every ESL surface Petal
|
||||
already has — the hover gloss, the right-click lookup, Ctrl/Cmd+D, the
|
||||
vocabulary garden capture — is built on `wordAt`, and `wordAt` is a regex
|
||||
over Latin letters. Segmentation is what replaces that regex, it runs on
|
||||
every hover, and a round-trip per hover is not a hover. So the word list
|
||||
ships to the browser: `web/public/dictionaries/zh/words.txt`.
|
||||
|
||||
* **The server holds the whole dictionary.** Pinyin and English senses are
|
||||
only ever wanted one word at a time, in answer to a hover or a click, which
|
||||
is exactly what `/api/gloss/{word}` already does for the other direction. So
|
||||
the readings stay in the binary — `internal/lexicon/data/hanzi.json.gz` —
|
||||
where their size costs a browser nothing.
|
||||
|
||||
That split is what makes the coverage decisions below come out *opposite* to
|
||||
each other, and both are deliberate.
|
||||
|
||||
Two sources, because neither one has both halves
|
||||
------------------------------------------------
|
||||
* **CC-CEDICT** (CC BY-SA 4.0, https://www.mdbg.net/) has the headwords,
|
||||
pinyin and English senses, and no frequency information at all.
|
||||
* **jieba's `dict.txt`** (MIT, https://github.com/fxsjy/jieba) has ~349k
|
||||
headwords with corpus frequencies, and no definitions.
|
||||
|
||||
Segmentation needs the frequencies: the standard algorithm is a shortest-path
|
||||
walk over log-probabilities, not longest-match, and without frequencies the
|
||||
classic ambiguities go the wrong way. The client list therefore carries
|
||||
`word freq` per line; the gloss map carries readings.
|
||||
|
||||
The size decision is the client list, and it is a size decision only
|
||||
--------------------------------------------------------------------
|
||||
Measured on ordinary learner prose, the segmentation produced by the full jieba
|
||||
dictionary (381,886 hanzi headwords once CC-CEDICT is unioned in) and by a
|
||||
frequency-gated one is **identical**, including on the textbook ambiguities
|
||||
(研究生命的起源, 乒乓球拍卖完了, 南京市长江大桥). What the long tail contains is
|
||||
rare proper nouns, and the max-probability walk almost never chooses one: a
|
||||
freq-3 name loses to two common words every time. The cases where a missing word
|
||||
does change the answer degrade *gracefully* — the sentence splits into smaller
|
||||
real words, which is a slightly clumsier gloss, not a wrong underline.
|
||||
|
||||
So the gate is set where the size is, at **freq >= 5**: 188,522 words, ~0.97 MB
|
||||
gzipped over the wire, in line with fr (1.19 MB) and es (1.74 MB) rather than in
|
||||
excess of them. Every CC-CEDICT headword is unioned back in regardless of
|
||||
frequency, so the segmenter can always see a word the server can explain.
|
||||
|
||||
The gloss map is gated by nothing, for the opposite reason
|
||||
-----------------------------------------------------------
|
||||
The es phase settled that a *spelling* dictionary should hold the union of every
|
||||
variety, because its only power is to underline and it must not underline
|
||||
correct writing. This asset's only power is to **explain**, and the word a
|
||||
learner stops on is precisely the one they do not know — which is to say, the
|
||||
rare one. Trimming this by frequency would remove exactly the entries it exists
|
||||
for. All 113,637 glossable headwords ship, ~3.1 MB gzipped, which is less than
|
||||
half of what `synonyms.json.gz` has embedded since Phase 9.
|
||||
|
||||
Simplified only, and said out loud
|
||||
-----------------------------------
|
||||
The zh langpack is written in simplified characters and jieba's frequencies are
|
||||
counted over simplified text, so the traditional headword in each CC-CEDICT line
|
||||
is dropped and simplified is what both assets are keyed by. Glossing traditional
|
||||
would be nearly free *here* and useless in the app: nothing would segment it, so
|
||||
nothing would ever ask. Traditional support is a real feature and it starts with
|
||||
a traditional word list, not with this file.
|
||||
|
||||
Usage:
|
||||
curl -sL https://www.mdbg.net/chinese/export/cedict/cedict_1_0_ts_utf-8_mdbg.txt.gz | gunzip > cedict.txt
|
||||
curl -sL https://raw.githubusercontent.com/fxsjy/jieba/master/jieba/dict.txt -o jieba.txt
|
||||
python3 scripts/build_cedict.py cedict.txt jieba.txt \
|
||||
web/public/dictionaries/zh/words.txt.gz \
|
||||
internal/lexicon/data/hanzi.json.gz
|
||||
"""
|
||||
import gzip
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
# Frequency gate for the *client* list only (see the module docstring). Words
|
||||
# below it survive if CC-CEDICT knows them, so "segmentable" is always a superset
|
||||
# of "glossable" and a hover can never land on a word the server cannot explain.
|
||||
MIN_FREQ = 5
|
||||
|
||||
# A CC-CEDICT headword we keep must be nothing but han characters. This drops the
|
||||
# entries that are really English or numerals with a Chinese gloss attached
|
||||
# ("AA制", "PM2.5", "11区"): the segmenter walks runs of hanzi, so a mixed
|
||||
# headword can never be matched anyway, and a Latin one would collide with the
|
||||
# English tokenizer that is still running on the same paragraph.
|
||||
HANZI_ONLY = re.compile(r'^[一-鿿]+$')
|
||||
|
||||
CEDICT_LINE = re.compile(r'^(\S+) (\S+) \[(.*?)\] /(.*)/$')
|
||||
|
||||
# At most this many readings per word, and this many senses per reading. Two
|
||||
# readings is not an arbitrary cap: it is what the particles need. 得 is dé "to
|
||||
# obtain" *and* de, the complement marker — and a learner who hovers 得 in
|
||||
# 说得很好 and is told only "to obtain" has been actively misinformed. Beyond two
|
||||
# the tail is dialect and surnames, which crowd out the sense actually wanted.
|
||||
MAX_READINGS = 2
|
||||
MAX_SENSES = 3
|
||||
MAX_SENSE_CHARS = 110
|
||||
|
||||
# Senses that describe the *dictionary* rather than the word. A learner hovering
|
||||
# a word wants to know what it means, not that it is an orthographic variant of
|
||||
# another headword they also do not know.
|
||||
SKIP_SENSE_PREFIXES = ('variant of', 'old variant', 'see ', 'used in', 'abbr. for')
|
||||
|
||||
# ── pinyin: numbered syllables to tone marks ────────────────────────────────
|
||||
# CC-CEDICT stores "gong1 yuan2". A learner reading their own writing back wants
|
||||
# gōngyuán: the tone mark is the part that is hard to remember and the part that
|
||||
# changes the word. The placement rule is the standard one — a/o/e take the mark
|
||||
# if present, otherwise the last vowel of the final — and it is small enough to
|
||||
# do here rather than to take a dependency for.
|
||||
TONE_VOWELS = {
|
||||
'a': 'āáǎà',
|
||||
'e': 'ēéěè',
|
||||
'i': 'īíǐì',
|
||||
'o': 'ōóǒò',
|
||||
'u': 'ūúǔù',
|
||||
'ü': 'ǖǘǚǜ',
|
||||
}
|
||||
|
||||
SYLLABLE = re.compile(r'^([a-zA-Zü:]+)([1-5])$')
|
||||
|
||||
|
||||
def tone_mark(syllable: str) -> str:
|
||||
"""One numbered pinyin syllable to its tone-marked form."""
|
||||
m = SYLLABLE.match(syllable)
|
||||
if not m:
|
||||
# Punctuation, a bare letter (CC-CEDICT writes "X" for unknown), or an
|
||||
# already-marked syllable: pass it through rather than mangling it.
|
||||
return syllable
|
||||
body, tone = m.group(1), int(m.group(2))
|
||||
# CC-CEDICT writes ü as "u:" and, in a few entries, as "v".
|
||||
body = body.replace('u:', 'ü').replace('U:', 'Ü').replace('v', 'ü').replace('V', 'Ü')
|
||||
if tone == 5: # neutral tone carries no mark
|
||||
return body
|
||||
low = body.lower()
|
||||
idx = -1
|
||||
for vowel in ('a', 'o', 'e'):
|
||||
idx = low.find(vowel)
|
||||
if idx >= 0:
|
||||
break
|
||||
if idx < 0:
|
||||
# No a/o/e: the mark goes on the last of i/u/ü (liú, guǐ, nǚ).
|
||||
idx = max(low.rfind('i'), low.rfind('u'), low.rfind('ü'))
|
||||
if idx < 0:
|
||||
return body
|
||||
marked = TONE_VOWELS[low[idx]][tone - 1]
|
||||
if body[idx].isupper():
|
||||
marked = marked.upper()
|
||||
return body[:idx] + marked + body[idx + 1:]
|
||||
|
||||
|
||||
def pinyin(numbered: str) -> str:
|
||||
"""A whole CC-CEDICT pinyin field to tone marks, syllables joined up.
|
||||
|
||||
Joined rather than spaced because that is how a word is written when it is
|
||||
being read as a word (gōngyuán, not gōng yuán); the spaces in the source are
|
||||
a storage convention, not orthography.
|
||||
"""
|
||||
return ''.join(tone_mark(s) for s in numbered.split())
|
||||
|
||||
|
||||
def clean_senses(raw: list[str]) -> list[str]:
|
||||
"""Strip the apparatus CC-CEDICT carries for lexicographers, not learners."""
|
||||
out = []
|
||||
for sense in raw:
|
||||
# "CL:座[zuo4]" is the measure-word field, useful and not a definition.
|
||||
sense = re.sub(r'\s*CL:.*$', '', sense).strip()
|
||||
# Bracketed pinyin cross-references ("abbr. for 的士[di1 shi4]").
|
||||
sense = re.sub(r'\[[a-zA-Z0-9: ]+\]', '', sense).strip()
|
||||
# Both edits cut inside parentheses — "cat (CL:只)" loses its closing
|
||||
# bracket and leaves "cat (" on the card. Drop a dangling opener rather
|
||||
# than trying to rebalance: what it introduced is gone.
|
||||
if sense.count('(') > sense.count(')'):
|
||||
sense = re.sub(r'\s*\([^()]*$', '', sense).strip()
|
||||
if not sense or sense.startswith(SKIP_SENSE_PREFIXES):
|
||||
continue
|
||||
out.append(sense)
|
||||
return out
|
||||
|
||||
|
||||
def read_cedict(path: str) -> dict[str, list[tuple[str, list[str]]]]:
|
||||
entries: dict[str, list[tuple[str, list[str]]]] = {}
|
||||
for line in open(path, encoding='utf-8'):
|
||||
if line.startswith('#'):
|
||||
continue
|
||||
m = CEDICT_LINE.match(line.strip())
|
||||
if not m:
|
||||
continue
|
||||
_traditional, simplified, py, defs = m.groups()
|
||||
if not HANZI_ONLY.match(simplified):
|
||||
continue
|
||||
entries.setdefault(simplified, []).append((py, defs.split('/')))
|
||||
return entries
|
||||
|
||||
|
||||
def read_jieba(path: str) -> dict[str, int]:
|
||||
freqs: dict[str, int] = {}
|
||||
for line in open(path, encoding='utf-8'):
|
||||
parts = line.split()
|
||||
if len(parts) >= 2 and HANZI_ONLY.match(parts[0]):
|
||||
freqs[parts[0]] = int(parts[1])
|
||||
return freqs
|
||||
|
||||
|
||||
# ── the assertions ──────────────────────────────────────────────────────────
|
||||
# The es phase's lesson, in the place it applies here: a check that every
|
||||
# plausible input would pass is not a check. The Spanish MUST_ACCEPT list
|
||||
# asserted vocabulary that all twenty-four builds carried, so it could not tell
|
||||
# them apart. These assert the things that actually go wrong in *this* build —
|
||||
# a mis-parsed pinyin field, a missing particle reading, a word list gated so
|
||||
# hard the segmenter can no longer see a word the server can explain.
|
||||
|
||||
# Tone marking, including the three cases the placement rule exists for.
|
||||
MUST_MARK = {
|
||||
'gong1 yuan2': 'gōngyuán', # a/o/e rule, first syllable
|
||||
'pao3 bu4': 'pǎobù',
|
||||
'liu2': 'liú', # no a/o/e: mark the *last* of i/u
|
||||
'gui3': 'guǐ',
|
||||
'nu:3': 'nǚ', # u: is ü
|
||||
'lu:e4': 'lüè', # ü and an e in the same syllable: e wins
|
||||
'de5': 'de', # neutral tone takes no mark at all
|
||||
'Zhong1 wen2': 'Zhōngwén', # capitalised headword keeps its capital
|
||||
}
|
||||
|
||||
# The particles the 错别字 rules are about must each carry the *grammatical*
|
||||
# reading, not only the lexical one. 的/地/得 are the single most confused triple
|
||||
# in written Chinese and all three are neutral-tone "de" in the use that matters;
|
||||
# an entry that only knows 得 as dé is worse than no entry.
|
||||
MUST_READ_DE = ('的', '地', '得')
|
||||
|
||||
# Words the segmenter must be able to see. 图书馆 and 乒乓球 are ordinary
|
||||
# vocabulary; 我 and 的 are the two commonest words in the language and a gate
|
||||
# that dropped either would be visibly broken; 的士 is a CC-CEDICT headword rare
|
||||
# enough to fall below the frequency gate, and is here to prove the union.
|
||||
MUST_SEGMENT = ('我', '的', '图书馆', '乒乓球', '公园', '的士')
|
||||
|
||||
|
||||
def check(words: dict[str, int], gloss: dict[str, list[list[str]]]) -> None:
|
||||
for numbered, want in MUST_MARK.items():
|
||||
got = pinyin(numbered)
|
||||
assert got == want, f'pinyin({numbered!r}) = {got!r}, want {want!r}'
|
||||
|
||||
for particle in MUST_READ_DE:
|
||||
readings = gloss.get(particle)
|
||||
assert readings, f'{particle} has no gloss entry at all'
|
||||
assert any(r[0] == 'de' for r in readings), \
|
||||
f'{particle} never reads as neutral "de": {readings}'
|
||||
|
||||
for word in MUST_SEGMENT:
|
||||
assert word in words, f'{word} missing from the segmentation list'
|
||||
|
||||
# The invariant the two gates exist to keep: everything the server can
|
||||
# explain, the browser can find.
|
||||
missing = [w for w in gloss if w not in words]
|
||||
assert not missing, f'{len(missing)} glossable words are unsegmentable, e.g. {missing[:5]}'
|
||||
|
||||
# Nothing Latin leaked into either asset (see HANZI_ONLY).
|
||||
for name, keys in (('words', words), ('gloss', gloss)):
|
||||
bad = [k for k in keys if not HANZI_ONLY.match(k)]
|
||||
assert not bad, f'non-hanzi headwords in {name}: {bad[:5]}'
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) != 5:
|
||||
sys.exit(__doc__.strip().rsplit('Usage:', 1)[-1].strip())
|
||||
cedict_path, jieba_path, words_out, gloss_out = sys.argv[1:]
|
||||
|
||||
entries = read_cedict(cedict_path)
|
||||
freqs = read_jieba(jieba_path)
|
||||
|
||||
# The client list: frequency-gated, then unioned with every glossable word.
|
||||
# A CC-CEDICT word jieba has never seen gets frequency 1 — real, and rare
|
||||
# enough that the max-probability walk will only choose it when nothing else
|
||||
# fits, which is exactly the standing it should have.
|
||||
words = {w: f for w, f in freqs.items() if f >= MIN_FREQ}
|
||||
for w in entries:
|
||||
words.setdefault(w, 1)
|
||||
|
||||
gloss: dict[str, list[list[str]]] = {}
|
||||
for word, rows in entries.items():
|
||||
readings: list[list[str]] = []
|
||||
for numbered, defs in rows:
|
||||
senses = clean_senses(defs)
|
||||
if not senses:
|
||||
continue
|
||||
readings.append([pinyin(numbered), '; '.join(senses[:MAX_SENSES])[:MAX_SENSE_CHARS]])
|
||||
if len(readings) == MAX_READINGS:
|
||||
break
|
||||
if readings:
|
||||
gloss[word] = readings
|
||||
|
||||
check(words, gloss)
|
||||
|
||||
# Gzipped on disk, like the pt-PT/fr/es word lists: the browser inflates it
|
||||
# with DecompressionStream (see useSpellChecker.fetchText), which costs no
|
||||
# bundle bytes, and 0.97 MB over the wire rather than 2.23 MB is the whole
|
||||
# difference between this and the biggest asset Petal ships.
|
||||
body = ('\n'.join(f'{w} {words[w]}' for w in sorted(words)) + '\n').encode('utf-8')
|
||||
with gzip.open(words_out, 'wb', compresslevel=9) as fh:
|
||||
fh.write(body)
|
||||
|
||||
payload = json.dumps(gloss, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
|
||||
with gzip.open(gloss_out, 'wb', compresslevel=9) as fh:
|
||||
fh.write(payload)
|
||||
|
||||
print(f'{words_out}: {len(words)} words, {len(body) / 1e6:.2f} MB raw, '
|
||||
f'{len(gzip.compress(body, 9)) / 1e6:.2f} MB gzipped')
|
||||
print(f'{gloss_out}: {len(gloss)} entries, {len(gzip.compress(payload, 9)) / 1e6:.2f} MB gzipped')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -89,6 +89,45 @@ through the packaging rather than through the model. The authentic dictionary is
|
||||
the Projecto Natura one (Universidade do Minho) that LibreOffice ships and Debian
|
||||
packages as `hunspell-pt-pt`; its aff declares `LANG pt_PT`.
|
||||
|
||||
**es: the wrong country again, hidden one layer further down.** Spanish looked
|
||||
like it would repeat the pt trap — `hunspell-es` installs twenty country codes,
|
||||
`es_AR` through `es_VE` — and then looked like it did not, because every one of
|
||||
them is a symlink to a single `es_ES.aff`/`es_ES.dic`. Both readings were wrong.
|
||||
Debian collapses the twenty because it ships **one** of upstream's builds, and
|
||||
the one it ships is the **peninsular** `es_ES`. RLA (Santiago Bosio's project,
|
||||
`sbosio/rla-es`) publishes twenty-four dictionaries per release: one per country,
|
||||
plus a **generic `es`** that is the union of all of them. Debian packages neither
|
||||
the generic one nor a choice — it packages Spain, under a name that reads like
|
||||
"Spanish".
|
||||
|
||||
Measured against the v2.9 release: Debian's file is 659,085 expanded forms and
|
||||
upstream `es_ES` is 659,018; the generic `es` is **717,640**. The 58,622-form
|
||||
difference is almost entirely **voseo** — `vení`, `tenés`, `querés`, `sabés`,
|
||||
`andá` — the present tense of most of Latin America, which Debian's package
|
||||
rejects as misspellings. Petal ships the **generic** build.
|
||||
|
||||
**Vocabulary cannot detect this and morphology can.** The first version of the es
|
||||
profile asserted the pan-Hispanic lexicon — *computadora* and *ordenador*, *papa*
|
||||
and *patata* — and passed happily on the peninsular file, because **every** RLA
|
||||
variant carries the full pan-Hispanic vocabulary; only the verb paradigms are
|
||||
localised. The `REP` table is no help either: its `ll`↔`y` and `ás`↔`az` entries
|
||||
look like evidence of yeísmo and seseo, but they are shared by all twenty-four
|
||||
builds. What separates them is exactly two things, and the profile now demands
|
||||
both at once: **voseo** (absent from `es_ES`) and **vosotros** (largely absent
|
||||
from `es_MX`). Only the generic build has both, so only the generic build passes.
|
||||
|
||||
This is the same decision fr made between `-classical` and `-revised`, arriving
|
||||
by a different road. The only thing this dictionary can do is underline
|
||||
something, and *tienes* and *tenés* are both correct Spanish taught in different
|
||||
countries — so Petal takes the build that accepts every variety rather than one
|
||||
that makes a writer wrong for where she is from. Nothing is generated to get
|
||||
there: the forms come from a real upstream package, which is what lets the
|
||||
MUST_ACCEPT list prove which package it was.
|
||||
|
||||
Licensing note: RLA is tri-licensed GPL-3+ / LGPL-3+ / MPL-1.1+; Petal
|
||||
redistributes under the MPL. The upstream README and LICENSE are vendored beside
|
||||
the output.
|
||||
|
||||
**fr: the wrong side of an argument the French have not settled.** The regional
|
||||
question turns out to be a non-question — Debian's `fr_FR`, `fr_CA`, `fr_BE`,
|
||||
`fr_CH`, `fr_LU` and `fr_MC` are all symlinks to one `fr.dic`, so unlike pt there
|
||||
@@ -114,6 +153,14 @@ Usage
|
||||
src/usr/share/hunspell/fr.aff \\
|
||||
src/usr/share/hunspell/fr.dic \\
|
||||
web/public/dictionaries/fr
|
||||
|
||||
Spanish does not come from Debian — see below; `hunspell-es` is the peninsular
|
||||
build. Take the generic dictionary from an upstream release instead:
|
||||
|
||||
curl -LO https://github.com/sbosio/rla-es/releases/download/v2.9/es.oxt
|
||||
unzip -d src es.oxt # an .oxt is a zip
|
||||
python3 scripts/build_hunspell_dictionary.py es \\
|
||||
src/es.aff src/es.dic web/public/dictionaries/es
|
||||
"""
|
||||
import gzip
|
||||
import os
|
||||
@@ -404,6 +451,52 @@ PROFILES = {
|
||||
"reject": ("jardinn", "écrivaitz", "xyzzyque"),
|
||||
"wrong": "this does not look like the comprehensive French dictionary",
|
||||
},
|
||||
# The generic RLA build, and the accept list is written to reject the four
|
||||
# neighbouring builds rather than to describe this one.
|
||||
#
|
||||
# The first version of this profile demanded *computadora* and *ordenador*,
|
||||
# *papa* and *patata*, and passed — on the peninsular file, because **every**
|
||||
# RLA variant carries the whole pan-Hispanic vocabulary. Vocabulary does not
|
||||
# discriminate here at all; only morphology does, and it discriminates
|
||||
# completely:
|
||||
#
|
||||
# * **voseo** (`vení`, `tenés`, `querés`) is in `es` and `es_AR` and not in
|
||||
# `es_ES` or Debian's package. Demanding it rejects the peninsular build.
|
||||
# * **vosotros** (`tenéis`, `escribid`) is in `es`, `es_AR` and `es_ES`, and
|
||||
# largely absent from `es_MX`. Demanding it rejects the Mexican build.
|
||||
#
|
||||
# Requiring both at once leaves exactly one package standing: the generic
|
||||
# `es`, which is the only one that accepts every variety of Spanish. That is
|
||||
# the same reason fr ships `-comprehensive` — the only thing this dictionary
|
||||
# can do is underline something, and *tienes* and *tenés* are both correct
|
||||
# Spanish taught in different countries.
|
||||
#
|
||||
# The rest are shape checks: `escribiésemos` is the -se imperfect subjunctive,
|
||||
# `dámelo` proves the enclitic pronoun rules ran, and `jardín`/`niño` prove
|
||||
# FLAG UTF-8 was read as characters rather than bytes.
|
||||
"es": {
|
||||
"accept": (
|
||||
# Rejects es_ES and Debian's hunspell-es.
|
||||
"vení", "tenés", "querés", "sabés", "andá",
|
||||
# Rejects es_MX.
|
||||
"tenéis", "escribid",
|
||||
# Rejects es_AR, which has both voseo and vosotros and would
|
||||
# otherwise pass. Caribbean and Andean everyday words: the generic
|
||||
# build is the union of all twenty-four, so it is the only one that
|
||||
# holds another region's vocabulary as well as its own.
|
||||
"arepa", "chévere", "bacán",
|
||||
# Pan-Hispanic vocabulary. These pass on every RLA build, so they
|
||||
# prove nothing on their own — kept because a source that stopped
|
||||
# being RLA at all would fail them.
|
||||
"computadora", "ordenador", "papa", "patata", "jugo", "zumo",
|
||||
# Morphology and encoding.
|
||||
"escribiéramos", "escribiésemos", "escríbeme", "dámelo",
|
||||
"jardín", "niño", "corazón",
|
||||
),
|
||||
"reject": ("jardinn", "escribiz", "xyzzyque", "haiga"),
|
||||
"wrong": "this is not the generic RLA build (a per-country one accepts "
|
||||
"only some of these)",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
||||
Generated
+18
@@ -30,6 +30,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/vite": "^4.0.0",
|
||||
"@types/node": "^26.1.2",
|
||||
"@types/react": "^19.1.0",
|
||||
"@types/react-dom": "^19.1.0",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
@@ -2107,6 +2108,16 @@
|
||||
"integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "26.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.2.tgz",
|
||||
"integrity": "sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~8.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/react": {
|
||||
"version": "19.2.17",
|
||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz",
|
||||
@@ -3559,6 +3570,13 @@
|
||||
"integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "8.3.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
|
||||
"integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/update-browserslist-db": {
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz",
|
||||
|
||||
@@ -33,6 +33,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/vite": "^4.0.0",
|
||||
"@types/node": "^26.1.2",
|
||||
"@types/react": "^19.1.0",
|
||||
"@types/react-dom": "^19.1.0",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
Spanish spelling dictionary
|
||||
===========================
|
||||
|
||||
The word list in `es.dic.gz` and the suggestion directives in `es.aff` are
|
||||
derived from the **generic** Spanish Hunspell dictionary published by the RLA-ES
|
||||
project ("Recursos Lingüísticos Abiertos del Español"), release v2.9.
|
||||
|
||||
Copyright (C) Santiago Bosio and the RLA-ES contributors
|
||||
|
||||
License: GPL-3+ or LGPL-3+ or MPL-1.1+
|
||||
Tri-licensed; you may choose freely among the three. Petal
|
||||
redistributes under the MPL. Full texts:
|
||||
https://www.gnu.org/licenses/gpl-3.0.en.html
|
||||
https://www.gnu.org/licenses/lgpl-3.0.en.html
|
||||
https://www.mozilla.org/en-US/MPL/1.1/
|
||||
|
||||
Upstream: https://github.com/sbosio/rla-es
|
||||
Source: https://github.com/sbosio/rla-es/releases/download/v2.9/es.oxt
|
||||
(an .oxt is a zip; es.aff and es.dic are at its root)
|
||||
|
||||
Not the Debian package, and that is the point
|
||||
---------------------------------------------
|
||||
`hunspell-es` looks like the obvious source and is the wrong one. It installs
|
||||
twenty country codes — `es_AR` through `es_VE` — all symlinked to a single file,
|
||||
which reads like "one pan-Hispanic dictionary". It is not. RLA publishes
|
||||
twenty-four dictionaries per release: one per country, plus a **generic `es`**
|
||||
that is the union of all of them, and Debian ships the **peninsular `es_ES`**
|
||||
build under the collapsed name.
|
||||
|
||||
Measured against v2.9, expanded to surface forms:
|
||||
|
||||
Debian hunspell-es 659,085 forms voseo: no vosotros: yes
|
||||
upstream es_ES 659,018 forms voseo: no vosotros: yes
|
||||
upstream es_MX 554,923 forms voseo: no vosotros: no
|
||||
upstream es_AR 669,605 forms voseo: yes vosotros: yes
|
||||
upstream es (generic) 717,640 forms voseo: yes vosotros: yes <-- this
|
||||
|
||||
The 58,622-form gap between Debian's file and the generic one is essentially the
|
||||
**voseo** paradigm — `vení`, `tenés`, `querés`, `sabés`, `andá` — the ordinary
|
||||
present tense of Argentina, Uruguay, Paraguay and much of Central America. Under
|
||||
the Debian package, a writer using it would have had her own verbs underlined as
|
||||
misspellings.
|
||||
|
||||
Why the generic build rather than one country
|
||||
---------------------------------------------
|
||||
The only thing this dictionary can do is underline something. *Tienes* and
|
||||
*tenés* are both correct Spanish, taught in different countries, and a writing
|
||||
companion has no business marking one of them wrong — the same reasoning that
|
||||
makes the French dictionary here the `-comprehensive` packaging rather than
|
||||
`-classical` or `-revised`. The generic build accepts every variety, so Petal
|
||||
underlines only what no Spanish speaker anywhere would write.
|
||||
|
||||
How the build proves it got this file
|
||||
-------------------------------------
|
||||
Vocabulary cannot tell these builds apart: *every* RLA variant carries the full
|
||||
pan-Hispanic lexicon, so *computadora* alongside *ordenador* passes on the
|
||||
peninsular file too. (The `REP` table is likewise no evidence — its `ll`/`y` and
|
||||
`ás`/`az` entries look like yeísmo and seseo but are shared by all builds.) Only
|
||||
the verb paradigms are localised, so the `es` profile in
|
||||
`scripts/build_hunspell_dictionary.py` demands, all at once:
|
||||
|
||||
* **voseo** (`vení`, `tenés`, `querés`) — rejects `es_ES` and Debian's package;
|
||||
* **vosotros** (`tenéis`, `escribid`) — rejects `es_MX`;
|
||||
* **another region's everyday words** (`arepa`, `chévere`, `bacán`) — rejects
|
||||
`es_AR`, which has both paradigms and would otherwise pass.
|
||||
|
||||
Only the generic build satisfies all three. Each of the four neighbouring builds
|
||||
was run through the profile and confirmed to fail.
|
||||
@@ -0,0 +1,29 @@
|
||||
SET UTF-8
|
||||
TRY aeroinsctldumpbgfvhzóíjáqéñxyúükwAEROINSCTLDUMPBGFVHZÓÍJÁQÉÑXYÚÜKW
|
||||
REP 19
|
||||
REP ás az
|
||||
REP az ás
|
||||
REP cc x
|
||||
REP és ez
|
||||
REP ez és
|
||||
REP güe hue
|
||||
REP güi hui
|
||||
REP hue güe
|
||||
REP hui güi
|
||||
REP ís iz
|
||||
REP ío ido
|
||||
REP ke que
|
||||
REP ki qui
|
||||
REP ll y
|
||||
REP mb nv
|
||||
REP nv mb
|
||||
REP seci cesi
|
||||
REP x cc
|
||||
REP y ll
|
||||
MAP 6
|
||||
MAP aáAÁ
|
||||
MAP eéEÉ
|
||||
MAP iíIÍ
|
||||
MAP oóOÓ
|
||||
MAP uúüUÚÜ
|
||||
MAP nñNÑ
|
||||
Binary file not shown.
@@ -0,0 +1,59 @@
|
||||
Chinese word list (segmentation)
|
||||
================================
|
||||
|
||||
`words.txt.gz` is not a spelling dictionary — Chinese has no spelling to check
|
||||
in the Hunspell sense. It is the word list Petal's segmenter walks, so that a
|
||||
sentence written without spaces has words in it to hover, look up and capture.
|
||||
Each line is `word frequency`. See scripts/build_cedict.py for how it is built
|
||||
and why it is gated where it is.
|
||||
|
||||
It is derived from two upstream sources, both redistributable, both credited
|
||||
here because the file itself has no room for a header.
|
||||
|
||||
|
||||
CC-CEDICT — the headwords
|
||||
-------------------------
|
||||
Community maintained free Chinese-English dictionary, published by MDBG.
|
||||
https://www.mdbg.net/chinese/dictionary?page=cedict
|
||||
|
||||
Licensed under the Creative Commons Attribution-ShareAlike 4.0 International
|
||||
License — https://creativecommons.org/licenses/by-sa/4.0/
|
||||
|
||||
Referenced works:
|
||||
CEDICT — Copyright (C) 1997, 1998 Paul Andrew Denisowski
|
||||
|
||||
CC-CEDICT is also the source of `internal/lexicon/data/hanzi.json.gz`, the
|
||||
pinyin and English senses embedded in the Petal binary. The same attribution and
|
||||
the same ShareAlike terms apply to that file; it is named here because it has
|
||||
nowhere of its own to say so.
|
||||
|
||||
|
||||
jieba — the frequencies
|
||||
-----------------------
|
||||
"结巴" Chinese word segmentation, by Sun Junyi.
|
||||
https://github.com/fxsjy/jieba
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2013 Sun Junyi
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
Only the word/frequency columns are used; jieba's part-of-speech tags and its
|
||||
algorithm are not (Petal's segmenter is its own, in web/src/lib/segment.ts).
|
||||
Binary file not shown.
+35
-8
@@ -3,6 +3,7 @@ import { api, type DocSummary, type DocUpdate, type Document, type Suggestion, t
|
||||
import { useAutoSave } from './hooks/useAutoSave'
|
||||
import { findingKey, useCheckpoint } from './hooks/useCheckpoint'
|
||||
import { useSpellChecker } from './hooks/useSpellChecker'
|
||||
import { useSegmenter } from './hooks/useSegmenter'
|
||||
import { useTags } from './hooks/useTags'
|
||||
import { DocList } from './components/DocList/DocList'
|
||||
import { EditorCore, type EditorChange } from './components/Editor/EditorCore'
|
||||
@@ -22,6 +23,7 @@ import { PetalFall } from './effects/PetalFall'
|
||||
import { usePack } from './i18n'
|
||||
import { useNightMode } from './hooks/useNightMode'
|
||||
import { playSuggestionSound } from './audio/sounds'
|
||||
import { fromIME } from './lib/ime'
|
||||
|
||||
export default function App() {
|
||||
const updateAvailable = useVersionWatch()
|
||||
@@ -31,7 +33,7 @@ export default function App() {
|
||||
const night = useNightMode()
|
||||
// Who's writing, and whether the server still recognises them. `signedOut`
|
||||
// flips the moment any call comes back 401.
|
||||
const { me, signedOut } = useSession()
|
||||
const { me, signedOut, setDirection, setPair } = useSession()
|
||||
const t = usePack()
|
||||
// A real account to sign out of, as opposed to the hardcoded local user a
|
||||
// build without auth configured runs as.
|
||||
@@ -80,6 +82,13 @@ export default function App() {
|
||||
}, [])
|
||||
|
||||
const { status, schedule, saveNow } = useAutoSave(currentDoc?.id ?? null)
|
||||
// The Chinese word list, for a writer going the other way through the zh pair.
|
||||
// Gated on the account's own setting rather than on anything in the text: a
|
||||
// Mandarin native drafting English quotes Chinese constantly, and none of that
|
||||
// is what segmentation is for. Declared above the checkpoint because the
|
||||
// offline 错别字 pass reads it.
|
||||
const segmenter = useSegmenter(me?.direction === 'learning_pair')
|
||||
|
||||
const {
|
||||
suggestions,
|
||||
checking,
|
||||
@@ -91,7 +100,7 @@ export default function App() {
|
||||
runCollocation,
|
||||
removeSuggestion,
|
||||
resolveServerId,
|
||||
} = useCheckpoint(currentDoc?.id ?? null)
|
||||
} = useCheckpoint(currentDoc?.id ?? null, segmenter)
|
||||
// Browser-side spell checker — loads the en-US dictionary once per session.
|
||||
const { checker: spellChecker, addWord } = useSpellChecker()
|
||||
// The tag roster (with counts). Assignments live on the doc summaries below.
|
||||
@@ -323,14 +332,25 @@ export default function App() {
|
||||
|
||||
const handleEditorChange = useCallback(
|
||||
(change: EditorChange) => {
|
||||
const { composing, ...patch } = change
|
||||
setWordCount(change.word_count)
|
||||
setDocText(change.content_text)
|
||||
setEditTick((n) => n + 1)
|
||||
if (currentDoc) {
|
||||
patchSummary(currentDoc.id, { word_count: change.word_count })
|
||||
schedule(change)
|
||||
scheduleCheckpoint(change.content_text)
|
||||
// The save is never held: see EditorChange.composing. The flag itself
|
||||
// stays out of the patch — it describes the keyboard, not the document,
|
||||
// and the stashed draft a signed-out save leaves behind should be the
|
||||
// document alone.
|
||||
schedule(patch)
|
||||
}
|
||||
// Everything below reads the text as prose. While an IME composition is
|
||||
// in flight it is not prose yet — it is the pinyin she is converting — so
|
||||
// the checkpoint, the rule pack and the companion all wait for the word
|
||||
// to commit. EditorCore emits one more change the moment it does, so
|
||||
// nothing is skipped, only deferred by the length of a word.
|
||||
if (composing) return
|
||||
setDocText(change.content_text)
|
||||
setEditTick((n) => n + 1)
|
||||
if (currentDoc) scheduleCheckpoint(change.content_text)
|
||||
},
|
||||
[currentDoc, patchSummary, schedule, scheduleCheckpoint],
|
||||
)
|
||||
@@ -406,11 +426,14 @@ export default function App() {
|
||||
[patchSummary],
|
||||
)
|
||||
|
||||
// Escape always restores the sidebar while in distraction-free mode.
|
||||
// Escape always restores the sidebar while in distraction-free mode — unless
|
||||
// it belongs to an IME, where it cancels a candidate and never reaches Petal
|
||||
// at all. This is the writer typing Chinese in the very mode built for
|
||||
// uninterrupted writing, so it is the one worth getting right.
|
||||
useEffect(() => {
|
||||
if (!focusMode) return
|
||||
const onKey = (e: KeyboardEvent) => {
|
||||
if (e.key === 'Escape') setFocusMode(false)
|
||||
if (e.key === 'Escape' && !fromIME(e)) setFocusMode(false)
|
||||
}
|
||||
window.addEventListener('keydown', onKey)
|
||||
return () => window.removeEventListener('keydown', onKey)
|
||||
@@ -528,6 +551,9 @@ export default function App() {
|
||||
onToggleTag={handleToggleTag}
|
||||
onCreateTag={handleCreateTag}
|
||||
account={account}
|
||||
direction={me?.direction}
|
||||
onDirection={setDirection}
|
||||
onPair={setPair}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -591,6 +617,7 @@ export default function App() {
|
||||
docId={currentDoc.id}
|
||||
initialContent={currentDoc.content}
|
||||
onChange={handleEditorChange}
|
||||
segmenter={segmenter}
|
||||
suggestions={suggestions}
|
||||
onAccept={handleAccept}
|
||||
onAcceptMany={handleAcceptMany}
|
||||
|
||||
@@ -210,6 +210,21 @@ export interface PersonalWords {
|
||||
words: string[]
|
||||
}
|
||||
|
||||
// One pronunciation of a Chinese word, and what it means in that pronunciation.
|
||||
// A list, because 得 is dé "to obtain" and also the particle in 说得很好.
|
||||
export interface HanziReading {
|
||||
pinyin: string
|
||||
senses: string
|
||||
}
|
||||
|
||||
// A Chinese word lookup. `readings` is empty for a word with no headword, in
|
||||
// which case `chars` may carry the character-by-character reading.
|
||||
export interface HanziInfo {
|
||||
word: string
|
||||
readings: HanziReading[]
|
||||
chars: { char: string; pinyin: string; senses: string }[]
|
||||
}
|
||||
|
||||
// Who's writing. Mirrors the backend db.User.
|
||||
export interface Me {
|
||||
id: string
|
||||
@@ -217,6 +232,11 @@ export interface Me {
|
||||
display_name: string
|
||||
created_at: string
|
||||
pair_lang: string
|
||||
// Which half of the pair is being learned: 'learning_en' (the writer is
|
||||
// native in pair_lang and practising English) or 'learning_pair' (the other
|
||||
// way round). Mirrors users.direction; the server refuses 'learning_pair' for
|
||||
// a pair it has no word list for.
|
||||
direction: string
|
||||
}
|
||||
|
||||
// Thrown when the server says the session is gone. Callers can tell it apart
|
||||
@@ -269,6 +289,15 @@ export const api = {
|
||||
setPairLang: (lang: string) =>
|
||||
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ pair_lang: lang }) }),
|
||||
|
||||
// Turn the pair around. Same endpoint, same contract, and deliberately a
|
||||
// separate call: the two fields are validated together server-side, so a
|
||||
// client that wants to change both says both in one request rather than
|
||||
// sending two that each pass on their own.
|
||||
setDirection: (direction: string) =>
|
||||
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ direction }) }),
|
||||
setPair: (lang: string, direction: string) =>
|
||||
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ pair_lang: lang, direction }) }),
|
||||
|
||||
listDocs: () => req<DocSummary[]>('/docs'),
|
||||
createDoc: () => req<Document>('/docs', { method: 'POST' }),
|
||||
getDoc: (id: string) => req<Document>(`/docs/${id}`),
|
||||
@@ -348,6 +377,12 @@ export const api = {
|
||||
// Lightweight Chinese-only gloss for the inline hover/select tooltip — instant
|
||||
// and offline, so it fires on hover without spinning up the heavier lookup.
|
||||
glossWord: (word: string) => req<Gloss>(`/gloss/${encodeURIComponent(word)}`),
|
||||
// The same lookup pointing the other way: a Chinese word to its pinyin and
|
||||
// English senses, for an account learning the pair language rather than
|
||||
// English. A word the dictionary has no headword for comes back with empty
|
||||
// readings and — when its characters are known — a per-character reading
|
||||
// instead, which is a real second answer for a compound.
|
||||
hanziWord: (word: string) => req<HanziInfo>(`/hanzi/${encodeURIComponent(word)}`),
|
||||
// Tone-rewrite: rewrites a selected passage in the given style ('natural',
|
||||
// 'academic', …) and returns the rewritten text for an in-editor preview. Not
|
||||
// persisted — the editor applies it directly on accept.
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { gunzipSync } from 'node:zlib'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { CONFUSION_PAIRS, hanziFindings } from './hanzi'
|
||||
import { buildSegmenter } from '../../lib/segment'
|
||||
|
||||
// The 错别字 pack, held to the bar Phase 22 set for the English rule pack: every
|
||||
// rule pinned in *two* directions — the mistake it must catch, and the correct
|
||||
// writing next to it that it must leave alone.
|
||||
//
|
||||
// Here the second direction is the one that matters, and it is unusually easy to
|
||||
// get wrong. Chinese has no spaces, so every one of these rules is a substring
|
||||
// match on running text, and for most of them there exists an ordinary correct
|
||||
// sentence that contains the substring across a word boundary. Those sentences
|
||||
// are the real test.
|
||||
|
||||
const raw = gunzipSync(readFileSync(new URL('../../../public/dictionaries/zh/words.txt.gz', import.meta.url)))
|
||||
const seg = buildSegmenter(raw.toString('utf8'))
|
||||
|
||||
const flagged = (text: string) => hanziFindings(text, seg).map((f) => `${f.original}→${f.replacement}`)
|
||||
|
||||
describe('the gate that admits a rule', () => {
|
||||
// The pack's own claim about itself, checked against the shipped dictionary
|
||||
// rather than asserted in a comment. A pair whose wrong form is a real word
|
||||
// cannot be decided mechanically and does not belong here.
|
||||
it('every wrong form is not a word, and every right form is', () => {
|
||||
for (const { wrong, right } of CONFUSION_PAIRS) {
|
||||
expect(seg.has(wrong), `${wrong} is a dictionary word and must not be flagged`).toBe(false)
|
||||
expect(seg.has(right), `${right} is not a dictionary word`).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
// The errors this pack deliberately refuses, and why — each is a genuine
|
||||
// mistake by a modern standard whose wrong form is itself a headword. If a
|
||||
// dictionary rebuild ever drops one of these, this test fails and the pair
|
||||
// becomes admissible; that is the intended way to find out.
|
||||
it('refuses the well-known errors it cannot decide', () => {
|
||||
for (const undecidable of ['自已', '好象', '倒底', '帐号', '部份']) {
|
||||
expect(seg.has(undecidable), `${undecidable} is no longer a word — reconsider the rule`).toBe(true)
|
||||
expect(flagged(`这是${undecidable}的例子`)).toEqual([])
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('the mistakes it catches', () => {
|
||||
it('已 / 己 / 以', () => {
|
||||
expect(flagged('我己经写完了作业')).toEqual(['己经→已经'])
|
||||
expect(flagged('我以经吃过饭了')).toEqual(['以经→已经'])
|
||||
expect(flagged('下课已后我们去公园')).toEqual(['已后→以后'])
|
||||
})
|
||||
|
||||
it('在 / 再', () => {
|
||||
expect(flagged('明天在见')).toEqual(['在见→再见'])
|
||||
expect(flagged('他正再看书')).toEqual(['正再→正在'])
|
||||
expect(flagged('现再几点了')).toEqual(['现再→现在'])
|
||||
})
|
||||
|
||||
it('做 / 作', () => {
|
||||
expect(flagged('我的工做很忙')).toEqual(['工做→工作'])
|
||||
expect(flagged('老师给我们很多做业')).toEqual(['做业→作业'])
|
||||
expect(flagged('这本书的做者是谁')).toEqual(['做者→作者'])
|
||||
})
|
||||
|
||||
it('the rest', () => {
|
||||
expect(flagged('我觉的这个很好')).toEqual(['觉的→觉得'])
|
||||
expect(flagged('你因该早点睡')).toEqual(['因该→应该'])
|
||||
expect(flagged('即然你来了就坐下吧')).toEqual(['即然→既然'])
|
||||
expect(flagged('你知到吗')).toEqual(['知到→知道'])
|
||||
expect(flagged('请输入你的蜜码')).toEqual(['蜜码→密码'])
|
||||
})
|
||||
|
||||
it('reports an exact span, so the card replaces the right characters', () => {
|
||||
const text = '我己经到了'
|
||||
const [f] = hanziFindings(text, seg)
|
||||
expect(text.slice(f.from, f.to)).toBe('己经')
|
||||
expect(text.slice(0, f.from) + f.replacement + text.slice(f.to)).toBe('我已经到了')
|
||||
})
|
||||
|
||||
it('finds every occurrence, in document order', () => {
|
||||
expect(flagged('我己经吃了，他也己经吃了')).toEqual(['己经→已经', '己经→已经'])
|
||||
expect(flagged('我的工做很忙，所以我觉的很累')).toEqual(['工做→工作', '觉的→觉得'])
|
||||
})
|
||||
})
|
||||
|
||||
// ── the direction that matters ──────────────────────────────────────────────
|
||||
|
||||
describe('the correct writing it must not touch', () => {
|
||||
// Each of these is an ordinary sentence containing a flagged substring across
|
||||
// a word boundary. Without the boundary gate, every one would be corrupted —
|
||||
// and corrupted silently, into text that is still made of real characters.
|
||||
it('leaves two real words alone where they happen to abut', () => {
|
||||
// 自己 + 经常. The substring is 己经.
|
||||
expect(flagged('他自己经常做饭')).toEqual([])
|
||||
// 睡觉 + 的. The substring is 觉的.
|
||||
expect(flagged('睡觉的时候不要看手机')).toEqual([])
|
||||
// 感觉 + 的.
|
||||
expect(flagged('这是我感觉的方向')).toEqual([])
|
||||
// 不知 + 到底.
|
||||
expect(flagged('我不知到底该怎么办')).toEqual([])
|
||||
// 因 + 位置.
|
||||
expect(flagged('因位置不好我们换了座位')).toEqual([])
|
||||
// 已 + 后悔.
|
||||
expect(flagged('他已后悔了')).toEqual([])
|
||||
})
|
||||
|
||||
it('leaves ordinary correct prose entirely alone', () => {
|
||||
for (const good of [
|
||||
'我今天早上去公园跑步了',
|
||||
'他的中文说得很好',
|
||||
'我已经完成了我的作业',
|
||||
'现在几点了，我们再见面吧',
|
||||
'我觉得这个工作很有意思',
|
||||
'既然你已经知道了，就按照计划做',
|
||||
]) {
|
||||
expect(flagged(good), good).toEqual([])
|
||||
}
|
||||
})
|
||||
|
||||
// Where the gate costs the pack a real catch, and the trade it is making.
|
||||
// 不知 is itself a word, so 我不知到他在哪里 — which really is 知到 for 知道 —
|
||||
// reads to the segmenter as 不知 + 到 and is left alone. That is the gate
|
||||
// preferring a missed error to a corrupted sentence, which is the whole
|
||||
// premise: 我不知到底该怎么办 is the same three characters and is correct.
|
||||
it('declines a real error rather than risk the sentence beside it', () => {
|
||||
expect(flagged('我不知到他在哪里')).toEqual([])
|
||||
expect(flagged('你知到吗')).toEqual(['知到→知道'])
|
||||
})
|
||||
|
||||
it('says nothing about English, or about nothing', () => {
|
||||
expect(flagged('I already finished my homework')).toEqual([])
|
||||
expect(flagged('')).toEqual([])
|
||||
})
|
||||
|
||||
// The direction gate. The word list is loaded only for an account learning
|
||||
// Chinese, so without one this pack is silent — a writer practising English
|
||||
// must never be told her own quoted Chinese is wrong.
|
||||
it('is silent without a segmenter, which is how the direction gate works', () => {
|
||||
expect(hanziFindings('我己经写完了', null)).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,149 @@
|
||||
import type { MechanicsFinding } from '../../api/client'
|
||||
import type { Segmenter } from '../../lib/segment'
|
||||
|
||||
// 错别字 — wrong-character detection, the Chinese counterpart of the spell
|
||||
// checker, and a different problem from the one Hunspell solves.
|
||||
//
|
||||
// Chinese has no misspellings in the English sense: every character a writer can
|
||||
// type is a real character, correctly formed, and an IME will not offer one that
|
||||
// is not. What it *will* offer is the wrong one. Typing pinyin `yijing` and
|
||||
// taking the first candidate gives 已经 or 己经 depending on the moment, and both
|
||||
// are made of real characters. So the unit of error is not a malformed word but
|
||||
// a **substituted character inside a correct-looking one** — which is why this
|
||||
// is a rule pack over confusable pairs rather than a dictionary membership test.
|
||||
//
|
||||
// The discipline is Phase 22's, and the bar is the same: **precision over
|
||||
// recall**. A wrong nudge costs more trust than a missed one earns, and it costs
|
||||
// double here, because a learner has no way to know the tool is wrong. Two
|
||||
// mechanical gates enforce it, and both are checked in the tests rather than
|
||||
// asserted in prose.
|
||||
|
||||
// A confusable pair: `wrong` is never a word, `right` is what was meant.
|
||||
//
|
||||
// **Gate one — the pair must be decidable by the dictionary.** Each entry is
|
||||
// admitted only if `wrong` is absent from the 188k-word list *and* `right` is
|
||||
// present. That is what makes the correction a fact rather than a preference,
|
||||
// and it is checked against the shipped asset in hanzi.test.ts.
|
||||
//
|
||||
// It is also the gate that keeps out errors everyone knows are errors. 自已 for
|
||||
// 自己 is among the commonest slips in written Chinese, and 自已 is itself a
|
||||
// dictionary headword — so this pack does not flag it, exactly as Phase 22's
|
||||
// English pack left out `married with`. The same fate for 好象 (an older form of
|
||||
// 好像, still in the dictionary), 倒底, 帐号 and 部份: all real errors by a modern
|
||||
// standard, none of them decidable here.
|
||||
interface Confusion {
|
||||
wrong: string
|
||||
right: string
|
||||
// The note on the card. English, because this pack only ever runs for a writer
|
||||
// whose English is the language they think in — see the direction gate below.
|
||||
why: string
|
||||
}
|
||||
|
||||
const CONFUSIONS: Confusion[] = [
|
||||
// 已 / 己 / 以 — three characters that differ by one stroke and share a
|
||||
// syllable. The most productive source of 错别字 there is.
|
||||
{ wrong: '己经', right: '已经', why: '已经 (already) — 己 is the "self" character; the one you want is 已.' },
|
||||
{ wrong: '以经', right: '已经', why: '已经 (already) — 以 is a different word; 已 is the one that means "already".' },
|
||||
{ wrong: '已后', right: '以后', why: '以后 (afterwards) takes 以, not 已.' },
|
||||
|
||||
// 在 / 再 — same pinyin (zài), completely different jobs: one is location and
|
||||
// ongoing action, the other is repetition.
|
||||
{ wrong: '在见', right: '再见', why: '再见 (goodbye) — 再 is "again", which is what "see you again" needs.' },
|
||||
{ wrong: '正再', right: '正在', why: '正在 (in the middle of doing) takes 在, the one about being somewhere.' },
|
||||
{ wrong: '现再', right: '现在', why: '现在 (now) takes 在.' },
|
||||
|
||||
// 做 / 作 — both zuò, both "to do", and which one a compound takes is simply
|
||||
// fixed by convention. A learner cannot reason it out, which is what makes a
|
||||
// reminder worth having.
|
||||
{ wrong: '工做', right: '工作', why: '工作 (work) is written with 作.' },
|
||||
{ wrong: '做业', right: '作业', why: '作业 (homework) is written with 作.' },
|
||||
{ wrong: '做者', right: '作者', why: '作者 (author) is written with 作.' },
|
||||
{ wrong: '做文', right: '作文', why: '作文 (an essay) is written with 作.' },
|
||||
{ wrong: '做用', right: '作用', why: '作用 (effect, function) is written with 作.' },
|
||||
|
||||
// 得 / 的 — the pair everyone knows about. Only the fixed compound is flagged:
|
||||
// deciding 的 against 地 against 得 in the general case needs to know whether
|
||||
// the next word is a verb or a noun, which nothing here can tell.
|
||||
{ wrong: '觉的', right: '觉得', why: '觉得 (to feel, to think) ends in 得.' },
|
||||
|
||||
// 即 / 既 — one stroke apart, opposite meanings ("namely" against "since").
|
||||
{ wrong: '即然', right: '既然', why: '既然 (since, given that) takes 既.' },
|
||||
{ wrong: '既使', right: '即使', why: '即使 (even if) takes 即.' },
|
||||
|
||||
// The rest: ordinary IME slips where the wrong character is a homophone.
|
||||
{ wrong: '因该', right: '应该', why: '应该 (should) — 因 means "because"; the word you want starts with 应.' },
|
||||
{ wrong: '因位', right: '因为', why: '因为 (because) ends in 为.' },
|
||||
{ wrong: '知到', right: '知道', why: '知道 (to know) ends in 道.' },
|
||||
{ wrong: '安照', right: '按照', why: '按照 (according to) takes 按.' },
|
||||
{ wrong: '蜜码', right: '密码', why: '密码 (password) takes 密 — 蜜 is honey.' },
|
||||
{ wrong: '犹其', right: '尤其', why: '尤其 (especially) takes 尤.' },
|
||||
{ wrong: '甘净', right: '干净', why: '干净 (clean) takes 干.' },
|
||||
{ wrong: '什末', right: '什么', why: '什么 (what) ends in 么.' },
|
||||
{ wrong: '一像', right: '一样', why: '一样 (the same) ends in 样 — 像 is "to resemble".' },
|
||||
{ wrong: '必须品', right: '必需品', why: '必需品 (a necessity) takes 需. 必须 is "must", which is a different word.' },
|
||||
]
|
||||
|
||||
// **Gate two — the characters must not already belong to two different words.**
|
||||
//
|
||||
// This is the gate that stops the pack from destroying correct writing, and
|
||||
// without it every rule above is dangerous. 自己经常 ("oneself, often") contains
|
||||
// the string 己经. 睡觉的时候 ("when sleeping") contains 觉的. 不知到底 contains 知到.
|
||||
// A substring match would corrupt all three.
|
||||
//
|
||||
// The segmenter already knows the difference, so the test is: split the text,
|
||||
// and if the two characters land in different tokens *and* either token is a
|
||||
// real multi-character word, this is a word boundary and not an error. Two
|
||||
// adjacent single-character tokens is what the walk produces when it has nothing
|
||||
// better to offer — which is exactly what a mistyped compound looks like.
|
||||
function isWordBoundary(tokens: { word: string; from: number; to: number }[], at: number): boolean {
|
||||
const left = tokens.find((t) => at >= t.from && at < t.to)
|
||||
const right = tokens.find((t) => at + 1 >= t.from && at + 1 < t.to)
|
||||
if (!left || !right || left === right) return false
|
||||
return left.word.length > 1 || right.word.length > 1
|
||||
}
|
||||
|
||||
// hanziFindings returns the 错别字 in a piece of text, as ordinary mechanics
|
||||
// findings — the same shape, the same rail, the same cards, the same accept.
|
||||
//
|
||||
// It needs the segmenter and does nothing without one, which is also the
|
||||
// direction gate: the word list is loaded only for an account learning Chinese
|
||||
// (useSegmenter), so a writer practising English can never be told her quoted
|
||||
// Chinese is wrong. That is not a nicety. Petal deliberately never corrects the
|
||||
// pair language — the fr and es dictionaries are chosen to hold every variety
|
||||
// precisely so they cannot underline correct writing — and a Mandarin native
|
||||
// does not need her own language checked by a rule pack of two dozen entries.
|
||||
export function hanziFindings(text: string, segmenter: Segmenter | null): MechanicsFinding[] {
|
||||
if (!segmenter || !text) return []
|
||||
// One segmentation for the whole text, shared by every rule. The walk is
|
||||
// linear, but running it two dozen times over a long document would not be.
|
||||
const tokens = segmenter.segment(text)
|
||||
const found: MechanicsFinding[] = []
|
||||
|
||||
for (const c of CONFUSIONS) {
|
||||
let from = text.indexOf(c.wrong)
|
||||
while (from !== -1) {
|
||||
// The boundary test is asked at the seam the substitution sits on: the
|
||||
// gap between the first two characters, which is where a mistyped
|
||||
// compound and two adjacent words look different from each other.
|
||||
if (!isWordBoundary(tokens, from)) {
|
||||
found.push({
|
||||
from,
|
||||
to: from + c.wrong.length,
|
||||
original: c.wrong,
|
||||
replacement: c.right,
|
||||
explanation: c.why,
|
||||
type: 'mechanics',
|
||||
})
|
||||
}
|
||||
from = text.indexOf(c.wrong, from + 1)
|
||||
}
|
||||
}
|
||||
|
||||
// Document order, so the rail reads down the page rather than down this file.
|
||||
return found.sort((a, b) => a.from - b.from)
|
||||
}
|
||||
|
||||
// Exported for the tests, which check every pair against the shipped word list.
|
||||
// A pack whose own gate is only described in a comment is a pack whose gate can
|
||||
// rot; this is how the description is made to stay true.
|
||||
export const CONFUSION_PAIRS = CONFUSIONS.map((c) => ({ wrong: c.wrong, right: c.right }))
|
||||
@@ -20,6 +20,12 @@ interface Props {
|
||||
// The signed-in writer, when there is real auth to sign out of. Null in a
|
||||
// local-dev build, where there is nothing to leave.
|
||||
account: { name: string } | null
|
||||
// The account's learner direction and the way to change it, passed straight
|
||||
// through to the language picker in the footer — the sidebar is the drawer,
|
||||
// and the drawer is the only chrome always one tap away on a phone.
|
||||
direction?: string
|
||||
onDirection?: (direction: string) => Promise<void>
|
||||
onPair?: (lang: string, direction: string) => Promise<void>
|
||||
}
|
||||
|
||||
// Sidebar sort orders. 'recent' keeps the server's updated_at-desc ordering.
|
||||
@@ -43,6 +49,9 @@ export function DocList({
|
||||
onToggleTag,
|
||||
onCreateTag,
|
||||
account,
|
||||
direction,
|
||||
onDirection,
|
||||
onPair,
|
||||
}: Props) {
|
||||
const t = usePack()
|
||||
// Active tag filter (null = show all). Cleared automatically if the tag
|
||||
@@ -161,7 +170,7 @@ export function DocList({
|
||||
{/* The pair Petal speaks. Unlike the rows above it this is not about any
|
||||
document, and unlike sign-out it is offered whether or not there is an
|
||||
account behind the session — a local-dev build still has a langpack. */}
|
||||
<LanguagePicker />
|
||||
<LanguagePicker direction={direction} onDirection={onDirection} onPair={onPair} />
|
||||
|
||||
{/* Who's writing, and the way out. Shown only when there's a real account
|
||||
behind the session — a local-dev build has nobody to sign out as. */}
|
||||
|
||||
@@ -15,26 +15,73 @@ import { setPackLang, shippedPacks, usePack } from '../../i18n'
|
||||
// read a label that says "Portuguese" in Chinese, so the buttons say 中文 and
|
||||
// Português and nothing else — the one place in Petal where bilingual copy would
|
||||
// actively get in the way.
|
||||
export function LanguagePicker() {
|
||||
interface Props {
|
||||
// The account's current direction ('learning_en' | 'learning_pair'), and the
|
||||
// way to change it. Owned by App rather than here, because turning the pair
|
||||
// around changes what the *editor* does — it is what loads the word list —
|
||||
// and this control is only where the writer says so.
|
||||
direction?: string
|
||||
onDirection?: (direction: string) => Promise<void>
|
||||
// Move the pair itself. Owned by App for the same reason: the answer carries
|
||||
// the direction too, and the account's direction is what loads the word list.
|
||||
onPair?: (lang: string, direction: string) => Promise<void>
|
||||
}
|
||||
|
||||
export function LanguagePicker({ direction, onDirection, onPair }: Props = {}) {
|
||||
const t = usePack()
|
||||
const packs = shippedPacks()
|
||||
const [saving, setSaving] = useState<string | null>(null)
|
||||
const [failed, setFailed] = useState(false)
|
||||
const [turning, setTurning] = useState(false)
|
||||
const [turnFailed, setTurnFailed] = useState(false)
|
||||
|
||||
// Nothing to choose between — a deployment with one pack shows no picker
|
||||
// rather than a single button that does nothing.
|
||||
if (packs.length < 2) return null
|
||||
// rather than a single button that does nothing. The direction control is
|
||||
// still worth rendering in that case, so it is checked separately below.
|
||||
const showPacks = packs.length >= 2
|
||||
// `t.learner` is the pack's own statement that this pair can be learned
|
||||
// toward, and the server keeps the matching list (auth.learnerPairs). A pack
|
||||
// without it renders nothing here, which is the same failure mode as a pair
|
||||
// without copy: absent rather than broken.
|
||||
const learner = t.learner
|
||||
if (!showPacks && !learner) return null
|
||||
|
||||
const turn = async (next: string) => {
|
||||
if (!onDirection || next === (direction ?? 'learning_en') || turning) return
|
||||
setTurning(true)
|
||||
setTurnFailed(false)
|
||||
try {
|
||||
await onDirection(next)
|
||||
} catch {
|
||||
setTurnFailed(true)
|
||||
} finally {
|
||||
setTurning(false)
|
||||
}
|
||||
}
|
||||
|
||||
const choose = async (code: string) => {
|
||||
if (code === t.code || saving) return
|
||||
setSaving(code)
|
||||
setFailed(false)
|
||||
try {
|
||||
// Name the direction alongside the pair. The server validates the two as
|
||||
// one decision and refuses a learner direction for a pair it has no word
|
||||
// list for, so an account that is learning Chinese cannot move to French
|
||||
// by naming only the pair — that request is rejected outright, and the
|
||||
// writer is left on a picker whose buttons all fail. A pair with no
|
||||
// learner side can only be travelled toward English; saying so is how the
|
||||
// move is actually made.
|
||||
const target = packs.find((p) => p.code === code)
|
||||
const next = target?.learner ? (direction ?? 'learning_en') : 'learning_en'
|
||||
if (onPair) {
|
||||
await onPair(code, next)
|
||||
} else {
|
||||
const me = await api.setPairLang(code)
|
||||
// The server's answer, not the code we asked for. Everything downstream —
|
||||
// her dictionary, the read-aloud voice, the word lookups — follows the
|
||||
// The server's answer, not the code we asked for. Everything downstream
|
||||
// — her dictionary, the read-aloud voice, the word lookups — follows the
|
||||
// pack, so it must follow what was actually stored.
|
||||
setPackLang(me.pair_lang)
|
||||
}
|
||||
} catch {
|
||||
// A 401 has already surfaced as the sign-in overlay through the client's
|
||||
// interceptor; anything else leaves her on the pair she was already on,
|
||||
@@ -47,6 +94,8 @@ export function LanguagePicker() {
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-1 px-1">
|
||||
{showPacks && (
|
||||
<>
|
||||
{/* Label and buttons wrap as a pair: the label is itself bilingual
|
||||
("Langue · Language"), and three self-naming buttons beside it need
|
||||
more than the drawer is wide in every language Petal ships. When they
|
||||
@@ -89,6 +138,52 @@ export function LanguagePicker() {
|
||||
{t.docs.languageFailed}
|
||||
</span>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Which way round the pair is being learned. Below the language buttons
|
||||
because it only makes sense once the language is settled, and rendered
|
||||
at all only for a pair Petal has the learner-side data for. */}
|
||||
{learner && onDirection && (
|
||||
<div
|
||||
className="flex flex-wrap items-center gap-x-2 gap-y-1 text-xs"
|
||||
style={{ color: 'var(--color-muted)' }}
|
||||
>
|
||||
<span className="shrink-0 font-semibold">{learner.label}</span>
|
||||
<div className="ml-auto flex shrink-0 gap-1">
|
||||
{[
|
||||
{ code: 'learning_en', text: learner.toEn, en: `learning English` },
|
||||
{ code: 'learning_pair', text: learner.toPair, en: `learning ${t.nativeName}` },
|
||||
].map((opt) => {
|
||||
const active = (direction ?? 'learning_en') === opt.code
|
||||
return (
|
||||
<button
|
||||
key={opt.code}
|
||||
type="button"
|
||||
onClick={() => void turn(opt.code)}
|
||||
disabled={turning}
|
||||
aria-pressed={active}
|
||||
aria-label={`I am ${opt.en}`}
|
||||
className="petal-tap-sm px-2.5 py-1 text-xs font-bold transition-colors disabled:opacity-60"
|
||||
style={{
|
||||
borderRadius: 'var(--radius-pill)',
|
||||
background: active ? 'var(--color-accent)' : 'var(--color-surface)',
|
||||
color: active ? '#fff' : 'var(--color-plum)',
|
||||
boxShadow: active ? 'none' : 'var(--shadow-soft)',
|
||||
}}
|
||||
>
|
||||
{opt.text}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{turnFailed && learner && (
|
||||
<span className="text-[0.7rem]" style={{ color: 'var(--color-accent)' }}>
|
||||
{learner.failed}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import { tagColorVar, type Tag, type TagColor } from '../../api/client'
|
||||
import { usePack } from '../../i18n'
|
||||
import { fromIME } from '../../lib/ime'
|
||||
|
||||
const COLORS: TagColor[] = ['rose', 'mint', 'peach', 'lavender', 'sky', 'honey']
|
||||
|
||||
@@ -26,7 +27,9 @@ export function TagPicker({ roster, assignedIds, onToggle, onCreate, onClose }:
|
||||
if (!ref.current?.contains(e.target as Node)) onClose()
|
||||
}
|
||||
const onKey = (e: KeyboardEvent) => {
|
||||
if (e.key === 'Escape') onClose()
|
||||
// Not while an IME is open: a tag named in Chinese is composed in this
|
||||
// very field, and Escape there means "wrong candidate", not "close".
|
||||
if (e.key === 'Escape' && !fromIME(e)) onClose()
|
||||
}
|
||||
// Defer so the opening click doesn't immediately close it.
|
||||
const id = setTimeout(() => document.addEventListener('pointerdown', onDown), 0)
|
||||
@@ -114,7 +117,7 @@ export function TagPicker({ roster, assignedIds, onToggle, onCreate, onClose }:
|
||||
value={name}
|
||||
onChange={(e) => setName(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') submit()
|
||||
if (e.key === 'Enter' && !fromIME(e)) submit()
|
||||
}}
|
||||
placeholder={t.docs.newTagPlaceholder}
|
||||
aria-label="New tag name"
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from 'react'
|
||||
import { api, streamSuggestionChat, type ChatMessage } from '../../api/client'
|
||||
import { usePack } from '../../i18n'
|
||||
import { splitBilingual } from './bilingualReply'
|
||||
import { fromIME } from '../../lib/ime'
|
||||
|
||||
interface Props {
|
||||
suggestionId: string
|
||||
@@ -177,6 +178,13 @@ export function AskPetal({ suggestionId, explanation }: Props) {
|
||||
ref={inputRef}
|
||||
value={input}
|
||||
onChange={(e) => setInput(e.target.value)}
|
||||
// She asks Petal in Mandarin, so the Enter that commits an IME
|
||||
// candidate lands in this field constantly. Most browsers already
|
||||
// withhold implicit form submission during a composition; the ones
|
||||
// that don't would send her half-typed question. Cheap to be certain.
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter' && fromIME(e)) e.preventDefault()
|
||||
}}
|
||||
placeholder={t.editor.askPlaceholder}
|
||||
className="min-w-0 flex-1 rounded-full px-3 py-1.5 text-xs focus:outline-none"
|
||||
style={{
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { EditorState, TextSelection } from '@tiptap/pm/state'
|
||||
import type { Transaction } from '@tiptap/pm/state'
|
||||
import { Schema } from '@tiptap/pm/model'
|
||||
import { compositionKey, compositionPlugin, isComposing, holdRedraw } from './Composition'
|
||||
import { suggestionPlugin, suggestionPluginKey, setSuggestions } from './SuggestionHighlight'
|
||||
import { spellPlugin, spellPluginKey, setSpellChecker } from './SpellCheck'
|
||||
import { searchPlugin, searchPluginKey, setSearch } from './SearchHighlight'
|
||||
import type { Suggestion } from '../../api/client'
|
||||
import type { SpellChecker } from '../../hooks/useSpellChecker'
|
||||
|
||||
// These tests are about one moment: she is typing 公园 with a pinyin IME, so the
|
||||
// document briefly contains "gongyuan" and a candidate window sits over it. Every
|
||||
// decoration layer wants to recompute, and recomputing rewrites the DOM around
|
||||
// the node the browser is composing in — which is what eats half-typed input.
|
||||
//
|
||||
// Nothing here needs a real EditorView: composition is tracked in plugin state
|
||||
// by the compositionstart/compositionend handlers, so a plain EditorState with
|
||||
// the same plugins reproduces exactly the decisions the layers make.
|
||||
|
||||
const schema = new Schema({
|
||||
nodes: {
|
||||
doc: { content: 'block+' },
|
||||
paragraph: { group: 'block', content: 'inline*', toDOM: () => ['p', 0] },
|
||||
text: { group: 'inline' },
|
||||
},
|
||||
})
|
||||
|
||||
const doc = (text: string) =>
|
||||
schema.node('doc', null, [schema.node('paragraph', null, text ? [schema.text(text)] : [])])
|
||||
|
||||
// A dictionary that knows ordinary English and nothing else — so the pinyin run
|
||||
// an IME leaves in the document mid-composition is a misspelling to it, which is
|
||||
// precisely the risk this guard exists for.
|
||||
const english: SpellChecker = {
|
||||
correct: (w) => ['the', 'park', 'went', 'to', 'today'].includes(w.toLowerCase()),
|
||||
suggest: () => [],
|
||||
extendedAlphabet: false,
|
||||
}
|
||||
|
||||
const suggestion = (original: string, replacement: string): Suggestion => ({
|
||||
id: `s-${original}`,
|
||||
doc_id: 'd',
|
||||
from_pos: 0,
|
||||
to_pos: 0,
|
||||
original,
|
||||
replacement,
|
||||
explanation: '',
|
||||
type: 'grammar',
|
||||
status: 'pending',
|
||||
source: 'llm',
|
||||
created_at: new Date().toISOString(),
|
||||
})
|
||||
|
||||
function harness(text: string) {
|
||||
let state = EditorState.create({
|
||||
schema,
|
||||
doc: doc(text),
|
||||
plugins: [compositionPlugin({ onEnd: null }), suggestionPlugin(), spellPlugin(), searchPlugin()],
|
||||
})
|
||||
const api = {
|
||||
get state() {
|
||||
return state
|
||||
},
|
||||
tr: (f: (tr: Transaction) => Transaction) => {
|
||||
state = state.apply(f(state.tr))
|
||||
},
|
||||
dispatch: (tr: Transaction) => {
|
||||
state = state.apply(tr)
|
||||
},
|
||||
// The two ends of a composition, as the DOM handlers dispatch them.
|
||||
startComposing: () => api.tr((tr) => tr.setMeta(compositionKey, true)),
|
||||
endComposing: () => api.tr((tr) => tr.setMeta(compositionKey, false)),
|
||||
// Typing, whether by keystroke or by an IME writing into the document.
|
||||
type: (at: number, text: string) =>
|
||||
api.tr((tr) => tr.insertText(text, at).setSelection(TextSelection.create(tr.doc, at + text.length))),
|
||||
// Replace a span, the way committing an IME candidate does.
|
||||
commit: (from: number, to: number, text: string) => api.tr((tr) => tr.insertText(text, from, to)),
|
||||
spans: (key: typeof suggestionPluginKey | typeof spellPluginKey | typeof searchPluginKey) => {
|
||||
const deco = (key.getState(state) as { decorations: import('@tiptap/pm/view').DecorationSet }).decorations
|
||||
return deco.find().map((d) => [d.from, d.to] as const)
|
||||
},
|
||||
}
|
||||
return api
|
||||
}
|
||||
|
||||
describe('composition tracking', () => {
|
||||
it('is off until a composition starts, and off again once it ends', () => {
|
||||
const h = harness('I went to the ')
|
||||
expect(isComposing(h.state)).toBe(false)
|
||||
h.startComposing()
|
||||
expect(isComposing(h.state)).toBe(true)
|
||||
h.endComposing()
|
||||
expect(isComposing(h.state)).toBe(false)
|
||||
})
|
||||
|
||||
it('releases the redraw on the very transaction that ends the composition', () => {
|
||||
const h = harness('hello')
|
||||
h.startComposing()
|
||||
const before = h.state
|
||||
expect(holdRedraw(before.tr, before)).toBe(true)
|
||||
// The end transaction is dispatched while composing is still true; if it
|
||||
// held its own redraw like any other, nothing would ever release it.
|
||||
expect(holdRedraw(before.tr.setMeta(compositionKey, false), before)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('spell underlines during composition', () => {
|
||||
it('does not underline the pinyin she is part-way through converting', () => {
|
||||
const h = harness('I went to the ')
|
||||
h.dispatch(h.state.tr.setMeta(spellPluginKey, english))
|
||||
expect(h.spans(spellPluginKey)).toEqual([])
|
||||
|
||||
h.startComposing()
|
||||
// The IME writes its buffer into the document one letter at a time. The
|
||||
// caret sits inside the run, so the caret exemption would cover "gongyuan"
|
||||
// on its own — but not a second word, and not once she moves back to fix a
|
||||
// syllable. The guard is what makes that irrelevant.
|
||||
h.type(15, 'gong')
|
||||
h.type(19, 'yuan')
|
||||
expect(h.spans(spellPluginKey)).toEqual([])
|
||||
// And the caret has moved away, which normally forces a rebuild.
|
||||
h.tr((tr) => tr.setSelection(TextSelection.create(tr.doc, 1)))
|
||||
expect(h.spans(spellPluginKey)).toEqual([])
|
||||
})
|
||||
|
||||
it('re-checks the moment the candidate is committed', () => {
|
||||
const h = harness('I went to the ')
|
||||
h.dispatch(h.state.tr.setMeta(spellPluginKey, english))
|
||||
h.startComposing()
|
||||
h.type(15, 'gongyuan')
|
||||
h.commit(15, 23, '公园') // she picks 公园; the pinyin is gone
|
||||
h.endComposing()
|
||||
// Nothing to flag: the pinyin never existed by the time anyone looked, and
|
||||
// CJK is not tokenized at all.
|
||||
expect(h.spans(spellPluginKey)).toEqual([])
|
||||
|
||||
// A real misspelling typed afterwards still underlines, so the layer is
|
||||
// released rather than switched off. (The caret moves off it first: a word
|
||||
// under the cursor is exempt, mid-typing, IME or no IME.)
|
||||
h.type(17, ' parc')
|
||||
h.tr((tr) => tr.setSelection(TextSelection.create(tr.doc, 1)))
|
||||
expect(h.spans(spellPluginKey).length).toBe(1)
|
||||
})
|
||||
|
||||
it('underlines the same text immediately when no IME is involved', () => {
|
||||
const h = harness('I went to the ')
|
||||
h.dispatch(h.state.tr.setMeta(spellPluginKey, english))
|
||||
h.type(15, 'gongyuan')
|
||||
h.tr((tr) => tr.setSelection(TextSelection.create(tr.doc, 1)))
|
||||
expect(h.spans(spellPluginKey).length).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('suggestion highlights during composition', () => {
|
||||
it('carries existing highlights along with the text instead of re-anchoring', () => {
|
||||
const h = harness('I went to the park today')
|
||||
setSuggestions(h.state, h.dispatch, [suggestion('went to', 'go to')])
|
||||
expect(h.spans(suggestionPluginKey)).toEqual([[3, 10]])
|
||||
|
||||
h.startComposing()
|
||||
h.type(1, 'x') // insert before the highlight: it has to move with the text
|
||||
expect(h.spans(suggestionPluginKey)).toEqual([[4, 11]])
|
||||
})
|
||||
|
||||
it('holds a freshly arrived suggestion list until the composition ends', () => {
|
||||
const h = harness('I went to the park today')
|
||||
h.startComposing()
|
||||
setSuggestions(h.state, h.dispatch, [suggestion('the park', 'a park')])
|
||||
// The list is stored, but the page is not repainted under the IME.
|
||||
expect(h.spans(suggestionPluginKey)).toEqual([])
|
||||
h.endComposing()
|
||||
expect(h.spans(suggestionPluginKey)).toEqual([[11, 19]])
|
||||
})
|
||||
|
||||
it('re-anchors against the committed text, not the pinyin it replaced', () => {
|
||||
const h = harness('I went to ')
|
||||
setSuggestions(h.state, h.dispatch, [suggestion('公园', '花园')])
|
||||
expect(h.spans(suggestionPluginKey)).toEqual([]) // not there yet
|
||||
h.startComposing()
|
||||
h.type(11, 'gongyuan')
|
||||
h.commit(11, 19, '公园')
|
||||
h.endComposing()
|
||||
expect(h.spans(suggestionPluginKey)).toEqual([[11, 13]])
|
||||
})
|
||||
})
|
||||
|
||||
describe('find-and-replace highlights during composition', () => {
|
||||
it('holds the match set, then refreshes it against the committed text', () => {
|
||||
const h = harness('公园 and 公园')
|
||||
setSearch(h.state, h.dispatch, '公园', false)
|
||||
expect(h.spans(searchPluginKey).length).toBe(2)
|
||||
|
||||
h.startComposing()
|
||||
h.type(10, ' gongyuan') // at the end of the text, where the caret is
|
||||
expect(h.spans(searchPluginKey).length).toBe(2) // still two, not three
|
||||
h.commit(11, 19, '公园')
|
||||
h.endComposing()
|
||||
expect(h.spans(searchPluginKey).length).toBe(3)
|
||||
})
|
||||
|
||||
it('closing the bar clears immediately — a composition never holds a removal', () => {
|
||||
const h = harness('公园 and 公园')
|
||||
setSearch(h.state, h.dispatch, '公园', false)
|
||||
h.startComposing()
|
||||
h.dispatch(h.state.tr.setMeta(searchPluginKey, { kind: 'clear' }))
|
||||
expect(h.spans(searchPluginKey)).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('the layers are only paused, never left stale', () => {
|
||||
it('rebuilds even if the composition ends on a transaction of its own', () => {
|
||||
// The end signal is dispatched on a timer, after ProseMirror has flushed the
|
||||
// composition's last document change — so the releasing transaction usually
|
||||
// carries no document change at all. That must still be enough.
|
||||
const h = harness('I went to the ')
|
||||
h.dispatch(h.state.tr.setMeta(spellPluginKey, english))
|
||||
h.startComposing()
|
||||
h.type(15, 'parc')
|
||||
expect(h.spans(spellPluginKey)).toEqual([])
|
||||
h.tr((tr) => tr.setSelection(TextSelection.create(tr.doc, 1)))
|
||||
h.endComposing() // no doc change, no selection change
|
||||
expect(h.spans(spellPluginKey).length).toBe(1)
|
||||
})
|
||||
|
||||
it('a checker arriving mid-composition is applied once it ends', () => {
|
||||
const h = harness('公园 parc')
|
||||
h.startComposing()
|
||||
setSpellChecker(h.state, h.dispatch, english)
|
||||
expect(h.spans(spellPluginKey)).toEqual([])
|
||||
h.tr((tr) => tr.setSelection(TextSelection.create(tr.doc, 1)))
|
||||
h.endComposing()
|
||||
expect(h.spans(spellPluginKey).length).toBe(1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,112 @@
|
||||
import { Extension } from '@tiptap/core'
|
||||
import { Plugin, PluginKey } from '@tiptap/pm/state'
|
||||
import type { EditorState, Transaction } from '@tiptap/pm/state'
|
||||
import type { EditorView } from '@tiptap/pm/view'
|
||||
|
||||
// Composition tracks whether an IME composition is in flight, and is the one
|
||||
// place the rest of the editor asks.
|
||||
//
|
||||
// Why it exists: typing Chinese (or Japanese, or Korean) does not produce
|
||||
// characters a keystroke at a time. The IME opens a *composition* — the pinyin
|
||||
// she types goes into the document as it is typed, a candidate window sits over
|
||||
// it, and only when she picks a candidate is the run replaced with hanzi.
|
||||
// Petal's three decoration layers (SuggestionHighlight, SpellCheck,
|
||||
// SearchHighlight) all recompute from the live document on every change, so
|
||||
// mid-composition they would recompute over half-typed pinyin — and rebuilding
|
||||
// decorations means rewriting the DOM around the node the IME is composing in.
|
||||
// That is the classic bug that eats half-typed input: the composition is
|
||||
// abandoned by the browser and the letters vanish or double.
|
||||
//
|
||||
// The fix is to hold the redraws, not to skip them. Decorations that are due
|
||||
// while a composition is in flight are kept (mapped through the transaction, so
|
||||
// they follow the text that moved) and rebuilt the moment the composition ends.
|
||||
// Nothing is lost — the pause is measured in the length of one word.
|
||||
//
|
||||
// Input rules need no guard here: Tiptap's own input-rule plugin already returns
|
||||
// early while `view.composing` is true, which matters because pinyin uses an
|
||||
// apostrophe as a syllable separator (xi'an → 西安) and Typography.ts rewrites
|
||||
// every ' into a curly ’.
|
||||
|
||||
export const compositionKey = new PluginKey<boolean>('petalComposition')
|
||||
|
||||
// isComposing answers "was an IME composition in flight as of this state?".
|
||||
// Decoration plugins ask it of the state *before* the transaction they are
|
||||
// applying, which is what makes the answer independent of plugin ordering: the
|
||||
// flag was set by an earlier transaction (compositionstart), not by this one.
|
||||
export function isComposing(state: EditorState): boolean {
|
||||
return compositionKey.getState(state) === true
|
||||
}
|
||||
|
||||
// holdRedraw is the question every decoration layer asks in its `apply`: should
|
||||
// this rebuild wait? Yes while composing — except on the transaction that ends
|
||||
// the composition, which is precisely the one that releases the held redraws.
|
||||
export function holdRedraw(tr: Transaction, stateBefore: EditorState): boolean {
|
||||
if (tr.getMeta(compositionKey) === false) return false
|
||||
return isComposing(stateBefore)
|
||||
}
|
||||
|
||||
function setComposing(view: EditorView, composing: boolean) {
|
||||
if (compositionKey.getState(view.state) === composing) return
|
||||
view.dispatch(view.state.tr.setMeta(compositionKey, composing))
|
||||
}
|
||||
|
||||
export interface CompositionOptions {
|
||||
// Called once after a composition has ended and the document has settled.
|
||||
// EditorCore uses it to re-report the committed text, since the analysis
|
||||
// passes were told to ignore everything typed while composing.
|
||||
onEnd: (() => void) | null
|
||||
}
|
||||
|
||||
export function compositionPlugin(options: CompositionOptions): Plugin<boolean> {
|
||||
return new Plugin<boolean>({
|
||||
key: compositionKey,
|
||||
state: {
|
||||
init: () => false,
|
||||
apply(tr, value) {
|
||||
const meta = tr.getMeta(compositionKey)
|
||||
return typeof meta === 'boolean' ? meta : value
|
||||
},
|
||||
},
|
||||
props: {
|
||||
handleDOMEvents: {
|
||||
compositionstart: (view) => {
|
||||
setComposing(view, true)
|
||||
return false
|
||||
},
|
||||
// A custom handleDOMEvents handler runs *before* ProseMirror's own, and
|
||||
// ProseMirror's compositionend queues the composition's final DOM
|
||||
// changes as a microtask. Ending on a macrotask puts us after both, so
|
||||
// the rebuild we release sees the committed hanzi rather than the pinyin
|
||||
// it replaced. (If a transaction from that flush arrives first it
|
||||
// rebuilds anyway — by then `composing` is false. Both orders land.)
|
||||
compositionend: (view) => {
|
||||
setTimeout(() => {
|
||||
if (view.isDestroyed) return
|
||||
setComposing(view, false)
|
||||
options.onEnd?.()
|
||||
}, 0)
|
||||
return false
|
||||
},
|
||||
// Clicking away mid-candidate abandons the composition without a
|
||||
// compositionend in some browsers. Without this the layers would stay
|
||||
// held — silently, and until she typed again.
|
||||
blur: (view) => {
|
||||
setComposing(view, false)
|
||||
return false
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export const Composition = Extension.create<CompositionOptions>({
|
||||
name: 'composition',
|
||||
|
||||
addOptions() {
|
||||
return { onEnd: null }
|
||||
},
|
||||
|
||||
addProseMirrorPlugins() {
|
||||
return [compositionPlugin(this.options)]
|
||||
},
|
||||
})
|
||||
@@ -29,11 +29,14 @@ import { SelectionBubble } from './SelectionBubble'
|
||||
import { SearchHighlight } from './SearchHighlight'
|
||||
import { FindReplace } from './FindReplace'
|
||||
import { Typography } from './Typography'
|
||||
import { Composition } from './Composition'
|
||||
import { RewritePreview, type RewriteStatus } from './RewritePreview'
|
||||
import { planBatch } from './acceptBatch'
|
||||
import { api, type Suggestion, type SuggestionType, type WordInfo } from '../../api/client'
|
||||
import { speak, speechSupported } from '../../audio/speech'
|
||||
import type { SpellChecker } from '../../hooks/useSpellChecker'
|
||||
import type { Segmenter } from '../../lib/segment'
|
||||
import { hanziWordAt, hanziToWordInfo, hanziPinyin } from './hanziWord'
|
||||
import { usePack } from '../../i18n'
|
||||
|
||||
// Breathing room left below the last suggestion card when the rail's stack is what
|
||||
@@ -44,6 +47,17 @@ export interface EditorChange {
|
||||
content: string // Tiptap JSON, stringified
|
||||
content_text: string // flattened plain text for the LLM
|
||||
word_count: number
|
||||
// True while an IME composition is in flight: this text contains the pinyin
|
||||
// she is part-way through converting, not the sentence she is writing.
|
||||
//
|
||||
// The save is deliberately NOT gated on it — a tablet keyboard can hold one
|
||||
// composition open for a whole sentence, and Petal never makes writing wait
|
||||
// for anything. Saving an intermediate state costs nothing: the next change
|
||||
// supersedes it, and one always arrives (this component emits a final change
|
||||
// once the composition commits). What it gates is *analysis* — asking the
|
||||
// rule pack or the model to read half-typed pinyin can only produce advice
|
||||
// about text that is about to stop existing.
|
||||
composing: boolean
|
||||
}
|
||||
|
||||
interface Props {
|
||||
@@ -73,6 +87,11 @@ interface Props {
|
||||
// to the personal dictionary is bubbled up so it persists app-wide.
|
||||
spellChecker: SpellChecker | null
|
||||
onAddWord: (word: string) => void
|
||||
// The Chinese word list, non-null only for a writer learning the pair
|
||||
// language (users.direction = 'learning_pair'). Its presence is what turns on
|
||||
// every Chinese-side behaviour here: hanzi stops being text the editor steps
|
||||
// over and becomes words it can point at.
|
||||
segmenter: Segmenter | null
|
||||
}
|
||||
|
||||
interface MisspellState {
|
||||
@@ -95,6 +114,12 @@ interface WordInfoState {
|
||||
left: number
|
||||
loading: boolean
|
||||
info: WordInfo | null
|
||||
// The word's own pinyin, for a Chinese lookup. Kept beside `info` rather than
|
||||
// inside it because WordInfo is the English dictionary's shape and `phonetic`
|
||||
// there means IPA — printing pinyin between the slashes that say "this is
|
||||
// IPA" would be a small lie in the one place a learner is looking for the
|
||||
// truth about pronunciation.
|
||||
pinyin: string
|
||||
// Garden state: the captured word's id (null until the auto-capture returns or
|
||||
// after it's removed) and whether it's currently in the garden.
|
||||
vocabId: string | null
|
||||
@@ -188,6 +213,10 @@ interface GlossState {
|
||||
gloss: string
|
||||
// The other reading, when the token is a word in her language too.
|
||||
reverse?: string
|
||||
// A line shown *above* the meaning rather than below it: pinyin, for a
|
||||
// Chinese word. Above because it is read first — the meaning of 公园 may
|
||||
// already be clear to someone who cannot yet say it.
|
||||
lead?: string
|
||||
from: number
|
||||
to: number
|
||||
top: number
|
||||
@@ -234,6 +263,7 @@ export function EditorCore({
|
||||
onFocusMode,
|
||||
spellChecker,
|
||||
onAddWord,
|
||||
segmenter,
|
||||
}: Props) {
|
||||
// Her pair's copy — the hover tip labels the second reading with the language's
|
||||
// own name, so it says "português" rather than "pt-PT".
|
||||
@@ -312,6 +342,13 @@ export function EditorCore({
|
||||
// once at construction) can trigger a re-measure without stale closures.
|
||||
const recomputeRailRef = useRef<() => void>(() => {})
|
||||
|
||||
// Re-report the document once an IME composition commits. Everything typed
|
||||
// while composing was reported with `composing: true`, so the analysis passes
|
||||
// ignored it; without this nudge the committed sentence would wait for the
|
||||
// next keystroke to be looked at. Held in a ref because the extension list is
|
||||
// built once, at construction.
|
||||
const emitCommittedRef = useRef<() => void>(() => {})
|
||||
|
||||
const editor = useEditor({
|
||||
extensions: [
|
||||
StarterKit,
|
||||
@@ -329,6 +366,11 @@ export function EditorCore({
|
||||
TextAlign.configure({ types: ['heading', 'paragraph'] }),
|
||||
Placeholder.configure({ placeholder: 'Start writing…' }),
|
||||
CharacterCount,
|
||||
// First in the list so its state is settled before the layers that read
|
||||
// it — not that they depend on the ordering (they read the state as of
|
||||
// the transaction before), but the one that answers the question should
|
||||
// come before the ones that ask it.
|
||||
Composition.configure({ onEnd: () => emitCommittedRef.current() }),
|
||||
SuggestionHighlight,
|
||||
SpellCheck,
|
||||
SearchHighlight,
|
||||
@@ -373,6 +415,7 @@ export function EditorCore({
|
||||
content: JSON.stringify(editor.getJSON()),
|
||||
content_text: editor.getText(),
|
||||
word_count: editor.storage.characterCount.words(),
|
||||
composing: editor.view.composing,
|
||||
})
|
||||
// Edits reflow the text, so the rail anchors need re-measuring.
|
||||
recomputeRailRef.current()
|
||||
@@ -399,6 +442,26 @@ export function EditorCore({
|
||||
},
|
||||
})
|
||||
|
||||
// The composition-end nudge. Same payload as onUpdate's, with `composing`
|
||||
// false by construction — this runs after the composition has ended and its
|
||||
// final changes have been flushed, so the text here is the committed one.
|
||||
//
|
||||
// Written in an effect rather than during render, like recomputeRailRef
|
||||
// below: a render React throws away must not be the one that leaves its
|
||||
// closure behind for a DOM event to call later.
|
||||
useEffect(() => {
|
||||
emitCommittedRef.current = () => {
|
||||
if (!editor) return
|
||||
onChange({
|
||||
content: JSON.stringify(editor.getJSON()),
|
||||
content_text: editor.getText(),
|
||||
word_count: editor.storage.characterCount.words(),
|
||||
composing: false,
|
||||
})
|
||||
recomputeRailRef.current()
|
||||
}
|
||||
}, [editor, onChange])
|
||||
|
||||
// When the selected document changes, swap in its content without emitting an
|
||||
// update (false) so loading a doc doesn't trigger a spurious save.
|
||||
useEffect(() => {
|
||||
@@ -421,6 +484,26 @@ export function EditorCore({
|
||||
// popover would offer a definition of "cora".
|
||||
const wordAlphabet = spellChecker?.extendedAlphabet ?? false
|
||||
|
||||
// "The word under here", for a document that may hold two writing systems at
|
||||
// once — which every document in this pair does, because a learner's Chinese
|
||||
// practice is full of English and her English is full of quoted Chinese.
|
||||
//
|
||||
// Chinese is tried first and Latin second, and the order costs nothing to get
|
||||
// right: the two can never both answer, because a Han character is not a Latin
|
||||
// letter and neither tokenizer will cross into the other's run. `hanzi` rides
|
||||
// along because the two answers go to different dictionaries — the same
|
||||
// string is a word in exactly one of them.
|
||||
const resolveWord = useCallback(
|
||||
(pos: number): { from: number; to: number; word: string; hanzi: boolean } | null => {
|
||||
if (!editor) return null
|
||||
const han = hanziWordAt(editor.state.doc, pos, segmenter)
|
||||
if (han) return { ...han, hanzi: true }
|
||||
const latin = wordAt(editor.state.doc, pos, wordAlphabet)
|
||||
return latin ? { ...latin, hanzi: false } : null
|
||||
},
|
||||
[editor, segmenter, wordAlphabet],
|
||||
)
|
||||
|
||||
// Push the spell checker into its decoration plugin once the dictionary loads
|
||||
// (and again whenever the personal dictionary changes its identity).
|
||||
useEffect(() => {
|
||||
@@ -833,7 +916,7 @@ export function EditorCore({
|
||||
const openWordLookup = useCallback(
|
||||
(pos: number) => {
|
||||
if (!editor) return
|
||||
const range = wordAt(editor.state.doc, pos, wordAlphabet)
|
||||
const range = resolveWord(pos)
|
||||
if (!range) return
|
||||
const wrapper = wrapperRef.current
|
||||
if (!wrapper) return
|
||||
@@ -849,12 +932,18 @@ export function EditorCore({
|
||||
closeCard()
|
||||
setMisspell(null)
|
||||
const token = ++wordReqRef.current
|
||||
setWordInfo({ word: range.word, from: range.from, to: range.to, top, left, loading: true, info: null, vocabId: null, saved: false })
|
||||
setWordInfo({ word: range.word, from: range.from, to: range.to, top, left, loading: true, info: null, pinyin: '', vocabId: null, saved: false })
|
||||
// The sentence the word sits in, for review context in the garden.
|
||||
const example = exampleAt(range.from)
|
||||
api
|
||||
.lookupWord(range.word)
|
||||
.then((info) => {
|
||||
// Two dictionaries, one card. The Chinese lookup answers in English and
|
||||
// the English one answers in her language; which is wanted follows from
|
||||
// which script the word is written in, so nothing here has to consult the
|
||||
// account's direction a second time.
|
||||
const lookup: Promise<{ info: WordInfo; pinyin: string }> = range.hanzi
|
||||
? api.hanziWord(range.word).then((h) => ({ info: hanziToWordInfo(h), pinyin: hanziPinyin(h) }))
|
||||
: api.lookupWord(range.word).then((info) => ({ info, pinyin: '' }))
|
||||
lookup
|
||||
.then(({ info, pinyin }) => {
|
||||
if (token !== wordReqRef.current) return
|
||||
// Auto-capture into the vocabulary garden — only words the dictionary
|
||||
// actually knows (a real gloss or definition), so accidental lookups of
|
||||
@@ -864,14 +953,18 @@ export function EditorCore({
|
||||
// Reflect the saved state optimistically so the heart shows 💚 the
|
||||
// moment a known word loads, rather than flashing 🤍 until the capture
|
||||
// round-trips. vocabId is filled in when recordVocab returns.
|
||||
setWordInfo((w) => (w ? { ...w, loading: false, info, saved: known } : null))
|
||||
setWordInfo((w) => (w ? { ...w, loading: false, info, pinyin, saved: known } : null))
|
||||
if (!known) return
|
||||
api
|
||||
.recordVocab({
|
||||
word: range.word,
|
||||
gloss: info.gloss,
|
||||
definition: info.definitions[0]?.definition ?? '',
|
||||
phonetic: info.phonetic,
|
||||
// The garden's pronunciation field holds whichever this word has:
|
||||
// IPA for an English word, pinyin for a Chinese one. Both answer
|
||||
// the same question on a review card — how do I say this — and a
|
||||
// second column would only be a second thing to keep in sync.
|
||||
phonetic: pinyin || info.phonetic,
|
||||
example,
|
||||
doc_id: docId,
|
||||
})
|
||||
@@ -891,7 +984,7 @@ export function EditorCore({
|
||||
}
|
||||
})
|
||||
},
|
||||
[editor, closeCard, docId],
|
||||
[editor, closeCard, docId, resolveWord, exampleAt],
|
||||
)
|
||||
|
||||
// Toggle a looked-up word in/out of the vocabulary garden from the WordCard
|
||||
@@ -938,13 +1031,16 @@ export function EditorCore({
|
||||
if (!editor) return
|
||||
const coords = editor.view.posAtCoords({ left: e.clientX, top: e.clientY })
|
||||
if (!coords) return
|
||||
if (!wordAt(editor.state.doc, coords.pos, wordAlphabet)) return
|
||||
// Whichever script is under the pointer — the same resolver openWordLookup
|
||||
// uses, so a Chinese word gets the card here too rather than falling
|
||||
// through to the native menu.
|
||||
if (!resolveWord(coords.pos)) return
|
||||
e.preventDefault()
|
||||
// A misspelled word offers corrections first; otherwise look it up.
|
||||
if (openMisspellAt(coords.pos)) return
|
||||
openWordLookup(coords.pos)
|
||||
},
|
||||
[editor, wordAlphabet, openMisspellAt, openWordLookup],
|
||||
[editor, resolveWord, openMisspellAt, openWordLookup],
|
||||
)
|
||||
|
||||
// Touch has no hover or right-click, so a long-press (~500ms without moving)
|
||||
@@ -998,7 +1094,7 @@ export function EditorCore({
|
||||
clear()
|
||||
return
|
||||
}
|
||||
const range = wordAt(editor.state.doc, coords.pos, wordAlphabet)
|
||||
const range = resolveWord(coords.pos)
|
||||
if (!range) {
|
||||
clear()
|
||||
return
|
||||
@@ -1007,9 +1103,21 @@ export function EditorCore({
|
||||
if (gloss && gloss.from === range.from && gloss.to === range.to) return
|
||||
clearTimeout(glossTimer.current)
|
||||
const token = ++glossReqRef.current
|
||||
// The Chinese hover carries a second line the English one has no use for:
|
||||
// pinyin above the meaning. It is the thing a learner most often stops to
|
||||
// ask about their own writing — reading a character back is not the same
|
||||
// as being able to say it — and it is why this tooltip is worth having at
|
||||
// all for a script the writer can already read the meaning of half the
|
||||
// time.
|
||||
const ask = (): Promise<{ gloss: string; reverse?: string; lead?: string }> =>
|
||||
range.hanzi
|
||||
? api.hanziWord(range.word).then((h) => ({
|
||||
gloss: h.readings[0]?.senses ?? h.chars.map((c) => `${c.char} ${c.senses}`).join(' · '),
|
||||
lead: hanziPinyin(h),
|
||||
}))
|
||||
: api.glossWord(range.word).then((g) => ({ gloss: g.gloss, reverse: g.reverse }))
|
||||
glossTimer.current = setTimeout(() => {
|
||||
api
|
||||
.glossWord(range.word)
|
||||
ask()
|
||||
.then((g) => {
|
||||
if (token !== glossReqRef.current) return
|
||||
const wrapper = wrapperRef.current
|
||||
@@ -1024,14 +1132,14 @@ export function EditorCore({
|
||||
const wrapRect = wrapper.getBoundingClientRect()
|
||||
const left = Math.max(0, Math.min(start.left - wrapRect.left, wrapper.clientWidth - 280))
|
||||
const top = end.bottom - wrapRect.top + 6
|
||||
setGloss({ word: range.word, gloss: g.gloss, reverse: g.reverse, from: range.from, to: range.to, top, left })
|
||||
setGloss({ word: range.word, gloss: g.gloss, reverse: g.reverse, lead: g.lead, from: range.from, to: range.to, top, left })
|
||||
})
|
||||
.catch(() => {
|
||||
if (token === glossReqRef.current) setGloss(null)
|
||||
})
|
||||
}, 350)
|
||||
},
|
||||
[editor, wordAlphabet, selection, rewrite, misspell, wordInfo, pinned, gloss],
|
||||
[editor, resolveWord, selection, rewrite, misspell, wordInfo, pinned, gloss],
|
||||
)
|
||||
|
||||
// Leaving the editor surface drops any pending/shown gloss.
|
||||
@@ -1255,6 +1363,7 @@ export function EditorCore({
|
||||
{gloss && (
|
||||
<GlossTip
|
||||
gloss={gloss.gloss}
|
||||
lead={gloss.lead}
|
||||
reverse={gloss.reverse}
|
||||
reverseLang={pack.nativeName}
|
||||
style={{ top: gloss.top, left: gloss.left }}
|
||||
@@ -1287,6 +1396,7 @@ export function EditorCore({
|
||||
loading={wordInfo.loading}
|
||||
saved={wordInfo.saved}
|
||||
onToggleSave={toggleSaveWord}
|
||||
pinyin={wordInfo.pinyin}
|
||||
style={{ top: wordInfo.top, left: wordInfo.left }}
|
||||
onReplace={replaceWord}
|
||||
/>
|
||||
|
||||
@@ -2,6 +2,7 @@ import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import type { Editor } from '@tiptap/react'
|
||||
import { clearSearch, getSearchState, setActive, setSearch } from './SearchHighlight'
|
||||
import { usePack } from '../../i18n'
|
||||
import { fromIME } from '../../lib/ime'
|
||||
|
||||
// FindReplace is the in-document search bar (Ctrl/Cmd+F). It drives the
|
||||
// SearchHighlight decoration layer: typing updates the highlighted matches, the
|
||||
@@ -108,7 +109,10 @@ export function FindReplace({ editor, onClose }: Props) {
|
||||
role="dialog"
|
||||
aria-label="Find and replace"
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Escape') {
|
||||
// Both fields take Chinese, so both take an IME: Escape cancels a
|
||||
// candidate and Enter commits one. A key that belongs to the composition
|
||||
// is not a command here — see lib/ime.
|
||||
if (e.key === 'Escape' && !fromIME(e)) {
|
||||
e.preventDefault()
|
||||
onClose()
|
||||
}
|
||||
@@ -137,7 +141,7 @@ export function FindReplace({ editor, onClose }: Props) {
|
||||
value={query}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
if (e.key === 'Enter' && !fromIME(e)) {
|
||||
e.preventDefault()
|
||||
go(e.shiftKey ? -1 : 1)
|
||||
}
|
||||
@@ -171,7 +175,7 @@ export function FindReplace({ editor, onClose }: Props) {
|
||||
value={replacement}
|
||||
onChange={(e) => setReplacement(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
if (e.key === 'Enter' && !fromIME(e)) {
|
||||
e.preventDefault()
|
||||
replaceActive()
|
||||
}
|
||||
|
||||
@@ -7,6 +7,11 @@
|
||||
|
||||
interface Props {
|
||||
gloss: string
|
||||
// A line above the gloss, in a lighter weight: the pinyin of a Chinese word.
|
||||
// It leads because it is what is actually being asked — a learner reading
|
||||
// their own 公园 back may know it means a park and still not know how to say
|
||||
// it, which is the one thing the character does not tell them.
|
||||
lead?: string
|
||||
// The English meaning of the same token read as a word of the writer's own
|
||||
// language, when it is one. On a Latin-script pair "sale" is both, and the
|
||||
// bubble shows the two readings stacked rather than picking one — the same
|
||||
@@ -17,7 +22,7 @@ interface Props {
|
||||
style: React.CSSProperties
|
||||
}
|
||||
|
||||
export function GlossTip({ gloss, reverse, reverseLang, style }: Props) {
|
||||
export function GlossTip({ gloss, lead, reverse, reverseLang, style }: Props) {
|
||||
return (
|
||||
<div
|
||||
className="petal-gloss-tip pointer-events-none absolute z-20 px-2.5 py-1.5 text-sm"
|
||||
@@ -33,6 +38,11 @@ export function GlossTip({ gloss, reverse, reverseLang, style }: Props) {
|
||||
...style,
|
||||
}}
|
||||
>
|
||||
{lead && (
|
||||
<span className="mb-0.5 block font-semibold" style={{ opacity: 0.9 }}>
|
||||
{lead}
|
||||
</span>
|
||||
)}
|
||||
{gloss}
|
||||
{reverse && (
|
||||
<span className="mt-0.5 block" style={{ opacity: 0.72, fontSize: '0.85em' }}>
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { EditorState, Transaction } from '@tiptap/pm/state'
|
||||
import { Decoration, DecorationSet } from '@tiptap/pm/view'
|
||||
import type { Node as PMNode } from '@tiptap/pm/model'
|
||||
import { mapOffset } from './SuggestionHighlight'
|
||||
import { holdRedraw } from './Composition'
|
||||
|
||||
// SearchHighlight powers the in-document Find & Replace bar. Like the suggestion
|
||||
// layer it uses ProseMirror *decorations* (not stored marks), so matches are
|
||||
@@ -22,6 +23,11 @@ interface PluginState {
|
||||
matches: Match[]
|
||||
active: number // index into matches, or -1 when there are none
|
||||
decorations: DecorationSet
|
||||
// Held back while an IME composition was in flight — see Composition.ts.
|
||||
// `matches` is held with the decorations rather than recomputed on its own:
|
||||
// the Find bar's "3 / 7" and the wash on the page are one answer, and half of
|
||||
// it moving while the other half waits would be worse than both waiting.
|
||||
stale: boolean
|
||||
}
|
||||
|
||||
export const searchPluginKey = new PluginKey<PluginState>('petalSearch')
|
||||
@@ -57,7 +63,7 @@ function build(doc: PMNode, query: string, caseSensitive: boolean, preferred: nu
|
||||
class: i === active ? 'petal-find-match petal-find-match-active' : 'petal-find-match',
|
||||
}),
|
||||
)
|
||||
return { query, caseSensitive, matches, active, decorations: DecorationSet.create(doc, decos) }
|
||||
return { query, caseSensitive, matches, active, decorations: DecorationSet.create(doc, decos), stale: false }
|
||||
}
|
||||
|
||||
const EMPTY: PluginState = {
|
||||
@@ -66,6 +72,7 @@ const EMPTY: PluginState = {
|
||||
matches: [],
|
||||
active: -1,
|
||||
decorations: DecorationSet.empty,
|
||||
stale: false,
|
||||
}
|
||||
|
||||
// setSearch updates the query / case-sensitivity and recomputes matches. Passing
|
||||
@@ -100,36 +107,49 @@ type Meta =
|
||||
| { kind: 'active'; index: number }
|
||||
| { kind: 'clear' }
|
||||
|
||||
export const SearchHighlight = Extension.create({
|
||||
name: 'searchHighlight',
|
||||
|
||||
addProseMirrorPlugins() {
|
||||
return [
|
||||
new Plugin<PluginState>({
|
||||
export function searchPlugin(): Plugin<PluginState> {
|
||||
return new Plugin<PluginState>({
|
||||
key: searchPluginKey,
|
||||
state: {
|
||||
init: () => EMPTY,
|
||||
apply(tr, value, _oldState, newState): PluginState {
|
||||
apply(tr, value, oldState, newState): PluginState {
|
||||
const meta = tr.getMeta(searchPluginKey) as Meta | undefined
|
||||
if (meta?.kind === 'search') {
|
||||
return build(newState.doc, meta.query, meta.caseSensitive, value.active < 0 ? 0 : value.active)
|
||||
}
|
||||
// Clearing the layer is the one thing a composition never holds: it
|
||||
// removes decorations rather than adding them, and it is what closing
|
||||
// the Find bar does.
|
||||
if (meta?.kind === 'clear') return EMPTY
|
||||
|
||||
const held = holdRedraw(tr, oldState)
|
||||
const query = meta?.kind === 'search' ? meta.query : value.query
|
||||
const caseSensitive = meta?.kind === 'search' ? meta.caseSensitive : value.caseSensitive
|
||||
|
||||
if (meta?.kind === 'active') {
|
||||
if (value.matches.length === 0) return value
|
||||
const active = ((meta.index % value.matches.length) + value.matches.length) % value.matches.length
|
||||
if (held) return { ...value, active, stale: true }
|
||||
const decos = value.matches.map((m, i) =>
|
||||
Decoration.inline(m.from, m.to, {
|
||||
class: i === active ? 'petal-find-match petal-find-match-active' : 'petal-find-match',
|
||||
}),
|
||||
)
|
||||
return { ...value, active, decorations: DecorationSet.create(newState.doc, decos) }
|
||||
return { ...value, active, stale: false, decorations: DecorationSet.create(newState.doc, decos) }
|
||||
}
|
||||
if (meta?.kind === 'clear') return EMPTY
|
||||
// Re-anchor on any document change so highlights track edits/replaces.
|
||||
if (tr.docChanged && value.query) {
|
||||
return build(newState.doc, value.query, value.caseSensitive, value.active)
|
||||
|
||||
// A new query, or any document change: re-anchor so highlights track
|
||||
// edits and replaces. Once due, it stays due until it happens.
|
||||
const due = value.stale || meta?.kind === 'search' || (tr.docChanged && !!value.query)
|
||||
if (!due) return value
|
||||
if (held) {
|
||||
return {
|
||||
...value,
|
||||
query,
|
||||
caseSensitive,
|
||||
stale: true,
|
||||
decorations: tr.docChanged ? value.decorations.map(tr.mapping, tr.doc) : value.decorations,
|
||||
}
|
||||
return value
|
||||
}
|
||||
const preferred = meta?.kind === 'search' && value.active < 0 ? 0 : value.active
|
||||
return build(newState.doc, query, caseSensitive, preferred)
|
||||
},
|
||||
},
|
||||
props: {
|
||||
@@ -137,7 +157,13 @@ export const SearchHighlight = Extension.create({
|
||||
return searchPluginKey.getState(state)?.decorations
|
||||
},
|
||||
},
|
||||
}),
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
export const SearchHighlight = Extension.create({
|
||||
name: 'searchHighlight',
|
||||
|
||||
addProseMirrorPlugins() {
|
||||
return [searchPlugin()]
|
||||
},
|
||||
})
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { EditorState, Transaction } from '@tiptap/pm/state'
|
||||
import { Decoration, DecorationSet } from '@tiptap/pm/view'
|
||||
import type { Node as PMNode } from '@tiptap/pm/model'
|
||||
import { mapOffset } from './SuggestionHighlight'
|
||||
import { holdRedraw } from './Composition'
|
||||
import type { SpellChecker } from '../../hooks/useSpellChecker'
|
||||
|
||||
// SpellCheck renders browser-side nspell misspellings as ProseMirror
|
||||
@@ -18,6 +19,11 @@ export const spellPluginKey = new PluginKey<PluginState>('petalSpellCheck')
|
||||
interface PluginState {
|
||||
checker: SpellChecker | null
|
||||
decorations: DecorationSet
|
||||
// Held back while an IME composition was in flight — see Composition.ts. This
|
||||
// layer is the one with the most to gain from the guard: the pinyin she is
|
||||
// part-way through typing is Latin letters, so it is exactly what the
|
||||
// tokenizer picks up and exactly what an underline would redraw over.
|
||||
stale: boolean
|
||||
}
|
||||
|
||||
// A word is a run of Latin letters with optional internal/edge apostrophes
|
||||
@@ -151,25 +157,31 @@ export function setSpellChecker(
|
||||
dispatch(state.tr.setMeta(spellPluginKey, checker ?? null))
|
||||
}
|
||||
|
||||
export const SpellCheck = Extension.create({
|
||||
name: 'spellCheck',
|
||||
|
||||
addProseMirrorPlugins() {
|
||||
return [
|
||||
new Plugin<PluginState>({
|
||||
export function spellPlugin(): Plugin<PluginState> {
|
||||
return new Plugin<PluginState>({
|
||||
key: spellPluginKey,
|
||||
state: {
|
||||
init: () => ({ checker: null, decorations: DecorationSet.empty }),
|
||||
apply(tr, value, _oldState, newState) {
|
||||
init: () => ({ checker: null, decorations: DecorationSet.empty, stale: false }),
|
||||
apply(tr, value, oldState, newState) {
|
||||
const meta = tr.getMeta(spellPluginKey) as SpellChecker | null | undefined
|
||||
const checker = meta !== undefined ? meta : value.checker
|
||||
if (!checker) return { checker: null, decorations: DecorationSet.empty }
|
||||
if (!checker) return { checker: null, decorations: DecorationSet.empty, stale: false }
|
||||
// Rebuild on a checker swap, a doc edit, or a caret move (so the word
|
||||
// you just left gets re-evaluated and the new caret word is exempt).
|
||||
if (meta !== undefined || tr.docChanged || tr.selectionSet) {
|
||||
return { checker, decorations: buildDecorations(newState.doc, checker, newState.selection.head) }
|
||||
const due = value.stale || meta !== undefined || tr.docChanged || tr.selectionSet
|
||||
if (!due) return value
|
||||
if (holdRedraw(tr, oldState)) {
|
||||
return {
|
||||
checker,
|
||||
stale: true,
|
||||
decorations: tr.docChanged ? value.decorations.map(tr.mapping, tr.doc) : value.decorations,
|
||||
}
|
||||
}
|
||||
return {
|
||||
checker,
|
||||
stale: false,
|
||||
decorations: buildDecorations(newState.doc, checker, newState.selection.head),
|
||||
}
|
||||
return { checker, decorations: value.decorations }
|
||||
},
|
||||
},
|
||||
props: {
|
||||
@@ -177,7 +189,13 @@ export const SpellCheck = Extension.create({
|
||||
return spellPluginKey.getState(state)?.decorations
|
||||
},
|
||||
},
|
||||
}),
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
export const SpellCheck = Extension.create({
|
||||
name: 'spellCheck',
|
||||
|
||||
addProseMirrorPlugins() {
|
||||
return [spellPlugin()]
|
||||
},
|
||||
})
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { EditorState, Transaction } from '@tiptap/pm/state'
|
||||
import { Decoration, DecorationSet } from '@tiptap/pm/view'
|
||||
import type { Node as PMNode } from '@tiptap/pm/model'
|
||||
import type { Suggestion } from '../../api/client'
|
||||
import { holdRedraw } from './Composition'
|
||||
|
||||
// SuggestionHighlight renders LLM suggestions as ProseMirror *decorations*, not
|
||||
// stored marks. Decorations are ephemeral overlays recomputed from the live
|
||||
@@ -21,6 +22,10 @@ interface PluginState {
|
||||
// decoration repaints that fire on every document change.
|
||||
activeId: string | null
|
||||
decorations: DecorationSet
|
||||
// A rebuild fell due while an IME composition was in flight and was held back
|
||||
// (see Composition.ts). The decorations on screen are the previous ones,
|
||||
// mapped forward; this says they still owe a rebuild.
|
||||
stale: boolean
|
||||
}
|
||||
|
||||
// Meta carried on a transaction to update the plugin: either a fresh suggestion
|
||||
@@ -173,40 +178,37 @@ export function setActiveSuggestion(
|
||||
dispatch(state.tr.setMeta(suggestionPluginKey, { activeId } satisfies SuggestionMeta))
|
||||
}
|
||||
|
||||
export const SuggestionHighlight = Extension.create({
|
||||
name: 'suggestionHighlight',
|
||||
|
||||
addProseMirrorPlugins() {
|
||||
return [
|
||||
new Plugin<PluginState>({
|
||||
export function suggestionPlugin(): Plugin<PluginState> {
|
||||
return new Plugin<PluginState>({
|
||||
key: suggestionPluginKey,
|
||||
state: {
|
||||
init: () => ({ suggestions: [], activeId: null, decorations: DecorationSet.empty }),
|
||||
apply(tr, value, _oldState, newState) {
|
||||
init: () => ({ suggestions: [], activeId: null, decorations: DecorationSet.empty, stale: false }),
|
||||
apply(tr, value, oldState, newState) {
|
||||
const meta = tr.getMeta(suggestionPluginKey) as SuggestionMeta | undefined
|
||||
if (meta && 'suggestions' in meta) {
|
||||
const suggestions = meta && 'suggestions' in meta ? meta.suggestions : value.suggestions
|
||||
const activeId = meta && 'activeId' in meta ? meta.activeId : value.activeId
|
||||
// A rebuild is due on a new list, a new emphasis, or any document change
|
||||
// (which is how a suggestion re-anchors by string), and stays due until
|
||||
// it happens.
|
||||
const due = value.stale || meta !== undefined || tr.docChanged
|
||||
if (!due) return value
|
||||
if (holdRedraw(tr, oldState)) {
|
||||
return {
|
||||
suggestions: meta.suggestions,
|
||||
activeId: value.activeId,
|
||||
decorations: buildDecorations(newState.doc, meta.suggestions, value.activeId),
|
||||
suggestions,
|
||||
activeId,
|
||||
stale: true,
|
||||
// Map rather than keep: the composing text is growing under these
|
||||
// highlights, and an unmapped decoration would drift a character at
|
||||
// a time across a word she is still typing.
|
||||
decorations: tr.docChanged ? value.decorations.map(tr.mapping, tr.doc) : value.decorations,
|
||||
}
|
||||
}
|
||||
if (meta && 'activeId' in meta) {
|
||||
return {
|
||||
suggestions: value.suggestions,
|
||||
activeId: meta.activeId,
|
||||
decorations: buildDecorations(newState.doc, value.suggestions, meta.activeId),
|
||||
suggestions,
|
||||
activeId,
|
||||
stale: false,
|
||||
decorations: buildDecorations(newState.doc, suggestions, activeId),
|
||||
}
|
||||
}
|
||||
// On any document change, re-anchor by string against the new doc.
|
||||
if (tr.docChanged) {
|
||||
return {
|
||||
suggestions: value.suggestions,
|
||||
activeId: value.activeId,
|
||||
decorations: buildDecorations(newState.doc, value.suggestions, value.activeId),
|
||||
}
|
||||
}
|
||||
return value
|
||||
},
|
||||
},
|
||||
props: {
|
||||
@@ -214,7 +216,13 @@ export const SuggestionHighlight = Extension.create({
|
||||
return suggestionPluginKey.getState(state)?.decorations
|
||||
},
|
||||
},
|
||||
}),
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
export const SuggestionHighlight = Extension.create({
|
||||
name: 'suggestionHighlight',
|
||||
|
||||
addProseMirrorPlugins() {
|
||||
return [suggestionPlugin()]
|
||||
},
|
||||
})
|
||||
|
||||
@@ -17,11 +17,16 @@ interface Props {
|
||||
// heart toggles it; `onToggleSave` removes/re-adds it.
|
||||
saved: boolean
|
||||
onToggleSave: () => void
|
||||
// A Chinese word's pinyin. Shown in place of the IPA line and *without* the
|
||||
// slashes, because pinyin is not a phonetic transcription — it is how the word
|
||||
// is spelled in letters, and the slashes would say something untrue about it
|
||||
// in the one place a learner is looking for the truth about pronunciation.
|
||||
pinyin?: string
|
||||
style: React.CSSProperties
|
||||
onReplace: (synonym: string) => void
|
||||
}
|
||||
|
||||
export function WordCard({ word, info, loading, saved, onToggleSave, style, onReplace }: Props) {
|
||||
export function WordCard({ word, info, loading, saved, onToggleSave, pinyin, style, onReplace }: Props) {
|
||||
const t = usePack()
|
||||
const definitions = info?.definitions ?? []
|
||||
const synonyms = info?.synonyms ?? []
|
||||
@@ -117,9 +122,10 @@ export function WordCard({ word, info, loading, saved, onToggleSave, style, onRe
|
||||
when she has found a word she likes — "can I use this?". Both are
|
||||
quiet, muted lines: information she can take or leave, never a verdict
|
||||
on her writing. */}
|
||||
{(phonetic || band) && (
|
||||
{(phonetic || pinyin || band) && (
|
||||
<div className="mt-1.5 flex items-center gap-2 text-sm">
|
||||
{phonetic && <span style={{ color: 'var(--color-muted)' }}>/{phonetic}/</span>}
|
||||
{pinyin && <span style={{ color: 'var(--color-muted)' }}>{pinyin}</span>}
|
||||
{!pinyin && phonetic && <span style={{ color: 'var(--color-muted)' }}>/{phonetic}/</span>}
|
||||
{band && (
|
||||
<span
|
||||
className="rounded-full px-2 py-0.5 text-xs font-semibold"
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import type { Node as PMNode } from '@tiptap/pm/model'
|
||||
|
||||
import { mapOffset } from './SuggestionHighlight'
|
||||
import type { Segmenter } from '../../lib/segment'
|
||||
import type { HanziInfo, WordInfo } from '../../api/client'
|
||||
|
||||
// The Chinese counterpart of `wordAt` (SpellCheck.ts): given a position in the
|
||||
// document, which Chinese word is there.
|
||||
//
|
||||
// It lives in its own file rather than as a branch inside `wordAt` because the
|
||||
// two answer the same question by genuinely different means — one runs a regex
|
||||
// over the text, the other runs a shortest-path walk over a 188k-word list it
|
||||
// had to fetch — and only one of them is about spelling at all. What they share
|
||||
// is the part that matters for correctness: the offset→position mapping, which
|
||||
// is `mapOffset`, the same function the suggestion, spell and search decoration
|
||||
// layers all anchor through.
|
||||
|
||||
export interface HanziRange {
|
||||
from: number
|
||||
to: number
|
||||
word: string
|
||||
}
|
||||
|
||||
// blockAt finds the textblock containing pos, along with where that block starts
|
||||
// — everything else here is arithmetic within one block.
|
||||
//
|
||||
// Segmentation is per-block for the same reason the spell tokenizer is: a word
|
||||
// cannot span a paragraph break, and a block is the largest unit whose text is
|
||||
// contiguous in the document.
|
||||
function blockAt(doc: PMNode, pos: number): { node: PMNode; start: number } | null {
|
||||
let found: { node: PMNode; start: number } | null = null
|
||||
doc.descendants((node, nodePos) => {
|
||||
if (found) return false
|
||||
if (!node.isTextblock) return true
|
||||
if (pos <= nodePos || pos >= nodePos + node.nodeSize) return false
|
||||
found = { node, start: nodePos }
|
||||
return false
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
// offsetOf is the inverse of mapOffset: an absolute ProseMirror position to a
|
||||
// character offset within the block's flattened text. Inline atoms (a hard
|
||||
// break) occupy a position and contribute no text, so the two are not the same
|
||||
// number and subtracting the block position would be wrong in any paragraph
|
||||
// containing one.
|
||||
function offsetOf(block: PMNode, blockStart: number, pos: number): number {
|
||||
let textOffset = 0
|
||||
let pmPos = blockStart + 1
|
||||
let result = -1
|
||||
block.forEach((child) => {
|
||||
if (result >= 0) return
|
||||
const len = child.isText ? (child.text?.length ?? 0) : 0
|
||||
if (pos <= pmPos + child.nodeSize) {
|
||||
result = textOffset + Math.max(0, Math.min(pos - pmPos, len))
|
||||
return
|
||||
}
|
||||
textOffset += len
|
||||
pmPos += child.nodeSize
|
||||
})
|
||||
return result >= 0 ? result : textOffset
|
||||
}
|
||||
|
||||
// hanziWordAt resolves the Chinese word at a document position, or null when
|
||||
// there is no Chinese there — which is the ordinary case in a mixed paragraph
|
||||
// and is why the caller falls through to the Latin tokenizer.
|
||||
export function hanziWordAt(doc: PMNode, pos: number, segmenter: Segmenter | null): HanziRange | null {
|
||||
if (!segmenter) return null
|
||||
const block = blockAt(doc, pos)
|
||||
if (!block) return null
|
||||
const text = block.node.textContent
|
||||
if (!text) return null
|
||||
|
||||
const token = segmenter.wordAt(text, offsetOf(block.node, block.start, pos))
|
||||
if (!token) return null
|
||||
return {
|
||||
from: mapOffset(block.node, block.start, token.from),
|
||||
to: mapOffset(block.node, block.start, token.to),
|
||||
word: token.word,
|
||||
}
|
||||
}
|
||||
|
||||
// hanziToWordInfo adapts a Chinese lookup into the shape the word card already
|
||||
// renders.
|
||||
//
|
||||
// An adapter rather than a second card, because everything around the card is
|
||||
// the same in both directions: it opens the same way, anchors the same way,
|
||||
// captures into the same vocabulary garden, and reads aloud through the same
|
||||
// voice — the zh pair already speaks Chinese, so 🔊 needs nothing new to say
|
||||
// 公园 out loud. What differs is only which fields carry what.
|
||||
//
|
||||
// * `definitions` holds one entry per reading, labelled with its pinyin. The
|
||||
// part-of-speech slot is where the card puts a short italic prefix, which
|
||||
// is exactly the shape a reading label wants — and a reading *is* the thing
|
||||
// that distinguishes these senses from each other (得 dé "to obtain" from 得
|
||||
// de, the complement marker).
|
||||
// * `gloss` stays empty. It means "translated into the writer's language",
|
||||
// and for a writer learning Chinese that language is English, which is what
|
||||
// the senses already are. Putting the English there too would print it
|
||||
// twice.
|
||||
// * The per-character fallback fills the same list, labelled by character, so
|
||||
// a compound with no headword still says something true about itself.
|
||||
export function hanziToWordInfo(info: HanziInfo): WordInfo {
|
||||
const definitions =
|
||||
info.readings.length > 0
|
||||
? info.readings.map((r) => ({ part_of_speech: r.pinyin, definition: r.senses }))
|
||||
: info.chars.map((c) => ({ part_of_speech: `${c.char} ${c.pinyin}`, definition: c.senses }))
|
||||
return {
|
||||
word: info.word,
|
||||
gloss: '',
|
||||
phonetic: '',
|
||||
definitions,
|
||||
synonyms: [],
|
||||
frequency: 0,
|
||||
difficulty: -1,
|
||||
etymology: '',
|
||||
}
|
||||
}
|
||||
|
||||
// hanziPinyin is the word's own pronunciation, for the line under the headword.
|
||||
// Empty when only the character fallback answered: the characters' readings are
|
||||
// not the word's reading — 不 is bù alone and bú before a fourth tone — and
|
||||
// printing them joined up would be inventing a pronunciation.
|
||||
export function hanziPinyin(info: HanziInfo): string {
|
||||
return info.readings[0]?.pinyin ?? ''
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,39 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
|
||||
import { loadSegmenter, type Segmenter } from '../lib/segment'
|
||||
|
||||
// Loads the Chinese word list, once per session, and only for a writer who is
|
||||
// going to use it.
|
||||
//
|
||||
// Modelled on useSpellChecker, and gated harder. That hook loads for everyone,
|
||||
// because everyone's English gets spell-checked; this one loads a megabyte for
|
||||
// the one direction that needs it, and an account practising English would
|
||||
// never ask a single question of it. The gate is the writer's own setting rather
|
||||
// than a guess from their text: a Mandarin native drafting English quotes
|
||||
// Chinese in it constantly, and none of that is what this is for.
|
||||
//
|
||||
// A failure resolves to null, which every consumer already handles as "no
|
||||
// segmentation" — the Chinese hover quietly does nothing rather than the editor
|
||||
// refusing to open.
|
||||
export function useSegmenter(enabled: boolean): Segmenter | null {
|
||||
const [segmenter, setSegmenter] = useState<Segmenter | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled) {
|
||||
// Turning the direction back drops it. It is a megabyte of resident map
|
||||
// whose only consumer just switched off, and re-loading costs one fetch
|
||||
// that the browser cache answers.
|
||||
setSegmenter(null)
|
||||
return
|
||||
}
|
||||
let cancelled = false
|
||||
loadSegmenter().then((seg) => {
|
||||
if (!cancelled) setSegmenter(seg)
|
||||
})
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [enabled])
|
||||
|
||||
return segmenter
|
||||
}
|
||||
@@ -42,5 +42,27 @@ export function useSession() {
|
||||
}
|
||||
}, [])
|
||||
|
||||
return { me, signedOut }
|
||||
// Turn the pair around. The account is the source of truth for which
|
||||
// direction the editor is in — it decides whether the word list loads at all —
|
||||
// so the state moves only once the server has agreed, and it moves to what the
|
||||
// server *stored* rather than to what was asked for.
|
||||
const setDirection = async (direction: string) => {
|
||||
const updated = await api.setDirection(direction)
|
||||
setMe(updated)
|
||||
}
|
||||
|
||||
// Move the pair, naming the direction with it. The two are validated together
|
||||
// server-side, so an account that is learning Chinese cannot change pair by
|
||||
// sending `pair_lang` alone — the combination it would ask for (French with
|
||||
// segmentation) does not exist and is refused. Saying both is how that move is
|
||||
// made, and routing it through here rather than through the picker's own
|
||||
// `api` call is what keeps `me.direction` — which decides whether the word
|
||||
// list stays loaded — in step with what was actually stored.
|
||||
const setPair = async (lang: string, direction: string) => {
|
||||
const updated = await api.setPair(lang, direction)
|
||||
setPackLang(updated.pair_lang)
|
||||
setMe(updated)
|
||||
}
|
||||
|
||||
return { me, signedOut, setDirection, setPair }
|
||||
}
|
||||
|
||||
@@ -98,6 +98,23 @@ const PAIR_DICTS: Partial<Record<PairLang, DictSpec>> = {
|
||||
extendedAlphabet: true,
|
||||
elision: FR_ELISION,
|
||||
},
|
||||
// Spanish glues its pronouns onto the *end* of a verb rather than little words
|
||||
// onto the front, so there is no elision list here — and none is needed: the
|
||||
// upstream dictionary carries the enclitic forms itself (dámelo, hacérselo,
|
||||
// escribiéndolo), because unlike French's thirty-four prefix rules they do not
|
||||
// multiply the word list into the megabytes.
|
||||
//
|
||||
// This is RLA's *generic* build, not Debian's hunspell-es — the latter is the
|
||||
// peninsular one under a pan-Hispanic-looking name, and it rejects vení and
|
||||
// tenés. See web/public/dictionaries/es/LICENSE: the whole of Spanish is
|
||||
// accepted here, because underlining is the only thing this file can do.
|
||||
es: {
|
||||
lang: 'es',
|
||||
aff: '/dictionaries/es/es.aff',
|
||||
dic: '/dictionaries/es/es.dic.gz',
|
||||
gzipped: true,
|
||||
extendedAlphabet: true,
|
||||
},
|
||||
}
|
||||
|
||||
// Where the list lived before it had an owner (Phase 7). Read once, handed to
|
||||
|
||||
+114
-5
@@ -4,12 +4,13 @@ import { onPackChange, pack, resetPackForTests, setPackLang, shippedPacks } from
|
||||
import { zh } from './packs/zh'
|
||||
import { ptPT } from './packs/pt-PT'
|
||||
import { fr } from './packs/fr'
|
||||
import { es } from './packs/es'
|
||||
import type { Pack } from './types'
|
||||
|
||||
// Every pack that ships. Shape assertions run over all of them, because the
|
||||
// point of Phase 19 was that a language is data — and data that only the first
|
||||
// author's pack satisfies isn't a shape, it's a coincidence.
|
||||
const PACKS: Pack[] = [zh, ptPT, fr]
|
||||
const PACKS: Pack[] = [zh, ptPT, fr, es]
|
||||
|
||||
// Every string a pack would ever put on screen, and nothing else — field names
|
||||
// excluded (see the pt-PT grep below for what including them cost). Templates are
|
||||
@@ -49,8 +50,10 @@ describe('pack selection', () => {
|
||||
it('falls back rather than blanking on a pair with no pack yet', () => {
|
||||
// A pair_lang the deployment has no copy for is a deployment that got ahead
|
||||
// of its translation. She should still get a working editor. (This was 'fr'
|
||||
// until Phase 24 gave fr a pack; 'es' is the pair still waiting for one.)
|
||||
setPackLang('es')
|
||||
// until Phase 24, then 'es' until Phase 25 — every pair PairLang names now
|
||||
// has a pack, so the stand-in is a regional code Petal has not decided
|
||||
// about, which is the realistic version of this failure anyway.)
|
||||
setPackLang('es-ES')
|
||||
expect(pack()).toBe(zh)
|
||||
setPackLang('klingon')
|
||||
expect(pack()).toBe(zh)
|
||||
@@ -70,7 +73,7 @@ describe('pack selection', () => {
|
||||
expect(seen).not.toHaveBeenCalled()
|
||||
|
||||
// An unshipped pair resolves back to zh, which is also not a change.
|
||||
setPackLang('es')
|
||||
setPackLang('es-ES')
|
||||
expect(seen).not.toHaveBeenCalled()
|
||||
|
||||
setPackLang('pt-PT')
|
||||
@@ -83,7 +86,7 @@ describe('pack selection', () => {
|
||||
// matching allowlist exists to enforce from the other side.
|
||||
it('offers exactly the pairs it has copy for', () => {
|
||||
const codes = shippedPacks().map((p) => p.code)
|
||||
expect(codes.sort()).toEqual(['fr', 'pt-PT', 'zh'])
|
||||
expect(codes.sort()).toEqual(['es', 'fr', 'pt-PT', 'zh'])
|
||||
// Every offered pair names itself, because a writer stranded on the wrong
|
||||
// pack can only read the label that is in her own language.
|
||||
for (const p of shippedPacks()) expect(p.nativeName.length).toBeGreaterThan(0)
|
||||
@@ -364,6 +367,97 @@ describe('the fr pack', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('the es pack', () => {
|
||||
// Spanish has no regional question in its dictionary at all — hunspell-es
|
||||
// ships twenty country codes and every one is a symlink to one pan-Hispanic
|
||||
// word list — so, even more than with French, the entire regional decision
|
||||
// lives in this file. Neutral Latin American was chosen deliberately, and a
|
||||
// stray peninsular form is invisible to everyone reviewing the diff.
|
||||
it('is Latin American, not peninsular', () => {
|
||||
const text = copyOf(es).toLowerCase()
|
||||
for (const bad of [
|
||||
'ordenador', 'vosotros', 'zumo', 'patata', 'coche',
|
||||
'gafas', 'billete', 'chaval', 'guay',
|
||||
// The one that is not merely regional: *coger* is an everyday verb in
|
||||
// Spain and obscene through most of Latin America. A companion in a
|
||||
// private notebook must never produce it by accident.
|
||||
'coger',
|
||||
]) {
|
||||
expect(text, `peninsular form "${bad}" in the es pack`).not.toMatch(
|
||||
new RegExp(`\\b${bad}\\b`),
|
||||
)
|
||||
}
|
||||
// And the accent it is read aloud in. Six of Piper's nine Spanish voices
|
||||
// are es_ES, so the wrong country is the easy default here — the pt-PT
|
||||
// trap, not the fr non-question.
|
||||
expect(es.locale).toBe('es-MX')
|
||||
})
|
||||
|
||||
// Spanish opens its questions and exclamations, and the easiest place to
|
||||
// forget is exactly where a reviewer's eye slides past: inside a Line's
|
||||
// native half, and inside an interpolated template. Every opening mark in the
|
||||
// pack is deliberate; a missing one is a typo the type system cannot see.
|
||||
it('opens every question and exclamation it closes', () => {
|
||||
const lines: { native: string; where: string }[] = []
|
||||
const walk = (node: unknown, path: string) => {
|
||||
if (typeof node === 'function') return walk((node as (...a: unknown[]) => unknown)(1, 'x'), path)
|
||||
if (!node || typeof node !== 'object') return
|
||||
const rec = node as Record<string, unknown>
|
||||
// A Line is the one shape whose `native` is pure Spanish — the flat
|
||||
// "Spanish · English" strings carry English punctuation too, so they are
|
||||
// asserted by hand below rather than by rule.
|
||||
if (typeof rec.native === 'string' && typeof rec.en === 'string') {
|
||||
lines.push({ native: rec.native, where: path })
|
||||
return
|
||||
}
|
||||
for (const [k, v] of Object.entries(rec)) walk(v, path ? `${path}.${k}` : k)
|
||||
}
|
||||
walk(es, '')
|
||||
expect(lines.length).toBeGreaterThan(40)
|
||||
for (const { native, where } of lines) {
|
||||
if (native.includes('?')) expect(native, `${where} closes ? without ¿`).toContain('¿')
|
||||
if (native.includes('!')) expect(native, `${where} closes ! without ¡`).toContain('¡')
|
||||
}
|
||||
// An exclamative opening with Qué/Cómo/Cuánto is the case that slips past a
|
||||
// reader, because it carries no closing "!" to look wrong against — the
|
||||
// whole pair is simply absent. The quorum review caught exactly one of
|
||||
// these ("Qué linda elección de palabra"), and only one reviewer of four
|
||||
// saw it, which is the argument for asserting it instead of re-reviewing it.
|
||||
for (const { native, where } of lines) {
|
||||
expect(native, `${where} opens an exclamative without ¡`).not.toMatch(
|
||||
/^(Qué|Cómo|Cuánto|Cuánta)\b/,
|
||||
)
|
||||
}
|
||||
// And the two-language strings, by hand: both halves punctuate their own way.
|
||||
expect(es.garden.promptRecognition).toBe('¿Qué significa? · What does this mean?')
|
||||
expect(es.garden.promptProduction).toContain('¿Cuál es la palabra en inglés?')
|
||||
})
|
||||
|
||||
it('renders its interpolated lines with the value in place', () => {
|
||||
expect(es.app.duplicateTitle('Primavera')).toBe('Primavera (copia)')
|
||||
expect(es.companion.milestone(300).native).toContain('300 palabras')
|
||||
// Spanish agreement is the pack's business; the call site only ever passes
|
||||
// a number. `flor`/`flores` is the irregular one — it takes -es, not -s.
|
||||
expect(es.garden.reviewDue(1)).toContain('1 palabra ·')
|
||||
expect(es.garden.reviewDue(4)).toContain('4 palabras ·')
|
||||
expect(es.garden.growing(1)).toContain('1 flor en el jardín')
|
||||
expect(es.garden.growing(3)).toContain('3 flores en el jardín')
|
||||
expect(es.journal.kept(1)).toContain('1 cosa que te llevaste')
|
||||
expect(es.journal.kept(5)).toContain('5 cosas que te llevaste')
|
||||
expect(es.status.petalsToPolish(1).native).toContain('1 pétalo por pulir')
|
||||
expect(es.status.petalsToPolish(2).native).toContain('2 pétalos por pulir')
|
||||
})
|
||||
|
||||
it('says the collision line, which this pair meets constantly', () => {
|
||||
// real, red, once, pie, sin, pan, mayor, sale, ropa — Spanish and English
|
||||
// collide about as often as French and English do.
|
||||
expect(es.editor.alsoIn).toBeTruthy()
|
||||
expect(es.editor.alsoIn).not.toBe(zh.editor.alsoIn)
|
||||
expect(es.editor.alsoIn).not.toBe(fr.editor.alsoIn)
|
||||
expect(es.editor.alsoIn).not.toBe(ptPT.editor.alsoIn)
|
||||
})
|
||||
})
|
||||
|
||||
// False friends are a per-pair dataset rather than copy: the Latin pairs carry
|
||||
// the traps their writers actually fall into, and the zh pair legitimately has
|
||||
// none. Both halves of that are worth pinning.
|
||||
@@ -391,6 +485,21 @@ describe('false friends', () => {
|
||||
expect(Object.keys(fr.falseFriends).length).toBeGreaterThan(10)
|
||||
})
|
||||
|
||||
it('the es pair carries the ones that cost most', () => {
|
||||
// "embarrassed" is the reason this feature exists at all: *embarazada* is
|
||||
// "pregnant", and it is the single false friend most likely to be said out
|
||||
// loud to a room. "molest" is the other one that has to be here, because
|
||||
// *molestar* is an everyday word and the English is not.
|
||||
for (const word of ['embarrassed', 'molest', 'actually', 'realize', 'exit', 'carpet']) {
|
||||
expect(es.falseFriends[word], word).toBeDefined()
|
||||
}
|
||||
// Spanish shares more Latin with English than either of the other Latin
|
||||
// pairs, so this list is the longest of the four and should stay that way.
|
||||
expect(Object.keys(es.falseFriends).length).toBeGreaterThan(
|
||||
Object.keys(fr.falseFriends).length,
|
||||
)
|
||||
})
|
||||
|
||||
it('is keyed by the lowercase English word, so a lookup can find it', () => {
|
||||
for (const p of PACKS) {
|
||||
for (const key of Object.keys(p.falseFriends)) {
|
||||
|
||||
@@ -18,12 +18,15 @@ import type { Pack, PairLang } from './types'
|
||||
import { zh } from './packs/zh'
|
||||
import { ptPT } from './packs/pt-PT'
|
||||
import { fr } from './packs/fr'
|
||||
import { es } from './packs/es'
|
||||
|
||||
export type { Pack, PairLang, Line } from './types'
|
||||
|
||||
// Every pack Petal ships. es is the same two lines when its copy is written —
|
||||
// TypeScript names every string a new pack still owes.
|
||||
const PACKS: Partial<Record<PairLang, Pack>> = { zh, 'pt-PT': ptPT, fr }
|
||||
// Every pack Petal ships — and now every pair PairLang names, so this map is
|
||||
// no longer Partial by necessity. It stays Partial anyway: the next pair will
|
||||
// be declared in the type before its copy exists, exactly as es was, and the
|
||||
// gap between the two is the point.
|
||||
const PACKS: Partial<Record<PairLang, Pack>> = { zh, 'pt-PT': ptPT, fr, es }
|
||||
|
||||
const DEFAULT_LANG: PairLang = 'zh'
|
||||
|
||||
|
||||
@@ -0,0 +1,550 @@
|
||||
// The Spanish pack — the fourth pair, and the second written straight into the
|
||||
// groove Phase 24 cut.
|
||||
//
|
||||
// ⚠️ REVIEWED BY FOUR MODELS, NOT BY A NATIVE SPEAKER.
|
||||
// SUGGESTIONS.md §3 sets the bar: a pack should be reviewed by someone who
|
||||
// speaks the pair before it is trusted. That has still not happened. What has
|
||||
// happened (2026-07-28) is the same interim pass the fr and pt-PT packs got —
|
||||
// four models read this file independently as Latin American Spanish speakers,
|
||||
// and only findings at least two of them reached on their own were applied,
|
||||
// listed in BUILD_PLAN Phase 25. A quorum of models agreeing is agreement, not
|
||||
// authority: it can catch a verb form no one says and a register that slips into
|
||||
// Spain, and it cannot catch a line that is correct and lifeless. Treat this as a
|
||||
// better-checked draft.
|
||||
//
|
||||
// The choices this file makes, and why:
|
||||
//
|
||||
// * **Latin American neutral, not peninsular.** Chosen deliberately (user,
|
||||
// 2026-07-28) over es-ES: it is the Spanish far more people write, and the
|
||||
// "neutral" register is a real thing that Spanish-language publishing and
|
||||
// dubbing have spent decades stabilising. So: *tú* for the singular,
|
||||
// **ustedes** for the plural and no *vosotros* anywhere, and the pan-American
|
||||
// half of every vocabulary split — *computadora*, *celular*, *carro*, *jugo*,
|
||||
// *papa*, *departamento*, *lentes*, *boleto*. A vitest greps this file for the
|
||||
// peninsular twins the way the fr pack is grepped for québécismes, because a
|
||||
// stray *ordenador* is invisible to everyone reviewing the diff.
|
||||
// * **The dictionary makes the opposite choice on purpose, and that is not a
|
||||
// contradiction.** This copy is Latin American; the spelling dictionary
|
||||
// behind it accepts *every* variety of Spanish, peninsular and voseante
|
||||
// alike (RLA's generic build — see web/public/dictionaries/es/LICENSE). The
|
||||
// two answer different questions: the copy is Petal *speaking*, where a
|
||||
// register has to be chosen, and the dictionary is Petal *listening*, where
|
||||
// the only available action is to underline something. Choosing a register
|
||||
// to write in costs a reader nothing; choosing one to accept would tell her
|
||||
// that her own conjugation is a typo.
|
||||
// * **Tuteo.** Petal is a companion in someone's private notebook, and *usted*
|
||||
// would put a desk between them — the same call the pt-PT pack made about
|
||||
// *tu* over *você* and the fr pack about *tu* over *vous*.
|
||||
// * **Tuteo here, voseo accepted there.** This copy says *tú*, because neutral
|
||||
// Latin American is tuteo and something had to be chosen. The dictionary
|
||||
// nonetheless accepts *vení* and *tenés*, so a Rioplatense writer is never
|
||||
// told her own present tense is a misspelling — she simply reads a companion
|
||||
// that speaks a slightly different Spanish than she writes, which is true of
|
||||
// every Spanish speaker reading anything.
|
||||
// * **¿Inverted marks, siempre!** Spanish opens questions and exclamations and
|
||||
// this file does too — including inside interpolated lines, where it is
|
||||
// easiest to forget. It is also the one punctuation habit that costs her
|
||||
// nothing in English: unlike the French space before « ! », there is no
|
||||
// Spanish mark to carry across by accident, so no prose note has to warn
|
||||
// about it.
|
||||
// * Quotation marks are the curly “ ” rather than « », which is the American
|
||||
// convention and the commoner one in Spanish outside Spain.
|
||||
//
|
||||
// Spanish first, English underneath — same shape as the other packs, for the same
|
||||
// reason: she reads her own language faster, and the English half is what she is
|
||||
// here to learn.
|
||||
|
||||
import type { Pack } from '../types'
|
||||
|
||||
export const es: Pack = {
|
||||
code: 'es',
|
||||
nativeName: 'Español',
|
||||
// Mexican Spanish is the standard "neutral" broadcast variety and the one
|
||||
// Piper has a Latin American voice for (es_MX-ald-medium). The peninsular
|
||||
// es_ES-davefx-medium the plan originally named would have read this copy in
|
||||
// the accent it was written to avoid.
|
||||
locale: 'es-MX',
|
||||
|
||||
app: {
|
||||
duplicateTitle: (title) => `${title} (copia)`,
|
||||
garden: 'Jardín de palabras',
|
||||
history: 'Historial',
|
||||
},
|
||||
|
||||
auth: {
|
||||
title: 'Vuelve a entrar',
|
||||
titleEn: 'Please sign in again',
|
||||
bodyWithDraft:
|
||||
'Lo que acabas de escribir está guardado en este dispositivo — vuelve a entrar y se guardará solo.',
|
||||
bodyWithDraftEn: "What you just wrote is safe on this device — it'll save itself once you're back in.",
|
||||
bodyPlain: 'Tu sesión expiró. Todo lo que escribiste ya está guardado.',
|
||||
bodyPlainEn: 'Your session expired. Everything you wrote is already saved.',
|
||||
signIn: 'Iniciar sesión · Sign in',
|
||||
},
|
||||
|
||||
companion: {
|
||||
choose: 'Elige un compañero · Choose a companion',
|
||||
|
||||
encouragements: [
|
||||
{ native: '¡Eso! Esa oración fluye mucho mejor 🌸', en: 'Lovely — that reads so much smoother now.' },
|
||||
{ native: 'Cada vez escribes mejor ✨', en: "You're getting better and better." },
|
||||
{ native: 'Me gusta mucho ese cambio 💕', en: 'I really like that change.' },
|
||||
{ native: '¡Sigue así, lo estás logrando!', en: 'Keep going — you’ve got this!' },
|
||||
{ native: 'Mmm, así queda mucho más claro 👍', en: 'Mm, that’s much clearer.' },
|
||||
{ native: '¡Qué linda elección de palabra! 🌷', en: 'That’s such a good word choice.' },
|
||||
{ native: 'Ay, ese párrafo se lee solito ☁️', en: 'Ooh, that paragraph flows so nicely.' },
|
||||
{ native: 'Me encanta verte escribir con más confianza 💛', en: 'I love watching you write with more confidence.' },
|
||||
{ native: 'Cada avance cuenta, por chiquito que sea 🌱', en: 'Every little bit of progress counts.' },
|
||||
{ native: 'Hoy tus palabras están brillando ✨', en: 'Your words are sparkling today.' },
|
||||
],
|
||||
|
||||
tips: [
|
||||
{ native: 'Consejo: en inglés, las oraciones cortas se leen mejor.', en: 'Tip: shorter English sentences often read clearer.' },
|
||||
{ native: 'No olvides los artículos “the” y “a”.', en: "Don't forget articles like “the” and “a”." },
|
||||
{ native: 'Para el pasado, usa el pretérito: go → went.', en: 'For the past, use past tense: go → went.' },
|
||||
{ native: 'Leer en voz alta ayuda a notar lo que suena raro.', en: 'Reading aloud helps you catch awkward spots.' },
|
||||
{ native: 'Una idea por párrafo y todo queda clarito.', en: 'One idea per paragraph keeps it tidy.' },
|
||||
{ native: '¿Tienes una duda? Pregúntame ✨', en: 'Not sure about something? Just ask me. ✨' },
|
||||
{ native: 'El plural lleva “s”: two apples 🍎', en: 'Plurals take an “s”: two apples 🍎' },
|
||||
// Two tips the zh pack has no use for. Spanish and English share so much
|
||||
// Latin vocabulary that the false friends are a daily hazard, and the
|
||||
// subject pronoun is the habit Spanish speakers drop most often.
|
||||
{ native: 'Cuidado con los falsos amigos: “actually” no significa *actualmente*.', en: 'Careful with false friends — “actually” means *in fact*.' },
|
||||
{ native: 'En inglés el sujeto casi nunca se omite: “it is raining”, no “is raining”.', en: 'English almost always needs a subject: “it is raining”, not “is raining”.' },
|
||||
],
|
||||
|
||||
breaks: [
|
||||
{ native: 'Llevas rato escribiendo — estírate y descansa la vista 🍵', en: "You've been writing a while — stretch and rest your eyes. 🍵" },
|
||||
{ native: '¿Un vaso de agua y cinco minutos de pausa?', en: 'Sip some water and take five?' },
|
||||
{ native: 'Mira a lo lejos un momento, la vista te lo agradece 🌿', en: 'Look into the distance for a moment — give your eyes a break. 🌿' },
|
||||
],
|
||||
|
||||
// Late-night nudges. The English wit is the user's own and is kept word for
|
||||
// word across every pack; the Spanish line leads gently into it, exactly as
|
||||
// the Mandarin, Portuguese and French ones do.
|
||||
bedtime: [
|
||||
{ native: 'Tu cama ha de estar preguntándose dónde andas 🛏️', en: 'I bet your bed is missing you right now.' },
|
||||
{ native: 'Cansada, escribes mal — ve a descansar 🌙', en: 'A tired writer is a bad writer — get some rest.' },
|
||||
{ native: 'Mejor consúltalo con la almohada ✨', en: 'Sleep is a wondrous enabler.' },
|
||||
{ native: '¿Oyes? No… porque todos están dormidos, y tú deberías estarlo también 😴', en: "Hear that? No… you don't, because everyone is sleeping and you should be too." },
|
||||
// Spanish sayings about sleep and haste, in place of the French and
|
||||
// Portuguese ones — a pack is not a translation of another pack.
|
||||
{ native: 'Dormir es el mejor remedio.', en: 'Sleep is the best medicine.' },
|
||||
{ native: 'A quien madruga, Dios lo ayuda.', en: 'The early riser gets a hand from above.' },
|
||||
{ native: 'No por mucho madrugar amanece más temprano.', en: 'Rising earlier will not make the sun come up sooner.' },
|
||||
],
|
||||
|
||||
greeting: { native: '¡Hola! Aquí te hago compañía 🐱', en: "Hi! I'm right here keeping you company. 🐱" },
|
||||
welcomeBack: { native: '¡Volviste! ✨ Sigamos', en: 'Welcome back ✨ let’s keep going!' },
|
||||
|
||||
errors: [
|
||||
{ native: 'Uy — un tropiezo chiquito, pero tus palabras están a salvo.', en: 'Oops — a little hiccup, but your words are safe.' },
|
||||
{ native: 'Ay, me enredé un segundo — ya vuelvo.', en: 'Haiya, I got stuck for a sec — back in a moment.' },
|
||||
{ native: 'No te preocupes, lo intentamos de nuevo en un ratito 🍵', en: "Don't worry — let's try again in a bit. 🍵" },
|
||||
],
|
||||
|
||||
milestone: (words: number) => ({
|
||||
native: `¡Guau! Ya llevas ${words} palabras 🎉`,
|
||||
en: `Wow — ${words} words already! Amazing. 🎉`,
|
||||
}),
|
||||
|
||||
// Algo pequeño que escribir, ofrecido una vez al día a una página en blanco.
|
||||
// Recuerdos y opiniones, nunca ejercicios: aquí no hay nada que se pueda
|
||||
// reprobar, y esa es justamente la intención.
|
||||
invitations: [
|
||||
{ native: 'Escribe 50 palabras: algo pequeño que te hizo sonreír hoy 🌸', en: 'Write 50 words: one small thing that made you smile today.' },
|
||||
{ native: 'Escribe 50 palabras: lo más rico que comiste hoy', en: 'Write 50 words: the best thing you ate today.' },
|
||||
{ native: 'Escribe 50 palabras: lo que ves por tu ventana en este momento', en: 'Write 50 words: what you can see out of your window right now.' },
|
||||
{ native: 'Escribe 50 palabras: un lugar al que volverías con gusto', en: 'Write 50 words: somewhere you would happily go back to.' },
|
||||
{ native: 'Escribe 50 palabras: algo que aprendiste esta semana', en: 'Write 50 words: one thing you learned this week.' },
|
||||
{ native: 'Escribe 50 palabras: un mensaje para ti misma dentro de un año', en: 'Write 50 words: something to tell yourself a year from now.' },
|
||||
{ native: 'Escribe 50 palabras: una canción que no has parado de escuchar estos días', en: 'Write 50 words: a song you have had on lately.' },
|
||||
{ native: 'Escribe 50 palabras: alguien a quien te gustaría agradecerle hoy', en: 'Write 50 words: someone you would like to thank today.' },
|
||||
],
|
||||
inviteAccept: 'Vamos · Let’s write',
|
||||
inviteDecline: 'Hoy no · Not today',
|
||||
declined: { native: 'Está bien, me regreso a dormir 😴', en: 'Fair enough — back to my nap. 😴' },
|
||||
|
||||
names: {
|
||||
cat: 'Gato dormilón',
|
||||
dog: 'Perro alegre',
|
||||
'wiggle-dog': 'Perro meneacola',
|
||||
butterfly: 'Mariposa',
|
||||
parrot: 'Loro',
|
||||
},
|
||||
},
|
||||
|
||||
prose: {
|
||||
longSentence: 'Esta oración quedó un poco larga — partirla en dos o tres la hace más clara 🌸',
|
||||
commaSplice: 'Aquí hay dos oraciones unidas solo por una coma. Pon un punto, o únelas con “and / but”.',
|
||||
vagueThis: (word) => `No queda claro a qué se refiere “${word}” — precísalo (por ejemplo “${word} idea / change…”).`,
|
||||
oxfordComma: 'En una lista de tres o más elementos, una coma antes de “and / or” ayuda a leer (la coma de Oxford).',
|
||||
transitionComma: (word) => `Después de un conector al inicio de la oración va una coma: “${word}, …”.`,
|
||||
capitalizeSentence: 'Empieza cada oración con mayúscula.',
|
||||
repeatedWord: (word) => `Parece que “${word}” quedó escrito dos veces — échale un ojo.`,
|
||||
capitalizeI: 'En inglés, “I” (yo) siempre va con mayúscula.',
|
||||
spaceBeforePunct: 'En inglés no se deja espacio antes de la puntuación: la coma y el punto van pegados a la palabra, igual que en español.',
|
||||
spaceAfterPunct: 'Después de una coma o un punto, deja un espacio antes de la siguiente palabra.',
|
||||
articleAn: (word) => `Antes de un sonido de vocal se usa “an”: “an ${word}”.`,
|
||||
articleA: (word) => `Antes de un sonido de consonante se usa “a”: “a ${word}”.`,
|
||||
uncountable: (word, singular) => `“${word}” es incontable en inglés — no lleva s, basta con “${singular}”.`,
|
||||
capitalizeProper: (fixed) => `En inglés, los idiomas, las nacionalidades, los días y los meses van con mayúscula: “${fixed}”.`,
|
||||
thirdPersonS: (subject, verb) => `Con he/she/it, el verbo lleva -s: “${subject} ${verb}”.`,
|
||||
pluralAfter: (determiner, noun) => `Después de “${determiner}”, el sustantivo va en plural: “${determiner} ${noun}s”.`,
|
||||
doubleDeterminer: (first, second) => `“${first} ${second}” lleva dos determinantes — deja solo uno (quita “${first}”, por ejemplo).`,
|
||||
thereArePlural: (noun) => `En plural se dice “there are”: “there are ${noun}…”.`,
|
||||
itsOwn: '“it’s” = “it is”. Para decir “su”, es “its” — o sea “its own”.',
|
||||
itsIs: (rest) => `Aquí va “it’s ${rest}” (it is); “its” es el posesivo.`,
|
||||
thanNotThen: (word) => `En una comparación se escribe “than”, no “then”: “${word} than”.`,
|
||||
preposition: (wrong, right) => `En inglés se dice “${right}”, no “${wrong}” — esa preposición es fija.`,
|
||||
collocation: (wrong, right) => `En inglés estas palabras van juntas así: “${right}”, y no “${wrong}”.`,
|
||||
doubleComparative: (lead, word) => `“${word}” ya es el comparativo — no necesita “${lead}”: con “${word}” basta.`,
|
||||
peopleArePlural: (verb) => `“People” es plural en inglés: “people ${verb}”.`,
|
||||
// Interferencia del español. Las tres primeras son las que de verdad se ven
|
||||
// todos los días; el resto las tiene el pack aunque este par no las corra.
|
||||
ageIsNotHave: (years) => `En inglés la edad se dice con *to be*, no con *tener*: “I am ${years} years old”.`,
|
||||
agreeIsAVerb: '“Agree” ya es el verbo — no lleva *to be* delante: se dice “I agree”, no “I am agree”.',
|
||||
forNotSince: (duration) => `Para una duración se usa “for”: “for ${duration}”. “Since” marca el punto de partida (since 2020).`,
|
||||
veryBeforeVerb: (verb) => `“Very” solo acompaña adjetivos, no verbos: “really ${verb}”, o “${verb}… very much”.`,
|
||||
turnOnNotOpen: (thing, on) => `En inglés los aparatos no se abren, se encienden: “turn ${on ? 'on' : 'off'} the ${thing}”.`,
|
||||
althoughOrBut: (word) => `En inglés se pone “${word}” o “but”, nunca los dos en la misma oración.`,
|
||||
},
|
||||
|
||||
// Los falsos amigos entre el español y el inglés — la trampa que hace sentir
|
||||
// ridícula en vez de simplemente corregida. Por eso son solo un aviso: Petal
|
||||
// nunca cambia la palabra, porque “actually” bien pudo ser la que quería.
|
||||
//
|
||||
// El español comparte tanto latín con el inglés que esta lista es la más larga
|
||||
// de los cuatro packs, y *embarrassed* es la razón por la que existe la función.
|
||||
falseFriends: {
|
||||
embarrassed: {
|
||||
native: '“Embarrassed” significa *apenada, avergonzada*. *Embarazada* se dice “pregnant”.',
|
||||
en: '“Embarrassed” means ashamed; the Spanish *embarazada* is “pregnant”.',
|
||||
},
|
||||
actually: {
|
||||
native: '“Actually” significa *en realidad*, no *actualmente*. Para *actualmente* se dice “currently” o “nowadays”.',
|
||||
en: '“Actually” means *in fact*. For the Spanish *actualmente*, English uses “currently”.',
|
||||
},
|
||||
eventually: {
|
||||
native: '“Eventually” significa *al final, tarde o temprano* — no *eventualmente*. Para eso: “possibly” o “if necessary”.',
|
||||
en: '“Eventually” means *in the end*, not *possibly*.',
|
||||
},
|
||||
realize: {
|
||||
native: '“Realize” significa *darse cuenta*. Para *realizar* (llevar a cabo) se dice “carry out” o “do”.',
|
||||
en: '“Realize” means to become aware; *realizar* is “to carry out”.',
|
||||
},
|
||||
assist: {
|
||||
native: '“Assist” significa *ayudar*. Para *asistir* (ir a algo) se dice “attend”.',
|
||||
en: '“Assist” means to help; *asistir a* is “to attend”.',
|
||||
},
|
||||
attend: {
|
||||
native: '“Attend” significa *asistir a*. Para *atender* (a alguien) se dice “serve” o “take care of”.',
|
||||
en: '“Attend” means to go to something; *atender* is “to serve”.',
|
||||
},
|
||||
support: {
|
||||
native: '“Support” significa *apoyar*. Para *soportar* (aguantar algo molesto) se dice “put up with” o “bear”.',
|
||||
en: '“Support” means to back someone up; *soportar* is “to put up with”.',
|
||||
},
|
||||
sensible: {
|
||||
native: '“Sensible” significa *sensato*. Para *sensible* se dice “sensitive”.',
|
||||
en: '“Sensible” means level-headed; the Spanish *sensible* is “sensitive”.',
|
||||
},
|
||||
sympathetic: {
|
||||
native: '“Sympathetic” significa *comprensivo, solidario*. Para *simpático* se dice “nice” o “friendly”.',
|
||||
en: '“Sympathetic” means understanding; *simpático* is “nice”.',
|
||||
},
|
||||
library: {
|
||||
native: '“Library” es la *biblioteca*. La *librería* se dice “bookshop” o “bookstore”.',
|
||||
en: '“Library” is where books are lent; a shop that sells them is a “bookstore”.',
|
||||
},
|
||||
exit: {
|
||||
native: '“Exit” es la *salida*. El *éxito* se dice “success”.',
|
||||
en: '“Exit” is the way out; *éxito* is “success”.',
|
||||
},
|
||||
success: {
|
||||
native: '“Success” es el *éxito*. Un *suceso* se dice “event”.',
|
||||
en: '“Success” is achievement; a Spanish *suceso* is an “event”.',
|
||||
},
|
||||
carpet: {
|
||||
native: '“Carpet” es la *alfombra*. Una *carpeta* se dice “folder”.',
|
||||
en: '“Carpet” covers a floor; a *carpeta* is a “folder”.',
|
||||
},
|
||||
discussion: {
|
||||
native: '“Discussion” es una *conversación*, sin pelea. Una *discusión* (riña) se dice “argument”.',
|
||||
en: '“Discussion” is calm; a Spanish *discusión* is an “argument”.',
|
||||
},
|
||||
argument: {
|
||||
native: '“Argument” es una *discusión* o *pelea*. El *argumento* de una historia se dice “plot”.',
|
||||
en: '“Argument” is a quarrel; the *argumento* of a story is its “plot”.',
|
||||
},
|
||||
introduce: {
|
||||
native: '“Introduce” es *presentar* a alguien. Para *introducir* (meter) se dice “insert” o “put in”.',
|
||||
en: '“Introduce” is to present someone; *introducir* is “to insert”.',
|
||||
},
|
||||
molest: {
|
||||
native: '“Molest” significa *abusar sexualmente* — nunca se usa por *molestar*. Para eso: “bother” o “annoy”.',
|
||||
en: '“Molest” means to abuse; the everyday *molestar* is “to bother”.',
|
||||
},
|
||||
constipated: {
|
||||
native: '“Constipated” significa *estreñida*. Para *constipada* (resfriada) se dice “to have a cold”.',
|
||||
en: '“Constipated” is a bowel problem; *constipado* is “a cold”.',
|
||||
},
|
||||
large: {
|
||||
native: '“Large” significa *grande*. Para *largo* se dice “long”.',
|
||||
en: '“Large” means big; *largo* is “long”.',
|
||||
},
|
||||
lecture: {
|
||||
native: '“Lecture” es una *conferencia* o *clase*. La *lectura* se dice “reading”.',
|
||||
en: '“Lecture” is a talk; *lectura* is “reading”.',
|
||||
},
|
||||
parents: {
|
||||
native: '“Parents” son los *padres*. Los *parientes* se dicen “relatives”.',
|
||||
en: '“Parents” are your mother and father; *parientes* are “relatives”.',
|
||||
},
|
||||
record: {
|
||||
native: '“Record” significa *grabar* o *registrar*. Para *recordar* se dice “remember”.',
|
||||
en: '“Record” means to register; *recordar* is “to remember”.',
|
||||
},
|
||||
remove: {
|
||||
native: '“Remove” significa *quitar*. Para *remover* (revolver) se dice “stir”.',
|
||||
en: '“Remove” means to take away; *remover* is “to stir”.',
|
||||
},
|
||||
rope: {
|
||||
native: '“Rope” es la *cuerda*. La *ropa* se dice “clothes”.',
|
||||
en: '“Rope” is cord; *ropa* is “clothes”.',
|
||||
},
|
||||
once: {
|
||||
native: '“Once” significa *una vez*. El número *once* se dice “eleven”.',
|
||||
en: '“Once” means one time; the Spanish *once* is “eleven”.',
|
||||
},
|
||||
question: {
|
||||
native: '“Question” es una *pregunta*. Una *cuestión* (asunto) se dice “matter” o “issue”.',
|
||||
en: '“Question” is something you ask; a *cuestión* is a “matter”.',
|
||||
},
|
||||
compromise: {
|
||||
native: '“Compromise” es un *acuerdo con concesiones*. Un *compromiso* (obligación) se dice “commitment”.',
|
||||
en: '“Compromise” is meeting halfway; a *compromiso* is a “commitment”.',
|
||||
},
|
||||
career: {
|
||||
native: '“Career” es la *trayectoria profesional*. La *carrera* que se estudia se dice “degree” o “major”.',
|
||||
en: '“Career” is your working life; a university *carrera* is a “degree”.',
|
||||
},
|
||||
ultimately: {
|
||||
native: '“Ultimately” significa *a fin de cuentas*. Para *últimamente* se dice “lately”.',
|
||||
en: '“Ultimately” means in the end; *últimamente* is “lately”.',
|
||||
},
|
||||
idiom: {
|
||||
native: '“Idiom” es una *expresión hecha*. El *idioma* se dice “language”.',
|
||||
en: '“Idiom” is a set phrase; *idioma* is “language”.',
|
||||
},
|
||||
deception: {
|
||||
native: '“Deception” significa *engaño*. La *decepción* se dice “disappointment”.',
|
||||
en: '“Deception” means being misled; *decepción* is “disappointment”.',
|
||||
},
|
||||
pretend: {
|
||||
native: '“Pretend” significa *fingir*. Para *pretender* (aspirar a) se dice “intend” o “claim”.',
|
||||
en: '“Pretend” means to fake; *pretender* is “to intend”.',
|
||||
},
|
||||
},
|
||||
|
||||
docs: {
|
||||
sortRecent: 'Recientes · Recent',
|
||||
sortTitle: 'Título · Title',
|
||||
sortLongest: 'Los más largos · Longest',
|
||||
backUpAll: 'Respaldar todo · Back up all:',
|
||||
signOut: 'Cerrar sesión · Sign out',
|
||||
duplicate: 'Duplicar · Duplicate',
|
||||
searchPlaceholder: 'Buscar · Search',
|
||||
searching: 'Buscando… · Searching…',
|
||||
noMatches: 'Sin resultados · No matches',
|
||||
tags: 'Etiquetas · Tags',
|
||||
newTagPlaceholder: 'Nueva etiqueta · New tag',
|
||||
language: 'Idioma · Language',
|
||||
languageFailed: 'No se pudo cambiar el idioma — sigue igual · Couldn’t switch',
|
||||
},
|
||||
|
||||
editor: {
|
||||
askPlaceholder: 'Ask why… / Pregunta por qué…',
|
||||
chatFailed: 'No logré responder — vuelve a preguntarme, por favor. 🌸\n\nI had trouble answering just now — please ask me again. 🌸',
|
||||
findPlaceholder: 'Buscar · Find',
|
||||
findNone: 'Nada · 0',
|
||||
matchCase: 'Match case · Distinguir mayúsculas',
|
||||
close: 'Close · Cerrar',
|
||||
replacePlaceholder: 'Reemplazar por · Replace',
|
||||
replace: 'Reemplazar',
|
||||
replaceAll: 'Todo',
|
||||
translateLabel: 'Traducción · Translate',
|
||||
spelling: 'Ortografía · Spelling',
|
||||
noSuggestions: 'Sin sugerencias · No suggestions',
|
||||
addToDictionary: 'Agregar al diccionario · Add to dictionary',
|
||||
readSelection: 'Leer la selección en voz alta · Read selection aloud',
|
||||
rewrite: 'Reescribir · Rewrite',
|
||||
rewriting: 'Reescribiendo… · Rewriting…',
|
||||
rewriteFailed: 'No se pudo reescribir — inténtalo otra vez · Couldn’t rewrite',
|
||||
cancel: 'Cancelar · Cancel',
|
||||
retry: 'Reintentar · Retry',
|
||||
useThis: 'Usar esta · Use this',
|
||||
word: 'Palabra · Word',
|
||||
inGarden: 'Ya está en el jardín · In your garden (tap to remove)',
|
||||
saveToGarden: 'Guardar en el jardín · Save to garden',
|
||||
readAloud: 'Leer en voz alta · Read aloud',
|
||||
readSlowly: 'Leer despacio · Read slowly',
|
||||
readAloudNative: 'Leer en español · Read in Spanish',
|
||||
lookingUp: 'Buscando… · Looking up…',
|
||||
definition: 'Definición · Definition',
|
||||
synonyms: 'Sinónimos · Synonyms',
|
||||
tapToSwap: 'toca para cambiar · tap to swap',
|
||||
nothingFound: 'No encontré esta palabra · Nothing found for this word',
|
||||
origin: 'Origen · Origin',
|
||||
// El par español ve esto seguido: inglés y español comparten tanto latín que
|
||||
// los choques son la regla y no la excepción — real, red, once, pie, sin,
|
||||
// pan, mayor, sale, ropa.
|
||||
alsoIn: 'También es una palabra en español · Also a word in Spanish',
|
||||
wordBands: {
|
||||
simple: { native: 'De todos los días', en: 'Everyday word' },
|
||||
standard: { native: 'Común', en: 'Standard' },
|
||||
advanced: { native: 'Avanzada', en: 'Advanced' },
|
||||
},
|
||||
},
|
||||
|
||||
styles: {
|
||||
natural: { native: 'Más natural', en: 'Natural' },
|
||||
academic: { native: 'Académico', en: 'Academic' },
|
||||
professional: { native: 'Profesional', en: 'Professional' },
|
||||
casual: { native: 'Informal', en: 'Casual' },
|
||||
humorous: { native: 'Divertido', en: 'Humorous' },
|
||||
creative: { native: 'Creativo', en: 'Creative' },
|
||||
persuasive: { native: 'Persuasivo', en: 'Persuasive' },
|
||||
},
|
||||
|
||||
tones: {
|
||||
general: { native: 'General', en: 'General' },
|
||||
academic: { native: 'Académico', en: 'Academic' },
|
||||
professional: { native: 'Profesional', en: 'Professional' },
|
||||
casual: { native: 'Informal', en: 'Casual' },
|
||||
humorous: { native: 'Divertido', en: 'Humorous' },
|
||||
creative: { native: 'Creativo', en: 'Creative' },
|
||||
persuasive: { native: 'Persuasivo', en: 'Persuasive' },
|
||||
},
|
||||
|
||||
exports: {
|
||||
label: 'Exportar',
|
||||
print: 'Imprimir / PDF',
|
||||
formats: {
|
||||
md: { native: 'Markdown', en: 'Markdown (.md)' },
|
||||
docx: { native: 'Documento de Word', en: 'Word (.docx)' },
|
||||
html: { native: 'Página web', en: 'Web page (.html)' },
|
||||
txt: { native: 'Texto sin formato', en: 'Plain text (.txt)' },
|
||||
},
|
||||
},
|
||||
|
||||
garden: {
|
||||
title: 'Jardín de palabras · Vocabulary Garden',
|
||||
titleWithFlower: '🌷 Jardín de palabras · Vocabulary Garden',
|
||||
reviewing: 'Repaso · Reviewing — recall, then grade yourself',
|
||||
subtitle: 'Words you looked up, blooming as you learn them',
|
||||
reviewDue: (n) => `Repasar ${n} palabra${n === 1 ? '' : 's'} · Review ${n} due 🌸`,
|
||||
emptyLead: 'Tu jardín todavía está vacío.',
|
||||
emptyHint: 'Haz clic derecho en una palabra en inglés para buscarla — y aquí brotará.',
|
||||
due: 'por repasar · due',
|
||||
seen: (reps, intervalDays) => `repasada ${reps}× · seen ${reps}× · intervalo ${intervalDays} d`,
|
||||
readAloud: '🔊 Leer',
|
||||
readSlowly: '🐢 Despacio',
|
||||
source: '📄 Fuente',
|
||||
remove: '🗑 Quitar',
|
||||
growing: (n) => `🐱💤 ${n} flor${n === 1 ? '' : 'es'} en el jardín · ${n} blossom${n > 1 ? 's' : ''} growing`,
|
||||
end: 'Terminar · End',
|
||||
promptProduction: '¿Cuál es la palabra en inglés? · Which English word?',
|
||||
promptRecognition: '¿Qué significa? · What does this mean?',
|
||||
showAnswer: 'Ver la respuesta · Show answer',
|
||||
gradeAgain: { native: 'Repetir', en: 'Again' },
|
||||
gradeGood: { native: 'La recuerdo', en: 'Good' },
|
||||
gradeEasy: { native: 'Fácil', en: 'Easy' },
|
||||
},
|
||||
|
||||
journal: {
|
||||
tabGarden: '🌷 Jardín · Garden',
|
||||
tabJournal: '🌱 Avances · Growth',
|
||||
subtitle: 'Your own writing, month by month — only ever you and your past self',
|
||||
empty: 'Sigue escribiendo un poco más — esta página crece a partir de tu propio trabajo. · Keep writing; this page grows out of your own work.',
|
||||
keptHead: 'Este mes · This month',
|
||||
kept: (n) => `${n} cosa${n === 1 ? '' : 's'} que te llevaste · ${n} thing${n === 1 ? '' : 's'} you took on board`,
|
||||
keptBefore: (n) => `${n} el mes pasado · ${n} the month before`,
|
||||
stuckHead: 'Se te quedó · Stayed with you',
|
||||
stuck: (phrase, docs) =>
|
||||
`“${phrase}” — ya la escribes sola, en ${docs} de tus textos · now in ${docs} of your pieces`,
|
||||
fadedHead: 'Ya no necesitas corregir esto · You stopped needing this',
|
||||
faded: (pattern, times) =>
|
||||
`“${pattern}” — ${times}× antes, ninguna este mes · ${times}× back then, none this month`,
|
||||
cheerStuck: (phrase) => ({
|
||||
native: `¡Ya usas “${phrase}” tú solita! 🌱`,
|
||||
en: `You’re using “${phrase}” on your own now! 🌱`,
|
||||
}),
|
||||
cheerFaded: (pattern) => ({
|
||||
native: `Hace rato que “${pattern}” no necesita corrección 😌`,
|
||||
en: `“${pattern}” hasn’t needed fixing in a while 😌`,
|
||||
}),
|
||||
},
|
||||
|
||||
history: {
|
||||
title: 'Historial · History',
|
||||
kinds: {
|
||||
manual: { native: 'Punto guardado', en: 'Saved point' },
|
||||
auto: { native: 'Automático', en: 'Auto' },
|
||||
pre_restore: { native: 'Antes de restaurar', en: 'Before restore' },
|
||||
},
|
||||
justNow: 'ahora mismo · just now',
|
||||
minutesAgo: (n) => `hace ${n} min · ${n} min ago`,
|
||||
hoursAgo: (n) => `hace ${n} h · ${n} hr ago`,
|
||||
daysAgo: (n) => `hace ${n} día${n > 1 ? 's' : ''} · ${n} day${n > 1 ? 's' : ''} ago`,
|
||||
preview: 'Vista previa · Preview',
|
||||
restoring: 'Restaurando… · Restoring…',
|
||||
restoreThis: 'Restaurar esta versión · Restore this version',
|
||||
passport: '📜 Pasaporte de escritura · Writing passport',
|
||||
keepFullHistory: 'Guardar todo el historial · Keep full history',
|
||||
},
|
||||
|
||||
status: {
|
||||
savedLocally: 'Guardado en este dispositivo · Kept on this device',
|
||||
helperRestingNative: 'El ayudante está descansando',
|
||||
helperRestingEn: "· Petal's helper is resting · tu texto está guardado",
|
||||
petalsToPolish: (n) => ({
|
||||
native: `${n} ${n === 1 ? 'pétalo' : 'pétalos'} por pulir`,
|
||||
en: `${n} ${n === 1 ? 'petal' : 'petals'} to polish`,
|
||||
}),
|
||||
soundsOn: 'Sonidos activados · Sounds on',
|
||||
soundsOff: 'Sonidos desactivados · Sounds off',
|
||||
petalsOn: 'Pétalos activados · Petals on',
|
||||
petalsOff: 'Pétalos desactivados · Petals off',
|
||||
statsTitle: 'Estadísticas · Writing stats',
|
||||
stats: {
|
||||
words: { native: 'Palabras', en: 'Words' },
|
||||
characters: { native: 'Caracteres', en: 'Characters' },
|
||||
sentences: { native: 'Oraciones', en: 'Sentences' },
|
||||
paragraphs: { native: 'Párrafos', en: 'Paragraphs' },
|
||||
pages: { native: 'Páginas', en: 'Pages' },
|
||||
readingTime: { native: 'Tiempo de lectura', en: 'Reading time' },
|
||||
avgWordLength: { native: 'Longitud promedio', en: 'Avg word length' },
|
||||
variety: { native: 'Variedad de vocabulario', en: 'Word variety' },
|
||||
readability: { native: 'Nivel de lectura', en: 'Reading level' },
|
||||
},
|
||||
readability: {
|
||||
easy: { native: 'Fácil', en: 'Easy' },
|
||||
standard: { native: 'Común', en: 'Standard' },
|
||||
fairlyHard: { native: 'Algo difícil', en: 'Fairly hard' },
|
||||
advanced: { native: 'Avanzado', en: 'Advanced' },
|
||||
},
|
||||
},
|
||||
|
||||
toolbar: {
|
||||
untitledHeading: '(sin título)',
|
||||
outline: 'Esquema · Outline',
|
||||
outlineHint: 'Usa H1/H2/H3 para crear títulos, y el esquema aparecerá aquí.',
|
||||
},
|
||||
|
||||
update: {
|
||||
available: 'Hay una versión nueva disponible',
|
||||
refresh: 'Actualizar · Refresh',
|
||||
dismiss: 'Más tarde · Dismiss',
|
||||
},
|
||||
}
|
||||
@@ -15,6 +15,17 @@ export const zh: Pack = {
|
||||
nativeName: '中文',
|
||||
locale: 'zh-CN',
|
||||
|
||||
// The zh pair is the only one Petal can be *learned* toward, because it is the
|
||||
// only one with a word list and a Chinese→English dictionary (Phase 26). The
|
||||
// two labels are each written for the person who would pick them: she reads
|
||||
// the first, and the English speaker learning her language reads the second.
|
||||
learner: {
|
||||
label: '我在学 · I am learning',
|
||||
toEn: '英文',
|
||||
toPair: 'Chinese 中文',
|
||||
failed: '没能换成功 · Couldn’t switch — nothing changed',
|
||||
},
|
||||
|
||||
app: {
|
||||
duplicateTitle: (title) => `${title} (副本)`,
|
||||
garden: '词汇花园',
|
||||
|
||||
@@ -39,6 +39,29 @@ export interface Pack {
|
||||
// Portuguese voice anyone reaches for is Brazilian.
|
||||
locale: string
|
||||
|
||||
// Copy for turning this pair around — a writer who is native in English and
|
||||
// learning X, rather than the other way round.
|
||||
//
|
||||
// Optional, and its presence is the pack's half of the same fact
|
||||
// auth.learnerPairs holds server-side: a pair can only be learned toward if
|
||||
// Petal has a word list to segment it with and a dictionary that reads from it
|
||||
// into English. Chinese has both; the Latin pairs have neither yet, so their
|
||||
// packs simply leave this out and the control does not render.
|
||||
//
|
||||
// Each label is written in the language of the person who would *choose* it,
|
||||
// for the same reason the pair buttons name themselves: someone on the wrong
|
||||
// side of this switch cannot read the side they are trying to reach.
|
||||
learner?: {
|
||||
// The heading over the two choices.
|
||||
label: string
|
||||
// "I am practising English" — read by the writer who is native in X.
|
||||
toEn: string
|
||||
// "I am learning X" — read by the writer who is native in English.
|
||||
toPair: string
|
||||
// Shown when the server refuses the change.
|
||||
failed: string
|
||||
}
|
||||
|
||||
app: {
|
||||
// A duplicated document's title. A function, not a suffix: where the marker
|
||||
// goes is the pack's business.
|
||||
|
||||
+1
-1
@@ -425,7 +425,7 @@ button, a, input {
|
||||
.petal-companion {
|
||||
/* Mascot size scales with the viewport width: ~original on a laptop, up to
|
||||
~2× on a large desktop. Tune the middle (vw) term to taste. */
|
||||
--petal-companion-size: clamp(10rem, 17vw, 20rem);
|
||||
--petal-companion-size: clamp(9rem, 15.3vw, 18rem);
|
||||
animation: petal-bob 3.2s ease-in-out infinite;
|
||||
/* Shrink toward its corner when fading out of a card's way. `scale` is a
|
||||
separate property from `transform` so it composes with the bob keyframes. */
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
// fromIME answers whether a keydown belongs to an in-flight IME composition
|
||||
// rather than to the app.
|
||||
//
|
||||
// While a candidate window is open, Enter and Escape mean something to the IME
|
||||
// and nothing to Petal: Enter commits the candidate, Escape cancels it back to
|
||||
// the pinyin. A handler that acts on them anyway steals the key — she presses
|
||||
// Escape to fix a wrong candidate and the sidebar reappears; she presses Enter
|
||||
// to accept 公园 and the Find bar jumps to the next match instead. In neither
|
||||
// case does the IME get its keystroke.
|
||||
//
|
||||
// `isComposing` is the standard signal and is what modern browsers set. The 229
|
||||
// keyCode is the older one, still the only signal some Safari/IME combinations
|
||||
// give, and costs one comparison to honour.
|
||||
// React's synthetic keyboard event doesn't surface `isComposing`, so the native
|
||||
// event underneath it is what gets asked — the same object either way.
|
||||
export function fromIME(e: KeyboardEvent | { nativeEvent: KeyboardEvent }): boolean {
|
||||
const native = 'nativeEvent' in e ? e.nativeEvent : e
|
||||
return native.isComposing || native.keyCode === 229
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { gunzipSync } from 'node:zlib'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { buildSegmenter, isHan, type Segmenter } from './segment'
|
||||
|
||||
// Segmentation is tested twice over, and the two halves check different things.
|
||||
//
|
||||
// The hand-built dictionaries below pin the *algorithm*: given these words with
|
||||
// these frequencies, this is the split, and the reason is visible in the four
|
||||
// lines above the assertion. They would pass with any word list.
|
||||
//
|
||||
// The block at the bottom pins the *shipped asset*: the real 188,522-word list
|
||||
// this app serves, on the sentences a rebuild would plausibly break. Those are
|
||||
// the cases where being wrong is invisible — the app still works, it just
|
||||
// underlines and glosses the wrong thing.
|
||||
|
||||
// A dictionary written the way the asset is: "word freq" per line.
|
||||
function dict(entries: Record<string, number>): Segmenter {
|
||||
return buildSegmenter(
|
||||
Object.entries(entries)
|
||||
.map(([w, f]) => `${w} ${f}`)
|
||||
.join('\n'),
|
||||
)
|
||||
}
|
||||
|
||||
const words = (seg: Segmenter, text: string) => seg.segment(text).map((t) => t.word)
|
||||
|
||||
describe('isHan', () => {
|
||||
it('accepts Han across the extension blocks, and nothing else', () => {
|
||||
expect(isHan('中')).toBe(true)
|
||||
expect(isHan('龥')).toBe(true)
|
||||
// Beyond the basic block. A character Petal fails to recognise as Chinese is
|
||||
// one the English tokenizer then tries to make sense of.
|
||||
expect(isHan('𠀀')).toBe(true)
|
||||
for (const ch of ['a', '1', ' ', '，', '。', 'あ', '한']) {
|
||||
expect(isHan(ch), ch).toBe(false)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('the walk chooses the likeliest split, not the longest match', () => {
|
||||
// The textbook case, and the reason longest-match is not good enough: 研究生
|
||||
// ("graduate student") is a real word and a longer match than 研究 at position
|
||||
// 0 — but 研究/生命 ("research" + "life") is the likelier path, and it is the
|
||||
// sentence a person would read.
|
||||
it('研究生命的起源', () => {
|
||||
const seg = dict({ 研究: 6000, 研究生: 800, 生命: 4000, 命: 900, 的: 300000, 起源: 700 })
|
||||
expect(words(seg, '研究生命的起源')).toEqual(['研究', '生命', '的', '起源'])
|
||||
})
|
||||
|
||||
it('乒乓球拍卖完了 — the ambiguity is 球拍 against 拍卖', () => {
|
||||
const seg = dict({
|
||||
乒乓球: 500, 乒乓: 400, 球拍: 200, 拍卖: 900, 卖完: 50, 完: 3000, 了: 200000, 球: 2000, 拍: 800,
|
||||
})
|
||||
expect(words(seg, '乒乓球拍卖完了')).toEqual(['乒乓球', '拍卖', '完', '了'])
|
||||
})
|
||||
|
||||
it('keeps particles as their own words', () => {
|
||||
const seg = dict({ 他: 90000, 的: 300000, 中文: 3000, 说: 20000, 得: 60000, 很: 40000, 好: 50000 })
|
||||
expect(words(seg, '他的中文说得很好')).toEqual(['他', '的', '中文', '说', '得', '很', '好'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('what the walk does with what it does not know', () => {
|
||||
// A sentence with an unfamiliar character in it must still segment. Every
|
||||
// position needs *some* path through it, which is why an unknown character
|
||||
// scores badly rather than not scoring at all.
|
||||
it('an unknown character becomes its own token and the rest survives', () => {
|
||||
const seg = dict({ 我: 90000, 喜欢: 5000, 猫: 2000 })
|
||||
expect(words(seg, '我喜欢龥猫')).toEqual(['我', '喜欢', '龥', '猫'])
|
||||
})
|
||||
|
||||
// It must never *invent* a word: an unknown span of two characters is two
|
||||
// unknown characters, not a new headword.
|
||||
it('never joins unknown characters into a word', () => {
|
||||
const seg = dict({ 我: 90000 })
|
||||
expect(words(seg, '我龥龥')).toEqual(['我', '龥', '龥'])
|
||||
})
|
||||
|
||||
// A character above the BMP is two UTF-16 code units, and asking about either
|
||||
// half alone says "not Han". Getting this wrong is quiet: the run breaks in
|
||||
// two around the character, the words either side of it stop being looked up,
|
||||
// and nothing anywhere reports an error.
|
||||
it('a supplementary-plane character is one unknown token inside the run', () => {
|
||||
const seg = dict({ 我: 90000, 喜欢: 5000, 猫: 2000 })
|
||||
const text = '我喜欢𠀀猫'
|
||||
expect(words(seg, text)).toEqual(['我', '喜欢', '𠀀', '猫'])
|
||||
for (const t of seg.segment(text)) expect(text.slice(t.from, t.to)).toBe(t.word)
|
||||
// And it is hoverable from either code unit — a caret offset can land on
|
||||
// the low surrogate, which is not a character boundary but is a real index.
|
||||
expect(seg.wordAt(text, 3)?.word).toBe('𠀀')
|
||||
expect(seg.wordAt(text, 4)?.word).toBe('𠀀')
|
||||
expect(seg.wordAt(text, 5)?.word).toBe('猫')
|
||||
})
|
||||
|
||||
// A rare real word still loses to two common ones — this is the property that
|
||||
// lets the shipped list keep 100,000 rare CC-CEDICT headwords without them
|
||||
// distorting ordinary sentences.
|
||||
it('a rare long word loses to two common short ones', () => {
|
||||
const seg = dict({ 公园: 4000, 跑步: 3000, 公园跑: 1 })
|
||||
expect(words(seg, '公园跑步')).toEqual(['公园', '跑步'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('Chinese is not the only thing in the paragraph', () => {
|
||||
const seg = dict({ 我: 90000, 在: 50000, 写: 8000, 英文: 3000 })
|
||||
|
||||
// Latin runs are skipped, not returned. The English tokenizer is still running
|
||||
// over the same text and owns them; returning them here would mean two layers
|
||||
// claiming one word.
|
||||
it('skips Latin and punctuation, keeping offsets into the original string', () => {
|
||||
const tokens = seg.segment('我在写 English 英文。')
|
||||
expect(tokens.map((t) => t.word)).toEqual(['我', '在', '写', '英文'])
|
||||
for (const t of tokens) {
|
||||
expect('我在写 English 英文。'.slice(t.from, t.to)).toBe(t.word)
|
||||
}
|
||||
})
|
||||
|
||||
it('每 token reports the span it actually occupies', () => {
|
||||
const tokens = seg.segment('英文')
|
||||
expect(tokens).toEqual([{ word: '英文', from: 0, to: 2 }])
|
||||
})
|
||||
})
|
||||
|
||||
describe('wordAt — the hover and click path', () => {
|
||||
const seg = dict({ 我: 90000, 今天: 8000, 去: 30000, 公园: 4000, 跑步: 3000, 了: 200000 })
|
||||
const text = '我今天去公园跑步了'
|
||||
|
||||
it('finds the word covering a position anywhere inside it', () => {
|
||||
// 公园 occupies [4,6): either of its characters resolves to the whole word
|
||||
// rather than to one character.
|
||||
for (const i of [4, 5]) {
|
||||
expect(seg.wordAt(text, i)?.word, `index ${i}`).toBe('公园')
|
||||
}
|
||||
expect(seg.wordAt(text, 0)?.word).toBe('我')
|
||||
expect(seg.wordAt(text, 2)?.word).toBe('今天')
|
||||
})
|
||||
|
||||
// A position names a gap; a word covers characters. On a boundary the answer
|
||||
// is the word that *starts* there, because that is the character being pointed
|
||||
// at — index 6 is the 跑 under the mouse, not the 园 behind it.
|
||||
it('a boundary belongs to the word that starts there', () => {
|
||||
expect(seg.wordAt(text, 6)?.word).toBe('跑步')
|
||||
expect(seg.wordAt(text, 4)?.word).toBe('公园')
|
||||
})
|
||||
|
||||
// The caret after a just-typed word belongs to that word. Ctrl/Cmd+D at the
|
||||
// end of 跑步 must look up 跑步, which is the position the caret is actually in
|
||||
// the moment someone finishes typing it.
|
||||
it('a caret at the very end of the text still resolves', () => {
|
||||
expect(seg.wordAt(text, text.length)?.word).toBe('了')
|
||||
})
|
||||
|
||||
it('returns null outside Han text', () => {
|
||||
expect(seg.wordAt('hello world', 3)).toBeNull()
|
||||
expect(seg.wordAt('', 0)).toBeNull()
|
||||
expect(seg.wordAt('我 hello', 4)).toBeNull()
|
||||
})
|
||||
|
||||
// The window exists so that a pasted page of Chinese with no punctuation is
|
||||
// not walked on every hover. It must not change the answer for ordinary text.
|
||||
it('agrees with a full segmentation of the same string', () => {
|
||||
const long = '我今天去公园跑步了'.repeat(20)
|
||||
const full = seg.segment(long)
|
||||
for (const t of full) {
|
||||
expect(seg.wordAt(long, t.from)).toEqual(t)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
// ── the shipped asset ───────────────────────────────────────────────────────
|
||||
// Everything above would pass with a word list built wrong. These read the file
|
||||
// this app actually serves.
|
||||
|
||||
describe('the shipped word list', () => {
|
||||
const raw = gunzipSync(readFileSync(new URL('../../public/dictionaries/zh/words.txt.gz', import.meta.url)))
|
||||
const seg = buildSegmenter(raw.toString('utf8'))
|
||||
|
||||
it('is the size the build script says it is', () => {
|
||||
expect(seg.size).toBeGreaterThan(180_000)
|
||||
})
|
||||
|
||||
it('segments ordinary learner prose the way a reader would', () => {
|
||||
expect(words(seg, '我今天早上去公园跑步了')).toEqual(['我', '今天', '早上', '去', '公园', '跑步', '了'])
|
||||
expect(words(seg, '他的中文说得很好')).toEqual(['他', '的', '中文', '说', '得', '很', '好'])
|
||||
expect(words(seg, '北京大学的学生正在图书馆学习')).toEqual([
|
||||
'北京大学', '的', '学生', '正在', '图书馆', '学习',
|
||||
])
|
||||
})
|
||||
|
||||
it('gets the textbook ambiguities right', () => {
|
||||
expect(words(seg, '研究生命的起源')).toEqual(['研究', '生命', '的', '起源'])
|
||||
expect(words(seg, '乒乓球拍卖完了')).toEqual(['乒乓球', '拍卖', '完', '了'])
|
||||
})
|
||||
|
||||
// The minimal pair, and the one that says the line above was a decision rather
|
||||
// than a bias against long words: the same five characters open both
|
||||
// sentences, and 研究生 is the right answer in one of them.
|
||||
it('finds 研究生 where 研究生 is the word', () => {
|
||||
expect(words(seg, '研究生宿舍')).toEqual(['研究生', '宿舍'])
|
||||
expect(words(seg, '他们正在研究生物')).toEqual(['他们', '正在', '研究', '生物'])
|
||||
})
|
||||
|
||||
// The three particles the 错别字 rules are about have to survive as their own
|
||||
// tokens, or those rules have nothing to anchor to.
|
||||
it('keeps 的 / 地 / 得 separate', () => {
|
||||
expect(words(seg, '她高兴地笑了')).toContain('地')
|
||||
expect(words(seg, '这个问题需要认真地思考')).toContain('地')
|
||||
expect(words(seg, '他跑得很快')).toContain('得')
|
||||
expect(words(seg, '我的书')).toContain('的')
|
||||
})
|
||||
|
||||
it('knows the words the build script asserts it kept', () => {
|
||||
for (const w of ['我', '的', '图书馆', '乒乓球', '公园', '的士']) {
|
||||
expect(seg.has(w), w).toBe(true)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,271 @@
|
||||
// Chinese word segmentation — the thing that has to exist before any of Petal's
|
||||
// ESL surfaces can point at a Chinese word.
|
||||
//
|
||||
// Every one of them is built on `wordAt(doc, pos)`, and `wordAt` is a regex over
|
||||
// runs of Latin letters. That works because English writes its word boundaries
|
||||
// down. Chinese does not: 我今天早上去公园跑步了 is eleven characters and seven
|
||||
// words, and which seven is a question with a real answer that no regex can
|
||||
// reach. Until something answers it there is no "word under the cursor" to
|
||||
// hover, look up, read aloud, or plant in the vocabulary garden.
|
||||
//
|
||||
// **Why the answer is a shortest-path walk and not longest-match.** The obvious
|
||||
// algorithm — take the longest dictionary word at each position and move on —
|
||||
// gets the textbook cases wrong in both directions, because the longest match is
|
||||
// not the likeliest one. The standard fix is to score every possible split by
|
||||
// how probable its words are and take the best-scoring path, which is a
|
||||
// shortest-path problem over a small DAG and is what this does. It is why the
|
||||
// word list ships with a frequency column at all.
|
||||
//
|
||||
// **Why it runs in the browser.** It runs on hover. A round-trip per hover is
|
||||
// not a hover, and the whole point of the offline lexicon (SUGGESTIONS §6) is
|
||||
// that the daily reading aids keep working with the tunnel down.
|
||||
|
||||
// A word found in the text, with the offsets it occupies. Offsets are into the
|
||||
// string that was passed in — the caller maps them to ProseMirror positions the
|
||||
// same way the spell and suggestion layers already do.
|
||||
export interface Token {
|
||||
word: string
|
||||
from: number
|
||||
to: number
|
||||
}
|
||||
|
||||
// Han characters only. Not a hand-rolled U+4E00–U+9FFF range: that misses the
|
||||
// extension blocks, and a character Petal fails to recognise as Chinese is one
|
||||
// the English tokenizer then tries to make sense of.
|
||||
const HAN = /^\p{Script=Han}$/u
|
||||
|
||||
// `ch` is one character, but "one character" is a code point, not a UTF-16 code
|
||||
// unit: the extension blocks live above the BMP and `text[i]` there is half a
|
||||
// surrogate pair. Testing a lone surrogate against \p{Script=Han} says no —
|
||||
// which would silently undo the whole reason this is a property escape — so the
|
||||
// pair is joined back up before it is asked about. Anchored, so a two-code-unit
|
||||
// string has to *be* one Han character rather than merely contain one.
|
||||
export function isHan(ch: string): boolean {
|
||||
return HAN.test(ch)
|
||||
}
|
||||
|
||||
// charAt is isHan's companion for scanning a string: it returns the whole code
|
||||
// point beginning at `i`, so a surrogate pair is asked about as one character.
|
||||
function charAt(text: string, i: number): string {
|
||||
const code = text.codePointAt(i)
|
||||
return code === undefined ? '' : String.fromCodePoint(code)
|
||||
}
|
||||
|
||||
// isHanAt reports whether the code point *beginning* at `i` is Han. A low
|
||||
// surrogate (the second half of a pair) is never a start, so it answers for the
|
||||
// pair it belongs to instead — which keeps a run contiguous across it.
|
||||
export function isHanAt(text: string, i: number): boolean {
|
||||
const code = text.charCodeAt(i)
|
||||
if (code >= 0xdc00 && code <= 0xdfff && i > 0) return isHanAt(text, i - 1)
|
||||
return isHan(charAt(text, i))
|
||||
}
|
||||
|
||||
// How many code units the character beginning at `i` occupies: two for a
|
||||
// surrogate pair, one for everything else. Every step through a string here goes
|
||||
// through this, so a supplementary-plane character is never cut in half.
|
||||
function charLen(text: string, i: number): number {
|
||||
const code = text.charCodeAt(i)
|
||||
return code >= 0xd800 && code <= 0xdbff && i + 1 < text.length ? 2 : 1
|
||||
}
|
||||
|
||||
// Where the character *before* `i` begins, or -1 when there is none.
|
||||
function prevCharStart(text: string, i: number): number {
|
||||
if (i <= 0) return -1
|
||||
const j = i - 1
|
||||
const code = text.charCodeAt(j)
|
||||
return code >= 0xdc00 && code <= 0xdfff && j > 0 ? j - 1 : j
|
||||
}
|
||||
|
||||
// The longest word the walk will consider at any position. The dictionary
|
||||
// contains longer entries (chengyu, place names, a few titles), but the cost of
|
||||
// the walk is linear in this number and the entries beyond it are rare enough
|
||||
// that paying for them on every hover is the wrong trade. Six characters covers
|
||||
// every ordinary word and every four-character idiom.
|
||||
const MAX_WORD_LEN = 6
|
||||
|
||||
// What an unknown single character is worth, as a fraction of one occurrence.
|
||||
// It must be *positive* — every position needs some path through it, or a
|
||||
// sentence containing one unfamiliar character would have no segmentation at
|
||||
// all — and it must be small enough that a real one-character word always wins.
|
||||
// Half an occurrence is below the rarest thing in the list (which is 1) and
|
||||
// above zero, which is the whole specification.
|
||||
const UNKNOWN_WEIGHT = 0.5
|
||||
|
||||
export interface Segmenter {
|
||||
// segment splits a whole string. Runs of non-Han text are skipped rather than
|
||||
// returned: this is the Chinese tokenizer, and the Latin one is still running
|
||||
// over the same paragraph.
|
||||
segment(text: string): Token[]
|
||||
// wordAt returns the token covering `index`, or null when that position is
|
||||
// not inside Han text. This is the hover/click path, and it segments only the
|
||||
// run around the position rather than the whole document.
|
||||
wordAt(text: string, index: number): Token | null
|
||||
// has reports whether a word is in the list — the 错别字 rules ask, to check
|
||||
// that a correction they are about to propose is a real word.
|
||||
has(word: string): boolean
|
||||
size: number
|
||||
}
|
||||
|
||||
// buildSegmenter turns the raw `word freq` list into something that can answer
|
||||
// questions about it. Exported for tests, which build tiny dictionaries by hand;
|
||||
// the app reaches it through loadSegmenter.
|
||||
export function buildSegmenter(source: string): Segmenter {
|
||||
const freq = new Map<string, number>()
|
||||
let total = 0
|
||||
for (const line of source.split('\n')) {
|
||||
if (!line) continue
|
||||
const sp = line.lastIndexOf(' ')
|
||||
if (sp <= 0) continue
|
||||
const word = line.slice(0, sp)
|
||||
const n = Number(line.slice(sp + 1))
|
||||
if (!Number.isFinite(n) || n <= 0) continue
|
||||
freq.set(word, n)
|
||||
total += n
|
||||
}
|
||||
// A dictionary with nothing in it would make every log() below -Infinity.
|
||||
const logTotal = Math.log(Math.max(total, 1))
|
||||
const unknownScore = Math.log(UNKNOWN_WEIGHT) - logTotal
|
||||
|
||||
// The walk, over one run of Han characters.
|
||||
//
|
||||
// `best[i]` is the score of the best segmentation of run[i..], and `next[i]`
|
||||
// is where that segmentation's first word ends. Filling it right-to-left means
|
||||
// each position only ever reads answers that are already final, which is what
|
||||
// makes this linear rather than exponential in the number of possible splits.
|
||||
function walk(run: string, base: number, out: Token[]): void {
|
||||
const n = run.length
|
||||
const best = new Float64Array(n + 1)
|
||||
const next = new Int32Array(n + 1)
|
||||
best[n] = 0
|
||||
for (let i = n - 1; i >= 0; i--) {
|
||||
// Positions inside a surrogate pair are not character boundaries, so no
|
||||
// path ever arrives at one and nothing below would ever read the answer.
|
||||
if (i > 0 && charLen(run, i - 1) === 2) continue
|
||||
let bestScore = -Infinity
|
||||
let bestEnd = i + charLen(run, i)
|
||||
// `len` counts *characters*, which is what MAX_WORD_LEN is in and what the
|
||||
// dictionary is keyed by; `j` counts code units, which is what a slice is
|
||||
// in. The two differ exactly where a supplementary character sits.
|
||||
let j = bestEnd
|
||||
for (let len = 1; len <= MAX_WORD_LEN && j <= n; len++) {
|
||||
const f = freq.get(run.slice(i, j))
|
||||
let score: number
|
||||
if (f === undefined) {
|
||||
// Only a single unknown character is a candidate. Allowing unknown
|
||||
// multi-character spans would let the walk invent words.
|
||||
if (len > 1) {
|
||||
if (j >= n) break
|
||||
j += charLen(run, j)
|
||||
continue
|
||||
}
|
||||
score = unknownScore
|
||||
} else {
|
||||
score = Math.log(f) - logTotal
|
||||
}
|
||||
score += best[j]
|
||||
if (score > bestScore) {
|
||||
bestScore = score
|
||||
bestEnd = j
|
||||
}
|
||||
if (j >= n) break
|
||||
j += charLen(run, j)
|
||||
}
|
||||
best[i] = bestScore
|
||||
next[i] = bestEnd
|
||||
}
|
||||
for (let i = 0; i < n; ) {
|
||||
const end = next[i]
|
||||
out.push({ word: run.slice(i, end), from: base + i, to: base + end })
|
||||
i = end
|
||||
}
|
||||
}
|
||||
|
||||
function segment(text: string): Token[] {
|
||||
const out: Token[] = []
|
||||
let i = 0
|
||||
while (i < text.length) {
|
||||
if (!isHanAt(text, i)) {
|
||||
i += charLen(text, i)
|
||||
continue
|
||||
}
|
||||
let j = i
|
||||
while (j < text.length && isHanAt(text, j)) j += charLen(text, j)
|
||||
walk(text.slice(i, j), i, out)
|
||||
i = j
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// How much context a hover segments. The run around the cursor is bounded
|
||||
// because a pasted page of Chinese with no punctuation is one run, and a hover
|
||||
// must not walk it. Segmentation is local enough that a window this size
|
||||
// reaches the same answer as the whole paragraph would: the walk's decisions
|
||||
// are dominated by the two or three characters either side, and a word longer
|
||||
// than MAX_WORD_LEN cannot span the window's edge anyway.
|
||||
const WINDOW = 60
|
||||
|
||||
function wordAt(text: string, index: number): Token | null {
|
||||
if (index < 0 || index > text.length) return null
|
||||
// `index` names a gap between characters; a word covers characters. So the
|
||||
// question is resolved on the character at `index` — the one to the *right*
|
||||
// of the caret — and a boundary belongs to the word that starts there rather
|
||||
// than the one that ends there. For a hover that is simply correct: index 6
|
||||
// of 我今天去公园跑步了 is the 跑 being pointed at.
|
||||
//
|
||||
// The step back covers the case where there is no character to the right:
|
||||
// the caret at the end of the text, or against following punctuation. That
|
||||
// is where the caret sits the instant an IME commits a word, and Ctrl/Cmd+D
|
||||
// there must look up the word just typed.
|
||||
let probe = index
|
||||
if (probe >= text.length || !isHanAt(text, probe)) {
|
||||
const prev = prevCharStart(text, probe)
|
||||
if (prev >= 0 && isHanAt(text, prev)) probe = prev
|
||||
else return null
|
||||
}
|
||||
// Never leave the probe inside a surrogate pair: the slice below starts
|
||||
// there, and half a character is not a character.
|
||||
if (probe > 0 && charLen(text, probe - 1) === 2) probe -= 1
|
||||
|
||||
let start = probe
|
||||
while (start > 0 && probe - start < WINDOW) {
|
||||
const prev = prevCharStart(text, start)
|
||||
if (prev < 0 || !isHanAt(text, prev)) break
|
||||
start = prev
|
||||
}
|
||||
let end = probe
|
||||
while (end < text.length && isHanAt(text, end) && end - probe < WINDOW) end += charLen(text, end)
|
||||
|
||||
const tokens: Token[] = []
|
||||
walk(text.slice(start, end), start, tokens)
|
||||
for (const t of tokens) {
|
||||
if (probe >= t.from && probe < t.to) return t
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
return { segment, wordAt, has: (w) => freq.has(w), size: freq.size }
|
||||
}
|
||||
|
||||
// Where the word list lives. Gzipped, like every dictionary Petal ships that is
|
||||
// bigger than English's.
|
||||
const WORDS_URL = '/dictionaries/zh/words.txt.gz'
|
||||
|
||||
// loadSegmenter fetches and builds the segmenter. One per session, like the
|
||||
// spelling dictionaries — the cost is the parse, not the download, and paying it
|
||||
// per document would be paying it per document for no reason.
|
||||
//
|
||||
// A failure resolves to null rather than throwing. Petal without segmentation is
|
||||
// Petal with no Chinese hover, which is a diminished editor; Petal that refused
|
||||
// to open because a static asset 404ed is no editor at all.
|
||||
export async function loadSegmenter(url = WORDS_URL): Promise<Segmenter | null> {
|
||||
try {
|
||||
const res = await fetch(url)
|
||||
if (!res.ok || !res.body) return null
|
||||
const stream = res.body.pipeThrough(new DecompressionStream('gzip'))
|
||||
const text = await new Response(stream).text()
|
||||
const seg = buildSegmenter(text)
|
||||
return seg.size > 0 ? seg : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user