From 8f2ad34a101b4465923aa88702e0648ac4e34d7a Mon Sep 17 00:00:00 2001 From: prosolis <5590409+prosolis@users.noreply.github.com> Date: Tue, 28 Jul 2026 20:03:45 -0700 Subject: [PATCH] Four ways the two scripts weren't the same app, and a smaller cat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A review of the pair work found the seams — every one of them a place where the Chinese half was written and the older Latin half was left standing. The right-click menu still asked the Latin tokenizer whether there was a word under the pointer, so right-clicking a hanzi opened the browser's own menu instead of the card. Hover, long-press and Ctrl+D had all moved to the shared resolver; this one hadn't, and it is the surface the segmenter's own header names first. isHan is a property escape precisely so the extension blocks are covered, and then every call site handed it one UTF-16 code unit — half a surrogate pair for anything above the BMP, which \p{Script=Han} rightly says is not Han. The run split in two around the character and the words either side stopped being looked up. The walk, the scan and wordAt now step by code point, the regex is anchored, and the test that passed by accident (unanchored, so it searched a two-unit string rather than testing one character) is joined by one that would have failed. The pair picker sent the pair alone. The server validates pair and direction as one decision and refuses a learner direction for a pair it has no word list for — so an English speaker learning Chinese could not move to French at all: every button failed with the generic message. It now names both, keeps her direction where the target pack has a learner side, and returns her to learning_en where it does not. Routed through useSession rather than the picker's own api call, so me.direction — which decides whether the word list stays loaded — moves with it. UpdateMe answered every error from Get with 401. A SQLite fault on a PATCH would have tripped the client's session interceptor and thrown a writer into the signed-out overlay while her session was fine. Only a missing row means not signed in, which is the distinction SetPair already made below it. emitCommittedRef was assigned during render and called later from compositionend; a render React discards must not leave its closure behind for a DOM event. And the kitten is 10% smaller — one clamp, three terms, everything else calc()s off it. vitest 297/297, tsc, go build/vet/test clean. --- internal/auth/users.go | 10 ++- web/src/App.tsx | 3 +- web/src/components/DocList/DocList.tsx | 4 +- web/src/components/DocList/LanguagePicker.tsx | 28 +++++-- web/src/components/Editor/EditorCore.tsx | 33 +++++--- web/src/hooks/useSession.ts | 15 +++- web/src/index.css | 2 +- web/src/lib/segment.test.ts | 16 ++++ web/src/lib/segment.ts | 83 ++++++++++++++++--- 9 files changed, 159 insertions(+), 35 deletions(-) diff --git a/internal/auth/users.go b/internal/auth/users.go index b857569..afa4078 100644 --- a/internal/auth/users.go +++ b/internal/auth/users.go @@ -190,7 +190,15 @@ func (u *UserStore) UpdateMeHandler() http.HandlerFunc { id := UserID(r.Context()) current, err := u.Get(id) if err != nil { - httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in") + // Only a missing row means "not signed in". A dictionary-file or + // SQLite fault answered as 401 would trip the client's session + // interceptor and throw a writer out of an app she is still signed + // in to — the same distinction SetPair's error branch makes below. + if errors.Is(err, sql.ErrNoRows) { + httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in") + return + } + httputil.ServerError(w, err) return } diff --git a/web/src/App.tsx b/web/src/App.tsx index 1f64eb0..cc0a656 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -33,7 +33,7 @@ export default function App() { const night = useNightMode() // Who's writing, and whether the server still recognises them. `signedOut` // flips the moment any call comes back 401. - const { me, signedOut, setDirection } = useSession() + const { me, signedOut, setDirection, setPair } = useSession() const t = usePack() // A real account to sign out of, as opposed to the hardcoded local user a // build without auth configured runs as. @@ -553,6 +553,7 @@ export default function App() { account={account} direction={me?.direction} onDirection={setDirection} + onPair={setPair} /> diff --git a/web/src/components/DocList/DocList.tsx b/web/src/components/DocList/DocList.tsx index ffbf0b5..fdf84b4 100644 --- a/web/src/components/DocList/DocList.tsx +++ b/web/src/components/DocList/DocList.tsx @@ -25,6 +25,7 @@ interface Props { // and the drawer is the only chrome always one tap away on a phone. direction?: string onDirection?: (direction: string) => Promise + onPair?: (lang: string, direction: string) => Promise } // Sidebar sort orders. 'recent' keeps the server's updated_at-desc ordering. @@ -50,6 +51,7 @@ export function DocList({ account, direction, onDirection, + onPair, }: Props) { const t = usePack() // Active tag filter (null = show all). Cleared automatically if the tag @@ -168,7 +170,7 @@ export function DocList({ {/* The pair Petal speaks. Unlike the rows above it this is not about any document, and unlike sign-out it is offered whether or not there is an account behind the session — a local-dev build still has a langpack. */} - + {/* Who's writing, and the way out. Shown only when there's a real account behind the session — a local-dev build has nobody to sign out as. */} diff --git a/web/src/components/DocList/LanguagePicker.tsx b/web/src/components/DocList/LanguagePicker.tsx index b8cc40f..5ac13bd 100644 --- a/web/src/components/DocList/LanguagePicker.tsx +++ b/web/src/components/DocList/LanguagePicker.tsx @@ -22,9 +22,12 @@ interface Props { // and this control is only where the writer says so. direction?: string onDirection?: (direction: string) => Promise + // Move the pair itself. Owned by App for the same reason: the answer carries + // the direction too, and the account's direction is what loads the word list. + onPair?: (lang: string, direction: string) => Promise } -export function LanguagePicker({ direction, onDirection }: Props = {}) { +export function LanguagePicker({ direction, onDirection, onPair }: Props = {}) { const t = usePack() const packs = shippedPacks() const [saving, setSaving] = useState(null) @@ -61,11 +64,24 @@ export function LanguagePicker({ direction, onDirection }: Props = {}) { setSaving(code) setFailed(false) try { - const me = await api.setPairLang(code) - // The server's answer, not the code we asked for. Everything downstream — - // her dictionary, the read-aloud voice, the word lookups — follows the - // pack, so it must follow what was actually stored. - setPackLang(me.pair_lang) + // Name the direction alongside the pair. The server validates the two as + // one decision and refuses a learner direction for a pair it has no word + // list for, so an account that is learning Chinese cannot move to French + // by naming only the pair — that request is rejected outright, and the + // writer is left on a picker whose buttons all fail. A pair with no + // learner side can only be travelled toward English; saying so is how the + // move is actually made. + const target = packs.find((p) => p.code === code) + const next = target?.learner ? (direction ?? 'learning_en') : 'learning_en' + if (onPair) { + await onPair(code, next) + } else { + const me = await api.setPairLang(code) + // The server's answer, not the code we asked for. Everything downstream + // — her dictionary, the read-aloud voice, the word lookups — follows the + // pack, so it must follow what was actually stored. + setPackLang(me.pair_lang) + } } catch { // A 401 has already surfaced as the sign-in overlay through the client's // interceptor; anything else leaves her on the pair she was already on, diff --git a/web/src/components/Editor/EditorCore.tsx b/web/src/components/Editor/EditorCore.tsx index c80ad12..b9b4579 100644 --- a/web/src/components/Editor/EditorCore.tsx +++ b/web/src/components/Editor/EditorCore.tsx @@ -445,16 +445,22 @@ export function EditorCore({ // The composition-end nudge. Same payload as onUpdate's, with `composing` // false by construction — this runs after the composition has ended and its // final changes have been flushed, so the text here is the committed one. - emitCommittedRef.current = () => { - if (!editor) return - onChange({ - content: JSON.stringify(editor.getJSON()), - content_text: editor.getText(), - word_count: editor.storage.characterCount.words(), - composing: false, - }) - recomputeRailRef.current() - } + // + // Written in an effect rather than during render, like recomputeRailRef + // below: a render React throws away must not be the one that leaves its + // closure behind for a DOM event to call later. + useEffect(() => { + emitCommittedRef.current = () => { + if (!editor) return + onChange({ + content: JSON.stringify(editor.getJSON()), + content_text: editor.getText(), + word_count: editor.storage.characterCount.words(), + composing: false, + }) + recomputeRailRef.current() + } + }, [editor, onChange]) // When the selected document changes, swap in its content without emitting an // update (false) so loading a doc doesn't trigger a spurious save. @@ -1025,13 +1031,16 @@ export function EditorCore({ if (!editor) return const coords = editor.view.posAtCoords({ left: e.clientX, top: e.clientY }) if (!coords) return - if (!wordAt(editor.state.doc, coords.pos, wordAlphabet)) return + // Whichever script is under the pointer — the same resolver openWordLookup + // uses, so a Chinese word gets the card here too rather than falling + // through to the native menu. + if (!resolveWord(coords.pos)) return e.preventDefault() // A misspelled word offers corrections first; otherwise look it up. if (openMisspellAt(coords.pos)) return openWordLookup(coords.pos) }, - [editor, wordAlphabet, openMisspellAt, openWordLookup], + [editor, resolveWord, openMisspellAt, openWordLookup], ) // Touch has no hover or right-click, so a long-press (~500ms without moving) diff --git a/web/src/hooks/useSession.ts b/web/src/hooks/useSession.ts index c719253..0f0a694 100644 --- a/web/src/hooks/useSession.ts +++ b/web/src/hooks/useSession.ts @@ -51,5 +51,18 @@ export function useSession() { setMe(updated) } - return { me, signedOut, setDirection } + // Move the pair, naming the direction with it. The two are validated together + // server-side, so an account that is learning Chinese cannot change pair by + // sending `pair_lang` alone — the combination it would ask for (French with + // segmentation) does not exist and is refused. Saying both is how that move is + // made, and routing it through here rather than through the picker's own + // `api` call is what keeps `me.direction` — which decides whether the word + // list stays loaded — in step with what was actually stored. + const setPair = async (lang: string, direction: string) => { + const updated = await api.setPair(lang, direction) + setPackLang(updated.pair_lang) + setMe(updated) + } + + return { me, signedOut, setDirection, setPair } } diff --git a/web/src/index.css b/web/src/index.css index c26044e..115a22a 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -425,7 +425,7 @@ button, a, input { .petal-companion { /* Mascot size scales with the viewport width: ~original on a laptop, up to ~2× on a large desktop. Tune the middle (vw) term to taste. */ - --petal-companion-size: clamp(10rem, 17vw, 20rem); + --petal-companion-size: clamp(9rem, 15.3vw, 18rem); animation: petal-bob 3.2s ease-in-out infinite; /* Shrink toward its corner when fading out of a card's way. `scale` is a separate property from `transform` so it composes with the bob keyframes. */ diff --git a/web/src/lib/segment.test.ts b/web/src/lib/segment.test.ts index 6546079..6f02fe1 100644 --- a/web/src/lib/segment.test.ts +++ b/web/src/lib/segment.test.ts @@ -78,6 +78,22 @@ describe('what the walk does with what it does not know', () => { expect(words(seg, '我龥龥')).toEqual(['我', '龥', '龥']) }) + // A character above the BMP is two UTF-16 code units, and asking about either + // half alone says "not Han". Getting this wrong is quiet: the run breaks in + // two around the character, the words either side of it stop being looked up, + // and nothing anywhere reports an error. + it('a supplementary-plane character is one unknown token inside the run', () => { + const seg = dict({ 我: 90000, 喜欢: 5000, 猫: 2000 }) + const text = '我喜欢𠀀猫' + expect(words(seg, text)).toEqual(['我', '喜欢', '𠀀', '猫']) + for (const t of seg.segment(text)) expect(text.slice(t.from, t.to)).toBe(t.word) + // And it is hoverable from either code unit — a caret offset can land on + // the low surrogate, which is not a character boundary but is a real index. + expect(seg.wordAt(text, 3)?.word).toBe('𠀀') + expect(seg.wordAt(text, 4)?.word).toBe('𠀀') + expect(seg.wordAt(text, 5)?.word).toBe('猫') + }) + // A rare real word still loses to two common ones — this is the property that // lets the shipped list keep 100,000 rare CC-CEDICT headwords without them // distorting ordinary sentences. diff --git a/web/src/lib/segment.ts b/web/src/lib/segment.ts index 082e709..959440d 100644 --- a/web/src/lib/segment.ts +++ b/web/src/lib/segment.ts @@ -32,12 +32,50 @@ export interface Token { // Han characters only. Not a hand-rolled U+4E00–U+9FFF range: that misses the // extension blocks, and a character Petal fails to recognise as Chinese is one // the English tokenizer then tries to make sense of. -const HAN = /\p{Script=Han}/u +const HAN = /^\p{Script=Han}$/u +// `ch` is one character, but "one character" is a code point, not a UTF-16 code +// unit: the extension blocks live above the BMP and `text[i]` there is half a +// surrogate pair. Testing a lone surrogate against \p{Script=Han} says no — +// which would silently undo the whole reason this is a property escape — so the +// pair is joined back up before it is asked about. Anchored, so a two-code-unit +// string has to *be* one Han character rather than merely contain one. export function isHan(ch: string): boolean { return HAN.test(ch) } +// charAt is isHan's companion for scanning a string: it returns the whole code +// point beginning at `i`, so a surrogate pair is asked about as one character. +function charAt(text: string, i: number): string { + const code = text.codePointAt(i) + return code === undefined ? '' : String.fromCodePoint(code) +} + +// isHanAt reports whether the code point *beginning* at `i` is Han. A low +// surrogate (the second half of a pair) is never a start, so it answers for the +// pair it belongs to instead — which keeps a run contiguous across it. +export function isHanAt(text: string, i: number): boolean { + const code = text.charCodeAt(i) + if (code >= 0xdc00 && code <= 0xdfff && i > 0) return isHanAt(text, i - 1) + return isHan(charAt(text, i)) +} + +// How many code units the character beginning at `i` occupies: two for a +// surrogate pair, one for everything else. Every step through a string here goes +// through this, so a supplementary-plane character is never cut in half. +function charLen(text: string, i: number): number { + const code = text.charCodeAt(i) + return code >= 0xd800 && code <= 0xdbff && i + 1 < text.length ? 2 : 1 +} + +// Where the character *before* `i` begins, or -1 when there is none. +function prevCharStart(text: string, i: number): number { + if (i <= 0) return -1 + const j = i - 1 + const code = text.charCodeAt(j) + return code >= 0xdc00 && code <= 0xdfff && j > 0 ? j - 1 : j +} + // The longest word the walk will consider at any position. The dictionary // contains longer entries (chengyu, place names, a few titles), but the cost of // the walk is linear in this number and the entries beyond it are rare enough @@ -100,16 +138,26 @@ export function buildSegmenter(source: string): Segmenter { const next = new Int32Array(n + 1) best[n] = 0 for (let i = n - 1; i >= 0; i--) { + // Positions inside a surrogate pair are not character boundaries, so no + // path ever arrives at one and nothing below would ever read the answer. + if (i > 0 && charLen(run, i - 1) === 2) continue let bestScore = -Infinity - let bestEnd = i + 1 - const limit = Math.min(n, i + MAX_WORD_LEN) - for (let j = i + 1; j <= limit; j++) { + let bestEnd = i + charLen(run, i) + // `len` counts *characters*, which is what MAX_WORD_LEN is in and what the + // dictionary is keyed by; `j` counts code units, which is what a slice is + // in. The two differ exactly where a supplementary character sits. + let j = bestEnd + for (let len = 1; len <= MAX_WORD_LEN && j <= n; len++) { const f = freq.get(run.slice(i, j)) let score: number if (f === undefined) { // Only a single unknown character is a candidate. Allowing unknown // multi-character spans would let the walk invent words. - if (j > i + 1) continue + if (len > 1) { + if (j >= n) break + j += charLen(run, j) + continue + } score = unknownScore } else { score = Math.log(f) - logTotal @@ -119,6 +167,8 @@ export function buildSegmenter(source: string): Segmenter { bestScore = score bestEnd = j } + if (j >= n) break + j += charLen(run, j) } best[i] = bestScore next[i] = bestEnd @@ -134,12 +184,12 @@ export function buildSegmenter(source: string): Segmenter { const out: Token[] = [] let i = 0 while (i < text.length) { - if (!isHan(text[i])) { - i++ + if (!isHanAt(text, i)) { + i += charLen(text, i) continue } let j = i - while (j < text.length && isHan(text[j])) j++ + while (j < text.length && isHanAt(text, j)) j += charLen(text, j) walk(text.slice(i, j), i, out) i = j } @@ -167,14 +217,23 @@ export function buildSegmenter(source: string): Segmenter { // is where the caret sits the instant an IME commits a word, and Ctrl/Cmd+D // there must look up the word just typed. let probe = index - if (probe >= text.length || !isHan(text[probe])) { - if (probe > 0 && isHan(text[probe - 1])) probe -= 1 + if (probe >= text.length || !isHanAt(text, probe)) { + const prev = prevCharStart(text, probe) + if (prev >= 0 && isHanAt(text, prev)) probe = prev else return null } + // Never leave the probe inside a surrogate pair: the slice below starts + // there, and half a character is not a character. + if (probe > 0 && charLen(text, probe - 1) === 2) probe -= 1 + let start = probe - while (start > 0 && isHan(text[start - 1]) && probe - start < WINDOW) start-- + while (start > 0 && probe - start < WINDOW) { + const prev = prevCharStart(text, start) + if (prev < 0 || !isHanAt(text, prev)) break + start = prev + } let end = probe - while (end < text.length && isHan(text[end]) && end - probe < WINDOW) end++ + while (end < text.length && isHanAt(text, end) && end - probe < WINDOW) end += charLen(text, end) const tokens: Token[] = [] walk(text.slice(start, end), start, tokens)