Four ways the two scripts weren't the same app, and a smaller cat

A review of the pair work found the seams — every one of them a place where
the Chinese half was written and the older Latin half was left standing.

The right-click menu still asked the Latin tokenizer whether there was a word
under the pointer, so right-clicking a hanzi opened the browser's own menu
instead of the card. Hover, long-press and Ctrl+D had all moved to the shared
resolver; this one hadn't, and it is the surface the segmenter's own header
names first.

isHan is a property escape precisely so the extension blocks are covered, and
then every call site handed it one UTF-16 code unit — half a surrogate pair
for anything above the BMP, which \p{Script=Han} rightly says is not Han. The
run split in two around the character and the words either side stopped being
looked up. The walk, the scan and wordAt now step by code point, the regex is
anchored, and the test that passed by accident (unanchored, so it searched a
two-unit string rather than testing one character) is joined by one that
would have failed.

The pair picker sent the pair alone. The server validates pair and direction
as one decision and refuses a learner direction for a pair it has no word
list for — so an English speaker learning Chinese could not move to French at
all: every button failed with the generic message. It now names both, keeps
her direction where the target pack has a learner side, and returns her to
learning_en where it does not. Routed through useSession rather than the
picker's own api call, so me.direction — which decides whether the word list
stays loaded — moves with it.

UpdateMe answered every error from Get with 401. A SQLite fault on a PATCH
would have tripped the client's session interceptor and thrown a writer into
the signed-out overlay while her session was fine. Only a missing row means
not signed in, which is the distinction SetPair already made below it.

emitCommittedRef was assigned during render and called later from
compositionend; a render React discards must not leave its closure behind for
a DOM event.

And the kitten is 10% smaller — one clamp, three terms, everything else
calc()s off it.

vitest 297/297, tsc, go build/vet/test clean.

internal/auth/users.go

@@ -190,9 +190,17 @@ func (u *UserStore) UpdateMeHandler() http.HandlerFunc {
 		id := UserID(r.Context())
 		current, err := u.Get(id)
 		if err != nil {
+			// Only a missing row means "not signed in". A dictionary-file or
+			// SQLite fault answered as 401 would trip the client's session
+			// interceptor and throw a writer out of an app she is still signed
+			// in to — the same distinction SetPair's error branch makes below.
+			if errors.Is(err, sql.ErrNoRows) {
 				httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
 				return
 			}
+			httputil.ServerError(w, err)
+			return
+		}
 
 		lang, direction := current.PairLang, current.Direction
 		if body.PairLang != nil {

web/src/App.tsx

@@ -33,7 +33,7 @@ export default function App() {
   const night = useNightMode()
   // Who's writing, and whether the server still recognises them. `signedOut`
   // flips the moment any call comes back 401.
-  const { me, signedOut, setDirection } = useSession()
+  const { me, signedOut, setDirection, setPair } = useSession()
   const t = usePack()
   // A real account to sign out of, as opposed to the hardcoded local user a
   // build without auth configured runs as.
@@ -553,6 +553,7 @@ export default function App() {
             account={account}
             direction={me?.direction}
             onDirection={setDirection}
+            onPair={setPair}
           />
         </div>
 

web/src/components/DocList/DocList.tsx

@@ -25,6 +25,7 @@ interface Props {
   // and the drawer is the only chrome always one tap away on a phone.
   direction?: string
   onDirection?: (direction: string) => Promise<void>
+  onPair?: (lang: string, direction: string) => Promise<void>
 }
 
 // Sidebar sort orders. 'recent' keeps the server's updated_at-desc ordering.
@@ -50,6 +51,7 @@ export function DocList({
   account,
   direction,
   onDirection,
+  onPair,
 }: Props) {
   const t = usePack()
   // Active tag filter (null = show all). Cleared automatically if the tag
@@ -168,7 +170,7 @@ export function DocList({
       {/* The pair Petal speaks. Unlike the rows above it this is not about any
           document, and unlike sign-out it is offered whether or not there is an
           account behind the session — a local-dev build still has a langpack. */}
-      <LanguagePicker direction={direction} onDirection={onDirection} />
+      <LanguagePicker direction={direction} onDirection={onDirection} onPair={onPair} />
 
       {/* Who's writing, and the way out. Shown only when there's a real account
           behind the session — a local-dev build has nobody to sign out as. */}

web/src/components/DocList/LanguagePicker.tsx

@@ -22,9 +22,12 @@ interface Props {
   // and this control is only where the writer says so.
   direction?: string
   onDirection?: (direction: string) => Promise<void>
+  // Move the pair itself. Owned by App for the same reason: the answer carries
+  // the direction too, and the account's direction is what loads the word list.
+  onPair?: (lang: string, direction: string) => Promise<void>
 }
 
-export function LanguagePicker({ direction, onDirection }: Props = {}) {
+export function LanguagePicker({ direction, onDirection, onPair }: Props = {}) {
   const t = usePack()
   const packs = shippedPacks()
   const [saving, setSaving] = useState<string | null>(null)
@@ -61,11 +64,24 @@ export function LanguagePicker({ direction, onDirection }: Props = {}) {
     setSaving(code)
     setFailed(false)
     try {
+      // Name the direction alongside the pair. The server validates the two as
+      // one decision and refuses a learner direction for a pair it has no word
+      // list for, so an account that is learning Chinese cannot move to French
+      // by naming only the pair — that request is rejected outright, and the
+      // writer is left on a picker whose buttons all fail. A pair with no
+      // learner side can only be travelled toward English; saying so is how the
+      // move is actually made.
+      const target = packs.find((p) => p.code === code)
+      const next = target?.learner ? (direction ?? 'learning_en') : 'learning_en'
+      if (onPair) {
+        await onPair(code, next)
+      } else {
         const me = await api.setPairLang(code)
-      // The server's answer, not the code we asked for. Everything downstream —
+        // The server's answer, not the code we asked for. Everything downstream
-      // her dictionary, the read-aloud voice, the word lookups — follows the
+        // — her dictionary, the read-aloud voice, the word lookups — follows the
         // pack, so it must follow what was actually stored.
         setPackLang(me.pair_lang)
+      }
     } catch {
       // A 401 has already surfaced as the sign-in overlay through the client's
       // interceptor; anything else leaves her on the pair she was already on,

web/src/components/Editor/EditorCore.tsx

@@ -445,6 +445,11 @@ export function EditorCore({
   // The composition-end nudge. Same payload as onUpdate's, with `composing`
   // false by construction — this runs after the composition has ended and its
   // final changes have been flushed, so the text here is the committed one.
+  //
+  // Written in an effect rather than during render, like recomputeRailRef
+  // below: a render React throws away must not be the one that leaves its
+  // closure behind for a DOM event to call later.
+  useEffect(() => {
     emitCommittedRef.current = () => {
       if (!editor) return
       onChange({
@@ -455,6 +460,7 @@ export function EditorCore({
       })
       recomputeRailRef.current()
     }
+  }, [editor, onChange])
 
   // When the selected document changes, swap in its content without emitting an
   // update (false) so loading a doc doesn't trigger a spurious save.
@@ -1025,13 +1031,16 @@ export function EditorCore({
       if (!editor) return
       const coords = editor.view.posAtCoords({ left: e.clientX, top: e.clientY })
       if (!coords) return
-      if (!wordAt(editor.state.doc, coords.pos, wordAlphabet)) return
+      // Whichever script is under the pointer — the same resolver openWordLookup
+      // uses, so a Chinese word gets the card here too rather than falling
+      // through to the native menu.
+      if (!resolveWord(coords.pos)) return
       e.preventDefault()
       // A misspelled word offers corrections first; otherwise look it up.
       if (openMisspellAt(coords.pos)) return
       openWordLookup(coords.pos)
     },
-    [editor, wordAlphabet, openMisspellAt, openWordLookup],
+    [editor, resolveWord, openMisspellAt, openWordLookup],
   )
 
   // Touch has no hover or right-click, so a long-press (~500ms without moving)

web/src/hooks/useSession.ts

@@ -51,5 +51,18 @@ export function useSession() {
     setMe(updated)
   }
 
-  return { me, signedOut, setDirection }
+  // Move the pair, naming the direction with it. The two are validated together
+  // server-side, so an account that is learning Chinese cannot change pair by
+  // sending `pair_lang` alone — the combination it would ask for (French with
+  // segmentation) does not exist and is refused. Saying both is how that move is
+  // made, and routing it through here rather than through the picker's own
+  // `api` call is what keeps `me.direction` — which decides whether the word
+  // list stays loaded — in step with what was actually stored.
+  const setPair = async (lang: string, direction: string) => {
+    const updated = await api.setPair(lang, direction)
+    setPackLang(updated.pair_lang)
+    setMe(updated)
+  }
+
+  return { me, signedOut, setDirection, setPair }
 }

web/src/index.css

@@ -425,7 +425,7 @@ button, a, input {
 .petal-companion {
   /* Mascot size scales with the viewport width: ~original on a laptop, up to
      ~2× on a large desktop. Tune the middle (vw) term to taste. */
-  --petal-companion-size: clamp(10rem, 17vw, 20rem);
+  --petal-companion-size: clamp(9rem, 15.3vw, 18rem);
   animation: petal-bob 3.2s ease-in-out infinite;
   /* Shrink toward its corner when fading out of a card's way. `scale` is a
      separate property from `transform` so it composes with the bob keyframes. */

web/src/lib/segment.test.ts

@@ -78,6 +78,22 @@ describe('what the walk does with what it does not know', () => {
     expect(words(seg, '我龥龥')).toEqual(['我', '龥', '龥'])
   })
 
+  // A character above the BMP is two UTF-16 code units, and asking about either
+  // half alone says "not Han". Getting this wrong is quiet: the run breaks in
+  // two around the character, the words either side of it stop being looked up,
+  // and nothing anywhere reports an error.
+  it('a supplementary-plane character is one unknown token inside the run', () => {
+    const seg = dict({ 我: 90000, 喜欢: 5000, 猫: 2000 })
+    const text = '我喜欢𠀀猫'
+    expect(words(seg, text)).toEqual(['我', '喜欢', '𠀀', '猫'])
+    for (const t of seg.segment(text)) expect(text.slice(t.from, t.to)).toBe(t.word)
+    // And it is hoverable from either code unit — a caret offset can land on
+    // the low surrogate, which is not a character boundary but is a real index.
+    expect(seg.wordAt(text, 3)?.word).toBe('𠀀')
+    expect(seg.wordAt(text, 4)?.word).toBe('𠀀')
+    expect(seg.wordAt(text, 5)?.word).toBe('猫')
+  })
+
   // A rare real word still loses to two common ones — this is the property that
   // lets the shipped list keep 100,000 rare CC-CEDICT headwords without them
   // distorting ordinary sentences.

web/src/lib/segment.ts

@@ -32,12 +32,50 @@ export interface Token {
 // Han characters only. Not a hand-rolled U+4E00–U+9FFF range: that misses the
 // extension blocks, and a character Petal fails to recognise as Chinese is one
 // the English tokenizer then tries to make sense of.
-const HAN = /\p{Script=Han}/u
+const HAN = /^\p{Script=Han}$/u
 
+// `ch` is one character, but "one character" is a code point, not a UTF-16 code
+// unit: the extension blocks live above the BMP and `text[i]` there is half a
+// surrogate pair. Testing a lone surrogate against \p{Script=Han} says no —
+// which would silently undo the whole reason this is a property escape — so the
+// pair is joined back up before it is asked about. Anchored, so a two-code-unit
+// string has to *be* one Han character rather than merely contain one.
 export function isHan(ch: string): boolean {
   return HAN.test(ch)
 }
 
+// charAt is isHan's companion for scanning a string: it returns the whole code
+// point beginning at `i`, so a surrogate pair is asked about as one character.
+function charAt(text: string, i: number): string {
+  const code = text.codePointAt(i)
+  return code === undefined ? '' : String.fromCodePoint(code)
+}
+
+// isHanAt reports whether the code point *beginning* at `i` is Han. A low
+// surrogate (the second half of a pair) is never a start, so it answers for the
+// pair it belongs to instead — which keeps a run contiguous across it.
+export function isHanAt(text: string, i: number): boolean {
+  const code = text.charCodeAt(i)
+  if (code >= 0xdc00 && code <= 0xdfff && i > 0) return isHanAt(text, i - 1)
+  return isHan(charAt(text, i))
+}
+
+// How many code units the character beginning at `i` occupies: two for a
+// surrogate pair, one for everything else. Every step through a string here goes
+// through this, so a supplementary-plane character is never cut in half.
+function charLen(text: string, i: number): number {
+  const code = text.charCodeAt(i)
+  return code >= 0xd800 && code <= 0xdbff && i + 1 < text.length ? 2 : 1
+}
+
+// Where the character *before* `i` begins, or -1 when there is none.
+function prevCharStart(text: string, i: number): number {
+  if (i <= 0) return -1
+  const j = i - 1
+  const code = text.charCodeAt(j)
+  return code >= 0xdc00 && code <= 0xdfff && j > 0 ? j - 1 : j
+}
+
 // The longest word the walk will consider at any position. The dictionary
 // contains longer entries (chengyu, place names, a few titles), but the cost of
 // the walk is linear in this number and the entries beyond it are rare enough
@@ -100,16 +138,26 @@ export function buildSegmenter(source: string): Segmenter {
     const next = new Int32Array(n + 1)
     best[n] = 0
     for (let i = n - 1; i >= 0; i--) {
+      // Positions inside a surrogate pair are not character boundaries, so no
+      // path ever arrives at one and nothing below would ever read the answer.
+      if (i > 0 && charLen(run, i - 1) === 2) continue
       let bestScore = -Infinity
-      let bestEnd = i + 1
+      let bestEnd = i + charLen(run, i)
-      const limit = Math.min(n, i + MAX_WORD_LEN)
+      // `len` counts *characters*, which is what MAX_WORD_LEN is in and what the
-      for (let j = i + 1; j <= limit; j++) {
+      // dictionary is keyed by; `j` counts code units, which is what a slice is
+      // in. The two differ exactly where a supplementary character sits.
+      let j = bestEnd
+      for (let len = 1; len <= MAX_WORD_LEN && j <= n; len++) {
         const f = freq.get(run.slice(i, j))
         let score: number
         if (f === undefined) {
           // Only a single unknown character is a candidate. Allowing unknown
           // multi-character spans would let the walk invent words.
-          if (j > i + 1) continue
+          if (len > 1) {
+            if (j >= n) break
+            j += charLen(run, j)
+            continue
+          }
           score = unknownScore
         } else {
           score = Math.log(f) - logTotal
@@ -119,6 +167,8 @@ export function buildSegmenter(source: string): Segmenter {
           bestScore = score
           bestEnd = j
         }
+        if (j >= n) break
+        j += charLen(run, j)
       }
       best[i] = bestScore
       next[i] = bestEnd
@@ -134,12 +184,12 @@ export function buildSegmenter(source: string): Segmenter {
     const out: Token[] = []
     let i = 0
     while (i < text.length) {
-      if (!isHan(text[i])) {
+      if (!isHanAt(text, i)) {
-        i++
+        i += charLen(text, i)
         continue
       }
       let j = i
-      while (j < text.length && isHan(text[j])) j++
+      while (j < text.length && isHanAt(text, j)) j += charLen(text, j)
       walk(text.slice(i, j), i, out)
       i = j
     }
@@ -167,14 +217,23 @@ export function buildSegmenter(source: string): Segmenter {
     // is where the caret sits the instant an IME commits a word, and Ctrl/Cmd+D
     // there must look up the word just typed.
     let probe = index
-    if (probe >= text.length || !isHan(text[probe])) {
+    if (probe >= text.length || !isHanAt(text, probe)) {
-      if (probe > 0 && isHan(text[probe - 1])) probe -= 1
+      const prev = prevCharStart(text, probe)
+      if (prev >= 0 && isHanAt(text, prev)) probe = prev
       else return null
     }
+    // Never leave the probe inside a surrogate pair: the slice below starts
+    // there, and half a character is not a character.
+    if (probe > 0 && charLen(text, probe - 1) === 2) probe -= 1
+
     let start = probe
-    while (start > 0 && isHan(text[start - 1]) && probe - start < WINDOW) start--
+    while (start > 0 && probe - start < WINDOW) {
+      const prev = prevCharStart(text, start)
+      if (prev < 0 || !isHanAt(text, prev)) break
+      start = prev
+    }
     let end = probe
-    while (end < text.length && isHan(text[end]) && end - probe < WINDOW) end++
+    while (end < text.length && isHanAt(text, end) && end - probe < WINDOW) end += charLen(text, end)
 
     const tokens: Token[] = []
     walk(text.slice(start, end), start, tokens)