The zh pair's other direction, and a rule pack that mostly says no
`pair_lang` had always been answering a second question nobody asked: it says which two languages, and every surface built on it assumed English was the one being learned. That is why hanzi is never tokenized, never spell-checked, never glossed — correct for a Mandarin native practising English, backwards for an English native practising Mandarin. `users.direction` (migration 0016) separates the two questions; a `zh-learner` pair code would have been cheaper and would have made two directions of one pair look like two unrelated languages to every query. Segmentation is what replaces `wordAt` where there are no spaces: a shortest-path walk over log-probabilities, 232 ms and 14 MB for 188,522 words. The browser gets the word list because segmentation runs on hover; the server keeps the whole dictionary. Their coverage gates come out opposite on purpose — the client list is frequency-gated because the segmentation is measurably identical without the tail, and the dictionary is gated by nothing, because its only power is to explain and the word a learner stops on is the rare one. The 错别字 pack is 24 confusable pairs behind two mechanical gates. One admits a pair only if the wrong form is not a dictionary word and the right form is, which is why it refuses 自已 for 自己 — a real error whose wrong form is a headword. The other asks the segmenter whether the two characters already belong to two different words, without which 自己经常, 睡觉的时候 and 不知到底 would all be corrupted silently into text still made of real characters. Not deployed (this carries a migration), not seen in a browser, and no account has ever been in the learner direction. The IME composition guards were in scope and are not done — see BUILD_PLAN Phase 26.
This commit is contained in:
prosolis
+13
-2
@@ -3,6 +3,7 @@ import { api, type DocSummary, type DocUpdate, type Document, type Suggestion, t
|
||||
import { useAutoSave } from './hooks/useAutoSave'
|
||||
import { findingKey, useCheckpoint } from './hooks/useCheckpoint'
|
||||
import { useSpellChecker } from './hooks/useSpellChecker'
|
||||
import { useSegmenter } from './hooks/useSegmenter'
|
||||
import { useTags } from './hooks/useTags'
|
||||
import { DocList } from './components/DocList/DocList'
|
||||
import { EditorCore, type EditorChange } from './components/Editor/EditorCore'
|
||||
@@ -31,7 +32,7 @@ export default function App() {
|
||||
const night = useNightMode()
|
||||
// Who's writing, and whether the server still recognises them. `signedOut`
|
||||
// flips the moment any call comes back 401.
|
||||
const { me, signedOut } = useSession()
|
||||
const { me, signedOut, setDirection } = useSession()
|
||||
const t = usePack()
|
||||
// A real account to sign out of, as opposed to the hardcoded local user a
|
||||
// build without auth configured runs as.
|
||||
@@ -80,6 +81,13 @@ export default function App() {
|
||||
}, [])
|
||||
|
||||
const { status, schedule, saveNow } = useAutoSave(currentDoc?.id ?? null)
|
||||
// The Chinese word list, for a writer going the other way through the zh pair.
|
||||
// Gated on the account's own setting rather than on anything in the text: a
|
||||
// Mandarin native drafting English quotes Chinese constantly, and none of that
|
||||
// is what segmentation is for. Declared above the checkpoint because the
|
||||
// offline 错别字 pass reads it.
|
||||
const segmenter = useSegmenter(me?.direction === 'learning_pair')
|
||||
|
||||
const {
|
||||
suggestions,
|
||||
checking,
|
||||
@@ -91,7 +99,7 @@ export default function App() {
|
||||
runCollocation,
|
||||
removeSuggestion,
|
||||
resolveServerId,
|
||||
} = useCheckpoint(currentDoc?.id ?? null)
|
||||
} = useCheckpoint(currentDoc?.id ?? null, segmenter)
|
||||
// Browser-side spell checker — loads the en-US dictionary once per session.
|
||||
const { checker: spellChecker, addWord } = useSpellChecker()
|
||||
// The tag roster (with counts). Assignments live on the doc summaries below.
|
||||
@@ -528,6 +536,8 @@ export default function App() {
|
||||
onToggleTag={handleToggleTag}
|
||||
onCreateTag={handleCreateTag}
|
||||
account={account}
|
||||
direction={me?.direction}
|
||||
onDirection={setDirection}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -591,6 +601,7 @@ export default function App() {
|
||||
docId={currentDoc.id}
|
||||
initialContent={currentDoc.content}
|
||||
onChange={handleEditorChange}
|
||||
segmenter={segmenter}
|
||||
suggestions={suggestions}
|
||||
onAccept={handleAccept}
|
||||
onAcceptMany={handleAcceptMany}
|
||||
|
||||
@@ -210,6 +210,21 @@ export interface PersonalWords {
|
||||
words: string[]
|
||||
}
|
||||
|
||||
// One pronunciation of a Chinese word, and what it means in that pronunciation.
|
||||
// A list, because 得 is dé "to obtain" and also the particle in 说得很好.
|
||||
export interface HanziReading {
|
||||
pinyin: string
|
||||
senses: string
|
||||
}
|
||||
|
||||
// A Chinese word lookup. `readings` is empty for a word with no headword, in
|
||||
// which case `chars` may carry the character-by-character reading.
|
||||
export interface HanziInfo {
|
||||
word: string
|
||||
readings: HanziReading[]
|
||||
chars: { char: string; pinyin: string; senses: string }[]
|
||||
}
|
||||
|
||||
// Who's writing. Mirrors the backend db.User.
|
||||
export interface Me {
|
||||
id: string
|
||||
@@ -217,6 +232,11 @@ export interface Me {
|
||||
display_name: string
|
||||
created_at: string
|
||||
pair_lang: string
|
||||
// Which half of the pair is being learned: 'learning_en' (the writer is
|
||||
// native in pair_lang and practising English) or 'learning_pair' (the other
|
||||
// way round). Mirrors users.direction; the server refuses 'learning_pair' for
|
||||
// a pair it has no word list for.
|
||||
direction: string
|
||||
}
|
||||
|
||||
// Thrown when the server says the session is gone. Callers can tell it apart
|
||||
@@ -269,6 +289,15 @@ export const api = {
|
||||
setPairLang: (lang: string) =>
|
||||
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ pair_lang: lang }) }),
|
||||
|
||||
// Turn the pair around. Same endpoint, same contract, and deliberately a
|
||||
// separate call: the two fields are validated together server-side, so a
|
||||
// client that wants to change both says both in one request rather than
|
||||
// sending two that each pass on their own.
|
||||
setDirection: (direction: string) =>
|
||||
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ direction }) }),
|
||||
setPair: (lang: string, direction: string) =>
|
||||
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ pair_lang: lang, direction }) }),
|
||||
|
||||
listDocs: () => req<DocSummary[]>('/docs'),
|
||||
createDoc: () => req<Document>('/docs', { method: 'POST' }),
|
||||
getDoc: (id: string) => req<Document>(`/docs/${id}`),
|
||||
@@ -348,6 +377,12 @@ export const api = {
|
||||
// Lightweight Chinese-only gloss for the inline hover/select tooltip — instant
|
||||
// and offline, so it fires on hover without spinning up the heavier lookup.
|
||||
glossWord: (word: string) => req<Gloss>(`/gloss/${encodeURIComponent(word)}`),
|
||||
// The same lookup pointing the other way: a Chinese word to its pinyin and
|
||||
// English senses, for an account learning the pair language rather than
|
||||
// English. A word the dictionary has no headword for comes back with empty
|
||||
// readings and — when its characters are known — a per-character reading
|
||||
// instead, which is a real second answer for a compound.
|
||||
hanziWord: (word: string) => req<HanziInfo>(`/hanzi/${encodeURIComponent(word)}`),
|
||||
// Tone-rewrite: rewrites a selected passage in the given style ('natural',
|
||||
// 'academic', …) and returns the rewritten text for an in-editor preview. Not
|
||||
// persisted — the editor applies it directly on accept.
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { gunzipSync } from 'node:zlib'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { CONFUSION_PAIRS, hanziFindings } from './hanzi'
|
||||
import { buildSegmenter } from '../../lib/segment'
|
||||
|
||||
// The 错别字 pack, held to the bar Phase 22 set for the English rule pack: every
|
||||
// rule pinned in *two* directions — the mistake it must catch, and the correct
|
||||
// writing next to it that it must leave alone.
|
||||
//
|
||||
// Here the second direction is the one that matters, and it is unusually easy to
|
||||
// get wrong. Chinese has no spaces, so every one of these rules is a substring
|
||||
// match on running text, and for most of them there exists an ordinary correct
|
||||
// sentence that contains the substring across a word boundary. Those sentences
|
||||
// are the real test.
|
||||
|
||||
const raw = gunzipSync(readFileSync(new URL('../../../public/dictionaries/zh/words.txt.gz', import.meta.url)))
|
||||
const seg = buildSegmenter(raw.toString('utf8'))
|
||||
|
||||
const flagged = (text: string) => hanziFindings(text, seg).map((f) => `${f.original}→${f.replacement}`)
|
||||
|
||||
describe('the gate that admits a rule', () => {
|
||||
// The pack's own claim about itself, checked against the shipped dictionary
|
||||
// rather than asserted in a comment. A pair whose wrong form is a real word
|
||||
// cannot be decided mechanically and does not belong here.
|
||||
it('every wrong form is not a word, and every right form is', () => {
|
||||
for (const { wrong, right } of CONFUSION_PAIRS) {
|
||||
expect(seg.has(wrong), `${wrong} is a dictionary word and must not be flagged`).toBe(false)
|
||||
expect(seg.has(right), `${right} is not a dictionary word`).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
// The errors this pack deliberately refuses, and why — each is a genuine
|
||||
// mistake by a modern standard whose wrong form is itself a headword. If a
|
||||
// dictionary rebuild ever drops one of these, this test fails and the pair
|
||||
// becomes admissible; that is the intended way to find out.
|
||||
it('refuses the well-known errors it cannot decide', () => {
|
||||
for (const undecidable of ['自已', '好象', '倒底', '帐号', '部份']) {
|
||||
expect(seg.has(undecidable), `${undecidable} is no longer a word — reconsider the rule`).toBe(true)
|
||||
expect(flagged(`这是${undecidable}的例子`)).toEqual([])
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('the mistakes it catches', () => {
|
||||
it('已 / 己 / 以', () => {
|
||||
expect(flagged('我己经写完了作业')).toEqual(['己经→已经'])
|
||||
expect(flagged('我以经吃过饭了')).toEqual(['以经→已经'])
|
||||
expect(flagged('下课已后我们去公园')).toEqual(['已后→以后'])
|
||||
})
|
||||
|
||||
it('在 / 再', () => {
|
||||
expect(flagged('明天在见')).toEqual(['在见→再见'])
|
||||
expect(flagged('他正再看书')).toEqual(['正再→正在'])
|
||||
expect(flagged('现再几点了')).toEqual(['现再→现在'])
|
||||
})
|
||||
|
||||
it('做 / 作', () => {
|
||||
expect(flagged('我的工做很忙')).toEqual(['工做→工作'])
|
||||
expect(flagged('老师给我们很多做业')).toEqual(['做业→作业'])
|
||||
expect(flagged('这本书的做者是谁')).toEqual(['做者→作者'])
|
||||
})
|
||||
|
||||
it('the rest', () => {
|
||||
expect(flagged('我觉的这个很好')).toEqual(['觉的→觉得'])
|
||||
expect(flagged('你因该早点睡')).toEqual(['因该→应该'])
|
||||
expect(flagged('即然你来了就坐下吧')).toEqual(['即然→既然'])
|
||||
expect(flagged('你知到吗')).toEqual(['知到→知道'])
|
||||
expect(flagged('请输入你的蜜码')).toEqual(['蜜码→密码'])
|
||||
})
|
||||
|
||||
it('reports an exact span, so the card replaces the right characters', () => {
|
||||
const text = '我己经到了'
|
||||
const [f] = hanziFindings(text, seg)
|
||||
expect(text.slice(f.from, f.to)).toBe('己经')
|
||||
expect(text.slice(0, f.from) + f.replacement + text.slice(f.to)).toBe('我已经到了')
|
||||
})
|
||||
|
||||
it('finds every occurrence, in document order', () => {
|
||||
expect(flagged('我己经吃了，他也己经吃了')).toEqual(['己经→已经', '己经→已经'])
|
||||
expect(flagged('我的工做很忙，所以我觉的很累')).toEqual(['工做→工作', '觉的→觉得'])
|
||||
})
|
||||
})
|
||||
|
||||
// ── the direction that matters ──────────────────────────────────────────────
|
||||
|
||||
describe('the correct writing it must not touch', () => {
|
||||
// Each of these is an ordinary sentence containing a flagged substring across
|
||||
// a word boundary. Without the boundary gate, every one would be corrupted —
|
||||
// and corrupted silently, into text that is still made of real characters.
|
||||
it('leaves two real words alone where they happen to abut', () => {
|
||||
// 自己 + 经常. The substring is 己经.
|
||||
expect(flagged('他自己经常做饭')).toEqual([])
|
||||
// 睡觉 + 的. The substring is 觉的.
|
||||
expect(flagged('睡觉的时候不要看手机')).toEqual([])
|
||||
// 感觉 + 的.
|
||||
expect(flagged('这是我感觉的方向')).toEqual([])
|
||||
// 不知 + 到底.
|
||||
expect(flagged('我不知到底该怎么办')).toEqual([])
|
||||
// 因 + 位置.
|
||||
expect(flagged('因位置不好我们换了座位')).toEqual([])
|
||||
// 已 + 后悔.
|
||||
expect(flagged('他已后悔了')).toEqual([])
|
||||
})
|
||||
|
||||
it('leaves ordinary correct prose entirely alone', () => {
|
||||
for (const good of [
|
||||
'我今天早上去公园跑步了',
|
||||
'他的中文说得很好',
|
||||
'我已经完成了我的作业',
|
||||
'现在几点了，我们再见面吧',
|
||||
'我觉得这个工作很有意思',
|
||||
'既然你已经知道了，就按照计划做',
|
||||
]) {
|
||||
expect(flagged(good), good).toEqual([])
|
||||
}
|
||||
})
|
||||
|
||||
// Where the gate costs the pack a real catch, and the trade it is making.
|
||||
// 不知 is itself a word, so 我不知到他在哪里 — which really is 知到 for 知道 —
|
||||
// reads to the segmenter as 不知 + 到 and is left alone. That is the gate
|
||||
// preferring a missed error to a corrupted sentence, which is the whole
|
||||
// premise: 我不知到底该怎么办 is the same three characters and is correct.
|
||||
it('declines a real error rather than risk the sentence beside it', () => {
|
||||
expect(flagged('我不知到他在哪里')).toEqual([])
|
||||
expect(flagged('你知到吗')).toEqual(['知到→知道'])
|
||||
})
|
||||
|
||||
it('says nothing about English, or about nothing', () => {
|
||||
expect(flagged('I already finished my homework')).toEqual([])
|
||||
expect(flagged('')).toEqual([])
|
||||
})
|
||||
|
||||
// The direction gate. The word list is loaded only for an account learning
|
||||
// Chinese, so without one this pack is silent — a writer practising English
|
||||
// must never be told her own quoted Chinese is wrong.
|
||||
it('is silent without a segmenter, which is how the direction gate works', () => {
|
||||
expect(hanziFindings('我己经写完了', null)).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,149 @@
|
||||
import type { MechanicsFinding } from '../../api/client'
|
||||
import type { Segmenter } from '../../lib/segment'
|
||||
|
||||
// 错别字 — wrong-character detection, the Chinese counterpart of the spell
|
||||
// checker, and a different problem from the one Hunspell solves.
|
||||
//
|
||||
// Chinese has no misspellings in the English sense: every character a writer can
|
||||
// type is a real character, correctly formed, and an IME will not offer one that
|
||||
// is not. What it *will* offer is the wrong one. Typing pinyin `yijing` and
|
||||
// taking the first candidate gives 已经 or 己经 depending on the moment, and both
|
||||
// are made of real characters. So the unit of error is not a malformed word but
|
||||
// a **substituted character inside a correct-looking one** — which is why this
|
||||
// is a rule pack over confusable pairs rather than a dictionary membership test.
|
||||
//
|
||||
// The discipline is Phase 22's, and the bar is the same: **precision over
|
||||
// recall**. A wrong nudge costs more trust than a missed one earns, and it costs
|
||||
// double here, because a learner has no way to know the tool is wrong. Two
|
||||
// mechanical gates enforce it, and both are checked in the tests rather than
|
||||
// asserted in prose.
|
||||
|
||||
// A confusable pair: `wrong` is never a word, `right` is what was meant.
|
||||
//
|
||||
// **Gate one — the pair must be decidable by the dictionary.** Each entry is
|
||||
// admitted only if `wrong` is absent from the 188k-word list *and* `right` is
|
||||
// present. That is what makes the correction a fact rather than a preference,
|
||||
// and it is checked against the shipped asset in hanzi.test.ts.
|
||||
//
|
||||
// It is also the gate that keeps out errors everyone knows are errors. 自已 for
|
||||
// 自己 is among the commonest slips in written Chinese, and 自已 is itself a
|
||||
// dictionary headword — so this pack does not flag it, exactly as Phase 22's
|
||||
// English pack left out `married with`. The same fate for 好象 (an older form of
|
||||
// 好像, still in the dictionary), 倒底, 帐号 and 部份: all real errors by a modern
|
||||
// standard, none of them decidable here.
|
||||
interface Confusion {
|
||||
wrong: string
|
||||
right: string
|
||||
// The note on the card. English, because this pack only ever runs for a writer
|
||||
// whose English is the language they think in — see the direction gate below.
|
||||
why: string
|
||||
}
|
||||
|
||||
const CONFUSIONS: Confusion[] = [
|
||||
// 已 / 己 / 以 — three characters that differ by one stroke and share a
|
||||
// syllable. The most productive source of 错别字 there is.
|
||||
{ wrong: '己经', right: '已经', why: '已经 (already) — 己 is the "self" character; the one you want is 已.' },
|
||||
{ wrong: '以经', right: '已经', why: '已经 (already) — 以 is a different word; 已 is the one that means "already".' },
|
||||
{ wrong: '已后', right: '以后', why: '以后 (afterwards) takes 以, not 已.' },
|
||||
|
||||
// 在 / 再 — same pinyin (zài), completely different jobs: one is location and
|
||||
// ongoing action, the other is repetition.
|
||||
{ wrong: '在见', right: '再见', why: '再见 (goodbye) — 再 is "again", which is what "see you again" needs.' },
|
||||
{ wrong: '正再', right: '正在', why: '正在 (in the middle of doing) takes 在, the one about being somewhere.' },
|
||||
{ wrong: '现再', right: '现在', why: '现在 (now) takes 在.' },
|
||||
|
||||
// 做 / 作 — both zuò, both "to do", and which one a compound takes is simply
|
||||
// fixed by convention. A learner cannot reason it out, which is what makes a
|
||||
// reminder worth having.
|
||||
{ wrong: '工做', right: '工作', why: '工作 (work) is written with 作.' },
|
||||
{ wrong: '做业', right: '作业', why: '作业 (homework) is written with 作.' },
|
||||
{ wrong: '做者', right: '作者', why: '作者 (author) is written with 作.' },
|
||||
{ wrong: '做文', right: '作文', why: '作文 (an essay) is written with 作.' },
|
||||
{ wrong: '做用', right: '作用', why: '作用 (effect, function) is written with 作.' },
|
||||
|
||||
// 得 / 的 — the pair everyone knows about. Only the fixed compound is flagged:
|
||||
// deciding 的 against 地 against 得 in the general case needs to know whether
|
||||
// the next word is a verb or a noun, which nothing here can tell.
|
||||
{ wrong: '觉的', right: '觉得', why: '觉得 (to feel, to think) ends in 得.' },
|
||||
|
||||
// 即 / 既 — one stroke apart, opposite meanings ("namely" against "since").
|
||||
{ wrong: '即然', right: '既然', why: '既然 (since, given that) takes 既.' },
|
||||
{ wrong: '既使', right: '即使', why: '即使 (even if) takes 即.' },
|
||||
|
||||
// The rest: ordinary IME slips where the wrong character is a homophone.
|
||||
{ wrong: '因该', right: '应该', why: '应该 (should) — 因 means "because"; the word you want starts with 应.' },
|
||||
{ wrong: '因位', right: '因为', why: '因为 (because) ends in 为.' },
|
||||
{ wrong: '知到', right: '知道', why: '知道 (to know) ends in 道.' },
|
||||
{ wrong: '安照', right: '按照', why: '按照 (according to) takes 按.' },
|
||||
{ wrong: '蜜码', right: '密码', why: '密码 (password) takes 密 — 蜜 is honey.' },
|
||||
{ wrong: '犹其', right: '尤其', why: '尤其 (especially) takes 尤.' },
|
||||
{ wrong: '甘净', right: '干净', why: '干净 (clean) takes 干.' },
|
||||
{ wrong: '什末', right: '什么', why: '什么 (what) ends in 么.' },
|
||||
{ wrong: '一像', right: '一样', why: '一样 (the same) ends in 样 — 像 is "to resemble".' },
|
||||
{ wrong: '必须品', right: '必需品', why: '必需品 (a necessity) takes 需. 必须 is "must", which is a different word.' },
|
||||
]
|
||||
|
||||
// **Gate two — the characters must not already belong to two different words.**
|
||||
//
|
||||
// This is the gate that stops the pack from destroying correct writing, and
|
||||
// without it every rule above is dangerous. 自己经常 ("oneself, often") contains
|
||||
// the string 己经. 睡觉的时候 ("when sleeping") contains 觉的. 不知到底 contains 知到.
|
||||
// A substring match would corrupt all three.
|
||||
//
|
||||
// The segmenter already knows the difference, so the test is: split the text,
|
||||
// and if the two characters land in different tokens *and* either token is a
|
||||
// real multi-character word, this is a word boundary and not an error. Two
|
||||
// adjacent single-character tokens is what the walk produces when it has nothing
|
||||
// better to offer — which is exactly what a mistyped compound looks like.
|
||||
function isWordBoundary(tokens: { word: string; from: number; to: number }[], at: number): boolean {
|
||||
const left = tokens.find((t) => at >= t.from && at < t.to)
|
||||
const right = tokens.find((t) => at + 1 >= t.from && at + 1 < t.to)
|
||||
if (!left || !right || left === right) return false
|
||||
return left.word.length > 1 || right.word.length > 1
|
||||
}
|
||||
|
||||
// hanziFindings returns the 错别字 in a piece of text, as ordinary mechanics
|
||||
// findings — the same shape, the same rail, the same cards, the same accept.
|
||||
//
|
||||
// It needs the segmenter and does nothing without one, which is also the
|
||||
// direction gate: the word list is loaded only for an account learning Chinese
|
||||
// (useSegmenter), so a writer practising English can never be told her quoted
|
||||
// Chinese is wrong. That is not a nicety. Petal deliberately never corrects the
|
||||
// pair language — the fr and es dictionaries are chosen to hold every variety
|
||||
// precisely so they cannot underline correct writing — and a Mandarin native
|
||||
// does not need her own language checked by a rule pack of two dozen entries.
|
||||
export function hanziFindings(text: string, segmenter: Segmenter | null): MechanicsFinding[] {
|
||||
if (!segmenter || !text) return []
|
||||
// One segmentation for the whole text, shared by every rule. The walk is
|
||||
// linear, but running it two dozen times over a long document would not be.
|
||||
const tokens = segmenter.segment(text)
|
||||
const found: MechanicsFinding[] = []
|
||||
|
||||
for (const c of CONFUSIONS) {
|
||||
let from = text.indexOf(c.wrong)
|
||||
while (from !== -1) {
|
||||
// The boundary test is asked at the seam the substitution sits on: the
|
||||
// gap between the first two characters, which is where a mistyped
|
||||
// compound and two adjacent words look different from each other.
|
||||
if (!isWordBoundary(tokens, from)) {
|
||||
found.push({
|
||||
from,
|
||||
to: from + c.wrong.length,
|
||||
original: c.wrong,
|
||||
replacement: c.right,
|
||||
explanation: c.why,
|
||||
type: 'mechanics',
|
||||
})
|
||||
}
|
||||
from = text.indexOf(c.wrong, from + 1)
|
||||
}
|
||||
}
|
||||
|
||||
// Document order, so the rail reads down the page rather than down this file.
|
||||
return found.sort((a, b) => a.from - b.from)
|
||||
}
|
||||
|
||||
// Exported for the tests, which check every pair against the shipped word list.
|
||||
// A pack whose own gate is only described in a comment is a pack whose gate can
|
||||
// rot; this is how the description is made to stay true.
|
||||
export const CONFUSION_PAIRS = CONFUSIONS.map((c) => ({ wrong: c.wrong, right: c.right }))
|
||||
@@ -20,6 +20,11 @@ interface Props {
|
||||
// The signed-in writer, when there is real auth to sign out of. Null in a
|
||||
// local-dev build, where there is nothing to leave.
|
||||
account: { name: string } | null
|
||||
// The account's learner direction and the way to change it, passed straight
|
||||
// through to the language picker in the footer — the sidebar is the drawer,
|
||||
// and the drawer is the only chrome always one tap away on a phone.
|
||||
direction?: string
|
||||
onDirection?: (direction: string) => Promise<void>
|
||||
}
|
||||
|
||||
// Sidebar sort orders. 'recent' keeps the server's updated_at-desc ordering.
|
||||
@@ -43,6 +48,8 @@ export function DocList({
|
||||
onToggleTag,
|
||||
onCreateTag,
|
||||
account,
|
||||
direction,
|
||||
onDirection,
|
||||
}: Props) {
|
||||
const t = usePack()
|
||||
// Active tag filter (null = show all). Cleared automatically if the tag
|
||||
@@ -161,7 +168,7 @@ export function DocList({
|
||||
{/* The pair Petal speaks. Unlike the rows above it this is not about any
|
||||
document, and unlike sign-out it is offered whether or not there is an
|
||||
account behind the session — a local-dev build still has a langpack. */}
|
||||
<LanguagePicker />
|
||||
<LanguagePicker direction={direction} onDirection={onDirection} />
|
||||
|
||||
{/* Who's writing, and the way out. Shown only when there's a real account
|
||||
behind the session — a local-dev build has nobody to sign out as. */}
|
||||
|
||||
@@ -15,15 +15,46 @@ import { setPackLang, shippedPacks, usePack } from '../../i18n'
|
||||
// read a label that says "Portuguese" in Chinese, so the buttons say 中文 and
|
||||
// Português and nothing else — the one place in Petal where bilingual copy would
|
||||
// actively get in the way.
|
||||
export function LanguagePicker() {
|
||||
interface Props {
|
||||
// The account's current direction ('learning_en' | 'learning_pair'), and the
|
||||
// way to change it. Owned by App rather than here, because turning the pair
|
||||
// around changes what the *editor* does — it is what loads the word list —
|
||||
// and this control is only where the writer says so.
|
||||
direction?: string
|
||||
onDirection?: (direction: string) => Promise<void>
|
||||
}
|
||||
|
||||
export function LanguagePicker({ direction, onDirection }: Props = {}) {
|
||||
const t = usePack()
|
||||
const packs = shippedPacks()
|
||||
const [saving, setSaving] = useState<string | null>(null)
|
||||
const [failed, setFailed] = useState(false)
|
||||
const [turning, setTurning] = useState(false)
|
||||
const [turnFailed, setTurnFailed] = useState(false)
|
||||
|
||||
// Nothing to choose between — a deployment with one pack shows no picker
|
||||
// rather than a single button that does nothing.
|
||||
if (packs.length < 2) return null
|
||||
// rather than a single button that does nothing. The direction control is
|
||||
// still worth rendering in that case, so it is checked separately below.
|
||||
const showPacks = packs.length >= 2
|
||||
// `t.learner` is the pack's own statement that this pair can be learned
|
||||
// toward, and the server keeps the matching list (auth.learnerPairs). A pack
|
||||
// without it renders nothing here, which is the same failure mode as a pair
|
||||
// without copy: absent rather than broken.
|
||||
const learner = t.learner
|
||||
if (!showPacks && !learner) return null
|
||||
|
||||
const turn = async (next: string) => {
|
||||
if (!onDirection || next === (direction ?? 'learning_en') || turning) return
|
||||
setTurning(true)
|
||||
setTurnFailed(false)
|
||||
try {
|
||||
await onDirection(next)
|
||||
} catch {
|
||||
setTurnFailed(true)
|
||||
} finally {
|
||||
setTurning(false)
|
||||
}
|
||||
}
|
||||
|
||||
const choose = async (code: string) => {
|
||||
if (code === t.code || saving) return
|
||||
@@ -47,6 +78,8 @@ export function LanguagePicker() {
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-1 px-1">
|
||||
{showPacks && (
|
||||
<>
|
||||
{/* Label and buttons wrap as a pair: the label is itself bilingual
|
||||
("Langue · Language"), and three self-naming buttons beside it need
|
||||
more than the drawer is wide in every language Petal ships. When they
|
||||
@@ -89,6 +122,52 @@ export function LanguagePicker() {
|
||||
{t.docs.languageFailed}
|
||||
</span>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{/* Which way round the pair is being learned. Below the language buttons
|
||||
because it only makes sense once the language is settled, and rendered
|
||||
at all only for a pair Petal has the learner-side data for. */}
|
||||
{learner && onDirection && (
|
||||
<div
|
||||
className="flex flex-wrap items-center gap-x-2 gap-y-1 text-xs"
|
||||
style={{ color: 'var(--color-muted)' }}
|
||||
>
|
||||
<span className="shrink-0 font-semibold">{learner.label}</span>
|
||||
<div className="ml-auto flex shrink-0 gap-1">
|
||||
{[
|
||||
{ code: 'learning_en', text: learner.toEn, en: `learning English` },
|
||||
{ code: 'learning_pair', text: learner.toPair, en: `learning ${t.nativeName}` },
|
||||
].map((opt) => {
|
||||
const active = (direction ?? 'learning_en') === opt.code
|
||||
return (
|
||||
<button
|
||||
key={opt.code}
|
||||
type="button"
|
||||
onClick={() => void turn(opt.code)}
|
||||
disabled={turning}
|
||||
aria-pressed={active}
|
||||
aria-label={`I am ${opt.en}`}
|
||||
className="petal-tap-sm px-2.5 py-1 text-xs font-bold transition-colors disabled:opacity-60"
|
||||
style={{
|
||||
borderRadius: 'var(--radius-pill)',
|
||||
background: active ? 'var(--color-accent)' : 'var(--color-surface)',
|
||||
color: active ? '#fff' : 'var(--color-plum)',
|
||||
boxShadow: active ? 'none' : 'var(--shadow-soft)',
|
||||
}}
|
||||
>
|
||||
{opt.text}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{turnFailed && learner && (
|
||||
<span className="text-[0.7rem]" style={{ color: 'var(--color-accent)' }}>
|
||||
{learner.failed}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -34,6 +34,8 @@ import { planBatch } from './acceptBatch'
|
||||
import { api, type Suggestion, type SuggestionType, type WordInfo } from '../../api/client'
|
||||
import { speak, speechSupported } from '../../audio/speech'
|
||||
import type { SpellChecker } from '../../hooks/useSpellChecker'
|
||||
import type { Segmenter } from '../../lib/segment'
|
||||
import { hanziWordAt, hanziToWordInfo, hanziPinyin } from './hanziWord'
|
||||
import { usePack } from '../../i18n'
|
||||
|
||||
// Breathing room left below the last suggestion card when the rail's stack is what
|
||||
@@ -73,6 +75,11 @@ interface Props {
|
||||
// to the personal dictionary is bubbled up so it persists app-wide.
|
||||
spellChecker: SpellChecker | null
|
||||
onAddWord: (word: string) => void
|
||||
// The Chinese word list, non-null only for a writer learning the pair
|
||||
// language (users.direction = 'learning_pair'). Its presence is what turns on
|
||||
// every Chinese-side behaviour here: hanzi stops being text the editor steps
|
||||
// over and becomes words it can point at.
|
||||
segmenter: Segmenter | null
|
||||
}
|
||||
|
||||
interface MisspellState {
|
||||
@@ -95,6 +102,12 @@ interface WordInfoState {
|
||||
left: number
|
||||
loading: boolean
|
||||
info: WordInfo | null
|
||||
// The word's own pinyin, for a Chinese lookup. Kept beside `info` rather than
|
||||
// inside it because WordInfo is the English dictionary's shape and `phonetic`
|
||||
// there means IPA — printing pinyin between the slashes that say "this is
|
||||
// IPA" would be a small lie in the one place a learner is looking for the
|
||||
// truth about pronunciation.
|
||||
pinyin: string
|
||||
// Garden state: the captured word's id (null until the auto-capture returns or
|
||||
// after it's removed) and whether it's currently in the garden.
|
||||
vocabId: string | null
|
||||
@@ -188,6 +201,10 @@ interface GlossState {
|
||||
gloss: string
|
||||
// The other reading, when the token is a word in her language too.
|
||||
reverse?: string
|
||||
// A line shown *above* the meaning rather than below it: pinyin, for a
|
||||
// Chinese word. Above because it is read first — the meaning of 公园 may
|
||||
// already be clear to someone who cannot yet say it.
|
||||
lead?: string
|
||||
from: number
|
||||
to: number
|
||||
top: number
|
||||
@@ -234,6 +251,7 @@ export function EditorCore({
|
||||
onFocusMode,
|
||||
spellChecker,
|
||||
onAddWord,
|
||||
segmenter,
|
||||
}: Props) {
|
||||
// Her pair's copy — the hover tip labels the second reading with the language's
|
||||
// own name, so it says "português" rather than "pt-PT".
|
||||
@@ -421,6 +439,26 @@ export function EditorCore({
|
||||
// popover would offer a definition of "cora".
|
||||
const wordAlphabet = spellChecker?.extendedAlphabet ?? false
|
||||
|
||||
// "The word under here", for a document that may hold two writing systems at
|
||||
// once — which every document in this pair does, because a learner's Chinese
|
||||
// practice is full of English and her English is full of quoted Chinese.
|
||||
//
|
||||
// Chinese is tried first and Latin second, and the order costs nothing to get
|
||||
// right: the two can never both answer, because a Han character is not a Latin
|
||||
// letter and neither tokenizer will cross into the other's run. `hanzi` rides
|
||||
// along because the two answers go to different dictionaries — the same
|
||||
// string is a word in exactly one of them.
|
||||
const resolveWord = useCallback(
|
||||
(pos: number): { from: number; to: number; word: string; hanzi: boolean } | null => {
|
||||
if (!editor) return null
|
||||
const han = hanziWordAt(editor.state.doc, pos, segmenter)
|
||||
if (han) return { ...han, hanzi: true }
|
||||
const latin = wordAt(editor.state.doc, pos, wordAlphabet)
|
||||
return latin ? { ...latin, hanzi: false } : null
|
||||
},
|
||||
[editor, segmenter, wordAlphabet],
|
||||
)
|
||||
|
||||
// Push the spell checker into its decoration plugin once the dictionary loads
|
||||
// (and again whenever the personal dictionary changes its identity).
|
||||
useEffect(() => {
|
||||
@@ -833,7 +871,7 @@ export function EditorCore({
|
||||
const openWordLookup = useCallback(
|
||||
(pos: number) => {
|
||||
if (!editor) return
|
||||
const range = wordAt(editor.state.doc, pos, wordAlphabet)
|
||||
const range = resolveWord(pos)
|
||||
if (!range) return
|
||||
const wrapper = wrapperRef.current
|
||||
if (!wrapper) return
|
||||
@@ -849,12 +887,18 @@ export function EditorCore({
|
||||
closeCard()
|
||||
setMisspell(null)
|
||||
const token = ++wordReqRef.current
|
||||
setWordInfo({ word: range.word, from: range.from, to: range.to, top, left, loading: true, info: null, vocabId: null, saved: false })
|
||||
setWordInfo({ word: range.word, from: range.from, to: range.to, top, left, loading: true, info: null, pinyin: '', vocabId: null, saved: false })
|
||||
// The sentence the word sits in, for review context in the garden.
|
||||
const example = exampleAt(range.from)
|
||||
api
|
||||
.lookupWord(range.word)
|
||||
.then((info) => {
|
||||
// Two dictionaries, one card. The Chinese lookup answers in English and
|
||||
// the English one answers in her language; which is wanted follows from
|
||||
// which script the word is written in, so nothing here has to consult the
|
||||
// account's direction a second time.
|
||||
const lookup: Promise<{ info: WordInfo; pinyin: string }> = range.hanzi
|
||||
? api.hanziWord(range.word).then((h) => ({ info: hanziToWordInfo(h), pinyin: hanziPinyin(h) }))
|
||||
: api.lookupWord(range.word).then((info) => ({ info, pinyin: '' }))
|
||||
lookup
|
||||
.then(({ info, pinyin }) => {
|
||||
if (token !== wordReqRef.current) return
|
||||
// Auto-capture into the vocabulary garden — only words the dictionary
|
||||
// actually knows (a real gloss or definition), so accidental lookups of
|
||||
@@ -864,14 +908,18 @@ export function EditorCore({
|
||||
// Reflect the saved state optimistically so the heart shows 💚 the
|
||||
// moment a known word loads, rather than flashing 🤍 until the capture
|
||||
// round-trips. vocabId is filled in when recordVocab returns.
|
||||
setWordInfo((w) => (w ? { ...w, loading: false, info, saved: known } : null))
|
||||
setWordInfo((w) => (w ? { ...w, loading: false, info, pinyin, saved: known } : null))
|
||||
if (!known) return
|
||||
api
|
||||
.recordVocab({
|
||||
word: range.word,
|
||||
gloss: info.gloss,
|
||||
definition: info.definitions[0]?.definition ?? '',
|
||||
phonetic: info.phonetic,
|
||||
// The garden's pronunciation field holds whichever this word has:
|
||||
// IPA for an English word, pinyin for a Chinese one. Both answer
|
||||
// the same question on a review card — how do I say this — and a
|
||||
// second column would only be a second thing to keep in sync.
|
||||
phonetic: pinyin || info.phonetic,
|
||||
example,
|
||||
doc_id: docId,
|
||||
})
|
||||
@@ -891,7 +939,7 @@ export function EditorCore({
|
||||
}
|
||||
})
|
||||
},
|
||||
[editor, closeCard, docId],
|
||||
[editor, closeCard, docId, resolveWord, exampleAt],
|
||||
)
|
||||
|
||||
// Toggle a looked-up word in/out of the vocabulary garden from the WordCard
|
||||
@@ -998,7 +1046,7 @@ export function EditorCore({
|
||||
clear()
|
||||
return
|
||||
}
|
||||
const range = wordAt(editor.state.doc, coords.pos, wordAlphabet)
|
||||
const range = resolveWord(coords.pos)
|
||||
if (!range) {
|
||||
clear()
|
||||
return
|
||||
@@ -1007,9 +1055,21 @@ export function EditorCore({
|
||||
if (gloss && gloss.from === range.from && gloss.to === range.to) return
|
||||
clearTimeout(glossTimer.current)
|
||||
const token = ++glossReqRef.current
|
||||
// The Chinese hover carries a second line the English one has no use for:
|
||||
// pinyin above the meaning. It is the thing a learner most often stops to
|
||||
// ask about their own writing — reading a character back is not the same
|
||||
// as being able to say it — and it is why this tooltip is worth having at
|
||||
// all for a script the writer can already read the meaning of half the
|
||||
// time.
|
||||
const ask = (): Promise<{ gloss: string; reverse?: string; lead?: string }> =>
|
||||
range.hanzi
|
||||
? api.hanziWord(range.word).then((h) => ({
|
||||
gloss: h.readings[0]?.senses ?? h.chars.map((c) => `${c.char} ${c.senses}`).join(' · '),
|
||||
lead: hanziPinyin(h),
|
||||
}))
|
||||
: api.glossWord(range.word).then((g) => ({ gloss: g.gloss, reverse: g.reverse }))
|
||||
glossTimer.current = setTimeout(() => {
|
||||
api
|
||||
.glossWord(range.word)
|
||||
ask()
|
||||
.then((g) => {
|
||||
if (token !== glossReqRef.current) return
|
||||
const wrapper = wrapperRef.current
|
||||
@@ -1024,14 +1084,14 @@ export function EditorCore({
|
||||
const wrapRect = wrapper.getBoundingClientRect()
|
||||
const left = Math.max(0, Math.min(start.left - wrapRect.left, wrapper.clientWidth - 280))
|
||||
const top = end.bottom - wrapRect.top + 6
|
||||
setGloss({ word: range.word, gloss: g.gloss, reverse: g.reverse, from: range.from, to: range.to, top, left })
|
||||
setGloss({ word: range.word, gloss: g.gloss, reverse: g.reverse, lead: g.lead, from: range.from, to: range.to, top, left })
|
||||
})
|
||||
.catch(() => {
|
||||
if (token === glossReqRef.current) setGloss(null)
|
||||
})
|
||||
}, 350)
|
||||
},
|
||||
[editor, wordAlphabet, selection, rewrite, misspell, wordInfo, pinned, gloss],
|
||||
[editor, resolveWord, selection, rewrite, misspell, wordInfo, pinned, gloss],
|
||||
)
|
||||
|
||||
// Leaving the editor surface drops any pending/shown gloss.
|
||||
@@ -1255,6 +1315,7 @@ export function EditorCore({
|
||||
{gloss && (
|
||||
<GlossTip
|
||||
gloss={gloss.gloss}
|
||||
lead={gloss.lead}
|
||||
reverse={gloss.reverse}
|
||||
reverseLang={pack.nativeName}
|
||||
style={{ top: gloss.top, left: gloss.left }}
|
||||
@@ -1287,6 +1348,7 @@ export function EditorCore({
|
||||
loading={wordInfo.loading}
|
||||
saved={wordInfo.saved}
|
||||
onToggleSave={toggleSaveWord}
|
||||
pinyin={wordInfo.pinyin}
|
||||
style={{ top: wordInfo.top, left: wordInfo.left }}
|
||||
onReplace={replaceWord}
|
||||
/>
|
||||
|
||||
@@ -7,6 +7,11 @@
|
||||
|
||||
interface Props {
|
||||
gloss: string
|
||||
// A line above the gloss, in a lighter weight: the pinyin of a Chinese word.
|
||||
// It leads because it is what is actually being asked — a learner reading
|
||||
// their own 公园 back may know it means a park and still not know how to say
|
||||
// it, which is the one thing the character does not tell them.
|
||||
lead?: string
|
||||
// The English meaning of the same token read as a word of the writer's own
|
||||
// language, when it is one. On a Latin-script pair "sale" is both, and the
|
||||
// bubble shows the two readings stacked rather than picking one — the same
|
||||
@@ -17,7 +22,7 @@ interface Props {
|
||||
style: React.CSSProperties
|
||||
}
|
||||
|
||||
export function GlossTip({ gloss, reverse, reverseLang, style }: Props) {
|
||||
export function GlossTip({ gloss, lead, reverse, reverseLang, style }: Props) {
|
||||
return (
|
||||
<div
|
||||
className="petal-gloss-tip pointer-events-none absolute z-20 px-2.5 py-1.5 text-sm"
|
||||
@@ -33,6 +38,11 @@ export function GlossTip({ gloss, reverse, reverseLang, style }: Props) {
|
||||
...style,
|
||||
}}
|
||||
>
|
||||
{lead && (
|
||||
<span className="mb-0.5 block font-semibold" style={{ opacity: 0.9 }}>
|
||||
{lead}
|
||||
</span>
|
||||
)}
|
||||
{gloss}
|
||||
{reverse && (
|
||||
<span className="mt-0.5 block" style={{ opacity: 0.72, fontSize: '0.85em' }}>
|
||||
|
||||
@@ -17,11 +17,16 @@ interface Props {
|
||||
// heart toggles it; `onToggleSave` removes/re-adds it.
|
||||
saved: boolean
|
||||
onToggleSave: () => void
|
||||
// A Chinese word's pinyin. Shown in place of the IPA line and *without* the
|
||||
// slashes, because pinyin is not a phonetic transcription — it is how the word
|
||||
// is spelled in letters, and the slashes would say something untrue about it
|
||||
// in the one place a learner is looking for the truth about pronunciation.
|
||||
pinyin?: string
|
||||
style: React.CSSProperties
|
||||
onReplace: (synonym: string) => void
|
||||
}
|
||||
|
||||
export function WordCard({ word, info, loading, saved, onToggleSave, style, onReplace }: Props) {
|
||||
export function WordCard({ word, info, loading, saved, onToggleSave, pinyin, style, onReplace }: Props) {
|
||||
const t = usePack()
|
||||
const definitions = info?.definitions ?? []
|
||||
const synonyms = info?.synonyms ?? []
|
||||
@@ -117,9 +122,10 @@ export function WordCard({ word, info, loading, saved, onToggleSave, style, onRe
|
||||
when she has found a word she likes — "can I use this?". Both are
|
||||
quiet, muted lines: information she can take or leave, never a verdict
|
||||
on her writing. */}
|
||||
{(phonetic || band) && (
|
||||
{(phonetic || pinyin || band) && (
|
||||
<div className="mt-1.5 flex items-center gap-2 text-sm">
|
||||
{phonetic && <span style={{ color: 'var(--color-muted)' }}>/{phonetic}/</span>}
|
||||
{pinyin && <span style={{ color: 'var(--color-muted)' }}>{pinyin}</span>}
|
||||
{!pinyin && phonetic && <span style={{ color: 'var(--color-muted)' }}>/{phonetic}/</span>}
|
||||
{band && (
|
||||
<span
|
||||
className="rounded-full px-2 py-0.5 text-xs font-semibold"
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import type { Node as PMNode } from '@tiptap/pm/model'
|
||||
|
||||
import { mapOffset } from './SuggestionHighlight'
|
||||
import type { Segmenter } from '../../lib/segment'
|
||||
import type { HanziInfo, WordInfo } from '../../api/client'
|
||||
|
||||
// The Chinese counterpart of `wordAt` (SpellCheck.ts): given a position in the
|
||||
// document, which Chinese word is there.
|
||||
//
|
||||
// It lives in its own file rather than as a branch inside `wordAt` because the
|
||||
// two answer the same question by genuinely different means — one runs a regex
|
||||
// over the text, the other runs a shortest-path walk over a 188k-word list it
|
||||
// had to fetch — and only one of them is about spelling at all. What they share
|
||||
// is the part that matters for correctness: the offset→position mapping, which
|
||||
// is `mapOffset`, the same function the suggestion, spell and search decoration
|
||||
// layers all anchor through.
|
||||
|
||||
export interface HanziRange {
|
||||
from: number
|
||||
to: number
|
||||
word: string
|
||||
}
|
||||
|
||||
// blockAt finds the textblock containing pos, along with where that block starts
|
||||
// — everything else here is arithmetic within one block.
|
||||
//
|
||||
// Segmentation is per-block for the same reason the spell tokenizer is: a word
|
||||
// cannot span a paragraph break, and a block is the largest unit whose text is
|
||||
// contiguous in the document.
|
||||
function blockAt(doc: PMNode, pos: number): { node: PMNode; start: number } | null {
|
||||
let found: { node: PMNode; start: number } | null = null
|
||||
doc.descendants((node, nodePos) => {
|
||||
if (found) return false
|
||||
if (!node.isTextblock) return true
|
||||
if (pos <= nodePos || pos >= nodePos + node.nodeSize) return false
|
||||
found = { node, start: nodePos }
|
||||
return false
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
// offsetOf is the inverse of mapOffset: an absolute ProseMirror position to a
|
||||
// character offset within the block's flattened text. Inline atoms (a hard
|
||||
// break) occupy a position and contribute no text, so the two are not the same
|
||||
// number and subtracting the block position would be wrong in any paragraph
|
||||
// containing one.
|
||||
function offsetOf(block: PMNode, blockStart: number, pos: number): number {
|
||||
let textOffset = 0
|
||||
let pmPos = blockStart + 1
|
||||
let result = -1
|
||||
block.forEach((child) => {
|
||||
if (result >= 0) return
|
||||
const len = child.isText ? (child.text?.length ?? 0) : 0
|
||||
if (pos <= pmPos + child.nodeSize) {
|
||||
result = textOffset + Math.max(0, Math.min(pos - pmPos, len))
|
||||
return
|
||||
}
|
||||
textOffset += len
|
||||
pmPos += child.nodeSize
|
||||
})
|
||||
return result >= 0 ? result : textOffset
|
||||
}
|
||||
|
||||
// hanziWordAt resolves the Chinese word at a document position, or null when
|
||||
// there is no Chinese there — which is the ordinary case in a mixed paragraph
|
||||
// and is why the caller falls through to the Latin tokenizer.
|
||||
export function hanziWordAt(doc: PMNode, pos: number, segmenter: Segmenter | null): HanziRange | null {
|
||||
if (!segmenter) return null
|
||||
const block = blockAt(doc, pos)
|
||||
if (!block) return null
|
||||
const text = block.node.textContent
|
||||
if (!text) return null
|
||||
|
||||
const token = segmenter.wordAt(text, offsetOf(block.node, block.start, pos))
|
||||
if (!token) return null
|
||||
return {
|
||||
from: mapOffset(block.node, block.start, token.from),
|
||||
to: mapOffset(block.node, block.start, token.to),
|
||||
word: token.word,
|
||||
}
|
||||
}
|
||||
|
||||
// hanziToWordInfo adapts a Chinese lookup into the shape the word card already
|
||||
// renders.
|
||||
//
|
||||
// An adapter rather than a second card, because everything around the card is
|
||||
// the same in both directions: it opens the same way, anchors the same way,
|
||||
// captures into the same vocabulary garden, and reads aloud through the same
|
||||
// voice — the zh pair already speaks Chinese, so 🔊 needs nothing new to say
|
||||
// 公园 out loud. What differs is only which fields carry what.
|
||||
//
|
||||
// * `definitions` holds one entry per reading, labelled with its pinyin. The
|
||||
// part-of-speech slot is where the card puts a short italic prefix, which
|
||||
// is exactly the shape a reading label wants — and a reading *is* the thing
|
||||
// that distinguishes these senses from each other (得 dé "to obtain" from 得
|
||||
// de, the complement marker).
|
||||
// * `gloss` stays empty. It means "translated into the writer's language",
|
||||
// and for a writer learning Chinese that language is English, which is what
|
||||
// the senses already are. Putting the English there too would print it
|
||||
// twice.
|
||||
// * The per-character fallback fills the same list, labelled by character, so
|
||||
// a compound with no headword still says something true about itself.
|
||||
export function hanziToWordInfo(info: HanziInfo): WordInfo {
|
||||
const definitions =
|
||||
info.readings.length > 0
|
||||
? info.readings.map((r) => ({ part_of_speech: r.pinyin, definition: r.senses }))
|
||||
: info.chars.map((c) => ({ part_of_speech: `${c.char} ${c.pinyin}`, definition: c.senses }))
|
||||
return {
|
||||
word: info.word,
|
||||
gloss: '',
|
||||
phonetic: '',
|
||||
definitions,
|
||||
synonyms: [],
|
||||
frequency: 0,
|
||||
difficulty: -1,
|
||||
etymology: '',
|
||||
}
|
||||
}
|
||||
|
||||
// hanziPinyin is the word's own pronunciation, for the line under the headword.
|
||||
// Empty when only the character fallback answered: the characters' readings are
|
||||
// not the word's reading — 不 is bù alone and bú before a fourth tone — and
|
||||
// printing them joined up would be inventing a pronunciation.
|
||||
export function hanziPinyin(info: HanziInfo): string {
|
||||
return info.readings[0]?.pinyin ?? ''
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,39 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
|
||||
import { loadSegmenter, type Segmenter } from '../lib/segment'
|
||||
|
||||
// Loads the Chinese word list, once per session, and only for a writer who is
|
||||
// going to use it.
|
||||
//
|
||||
// Modelled on useSpellChecker, and gated harder. That hook loads for everyone,
|
||||
// because everyone's English gets spell-checked; this one loads a megabyte for
|
||||
// the one direction that needs it, and an account practising English would
|
||||
// never ask a single question of it. The gate is the writer's own setting rather
|
||||
// than a guess from their text: a Mandarin native drafting English quotes
|
||||
// Chinese in it constantly, and none of that is what this is for.
|
||||
//
|
||||
// A failure resolves to null, which every consumer already handles as "no
|
||||
// segmentation" — the Chinese hover quietly does nothing rather than the editor
|
||||
// refusing to open.
|
||||
export function useSegmenter(enabled: boolean): Segmenter | null {
|
||||
const [segmenter, setSegmenter] = useState<Segmenter | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled) {
|
||||
// Turning the direction back drops it. It is a megabyte of resident map
|
||||
// whose only consumer just switched off, and re-loading costs one fetch
|
||||
// that the browser cache answers.
|
||||
setSegmenter(null)
|
||||
return
|
||||
}
|
||||
let cancelled = false
|
||||
loadSegmenter().then((seg) => {
|
||||
if (!cancelled) setSegmenter(seg)
|
||||
})
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [enabled])
|
||||
|
||||
return segmenter
|
||||
}
|
||||
@@ -42,5 +42,14 @@ export function useSession() {
|
||||
}
|
||||
}, [])
|
||||
|
||||
return { me, signedOut }
|
||||
// Turn the pair around. The account is the source of truth for which
|
||||
// direction the editor is in — it decides whether the word list loads at all —
|
||||
// so the state moves only once the server has agreed, and it moves to what the
|
||||
// server *stored* rather than to what was asked for.
|
||||
const setDirection = async (direction: string) => {
|
||||
const updated = await api.setDirection(direction)
|
||||
setMe(updated)
|
||||
}
|
||||
|
||||
return { me, signedOut, setDirection }
|
||||
}
|
||||
|
||||
@@ -15,6 +15,17 @@ export const zh: Pack = {
|
||||
nativeName: '中文',
|
||||
locale: 'zh-CN',
|
||||
|
||||
// The zh pair is the only one Petal can be *learned* toward, because it is the
|
||||
// only one with a word list and a Chinese→English dictionary (Phase 26). The
|
||||
// two labels are each written for the person who would pick them: she reads
|
||||
// the first, and the English speaker learning her language reads the second.
|
||||
learner: {
|
||||
label: '我在学 · I am learning',
|
||||
toEn: '英文',
|
||||
toPair: 'Chinese 中文',
|
||||
failed: '没能换成功 · Couldn’t switch — nothing changed',
|
||||
},
|
||||
|
||||
app: {
|
||||
duplicateTitle: (title) => `${title} (副本)`,
|
||||
garden: '词汇花园',
|
||||
|
||||
@@ -39,6 +39,29 @@ export interface Pack {
|
||||
// Portuguese voice anyone reaches for is Brazilian.
|
||||
locale: string
|
||||
|
||||
// Copy for turning this pair around — a writer who is native in English and
|
||||
// learning X, rather than the other way round.
|
||||
//
|
||||
// Optional, and its presence is the pack's half of the same fact
|
||||
// auth.learnerPairs holds server-side: a pair can only be learned toward if
|
||||
// Petal has a word list to segment it with and a dictionary that reads from it
|
||||
// into English. Chinese has both; the Latin pairs have neither yet, so their
|
||||
// packs simply leave this out and the control does not render.
|
||||
//
|
||||
// Each label is written in the language of the person who would *choose* it,
|
||||
// for the same reason the pair buttons name themselves: someone on the wrong
|
||||
// side of this switch cannot read the side they are trying to reach.
|
||||
learner?: {
|
||||
// The heading over the two choices.
|
||||
label: string
|
||||
// "I am practising English" — read by the writer who is native in X.
|
||||
toEn: string
|
||||
// "I am learning X" — read by the writer who is native in English.
|
||||
toPair: string
|
||||
// Shown when the server refuses the change.
|
||||
failed: string
|
||||
}
|
||||
|
||||
app: {
|
||||
// A duplicated document's title. A function, not a suffix: where the marker
|
||||
// goes is the pack's business.
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { gunzipSync } from 'node:zlib'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { buildSegmenter, isHan, type Segmenter } from './segment'
|
||||
|
||||
// Segmentation is tested twice over, and the two halves check different things.
|
||||
//
|
||||
// The hand-built dictionaries below pin the *algorithm*: given these words with
|
||||
// these frequencies, this is the split, and the reason is visible in the four
|
||||
// lines above the assertion. They would pass with any word list.
|
||||
//
|
||||
// The block at the bottom pins the *shipped asset*: the real 188,522-word list
|
||||
// this app serves, on the sentences a rebuild would plausibly break. Those are
|
||||
// the cases where being wrong is invisible — the app still works, it just
|
||||
// underlines and glosses the wrong thing.
|
||||
|
||||
// A dictionary written the way the asset is: "word freq" per line.
|
||||
function dict(entries: Record<string, number>): Segmenter {
|
||||
return buildSegmenter(
|
||||
Object.entries(entries)
|
||||
.map(([w, f]) => `${w} ${f}`)
|
||||
.join('\n'),
|
||||
)
|
||||
}
|
||||
|
||||
const words = (seg: Segmenter, text: string) => seg.segment(text).map((t) => t.word)
|
||||
|
||||
describe('isHan', () => {
|
||||
it('accepts Han across the extension blocks, and nothing else', () => {
|
||||
expect(isHan('中')).toBe(true)
|
||||
expect(isHan('龥')).toBe(true)
|
||||
// Beyond the basic block. A character Petal fails to recognise as Chinese is
|
||||
// one the English tokenizer then tries to make sense of.
|
||||
expect(isHan('𠀀')).toBe(true)
|
||||
for (const ch of ['a', '1', ' ', '，', '。', 'あ', '한']) {
|
||||
expect(isHan(ch), ch).toBe(false)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('the walk chooses the likeliest split, not the longest match', () => {
|
||||
// The textbook case, and the reason longest-match is not good enough: 研究生
|
||||
// ("graduate student") is a real word and a longer match than 研究 at position
|
||||
// 0 — but 研究/生命 ("research" + "life") is the likelier path, and it is the
|
||||
// sentence a person would read.
|
||||
it('研究生命的起源', () => {
|
||||
const seg = dict({ 研究: 6000, 研究生: 800, 生命: 4000, 命: 900, 的: 300000, 起源: 700 })
|
||||
expect(words(seg, '研究生命的起源')).toEqual(['研究', '生命', '的', '起源'])
|
||||
})
|
||||
|
||||
it('乒乓球拍卖完了 — the ambiguity is 球拍 against 拍卖', () => {
|
||||
const seg = dict({
|
||||
乒乓球: 500, 乒乓: 400, 球拍: 200, 拍卖: 900, 卖完: 50, 完: 3000, 了: 200000, 球: 2000, 拍: 800,
|
||||
})
|
||||
expect(words(seg, '乒乓球拍卖完了')).toEqual(['乒乓球', '拍卖', '完', '了'])
|
||||
})
|
||||
|
||||
it('keeps particles as their own words', () => {
|
||||
const seg = dict({ 他: 90000, 的: 300000, 中文: 3000, 说: 20000, 得: 60000, 很: 40000, 好: 50000 })
|
||||
expect(words(seg, '他的中文说得很好')).toEqual(['他', '的', '中文', '说', '得', '很', '好'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('what the walk does with what it does not know', () => {
|
||||
// A sentence with an unfamiliar character in it must still segment. Every
|
||||
// position needs *some* path through it, which is why an unknown character
|
||||
// scores badly rather than not scoring at all.
|
||||
it('an unknown character becomes its own token and the rest survives', () => {
|
||||
const seg = dict({ 我: 90000, 喜欢: 5000, 猫: 2000 })
|
||||
expect(words(seg, '我喜欢龥猫')).toEqual(['我', '喜欢', '龥', '猫'])
|
||||
})
|
||||
|
||||
// It must never *invent* a word: an unknown span of two characters is two
|
||||
// unknown characters, not a new headword.
|
||||
it('never joins unknown characters into a word', () => {
|
||||
const seg = dict({ 我: 90000 })
|
||||
expect(words(seg, '我龥龥')).toEqual(['我', '龥', '龥'])
|
||||
})
|
||||
|
||||
// A rare real word still loses to two common ones — this is the property that
|
||||
// lets the shipped list keep 100,000 rare CC-CEDICT headwords without them
|
||||
// distorting ordinary sentences.
|
||||
it('a rare long word loses to two common short ones', () => {
|
||||
const seg = dict({ 公园: 4000, 跑步: 3000, 公园跑: 1 })
|
||||
expect(words(seg, '公园跑步')).toEqual(['公园', '跑步'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('Chinese is not the only thing in the paragraph', () => {
|
||||
const seg = dict({ 我: 90000, 在: 50000, 写: 8000, 英文: 3000 })
|
||||
|
||||
// Latin runs are skipped, not returned. The English tokenizer is still running
|
||||
// over the same text and owns them; returning them here would mean two layers
|
||||
// claiming one word.
|
||||
it('skips Latin and punctuation, keeping offsets into the original string', () => {
|
||||
const tokens = seg.segment('我在写 English 英文。')
|
||||
expect(tokens.map((t) => t.word)).toEqual(['我', '在', '写', '英文'])
|
||||
for (const t of tokens) {
|
||||
expect('我在写 English 英文。'.slice(t.from, t.to)).toBe(t.word)
|
||||
}
|
||||
})
|
||||
|
||||
it('每 token reports the span it actually occupies', () => {
|
||||
const tokens = seg.segment('英文')
|
||||
expect(tokens).toEqual([{ word: '英文', from: 0, to: 2 }])
|
||||
})
|
||||
})
|
||||
|
||||
describe('wordAt — the hover and click path', () => {
|
||||
const seg = dict({ 我: 90000, 今天: 8000, 去: 30000, 公园: 4000, 跑步: 3000, 了: 200000 })
|
||||
const text = '我今天去公园跑步了'
|
||||
|
||||
it('finds the word covering a position anywhere inside it', () => {
|
||||
// 公园 occupies [4,6): either of its characters resolves to the whole word
|
||||
// rather than to one character.
|
||||
for (const i of [4, 5]) {
|
||||
expect(seg.wordAt(text, i)?.word, `index ${i}`).toBe('公园')
|
||||
}
|
||||
expect(seg.wordAt(text, 0)?.word).toBe('我')
|
||||
expect(seg.wordAt(text, 2)?.word).toBe('今天')
|
||||
})
|
||||
|
||||
// A position names a gap; a word covers characters. On a boundary the answer
|
||||
// is the word that *starts* there, because that is the character being pointed
|
||||
// at — index 6 is the 跑 under the mouse, not the 园 behind it.
|
||||
it('a boundary belongs to the word that starts there', () => {
|
||||
expect(seg.wordAt(text, 6)?.word).toBe('跑步')
|
||||
expect(seg.wordAt(text, 4)?.word).toBe('公园')
|
||||
})
|
||||
|
||||
// The caret after a just-typed word belongs to that word. Ctrl/Cmd+D at the
|
||||
// end of 跑步 must look up 跑步, which is the position the caret is actually in
|
||||
// the moment someone finishes typing it.
|
||||
it('a caret at the very end of the text still resolves', () => {
|
||||
expect(seg.wordAt(text, text.length)?.word).toBe('了')
|
||||
})
|
||||
|
||||
it('returns null outside Han text', () => {
|
||||
expect(seg.wordAt('hello world', 3)).toBeNull()
|
||||
expect(seg.wordAt('', 0)).toBeNull()
|
||||
expect(seg.wordAt('我 hello', 4)).toBeNull()
|
||||
})
|
||||
|
||||
// The window exists so that a pasted page of Chinese with no punctuation is
|
||||
// not walked on every hover. It must not change the answer for ordinary text.
|
||||
it('agrees with a full segmentation of the same string', () => {
|
||||
const long = '我今天去公园跑步了'.repeat(20)
|
||||
const full = seg.segment(long)
|
||||
for (const t of full) {
|
||||
expect(seg.wordAt(long, t.from)).toEqual(t)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
// ── the shipped asset ───────────────────────────────────────────────────────
|
||||
// Everything above would pass with a word list built wrong. These read the file
|
||||
// this app actually serves.
|
||||
|
||||
describe('the shipped word list', () => {
|
||||
const raw = gunzipSync(readFileSync(new URL('../../public/dictionaries/zh/words.txt.gz', import.meta.url)))
|
||||
const seg = buildSegmenter(raw.toString('utf8'))
|
||||
|
||||
it('is the size the build script says it is', () => {
|
||||
expect(seg.size).toBeGreaterThan(180_000)
|
||||
})
|
||||
|
||||
it('segments ordinary learner prose the way a reader would', () => {
|
||||
expect(words(seg, '我今天早上去公园跑步了')).toEqual(['我', '今天', '早上', '去', '公园', '跑步', '了'])
|
||||
expect(words(seg, '他的中文说得很好')).toEqual(['他', '的', '中文', '说', '得', '很', '好'])
|
||||
expect(words(seg, '北京大学的学生正在图书馆学习')).toEqual([
|
||||
'北京大学', '的', '学生', '正在', '图书馆', '学习',
|
||||
])
|
||||
})
|
||||
|
||||
it('gets the textbook ambiguities right', () => {
|
||||
expect(words(seg, '研究生命的起源')).toEqual(['研究', '生命', '的', '起源'])
|
||||
expect(words(seg, '乒乓球拍卖完了')).toEqual(['乒乓球', '拍卖', '完', '了'])
|
||||
})
|
||||
|
||||
// The minimal pair, and the one that says the line above was a decision rather
|
||||
// than a bias against long words: the same five characters open both
|
||||
// sentences, and 研究生 is the right answer in one of them.
|
||||
it('finds 研究生 where 研究生 is the word', () => {
|
||||
expect(words(seg, '研究生宿舍')).toEqual(['研究生', '宿舍'])
|
||||
expect(words(seg, '他们正在研究生物')).toEqual(['他们', '正在', '研究', '生物'])
|
||||
})
|
||||
|
||||
// The three particles the 错别字 rules are about have to survive as their own
|
||||
// tokens, or those rules have nothing to anchor to.
|
||||
it('keeps 的 / 地 / 得 separate', () => {
|
||||
expect(words(seg, '她高兴地笑了')).toContain('地')
|
||||
expect(words(seg, '这个问题需要认真地思考')).toContain('地')
|
||||
expect(words(seg, '他跑得很快')).toContain('得')
|
||||
expect(words(seg, '我的书')).toContain('的')
|
||||
})
|
||||
|
||||
it('knows the words the build script asserts it kept', () => {
|
||||
for (const w of ['我', '的', '图书馆', '乒乓球', '公园', '的士']) {
|
||||
expect(seg.has(w), w).toBe(true)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,212 @@
|
||||
// Chinese word segmentation — the thing that has to exist before any of Petal's
|
||||
// ESL surfaces can point at a Chinese word.
|
||||
//
|
||||
// Every one of them is built on `wordAt(doc, pos)`, and `wordAt` is a regex over
|
||||
// runs of Latin letters. That works because English writes its word boundaries
|
||||
// down. Chinese does not: 我今天早上去公园跑步了 is eleven characters and seven
|
||||
// words, and which seven is a question with a real answer that no regex can
|
||||
// reach. Until something answers it there is no "word under the cursor" to
|
||||
// hover, look up, read aloud, or plant in the vocabulary garden.
|
||||
//
|
||||
// **Why the answer is a shortest-path walk and not longest-match.** The obvious
|
||||
// algorithm — take the longest dictionary word at each position and move on —
|
||||
// gets the textbook cases wrong in both directions, because the longest match is
|
||||
// not the likeliest one. The standard fix is to score every possible split by
|
||||
// how probable its words are and take the best-scoring path, which is a
|
||||
// shortest-path problem over a small DAG and is what this does. It is why the
|
||||
// word list ships with a frequency column at all.
|
||||
//
|
||||
// **Why it runs in the browser.** It runs on hover. A round-trip per hover is
|
||||
// not a hover, and the whole point of the offline lexicon (SUGGESTIONS §6) is
|
||||
// that the daily reading aids keep working with the tunnel down.
|
||||
|
||||
// A word found in the text, with the offsets it occupies. Offsets are into the
|
||||
// string that was passed in — the caller maps them to ProseMirror positions the
|
||||
// same way the spell and suggestion layers already do.
|
||||
export interface Token {
|
||||
word: string
|
||||
from: number
|
||||
to: number
|
||||
}
|
||||
|
||||
// Han characters only. Not a hand-rolled U+4E00–U+9FFF range: that misses the
|
||||
// extension blocks, and a character Petal fails to recognise as Chinese is one
|
||||
// the English tokenizer then tries to make sense of.
|
||||
const HAN = /\p{Script=Han}/u
|
||||
|
||||
export function isHan(ch: string): boolean {
|
||||
return HAN.test(ch)
|
||||
}
|
||||
|
||||
// The longest word the walk will consider at any position. The dictionary
|
||||
// contains longer entries (chengyu, place names, a few titles), but the cost of
|
||||
// the walk is linear in this number and the entries beyond it are rare enough
|
||||
// that paying for them on every hover is the wrong trade. Six characters covers
|
||||
// every ordinary word and every four-character idiom.
|
||||
const MAX_WORD_LEN = 6
|
||||
|
||||
// What an unknown single character is worth, as a fraction of one occurrence.
|
||||
// It must be *positive* — every position needs some path through it, or a
|
||||
// sentence containing one unfamiliar character would have no segmentation at
|
||||
// all — and it must be small enough that a real one-character word always wins.
|
||||
// Half an occurrence is below the rarest thing in the list (which is 1) and
|
||||
// above zero, which is the whole specification.
|
||||
const UNKNOWN_WEIGHT = 0.5
|
||||
|
||||
export interface Segmenter {
|
||||
// segment splits a whole string. Runs of non-Han text are skipped rather than
|
||||
// returned: this is the Chinese tokenizer, and the Latin one is still running
|
||||
// over the same paragraph.
|
||||
segment(text: string): Token[]
|
||||
// wordAt returns the token covering `index`, or null when that position is
|
||||
// not inside Han text. This is the hover/click path, and it segments only the
|
||||
// run around the position rather than the whole document.
|
||||
wordAt(text: string, index: number): Token | null
|
||||
// has reports whether a word is in the list — the 错别字 rules ask, to check
|
||||
// that a correction they are about to propose is a real word.
|
||||
has(word: string): boolean
|
||||
size: number
|
||||
}
|
||||
|
||||
// buildSegmenter turns the raw `word freq` list into something that can answer
|
||||
// questions about it. Exported for tests, which build tiny dictionaries by hand;
|
||||
// the app reaches it through loadSegmenter.
|
||||
export function buildSegmenter(source: string): Segmenter {
|
||||
const freq = new Map<string, number>()
|
||||
let total = 0
|
||||
for (const line of source.split('\n')) {
|
||||
if (!line) continue
|
||||
const sp = line.lastIndexOf(' ')
|
||||
if (sp <= 0) continue
|
||||
const word = line.slice(0, sp)
|
||||
const n = Number(line.slice(sp + 1))
|
||||
if (!Number.isFinite(n) || n <= 0) continue
|
||||
freq.set(word, n)
|
||||
total += n
|
||||
}
|
||||
// A dictionary with nothing in it would make every log() below -Infinity.
|
||||
const logTotal = Math.log(Math.max(total, 1))
|
||||
const unknownScore = Math.log(UNKNOWN_WEIGHT) - logTotal
|
||||
|
||||
// The walk, over one run of Han characters.
|
||||
//
|
||||
// `best[i]` is the score of the best segmentation of run[i..], and `next[i]`
|
||||
// is where that segmentation's first word ends. Filling it right-to-left means
|
||||
// each position only ever reads answers that are already final, which is what
|
||||
// makes this linear rather than exponential in the number of possible splits.
|
||||
function walk(run: string, base: number, out: Token[]): void {
|
||||
const n = run.length
|
||||
const best = new Float64Array(n + 1)
|
||||
const next = new Int32Array(n + 1)
|
||||
best[n] = 0
|
||||
for (let i = n - 1; i >= 0; i--) {
|
||||
let bestScore = -Infinity
|
||||
let bestEnd = i + 1
|
||||
const limit = Math.min(n, i + MAX_WORD_LEN)
|
||||
for (let j = i + 1; j <= limit; j++) {
|
||||
const f = freq.get(run.slice(i, j))
|
||||
let score: number
|
||||
if (f === undefined) {
|
||||
// Only a single unknown character is a candidate. Allowing unknown
|
||||
// multi-character spans would let the walk invent words.
|
||||
if (j > i + 1) continue
|
||||
score = unknownScore
|
||||
} else {
|
||||
score = Math.log(f) - logTotal
|
||||
}
|
||||
score += best[j]
|
||||
if (score > bestScore) {
|
||||
bestScore = score
|
||||
bestEnd = j
|
||||
}
|
||||
}
|
||||
best[i] = bestScore
|
||||
next[i] = bestEnd
|
||||
}
|
||||
for (let i = 0; i < n; ) {
|
||||
const end = next[i]
|
||||
out.push({ word: run.slice(i, end), from: base + i, to: base + end })
|
||||
i = end
|
||||
}
|
||||
}
|
||||
|
||||
function segment(text: string): Token[] {
|
||||
const out: Token[] = []
|
||||
let i = 0
|
||||
while (i < text.length) {
|
||||
if (!isHan(text[i])) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
let j = i
|
||||
while (j < text.length && isHan(text[j])) j++
|
||||
walk(text.slice(i, j), i, out)
|
||||
i = j
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// How much context a hover segments. The run around the cursor is bounded
|
||||
// because a pasted page of Chinese with no punctuation is one run, and a hover
|
||||
// must not walk it. Segmentation is local enough that a window this size
|
||||
// reaches the same answer as the whole paragraph would: the walk's decisions
|
||||
// are dominated by the two or three characters either side, and a word longer
|
||||
// than MAX_WORD_LEN cannot span the window's edge anyway.
|
||||
const WINDOW = 60
|
||||
|
||||
function wordAt(text: string, index: number): Token | null {
|
||||
if (index < 0 || index > text.length) return null
|
||||
// `index` names a gap between characters; a word covers characters. So the
|
||||
// question is resolved on the character at `index` — the one to the *right*
|
||||
// of the caret — and a boundary belongs to the word that starts there rather
|
||||
// than the one that ends there. For a hover that is simply correct: index 6
|
||||
// of 我今天去公园跑步了 is the 跑 being pointed at.
|
||||
//
|
||||
// The step back covers the case where there is no character to the right:
|
||||
// the caret at the end of the text, or against following punctuation. That
|
||||
// is where the caret sits the instant an IME commits a word, and Ctrl/Cmd+D
|
||||
// there must look up the word just typed.
|
||||
let probe = index
|
||||
if (probe >= text.length || !isHan(text[probe])) {
|
||||
if (probe > 0 && isHan(text[probe - 1])) probe -= 1
|
||||
else return null
|
||||
}
|
||||
let start = probe
|
||||
while (start > 0 && isHan(text[start - 1]) && probe - start < WINDOW) start--
|
||||
let end = probe
|
||||
while (end < text.length && isHan(text[end]) && end - probe < WINDOW) end++
|
||||
|
||||
const tokens: Token[] = []
|
||||
walk(text.slice(start, end), start, tokens)
|
||||
for (const t of tokens) {
|
||||
if (probe >= t.from && probe < t.to) return t
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
return { segment, wordAt, has: (w) => freq.has(w), size: freq.size }
|
||||
}
|
||||
|
||||
// Where the word list lives. Gzipped, like every dictionary Petal ships that is
|
||||
// bigger than English's.
|
||||
const WORDS_URL = '/dictionaries/zh/words.txt.gz'
|
||||
|
||||
// loadSegmenter fetches and builds the segmenter. One per session, like the
|
||||
// spelling dictionaries — the cost is the parse, not the download, and paying it
|
||||
// per document would be paying it per document for no reason.
|
||||
//
|
||||
// A failure resolves to null rather than throwing. Petal without segmentation is
|
||||
// Petal with no Chinese hover, which is a diminished editor; Petal that refused
|
||||
// to open because a static asset 404ed is no editor at all.
|
||||
export async function loadSegmenter(url = WORDS_URL): Promise<Segmenter | null> {
|
||||
try {
|
||||
const res = await fetch(url)
|
||||
if (!res.ok || !res.body) return null
|
||||
const stream = res.body.pipeThrough(new DecompressionStream('gzip'))
|
||||
const text = await new Response(stream).text()
|
||||
const seg = buildSegmenter(text)
|
||||
return seg.size > 0 ? seg : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user