import { readFileSync } from 'node:fs' import { gunzipSync } from 'node:zlib' import { describe, expect, it } from 'vitest' import { CONFUSION_PAIRS, hanziFindings } from './hanzi' import { buildSegmenter } from '../../lib/segment' // The 错别字 pack, held to the bar Phase 22 set for the English rule pack: every // rule pinned in *two* directions — the mistake it must catch, and the correct // writing next to it that it must leave alone. // // Here the second direction is the one that matters, and it is unusually easy to // get wrong. Chinese has no spaces, so every one of these rules is a substring // match on running text, and for most of them there exists an ordinary correct // sentence that contains the substring across a word boundary. Those sentences // are the real test. const raw = gunzipSync(readFileSync(new URL('../../../public/dictionaries/zh/words.txt.gz', import.meta.url))) const seg = buildSegmenter(raw.toString('utf8')) const flagged = (text: string) => hanziFindings(text, seg).map((f) => `${f.original}→${f.replacement}`) describe('the gate that admits a rule', () => { // The pack's own claim about itself, checked against the shipped dictionary // rather than asserted in a comment. A pair whose wrong form is a real word // cannot be decided mechanically and does not belong here. it('every wrong form is not a word, and every right form is', () => { for (const { wrong, right } of CONFUSION_PAIRS) { expect(seg.has(wrong), `${wrong} is a dictionary word and must not be flagged`).toBe(false) expect(seg.has(right), `${right} is not a dictionary word`).toBe(true) } }) // The errors this pack deliberately refuses, and why — each is a genuine // mistake by a modern standard whose wrong form is itself a headword. If a // dictionary rebuild ever drops one of these, this test fails and the pair // becomes admissible; that is the intended way to find out. it('refuses the well-known errors it cannot decide', () => { for (const undecidable of ['自已', '好象', '倒底', '帐号', '部份']) { expect(seg.has(undecidable), `${undecidable} is no longer a word — reconsider the rule`).toBe(true) expect(flagged(`这是${undecidable}的例子`)).toEqual([]) } }) }) describe('the mistakes it catches', () => { it('已 / 己 / 以', () => { expect(flagged('我己经写完了作业')).toEqual(['己经→已经']) expect(flagged('我以经吃过饭了')).toEqual(['以经→已经']) expect(flagged('下课已后我们去公园')).toEqual(['已后→以后']) }) it('在 / 再', () => { expect(flagged('明天在见')).toEqual(['在见→再见']) expect(flagged('他正再看书')).toEqual(['正再→正在']) expect(flagged('现再几点了')).toEqual(['现再→现在']) }) it('做 / 作', () => { expect(flagged('我的工做很忙')).toEqual(['工做→工作']) expect(flagged('老师给我们很多做业')).toEqual(['做业→作业']) expect(flagged('这本书的做者是谁')).toEqual(['做者→作者']) }) it('the rest', () => { expect(flagged('我觉的这个很好')).toEqual(['觉的→觉得']) expect(flagged('你因该早点睡')).toEqual(['因该→应该']) expect(flagged('即然你来了就坐下吧')).toEqual(['即然→既然']) expect(flagged('你知到吗')).toEqual(['知到→知道']) expect(flagged('请输入你的蜜码')).toEqual(['蜜码→密码']) }) it('reports an exact span, so the card replaces the right characters', () => { const text = '我己经到了' const [f] = hanziFindings(text, seg) expect(text.slice(f.from, f.to)).toBe('己经') expect(text.slice(0, f.from) + f.replacement + text.slice(f.to)).toBe('我已经到了') }) it('finds every occurrence, in document order', () => { expect(flagged('我己经吃了,他也己经吃了')).toEqual(['己经→已经', '己经→已经']) expect(flagged('我的工做很忙,所以我觉的很累')).toEqual(['工做→工作', '觉的→觉得']) }) }) // ── the direction that matters ────────────────────────────────────────────── describe('the correct writing it must not touch', () => { // Each of these is an ordinary sentence containing a flagged substring across // a word boundary. Without the boundary gate, every one would be corrupted — // and corrupted silently, into text that is still made of real characters. it('leaves two real words alone where they happen to abut', () => { // 自己 + 经常. The substring is 己经. expect(flagged('他自己经常做饭')).toEqual([]) // 睡觉 + 的. The substring is 觉的. expect(flagged('睡觉的时候不要看手机')).toEqual([]) // 感觉 + 的. expect(flagged('这是我感觉的方向')).toEqual([]) // 不知 + 到底. expect(flagged('我不知到底该怎么办')).toEqual([]) // 因 + 位置. expect(flagged('因位置不好我们换了座位')).toEqual([]) // 已 + 后悔. expect(flagged('他已后悔了')).toEqual([]) }) it('leaves ordinary correct prose entirely alone', () => { for (const good of [ '我今天早上去公园跑步了', '他的中文说得很好', '我已经完成了我的作业', '现在几点了,我们再见面吧', '我觉得这个工作很有意思', '既然你已经知道了,就按照计划做', ]) { expect(flagged(good), good).toEqual([]) } }) // Where the gate costs the pack a real catch, and the trade it is making. // 不知 is itself a word, so 我不知到他在哪里 — which really is 知到 for 知道 — // reads to the segmenter as 不知 + 到 and is left alone. That is the gate // preferring a missed error to a corrupted sentence, which is the whole // premise: 我不知到底该怎么办 is the same three characters and is correct. it('declines a real error rather than risk the sentence beside it', () => { expect(flagged('我不知到他在哪里')).toEqual([]) expect(flagged('你知到吗')).toEqual(['知到→知道']) }) it('says nothing about English, or about nothing', () => { expect(flagged('I already finished my homework')).toEqual([]) expect(flagged('')).toEqual([]) }) // The direction gate. The word list is loaded only for an account learning // Chinese, so without one this pack is silent — a writer practising English // must never be told her own quoted Chinese is wrong. it('is silent without a segmenter, which is how the direction gate works', () => { expect(hanziFindings('我己经写完了', null)).toEqual([]) }) })