#!/usr/bin/env python3 """Build the two Chinese assets the learner direction of the zh pair needs. Why two, and why they are split the way they are ------------------------------------------------ Every other pair Petal ships needs one asset: a word list the browser loads so it can underline. Chinese needs two, because the browser and the server want different halves of the same dictionary and for different reasons. * **The browser needs a word list, and it needs it offline.** Chinese is written without spaces, so there is no such thing as "the word under the cursor" until something segments the sentence. Every ESL surface Petal already has — the hover gloss, the right-click lookup, Ctrl/Cmd+D, the vocabulary garden capture — is built on `wordAt`, and `wordAt` is a regex over Latin letters. Segmentation is what replaces that regex, it runs on every hover, and a round-trip per hover is not a hover. So the word list ships to the browser: `web/public/dictionaries/zh/words.txt`. * **The server holds the whole dictionary.** Pinyin and English senses are only ever wanted one word at a time, in answer to a hover or a click, which is exactly what `/api/gloss/{word}` already does for the other direction. So the readings stay in the binary — `internal/lexicon/data/hanzi.json.gz` — where their size costs a browser nothing. That split is what makes the coverage decisions below come out *opposite* to each other, and both are deliberate. Two sources, because neither one has both halves ------------------------------------------------ * **CC-CEDICT** (CC BY-SA 4.0, https://www.mdbg.net/) has the headwords, pinyin and English senses, and no frequency information at all. * **jieba's `dict.txt`** (MIT, https://github.com/fxsjy/jieba) has ~349k headwords with corpus frequencies, and no definitions. Segmentation needs the frequencies: the standard algorithm is a shortest-path walk over log-probabilities, not longest-match, and without frequencies the classic ambiguities go the wrong way. The client list therefore carries `word freq` per line; the gloss map carries readings. The size decision is the client list, and it is a size decision only -------------------------------------------------------------------- Measured on ordinary learner prose, the segmentation produced by the full jieba dictionary (381,886 hanzi headwords once CC-CEDICT is unioned in) and by a frequency-gated one is **identical**, including on the textbook ambiguities (研究生命的起源, 乒乓球拍卖完了, 南京市长江大桥). What the long tail contains is rare proper nouns, and the max-probability walk almost never chooses one: a freq-3 name loses to two common words every time. The cases where a missing word does change the answer degrade *gracefully* — the sentence splits into smaller real words, which is a slightly clumsier gloss, not a wrong underline. So the gate is set where the size is, at **freq >= 5**: 188,522 words, ~0.97 MB gzipped over the wire, in line with fr (1.19 MB) and es (1.74 MB) rather than in excess of them. Every CC-CEDICT headword is unioned back in regardless of frequency, so the segmenter can always see a word the server can explain. The gloss map is gated by nothing, for the opposite reason ----------------------------------------------------------- The es phase settled that a *spelling* dictionary should hold the union of every variety, because its only power is to underline and it must not underline correct writing. This asset's only power is to **explain**, and the word a learner stops on is precisely the one they do not know — which is to say, the rare one. Trimming this by frequency would remove exactly the entries it exists for. All 113,637 glossable headwords ship, ~3.1 MB gzipped, which is less than half of what `synonyms.json.gz` has embedded since Phase 9. Simplified only, and said out loud ----------------------------------- The zh langpack is written in simplified characters and jieba's frequencies are counted over simplified text, so the traditional headword in each CC-CEDICT line is dropped and simplified is what both assets are keyed by. Glossing traditional would be nearly free *here* and useless in the app: nothing would segment it, so nothing would ever ask. Traditional support is a real feature and it starts with a traditional word list, not with this file. Usage: curl -sL https://www.mdbg.net/chinese/export/cedict/cedict_1_0_ts_utf-8_mdbg.txt.gz | gunzip > cedict.txt curl -sL https://raw.githubusercontent.com/fxsjy/jieba/master/jieba/dict.txt -o jieba.txt python3 scripts/build_cedict.py cedict.txt jieba.txt \ web/public/dictionaries/zh/words.txt.gz \ internal/lexicon/data/hanzi.json.gz """ import gzip import json import re import sys # Frequency gate for the *client* list only (see the module docstring). Words # below it survive if CC-CEDICT knows them, so "segmentable" is always a superset # of "glossable" and a hover can never land on a word the server cannot explain. MIN_FREQ = 5 # A CC-CEDICT headword we keep must be nothing but han characters. This drops the # entries that are really English or numerals with a Chinese gloss attached # ("AA制", "PM2.5", "11区"): the segmenter walks runs of hanzi, so a mixed # headword can never be matched anyway, and a Latin one would collide with the # English tokenizer that is still running on the same paragraph. HANZI_ONLY = re.compile(r'^[一-鿿]+$') CEDICT_LINE = re.compile(r'^(\S+) (\S+) \[(.*?)\] /(.*)/$') # At most this many readings per word, and this many senses per reading. Two # readings is not an arbitrary cap: it is what the particles need. 得 is dé "to # obtain" *and* de, the complement marker — and a learner who hovers 得 in # 说得很好 and is told only "to obtain" has been actively misinformed. Beyond two # the tail is dialect and surnames, which crowd out the sense actually wanted. MAX_READINGS = 2 MAX_SENSES = 3 MAX_SENSE_CHARS = 110 # Senses that describe the *dictionary* rather than the word. A learner hovering # a word wants to know what it means, not that it is an orthographic variant of # another headword they also do not know. SKIP_SENSE_PREFIXES = ('variant of', 'old variant', 'see ', 'used in', 'abbr. for') # ── pinyin: numbered syllables to tone marks ──────────────────────────────── # CC-CEDICT stores "gong1 yuan2". A learner reading their own writing back wants # gōngyuán: the tone mark is the part that is hard to remember and the part that # changes the word. The placement rule is the standard one — a/o/e take the mark # if present, otherwise the last vowel of the final — and it is small enough to # do here rather than to take a dependency for. TONE_VOWELS = { 'a': 'āáǎà', 'e': 'ēéěè', 'i': 'īíǐì', 'o': 'ōóǒò', 'u': 'ūúǔù', 'ü': 'ǖǘǚǜ', } SYLLABLE = re.compile(r'^([a-zA-Zü:]+)([1-5])$') def tone_mark(syllable: str) -> str: """One numbered pinyin syllable to its tone-marked form.""" m = SYLLABLE.match(syllable) if not m: # Punctuation, a bare letter (CC-CEDICT writes "X" for unknown), or an # already-marked syllable: pass it through rather than mangling it. return syllable body, tone = m.group(1), int(m.group(2)) # CC-CEDICT writes ü as "u:" and, in a few entries, as "v". body = body.replace('u:', 'ü').replace('U:', 'Ü').replace('v', 'ü').replace('V', 'Ü') if tone == 5: # neutral tone carries no mark return body low = body.lower() idx = -1 for vowel in ('a', 'o', 'e'): idx = low.find(vowel) if idx >= 0: break if idx < 0: # No a/o/e: the mark goes on the last of i/u/ü (liú, guǐ, nǚ). idx = max(low.rfind('i'), low.rfind('u'), low.rfind('ü')) if idx < 0: return body marked = TONE_VOWELS[low[idx]][tone - 1] if body[idx].isupper(): marked = marked.upper() return body[:idx] + marked + body[idx + 1:] def pinyin(numbered: str) -> str: """A whole CC-CEDICT pinyin field to tone marks, syllables joined up. Joined rather than spaced because that is how a word is written when it is being read as a word (gōngyuán, not gōng yuán); the spaces in the source are a storage convention, not orthography. """ return ''.join(tone_mark(s) for s in numbered.split()) def clean_senses(raw: list[str]) -> list[str]: """Strip the apparatus CC-CEDICT carries for lexicographers, not learners.""" out = [] for sense in raw: # "CL:座[zuo4]" is the measure-word field, useful and not a definition. sense = re.sub(r'\s*CL:.*$', '', sense).strip() # Bracketed pinyin cross-references ("abbr. for 的士[di1 shi4]"). sense = re.sub(r'\[[a-zA-Z0-9: ]+\]', '', sense).strip() # Both edits cut inside parentheses — "cat (CL:只)" loses its closing # bracket and leaves "cat (" on the card. Drop a dangling opener rather # than trying to rebalance: what it introduced is gone. if sense.count('(') > sense.count(')'): sense = re.sub(r'\s*\([^()]*$', '', sense).strip() if not sense or sense.startswith(SKIP_SENSE_PREFIXES): continue out.append(sense) return out def read_cedict(path: str) -> dict[str, list[tuple[str, list[str]]]]: entries: dict[str, list[tuple[str, list[str]]]] = {} for line in open(path, encoding='utf-8'): if line.startswith('#'): continue m = CEDICT_LINE.match(line.strip()) if not m: continue _traditional, simplified, py, defs = m.groups() if not HANZI_ONLY.match(simplified): continue entries.setdefault(simplified, []).append((py, defs.split('/'))) return entries def read_jieba(path: str) -> dict[str, int]: freqs: dict[str, int] = {} for line in open(path, encoding='utf-8'): parts = line.split() if len(parts) >= 2 and HANZI_ONLY.match(parts[0]): freqs[parts[0]] = int(parts[1]) return freqs # ── the assertions ────────────────────────────────────────────────────────── # The es phase's lesson, in the place it applies here: a check that every # plausible input would pass is not a check. The Spanish MUST_ACCEPT list # asserted vocabulary that all twenty-four builds carried, so it could not tell # them apart. These assert the things that actually go wrong in *this* build — # a mis-parsed pinyin field, a missing particle reading, a word list gated so # hard the segmenter can no longer see a word the server can explain. # Tone marking, including the three cases the placement rule exists for. MUST_MARK = { 'gong1 yuan2': 'gōngyuán', # a/o/e rule, first syllable 'pao3 bu4': 'pǎobù', 'liu2': 'liú', # no a/o/e: mark the *last* of i/u 'gui3': 'guǐ', 'nu:3': 'nǚ', # u: is ü 'lu:e4': 'lüè', # ü and an e in the same syllable: e wins 'de5': 'de', # neutral tone takes no mark at all 'Zhong1 wen2': 'Zhōngwén', # capitalised headword keeps its capital } # The particles the 错别字 rules are about must each carry the *grammatical* # reading, not only the lexical one. 的/地/得 are the single most confused triple # in written Chinese and all three are neutral-tone "de" in the use that matters; # an entry that only knows 得 as dé is worse than no entry. MUST_READ_DE = ('的', '地', '得') # Words the segmenter must be able to see. 图书馆 and 乒乓球 are ordinary # vocabulary; 我 and 的 are the two commonest words in the language and a gate # that dropped either would be visibly broken; 的士 is a CC-CEDICT headword rare # enough to fall below the frequency gate, and is here to prove the union. MUST_SEGMENT = ('我', '的', '图书馆', '乒乓球', '公园', '的士') def check(words: dict[str, int], gloss: dict[str, list[list[str]]]) -> None: for numbered, want in MUST_MARK.items(): got = pinyin(numbered) assert got == want, f'pinyin({numbered!r}) = {got!r}, want {want!r}' for particle in MUST_READ_DE: readings = gloss.get(particle) assert readings, f'{particle} has no gloss entry at all' assert any(r[0] == 'de' for r in readings), \ f'{particle} never reads as neutral "de": {readings}' for word in MUST_SEGMENT: assert word in words, f'{word} missing from the segmentation list' # The invariant the two gates exist to keep: everything the server can # explain, the browser can find. missing = [w for w in gloss if w not in words] assert not missing, f'{len(missing)} glossable words are unsegmentable, e.g. {missing[:5]}' # Nothing Latin leaked into either asset (see HANZI_ONLY). for name, keys in (('words', words), ('gloss', gloss)): bad = [k for k in keys if not HANZI_ONLY.match(k)] assert not bad, f'non-hanzi headwords in {name}: {bad[:5]}' def main() -> None: if len(sys.argv) != 5: sys.exit(__doc__.strip().rsplit('Usage:', 1)[-1].strip()) cedict_path, jieba_path, words_out, gloss_out = sys.argv[1:] entries = read_cedict(cedict_path) freqs = read_jieba(jieba_path) # The client list: frequency-gated, then unioned with every glossable word. # A CC-CEDICT word jieba has never seen gets frequency 1 — real, and rare # enough that the max-probability walk will only choose it when nothing else # fits, which is exactly the standing it should have. words = {w: f for w, f in freqs.items() if f >= MIN_FREQ} for w in entries: words.setdefault(w, 1) gloss: dict[str, list[list[str]]] = {} for word, rows in entries.items(): readings: list[list[str]] = [] for numbered, defs in rows: senses = clean_senses(defs) if not senses: continue readings.append([pinyin(numbered), '; '.join(senses[:MAX_SENSES])[:MAX_SENSE_CHARS]]) if len(readings) == MAX_READINGS: break if readings: gloss[word] = readings check(words, gloss) # Gzipped on disk, like the pt-PT/fr/es word lists: the browser inflates it # with DecompressionStream (see useSpellChecker.fetchText), which costs no # bundle bytes, and 0.97 MB over the wire rather than 2.23 MB is the whole # difference between this and the biggest asset Petal ships. body = ('\n'.join(f'{w} {words[w]}' for w in sorted(words)) + '\n').encode('utf-8') with gzip.open(words_out, 'wb', compresslevel=9) as fh: fh.write(body) payload = json.dumps(gloss, ensure_ascii=False, separators=(',', ':')).encode('utf-8') with gzip.open(gloss_out, 'wb', compresslevel=9) as fh: fh.write(payload) print(f'{words_out}: {len(words)} words, {len(body) / 1e6:.2f} MB raw, ' f'{len(gzip.compress(body, 9)) / 1e6:.2f} MB gzipped') print(f'{gloss_out}: {len(gloss)} entries, {len(gzip.compress(payload, 9)) / 1e6:.2f} MB gzipped') if __name__ == '__main__': main()