package lexicon import ( "fmt" "strings" "sync" "unicode" ) // The Chinese half of the lexicon: a word written in hanzi to its pinyin and // English senses. This is the mirror image of `gloss` — that one reads English // and answers in Chinese, for a Mandarin native practising English; this one // reads Chinese and answers in English, for the other direction of the same // pair (`users.direction = 'learning_pair'`). // // It is deliberately not folded into [Lexicon.load]. That method reads four // datasets on the first lookup of any kind, and this one is 3.1 MB gzipped that // only a learner-direction account will ever ask for — every other writer would // pay the decompression and the resident memory for a map they never touch. Its // own sync.Once means the cost lands on the first Chinese hover and nowhere // else. // HanziReading is one pronunciation of a word and the senses it carries in that // pronunciation. A word usually has one; the ones that have two are why this is // a list rather than a pair of strings. 得 is dé, "to obtain", *and* de, the // particle that makes 说得很好 mean "speaks well" — a learner shown only the // first has been told something false about the sentence in front of them. type HanziReading struct { Pinyin string `json:"pinyin"` Senses string `json:"senses"` } // HanziChar is one character of a word that the dictionary could not answer as // a whole. See [Lexicon.Hanzi]. type HanziChar struct { Char string `json:"char"` Pinyin string `json:"pinyin"` Senses string `json:"senses"` } // HanziResult is what a Chinese word lookup answers. Readings is empty for a // word the dictionary does not have, in which case Chars may carry the // character-by-character reading instead. type HanziResult struct { Word string `json:"word"` Readings []HanziReading `json:"readings"` Chars []HanziChar `json:"chars"` } type hanziStore struct { once sync.Once err error // word → [[pinyin, senses], …], exactly as scripts/build_cedict.py writes it. entries map[string][][]string } var hanzi hanziStore func (h *hanziStore) load() { h.once.Do(func() { if err := gunzipJSON(hanziGz, &h.entries); err != nil { h.err = fmt.Errorf("load hanzi: %w", err) } }) } // maxHanziChars caps the per-character fallback. A run longer than this is // almost certainly a phrase the segmenter split badly rather than a word, and // spelling out eight characters one at a time is a wall, not a hint. const maxHanziChars = 6 // Hanzi returns the pinyin and English senses of a Chinese word. // // There is no de-inflection walk here, and its absence is a fact about the // language rather than an omission: Chinese words do not inflect, so the // candidate forms [lookupGloss] tries for "running" → "run" have no analogue. // A lookup either hits the headword or it does not. // // What it does instead is fall back to the characters. The segmentation word // list is a superset of this dictionary — every glossable word can be // segmented, but jieba knows ordinary compounds CC-CEDICT has no entry for — so // a hover really can land on a word with nothing to say about it. Chinese // compounds are usually transparent from their parts (电脑 is "electric brain"), // which makes the character reading a genuinely useful second answer rather // than a consolation prize. It is returned as its own field so the surface can // say which of the two it is showing; a caller that only wants whole words can // ignore it. func (l *Lexicon) Hanzi(word string) (HanziResult, error) { hanzi.load() if hanzi.err != nil { return HanziResult{}, hanzi.err } norm := strings.TrimSpace(word) res := HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}} if norm == "" { return res, nil } if rows, ok := hanzi.entries[norm]; ok { res.Readings = toReadings(rows) return res, nil } chars := []rune(norm) if len(chars) < 2 || len(chars) > maxHanziChars { // A single character that missed has no parts to fall back to, and a long // run is not a word. Either way the honest answer is nothing. return res, nil } for _, r := range chars { if !unicode.Is(unicode.Han, r) { // Mixed input (a stray letter or digit inside the run) is not something // the character reading can explain, and guessing at the hanzi parts of // it would be worse than silence. return HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}, nil } rows, ok := hanzi.entries[string(r)] if !ok { continue } first := toReadings(rows) if len(first) == 0 { continue } res.Chars = append(res.Chars, HanziChar{ Char: string(r), Pinyin: first[0].Pinyin, Senses: first[0].Senses, }) } return res, nil } func toReadings(rows [][]string) []HanziReading { out := make([]HanziReading, 0, len(rows)) for _, row := range rows { if len(row) < 2 { continue } out = append(out, HanziReading{Pinyin: row[0], Senses: row[1]}) } return out }