petal/internal/lexicon/hanzi.go

package lexicon

import (
	"fmt"
	"strings"
	"sync"
	"unicode"
)

// The Chinese half of the lexicon: a word written in hanzi to its pinyin and
// English senses. This is the mirror image of `gloss` — that one reads English
// and answers in Chinese, for a Mandarin native practising English; this one
// reads Chinese and answers in English, for the other direction of the same
// pair (`users.direction = 'learning_pair'`).
//
// It is deliberately not folded into [Lexicon.load]. That method reads four
// datasets on the first lookup of any kind, and this one is 3.1 MB gzipped that
// only a learner-direction account will ever ask for — every other writer would
// pay the decompression and the resident memory for a map they never touch. Its
// own sync.Once means the cost lands on the first Chinese hover and nowhere
// else.

// HanziReading is one pronunciation of a word and the senses it carries in that
// pronunciation. A word usually has one; the ones that have two are why this is
// a list rather than a pair of strings. 得 is dé, "to obtain", *and* de, the
// particle that makes 说得很好 mean "speaks well" — a learner shown only the
// first has been told something false about the sentence in front of them.
type HanziReading struct {
	Pinyin string `json:"pinyin"`
	Senses string `json:"senses"`
}

// HanziChar is one character of a word that the dictionary could not answer as
// a whole. See [Lexicon.Hanzi].
type HanziChar struct {
	Char   string `json:"char"`
	Pinyin string `json:"pinyin"`
	Senses string `json:"senses"`
}

// HanziResult is what a Chinese word lookup answers. Readings is empty for a
// word the dictionary does not have, in which case Chars may carry the
// character-by-character reading instead.
type HanziResult struct {
	Word     string         `json:"word"`
	Readings []HanziReading `json:"readings"`
	Chars    []HanziChar    `json:"chars"`
}

type hanziStore struct {
	once sync.Once
	err  error
	// word → [[pinyin, senses], …], exactly as scripts/build_cedict.py writes it.
	entries map[string][][]string
}

var hanzi hanziStore

func (h *hanziStore) load() {
	h.once.Do(func() {
		if err := gunzipJSON(hanziGz, &h.entries); err != nil {
			h.err = fmt.Errorf("load hanzi: %w", err)
		}
	})
}

// maxHanziChars caps the per-character fallback. A run longer than this is
// almost certainly a phrase the segmenter split badly rather than a word, and
// spelling out eight characters one at a time is a wall, not a hint.
const maxHanziChars = 6

// Hanzi returns the pinyin and English senses of a Chinese word.
//
// There is no de-inflection walk here, and its absence is a fact about the
// language rather than an omission: Chinese words do not inflect, so the
// candidate forms [lookupGloss] tries for "running" → "run" have no analogue.
// A lookup either hits the headword or it does not.
//
// What it does instead is fall back to the characters. The segmentation word
// list is a superset of this dictionary — every glossable word can be
// segmented, but jieba knows ordinary compounds CC-CEDICT has no entry for — so
// a hover really can land on a word with nothing to say about it. Chinese
// compounds are usually transparent from their parts (电脑 is "electric brain"),
// which makes the character reading a genuinely useful second answer rather
// than a consolation prize. It is returned as its own field so the surface can
// say which of the two it is showing; a caller that only wants whole words can
// ignore it.
func (l *Lexicon) Hanzi(word string) (HanziResult, error) {
	hanzi.load()
	if hanzi.err != nil {
		return HanziResult{}, hanzi.err
	}

	norm := strings.TrimSpace(word)
	res := HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}
	if norm == "" {
		return res, nil
	}

	if rows, ok := hanzi.entries[norm]; ok {
		res.Readings = toReadings(rows)
		return res, nil
	}

	chars := []rune(norm)
	if len(chars) < 2 || len(chars) > maxHanziChars {
		// A single character that missed has no parts to fall back to, and a long
		// run is not a word. Either way the honest answer is nothing.
		return res, nil
	}
	for _, r := range chars {
		if !unicode.Is(unicode.Han, r) {
			// Mixed input (a stray letter or digit inside the run) is not something
			// the character reading can explain, and guessing at the hanzi parts of
			// it would be worse than silence.
			return HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}, nil
		}
		rows, ok := hanzi.entries[string(r)]
		if !ok {
			continue
		}
		first := toReadings(rows)
		if len(first) == 0 {
			continue
		}
		res.Chars = append(res.Chars, HanziChar{
			Char:   string(r),
			Pinyin: first[0].Pinyin,
			Senses: first[0].Senses,
		})
	}
	return res, nil
}

func toReadings(rows [][]string) []HanziReading {
	out := make([]HanziReading, 0, len(rows))
	for _, row := range rows {
		if len(row) < 2 {
			continue
		}
		out = append(out, HanziReading{Pinyin: row[0], Senses: row[1]})
	}
	return out
}