petal/internal/lexicon/hanzi_test.go

package lexicon

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/go-chi/chi/v5"
)

// The Chinese direction of the lexicon, against the real embedded asset — not a
// fixture. The dataset is built by scripts/build_cedict.py, which asserts its
// own invariants at build time; what these assert is that the *lookup* over it
// behaves, including on the entries the build script goes out of its way to keep.

func TestHanziLookup(t *testing.T) {
	l := New()

	res, err := l.Hanzi("公园")
	if err != nil {
		t.Fatalf("lookup 公园: %v", err)
	}
	if len(res.Readings) == 0 {
		t.Fatal("公园 has no readings")
	}
	// Tone marks, not the numbered pinyin CC-CEDICT stores. The number is the
	// storage format; the marks are what a learner reads.
	if got := res.Readings[0].Pinyin; got != "gōngyuán" {
		t.Errorf("公园 pinyin = %q, want gōngyuán", got)
	}
	if !strings.Contains(res.Readings[0].Senses, "park") {
		t.Errorf("公园 senses = %q, want something about a park", res.Readings[0].Senses)
	}
	// A word answered whole says nothing about its characters — the fallback is
	// the other branch, and sending both would double the payload of the common
	// case to no purpose.
	if len(res.Chars) != 0 {
		t.Errorf("a whole-word hit also returned %d characters", len(res.Chars))
	}
}

// 得 is the reason readings are a list. Answered with only dé "to obtain", a
// learner hovering it in 说得很好 has been told something false about the
// sentence they are looking at.
func TestHanziParticleCarriesItsGrammaticalReading(t *testing.T) {
	l := New()

	for _, particle := range []string{"的", "地", "得"} {
		res, err := l.Hanzi(particle)
		if err != nil {
			t.Fatalf("lookup %s: %v", particle, err)
		}
		var found bool
		for _, r := range res.Readings {
			if r.Pinyin == "de" {
				found = true
			}
		}
		if !found {
			t.Errorf("%s never reads as neutral \"de\": %+v", particle, res.Readings)
		}
	}
}

// The fallback the segmentation gap makes necessary: jieba knows ordinary
// compounds CC-CEDICT has no headword for, so a hover can land on a real word
// with no entry. Chinese compounds are usually transparent from their parts, so
// the characters are a real second answer.
func TestHanziFallsBackToCharacters(t *testing.T) {
	l := New()

	// Constructed rather than borrowed from the corpus: a word that CC-CEDICT
	// *does* carry would test the other branch, and which compounds it happens to
	// omit is not something a test should pin.
	const made = "猫书"
	if _, ok := hanzi.entries[made]; ok {
		t.Skipf("%s has become a real headword; pick another compound", made)
	}
	res, err := l.Hanzi(made)
	if err != nil {
		t.Fatalf("lookup %s: %v", made, err)
	}
	if len(res.Readings) != 0 {
		t.Fatalf("%s answered as a whole word: %+v", made, res.Readings)
	}
	if len(res.Chars) != 2 {
		t.Fatalf("character fallback gave %d entries, want 2: %+v", len(res.Chars), res.Chars)
	}
	if res.Chars[0].Char != "猫" || !strings.Contains(res.Chars[0].Senses, "cat") {
		t.Errorf("first character = %+v, want 猫 ~ cat", res.Chars[0])
	}
	if res.Chars[0].Pinyin != "māo" {
		t.Errorf("猫 pinyin = %q, want māo", res.Chars[0].Pinyin)
	}
}

func TestHanziMisses(t *testing.T) {
	l := New()

	for name, word := range map[string]string{
		// A single character with no entry has no parts to fall back to.
		"lone unknown character": "龥",
		"empty":                  "",
		"whitespace":             "  ",
		// Not Chinese at all: the English tokenizer owns these, and answering
		// would mean guessing.
		"english": "hello",
		"mixed":   "猫cat",
		// Longer than a word: a bad segmentation, not something to spell out
		// character by character.
		"a whole clause": "我今天早上去公园跑步了",
	} {
		res, err := l.Hanzi(word)
		if err != nil {
			t.Fatalf("%s: %v", name, err)
		}
		if len(res.Readings) != 0 || len(res.Chars) != 0 {
			t.Errorf("%s (%q) answered with %+v / %+v", name, word, res.Readings, res.Chars)
		}
	}
}

func TestHanziEndpoint(t *testing.T) {
	h := NewHandler(nil, NewSet(nil))
	r := chi.NewRouter()
	r.Mount("/hanzi", h.HanziRoutes())

	w := httptest.NewRecorder()
	r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/"+"跑步", nil))
	if w.Code != http.StatusOK {
		t.Fatalf("status = %d", w.Code)
	}
	var got HanziResult
	if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
		t.Fatalf("decode: %v", err)
	}
	if got.Word != "跑步" || len(got.Readings) == 0 || got.Readings[0].Pinyin != "pǎobù" {
		t.Fatalf("response = %+v", got)
	}

	// A miss is a 200 with empty lists, like the other two lookups — the tooltip
	// quietly doesn't open rather than showing an error over her writing.
	w = httptest.NewRecorder()
	r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/zzz", nil))
	if w.Code != http.StatusOK {
		t.Fatalf("miss: status = %d, want 200", w.Code)
	}
}