petal/internal/lexicon/dreamdict_test.go

package lexicon

import (
	"database/sql"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"unicode/utf8"

	"github.com/go-chi/chi/v5"
	"github.com/prosolis/dreamdict/dictionary"
	_ "modernc.org/sqlite"

	"gitea.parodia.dev/drwily/petal/internal/auth"
	"gitea.parodia.dev/drwily/petal/internal/db"
)

// The fixture is a real dict.db on disk rather than an in-memory handle, so
// these tests exercise the path production takes: stat the file, open it
// read-only, find it seeded. A fake would have skipped every one of those.
//
// "ephemeral" is the worked example throughout: it has definitions only under
// its own headword, translations into two languages, IPA alongside a CMU
// pronunciation Petal must not show, a frequency, a difficulty and an etymology
// long enough to need trimming.
func writeFixture(t *testing.T, seeded bool) string {
	t.Helper()
	path := filepath.Join(t.TempDir(), "dict.db")
	sqldb, err := sql.Open("sqlite", path)
	if err != nil {
		t.Fatalf("open fixture: %v", err)
	}
	defer sqldb.Close()
	if err := dictionary.BootstrapSchema(sqldb); err != nil {
		t.Fatalf("bootstrap: %v", err)
	}
	if !seeded {
		return path
	}

	exec := func(q string, args ...any) {
		t.Helper()
		if _, err := sqldb.Exec(q, args...); err != nil {
			t.Fatalf("seed %q: %v", q, err)
		}
	}
	exec(`INSERT INTO meta (key, value) VALUES ('schema_version', '2')`)

	exec(`INSERT INTO words (id, word, lang, pos, frequency, difficulty) VALUES
		(1, 'ephemeral', 'en', 'adjective', 50, 0.72),
		(2, 'run',       'en', 'verb',     900, 0.05),
		(3, 'plain',     'en', 'adjective',  0, NULL),
		(4, 'efémero',   'pt-PT', 'adjective', 12, 0.6)`)

	exec(`INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES
		(1, 'adjective', 'lasting a very short time', 'wordnet',    10),
		(1, 'adjective', 'short-lived',                'wiktionary', 99),
		(1, 'adjective', 'transitory',                 'wiktionary', 99),
		(1, 'adjective', 'fleeting',                   'wiktionary', 99),
		(1, 'adjective', 'evanescent',                 'wiktionary', 99),
		(2, 'verb',      'move fast on foot',          'wordnet',    10),
		(3, 'adjective', 'without decoration',         'wordnet',    10)`)

	exec(`INSERT INTO synonyms (word_id, synonym, source) VALUES
		(1, 'fleeting', 'wordnet'), (1, 'transient', 'wordnet'),
		(2, 'sprint', 'wordnet')`)

	exec(`INSERT INTO translations (word_id, translation, target_lang, source) VALUES
		(1, 'efémero',  'pt-PT', 'kaikki'),
		(1, 'passageiro','pt-PT', 'kaikki'),
		(1, 'éphémère', 'fr',    'kaikki'),
		(1, '短暂的',    'zh',    'cedict'),
		(2, 'correr',   'pt-PT', 'kaikki')`)

	// CMU is listed first deliberately: picking the first row would show a
	// learner "IH0 F EH1 M ER0 AH0 L", which is a machine format, not help.
	exec(`INSERT INTO pronunciations (word_id, format, value, source) VALUES
		(1, 'cmu', 'IH0 F EH1 M ER0 AH0 L', 'cmudict'),
		(1, 'ipa', '/ɪˈfɛm.ər.əl/',          'wiktionary')`)

	// "brief" carries no translation row at all — only a shared WordNet synset
	// with two pt-PT words. On the real database that is the *usual* case, not
	// the exotic one, so Petal must reach a gloss this way or the pt-PT pair
	// has almost no glosses. "breve" is the commoner of the two and leads.
	exec(`INSERT INTO words (id, word, lang, pos, frequency) VALUES
		(5, 'brief',   'en',    'adjective', 400),
		(6, 'breve',   'pt-PT', 'adjective', 300),
		(7, 'sucinto', 'pt-PT', 'adjective',  20)`)
	exec(`INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES
		(5, 'adjective', 'of short duration', 'wordnet', 10)`)
	exec(`INSERT INTO synsets (id, synset_id, pos) VALUES (1, '00751145-a', 'adjective')`)
	exec(`INSERT INTO word_synsets (word_id, synset_id, source) VALUES
		(5, 1, 'wordnet'), (6, 1, 'omw'), (7, 1, 'omw')`)

	exec(`INSERT INTO etymology (word_id, text, source) VALUES
		(1, 'From Medieval Latin ephemerus, from Ancient Greek ἐφήμερος (ephḗmeros, "lasting only a day"), from ἐπί (epí, "upon") and ἡμέρα (hēméra, "day"). The sense of transience is attested in English from the late sixteenth century onwards.', 'wiktionary')`)

	return path
}

func openFixture(t *testing.T) *DreamDict {
	t.Helper()
	dd, err := OpenDreamDict(writeFixture(t, true))
	if err != nil {
		t.Fatalf("OpenDreamDict: %v", err)
	}
	if dd == nil {
		t.Fatal("OpenDreamDict returned no dictionary for a seeded file")
	}
	t.Cleanup(func() { dd.Close() })
	return dd
}

func TestOpenMissingFileIsNotAnError(t *testing.T) {
	dd, err := OpenDreamDict(filepath.Join(t.TempDir(), "absent.db"))
	if err != nil {
		t.Fatalf("a missing dict.db must not be an error: %v", err)
	}
	if dd != nil {
		t.Fatal("a missing dict.db must yield no dictionary")
	}
	// An unset path is the laptop default and must behave the same way.
	if dd, err := OpenDreamDict(""); err != nil || dd != nil {
		t.Fatalf(`OpenDreamDict("") = %v, %v; want nil, nil`, dd, err)
	}
	// Close on the nil handle is what main.go defers unconditionally.
	if err := dd.Close(); err != nil {
		t.Fatalf("Close on absent dictionary: %v", err)
	}
}

func TestOpenPresentButUnseededIsAnError(t *testing.T) {
	// A file that exists but was never imported is somebody's mistake — a
	// half-finished deploy — and must be loud, unlike a file that isn't there.
	dd, err := OpenDreamDict(writeFixture(t, false))
	if err == nil {
		dd.Close()
		t.Fatal("an unseeded dict.db must report an error")
	}
	if dd != nil {
		t.Fatal("an unseeded dict.db must not yield a usable dictionary")
	}
}

func TestOpenUnreadablePathIsAnError(t *testing.T) {
	// Not a missing file: a directory where dict.db should be. Distinguishing
	// this from ErrNotExist is the whole point of the stat.
	dir := filepath.Join(t.TempDir(), "dict.db")
	if err := os.Mkdir(dir, 0o755); err != nil {
		t.Fatal(err)
	}
	if _, err := OpenDreamDict(dir); err == nil {
		t.Fatal("a directory in place of dict.db must report an error")
	}
}

func TestDreamLookupFillsEveryField(t *testing.T) {
	p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
	res, err := p.Lookup("ephemeral")
	if err != nil {
		t.Fatalf("Lookup: %v", err)
	}
	if res.Gloss != "efémero; passageiro" {
		t.Errorf("Gloss = %q, want the pt-PT translations joined", res.Gloss)
	}
	if res.Phonetic != "ɪˈfɛm.ər.əl" {
		t.Errorf("Phonetic = %q, want the IPA without its slashes", res.Phonetic)
	}
	if len(res.Definitions) != maxDefinitions {
		t.Fatalf("Definitions = %d, want them capped at %d", len(res.Definitions), maxDefinitions)
	}
	if res.Definitions[0].Definition != "lasting a very short time" {
		t.Errorf("first definition = %q, want the curated (wordnet) sense first",
			res.Definitions[0].Definition)
	}
	if res.Definitions[0].PartOfSpeech != "adjective" {
		t.Errorf("part of speech = %q, want adjective", res.Definitions[0].PartOfSpeech)
	}
	if len(res.Synonyms) != 2 {
		t.Errorf("Synonyms = %v, want both", res.Synonyms)
	}
	if res.Frequency != 50 {
		t.Errorf("Frequency = %d, want 50", res.Frequency)
	}
	if res.Difficulty != 0.72 {
		t.Errorf("Difficulty = %v, want 0.72", res.Difficulty)
	}
	if !strings.HasPrefix(res.Etymology, "From Medieval Latin ephemerus") {
		t.Errorf("Etymology = %q, want the Wiktionary text", res.Etymology)
	}
	if len(res.Etymology) > maxEtymology {
		t.Errorf("Etymology not trimmed: %d chars", len(res.Etymology))
	}
}

func TestDreamGlossFollowsTheWriterNotTheWord(t *testing.T) {
	dd := openFixture(t)
	for lang, want := range map[string]string{
		"pt-PT": "efémero; passageiro",
		"fr":    "éphémère",
		"es":    "", // DreamDict supports Spanish; this database wasn't built with it
		"de":    "", // never a Petal pair, and must not silently borrow another's
	} {
		got, err := dreamProvider{dict: dd, native: lang}.Gloss("ephemeral")
		if err != nil {
			t.Fatalf("Gloss(%s): %v", lang, err)
		}
		if got.Gloss != want {
			t.Errorf("Gloss for %s = %q, want %q", lang, got.Gloss, want)
		}
		if got.Word != "ephemeral" {
			t.Errorf("Word = %q, want the word as asked", got.Word)
		}
	}
}

func TestDreamGlossesThroughSharedSynsets(t *testing.T) {
	// The measurement that drove this: on the real dict.db, Wiktionary's
	// en→pt-PT translation table answers for 17% of the 2,000 commonest English
	// words and the shared-synset path answers for 62%. A word with no
	// translation row must still get a gloss, commonest sense first.
	p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
	res, err := p.Lookup("brief")
	if err != nil {
		t.Fatalf("Lookup: %v", err)
	}
	if res.Gloss != "breve; sucinto" {
		t.Errorf("Gloss = %q, want the synset equivalents, commonest first", res.Gloss)
	}
}

func TestDreamDeinflectsToTheHeadword(t *testing.T) {
	// dict.db stores headwords: "running" has no row of its own. The candidate
	// walk is what makes a right-click on real prose work at all.
	p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
	res, err := p.Lookup("running")
	if err != nil {
		t.Fatalf("Lookup: %v", err)
	}
	if len(res.Definitions) == 0 || res.Definitions[0].Definition != "move fast on foot" {
		t.Fatalf("Definitions = %+v, want run's", res.Definitions)
	}
	// Every other field must come from the same headword — a popover that mixed
	// "running"'s (absent) frequency with "run"'s definitions would be lying.
	if res.Frequency != 900 {
		t.Errorf("Frequency = %d, want run's 900", res.Frequency)
	}
	if res.Difficulty != 0.05 {
		t.Errorf("Difficulty = %v, want run's 0.05", res.Difficulty)
	}
	if res.Synonyms[0] != "sprint" {
		t.Errorf("Synonyms = %v, want run's", res.Synonyms)
	}
	if res.Gloss != "correr" {
		t.Errorf("Gloss = %q, want run's", res.Gloss)
	}
}

func TestDreamMissIsAnEmptyResultNotAnError(t *testing.T) {
	p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
	res, err := p.Lookup("zzzxqqq")
	if err != nil {
		t.Fatalf("Lookup: %v", err)
	}
	if len(res.Definitions) != 0 || len(res.Synonyms) != 0 || res.Gloss != "" {
		t.Errorf("expected an empty result, got %+v", res)
	}
	// The frontend renders [] and never null.
	if res.Definitions == nil || res.Synonyms == nil {
		t.Errorf("empty slices must be non-nil: %+v", res)
	}
	if res.Difficulty != unknownDifficulty {
		t.Errorf("Difficulty = %v, want the unknown sentinel", res.Difficulty)
	}
	// Empty input is a miss, not a crash.
	if res, err := p.Lookup("   "); err != nil || res.Gloss != "" {
		t.Errorf("Lookup(blank) = %+v, %v", res, err)
	}
	if res, err := p.Gloss(""); err != nil || res.Gloss != "" {
		t.Errorf("Gloss(empty) = %+v, %v", res, err)
	}
}

func TestDreamUnscoredWordKeepsTheUnknownSentinel(t *testing.T) {
	// "plain" is in the database with no frequency and a NULL difficulty. The
	// popover must be able to tell that apart from "difficulty 0.0, the easiest
	// word there is" — which is why the sentinel is -1 and not omitempty.
	p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
	res, err := p.Lookup("plain")
	if err != nil {
		t.Fatalf("Lookup: %v", err)
	}
	if len(res.Definitions) == 0 {
		t.Fatal("expected plain to be found")
	}
	if res.Difficulty != unknownDifficulty {
		t.Errorf("Difficulty = %v, want the unknown sentinel for a NULL score", res.Difficulty)
	}
	if res.Frequency != 0 {
		t.Errorf("Frequency = %d, want 0 for no count", res.Frequency)
	}
}

func TestPickIPASkipsMachineFormats(t *testing.T) {
	if got := pickIPA([]dictionary.Pronunciation{{Format: "cmu", Value: "K AE1 T"}}); got != "" {
		t.Errorf("pickIPA on CMU alone = %q, want empty — CMU is not for a reader", got)
	}
	if got := pickIPA(nil); got != "" {
		t.Errorf("pickIPA(nil) = %q", got)
	}
	if got := pickIPA([]dictionary.Pronunciation{{Format: "IPA", Value: " [kæt] "}}); got != "kæt" {
		t.Errorf("pickIPA = %q, want the bare IPA regardless of case or brackets", got)
	}
}

func TestTrimEtymologyPrefersASentence(t *testing.T) {
	// A sentence that ends past halfway is the good cut: keep it, drop the rest.
	long := strings.Repeat("padding word ", 14) + "end. " + strings.Repeat("more ", 40)
	got := trimEtymology(long)
	if utf8.RuneCountInString(got) > maxEtymology {
		t.Errorf("not trimmed: %d runes", utf8.RuneCountInString(got))
	}
	if !strings.HasSuffix(got, "end.") {
		t.Errorf("trimEtymology = %q, want it to stop at the sentence", got)
	}

	// An early full stop is not a summary — cutting there would throw away
	// almost the whole line — so this falls through to a word boundary.
	got = trimEtymology("From Latin. " + strings.Repeat("padding word ", 40))
	if strings.HasSuffix(got, "Latin.") {
		t.Errorf("trimEtymology = %q, want more than the first four words", got)
	}
	if !strings.HasSuffix(got, "…") {
		t.Errorf("trimEtymology = %q, want an ellipsis when cut mid-thought", got)
	}

	// Multi-byte text must be cut on rune boundaries: a byte slice through
	// ἐφήμερος would put invalid UTF-8 in the JSON.
	greek := trimEtymology(strings.Repeat("ἐφήμερος ", 60))
	if !utf8.ValidString(greek) {
		t.Errorf("trimEtymology produced invalid UTF-8: %q", greek)
	}
	if n := utf8.RuneCountInString(greek); n > maxEtymology {
		t.Errorf("trimmed to %d runes, want at most %d", n, maxEtymology)
	}

	if strings.Contains(got, "  ") || strings.Contains(trimEtymology("a\n  b"), "\n") {
		t.Error("whitespace should be collapsed to a single line")
	}
	// Short text passes through untouched.
	if got := trimEtymology("From Old English."); got != "From Old English." {
		t.Errorf("trimEtymology = %q", got)
	}
}

// --- the Set: which provider answers, and what happens when dict.db is absent

func TestSetRoutesByPairLanguage(t *testing.T) {
	set := NewSet(openFixture(t))
	if !set.HasDreamDict() {
		t.Fatal("HasDreamDict = false with a dictionary open")
	}
	// zh — and an empty column, which is what a pre-auth row reads as — stays
	// on the embedded datasets until the two have been compared on real
	// lookups. This test is the guard on that decision.
	for _, lang := range []string{"", LangZh} {
		if _, ok := set.For(lang).(*Lexicon); !ok {
			t.Errorf("For(%q) = %T, want the embedded Lexicon", lang, set.For(lang))
		}
	}
	for _, lang := range []string{"pt-PT", "fr", "es"} {
		p, ok := set.For(lang).(dreamProvider)
		if !ok {
			t.Fatalf("For(%q) = %T, want DreamDict", lang, set.For(lang))
		}
		if p.native != lang {
			t.Errorf("For(%q) glosses into %q", lang, p.native)
		}
	}
}

func TestSetWithoutDictKeepsTheEnglishHalf(t *testing.T) {
	// The interesting degradation: dict.db never got deployed. A pt-PT writer
	// should still get definitions, synonyms and phonetics — all compiled into
	// the binary and all correct for her — and lose only the translation.
	set := NewSet(nil)
	if set.HasDreamDict() {
		t.Fatal("HasDreamDict = true with no dictionary")
	}
	p := set.For("pt-PT")
	res, err := p.Lookup("happy")
	if err != nil {
		t.Fatalf("Lookup: %v", err)
	}
	if len(res.Definitions) == 0 || len(res.Synonyms) == 0 {
		t.Error("expected the embedded English half to survive a missing dict.db")
	}
	if res.Gloss != "" {
		t.Errorf("Gloss = %q — a pt-PT writer must never be handed the Chinese gloss", res.Gloss)
	}
	g, err := p.Gloss("happy")
	if err != nil {
		t.Fatalf("Gloss: %v", err)
	}
	if g.Gloss != "" {
		t.Errorf("Gloss = %q, want empty", g.Gloss)
	}
	if g.Word != "happy" {
		t.Errorf("Word = %q, want the word as asked", g.Word)
	}
	// The zh writer is untouched by any of this.
	zh, err := set.For(LangZh).Lookup("happy")
	if err != nil {
		t.Fatalf("Lookup(zh): %v", err)
	}
	if zh.Gloss == "" {
		t.Error("the zh pair must keep its embedded gloss with no dict.db")
	}
}

// --- the handler: the pair language is read per request, from the caller's row

func mountLexicon(t *testing.T, set *Set) (*chi.Mux, *db.DB) {
	t.Helper()
	database, err := db.Open(filepath.Join(t.TempDir(), "petal.db"))
	if err != nil {
		t.Fatalf("db.Open: %v", err)
	}
	t.Cleanup(func() { database.Close() })

	for _, u := range []struct{ id, lang string }{
		{"alice", LangZh}, {"bob", "pt-PT"},
	} {
		if _, err := database.Exec(
			`INSERT INTO users (id, email, display_name, pair_lang) VALUES (?, ?, ?, ?)`,
			u.id, u.id+"@example.com", u.id, u.lang,
		); err != nil {
			t.Fatalf("seed user: %v", err)
		}
	}

	h := NewHandler(database.DB, set)
	r := chi.NewMux()
	r.Mount("/word", h.Routes())
	r.Mount("/gloss", h.GlossRoutes())
	return r, database
}

func getAs(t *testing.T, r http.Handler, userID, path string) Result {
	t.Helper()
	req := httptest.NewRequest(http.MethodGet, path, nil)
	req = req.WithContext(auth.WithUser(req.Context(), userID))
	rec := httptest.NewRecorder()
	r.ServeHTTP(rec, req)
	if rec.Code != http.StatusOK {
		t.Fatalf("GET %s as %s = %d: %s", path, userID, rec.Code, rec.Body)
	}
	var res Result
	if err := json.Unmarshal(rec.Body.Bytes(), &res); err != nil {
		t.Fatalf("decode: %v", err)
	}
	return res
}

func TestHandlerGlossesInTheCallersLanguage(t *testing.T) {
	r, _ := mountLexicon(t, NewSet(openFixture(t)))

	// Same URL, two writers, two languages. This is why the response is no
	// longer cacheable as `public`.
	bob := getAs(t, r, "bob", "/word/ephemeral")
	if bob.Gloss != "efémero; passageiro" {
		t.Errorf("bob's gloss = %q, want pt-PT", bob.Gloss)
	}
	alice := getAs(t, r, "alice", "/word/ephemeral")
	if !strings.ContainsAny(alice.Gloss, "短暂的") && alice.Gloss != "" {
		// alice is on the embedded ECDICT dataset, not the fixture's zh row —
		// what matters is that she is *not* served bob's Portuguese.
		t.Logf("alice's embedded gloss: %q", alice.Gloss)
	}
	if alice.Gloss == bob.Gloss && bob.Gloss != "" {
		t.Error("the zh writer was served the pt-PT gloss")
	}
	if strings.Contains(alice.Gloss, "efémero") {
		t.Errorf("alice's gloss = %q, want the embedded Chinese one", alice.Gloss)
	}
}

func TestHandlerUnknownCallerFallsBackRatherThanFailing(t *testing.T) {
	// No session, or a user row that has gone: the lookup still answers, from
	// the embedded datasets. A dictionary that fails closed would be worse than
	// one that answers in the wrong language, because nothing at all is not a
	// dictionary.
	r, _ := mountLexicon(t, NewSet(openFixture(t)))
	res := getAs(t, r, "nobody", "/word/happy")
	if len(res.Definitions) == 0 {
		t.Error("expected the embedded fallback to answer for an unknown caller")
	}
}

func TestHandlerCachesPrivately(t *testing.T) {
	r, _ := mountLexicon(t, NewSet(nil))
	req := httptest.NewRequest(http.MethodGet, "/gloss/happy", nil)
	req = req.WithContext(auth.WithUser(req.Context(), "alice"))
	rec := httptest.NewRecorder()
	r.ServeHTTP(rec, req)
	if got := rec.Header().Get("Cache-Control"); !strings.HasPrefix(got, "private") {
		t.Errorf("Cache-Control = %q — a per-writer gloss must not go in a shared cache", got)
	}
}

func TestHandlerDecodesPunctuatedWords(t *testing.T) {
	r, _ := mountLexicon(t, NewSet(openFixture(t)))
	res := getAs(t, r, "bob", "/word/"+"caf%C3%A9")
	if res.Word != "café" {
		t.Errorf("Word = %q, want the decoded word", res.Word)
	}
}

func TestContentsCountsRowsNotSupportedLanguages(t *testing.T) {
	// The fixture is seeded with English and pt-PT only. DreamDict *supports*
	// French, Spanish and Chinese too — and a startup line that reported the
	// supported set would have named all five while every French lookup came
	// back empty. That is the failure this log line exists to catch, so it must
	// count rows.
	got := NewSet(openFixture(t))
	summary := got.Contents()
	if !strings.Contains(summary, "en=") || !strings.Contains(summary, "pt-PT=") {
		t.Errorf("Contents = %q, want the languages the fixture actually holds", summary)
	}
	for _, absent := range []string{"fr=", "es=", "zh="} {
		if strings.Contains(summary, absent) {
			t.Errorf("Contents = %q, must not name %q — no rows exist for it", summary, absent)
		}
	}
	// No dictionary at all still has to answer something printable.
	if s := NewSet(nil).Contents(); s == "" {
		t.Error("Contents with no dictionary must still say something")
	}
}