petal/internal/suggestions/doclang_test.go

package suggestions

import (
	"encoding/json"
	"net/http"
	"path/filepath"
	"strings"
	"testing"

	"github.com/go-chi/chi/v5"

	"gitea.parodia.dev/drwily/petal/internal/auth"
	"gitea.parodia.dev/drwily/petal/internal/db"
	"gitea.parodia.dev/drwily/petal/internal/llm"
)

// A monolingual document in either language has to be read as that language, and
// the mixed cases in between are where the whole design lives: one quotation
// must not move a document, and one leftover English line must not hold a
// journal in English.
func TestDocumentLangReadsWholeDocuments(t *testing.T) {
	const ptJournal = "Hoje foi um dia muito bom. Eu gosto de escrever aqui todas as noites. " +
		"A minha irmã também quer aprender. Não sei porque isso é tão difícil para mim."
	const enEssay = "The weather was very cold this morning. I think that the bus was late again. " +
		"She told me about the meeting, but I could not hear what they said."
	const zhJournal = "今天天气很好。我和妹妹一起去公园散步。我们看到很多花。"

	tests := []struct {
		name     string
		text     string
		pairLang string
		prev     string
		want     string
	}{
		{"portuguese journal", ptJournal, "pt-PT", "", docLangPair},
		{"english essay", enEssay, "pt-PT", "", docLangEnglish},
		{"chinese journal", zhJournal, "zh", "", docLangPair},
		{"english essay, zh writer", enEssay, "zh", "", docLangEnglish},

		// One English sentence at the end of a Portuguese journal is the case that
		// motivated the whole phase: the pass must stay in Portuguese.
		{
			"portuguese with one english line",
			ptJournal + " I will write more tomorrow.",
			"pt-PT", "", docLangPair,
		},
		// And the mirror: an English essay quoting a line of Portuguese is still an
		// English essay.
		{
			"english quoting portuguese",
			enEssay + " She wrote: \"Eu não sei o que dizer.\"",
			"pt-PT", "", docLangEnglish,
		},
		// A pair Petal has no test for cannot flip anything. Saying English is what
		// every surface did before this phase.
		{"untested pair", ptJournal, "de", "", docLangEnglish},
		// Nothing to go on holds the previous answer rather than resetting a
		// journal because she cleared it to start again.
		{"emptied portuguese journal", "", "pt-PT", docLangPair, docLangPair},
		{"emptied english essay", "   \n  ", "pt-PT", docLangEnglish, docLangEnglish},
		// Proportion, not presence: a couple of Portuguese words are not a
		// Portuguese document even though readsAsPairLang would label that span.
		{
			"english with a portuguese phrase",
			enEssay + " The sign said pão com manteiga.",
			"pt-PT", "", docLangEnglish,
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			if got := documentLang(tc.text, tc.pairLang, tc.prev); got != tc.want {
				t.Fatalf("documentLang = %q, want %q", got, tc.want)
			}
		})
	}
}

// The band, from both directions. A document sitting inside it keeps whatever it
// was, and that is the point: without it, a bilingual paragraph would alternate
// its cards' language every few keystrokes as she typed across the threshold.
func TestDocumentLangHysteresis(t *testing.T) {
	// Half and half: two Portuguese sentences, two English ones. Inside the band
	// from either side.
	const mixed = "Eu gosto muito de escrever aqui. A minha irmã não sabe porque é difícil. " +
		"The weather was very cold this morning. I think that they said the same thing."

	if got := documentLang(mixed, "pt-PT", docLangEnglish); got != docLangEnglish {
		t.Fatalf("mixed document from english = %q, want it to stay %q", got, docLangEnglish)
	}
	if got := documentLang(mixed, "pt-PT", docLangPair); got != docLangPair {
		t.Fatalf("mixed document from pair = %q, want it to stay %q", got, docLangPair)
	}

	// Above the upper threshold it flips regardless of where it came from; below
	// the lower one it flips back regardless.
	const mostlyPT = "Eu gosto muito de escrever aqui. A minha irmã não sabe porque é difícil. " +
		"Hoje foi um dia bom para mim. Amanhã também quero escrever mais uma coisa. " +
		"I think so too."
	if got := documentLang(mostlyPT, "pt-PT", docLangEnglish); got != docLangPair {
		t.Fatalf("mostly-portuguese from english = %q, want %q", got, docLangPair)
	}
	const mostlyEN = "The weather was very cold this morning. I think that they said the same thing. " +
		"She could not hear what the other people were saying about it. Eu não sei."
	if got := documentLang(mostlyEN, "pt-PT", docLangPair); got != docLangEnglish {
		t.Fatalf("mostly-english from pair = %q, want %q", got, docLangEnglish)
	}
}

// Corroboration: a proportion computed over almost nothing is not evidence. Two
// bare words at 100% must not flip a document, because a flip rewrites every
// card in it.
func TestDocumentLangNeedsCorroboration(t *testing.T) {
	if got := documentLang("Não. Eu.", "pt-PT", docLangEnglish); got != docLangEnglish {
		t.Fatalf("two bare words flipped the document: %q", got)
	}
	if got := documentLang("我。", "zh", docLangEnglish); got != docLangEnglish {
		t.Fatalf("two Han runes flipped the document: %q", got)
	}
}

// The two language decisions are genuinely independent, and only the zh pair can
// prove it today — it is the one pair that can be travelled in both directions.
//
// A Mandarin native practising English who writes Chinese wants Chinese
// corrections explained in Chinese. An English native learning Chinese who writes
// Chinese wants the same Chinese corrections explained in English. Same document,
// same Correct, different Explain.
func TestTargetSeparatesCorrectedFromExplained(t *testing.T) {
	learningEn := targetFor("zh", auth.DirectionLearningEn, docLangPair)
	if learningEn.Correct.Code != "zh" || learningEn.Explain.Code != "zh" {
		t.Fatalf("learning_en on a Chinese document: correct=%s explain=%s", learningEn.Correct.Code, learningEn.Explain.Code)
	}

	learningPair := targetFor("zh", auth.DirectionLearningPair, docLangPair)
	if learningPair.Correct.Code != "zh" {
		t.Fatalf("learner direction changed what gets corrected: %s", learningPair.Correct.Code)
	}
	if learningPair.Explain.Code != "en" {
		t.Fatalf("learner direction explained in %s, want English", learningPair.Explain.Code)
	}

	// An English document is the path every account is on today, in either
	// direction: English corrections, English explanations, her language still on
	// the Ask Petal and translate taps.
	for _, dir := range []string{auth.DirectionLearningEn, auth.DirectionLearningPair} {
		got := targetFor("zh", dir, docLangEnglish)
		if got.Flipped() || got.Explain.Code != "en" {
			t.Fatalf("english document with direction %s: %+v", dir, got)
		}
		if got.Pair.Code != "zh" {
			t.Fatalf("english document lost the writer's pair: %+v", got)
		}
	}
}

// newDirectedServer seeds one writer on a given pair and direction, with a
// document of her own. Like newPairServer, but the direction is the variable.
func newDirectedServer(t *testing.T, client llm.LLMClient, pairLang, direction, text string) (http.Handler, string, *Handler) {
	t.Helper()
	database, err := db.Open(filepath.Join(t.TempDir(), "doclang.db"))
	if err != nil {
		t.Fatalf("open db: %v", err)
	}
	t.Cleanup(func() { database.Close() })

	const userID = "writer-directed"
	if _, err := database.Exec(
		`INSERT INTO users (id, email, display_name, pair_lang, direction) VALUES (?, ?, ?, ?, ?)`,
		userID, "d@example.com", "Writer", pairLang, direction,
	); err != nil {
		t.Fatalf("seed user: %v", err)
	}

	var docID string
	if err := database.QueryRow(
		`INSERT INTO documents (user_id, content_text) VALUES (?, ?) RETURNING id`,
		userID, text,
	).Scan(&docID); err != nil {
		t.Fatalf("seed doc: %v", err)
	}

	h := New(database, client)
	h.Limit = llm.NewRateLimiter(0)
	h.VoiceLimit = llm.NewRateLimiter(0)
	r := chi.NewRouter()
	r.Route("/docs", func(dr chi.Router) { h.RegisterDocRoutes(dr) })
	r.Mount("/suggestions", h.Routes())
	return auth.Middleware(auth.StaticResolver(userID))(r), docID, h
}

const ptDocument = "Hoje foi um dia muito bom. Eu gosto de escrever aqui todas as noites. " +
	"A minha irmã também quer aprender comigo. Não sei porque isso é tão difícil para mim."

// End to end: a Portuguese document reaches the model as a Portuguese
// checkpoint. This is the observed bug from 2026-07-28 — two pt-PT sentences
// drew no cards at all, because Petal was reading them as bad English.
func TestCheckpointFollowsTheDocumentLanguage(t *testing.T) {
	client := &stubClient{response: `{"suggestions":[]}`}
	srv, docID, _ := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)

	if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
		t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
	}
	if !strings.Contains(client.lastPrompt, "European Portuguese") {
		t.Fatalf("checkpoint didn't follow the document into Portuguese:\n%s", client.lastPrompt)
	}
	if strings.Contains(client.lastPrompt, "second language") {
		t.Fatalf("checkpoint kept the ESL framing on a Portuguese document:\n%s", client.lastPrompt)
	}

	// And the voice pass, which had no language argument at all before this phase.
	if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/voice", ""); rec.Code != http.StatusOK {
		t.Fatalf("voice: code=%d body=%s", rec.Code, rec.Body)
	}
	if !strings.Contains(client.lastPrompt, "European Portuguese") {
		t.Fatalf("voice pass didn't follow the document:\n%s", client.lastPrompt)
	}
}

// The verdict is persisted, because hysteresis needs a yesterday.
func TestDocumentLangIsRemembered(t *testing.T) {
	client := &stubClient{response: `{"suggestions":[]}`}
	srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)

	if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
		t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
	}
	var stored string
	if err := h.DB.QueryRow(`SELECT doc_lang FROM documents WHERE id = ?`, docID).Scan(&stored); err != nil {
		t.Fatalf("read doc_lang: %v", err)
	}
	if stored != docLangPair {
		t.Fatalf("doc_lang = %q, want %q", stored, docLangPair)
	}
}

// A document that changes language re-opens every sentence. Without the verdict
// in the chunk salt, the sentences she didn't touch would keep serving cards
// written in the language the document no longer speaks.
func TestLanguageFlipReopensCheckedSentences(t *testing.T) {
	client := &stubClient{response: `{"suggestions":[]}`}
	const enStart = "The weather was very cold this morning."
	srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, enStart)

	if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
		t.Fatalf("first check: code=%d body=%s", rec.Code, rec.Body)
	}
	first := client.calls

	// She rewrites the document in Portuguese, keeping the first sentence.
	setDocText(t, h, docID, enStart+" "+ptDocument)
	if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
		t.Fatalf("second check: code=%d body=%s", rec.Code, rec.Body)
	}
	if client.calls == first {
		t.Fatal("the flipped document was never sent to the model")
	}
	if !strings.Contains(client.lastPrompt, "The weather was very cold") {
		t.Fatalf("the already-checked sentence was not re-opened by the flip:\n%s", client.lastPrompt)
	}
}

// The translate card, pointed the other way. She is writing her journal in
// Portuguese and drops in the one English sentence she knows; Petal renders it
// into Portuguese, and that card is a translation — not a correction to prose
// that was never wrong.
func TestEnglishSpanBecomesATranslateCardInAPortugueseDocument(t *testing.T) {
	const english = "I want to say this but I don't know how to say it."
	// The model volunteers "clarity", as it did for the zh case. Not consulted.
	client := &stubClient{response: `{"suggestions":[
		{"original":"` + english + `","replacement":"Eu quero dizer isto mas não sei como o dizer.","explanation":"Aqui está em português.","type":"clarity"}
	]}`}
	srv, docID, _ := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument+" "+english)

	var out []db.Suggestion
	rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
	if rec.Code != http.StatusOK {
		t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
	}
	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
		t.Fatalf("decode: %v", err)
	}
	if len(out) != 1 {
		t.Fatalf("want 1 card, got %d: %+v", len(out), out)
	}
	if out[0].Type != db.SuggestionTypeTranslate {
		t.Fatalf("card type = %q, want %q", out[0].Type, db.SuggestionTypeTranslate)
	}
}

// And the half that keeps it honest: a genuine Portuguese correction in the same
// document stays a correction. Reading the English-document test backwards would
// have called this a translation, because every Portuguese sentence also "reads
// as English" by that test's deliberately low bar.
func TestPortugueseCorrectionKeepsItsTypeInAPortugueseDocument(t *testing.T) {
	client := &stubClient{response: `{"suggestions":[
		{"original":"Não sei porque isso é tão difícil para mim.","replacement":"Não sei porque isto é tão difícil para mim.","explanation":"Aqui usa-se isto.","type":"grammar"}
	]}`}
	srv, docID, _ := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)

	var out []db.Suggestion
	rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
	if rec.Code != http.StatusOK {
		t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
	}
	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
		t.Fatalf("decode: %v", err)
	}
	if len(out) != 1 {
		t.Fatalf("want 1 card, got %d: %+v", len(out), out)
	}
	if out[0].Type == db.SuggestionTypeTranslate {
		t.Fatal("a Portuguese correction inside a Portuguese document was labelled a translation")
	}
}

// setDocLang writes a document's language verdict directly, so a test of the
// tap-through doesn't have to run a checkpoint through the same stub client to
// get one.
func setDocLang(t *testing.T, h *Handler, docID, lang string) {
	t.Helper()
	if _, err := h.DB.Exec(`UPDATE documents SET doc_lang = ? WHERE id = ?`, lang, docID); err != nil {
		t.Fatalf("set doc_lang: %v", err)
	}
}

// seedExplanation files one card carrying a given explanation and returns its
// id — the shape the translate tap-through needs, where only the explanation and
// the document it hangs off matter.
func seedExplanation(t *testing.T, h *Handler, docID, explanation string) string {
	t.Helper()
	var sugID string
	if err := h.DB.QueryRow(
		`INSERT INTO suggestions (doc_id, original, replacement, explanation, type, from_pos, to_pos)
		 VALUES (?, ?, ?, ?, ?, 0, 5) RETURNING id`,
		docID, "isso", "isto", explanation, "grammar",
	).Scan(&sugID); err != nil {
		t.Fatalf("seed suggestion: %v", err)
	}
	return sugID
}

// The tap-through has to read the same decision the card was written under. On a
// Portuguese document by a Portuguese writer the explanation already arrived in
// Portuguese, and the old endpoint would have sent it to the model to be
// rendered into Portuguese again.
func TestTranslateSkipsWhenTheExplanationIsAlreadyHers(t *testing.T) {
	client := &stubClient{response: "Não devia ser chamado."}
	srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)
	setDocLang(t, h, docID, docLangPair)
	sugID := seedExplanation(t, h, docID, "Aqui usa-se isto.")

	rec := do(t, srv, http.MethodPost, "/suggestions/"+sugID+"/translate", "")
	if rec.Code != http.StatusOK {
		t.Fatalf("translate: code=%d body=%s", rec.Code, rec.Body)
	}
	var out translateResponse
	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
		t.Fatalf("decode: %v", err)
	}
	if out.Translation != "" {
		t.Fatalf("translation = %q, want empty: the bubble seeds from the explanation itself", out.Translation)
	}
	if client.calls != 0 {
		t.Fatal("the model was asked to render Portuguese into Portuguese")
	}
}

// The learner travelling the other way is the case that proves the endpoint
// derives its destination rather than skipping whenever a document is flipped: a
// native English speaker learning Chinese, writing Chinese, gets her
// explanations in English — and the tap still has somewhere to go.
func TestTranslateStillRendersForALearnersEnglishExplanation(t *testing.T) {
	const zhDocument = "今天天气很好。我早上去公园散步。下午我在家里写作业。晚上我和朋友一起吃饭。"
	client := &stubClient{response: "这里应该用这个。"}
	srv, docID, h := newDirectedServer(t, client, "zh", auth.DirectionLearningPair, zhDocument)
	setDocLang(t, h, docID, docLangPair)
	sugID := seedExplanation(t, h, docID, "This measure word doesn't fit here.")

	rec := do(t, srv, http.MethodPost, "/suggestions/"+sugID+"/translate", "")
	if rec.Code != http.StatusOK {
		t.Fatalf("translate: code=%d body=%s", rec.Code, rec.Body)
	}
	var out translateResponse
	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
		t.Fatalf("decode: %v", err)
	}
	if out.Translation == "" {
		t.Fatal("a learner's English explanation was left untranslated")
	}
	if !strings.Contains(client.lastPrompt, "Simplified Chinese") {
		t.Fatalf("translate didn't render into the pair language:\n%s", client.lastPrompt)
	}
}

// TestPassAnnouncesItsVerdict pins the header the client reads. Storing the
// verdict on the document row is not enough on its own: the editor sees that row
// only when the document is opened or saved, and the pass that decides the
// verdict runs *after* a save — so the client would always be one save behind,
// and read-aloud is reached for exactly when she has stopped typing and no
// further save is coming. Caught in a browser: a Portuguese paragraph read in an
// American voice, twice, until another keystroke went in.
//
// Asserted on both endpoints that can flip it, and on the empty-document early
// return, which answers without ever reaching the model.
func TestPassAnnouncesItsVerdict(t *testing.T) {
	client := &stubClient{response: `{"suggestions":[]}`}
	srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)

	for _, path := range []string{"/check", "/voice"} {
		rec := do(t, srv, http.MethodPost, "/docs/"+docID+path, "")
		if rec.Code != http.StatusOK {
			t.Fatalf("%s: code=%d body=%s", path, rec.Code, rec.Body)
		}
		if got := rec.Header().Get("X-Petal-Doc-Lang"); got != docLangPair {
			t.Fatalf("%s: X-Petal-Doc-Lang = %q, want %q", path, got, docLangPair)
		}
	}

	// An English document says so rather than saying nothing — the client has to
	// be able to hear a flip back, not just a flip away.
	if _, err := h.DB.Exec(
		`UPDATE documents SET content_text = ?, doc_lang = '' WHERE id = ?`,
		"The weather was very cold this morning. I walked to the shop and bought some bread.", docID,
	); err != nil {
		t.Fatalf("rewrite doc: %v", err)
	}
	rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
	if rec.Code != http.StatusOK {
		t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
	}
	if got := rec.Header().Get("X-Petal-Doc-Lang"); got != docLangEnglish {
		t.Fatalf("English document: X-Petal-Doc-Lang = %q, want %q", got, docLangEnglish)
	}

	// The empty-document path returns before the model call, and still answers.
	if _, err := h.DB.Exec(`UPDATE documents SET content_text = '' WHERE id = ?`, docID); err != nil {
		t.Fatalf("empty doc: %v", err)
	}
	rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
	if rec.Code != http.StatusOK {
		t.Fatalf("empty check: code=%d body=%s", rec.Code, rec.Body)
	}
	if got := rec.Header().Get("X-Petal-Doc-Lang"); got == "" {
		t.Fatal("empty document answered with no verdict header at all")
	}
}

// TestOrdinaryProseIsEnoughEvidence is the regression for what the marker lists
// were caught doing on 2026-07-29, live, in a browser: unremarkable Portuguese
// read as English, because the list was curated against English so tightly that
// it had also been curated against ordinary writing. The document below scored
// two pair markers and zero English ones, and two is below the corroboration
// floor — so a paragraph with no evidence of English in it at all came back
// English, and was corrected and read aloud as English.
//
// Every sample here is prose a person might actually write, not prose chosen to
// contain markers. That is the whole point of the test: the failure was invisible
// to a suite whose fixtures all argued their own case.
func TestOrdinaryProseIsEnoughEvidence(t *testing.T) {
	samples := []struct{ name, text string }{
		{"the one seen live", "Esta manhã acordei cedo e fui correr ao longo da marginal. O ar estava fresco e havia poucas pessoas na rua. Depois comprei um jornal e li-o sentado num banco ao sol."},
		{"an afternoon out", "Hoje o céu estava limpo e fomos até ao jardim junto ao rio. A minha mãe trouxe uma manta velha e sentámos-nos debaixo de uma árvore."},
		{"plans", "Amanhã vamos ao cinema depois do trabalho. Ontem estava demasiado cansada para sair de casa."},
	}
	for _, s := range samples {
		if got := documentLang(s.text, "pt-PT", ""); got != docLangPair {
			t.Errorf("%s: documentLang = %q, want %q — ordinary Portuguese must not read as English\n%s",
				s.name, got, docLangPair, s.text)
		}
	}
}

// TestEnglishDidNotGetEasierToMistake is the other half, and the reason the
// additions were held to "a word an English sentence has no reason to contain".
// Widening a marker list is only safe if it widens in one direction: these are
// English documents, including ones about Portugal and ones quoting Portuguese,
// and every one of them must still come back English.
func TestEnglishDidNotGetEasierToMistake(t *testing.T) {
	samples := []struct{ name, text string }{
		{"plain English", "This morning I woke up early and went for a run along the seafront. The air was fresh and there were few people about. Afterwards I bought a newspaper and read it on a bench."},
		{"English about Portugal", "We spent a week in Lisbon last summer. The trams were crowded but the food was wonderful, and we walked up to the castle every evening."},
		{"English quoting her", "My mother always says \"até amanhã\" when she leaves, never goodbye. I asked her why once and she said it sounded less final to her."},
		{"an English diary", "Today was long. I had two meetings before lunch and another one after, and by the time I got home I could not think straight. Tomorrow should be quieter."},
	}
	for _, s := range samples {
		if got := documentLang(s.text, "pt-PT", ""); got != docLangEnglish {
			t.Errorf("%s: documentLang = %q, want %q — the widened list must not pull English across\n%s",
				s.name, got, docLangEnglish, s.text)
		}
	}
	// And the same document must not flip once it is already sitting in English:
	// the hysteresis band is only a safety net if the low side holds too.
	for _, s := range samples {
		if got := documentLang(s.text, "pt-PT", docLangEnglish); got != docLangEnglish {
			t.Errorf("%s: held verdict flipped to %q", s.name, got)
		}
	}
}