package suggestions import ( "net/http" "path/filepath" "strings" "testing" "github.com/go-chi/chi/v5" "gitea.parodia.dev/drwily/petal/internal/auth" "gitea.parodia.dev/drwily/petal/internal/db" "gitea.parodia.dev/drwily/petal/internal/llm" ) // A monolingual document in either language has to be read as that language, and // the mixed cases in between are where the whole design lives: one quotation // must not move a document, and one leftover English line must not hold a // journal in English. func TestDocumentLangReadsWholeDocuments(t *testing.T) { const ptJournal = "Hoje foi um dia muito bom. Eu gosto de escrever aqui todas as noites. " + "A minha irmã também quer aprender. Não sei porque isso é tão difícil para mim." const enEssay = "The weather was very cold this morning. I think that the bus was late again. " + "She told me about the meeting, but I could not hear what they said." const zhJournal = "今天天气很好。我和妹妹一起去公园散步。我们看到很多花。" tests := []struct { name string text string pairLang string prev string want string }{ {"portuguese journal", ptJournal, "pt-PT", "", docLangPair}, {"english essay", enEssay, "pt-PT", "", docLangEnglish}, {"chinese journal", zhJournal, "zh", "", docLangPair}, {"english essay, zh writer", enEssay, "zh", "", docLangEnglish}, // One English sentence at the end of a Portuguese journal is the case that // motivated the whole phase: the pass must stay in Portuguese. { "portuguese with one english line", ptJournal + " I will write more tomorrow.", "pt-PT", "", docLangPair, }, // And the mirror: an English essay quoting a line of Portuguese is still an // English essay. { "english quoting portuguese", enEssay + " She wrote: \"Eu não sei o que dizer.\"", "pt-PT", "", docLangEnglish, }, // A pair Petal has no test for cannot flip anything. Saying English is what // every surface did before this phase. {"untested pair", ptJournal, "de", "", docLangEnglish}, // Nothing to go on holds the previous answer rather than resetting a // journal because she cleared it to start again. {"emptied portuguese journal", "", "pt-PT", docLangPair, docLangPair}, {"emptied english essay", " \n ", "pt-PT", docLangEnglish, docLangEnglish}, // Proportion, not presence: a couple of Portuguese words are not a // Portuguese document even though readsAsPairLang would label that span. { "english with a portuguese phrase", enEssay + " The sign said pão com manteiga.", "pt-PT", "", docLangEnglish, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { if got := documentLang(tc.text, tc.pairLang, tc.prev); got != tc.want { t.Fatalf("documentLang = %q, want %q", got, tc.want) } }) } } // The band, from both directions. A document sitting inside it keeps whatever it // was, and that is the point: without it, a bilingual paragraph would alternate // its cards' language every few keystrokes as she typed across the threshold. func TestDocumentLangHysteresis(t *testing.T) { // Half and half: two Portuguese sentences, two English ones. Inside the band // from either side. const mixed = "Eu gosto muito de escrever aqui. A minha irmã não sabe porque é difícil. " + "The weather was very cold this morning. I think that they said the same thing." if got := documentLang(mixed, "pt-PT", docLangEnglish); got != docLangEnglish { t.Fatalf("mixed document from english = %q, want it to stay %q", got, docLangEnglish) } if got := documentLang(mixed, "pt-PT", docLangPair); got != docLangPair { t.Fatalf("mixed document from pair = %q, want it to stay %q", got, docLangPair) } // Above the upper threshold it flips regardless of where it came from; below // the lower one it flips back regardless. const mostlyPT = "Eu gosto muito de escrever aqui. A minha irmã não sabe porque é difícil. " + "Hoje foi um dia bom para mim. Amanhã também quero escrever mais uma coisa. " + "I think so too." if got := documentLang(mostlyPT, "pt-PT", docLangEnglish); got != docLangPair { t.Fatalf("mostly-portuguese from english = %q, want %q", got, docLangPair) } const mostlyEN = "The weather was very cold this morning. I think that they said the same thing. " + "She could not hear what the other people were saying about it. Eu não sei." if got := documentLang(mostlyEN, "pt-PT", docLangPair); got != docLangEnglish { t.Fatalf("mostly-english from pair = %q, want %q", got, docLangEnglish) } } // Corroboration: a proportion computed over almost nothing is not evidence. Two // bare words at 100% must not flip a document, because a flip rewrites every // card in it. func TestDocumentLangNeedsCorroboration(t *testing.T) { if got := documentLang("Não. Eu.", "pt-PT", docLangEnglish); got != docLangEnglish { t.Fatalf("two bare words flipped the document: %q", got) } if got := documentLang("我。", "zh", docLangEnglish); got != docLangEnglish { t.Fatalf("two Han runes flipped the document: %q", got) } } // The two language decisions are genuinely independent, and only the zh pair can // prove it today — it is the one pair that can be travelled in both directions. // // A Mandarin native practising English who writes Chinese wants Chinese // corrections explained in Chinese. An English native learning Chinese who writes // Chinese wants the same Chinese corrections explained in English. Same document, // same Correct, different Explain. func TestTargetSeparatesCorrectedFromExplained(t *testing.T) { learningEn := targetFor("zh", auth.DirectionLearningEn, docLangPair) if learningEn.Correct.Code != "zh" || learningEn.Explain.Code != "zh" { t.Fatalf("learning_en on a Chinese document: correct=%s explain=%s", learningEn.Correct.Code, learningEn.Explain.Code) } learningPair := targetFor("zh", auth.DirectionLearningPair, docLangPair) if learningPair.Correct.Code != "zh" { t.Fatalf("learner direction changed what gets corrected: %s", learningPair.Correct.Code) } if learningPair.Explain.Code != "en" { t.Fatalf("learner direction explained in %s, want English", learningPair.Explain.Code) } // An English document is the path every account is on today, in either // direction: English corrections, English explanations, her language still on // the Ask Petal and translate taps. for _, dir := range []string{auth.DirectionLearningEn, auth.DirectionLearningPair} { got := targetFor("zh", dir, docLangEnglish) if got.Flipped() || got.Explain.Code != "en" { t.Fatalf("english document with direction %s: %+v", dir, got) } if got.Pair.Code != "zh" { t.Fatalf("english document lost the writer's pair: %+v", got) } } } // newDirectedServer seeds one writer on a given pair and direction, with a // document of her own. Like newPairServer, but the direction is the variable. func newDirectedServer(t *testing.T, client llm.LLMClient, pairLang, direction, text string) (http.Handler, string, *Handler) { t.Helper() database, err := db.Open(filepath.Join(t.TempDir(), "doclang.db")) if err != nil { t.Fatalf("open db: %v", err) } t.Cleanup(func() { database.Close() }) const userID = "writer-directed" if _, err := database.Exec( `INSERT INTO users (id, email, display_name, pair_lang, direction) VALUES (?, ?, ?, ?, ?)`, userID, "d@example.com", "Writer", pairLang, direction, ); err != nil { t.Fatalf("seed user: %v", err) } var docID string if err := database.QueryRow( `INSERT INTO documents (user_id, content_text) VALUES (?, ?) RETURNING id`, userID, text, ).Scan(&docID); err != nil { t.Fatalf("seed doc: %v", err) } h := New(database, client) h.Limit = llm.NewRateLimiter(0) h.VoiceLimit = llm.NewRateLimiter(0) r := chi.NewRouter() r.Route("/docs", func(dr chi.Router) { h.RegisterDocRoutes(dr) }) r.Mount("/suggestions", h.Routes()) return auth.Middleware(auth.StaticResolver(userID))(r), docID, h } const ptDocument = "Hoje foi um dia muito bom. Eu gosto de escrever aqui todas as noites. " + "A minha irmã também quer aprender comigo. Não sei porque isso é tão difícil para mim." // End to end: a Portuguese document reaches the model as a Portuguese // checkpoint. This is the observed bug from 2026-07-28 — two pt-PT sentences // drew no cards at all, because Petal was reading them as bad English. func TestCheckpointFollowsTheDocumentLanguage(t *testing.T) { client := &stubClient{response: `{"suggestions":[]}`} srv, docID, _ := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument) if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK { t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body) } if !strings.Contains(client.lastPrompt, "European Portuguese") { t.Fatalf("checkpoint didn't follow the document into Portuguese:\n%s", client.lastPrompt) } if strings.Contains(client.lastPrompt, "second language") { t.Fatalf("checkpoint kept the ESL framing on a Portuguese document:\n%s", client.lastPrompt) } // And the voice pass, which had no language argument at all before this phase. if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/voice", ""); rec.Code != http.StatusOK { t.Fatalf("voice: code=%d body=%s", rec.Code, rec.Body) } if !strings.Contains(client.lastPrompt, "European Portuguese") { t.Fatalf("voice pass didn't follow the document:\n%s", client.lastPrompt) } } // The verdict is persisted, because hysteresis needs a yesterday. func TestDocumentLangIsRemembered(t *testing.T) { client := &stubClient{response: `{"suggestions":[]}`} srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument) if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK { t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body) } var stored string if err := h.DB.QueryRow(`SELECT doc_lang FROM documents WHERE id = ?`, docID).Scan(&stored); err != nil { t.Fatalf("read doc_lang: %v", err) } if stored != docLangPair { t.Fatalf("doc_lang = %q, want %q", stored, docLangPair) } } // A document that changes language re-opens every sentence. Without the verdict // in the chunk salt, the sentences she didn't touch would keep serving cards // written in the language the document no longer speaks. func TestLanguageFlipReopensCheckedSentences(t *testing.T) { client := &stubClient{response: `{"suggestions":[]}`} const enStart = "The weather was very cold this morning." srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, enStart) if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK { t.Fatalf("first check: code=%d body=%s", rec.Code, rec.Body) } first := client.calls // She rewrites the document in Portuguese, keeping the first sentence. setDocText(t, h, docID, enStart+" "+ptDocument) if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK { t.Fatalf("second check: code=%d body=%s", rec.Code, rec.Body) } if client.calls == first { t.Fatal("the flipped document was never sent to the model") } if !strings.Contains(client.lastPrompt, "The weather was very cold") { t.Fatalf("the already-checked sentence was not re-opened by the flip:\n%s", client.lastPrompt) } }