petal/internal/suggestions/language_test.go

package suggestions

import "testing"

// The flagship case, and the ones next to it that must NOT become translations.
//
// Every case here is an ENGLISH document — the path every account was on before
// Phase 28 — so `docLang` is left at "". The mirror image lives in
// TestIsTranslationInAPairLanguageDocument below.
func TestIsTranslation(t *testing.T) {
	cases := []struct {
		name        string
		original    string
		replacement string
		pairLang    string
		want        bool
	}{
		{
			// The sentence from the UX review, verbatim.
			name:        "whole Chinese sentence rendered into English",
			original:    "我想说这句话但是不知道用英语怎么说。",
			replacement: "I want to say this but I don't know how to say it in English.",
			pairLang:    "zh",
			want:        true,
		},
		{
			name:        "ordinary English correction is not a translation",
			original:    "She goes to market yesterday",
			replacement: "She went to the market yesterday",
			pairLang:    "zh",
			want:        false,
		},
		{
			// One Chinese word inside English prose. The sentence around it is
			// still English with its own grammar to fix, and calling the card a
			// translation would mislabel a grammar fix.
			name:        "single Chinese word inside an English sentence",
			original:    "I bought a 苹果 at the store",
			replacement: "I bought an apple at the store",
			pairLang:    "zh",
			want:        false,
		},
		{
			name:        "a lone stray Han rune is not a sentence",
			original:    "的",
			replacement: "of",
			pairLang:    "zh",
			want:        false,
		},
		{
			// Chinese in, Chinese out: whatever this is, Petal is not translating.
			name:        "Chinese rewritten as Chinese",
			original:    "我想说这句话",
			replacement: "我要说这句话",
			pairLang:    "zh",
			want:        false,
		},
		{
			// The same Chinese span, but the writer is on the French pair. Petal
			// has no business offering to translate a language she never claimed.
			name:        "Chinese span on a non-zh pair",
			original:    "我想说这句话但是不知道用英语怎么说。",
			replacement: "I want to say this in English.",
			pairLang:    "fr",
			want:        false,
		},
		{
			name:        "French sentence rendered into English",
			original:    "Je ne sais pas comment le dire en anglais.",
			replacement: "I don't know how to say it in English.",
			pairLang:    "fr",
			want:        true,
		},
		{
			name:        "Portuguese sentence rendered into English",
			original:    "Eu quero dizer isso mas não sei como.",
			replacement: "I want to say this but I don't know how.",
			pairLang:    "pt-PT",
			want:        true,
		},
		{
			name:        "Spanish sentence rendered into English",
			original:    "Yo quiero decir esto pero no sé cómo.",
			replacement: "I want to say this but I don't know how.",
			pairLang:    "es",
			want:        true,
		},
		{
			// A single marker is not evidence. "Que" appears in English writing
			// about other languages, in names, in quoted phrases.
			name:        "one Latin marker is not enough",
			original:    "The word que confused me",
			replacement: "The word que confuses me",
			pairLang:    "pt-PT",
			want:        false,
		},
		{
			// The words most likely to sink this heuristic: English function words
			// that are also Romance function words. They are kept out of the lists
			// precisely so this sentence stays a grammar fix.
			name:        "English full of pan-Romance lookalikes",
			original:    "I do not know if a con man on the plus side as no era",
			replacement: "I do not know whether a con man, on the plus side, is no era",
			pairLang:    "es",
			want:        false,
		},
		{
			name:        "English with a borrowed French phrase stays English",
			original:    "It was a pas de deux, more or less",
			replacement: "It was a pas de deux, more or less.",
			pairLang:    "fr",
			want:        false,
		},
		{
			name:        "empty replacement (an awareness-only finding)",
			original:    "我想说这句话但是不知道用英语怎么说。",
			replacement: "",
			pairLang:    "zh",
			want:        false,
		},
		{
			// A document whose owner has no pair recorded. No test, no label.
			name:        "no pair language",
			original:    "我想说这句话但是不知道用英语怎么说。",
			replacement: "I want to say this in English.",
			pairLang:    "",
			want:        false,
		},
		{
			// An unshipped pair. Same rule: decline rather than guess.
			name:        "unknown pair language",
			original:    "Ich weiß nicht wie man das sagt.",
			replacement: "I don't know how to say that.",
			pairLang:    "de",
			want:        false,
		},
	}

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			if got := isTranslation(c.original, c.replacement, c.pairLang, ""); got != c.want {
				t.Errorf("isTranslation(%q, %q, %q, en) = %v, want %v",
					c.original, c.replacement, c.pairLang, got, c.want)
			}
		})
	}
}

// The mirror image (Phase 28): in a document she wrote in her own language, the
// translate card is the English sentence rendered into her language — and the
// English-document question, asked here, would label nothing.
//
// The case this file exists to pin is the third one: a genuine Portuguese
// correction inside a Portuguese document. Reading the English-document test
// backwards would call it a translation, because `readsAsEnglish` is a low bar
// that Portuguese clears too. It has to stay a correction.
func TestIsTranslationInAPairLanguageDocument(t *testing.T) {
	cases := []struct {
		name        string
		original    string
		replacement string
		pairLang    string
		want        bool
	}{
		{
			name:        "English sentence rendered into Portuguese",
			original:    "I want to say this but I don't know how to say it.",
			replacement: "Eu quero dizer isso mas não sei como o dizer.",
			pairLang:    "pt-PT",
			want:        true,
		},
		{
			name:        "English sentence rendered into Chinese",
			original:    "I don't know how to say this in Chinese.",
			replacement: "我不知道这句话用中文怎么说。",
			pairLang:    "zh",
			want:        true,
		},
		{
			// The one that matters. Portuguese in, Portuguese out, inside a
			// Portuguese document: a correction, and nothing else.
			name:        "Portuguese corrected as Portuguese",
			original:    "Eu quero dizer isso mas não sei como.",
			replacement: "Eu quero dizer isto mas não sei como.",
			pairLang:    "pt-PT",
			want:        false,
		},
		{
			// And its Chinese twin, which the script test already caught.
			name:        "Chinese corrected as Chinese",
			original:    "我想说这句话",
			replacement: "我要说这句话",
			pairLang:    "zh",
			want:        false,
		},
		{
			// The old direction, asked in the new document. She quoted English in
			// her Portuguese and Petal rendered it into Portuguese — which IS a
			// translation, and is the case above. This is its reverse: Portuguese
			// out of an English document that isn't one. No label.
			name:        "Portuguese rendered into English is not this document's translation",
			original:    "Eu quero dizer isso mas não sei como.",
			replacement: "I want to say this but I don't know how.",
			pairLang:    "pt-PT",
			want:        false,
		},
		{
			// English prose without enough evidence to vote. Silence, not a guess.
			name:        "too short to read as English",
			original:    "OK",
			replacement: "Está bem, muito obrigado.",
			pairLang:    "pt-PT",
			want:        false,
		},
		{
			name:        "unknown pair language declines in both directions",
			original:    "I don't know how to say that.",
			replacement: "Ich weiß nicht wie man das sagt.",
			pairLang:    "de",
			want:        false,
		},
	}

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			if got := isTranslation(c.original, c.replacement, c.pairLang, docLangPair); got != c.want {
				t.Errorf("isTranslation(%q, %q, %q, pair) = %v, want %v",
					c.original, c.replacement, c.pairLang, got, c.want)
			}
		})
	}
}

// pair_lang is stored as the pack code, but a stored value has drifted before
// (see the picker's history), so the fold is tested rather than assumed.
func TestNormalizePairLang(t *testing.T) {
	for in, want := range map[string]string{
		"zh": "zh", "zh-CN": "zh", "ZH": "zh",
		"pt": "pt-PT", "pt-PT": "pt-PT", "pt-pt": "pt-PT",
		"fr": "fr", "fr-FR": "fr",
		"es": "es", "es-ES": "es",
		" zh ": "zh",
		"":     "",
		"de":   "de",
	} {
		if got := normalizePairLang(in); got != want {
			t.Errorf("normalizePairLang(%q) = %q, want %q", in, got, want)
		}
	}
}

// French elision must yield its parts, or "j'ai" and "n'est" — two of the
// commonest shapes in the language — count for nothing.
func TestElisionYieldsMarkers(t *testing.T) {
	if n := distinctMarkers("Je n'est pas", latinMarkers["fr"]); n < 3 {
		t.Errorf("elided French: got %d markers, want >= 3 (je, est, pas)", n)
	}
}

// Distinct, not total: one word repeated is one piece of evidence.
func TestRepeatedMarkerCountsOnce(t *testing.T) {
	if n := distinctMarkers("que que que", latinMarkers["pt-PT"]); n != 1 {
		t.Errorf("repeated marker: got %d, want 1", n)
	}
}