petal/internal/suggestions/language.go

package suggestions

import (
	"strings"
	"unicode"
)

// Telling her language from English, well enough to label a card.
//
// When the checkpoint quotes a span she wrote in her own language and hands back
// an English rendering, that is not a correction — nothing was wrong with what
// she wrote — and it should not be filed under 'clarity'. The label is derived
// here rather than asked of the model: a type is structural, and a model that
// re-reasons every pass would drift between labels for the same sentence.
//
// The failure mode is deliberately cheap. Getting this wrong changes a card's
// coloured pill and nothing else — the replacement, the explanation and the
// Accept button are identical either way — so a heuristic is the right tool. It
// is written to under-claim: a span it isn't sure about stays whatever the model
// called it.
//
// The two pair families need genuinely different tests, and pretending otherwise
// would be the bug:
//
//   - zh is a different script. Counting Han runes is close to certain.
//   - pt-PT, fr and es share the Latin alphabet with English, where no such
//     signal exists. Those fall back to function words — the short, extremely
//     common words a sentence in that language can hardly avoid and an English
//     sentence has no reason to contain.

// isTranslation reports whether this edit is a rendering of one language into
// the other, rather than a correction. Both halves must hold: the quoted span
// reads as one language, and what Petal offers back reads as the other. The
// second half matters — a Chinese span rewritten into different Chinese is
// something else entirely, and Petal has no business calling it a translation.
//
// Which way it points follows the document (Phase 28). In an English document
// the translate card is her language rendered into English — she reached for a
// sentence she couldn't say yet, and Petal said it for her. In a document she
// wrote in her own language the useful card is the mirror image: an English
// sentence she dropped into her Portuguese, rendered into Portuguese. Asking the
// English-document question there would label nothing, and the card would file
// as a correction to prose that was never wrong.
//
// The flipped direction cannot be the same test read backwards. `readsAsEnglish`
// is a low bar on purpose — Latin letters, not swamped by another script — which
// every Portuguese sentence also clears, so using it on the *original* would
// call every genuine Portuguese correction a translation. The flipped test
// instead uses the sentence-level vote from doclang.go, where English has its
// own marker list and has to out-evidence the pair language to win.
func isTranslation(original, replacement, pairLang, docLang string) bool {
	if strings.TrimSpace(original) == "" || strings.TrimSpace(replacement) == "" {
		return false
	}
	if normalizeDocLang(docLang) == docLangPair {
		p := normalizePairLang(pairLang)
		return sentenceLang(original, p) == docLangEnglish &&
			sentenceLang(replacement, p) == docLangPair
	}
	return readsAsPairLang(original, pairLang) && readsAsEnglish(replacement)
}

// readsAsPairLang reports whether s is predominantly in the writer's language.
func readsAsPairLang(s, pairLang string) bool {
	switch normalizePairLang(pairLang) {
	case "zh":
		han, latin := scriptCounts(s)
		// Predominantly, not merely partly: one Chinese word inside an English
		// sentence is a vocabulary question, and the sentence around it is still
		// English prose with its own grammar to correct. Two runes is the floor
		// because a single Han character is as likely to be a stray keystroke.
		return han >= 2 && han > latin
	case "pt-PT", "fr", "es":
		return distinctMarkers(s, latinMarkers[normalizePairLang(pairLang)]) >= 2
	}
	// A pair Petal has no test for. Say no: an unlabelled card is a card that
	// reads as it did yesterday, and a wrongly-labelled one is a new defect.
	return false
}

// readsAsEnglish reports whether s is English prose rather than more of her own
// language. It is not a language identifier — it only has to separate "English"
// from "the pair language", and it is only ever asked about text Petal itself
// generated, so the bar is low on purpose: Latin letters present, and not
// swamped by another script.
func readsAsEnglish(s string) bool {
	han, latin := scriptCounts(s)
	return latin > 0 && latin > han
}

// normalizePairLang folds the stored `users.pair_lang` into the codes below.
// Empty (a document whose owner has no pair recorded) falls through to no test.
func normalizePairLang(pairLang string) string {
	switch p := strings.ToLower(strings.TrimSpace(pairLang)); p {
	case "zh", "zh-cn", "zh-hans":
		return "zh"
	case "pt", "pt-pt":
		return "pt-PT"
	case "fr", "fr-fr":
		return "fr"
	case "es", "es-es":
		return "es"
	default:
		return p
	}
}

// scriptCounts counts Han runes and ASCII letters. Everything else — digits,
// punctuation, spaces, emoji — is ignored, so trailing 。or a stray comma
// changes nothing.
func scriptCounts(s string) (han, latin int) {
	for _, r := range s {
		switch {
		case unicode.Is(unicode.Han, r):
			han++
		case r < unicode.MaxASCII && unicode.IsLetter(r):
			latin++
		}
	}
	return han, latin
}

// distinctMarkers counts how many *different* marker words appear in s. Distinct
// rather than total: "que ... que" is one writer's habit, while "eu quero" is two
// independent pieces of evidence.
func distinctMarkers(s string, markers map[string]bool) int {
	if len(markers) == 0 {
		return 0
	}
	seen := map[string]bool{}
	for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
		// Split on anything that isn't a letter, so punctuation and digits are
		// separators. Apostrophes included: French elision (j'ai, n'est) should
		// yield its parts.
		return !unicode.IsLetter(r)
	}) {
		if markers[w] {
			seen[w] = true
		}
	}
	return len(seen)
}

// Function words that a sentence in each Latin pair can hardly avoid.
//
// Curated against English, not for coverage: every entry here is a word an
// English sentence has essentially no reason to contain, which is why the lists
// omit plenty of far more common words. Deliberately absent — each of them a
// false positive waiting to happen — is anything that is *also* an English word:
// the pan-Romance shorts (a, o, e, as, no, on, en, de, se, na, mi, son, era,
// plus, pour, si, ma, ce, ne), Portuguese "do", Spanish "con", "ya" and "todo".
// Dropping "con" costs the Spanish list one of its commonest words, and that is
// the right trade — a marker that fires on English corroborates the wrong
// answer, which is worse than a sentence Petal declines to label.
//
// A single marker is not enough (see readsAsPairLang), so these lists are read
// as evidence to be corroborated rather than as a decision.
var latinMarkers = map[string]map[string]bool{
	"fr": words(
		"je", "tu", "il", "elle", "ils", "elles", "nous", "vous", "est", "sont",
		"était", "étais", "une", "des", "les", "du", "dans", "avec", "que", "qui",
		"mais", "très", "être", "avoir", "pas", "cette", "cet", "ces", "mon",
		"mes", "notre", "votre", "leur", "aussi", "alors", "parce", "comme",
		"beaucoup", "toujours", "jamais", "quand", "bien", "chose", "temps",
		"moi", "toi", "lui", "peux", "veux", "sais", "faire", "dit", "aujourd",
		"hui", "quelque", "chez", "tout", "tous", "rien", "déjà", "encore",
	),
	"pt-PT": words(
		"eu", "você", "ele", "ela", "eles", "elas", "nós", "são", "uma", "os",
		"da", "dos", "das", "com", "que", "mas", "muito", "não", "meu",
		"minha", "seu", "sua", "isso", "este", "esta", "está", "estou", "quero",
		"também", "quando", "porque", "coisa", "tempo", "fazer", "sempre",
		"nunca", "bem", "obrigado", "obrigada", "gosto", "tenho", "tem", "foi",
		"ser", "ter", "mais", "já", "ainda", "aqui", "ali", "nada", "tudo",
		"todos", "para", "pela", "pelo", "sobre", "assim",
	),
	"es": words(
		"yo", "él", "ella", "ellos", "ellas", "nosotros", "una", "los", "las",
		"del", "que", "pero", "muy", "esto", "esta", "este", "está",
		"estoy", "quiero", "también", "cuando", "porque", "cosa", "tiempo",
		"hacer", "siempre", "nunca", "bien", "gracias", "tengo", "tiene", "fue",
		"ser", "tener", "más", "aquí", "allí", "nada", "todos",
		"para", "sobre", "así", "hola", "señor", "usted", "muchas",
	),
}

func words(list ...string) map[string]bool {
	out := make(map[string]bool, len(list))
	for _, w := range list {
		out[w] = true
	}
	return out
}