petal/internal/suggestions/language.go

package suggestions

import (
	"strings"
	"unicode"
)

// Telling her language from English, well enough to label a card.
//
// When the checkpoint quotes a span she wrote in her own language and hands back
// an English rendering, that is not a correction — nothing was wrong with what
// she wrote — and it should not be filed under 'clarity'. The label is derived
// here rather than asked of the model: a type is structural, and a model that
// re-reasons every pass would drift between labels for the same sentence.
//
// The failure mode is deliberately cheap. Getting this wrong changes a card's
// coloured pill and nothing else — the replacement, the explanation and the
// Accept button are identical either way — so a heuristic is the right tool. It
// is written to under-claim: a span it isn't sure about stays whatever the model
// called it.
//
// The two pair families need genuinely different tests, and pretending otherwise
// would be the bug:
//
//   - zh is a different script. Counting Han runes is close to certain.
//   - pt-PT, fr and es share the Latin alphabet with English, where no such
//     signal exists. Those fall back to function words — the short, extremely
//     common words a sentence in that language can hardly avoid and an English
//     sentence has no reason to contain.

// isTranslation reports whether this edit is a rendering of one language into
// the other, rather than a correction. Both halves must hold: the quoted span
// reads as one language, and what Petal offers back reads as the other. The
// second half matters — a Chinese span rewritten into different Chinese is
// something else entirely, and Petal has no business calling it a translation.
//
// Which way it points follows the document (Phase 28). In an English document
// the translate card is her language rendered into English — she reached for a
// sentence she couldn't say yet, and Petal said it for her. In a document she
// wrote in her own language the useful card is the mirror image: an English
// sentence she dropped into her Portuguese, rendered into Portuguese. Asking the
// English-document question there would label nothing, and the card would file
// as a correction to prose that was never wrong.
//
// The flipped direction cannot be the same test read backwards. `readsAsEnglish`
// is a low bar on purpose — Latin letters, not swamped by another script — which
// every Portuguese sentence also clears, so using it on the *original* would
// call every genuine Portuguese correction a translation. The flipped test
// instead uses the sentence-level vote from doclang.go, where English has its
// own marker list and has to out-evidence the pair language to win.
func isTranslation(original, replacement, pairLang, docLang string) bool {
	if strings.TrimSpace(original) == "" || strings.TrimSpace(replacement) == "" {
		return false
	}
	if normalizeDocLang(docLang) == docLangPair {
		p := normalizePairLang(pairLang)
		return sentenceLang(original, p) == docLangEnglish &&
			sentenceLang(replacement, p) == docLangPair
	}
	return readsAsPairLang(original, pairLang) && readsAsEnglish(replacement)
}

// readsAsPairLang reports whether s is predominantly in the writer's language.
func readsAsPairLang(s, pairLang string) bool {
	switch normalizePairLang(pairLang) {
	case "zh":
		han, latin := scriptCounts(s)
		// Predominantly, not merely partly: one Chinese word inside an English
		// sentence is a vocabulary question, and the sentence around it is still
		// English prose with its own grammar to correct. Two runes is the floor
		// because a single Han character is as likely to be a stray keystroke.
		return han >= 2 && han > latin
	case "pt-PT", "fr", "es":
		return distinctMarkers(s, latinMarkers[normalizePairLang(pairLang)]) >= 2
	}
	// A pair Petal has no test for. Say no: an unlabelled card is a card that
	// reads as it did yesterday, and a wrongly-labelled one is a new defect.
	return false
}

// readsAsEnglish reports whether s is English prose rather than more of her own
// language. It is not a language identifier — it only has to separate "English"
// from "the pair language", and it is only ever asked about text Petal itself
// generated, so the bar is low on purpose: Latin letters present, and not
// swamped by another script.
func readsAsEnglish(s string) bool {
	han, latin := scriptCounts(s)
	return latin > 0 && latin > han
}

// normalizePairLang folds the stored `users.pair_lang` into the codes below.
// Empty (a document whose owner has no pair recorded) falls through to no test.
func normalizePairLang(pairLang string) string {
	switch p := strings.ToLower(strings.TrimSpace(pairLang)); p {
	case "zh", "zh-cn", "zh-hans":
		return "zh"
	case "pt", "pt-pt":
		return "pt-PT"
	case "fr", "fr-fr":
		return "fr"
	case "es", "es-es":
		return "es"
	default:
		return p
	}
}

// scriptCounts counts Han runes and ASCII letters. Everything else — digits,
// punctuation, spaces, emoji — is ignored, so trailing 。or a stray comma
// changes nothing.
func scriptCounts(s string) (han, latin int) {
	for _, r := range s {
		switch {
		case unicode.Is(unicode.Han, r):
			han++
		case r < unicode.MaxASCII && unicode.IsLetter(r):
			latin++
		}
	}
	return han, latin
}

// distinctMarkers counts how many *different* marker words appear in s. Distinct
// rather than total: "que ... que" is one writer's habit, while "eu quero" is two
// independent pieces of evidence.
func distinctMarkers(s string, markers map[string]bool) int {
	if len(markers) == 0 {
		return 0
	}
	seen := map[string]bool{}
	for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
		// Split on anything that isn't a letter, so punctuation and digits are
		// separators. Apostrophes included: French elision (j'ai, n'est) should
		// yield its parts.
		return !unicode.IsLetter(r)
	}) {
		if markers[w] {
			seen[w] = true
		}
	}
	return len(seen)
}

// Function words that a sentence in each Latin pair can hardly avoid.
//
// Curated against English, not for coverage: every entry here is a word an
// English sentence has essentially no reason to contain, which is why the lists
// omit plenty of far more common words. Deliberately absent — each of them a
// false positive waiting to happen — is anything that is *also* an English word:
// the pan-Romance shorts (a, o, e, as, no, on, en, de, se, na, mi, son, era,
// plus, pour, si, ma, ce, ne), Portuguese "do", Spanish "con", "ya" and "todo".
// Dropping "con" costs the Spanish list one of its commonest words, and that is
// the right trade — a marker that fires on English corroborates the wrong
// answer, which is worse than a sentence Petal declines to label.
//
// A single marker is not enough (see readsAsPairLang), so these lists are read
// as evidence to be corroborated rather than as a decision.
//
// **Curated against English is not the same as curated thinly**, and the first
// version of these lists confused the two. Seen live 2026-07-29: "Esta manhã
// acordei cedo e fui correr ao longo da marginal. O ar estava fresco e havia
// poucas pessoas na rua." — unremarkable Portuguese, two marker hits, *zero*
// English hits, and a verdict of English, because the document-level floor wants
// three. The list was missing the ordinary machinery of the language: the
// contractions (ao, à, num), the past tenses a diary is written in (estava,
// havia, fomos), and the words that join two clauses (até, depois, então,
// onde). Every one of them clears the bar above — an English sentence has no
// reason to contain them — so their absence bought nothing and cost the verdict.
// The floor stays at three; what changed is that three is now reachable by
// prose rather than only by a paragraph that happens to argue with itself.
var latinMarkers = map[string]map[string]bool{
	"fr": words(
		"je", "tu", "il", "elle", "ils", "elles", "nous", "vous", "est", "sont",
		"était", "étais", "une", "des", "les", "du", "dans", "avec", "que", "qui",
		"mais", "très", "être", "avoir", "pas", "cette", "cet", "ces", "mon",
		"mes", "notre", "votre", "leur", "aussi", "alors", "parce", "comme",
		"beaucoup", "toujours", "jamais", "quand", "bien", "chose", "temps",
		"moi", "toi", "lui", "peux", "veux", "sais", "faire", "dit", "aujourd",
		"hui", "quelque", "chez", "tout", "tous", "rien", "déjà", "encore",
		// The same gap the pt-PT list was caught with, closed by analogy rather
		// than by observation — no fr account exists yet to catch it live.
		"aux", "après", "où", "avait", "étaient", "depuis", "jusqu", "chaque",
		"autre", "même", "hier", "demain", "matin", "soir", "nôtre", "leurs",
	),
	"pt-PT": words(
		"eu", "você", "ele", "ela", "eles", "elas", "nós", "são", "uma", "os",
		"da", "dos", "das", "com", "que", "mas", "muito", "não", "meu",
		"minha", "seu", "sua", "isso", "este", "esta", "está", "estou", "quero",
		"também", "quando", "porque", "coisa", "tempo", "fazer", "sempre",
		"nunca", "bem", "obrigado", "obrigada", "gosto", "tenho", "tem", "foi",
		"ser", "ter", "mais", "já", "ainda", "aqui", "ali", "nada", "tudo",
		"todos", "para", "pela", "pelo", "sobre", "assim",
		// The contractions, which no English sentence has any use for.
		"ao", "aos", "à", "às", "num", "numa", "dum", "duma", "pelos", "pelas",
		"neste", "nesta", "disso", "deste", "desta",
		// The tenses a journal is actually written in.
		"estava", "estavam", "estão", "estamos", "havia", "houve", "era", "eram",
		"fui", "fomos", "foram", "vai", "vamos", "tinha", "tinham",
		// The joins between two clauses.
		"até", "depois", "antes", "onde", "então", "enquanto", "embora",
		"sem", "quem", "entre",
		// And the everyday determiners and time words a diary can hardly avoid.
		"nosso", "nossa", "outro", "outra", "mesmo", "mesma", "tão",
		"muitos", "muitas", "poucos", "poucas", "hoje", "ontem", "amanhã",
	),
	"es": words(
		"yo", "él", "ella", "ellos", "ellas", "nosotros", "una", "los", "las",
		"del", "que", "pero", "muy", "esto", "esta", "este", "está",
		"estoy", "quiero", "también", "cuando", "porque", "cosa", "tiempo",
		"hacer", "siempre", "nunca", "bien", "gracias", "tengo", "tiene", "fue",
		"ser", "tener", "más", "aquí", "allí", "nada", "todos",
		"para", "sobre", "así", "hola", "señor", "usted", "muchas",
		// Likewise by analogy: no es account exists yet either. "sin" and "tan"
		// stay out — both are English words, which is the one disqualification.
		"al", "después", "antes", "donde", "entonces", "mientras", "aunque",
		"estaba", "estaban", "están", "había", "hubo", "fuimos", "fueron",
		"nuestro", "nuestra", "otro", "otra", "mismo", "misma", "quién", "quien",
		"muchos", "pocas", "pocos", "hoy", "ayer", "mañana",
	),
}

func words(list ...string) map[string]bool {
	out := make(map[string]bool, len(list))
	for _, w := range list {
		out[w] = true
	}
	return out
}