diff --git a/.gitignore b/.gitignore index f4a6dc9..bcb879a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ web/dist/* !web/dist/.gitkeep *.log +# Python +__pycache__/ +*.pyc + # Local env & data .env *.db diff --git a/scripts/build_hunspell_dictionary.py b/scripts/build_hunspell_dictionary.py index 42a4dcf..401fd67 100644 --- a/scripts/build_hunspell_dictionary.py +++ b/scripts/build_hunspell_dictionary.py @@ -145,7 +145,12 @@ class Aff: def __init__(self): self.pfx = {} # flag -> [(strip, append, condition, continuation)] self.sfx = {} - self.cross = {} # flag -> bool + # Cross-product, per flag — and kept per table, because PFX and SFX are + # separate flag namespaces in hunspell: the same flag may name a prefix + # table and a suffix table, with different cross-product settings. One + # shared dict let the second block silently overwrite the first. + self.cross_pfx = {} # flag -> bool + self.cross_sfx = {} self.flag_kind = "char" self.needaffix = None self.circumfix = None @@ -159,7 +164,13 @@ def parse_flags(raw, kind): if not raw: return set() if kind == "long": - return {raw[i:i + 2] for i in range(0, len(raw) - 1, 2)} + # Two characters per flag, exactly. An odd length is a malformed flag + # string, and silently dropping the trailing character would quietly + # expand an entry through the wrong paradigm — the failure mode this + # whole FLAG-aware rewrite exists to avoid. + if len(raw) % 2: + raise SystemExit(f"odd-length long flag string {raw!r}") + return {raw[i:i + 2] for i in range(0, len(raw), 2)} if kind == "num": return {f for f in raw.split(",") if f} return set(raw) @@ -204,7 +215,8 @@ def parse_aff(path): if parts and parts[0] in ("PFX", "SFX"): kind, flag, cross_flag, count = parts[0], parts[1], parts[2], int(parts[3]) table = aff.pfx if kind == "PFX" else aff.sfx - aff.cross[flag] = cross_flag == "Y" + cross = aff.cross_pfx if kind == "PFX" else aff.cross_sfx + cross[flag] = cross_flag == "Y" rules = table.setdefault(flag, []) for j in range(1, count + 1): p = lines[i + j].split() @@ -298,24 +310,30 @@ def expand_entry(word, flags, aff, out): # Prefix then suffix. The suffix flag may come from the stem or from the # prefix's own continuation (`PFX Um 0 0/S.`), and the suffix condition is # matched against the whole prefixed word, which is what hunspell does. + # NEEDAFFIX is checked here too, exactly as on the single-affix paths above: + # a doubly-affixed form whose last continuation still carries the flag is + # "not a word on its own", and without compounding there is no third affix + # left to make it one. for form, pf, pcont in prefixed: - if not aff.cross.get(pf): + if not aff.cross_pfx.get(pf): continue for f in flags | pcont: - if f in aff.sfx and aff.cross.get(f): - for full, _ in apply_suffix(form, aff.sfx[f]): - out.add(full) + if f in aff.sfx and aff.cross_sfx.get(f): + for full, fcont in apply_suffix(form, aff.sfx[f]): + if is_word(fcont): + out.add(full) # Suffix then prefix — the same pair reached from the other side, which is # how the elision prefixes arrive in French. Only the flags the suffix hands # forward are new here; the stem's own were covered above. for form, sf, scont in suffixed: - if not aff.cross.get(sf): + if not aff.cross_sfx.get(sf): continue for f in scont: - if f in aff.pfx and aff.cross.get(f): - for full, _ in apply_prefix(form, aff.pfx[f]): - out.add(full) + if f in aff.pfx and aff.cross_pfx.get(f): + for full, fcont in apply_prefix(form, aff.pfx[f]): + if is_word(fcont): + out.add(full) def expand(aff_path, dic_path): diff --git a/web/src/components/Editor/SpellCheck.test.ts b/web/src/components/Editor/SpellCheck.test.ts index 44b53e5..be0f92f 100644 --- a/web/src/components/Editor/SpellCheck.test.ts +++ b/web/src/components/Editor/SpellCheck.test.ts @@ -50,6 +50,25 @@ describe('wordAt and the pair alphabet', () => { expect(wordAt(doc, posOf(6), true)?.word).toBe('today') }) + it('keeps a curly apostrophe inside the word, because that is the only kind here', () => { + // Typography.ts rewrites every ' typed in this editor into ’, so a token + // that only matched the straight mark never saw an apostrophe at all. + // "aujourd’hui" cut into "aujourd" + "hui" — neither one a French word, both + // underlined, and withElision never reached. + const doc = para('c’est aujourd’hui') + expect(wordAt(doc, posOf(8), true)?.word).toBe('aujourd’hui') + // English pays the same debt: "don’t" must stay whole to be looked up. + expect(wordAt(doc, posOf(1))?.word).toBe('c’est') + }) + + it('reads œ as a letter, not as a word boundary', () => { + // U+0153 sits outside the Latin-1 ranges, so "cœur" tokenized as "c" + "ur" + // and the orphan "ur" was long enough to be underlined. 586 œ forms ship in + // the French word list and none of them were reachable. + const doc = para('mon cœur') + expect(wordAt(doc, posOf(5), true)?.word).toBe('cœur') + }) + it('stops the wide alphabet at the maths symbols hiding in Latin-1', () => { // × (U+00D7) and ÷ (U+00F7) sit inside the accented-letter block. A range // written À-ÿ would swallow them and glue "3×4" into one token. diff --git a/web/src/components/Editor/SpellCheck.ts b/web/src/components/Editor/SpellCheck.ts index e5eb239..b1450f9 100644 --- a/web/src/components/Editor/SpellCheck.ts +++ b/web/src/components/Editor/SpellCheck.ts @@ -32,8 +32,19 @@ interface PluginState { // writer with no Latin second language, adding accented letters can only find // new words to underline (the "café" and "naïve" she borrows), and finds no // mistakes she has actually made. -const WORD_RE = /[A-Za-z][A-Za-z']*/g -const WORD_RE_LATIN = /[A-Za-zÀ-ÖØ-öø-ÿ][A-Za-zÀ-ÖØ-öø-ÿ']*/g +// +// Both apostrophes count as word characters, because Typography.ts rewrites +// every ' typed in this editor into a curly ’ — so by the time text reaches the +// tokenizer the straight mark is only ever seen in pasted content. Matching only +// the straight one cut "aujourd’hui" into "aujourd" + "hui", neither of which is +// a French word, and left the elision handling in useSpellChecker unreachable. +// The dictionaries spell theirs straight; `combine` normalises on lookup. +// +// Œ/œ are named explicitly: they sit at U+0152/U+0153, outside the Latin-1 +// ranges below, so without them "cœur" tokenized as "c" + "ur" and the bare +// "ur" was underlined. Æ/æ need no help — they are inside À-Ö and Ø-ö. +const WORD_RE = /[A-Za-z][A-Za-z'’]*/g +const WORD_RE_LATIN = /[A-Za-zÀ-ÖØ-öø-ÿŒœ][A-Za-zÀ-ÖØ-öø-ÿŒœ'’]*/g // wordRe returns a fresh matcher for the alphabet in force. Fresh because these // are /g regexes carrying lastIndex, and two scans sharing one would interleave. @@ -55,8 +66,8 @@ function isCheckable(word: string): boolean { // lookup sees the bare token; returns the core plus how many chars were trimmed // off the front (to re-anchor the decoration). function coreOf(word: string): { core: string; lead: number } { - const lead = word.match(/^'+/)?.[0].length ?? 0 - const trail = word.match(/'+$/)?.[0].length ?? 0 + const lead = word.match(/^['’]+/)?.[0].length ?? 0 + const trail = word.match(/['’]+$/)?.[0].length ?? 0 return { core: word.slice(lead, word.length - trail), lead } } diff --git a/web/src/hooks/useSpellChecker.ts b/web/src/hooks/useSpellChecker.ts index 590863d..43bf52e 100644 --- a/web/src/hooks/useSpellChecker.ts +++ b/web/src/hooks/useSpellChecker.ts @@ -252,6 +252,19 @@ export function interleave(lists: string[][]): string[] { return out } +// Typography.ts turns every apostrophe typed in the editor into a curly U+2019, +// and every word list Petal ships spells its apostrophes straight ("aujourd'hui", +// "don't", and the elision heads in FR_ELISION). Normalising here — the one place +// every lookup passes through — keeps that a rendering detail rather than a +// spelling one; corrections come back wearing whichever mark she actually used, +// so accepting a pill never swaps her curly apostrophe for a straight one. +const CURLY_APOSTROPHE = /’/g +const STRAIGHT_APOSTROPHE = /'/g + +function straighten(word: string): string { + return word.replace(CURLY_APOSTROPHE, "'") +} + // combine is the both-dictionaries rule itself, separated from the loading so it // can be reasoned about (and tested) on its own. `dicts` is a getter because the // writer's dictionary lands after English and the checker must see it when it @@ -266,9 +279,14 @@ export function combine(dicts: () => Loaded[]): SpellChecker { correct: (w) => { const loaded = dicts() if (loaded.length === 0) return true - return loaded.some((d) => d.spell.correct(w)) + const word = straighten(w) + return loaded.some((d) => d.spell.correct(word)) + }, + suggest: (w) => { + const word = straighten(w) + const out = interleave(dicts().map((d) => d.spell.suggest(word))) + return word === w ? out : out.map((s) => s.replace(STRAIGHT_APOSTROPHE, '’')) }, - suggest: (w) => interleave(dicts().map((d) => d.spell.suggest(w))), // A getter, not a snapshot. The alphabet is read by every surface that // resolves a word, and if it were captured when the checker was built it // would still say A-Z after her dictionary arrived — so a pt-PT writer's diff --git a/web/src/i18n/i18n.test.ts b/web/src/i18n/i18n.test.ts index 02f38cf..457e1d7 100644 --- a/web/src/i18n/i18n.test.ts +++ b/web/src/i18n/i18n.test.ts @@ -194,7 +194,10 @@ describe('the pt-PT pack', () => { // is invisible to anyone who doesn't read Portuguese — including whoever // reviews this diff. it('is European Portuguese, not Brazilian', () => { - const text = JSON.stringify(ptPT, (_k, v) => (typeof v === 'function' ? v(1, 'x') : v)) + // Lowercased: half this copy is sentences, and a form that only ever appears + // at the start of one ("Actualmente…") would otherwise walk straight past + // both greps below. + const text = JSON.stringify(ptPT, (_k, v) => (typeof v === 'function' ? v(1, 'x') : v)).toLowerCase() // Brazilian spellings and vocabulary that would give the pack away. for (const bad of ['sinônimo', 'acadêmico', 'arquivo', 'tela', 'salvar', 'deletar', 'usuário', 'você']) { @@ -259,7 +262,7 @@ describe('the fr pack', () => { // this file, which makes it exactly as invisible to a reviewer as the pt-BR // forms were, and worth pinning the same way. it('is metropolitan French, not Québécois', () => { - const text = JSON.stringify(fr, (_k, v) => (typeof v === 'function' ? v(1, 'x') : v)) + const text = JSON.stringify(fr, (_k, v) => (typeof v === 'function' ? v(1, 'x') : v)).toLowerCase() for (const bad of ['courriel', 'clavarder', 'magasiner', 'fin de semaine', 'baladodiffusion']) { expect(text, `Québécois form "${bad}" in the fr pack`).not.toContain(bad) }