Compare commits
67
Commits
3640ce9324
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d961a89bf8 | ||
|
|
7fa98d03c7 | ||
|
|
6bed33c27e | ||
|
|
7383bdb403 | ||
|
|
3cc23b8ea4 | ||
|
|
6026d98598 | ||
|
|
15398eab4d | ||
|
|
2c5b05b398 | ||
|
|
acb35108c0 | ||
|
|
e67f77eb05 | ||
|
|
c719effe1d | ||
|
|
1fdc206576 | ||
|
|
466055020f | ||
|
|
29eb2fe1fc | ||
|
|
76dede8856 | ||
|
|
db9cfb7abf | ||
|
|
f82f2b589d | ||
|
|
5cd3aeabde | ||
|
|
047f4ae67f | ||
|
|
f2dd30628a | ||
|
|
c6bf36bddf | ||
|
|
178cb7ae67 | ||
|
|
8f2ad34a10 | ||
|
|
5659312358 | ||
|
|
a216614c81 | ||
|
|
c348a9b8ae | ||
|
|
77f284f65c | ||
|
|
9224c44fff | ||
|
|
39d4e4770a | ||
|
|
bd92cdc9b6 | ||
|
|
1acc23244e | ||
|
|
40de65b3d1 | ||
|
|
b2d50e9136 | ||
|
|
1aa3a14030 | ||
|
|
978cb80642 | ||
|
|
f082a930cb | ||
|
|
ce5de68b9b | ||
|
|
b23c5a9a13 | ||
|
|
25e415daa2 | ||
|
|
3bcc967f51 | ||
|
|
963fc1754d | ||
|
|
de251ceae2 | ||
|
|
10e8aef86c | ||
|
|
c33de1175b | ||
|
|
ba06d904f0 | ||
|
|
ea14eb5e88 | ||
|
|
aac15b5ac5 | ||
|
|
be9aa13287 | ||
|
|
ec9fba9252 | ||
|
|
9c40a8ad3f | ||
|
|
ac1c6cddb0 | ||
|
|
3e714b6f00 | ||
|
|
69bf3ffde1 | ||
|
|
9a0edd6679 | ||
|
|
be1ab5cef7 | ||
|
|
071ea7b835 | ||
|
|
9a2e909b85 | ||
|
|
1f4ca4775a | ||
|
|
1bbc8fc8d3 | ||
|
|
e9b8595456 | ||
|
|
7b845644be | ||
|
|
3b714e297a | ||
|
|
24c3533e18 | ||
|
|
ccb43e5a4d | ||
|
|
4de83d0da5 | ||
|
|
74bf600593 | ||
|
|
86175f1559 |
+14
-1
@@ -44,6 +44,12 @@ TTS_AUDIO_FORMAT=mp3 # mp3 | opus | wav — mp3/opus transcode Pipe
|
||||
# them commented out for local development and Petal runs as the single
|
||||
# hardcoded `local` user, exactly as it did before auth landed.
|
||||
#
|
||||
# That fallback is scoped to development on purpose. With a BASE_URL naming
|
||||
# anything but localhost, Petal refuses to start rather than run open — see
|
||||
# PETAL_REQUIRE_AUTH — because the fallback on a reachable host means every
|
||||
# anonymous visitor is the `local` user, with full read and write over every
|
||||
# document in the database.
|
||||
#
|
||||
# AUTHENTIK_URL is the issuer of the Petal provider in Authentik (the value of
|
||||
# its "OpenID Configuration Issuer" field). The redirect URI to register there
|
||||
# is BASE_URL + /auth/callback.
|
||||
@@ -52,8 +58,15 @@ TTS_AUDIO_FORMAT=mp3 # mp3 | opus | wav — mp3/opus transcode Pipe
|
||||
# AUTHENTIK_CLIENT_SECRET=
|
||||
#
|
||||
# Who may sign in: comma-separated OIDC subject ids and/or email addresses.
|
||||
# Empty = anyone Authentik authenticates.
|
||||
# Empty = anyone Authentik authenticates, which is right for a single-household
|
||||
# instance and wrong the moment the IdP serves a wider audience than Petal. An
|
||||
# empty list is warned about at every boot rather than assumed either way.
|
||||
# PETAL_ALLOWED_SUBS=her@example.com,me@example.com
|
||||
#
|
||||
# Whether a missing OIDC configuration is fatal. Defaults to false for a
|
||||
# loopback BASE_URL and true for anything else, so neither a laptop nor a
|
||||
# deployment normally has to name it.
|
||||
# PETAL_REQUIRE_AUTH=true
|
||||
|
||||
# --- Deferred (not wired in the local-dev build) ---
|
||||
|
||||
|
||||
@@ -10,6 +10,10 @@ web/dist/*
|
||||
!web/dist/.gitkeep
|
||||
*.log
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Local env & data
|
||||
.env
|
||||
*.db
|
||||
|
||||
+258
-17
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The baseline policy has to actually permit the frontend Vite builds. These
|
||||
// are the allowances dist/index.html needs; if a future build starts emitting
|
||||
// an inline script or pulling from a new host, that shows up here rather than
|
||||
// as a blank page in production.
|
||||
func TestBaselineCSPCoversTheBuiltFrontend(t *testing.T) {
|
||||
required := []string{
|
||||
"script-src 'self'", // Vite emits no inline script
|
||||
"'unsafe-inline' https://fonts.googleapis.com", // React style={{…}} + the font link
|
||||
"https://fonts.gstatic.com", // the font files themselves
|
||||
"blob:", // read-aloud plays an object URL
|
||||
"object-src 'none'",
|
||||
"base-uri 'self'",
|
||||
"frame-ancestors 'self'",
|
||||
}
|
||||
for _, want := range required {
|
||||
if !strings.Contains(contentSecurityPolicy, want) {
|
||||
t.Errorf("baseline CSP is missing %q:\n%s", want, contentSecurityPolicy)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The middleware is a floor, not a ceiling: it runs *before* the handler
|
||||
// precisely so a route serving untrusted bytes can overwrite the policy with a
|
||||
// stricter one. This is the contract the image store depends on, and the reason
|
||||
// the policy no longer lives in the Traefik labels — customresponseheaders
|
||||
// would overwrite it in the other direction.
|
||||
func TestRouteMayTightenTheBaselineCSP(t *testing.T) {
|
||||
strict := "default-src 'none'; sandbox"
|
||||
handler := securityHeaders(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Security-Policy", strict)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/images/x.svg", nil))
|
||||
|
||||
if got := rec.Header().Get("Content-Security-Policy"); got != strict {
|
||||
t.Fatalf("handler's policy was not honoured: got %q, want %q", got, strict)
|
||||
}
|
||||
// The headers it didn't touch still stand.
|
||||
if rec.Header().Get("X-Content-Type-Options") != "nosniff" {
|
||||
t.Error("baseline nosniff was lost")
|
||||
}
|
||||
}
|
||||
|
||||
// Every ordinary response carries the baseline.
|
||||
func TestBaselineHeadersOnAPlainResponse(t *testing.T) {
|
||||
handler := securityHeaders(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
||||
|
||||
for header, want := range map[string]string{
|
||||
"Content-Security-Policy": contentSecurityPolicy,
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Referrer-Policy": "same-origin",
|
||||
} {
|
||||
if got := rec.Header().Get(header); got != want {
|
||||
t.Errorf("%s = %q, want %q", header, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
+82
-3
@@ -64,21 +64,45 @@ func main() {
|
||||
sessions := auth.NewSessionStore(database.DB)
|
||||
users := auth.NewUserStore(database.DB)
|
||||
|
||||
// …and the fallback is exactly what must not happen quietly on a public
|
||||
// host. Refuse to start rather than serve someone's journals to the open
|
||||
// internet because one environment variable was misspelled. See
|
||||
// config.RequireAuth for why this defaults on for any non-loopback BASE_URL.
|
||||
if !cfg.AuthEnabled() && cfg.RequireAuth {
|
||||
log.Fatalf("auth: refusing to start unauthenticated at %s.\n"+
|
||||
" Petal would resolve every anonymous request to the single %q user, with full\n"+
|
||||
" read and write over every document in the database.\n"+
|
||||
" Set AUTHENTIK_URL, AUTHENTIK_CLIENT_ID and AUTHENTIK_CLIENT_SECRET, or set\n"+
|
||||
" PETAL_REQUIRE_AUTH=false if this really is a trusted private network.",
|
||||
cfg.BaseURL, db.LocalUserID)
|
||||
}
|
||||
|
||||
var resolver auth.Resolver = auth.StaticResolver(db.LocalUserID)
|
||||
var oidcClient *auth.OIDC
|
||||
if cfg.AuthEnabled() {
|
||||
allowed := auth.ParseAllowlist(cfg.AllowedSubs)
|
||||
oidcClient = auth.NewOIDC(context.Background(), auth.Options{
|
||||
IssuerURL: cfg.AuthentikURL,
|
||||
ClientID: cfg.AuthentikClientID,
|
||||
ClientSecret: cfg.AuthentikClientSecret,
|
||||
BaseURL: cfg.BaseURL,
|
||||
Allowed: auth.ParseAllowlist(cfg.AllowedSubs),
|
||||
Allowed: allowed,
|
||||
}, sessions, users)
|
||||
resolver = sessions
|
||||
if n, err := sessions.Prune(); err == nil && n > 0 {
|
||||
log.Printf("auth: pruned %d expired session(s)", n)
|
||||
}
|
||||
log.Printf("auth: OIDC enabled (issuer=%s, redirect=%s)", cfg.AuthentikURL, oidcClient.RedirectURI())
|
||||
// An empty allowlist is a legitimate choice for a single-household
|
||||
// instance and a wide-open door in front of an IdP that fronts anything
|
||||
// else. Petal cannot tell which it is, so it says so every boot rather
|
||||
// than assuming.
|
||||
if len(allowed) == 0 {
|
||||
log.Printf("auth: WARNING — PETAL_ALLOWED_SUBS is empty, so EVERY account %s "+
|
||||
"authenticates may sign in and start writing here. Set it to the "+
|
||||
"comma-separated emails (or subject ids) that belong in this Petal.",
|
||||
cfg.AuthentikURL)
|
||||
}
|
||||
} else {
|
||||
log.Printf("auth: OIDC not configured — running as the single %q user", db.LocalUserID)
|
||||
}
|
||||
@@ -96,7 +120,7 @@ func main() {
|
||||
defer dict.Close()
|
||||
lexSet := lexicon.NewSet(dict)
|
||||
if lexSet.HasDreamDict() {
|
||||
log.Printf("dictionary: DreamDict open at %s (%v)", cfg.DictPath, dict.Langs())
|
||||
log.Printf("dictionary: DreamDict open at %s (%s)", cfg.DictPath, lexSet.Contents())
|
||||
} else {
|
||||
log.Printf("dictionary: no dict.db at %s — English/Chinese only", cfg.DictPath)
|
||||
}
|
||||
@@ -106,6 +130,7 @@ func main() {
|
||||
r.Use(middleware.RealIP)
|
||||
r.Use(middleware.Logger)
|
||||
r.Use(middleware.Recoverer)
|
||||
r.Use(securityHeaders)
|
||||
|
||||
// Build version: a hash of the embedded SPA shell. Vite rewrites index.html
|
||||
// with content-hashed asset names on every build, so this string changes
|
||||
@@ -149,6 +174,12 @@ func main() {
|
||||
// this id and shows the signed-in writer.
|
||||
pr.Get("/me", users.MeHandler())
|
||||
|
||||
// …and the one thing about herself she can change: which language
|
||||
// Petal is her pair in. It lives here rather than under a /settings
|
||||
// tree because there is exactly one setting and it is a property of
|
||||
// the user row — the same row /me reads back.
|
||||
pr.Patch("/me", users.UpdateMeHandler())
|
||||
|
||||
llmClient := llm.NewLLMClient(cfg)
|
||||
sug := suggestions.New(database, llmClient)
|
||||
|
||||
@@ -176,6 +207,9 @@ func main() {
|
||||
lex := lexicon.NewHandler(database.DB, lexSet)
|
||||
pr.Mount("/word", lex.Routes())
|
||||
pr.Mount("/gloss", lex.GlossRoutes())
|
||||
// The same lookup pointing the other way: a Chinese word to its pinyin
|
||||
// and English senses, for an account whose direction is learning_pair.
|
||||
pr.Mount("/hanzi", lex.HanziRoutes())
|
||||
|
||||
// Vocabulary garden: words the writer looks up are captured here and
|
||||
// surfaced for gentle spaced-repetition review.
|
||||
@@ -200,7 +234,11 @@ func main() {
|
||||
// back to the browser's Web Speech API on its own.
|
||||
if ttsHandler, ok := tts.New(cfg); ok {
|
||||
pr.Mount("/tts", ttsHandler.Routes())
|
||||
log.Printf("read-aloud enabled (TTS endpoint=%s)", cfg.TTSEndpoint)
|
||||
// Name the languages, not just the English endpoint: which
|
||||
// voices a deployment actually reached is the thing worth
|
||||
// seeing at boot, and a missing sidecar is silent otherwise
|
||||
// (a 404 the client answers by quietly using Web Speech).
|
||||
log.Printf("read-aloud enabled (voices: %s)", strings.Join(ttsHandler.Languages(), ", "))
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -221,6 +259,47 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// contentSecurityPolicy is the default policy for everything Petal serves.
|
||||
//
|
||||
// It lives here rather than in the Traefik labels, and that move is the point:
|
||||
// Traefik's customResponseHeaders *sets* a header, overwriting whatever the
|
||||
// application chose, so a policy declared at the edge silently replaces the
|
||||
// stricter one an individual route needs. Stored images need exactly that (an
|
||||
// uploaded SVG is a document that can carry script — see internal/images), and
|
||||
// a rule the edge can quietly undo is not a rule.
|
||||
//
|
||||
// The allowances are what the built frontend actually uses, no more: script
|
||||
// only from Petal itself (Vite emits no inline script — this policy is checked
|
||||
// against dist/index.html), inline *styles* because React's style={{…}} props
|
||||
// compile to style attributes, and Google's font hosts because index.html links
|
||||
// them. object-src and base-uri close the two attribute-injection routes that
|
||||
// survive HTML escaping.
|
||||
const contentSecurityPolicy = "default-src 'self'; " +
|
||||
"script-src 'self'; " +
|
||||
"style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " +
|
||||
"font-src 'self' data: https://fonts.gstatic.com; " +
|
||||
"img-src 'self' data: blob:; " +
|
||||
"media-src 'self' data: blob:; " +
|
||||
"connect-src 'self'; " +
|
||||
"object-src 'none'; " +
|
||||
"base-uri 'self'; " +
|
||||
"form-action 'self'; " +
|
||||
"frame-ancestors 'self'"
|
||||
|
||||
// securityHeaders lays down the baseline response headers before the handler
|
||||
// runs, so a route that needs something stricter — the image store — simply
|
||||
// overwrites its own copy on the way past. Ordering is the mechanism: this is a
|
||||
// floor, not a ceiling.
|
||||
func securityHeaders(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
h := w.Header()
|
||||
h.Set("Content-Security-Policy", contentSecurityPolicy)
|
||||
h.Set("X-Content-Type-Options", "nosniff")
|
||||
h.Set("Referrer-Policy", "same-origin")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// maxAPIBodyBytes caps a JSON API request body at 2 MiB. That's far above any
|
||||
// real document save (the body is text plus lightweight marks; images upload
|
||||
// separately by reference) while still bounding abuse. Exceeding it makes the
|
||||
|
||||
+111
-13
@@ -23,11 +23,13 @@ runs them.
|
||||
|
||||
- **petal** — the single Go binary with the frontend embedded. Publishes no host
|
||||
port; Traefik is the only way in.
|
||||
- **piper-en** / **piper-zh** — read-aloud. Each Piper HTTP server loads exactly
|
||||
one voice, so English and Chinese are separate containers off one image, with
|
||||
the models cached in a shared volume. They sit on an internal network with no
|
||||
published ports, so only Petal can reach them. Adding pt-PT in Phase 21 is a
|
||||
fourth service, not a new image.
|
||||
- **piper-en** / **piper-zh** / **piper-pt** / **piper-fr** — read-aloud. Each
|
||||
Piper HTTP server loads exactly one voice, so every language is its own
|
||||
container off one image, with the models cached in a shared volume. They sit
|
||||
on an internal network with no published ports, so only Petal can reach them.
|
||||
Adding pt-PT in Phase 21 was a third service and fr in Phase 24 a fourth —
|
||||
never a new image, and since Phase 21 never any Go either (the languages are
|
||||
discovered from `TTS_ENDPOINT_<LANG>`/`TTS_VOICE_<LANG>`).
|
||||
|
||||
They run as containers rather than the host systemd units millenia uses because
|
||||
Piper was never actually installed on the VPS, and the `reala` account has no
|
||||
@@ -231,8 +233,21 @@ unauthenticated `/api/health` router that existed only to escape it: every `/api
|
||||
route now answers 401 without a session, and the only thing an anonymous visitor
|
||||
gets is the app shell and a redirect to sign in.
|
||||
|
||||
If you ever run this stack *without* `AUTHENTIK_*` configured — Petal then falls
|
||||
back to the single `local` user — put the gate back before pointing DNS at it:
|
||||
Removing that gate meant a missing `AUTHENTIK_*` variable stopped being a
|
||||
nuisance and became an exposure: Petal would fall back to the single `local`
|
||||
user and hand every anonymous visitor full read and write over the database,
|
||||
saying so only in a log line. So **it now refuses to start instead**:
|
||||
|
||||
```
|
||||
auth: refusing to start unauthenticated at https://petal.parodia.dev.
|
||||
Petal would resolve every anonymous request to the single "local" user, ...
|
||||
```
|
||||
|
||||
The guard defaults on for any `BASE_URL` that isn't loopback, so a laptop
|
||||
checkout still runs open and a deployment cannot. If you genuinely want an
|
||||
unauthenticated instance on a trusted private network, say so out loud with
|
||||
`PETAL_REQUIRE_AUTH=false` — and if it is reachable from anywhere else, put a
|
||||
gate back in front of it first:
|
||||
|
||||
```yaml
|
||||
traefik.http.routers.petal.middlewares: compression@file,petal-headers,petal-auth
|
||||
@@ -241,6 +256,28 @@ traefik.http.middlewares.petal-auth.basicauth.users: ${PETAL_BASIC_AUTH:?}
|
||||
|
||||
with `htpasswd -nbB petal 'your-password'` in `.env` as `PETAL_BASIC_AUTH`.
|
||||
|
||||
### Who gets in
|
||||
|
||||
`PETAL_ALLOWED_SUBS` is a comma-separated list of emails and/or OIDC subject
|
||||
ids. **Set it.** Empty means everyone authentik authenticates, and authentik on
|
||||
this host fronts several applications — a valid account there is not the same as
|
||||
belonging in someone's private journal. An empty list is legal (a single-
|
||||
household instance may want it) and warns at every boot:
|
||||
|
||||
```
|
||||
auth: WARNING — PETAL_ALLOWED_SUBS is empty, so EVERY account ... may sign in
|
||||
```
|
||||
|
||||
### Response headers
|
||||
|
||||
`Content-Security-Policy`, `X-Content-Type-Options` and `Referrer-Policy` are set
|
||||
by the binary, not by the Traefik labels. Traefik's `customresponseheaders`
|
||||
*overwrites*, which would silently replace the stricter policy an individual
|
||||
route picks for itself — the image store serves stored uploads under
|
||||
`default-src 'none'; sandbox` so that an SVG someone pasted into a document
|
||||
cannot run as a page on Petal's own origin. Only HSTS stays at the edge, where
|
||||
TLS is actually terminated.
|
||||
|
||||
---
|
||||
|
||||
## 4a. Moving an account (`scripts/migrate_local_user.py`)
|
||||
@@ -332,13 +369,38 @@ commonest English words; ECDICT covers essentially all of them and is in daily
|
||||
use by a real writer. `lexicon.Set.For` is where that decision lives — one
|
||||
`switch`, changed the day a comparison on her actual lookups says otherwise.
|
||||
|
||||
For pt-PT and French the same measurement reads 62%, which is why they use
|
||||
DreamDict: there is no alternative source for them at all.
|
||||
For pt-PT and French the same measurement reads 62% and 63%, which is why they
|
||||
use DreamDict: there is no alternative source for them at all.
|
||||
|
||||
**The deployed `dict.db` is from 2026-04-04 and has no Spanish data**, because
|
||||
DreamDict grew Spanish support after it was built. A Spanish-pair writer gets
|
||||
empty glosses (and English definitions) until it is rebuilt — do that before
|
||||
the Spanish pair ships.
|
||||
### Rebuilding it
|
||||
|
||||
Rebuilt 2026-07-27 to add Spanish (the previous file predated DreamDict's
|
||||
Spanish support). The recipe, since it will be needed again:
|
||||
|
||||
```bash
|
||||
# on millenia, from a clean checkout of dreamdict main
|
||||
./scripts/download-dict-data.sh ~/dreamdict/data # idempotent; skips what's there
|
||||
go run ./cmd/dictimport --data ~/dreamdict/data --db ./dict.db --clean
|
||||
```
|
||||
|
||||
~6 minutes on 32 cores; the data directory is ~7 GB and mostly already
|
||||
downloaded. **Build to a new path, never over a file in use** — then verify by
|
||||
hash on both ends before swapping.
|
||||
|
||||
Two things worth knowing before trusting a rebuild:
|
||||
|
||||
- The SUBTLEX-US download fails (the source moved behind a manual export). It
|
||||
does not matter: the loader falls back to `SUBTLEX-US.txt`, which is present,
|
||||
and English "frequency" is mostly SCOWL's commonness bucket anyway —
|
||||
1000/800/600/…/50, refined by SUBTLEX for only ~1,600 words. That is why the
|
||||
word-difficulty chip reads `difficulty`, not `frequency`.
|
||||
- Check the *other* languages' counts are unchanged before shipping. The 2026-07
|
||||
rebuild came out byte-identical for en/fr/pt-PT/zh, which is what says it
|
||||
added a language rather than quietly shifting the rest.
|
||||
|
||||
Gloss coverage of the 2,000 commonest English words, after the rebuild:
|
||||
**es 68.6%**, fr 63.1%, pt-PT 62.1%, zh 53.2%. The startup line reports actual
|
||||
per-language row counts, so a database missing a language says so.
|
||||
|
||||
---
|
||||
|
||||
@@ -554,6 +616,42 @@ Petal's env then carries `TTS_ENDPOINT=http://127.0.0.1:5005`,
|
||||
maps language → instance from config, so another language is another instance
|
||||
plus an env pair, no code change.
|
||||
|
||||
**Adding a language (Phase 21 made this literal).** Petal discovers its Piper
|
||||
instances from the environment: English is the unsuffixed
|
||||
`TTS_ENDPOINT`/`TTS_VOICE_EN`, and every other language is a
|
||||
`TTS_ENDPOINT_<LANG>`/`TTS_VOICE_<LANG>` pair. `<LANG>` is the *base* tag —
|
||||
`PT`, not `PT_PT`, because an environment variable name cannot hold a hyphen and
|
||||
only one Portuguese model is loaded regardless. Both halves must be set: an
|
||||
endpoint with no voice is dropped, so a half-finished language reads to the
|
||||
browser as "no voice here, use Web Speech" instead of erroring on every tap. The
|
||||
startup line names what it actually resolved:
|
||||
|
||||
```
|
||||
read-aloud enabled (voices: en=en_US-amy-medium, pt=pt_PT-tugão-medium, zh=zh_CN-huayan-medium)
|
||||
```
|
||||
|
||||
**Portuguese: `pt_PT-tugão-medium` is the only European voice Piper ships.** The
|
||||
other five `pt_*` models in the catalogue are all Brazilian, so the voice has to
|
||||
be named explicitly for the same reason the Hunspell dictionary did (Phase 21):
|
||||
the obvious default is the wrong country. Check what exists before assuming:
|
||||
|
||||
```bash
|
||||
docker exec petal-piper-en python -c "import urllib.request,json; \
|
||||
d=json.load(urllib.request.urlopen('https://huggingface.co/rhasspy/piper-voices/resolve/main/voices.json')); \
|
||||
print([k for k in d if k.startswith('pt')])"
|
||||
```
|
||||
|
||||
**French: the opposite situation, and worth knowing it is.** Every `fr_*` voice
|
||||
in the catalogue is `fr_FR`, so there is no wrong country to land on by default
|
||||
and no Québec voice to choose instead; `fr_FR-siwis-medium` is picked to match
|
||||
the register of the other three rather than to avoid anything. The name is also
|
||||
plain ASCII, so the entrypoint's percent-encoded download fallback — which
|
||||
exists only because `tugão` broke `piper.download_voices` — never fires here.
|
||||
|
||||
**Slow replay.** `POST /api/tts` takes `slow: true`, which raises Piper's
|
||||
`length_scale` to about 4/3 (≈0.75× pace). It is a separate cache entry, not a
|
||||
playback-rate trick, so the slow clip is synthesized once and then instant.
|
||||
|
||||
**Piper version note:** piper-tts moved synthesis from `POST /` to
|
||||
`POST /synthesize` in 1.6.0, with an identical request body. `TTS_PATH` selects
|
||||
which — it defaults to `/`, and both the VPS compose and millenia's `start.sh`
|
||||
|
||||
+48
-16
@@ -20,13 +20,9 @@ TZ=Europe/Lisbon
|
||||
PETAL_UID=1001
|
||||
PETAL_GID=1001
|
||||
|
||||
# --- Interim edge gate (delete when Phase 16 auth lands) ---------------------
|
||||
# Petal has no authentication of its own yet — StaticResolver hands every
|
||||
# request the same local user — so Traefik holds the door with basic auth until
|
||||
# the OIDC flow exists. user:bcrypt-hash, as produced by:
|
||||
# htpasswd -nbB petal 'your-password'
|
||||
# /api/health is deliberately exempt (its own router) so monitoring still works.
|
||||
PETAL_BASIC_AUTH=
|
||||
# (The interim PETAL_BASIC_AUTH edge gate is gone: Petal authenticates for
|
||||
# itself now, and the auth block at the bottom of this file is what holds the
|
||||
# door. A second password in front of a real login is one more thing to lose.)
|
||||
|
||||
# --- LLM (millenia, over headscale) ------------------------------------------
|
||||
# The only cross-VPN dependency. Petal degrades warmly when it's unreachable:
|
||||
@@ -49,22 +45,58 @@ LLM_TIMEOUT=90s
|
||||
# --- Read-aloud (Piper sidecars) ---------------------------------------------
|
||||
# Endpoints are wired in docker-compose.yml; these pick the voice each sidecar
|
||||
# loads. Changing one means recreating that container so it downloads the model.
|
||||
#
|
||||
# A language is routable only when both halves are set — a TTS_ENDPOINT_XX with
|
||||
# no TTS_VOICE_XX reads as "no voice for this language" and the browser's own
|
||||
# synthesizer takes over, rather than as an instance that errors on every
|
||||
# request. Adding es is a compose service plus a pair of lines here.
|
||||
#
|
||||
# pt_PT-tugão-medium is the only European Portuguese voice Piper ships; every
|
||||
# other pt model in the catalogue is Brazilian. French has the opposite
|
||||
# property — every fr voice in the catalogue is fr_FR — so there is no wrong
|
||||
# country to land on and no non-ASCII name to trip the downloader.
|
||||
TTS_VOICE_EN=en_US-amy-medium
|
||||
TTS_VOICE_ZH=zh_CN-huayan-medium
|
||||
TTS_VOICE_PT=pt_PT-tugão-medium
|
||||
TTS_VOICE_FR=fr_FR-siwis-medium
|
||||
# Mexican, not peninsular — the es pack is written in neutral Latin American
|
||||
# Spanish, and es_ES-davefx-medium would read it in the accent it avoids.
|
||||
TTS_VOICE_ES=es_MX-ald-medium
|
||||
TTS_AUDIO_FORMAT=mp3
|
||||
TTS_TIMEOUT=15s
|
||||
|
||||
# --- Auth (Authentik OIDC) ---------------------------------------------------
|
||||
# Authentik already runs on this host. Set all three and Petal authenticates
|
||||
# for itself; leave any unset and it falls back to the single `local` user
|
||||
# (which on a public host means the Traefik basic-auth gate must stay).
|
||||
# NOT OPTIONAL HERE. Authentik already runs on this host; set all three and
|
||||
# Petal authenticates for itself.
|
||||
#
|
||||
# Leave any of them unset and Petal REFUSES TO START, because the alternative is
|
||||
# worse: it would otherwise fall back to resolving every anonymous request to
|
||||
# the single `local` user, handing the open internet full read and write over
|
||||
# every document in the database. That fallback is right on a laptop and a
|
||||
# catastrophe on this host, so the guard is on for any non-loopback BASE_URL.
|
||||
# See PETAL_REQUIRE_AUTH below.
|
||||
#
|
||||
# AUTHENTIK_URL is the provider's issuer, and the redirect URI to register in
|
||||
# Authentik is https://petal.parodia.dev/auth/callback.
|
||||
# AUTHENTIK_URL=https://auth.parodia.dev/application/o/petal/
|
||||
# AUTHENTIK_CLIENT_ID=petal
|
||||
# AUTHENTIK_CLIENT_SECRET=
|
||||
AUTHENTIK_URL=https://auth.parodia.dev/application/o/petal/
|
||||
AUTHENTIK_CLIENT_ID=petal
|
||||
AUTHENTIK_CLIENT_SECRET=
|
||||
|
||||
# Who may sign in: comma-separated subject ids and/or emails.
|
||||
#
|
||||
# Who may sign in: comma-separated subject ids and/or emails. Empty = anyone
|
||||
# Authentik authenticates, which is wider than this instance wants.
|
||||
# PETAL_ALLOWED_SUBS=
|
||||
# SET THIS. Empty means everyone Authentik authenticates, and Authentik on this
|
||||
# host fronts half a dozen applications — being a valid user there is not the
|
||||
# same as belonging in someone's private journal. An empty value is legal (a
|
||||
# single-household instance may genuinely want it) and says so loudly in the
|
||||
# startup log every boot.
|
||||
#
|
||||
# An email is knowable in advance; a subject id is an opaque uuid nobody can
|
||||
# know before that person's first login. Use emails to invite, subject ids to
|
||||
# pin.
|
||||
PETAL_ALLOWED_SUBS=her@example.com,me@example.com
|
||||
|
||||
# The guard itself. Defaulted from BASE_URL — loopback origins run open, real
|
||||
# ones demand a login — so it does not normally need setting. Set it to false
|
||||
# only for a deployment genuinely reachable from nowhere but a trusted network,
|
||||
# and understand that it means anyone who reaches Petal is the `local` user.
|
||||
# PETAL_REQUIRE_AUTH=true
|
||||
|
||||
@@ -10,7 +10,42 @@ port="${PIPER_PORT:-5000}"
|
||||
|
||||
if [ ! -f "${data_dir}/${voice}.onnx" ]; then
|
||||
echo ">> downloading voice ${voice} into ${data_dir}"
|
||||
python -m piper.download_voices "${voice}" --data-dir "${data_dir}"
|
||||
# piper.download_voices cannot fetch a voice whose name isn't ASCII, and the
|
||||
# only European Portuguese voice in the catalogue is pt_PT-tugão-medium:
|
||||
# the downloader pastes the name straight into the request line, and
|
||||
# http.client encodes that as ASCII, so it dies with UnicodeEncodeError on
|
||||
# the ã before a byte leaves the container. Every pt_BR voice downloads
|
||||
# fine — the failure lands precisely on the voice the pt-PT pair needs.
|
||||
#
|
||||
# So: try the supported path, and fall back to fetching the two files
|
||||
# ourselves with the URL percent-encoded, which is all the downloader was
|
||||
# missing. Same host, same files, same destination names.
|
||||
python -m piper.download_voices "${voice}" --data-dir "${data_dir}" || {
|
||||
echo ">> download_voices failed for ${voice}; fetching directly (non-ASCII voice name)"
|
||||
python - "${voice}" "${data_dir}" <<'PY'
|
||||
import json, sys, urllib.parse, urllib.request
|
||||
|
||||
voice, data_dir = sys.argv[1], sys.argv[2]
|
||||
BASE = "https://huggingface.co/rhasspy/piper-voices/resolve/main/"
|
||||
|
||||
catalogue = json.load(urllib.request.urlopen(BASE + "voices.json", timeout=120))
|
||||
entry = catalogue.get(voice)
|
||||
if entry is None:
|
||||
sys.exit(f"no voice named {voice!r} in the catalogue")
|
||||
|
||||
# The catalogue keys the files by repo path; only the model and its config are
|
||||
# needed to serve (MODEL_CARD is licence text).
|
||||
for path in entry["files"]:
|
||||
if not path.endswith((".onnx", ".onnx.json")):
|
||||
continue
|
||||
url = BASE + urllib.parse.quote(path)
|
||||
dest = f"{data_dir}/{path.rsplit('/', 1)[-1]}"
|
||||
print(f">> {url} -> {dest}", flush=True)
|
||||
with urllib.request.urlopen(url, timeout=600) as r, open(dest, "wb") as out:
|
||||
while chunk := r.read(1 << 20):
|
||||
out.write(chunk)
|
||||
PY
|
||||
}
|
||||
fi
|
||||
|
||||
echo ">> serving ${voice} on :${port}"
|
||||
|
||||
+78
-4
@@ -46,6 +46,13 @@ services:
|
||||
# separate containers; the handler maps language → instance from config.
|
||||
TTS_ENDPOINT: http://piper-en:5000
|
||||
TTS_ENDPOINT_ZH: http://piper-zh:5000
|
||||
# A language is discovered from the TTS_ENDPOINT_<LANG>/TTS_VOICE_<LANG>
|
||||
# pair, so fr and es cost a service and two lines rather than a code
|
||||
# change. <LANG> is the base tag — an env var name can't hold pt-PT's
|
||||
# hyphen, and there is one Portuguese voice loaded either way.
|
||||
TTS_ENDPOINT_PT: http://piper-pt:5000
|
||||
TTS_ENDPOINT_FR: http://piper-fr:5000
|
||||
TTS_ENDPOINT_ES: http://piper-es:5000
|
||||
# The sidecars run piper-tts 1.6.0, which serves synthesis on
|
||||
# /synthesize; millenia's older server keeps the default "/".
|
||||
TTS_PATH: /synthesize
|
||||
@@ -75,6 +82,7 @@ services:
|
||||
depends_on:
|
||||
- piper-en
|
||||
- piper-zh
|
||||
- piper-pt
|
||||
labels:
|
||||
traefik.enable: "true"
|
||||
traefik.docker.network: traefik
|
||||
@@ -91,11 +99,18 @@ services:
|
||||
# second password in front of a real login is just one more thing to lose.
|
||||
traefik.http.routers.petal.middlewares: compression@file,petal-headers
|
||||
traefik.http.services.petal.loadbalancer.server.port: "8080"
|
||||
# Petal is a private writing space: no framing, no sniffing, HSTS on.
|
||||
traefik.http.middlewares.petal-headers.headers.customresponseheaders.Content-Security-Policy: frame-ancestors 'self'
|
||||
# HSTS is the edge's business — it is a statement about the TLS
|
||||
# termination, which happens here and not in the container.
|
||||
#
|
||||
# The Content-Security-Policy that used to sit alongside it has moved into
|
||||
# the app (see securityHeaders in cmd/server/main.go). customresponseheaders
|
||||
# *overwrites*, so a policy set here would silently replace the stricter
|
||||
# one an individual route chooses for itself — which is exactly what the
|
||||
# image store does to keep an uploaded SVG from running as a page. A rule
|
||||
# the edge can quietly undo is not a rule. X-Content-Type-Options and
|
||||
# Referrer-Policy moved with it for the same reason: one place to read,
|
||||
# and no dependence on this file being deployed alongside the binary.
|
||||
traefik.http.middlewares.petal-headers.headers.customresponseheaders.Strict-Transport-Security: max-age=31536000; includeSubDomains
|
||||
traefik.http.middlewares.petal-headers.headers.customresponseheaders.X-Content-Type-Options: nosniff
|
||||
traefik.http.middlewares.petal-headers.headers.customresponseheaders.Referrer-Policy: same-origin
|
||||
|
||||
piper-en:
|
||||
build:
|
||||
@@ -123,6 +138,65 @@ services:
|
||||
networks:
|
||||
- internal
|
||||
|
||||
# European Portuguese, for the pt-PT pair. pt_PT-tugão-medium is the *only*
|
||||
# European voice in Piper's catalogue — the other five Portuguese models are
|
||||
# all pt_BR — so the default anyone reaches for is the Brazilian one, exactly
|
||||
# as it was with the Hunspell dictionary in Phase 21. Named here rather than
|
||||
# left to the image default for that reason.
|
||||
piper-pt:
|
||||
build:
|
||||
context: deploy/piper
|
||||
image: petal-piper:local
|
||||
container_name: petal-piper-pt
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PIPER_VOICE: ${TTS_VOICE_PT:-pt_PT-tugão-medium}
|
||||
volumes:
|
||||
- piper-voices:/voices
|
||||
networks:
|
||||
- internal
|
||||
|
||||
# French, for the fr pair. The opposite situation to Portuguese: every French
|
||||
# voice Piper ships is fr_FR, so there is no wrong country to land on by
|
||||
# default, and the name is plain ASCII so the entrypoint's percent-encoded
|
||||
# fallback (added for tugão) never has to fire. siwis-medium to match the
|
||||
# register of the other three.
|
||||
piper-fr:
|
||||
build:
|
||||
context: deploy/piper
|
||||
image: petal-piper:local
|
||||
container_name: petal-piper-fr
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PIPER_VOICE: ${TTS_VOICE_FR:-fr_FR-siwis-medium}
|
||||
volumes:
|
||||
- piper-voices:/voices
|
||||
networks:
|
||||
- internal
|
||||
|
||||
# Spanish, for the es pair — and the Portuguese trap rather than the French
|
||||
# one. Piper's catalogue has nine Spanish voices, six of them es_ES, and the
|
||||
# obvious pick (es_ES-davefx-medium, which the build plan itself named) is
|
||||
# peninsular. The es pack is written in neutral Latin American Spanish, so a
|
||||
# Castilian voice would read it aloud in the accent the copy was written to
|
||||
# avoid — the same wrong-country default that pt-PT hit through packaging,
|
||||
# arriving here through the voice list. Only two Latin American voices exist,
|
||||
# es_AR-daniela-high and es_MX; Mexican is the neutral broadcast standard and
|
||||
# ald-medium matches the register of the other four. ASCII, so the
|
||||
# percent-encoded download fallback added for tugão never has to fire.
|
||||
piper-es:
|
||||
build:
|
||||
context: deploy/piper
|
||||
image: petal-piper:local
|
||||
container_name: petal-piper-es
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PIPER_VOICE: ${TTS_VOICE_ES:-es_MX-ald-medium}
|
||||
volumes:
|
||||
- piper-voices:/voices
|
||||
networks:
|
||||
- internal
|
||||
|
||||
networks:
|
||||
# Created and owned by the host's Traefik stack.
|
||||
traefik:
|
||||
|
||||
@@ -129,7 +129,10 @@ func (o *OIDC) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/login", o.login)
|
||||
r.Get("/callback", o.callback)
|
||||
r.Get("/logout", o.logout)
|
||||
// POST only. Signing out is a state change, and SameSite=Lax deliberately
|
||||
// *does* send the session cookie on a top-level cross-site GET — so a GET
|
||||
// route here means any page on the internet can sign her out mid-draft by
|
||||
// linking to it, or embedding it as an image. Small harm, free to remove.
|
||||
r.Post("/logout", o.logout)
|
||||
return r
|
||||
}
|
||||
@@ -275,8 +278,8 @@ func (o *OIDC) callback(w http.ResponseWriter, r *http.Request) {
|
||||
// matters: clearing only the cookie leaves a token that still works if it was
|
||||
// ever captured.
|
||||
func (o *OIDC) logout(w http.ResponseWriter, r *http.Request) {
|
||||
if c, err := r.Cookie(SessionCookie); err == nil && c.Value != "" {
|
||||
if err := o.sessions.Revoke(c.Value); err != nil {
|
||||
if token := SessionToken(r); token != "" {
|
||||
if err := o.sessions.Revoke(token); err != nil {
|
||||
log.Printf("auth: revoke failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -234,7 +234,7 @@ func TestLoginRoundTrip(t *testing.T) {
|
||||
|
||||
// Signing out revokes server-side, not just in the browser.
|
||||
out := httptest.NewRecorder()
|
||||
flow.ServeHTTP(out, jar.attach(httptest.NewRequest(http.MethodGet, "/logout", nil)))
|
||||
flow.ServeHTTP(out, jar.attach(httptest.NewRequest(http.MethodPost, "/logout", nil)))
|
||||
if out.Code != http.StatusFound {
|
||||
t.Fatalf("logout status=%d", out.Code)
|
||||
}
|
||||
@@ -243,6 +243,19 @@ func TestLoginRoundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Signing out is a state change, so it must not be reachable by GET: with
|
||||
// SameSite=Lax the session cookie *is* sent on a top-level cross-site
|
||||
// navigation, which would let any page on the internet sign her out mid-draft.
|
||||
func TestLogoutRejectsGET(t *testing.T) {
|
||||
_, flow, _, _ := newFlow(t, nil)
|
||||
|
||||
out := httptest.NewRecorder()
|
||||
flow.ServeHTTP(out, httptest.NewRequest(http.MethodGet, "/logout", nil))
|
||||
if out.Code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("GET /logout status=%d, want 405", out.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// A callback whose state doesn't match the cookie is a forged one.
|
||||
func TestCallbackRejectsBadState(t *testing.T) {
|
||||
idp, flow, sessions, _ := newFlow(t, nil)
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// patchMe drives UpdateMeHandler as the given user would reach it: behind the
|
||||
// middleware, which is the only thing that puts an id in the context.
|
||||
func patchMe(t *testing.T, users *UserStore, id, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
r := httptest.NewRequest(http.MethodPatch, "/me", strings.NewReader(body))
|
||||
r = r.WithContext(WithUser(r.Context(), id))
|
||||
w := httptest.NewRecorder()
|
||||
users.UpdateMeHandler()(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
func TestSetPairLang(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPair("bob", "pt-PT", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set pt-PT: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "pt-PT" {
|
||||
t.Fatalf("pair_lang = %q, want pt-PT", u.PairLang)
|
||||
}
|
||||
|
||||
// Every pair with a langpack, not just the first one: this list and the
|
||||
// frontend's PACKS are two copies of the same fact, and the day they
|
||||
// disagree is the day she can pick a pair the app cannot render.
|
||||
if err := users.SetPair("bob", "fr", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set fr: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "fr" {
|
||||
t.Fatalf("pair_lang = %q, want fr", u.PairLang)
|
||||
}
|
||||
|
||||
if err := users.SetPair("bob", "es", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set es: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "es" {
|
||||
t.Fatalf("pair_lang = %q, want es", u.PairLang)
|
||||
}
|
||||
|
||||
// And back — a writer who tries a pair and doesn't like it must be able to
|
||||
// return, which is the whole reason the picker exists.
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set zh: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "zh" {
|
||||
t.Fatalf("pair_lang = %q, want zh", u.PairLang)
|
||||
}
|
||||
}
|
||||
|
||||
// A pair the frontend has no langpack for must not be storable. Accepting it
|
||||
// would leave her looking at Chinese copy with no way back except a lucky guess.
|
||||
func TestSetPairLangRejectsUnshippedPairs(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
// The near-misses are the ones that matter, and there are two of them now.
|
||||
// "pt-BR" must not be quietly served European copy and a European voice;
|
||||
// "es-ES" is the same mistake pointing the other way, because the es pack is
|
||||
// deliberately Latin American and reads itself aloud in a Mexican voice. A
|
||||
// regional code Petal has not decided about is refused rather than rounded
|
||||
// to the nearest pack it happens to have.
|
||||
for _, lang := range []string{"es-ES", "pt-BR", "fr-CA", "de", "klingon", "", " "} {
|
||||
if err := users.SetPair("bob", lang, DirectionLearningEn); err == nil {
|
||||
t.Fatalf("stored unshipped pair %q", lang)
|
||||
}
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "zh" {
|
||||
t.Fatalf("a refused write still moved pair_lang to %q", u.PairLang)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetPairLangUnknownUser(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
if err := users.SetPair("nobody", "pt-PT", DirectionLearningEn); err == nil {
|
||||
t.Fatal("set a pair language on an account that does not exist")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMeHandler(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
w := patchMe(t, users, "bob", `{"pair_lang":"pt-PT"}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
// The whole user comes back, so the client can re-read the pair from the
|
||||
// server instead of assuming its request took.
|
||||
var got db.User
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.ID != "bob" || got.PairLang != "pt-PT" {
|
||||
t.Fatalf("response = %+v, want bob on pt-PT", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMeHandlerRejects(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
for name, body := range map[string]string{
|
||||
"unshipped pair": `{"pair_lang":"es-ES"}`,
|
||||
"unknown direction": `{"direction":"learning_klingon"}`,
|
||||
"not json": `pt-PT`,
|
||||
} {
|
||||
if w := patchMe(t, users, "bob", body); w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: status = %d, want 400", name, w.Code)
|
||||
}
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "zh" {
|
||||
t.Fatalf("a rejected request still moved pair_lang to %q", u.PairLang)
|
||||
}
|
||||
|
||||
// A caller the middleware never resolved (or whose row is gone) is a lapsed
|
||||
// session, not a bad request — the client turns 401 into the sign-in overlay.
|
||||
if w := patchMe(t, users, "nobody", `{"pair_lang":"pt-PT"}`); w.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("unknown user: status = %d, want 401", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// An empty body used to be a 400, back when pair_lang was the only field and a
|
||||
// request that named none of it could only be a client bug. With two optional
|
||||
// fields it is an ordinary PATCH that changes nothing, and it has to be: the
|
||||
// picker sends one field without knowing the other, and "omitted" has to mean
|
||||
// "leave it alone" for that to be safe.
|
||||
func TestUpdateMeHandlerEmptyBodyChangesNothing(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningPair); err != nil {
|
||||
t.Fatalf("set up: %v", err)
|
||||
}
|
||||
w := patchMe(t, users, "bob", `{}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
u, _ := users.Get("bob")
|
||||
if u.PairLang != "zh" || u.Direction != DirectionLearningPair {
|
||||
t.Fatalf("empty PATCH moved the account to %q/%q", u.PairLang, u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The direction axis: an account can be turned around and turned back, and the
|
||||
// default every existing row already carries is the one it had before the column
|
||||
// existed.
|
||||
func TestDirectionRoundTrip(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("a fresh account starts at %q, want %q", u.Direction, DirectionLearningEn)
|
||||
}
|
||||
|
||||
w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("turn around: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
var got db.User
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
// The response carries the direction, not just the pair — the client reads
|
||||
// its whole state back from here rather than assuming the write took.
|
||||
if got.Direction != DirectionLearningPair || got.PairLang != "zh" {
|
||||
t.Fatalf("response = %+v, want bob learning zh", got)
|
||||
}
|
||||
|
||||
if w := patchMe(t, users, "bob", `{"direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||
t.Fatalf("turn back: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("direction = %q after turning back", u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The refusal this axis exists to make: a pair with no learner-side data cannot
|
||||
// be learned toward, however good its langpack is. fr and es have copy, voices
|
||||
// and spelling dictionaries, and no `learner` block in their packs to offer the
|
||||
// choice with — so the server keeps saying no until one is written.
|
||||
//
|
||||
// pt-PT is deliberately no longer in this list; see TestLearnerDirectionForPtPT
|
||||
// below and the argument in `learnerPairs`.
|
||||
func TestLearnerDirectionRefusedForPairsWithoutData(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
for _, lang := range []string{"fr", "es"} {
|
||||
if err := users.SetPair("bob", lang, DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set %s: %v", lang, err)
|
||||
}
|
||||
w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: status = %d, want 400", lang, w.Code)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("%s: a refused write still moved direction to %q", lang, u.Direction)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The other direction of that same rule, and the one a native English speaker
|
||||
// writing Portuguese depends on.
|
||||
//
|
||||
// This is not only a settings toggle: `direction` is what decides which language
|
||||
// Petal *explains* in (see suggestions.targetFor), so an account that cannot
|
||||
// reach learning_pair gets its Portuguese annotated in Portuguese with no way to
|
||||
// ask for English. Pinned in both directions — the move must take, and it must
|
||||
// still be there when the account is read back.
|
||||
func TestLearnerDirectionForPtPT(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPair("bob", "pt-PT", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set pt-PT: %v", err)
|
||||
}
|
||||
if w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`); w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d (%s), want 200", w.Code, w.Body.String())
|
||||
}
|
||||
u, _ := users.Get("bob")
|
||||
if u.Direction != DirectionLearningPair || u.PairLang != "pt-PT" {
|
||||
t.Fatalf("account = %+v, want pt-PT learning_pair", u)
|
||||
}
|
||||
|
||||
// And it can be turned back, the same as zh.
|
||||
if w := patchMe(t, users, "bob", `{"direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||
t.Fatalf("turn back: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("direction = %q after turning back", u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The two-field combination the handler validates as one decision. An account
|
||||
// already learning Chinese that asks only to change pair is asking for a state
|
||||
// neither field names on its own — French with segmentation — and it must not
|
||||
// arrive by leaving one field out.
|
||||
func TestPairChangeCannotStrandTheLearnerDirection(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningPair); err != nil {
|
||||
t.Fatalf("set up: %v", err)
|
||||
}
|
||||
if w := patchMe(t, users, "bob", `{"pair_lang":"fr"}`); w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", w.Code)
|
||||
}
|
||||
u, _ := users.Get("bob")
|
||||
if u.PairLang != "zh" || u.Direction != DirectionLearningPair {
|
||||
t.Fatalf("refused write left the account at %q/%q", u.PairLang, u.Direction)
|
||||
}
|
||||
|
||||
// Naming both at once is how that move is actually made, and it works.
|
||||
if w := patchMe(t, users, "bob", `{"pair_lang":"fr","direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||
t.Fatalf("both fields: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "fr" || u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("account = %q/%q, want fr/learning_en", u.PairLang, u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The CHECK constraint is the last line, below the handler and below SetPair:
|
||||
// a direction that reaches the column by any other route is still refused.
|
||||
func TestDirectionCheckConstraint(t *testing.T) {
|
||||
_, users, database := newStores(t)
|
||||
if _, err := database.Exec(`UPDATE users SET direction = 'sideways' WHERE id = 'bob'`); err == nil {
|
||||
t.Fatal("the users.direction CHECK accepted 'sideways'")
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("direction = %q after a refused UPDATE", u.Direction)
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,31 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// SessionCookie is the cookie carrying the opaque session token.
|
||||
const SessionCookie = "petal_session"
|
||||
// The cookie carrying the opaque session token, in its two spellings.
|
||||
//
|
||||
// Over https the name takes the __Host- prefix, which is not decoration: the
|
||||
// browser will only accept such a cookie if it is Secure, Path=/, and carries
|
||||
// no Domain attribute — and, crucially, refuses to let any other host set it.
|
||||
// Without the prefix, anything that can write cookies for a sibling name under
|
||||
// parodia.dev (another service on the box, a subdomain takeover) can plant a
|
||||
// session cookie in her browser that Petal will then read as hers.
|
||||
//
|
||||
// The prefix is impossible over plain http, because it requires Secure and a
|
||||
// browser drops a Secure cookie on an insecure origin. So local development
|
||||
// keeps the bare name, and the name in use follows the same `secure` flag the
|
||||
// rest of the cookie does.
|
||||
const (
|
||||
SessionCookie = "petal_session"
|
||||
HostSessionCookie = "__Host-petal_session"
|
||||
)
|
||||
|
||||
// sessionCookieName is the name to *write* under this scheme.
|
||||
func sessionCookieName(secure bool) string {
|
||||
if secure {
|
||||
return HostSessionCookie
|
||||
}
|
||||
return SessionCookie
|
||||
}
|
||||
|
||||
const (
|
||||
// sessionTTL is how long a session lives without use. Thirty days, sliding:
|
||||
@@ -78,11 +101,28 @@ func (s *SessionStore) Create(userID, userAgent string) (string, error) {
|
||||
// Resolve implements [Resolver]: it reads the session cookie, validates it, and
|
||||
// returns the user it belongs to — extending the session's life while it does.
|
||||
func (s *SessionStore) Resolve(r *http.Request) (string, error) {
|
||||
c, err := r.Cookie(SessionCookie)
|
||||
if err != nil || c.Value == "" {
|
||||
token := SessionToken(r)
|
||||
if token == "" {
|
||||
return "", ErrNoSession
|
||||
}
|
||||
return s.userFor(c.Value)
|
||||
return s.userFor(token)
|
||||
}
|
||||
|
||||
// SessionToken pulls the raw session token out of a request, preferring the
|
||||
// __Host- spelling.
|
||||
//
|
||||
// Both are read because a deployment that was signing people in before the
|
||||
// prefix existed has browsers holding the old name; those sessions stay valid
|
||||
// and quietly re-issue under the new name at the next sign-in. The prefixed one
|
||||
// wins where both are present, since it is the one another host could not have
|
||||
// planted.
|
||||
func SessionToken(r *http.Request) string {
|
||||
for _, name := range []string{HostSessionCookie, SessionCookie} {
|
||||
if c, err := r.Cookie(name); err == nil && c.Value != "" {
|
||||
return c.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// userFor validates a raw token and slides its expiry forward.
|
||||
@@ -146,7 +186,7 @@ func hashToken(token string) string {
|
||||
// browser drop the cookie and silently break local login.
|
||||
func SetSessionCookie(w http.ResponseWriter, token string, secure bool) {
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: SessionCookie,
|
||||
Name: sessionCookieName(secure),
|
||||
Value: token,
|
||||
Path: "/",
|
||||
HttpOnly: true,
|
||||
@@ -158,9 +198,18 @@ func SetSessionCookie(w http.ResponseWriter, token string, secure bool) {
|
||||
|
||||
// ClearSessionCookie expires the session cookie in the browser. The matching
|
||||
// server-side row must be revoked separately — that's the half that counts.
|
||||
//
|
||||
// Both spellings are expired, not just the one currently written: a browser
|
||||
// carrying a pre-prefix cookie must not be left holding it after signing out,
|
||||
// which is precisely the case where "clear the cookie" is the part the user can
|
||||
// see working.
|
||||
func ClearSessionCookie(w http.ResponseWriter, secure bool) {
|
||||
for _, name := range []string{HostSessionCookie, SessionCookie} {
|
||||
if name == HostSessionCookie && !secure {
|
||||
continue // the browser would reject a non-Secure __Host- cookie
|
||||
}
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: SessionCookie,
|
||||
Name: name,
|
||||
Value: "",
|
||||
Path: "/",
|
||||
HttpOnly: true,
|
||||
@@ -168,4 +217,5 @@ func ClearSessionCookie(w http.ResponseWriter, secure bool) {
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
MaxAge: -1,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -309,3 +309,67 @@ func TestAllowlist(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Over https the cookie takes the __Host- prefix, which the browser will only
|
||||
// accept from the exact host that set it — closing the door on a sibling
|
||||
// service under the same registrable domain planting a session in her browser.
|
||||
// Over plain http it cannot: the prefix requires Secure, and a browser drops a
|
||||
// Secure cookie on an insecure origin, so local development would silently stop
|
||||
// logging in.
|
||||
func TestSessionCookieNamePerScheme(t *testing.T) {
|
||||
secure := httptest.NewRecorder()
|
||||
SetSessionCookie(secure, "tok", true)
|
||||
c := secure.Result().Cookies()[0]
|
||||
if c.Name != HostSessionCookie {
|
||||
t.Fatalf("https cookie name=%q, want %q", c.Name, HostSessionCookie)
|
||||
}
|
||||
// The prefix is a promise about these three attributes; a browser rejects
|
||||
// the cookie outright if any is wrong.
|
||||
if !c.Secure || c.Path != "/" || c.Domain != "" {
|
||||
t.Fatalf("__Host- cookie violates its own contract: %+v", c)
|
||||
}
|
||||
|
||||
insecure := httptest.NewRecorder()
|
||||
SetSessionCookie(insecure, "tok", false)
|
||||
if name := insecure.Result().Cookies()[0].Name; name != SessionCookie {
|
||||
t.Fatalf("http cookie name=%q, want %q", name, SessionCookie)
|
||||
}
|
||||
}
|
||||
|
||||
// A browser holding a cookie issued before the prefix existed must stay signed
|
||||
// in — and start using the new name at its next sign-in, not be logged out to
|
||||
// get there.
|
||||
func TestResolveAcceptsEitherCookieName(t *testing.T) {
|
||||
store, _, _ := newStores(t)
|
||||
token, err := store.Create("bob", "test-agent")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, name := range []string{SessionCookie, HostSessionCookie} {
|
||||
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
r.AddCookie(&http.Cookie{Name: name, Value: token})
|
||||
got, err := store.Resolve(r)
|
||||
if err != nil || got != "bob" {
|
||||
t.Fatalf("%s: resolved to %q (err=%v)", name, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Signing out must not leave the browser holding either spelling.
|
||||
func TestClearSessionCookieExpiresBothNames(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
ClearSessionCookie(rec, true)
|
||||
|
||||
cleared := map[string]bool{}
|
||||
for _, c := range rec.Result().Cookies() {
|
||||
if c.MaxAge < 0 {
|
||||
cleared[c.Name] = true
|
||||
}
|
||||
}
|
||||
for _, name := range []string{SessionCookie, HostSessionCookie} {
|
||||
if !cleared[name] {
|
||||
t.Errorf("%s was left in the browser after signing out", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+196
-2
@@ -2,6 +2,7 @@ package auth
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -48,9 +49,9 @@ func (u *UserStore) Upsert(sub, email, displayName string) error {
|
||||
func (u *UserStore) Get(id string) (db.User, error) {
|
||||
var user db.User
|
||||
err := u.db.QueryRow(
|
||||
`SELECT id, email, COALESCE(display_name, ''), created_at, pair_lang
|
||||
`SELECT id, email, COALESCE(display_name, ''), created_at, pair_lang, direction
|
||||
FROM users WHERE id = ?`, id,
|
||||
).Scan(&user.ID, &user.Email, &user.DisplayName, &user.CreatedAt, &user.PairLang)
|
||||
).Scan(&user.ID, &user.Email, &user.DisplayName, &user.CreatedAt, &user.PairLang, &user.Direction)
|
||||
return user, err
|
||||
}
|
||||
|
||||
@@ -68,6 +69,199 @@ func (u *UserStore) MeHandler() http.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// The pairs a writer may actually choose, in the order the picker offers them.
|
||||
//
|
||||
// This is deliberately *not* internal/llm's list of languages. That one names
|
||||
// every pair the prompts know how to talk about, which is a cheap thing to add;
|
||||
// this one names the pairs Petal can render itself in, which requires a langpack
|
||||
// on the frontend. Accepting a code with no pack would leave her looking at
|
||||
// Chinese with no way back except another guess, so the server refuses it. es
|
||||
// joined on the day its pack landed, not before.
|
||||
//
|
||||
// These four are now every pair PairLang names on the frontend, which makes the
|
||||
// two lists look redundant. They are not: the next pair will exist in the type
|
||||
// and in the prompts long before it has copy, and this list is the one that
|
||||
// says a writer may actually be sent there.
|
||||
var shippedPairs = []string{"zh", "pt-PT", "fr", "es"}
|
||||
|
||||
func pairIsShipped(lang string) bool {
|
||||
for _, p := range shippedPairs {
|
||||
if p == lang {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The two directions a pair can be travelled in. `DirectionLearningEn` is the
|
||||
// original assumption made explicit: the writer is native in X and practising
|
||||
// English. `DirectionLearningPair` is the other way round.
|
||||
const (
|
||||
DirectionLearningEn = "learning_en"
|
||||
DirectionLearningPair = "learning_pair"
|
||||
)
|
||||
|
||||
// The pairs whose *learner* direction Petal can actually serve, which is a
|
||||
// narrower thing than a shipped pair and narrower again than a langpack.
|
||||
//
|
||||
// Turning a pair around needs data no langpack carries: a way to find word
|
||||
// boundaries, and a dictionary that reads from the pair language into English. A
|
||||
// pair missing either would leave a writer looking at an editor that silently
|
||||
// does nothing when she hovers — worse than a missing pack, which at least reads
|
||||
// as a bug rather than as an absence. So the server refuses, for the same reason
|
||||
// and by the same mechanism as `shippedPairs`.
|
||||
//
|
||||
// Chinese has both as of Phase 26 (CC-CEDICT + jieba). Portuguese turns out to
|
||||
// have both as well, and the original note here — "French, Spanish and
|
||||
// Portuguese have neither" — was written one phase too early to see it:
|
||||
//
|
||||
// - Word boundaries are spaces. The megabyte word list jieba needs is a
|
||||
// property of a writing system that doesn't use them, not a debt every pair
|
||||
// owes; a Latin-script pair needs nothing loaded to be segmented.
|
||||
// - The dictionary arrived with dict.db, which reads pt→en as readily as
|
||||
// en→pt (see lexicon.dreamProvider.reverse). The reverse lookup the hover
|
||||
// and the word card need is already there and already answering.
|
||||
//
|
||||
// So the pair a native English speaker learning Portuguese needs is real, and
|
||||
// what was actually blocking it was this list. French and Spanish clear the same
|
||||
// two bars through the same dict.db; they are held back only by their packs
|
||||
// carrying no `learner` copy yet (see Pack.learner), which is a translation
|
||||
// question rather than a data one.
|
||||
//
|
||||
// This list is still expected to grow one pair at a time and never to be
|
||||
// inferred: segmentation is a property of a writing system, and there is no rule
|
||||
// that derives "has a word list" from a language code.
|
||||
var learnerPairs = []string{"zh", "pt-PT"}
|
||||
|
||||
// SupportsLearnerDirection reports whether a pair can be turned around.
|
||||
func SupportsLearnerDirection(lang string) bool {
|
||||
for _, p := range learnerPairs {
|
||||
if p == lang {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func directionIsKnown(d string) bool {
|
||||
return d == DirectionLearningEn || d == DirectionLearningPair
|
||||
}
|
||||
|
||||
// SetPair moves an account to another (English + X) pair, in a given direction.
|
||||
//
|
||||
// The two are written together because they constrain each other: a direction is
|
||||
// only meaningful for a pair that can be travelled in it, and validating them a
|
||||
// field at a time would let a two-step change pass through a state that neither
|
||||
// step is allowed to leave behind.
|
||||
func (u *UserStore) SetPair(id, lang, direction string) error {
|
||||
if !pairIsShipped(lang) {
|
||||
return errors.New("auth: unshipped pair language " + lang)
|
||||
}
|
||||
if !directionIsKnown(direction) {
|
||||
return errors.New("auth: unknown direction " + direction)
|
||||
}
|
||||
if direction == DirectionLearningPair && !SupportsLearnerDirection(lang) {
|
||||
return errors.New("auth: no learner direction for " + lang)
|
||||
}
|
||||
res, err := u.db.Exec(
|
||||
`UPDATE users SET pair_lang = ?, direction = ? WHERE id = ?`, lang, direction, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n, err := res.RowsAffected(); err == nil && n == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateMeHandler changes the caller's own settings: which language Petal
|
||||
// speaks alongside her English, and which of the two she is learning.
|
||||
//
|
||||
// It answers with the whole updated user rather than an empty 204 so the client
|
||||
// has one shape to trust: /api/me and this return the same thing, and the app
|
||||
// re-reads the pair from the response instead of assuming its request took.
|
||||
//
|
||||
// The pair language reaches further than the UI copy — it picks her Hunspell
|
||||
// dictionary, her read-aloud voice, which word-lookup provider answers, and the
|
||||
// language the prompts ask the model to explain in. All of those read
|
||||
// `users.pair_lang` at use time, so all of them follow from this one write.
|
||||
//
|
||||
// Both fields are optional and each defaults to what the account already has, so
|
||||
// the picker can send one without knowing the other. That matters for the
|
||||
// combination this endpoint exists to prevent: a client that sent only
|
||||
// `pair_lang: "fr"` while the account sat on `learning_pair` would otherwise ask
|
||||
// for French-with-segmentation, which does not exist. Here it is one decision
|
||||
// with one validation, and the answer carries whatever actually landed.
|
||||
func (u *UserStore) UpdateMeHandler() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
PairLang *string `json:"pair_lang"`
|
||||
Direction *string `json:"direction"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
httputil.BadRequest(w, "invalid request body")
|
||||
return
|
||||
}
|
||||
id := UserID(r.Context())
|
||||
current, err := u.Get(id)
|
||||
if err != nil {
|
||||
// Only a missing row means "not signed in". A dictionary-file or
|
||||
// SQLite fault answered as 401 would trip the client's session
|
||||
// interceptor and throw a writer out of an app she is still signed
|
||||
// in to — the same distinction SetPair's error branch makes below.
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||
return
|
||||
}
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
lang, direction := current.PairLang, current.Direction
|
||||
if body.PairLang != nil {
|
||||
lang = strings.TrimSpace(*body.PairLang)
|
||||
}
|
||||
if body.Direction != nil {
|
||||
direction = strings.TrimSpace(*body.Direction)
|
||||
}
|
||||
|
||||
if !pairIsShipped(lang) {
|
||||
// Name the ones that work. A writer who lands here has picked from a
|
||||
// stale client, and "not a language" tells her nothing.
|
||||
httputil.BadRequest(w, "unsupported language pair — Petal speaks "+strings.Join(shippedPairs, ", "))
|
||||
return
|
||||
}
|
||||
if !directionIsKnown(direction) {
|
||||
httputil.BadRequest(w, "unknown direction — expected "+DirectionLearningEn+" or "+DirectionLearningPair)
|
||||
return
|
||||
}
|
||||
if direction == DirectionLearningPair && !SupportsLearnerDirection(lang) {
|
||||
// Refused rather than quietly downgraded to learning_en. A silent
|
||||
// downgrade would leave the writer looking at an editor that behaves
|
||||
// like the one she just tried to leave, with nothing to read as an
|
||||
// explanation — and the caller cannot tell the two outcomes apart
|
||||
// without diffing the response it was given.
|
||||
httputil.BadRequest(w, "Petal can only be learned toward "+strings.Join(learnerPairs, ", ")+" so far")
|
||||
return
|
||||
}
|
||||
|
||||
if err := u.SetPair(id, lang, direction); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||
return
|
||||
}
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
user, err := u.Get(id)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, user)
|
||||
}
|
||||
}
|
||||
|
||||
// Allowlist decides which of Authentik's users may write in this Petal.
|
||||
// Authentik fronts several applications; being a valid user there does not mean
|
||||
// being a user here.
|
||||
|
||||
+125
-8
@@ -1,7 +1,10 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -29,10 +32,16 @@ type Config struct {
|
||||
// TTS (read-aloud). Off unless TTSEndpoint is set — when empty, the /api/tts
|
||||
// route isn't mounted and the frontend falls back to the browser's Web Speech
|
||||
// API. Endpoint points at a local Piper HTTP server.
|
||||
TTSEndpoint string // Piper instance serving the English voice
|
||||
TTSEndpointZH string // Piper instance serving the Chinese voice; empty = zh falls back to Web Speech
|
||||
TTSVoiceEN string // Piper voice id for English (e.g. en_US-amy-medium)
|
||||
TTSVoiceZH string // Piper voice id for Chinese (e.g. zh_CN-huayan-medium)
|
||||
TTSEndpoint string // Piper instance serving the English voice; also the on/off switch
|
||||
// TTSVoices is every language Petal can read aloud, keyed by base language
|
||||
// tag ("en", "zh", "pt", …). Each Piper server loads exactly one model, so
|
||||
// a language *is* an instance — and the instances are discovered from the
|
||||
// environment rather than named in this struct: one
|
||||
// TTS_ENDPOINT_<LANG>/TTS_VOICE_<LANG> pair per language, so the fr and es
|
||||
// pairs cost a compose service and two lines of .env rather than a code
|
||||
// change. English keeps the unsuffixed TTS_ENDPOINT/TTS_VOICE_EN it has
|
||||
// always had.
|
||||
TTSVoices map[string]TTSVoice
|
||||
// TTSPath is the path Piper serves synthesis on. Piper moved it from "/" to
|
||||
// "/synthesize" in 1.6.0 with an unchanged request body, so this is a
|
||||
// version knob, not a feature: millenia's older server keeps the default,
|
||||
@@ -54,6 +63,28 @@ type Config struct {
|
||||
// authenticates — right for a single-household instance, wrong the moment
|
||||
// the IdP serves an audience wider than Petal's.
|
||||
AllowedSubs string
|
||||
// RequireAuth refuses to start when OIDC isn't configured, instead of
|
||||
// falling back to the single local user.
|
||||
//
|
||||
// The fallback is the right behaviour on a laptop and a catastrophe on a
|
||||
// public host: a typo in AUTHENTIK_CLIENT_SECRET turns every anonymous
|
||||
// visitor into the `local` user, with full read and write over someone's
|
||||
// private journals, and says so only in a log line nobody is reading. The
|
||||
// Traefik basic-auth gate that used to stand behind that mistake was
|
||||
// removed when Petal learned to authenticate for itself, so nothing catches
|
||||
// it now.
|
||||
//
|
||||
// Defaulted from BASE_URL rather than declared: a Petal that knows itself by
|
||||
// a real public origin has no business running open, and one on localhost
|
||||
// has no business demanding an IdP. Set PETAL_REQUIRE_AUTH explicitly to
|
||||
// override in either direction.
|
||||
RequireAuth bool
|
||||
}
|
||||
|
||||
// TTSVoice is one Piper instance and the single voice it has loaded.
|
||||
type TTSVoice struct {
|
||||
Endpoint string
|
||||
Voice string
|
||||
}
|
||||
|
||||
// AuthEnabled reports whether real logins are configured. When false, Petal
|
||||
@@ -64,9 +95,10 @@ func (c *Config) AuthEnabled() bool {
|
||||
|
||||
// Load reads configuration from the environment, applying sane local-dev defaults.
|
||||
func Load() *Config {
|
||||
baseURL := env("BASE_URL", "http://localhost:8080")
|
||||
return &Config{
|
||||
Port: env("PORT", "8080"),
|
||||
BaseURL: env("BASE_URL", "http://localhost:8080"),
|
||||
BaseURL: baseURL,
|
||||
DatabasePath: env("DATABASE_PATH", "./data/petal.db"),
|
||||
ImageDir: env("IMAGE_DIR", "./data/images"),
|
||||
DictPath: env("DICT_PATH", "./data/dict.db"),
|
||||
@@ -78,9 +110,7 @@ func Load() *Config {
|
||||
LLMTimeout: envDuration("LLM_TIMEOUT", 30*time.Second),
|
||||
|
||||
TTSEndpoint: env("TTS_ENDPOINT", ""),
|
||||
TTSEndpointZH: env("TTS_ENDPOINT_ZH", ""),
|
||||
TTSVoiceEN: env("TTS_VOICE_EN", "en_US-amy-medium"),
|
||||
TTSVoiceZH: env("TTS_VOICE_ZH", "zh_CN-huayan-medium"),
|
||||
TTSVoices: ttsVoices(os.Environ()),
|
||||
TTSPath: env("TTS_PATH", "/"),
|
||||
TTSCacheDir: env("TTS_CACHE_DIR", "./data/tts"),
|
||||
TTSTimeout: envDuration("TTS_TIMEOUT", 15*time.Second),
|
||||
@@ -90,9 +120,84 @@ func Load() *Config {
|
||||
AuthentikClientID: env("AUTHENTIK_CLIENT_ID", ""),
|
||||
AuthentikClientSecret: env("AUTHENTIK_CLIENT_SECRET", ""),
|
||||
AllowedSubs: env("PETAL_ALLOWED_SUBS", ""),
|
||||
RequireAuth: envBool("PETAL_REQUIRE_AUTH", !isLoopbackOrigin(baseURL)),
|
||||
}
|
||||
}
|
||||
|
||||
// isLoopbackOrigin reports whether a base URL names this machine — the shape a
|
||||
// development checkout has, and the only shape where running without a login is
|
||||
// a reasonable default. Anything else (a hostname, a public origin) is a
|
||||
// deployment, however small.
|
||||
func isLoopbackOrigin(baseURL string) bool {
|
||||
u, err := url.Parse(strings.TrimSpace(baseURL))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(u.Hostname()) {
|
||||
case "localhost", "127.0.0.1", "::1", "":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ttsVoices reads the Piper instances out of an environment slice (as returned
|
||||
// by os.Environ) into a map keyed by base language tag.
|
||||
//
|
||||
// English is the unsuffixed pair, TTS_ENDPOINT + TTS_VOICE_EN, because that is
|
||||
// what every deployment already sets and read-aloud has always been English
|
||||
// first. Every other language is a TTS_ENDPOINT_<LANG>/TTS_VOICE_<LANG> pair,
|
||||
// discovered rather than enumerated — TTS_ENDPOINT_ZH is what millenia and the
|
||||
// VPS already use, and TTS_ENDPOINT_PT is all the Portuguese pair needs.
|
||||
//
|
||||
// <LANG> is the *base* tag: an environment variable name cannot hold the hyphen
|
||||
// in "pt-PT", and the handler routes on the base tag anyway (a request for
|
||||
// pt-PT, pt-BR or bare pt reaches the same instance, because there is only one
|
||||
// Portuguese voice loaded). A pair is ignored unless both halves are set: half
|
||||
// a configuration should read as "no voice for this language" and fall back to
|
||||
// the browser, not as an instance that answers every request with an error.
|
||||
func ttsVoices(environ []string) map[string]TTSVoice {
|
||||
vals := make(map[string]string, len(environ))
|
||||
for _, kv := range environ {
|
||||
if k, v, ok := strings.Cut(kv, "="); ok {
|
||||
vals[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
voices := map[string]TTSVoice{}
|
||||
add := func(lang, endpoint, voice string) {
|
||||
endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/")
|
||||
voice = strings.TrimSpace(voice)
|
||||
if endpoint == "" || voice == "" {
|
||||
return
|
||||
}
|
||||
voices[lang] = TTSVoice{Endpoint: endpoint, Voice: voice}
|
||||
}
|
||||
|
||||
// The two languages that shipped before this was a map keep their voice
|
||||
// defaults, so an existing deployment that names only the endpoints (as
|
||||
// millenia's unit does) sounds exactly as it did.
|
||||
voiceOr := func(key, fallback string) string {
|
||||
if v := strings.TrimSpace(vals[key]); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
add("en", vals["TTS_ENDPOINT"], voiceOr("TTS_VOICE_EN", "en_US-amy-medium"))
|
||||
for k, endpoint := range vals {
|
||||
suffix, ok := strings.CutPrefix(k, "TTS_ENDPOINT_")
|
||||
if !ok || suffix == "" {
|
||||
continue
|
||||
}
|
||||
voice := vals["TTS_VOICE_"+suffix]
|
||||
if suffix == "ZH" {
|
||||
voice = voiceOr("TTS_VOICE_ZH", "zh_CN-huayan-medium")
|
||||
}
|
||||
add(strings.ToLower(suffix), endpoint, voice)
|
||||
}
|
||||
return voices
|
||||
}
|
||||
|
||||
func env(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
@@ -100,6 +205,18 @@ func env(key, fallback string) string {
|
||||
return fallback
|
||||
}
|
||||
|
||||
// envBool reads a boolean knob. Anything unparseable keeps the default rather
|
||||
// than silently reading as false — a mistyped PETAL_REQUIRE_AUTH must not be the
|
||||
// thing that turns the guard off.
|
||||
func envBool(key string, fallback bool) bool {
|
||||
if v := strings.TrimSpace(os.Getenv(key)); v != "" {
|
||||
if b, err := strconv.ParseBool(v); err == nil {
|
||||
return b
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envDuration(key string, fallback time.Duration) time.Duration {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package config
|
||||
|
||||
import "testing"
|
||||
|
||||
// The Piper instances are discovered from the environment rather than named in
|
||||
// code, so that a new pair costs a compose service and two .env lines. These
|
||||
// assert the discovery rule, including the two shapes that already exist in the
|
||||
// wild (millenia's systemd unit and the VPS compose file).
|
||||
func TestTTSVoicesDiscovery(t *testing.T) {
|
||||
voices := ttsVoices([]string{
|
||||
"TTS_ENDPOINT=http://piper-en:5000",
|
||||
"TTS_VOICE_EN=en_US-amy-medium",
|
||||
"TTS_ENDPOINT_ZH=http://piper-zh:5000/",
|
||||
"TTS_VOICE_ZH=zh_CN-huayan-medium",
|
||||
"TTS_ENDPOINT_PT=http://piper-pt:5000",
|
||||
"TTS_VOICE_PT=pt_PT-tugão-medium",
|
||||
"TTS_ENDPOINT_FR=http://piper-fr:5000",
|
||||
"TTS_VOICE_FR=fr_FR-siwis-medium",
|
||||
"TTS_ENDPOINT_ES=http://piper-es:5000",
|
||||
"TTS_VOICE_ES=es_MX-ald-medium",
|
||||
// Noise that must not become a language.
|
||||
"TTS_PATH=/synthesize",
|
||||
"PATH=/usr/bin",
|
||||
})
|
||||
|
||||
want := map[string]TTSVoice{
|
||||
"en": {"http://piper-en:5000", "en_US-amy-medium"},
|
||||
// The trailing slash is trimmed here so the synthesis path concatenates
|
||||
// cleanly rather than producing a double slash at every call site.
|
||||
"zh": {"http://piper-zh:5000", "zh_CN-huayan-medium"},
|
||||
"pt": {"http://piper-pt:5000", "pt_PT-tugão-medium"},
|
||||
// Phase 24's whole TTS change: a fourth language costs two lines here
|
||||
// and a compose service, and no Go at all.
|
||||
"fr": {"http://piper-fr:5000", "fr_FR-siwis-medium"},
|
||||
// And a fifth cost exactly the same, which is the claim actually being
|
||||
// tested. The voice is Mexican on purpose: the es pack is Latin
|
||||
// American, and es_ES-davefx-medium would read it in the wrong accent.
|
||||
"es": {"http://piper-es:5000", "es_MX-ald-medium"},
|
||||
}
|
||||
if len(voices) != len(want) {
|
||||
t.Fatalf("discovered %v, want %v", voices, want)
|
||||
}
|
||||
for lang, w := range want {
|
||||
if voices[lang] != w {
|
||||
t.Errorf("%s = %+v, want %+v", lang, voices[lang], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Half a configuration is not a language. An endpoint with no voice (or the
|
||||
// reverse) must read as "no voice for this language" — a 404 the client answers
|
||||
// by falling back to Web Speech — rather than as an instance that exists and
|
||||
// errors on every request.
|
||||
func TestTTSVoicesIgnoresHalfConfiguredLanguages(t *testing.T) {
|
||||
voices := ttsVoices([]string{
|
||||
"TTS_ENDPOINT=http://piper-en:5000",
|
||||
"TTS_VOICE_EN=en_US-amy-medium",
|
||||
"TTS_ENDPOINT_FR=http://piper-fr:5000", // no TTS_VOICE_FR
|
||||
"TTS_VOICE_ES=es_ES-davefx-medium", // no TTS_ENDPOINT_ES
|
||||
})
|
||||
if _, ok := voices["fr"]; ok {
|
||||
t.Errorf("fr routed with no voice configured")
|
||||
}
|
||||
if _, ok := voices["es"]; ok {
|
||||
t.Errorf("es routed with no endpoint configured")
|
||||
}
|
||||
if len(voices) != 1 {
|
||||
t.Errorf("discovered %v, want English only", voices)
|
||||
}
|
||||
}
|
||||
|
||||
// A deployment that predates the map names only the endpoints and relies on the
|
||||
// voice defaults; it must sound exactly as it did.
|
||||
func TestTTSVoicesKeepsTheOriginalDefaults(t *testing.T) {
|
||||
voices := ttsVoices([]string{
|
||||
"TTS_ENDPOINT=http://127.0.0.1:5005",
|
||||
"TTS_ENDPOINT_ZH=http://127.0.0.1:5006",
|
||||
})
|
||||
if got := voices["en"].Voice; got != "en_US-amy-medium" {
|
||||
t.Errorf("en voice = %q, want the default", got)
|
||||
}
|
||||
if got := voices["zh"].Voice; got != "zh_CN-huayan-medium" {
|
||||
t.Errorf("zh voice = %q, want the default", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Read-aloud is off when no English instance is configured; nothing else may
|
||||
// switch it on. (tts.New gates on TTSEndpoint, so a stray TTS_ENDPOINT_PT with
|
||||
// no English sibling must not produce a routable map that outlives that gate.)
|
||||
func TestTTSVoicesEmptyWithoutEndpoints(t *testing.T) {
|
||||
if voices := ttsVoices([]string{"TTS_VOICE_EN=en_US-amy-medium"}); len(voices) != 0 {
|
||||
t.Errorf("discovered %v, want none", voices)
|
||||
}
|
||||
}
|
||||
|
||||
// The fallback to the single local user is right on a laptop and a catastrophe
|
||||
// on a public host, so it is defaulted from the origin Petal knows itself by
|
||||
// rather than left to be remembered.
|
||||
func TestRequireAuthDefaultsFromBaseURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
baseURL string
|
||||
want bool
|
||||
}{
|
||||
{"http://localhost:8080", false},
|
||||
{"http://127.0.0.1:8080", false},
|
||||
{"http://[::1]:8080", false},
|
||||
{"", false}, // no BASE_URL set at all: the local-dev default
|
||||
{"https://petal.parodia.dev", true},
|
||||
{"http://petal.parodia.dev", true},
|
||||
{"https://petal.example.com/", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Setenv("BASE_URL", c.baseURL)
|
||||
t.Setenv("PETAL_REQUIRE_AUTH", "")
|
||||
if got := Load().RequireAuth; got != c.want {
|
||||
t.Errorf("BASE_URL=%q: RequireAuth=%v, want %v", c.baseURL, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The default is a default, not a rule: a trusted private network is a real
|
||||
// deployment shape, and so is wanting the guard on locally.
|
||||
func TestRequireAuthExplicitOverride(t *testing.T) {
|
||||
t.Setenv("BASE_URL", "https://petal.parodia.dev")
|
||||
t.Setenv("PETAL_REQUIRE_AUTH", "false")
|
||||
if Load().RequireAuth {
|
||||
t.Error("an explicit false must be honoured on a public origin")
|
||||
}
|
||||
|
||||
t.Setenv("BASE_URL", "http://localhost:8080")
|
||||
t.Setenv("PETAL_REQUIRE_AUTH", "true")
|
||||
if !Load().RequireAuth {
|
||||
t.Error("an explicit true must be honoured on localhost")
|
||||
}
|
||||
|
||||
// A typo must not be the thing that disables the guard.
|
||||
t.Setenv("BASE_URL", "https://petal.parodia.dev")
|
||||
t.Setenv("PETAL_REQUIRE_AUTH", "nope")
|
||||
if !Load().RequireAuth {
|
||||
t.Error("an unparseable value must keep the default, not read as false")
|
||||
}
|
||||
}
|
||||
@@ -459,6 +459,185 @@ CREATE TABLE personal_words (
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (user_id, lang, word)
|
||||
);
|
||||
`,
|
||||
},
|
||||
{
|
||||
// The growth journal reads the suggestions table as a record of what the
|
||||
// writer has been learning, and that reading only works if a row is dated
|
||||
// by *her decision* rather than by the model's proposal. `created_at` is
|
||||
// when a checkpoint offered the edit; a suggestion offered in April and
|
||||
// accepted in June is June's growth, not April's.
|
||||
//
|
||||
// Existing rows are backfilled to created_at — which is exactly the
|
||||
// approximation the journal would have had to make anyway, and is very
|
||||
// nearly right in practice since edits are settled minutes after a
|
||||
// checkpoint. Only pending rows keep a NULL: nothing has been decided.
|
||||
name: "0012_suggestion_resolved_at",
|
||||
stmt: `
|
||||
ALTER TABLE suggestions ADD COLUMN resolved_at DATETIME;
|
||||
UPDATE suggestions SET resolved_at = created_at WHERE status != 'pending';
|
||||
CREATE INDEX idx_suggestions_resolved ON suggestions(status, resolved_at);
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Which engine proposed a row. Until now `type` doubled as that answer —
|
||||
// 'mechanics' meant "the offline rule pack found this" and everything else
|
||||
// meant "the model did". That breaks the moment an offline rule proposes a
|
||||
// *collocation*: the miscollocation list (SUGGESTIONS §6) is the same
|
||||
// family, the same rail and the same warm phrasing as the LLM coach, and it
|
||||
// must stay type='collocation' so an accepted chunk still plants in the
|
||||
// garden and still counts in the journal. With type no longer naming the
|
||||
// engine, the two passes could not scope their own DELETEs — the coach
|
||||
// would wipe the offline flags, and the offline pass would leave the
|
||||
// coach's behind to accumulate.
|
||||
//
|
||||
// Existing mechanics rows are local by definition; everything else came
|
||||
// from a model.
|
||||
name: "0013_suggestion_source",
|
||||
stmt: `
|
||||
ALTER TABLE suggestions ADD COLUMN source TEXT NOT NULL DEFAULT 'llm';
|
||||
UPDATE suggestions SET source = 'local' WHERE type = 'mechanics';
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Sentence-level identity, so a re-check stops regenerating the world.
|
||||
// Every pass used to delete its whole family and re-insert it, which
|
||||
// meant accepting one edit gave every other card a new id and a newly
|
||||
// worded explanation — the rail visibly emptied and refilled, and the
|
||||
// model was asked again about sentences nobody had touched.
|
||||
//
|
||||
// `chunk_hash` records which sentence a suggestion belongs to, and
|
||||
// checked_chunks records which sentences a family has already read. A
|
||||
// re-check then asks only about the difference and keeps the rest of
|
||||
// the rows exactly as they are, id and wording included.
|
||||
//
|
||||
// Existing rows get '' — "sentence unknown", which reads as in-play, so
|
||||
// they are simply reconciled on the next pass like any fresh finding.
|
||||
name: "0014_suggestion_chunk_hash",
|
||||
stmt: `
|
||||
ALTER TABLE suggestions ADD COLUMN chunk_hash TEXT NOT NULL DEFAULT '';
|
||||
|
||||
CREATE TABLE checked_chunks (
|
||||
doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
family TEXT NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
PRIMARY KEY (doc_id, family, hash)
|
||||
);
|
||||
`,
|
||||
},
|
||||
{
|
||||
// A sentence she wrote in her own language gets its own type. Petal already
|
||||
// detected such spans and already rendered them into English — it just
|
||||
// filed the result under 'clarity', so the pair model's flagship moment
|
||||
// read as tidying up her Chinese. As with 0005 and 0008, the `type` CHECK
|
||||
// can't be ALTERed in place, so rebuild the table with the extended
|
||||
// constraint, copy every row across, and recreate both indexes.
|
||||
//
|
||||
// Existing rows are left on whatever type they have. A card she is already
|
||||
// reading keeps the label she has already read (the same rule reconcile.go
|
||||
// follows for a re-proposed edit); new findings get the new label.
|
||||
name: "0015_translate_suggestion_type",
|
||||
stmt: `
|
||||
CREATE TABLE suggestions_new (
|
||||
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
||||
doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
from_pos INTEGER NOT NULL,
|
||||
to_pos INTEGER NOT NULL,
|
||||
original TEXT NOT NULL,
|
||||
replacement TEXT NOT NULL,
|
||||
explanation TEXT NOT NULL,
|
||||
type TEXT NOT NULL CHECK(type IN ('grammar','phrasing','idiom','clarity','translate','voice','collocation','mechanics')),
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','accepted','rejected')),
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
resolved_at DATETIME,
|
||||
source TEXT NOT NULL DEFAULT 'llm',
|
||||
chunk_hash TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
INSERT INTO suggestions_new (id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash)
|
||||
SELECT id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash FROM suggestions;
|
||||
|
||||
DROP TABLE suggestions;
|
||||
ALTER TABLE suggestions_new RENAME TO suggestions;
|
||||
CREATE INDEX idx_suggestions_doc_id ON suggestions(doc_id);
|
||||
CREATE INDEX idx_suggestions_resolved ON suggestions(status, resolved_at);
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Which half of the pair is being learned.
|
||||
//
|
||||
// `pair_lang` (0010) has always answered "which two languages", and every
|
||||
// surface built on it assumed the answer to a second question nobody had
|
||||
// asked: that English is the language being *learned*. That assumption is
|
||||
// load-bearing in a dozen places — CJK is deliberately never tokenized,
|
||||
// never spell-checked, never glossed; the prompts explain English in her
|
||||
// language; the vocabulary garden captures English words. All correct for
|
||||
// a Mandarin native practising English, and all backwards for an English
|
||||
// native practising Mandarin.
|
||||
//
|
||||
// A second pair code ('zh-learner') was the cheaper option and is the
|
||||
// wrong shape: it would make the two directions of one pair look like two
|
||||
// unrelated languages to every query, and it would have to be repeated for
|
||||
// fr, es and pt-PT before any of them could turn around. A column keeps
|
||||
// the two questions separate, which is what they are.
|
||||
//
|
||||
// 'learning_en' is the default and is what every existing row means — the
|
||||
// backfill is the DEFAULT itself, and it is right rather than merely
|
||||
// convenient: all three accounts today are Mandarin natives writing
|
||||
// English.
|
||||
name: "0016_user_direction",
|
||||
stmt: `
|
||||
ALTER TABLE users ADD COLUMN direction TEXT NOT NULL DEFAULT 'learning_en'
|
||||
CHECK(direction IN ('learning_en','learning_pair'));
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Which language this document is written in — 'en' or 'pair'.
|
||||
//
|
||||
// It is stored, rather than recomputed per pass and forgotten, for one
|
||||
// reason: the verdict has hysteresis (see suggestions/doclang.go). A
|
||||
// bilingual document sits between the two thresholds, and "whatever it
|
||||
// was last time" is only an answer if last time was written down. Without
|
||||
// the column a mixed paragraph would alternate its cards' language
|
||||
// between passes.
|
||||
//
|
||||
// 'pair' rather than a language code, deliberately. Which language "pair"
|
||||
// names is the owner's users.pair_lang, so changing her pair re-reads her
|
||||
// documents instead of stranding a stale language name on every one of
|
||||
// them.
|
||||
//
|
||||
// Empty is the backfill and means English: every document that exists
|
||||
// today was written by a Mandarin native practising English, and English
|
||||
// is what every surface assumed before this phase.
|
||||
name: "0017_document_lang",
|
||||
stmt: `
|
||||
ALTER TABLE documents ADD COLUMN doc_lang TEXT NOT NULL DEFAULT ''
|
||||
CHECK(doc_lang IN ('', 'en', 'pair'));
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Which language a garden card is in — the same '' | 'en' | 'pair'
|
||||
// vocabulary as documents.doc_lang, and set from it: a word is captured
|
||||
// (or a phrase planted) out of a document, so the document's verdict is
|
||||
// the card's language. A card with no document keeps '', which reads as
|
||||
// English like every other empty here.
|
||||
//
|
||||
// The garden needed this the moment a document could be written in her
|
||||
// own language. Before Phase 28 every card was English by construction;
|
||||
// now a Portuguese lookup lands beside an English one with nothing to
|
||||
// tell them apart, and two surfaces get it wrong without the tag — the
|
||||
// review card's read-aloud (which would say a Portuguese word in a US
|
||||
// English voice) and the panel, where a mixed garden is illegible.
|
||||
//
|
||||
// Every card is reviewed regardless. Filtering the queue to the half she
|
||||
// is learning was the alternative and is wrong for the writer this is
|
||||
// for: the words she met while writing Portuguese are still words she
|
||||
// met, and a garden that quietly drops them is a garden that stops being
|
||||
// a record of her reading.
|
||||
name: "0018_vocab_lang",
|
||||
stmt: `
|
||||
ALTER TABLE vocab_words ADD COLUMN lang TEXT NOT NULL DEFAULT ''
|
||||
CHECK(lang IN ('', 'en', 'pair'));
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package db
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestOpenMigratesAndSeeds(t *testing.T) {
|
||||
@@ -83,3 +84,283 @@ func TestOpenMigratesAndSeeds(t *testing.T) {
|
||||
t.Errorf("expected exactly 1 local user after reopen, got %d", users)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolvedAtBackfill runs migration 0012 against a database that predates
|
||||
// it, which is the only shape that matters: on the live box the suggestions
|
||||
// table is years of settled edits with no resolved_at to their name. Backfilling
|
||||
// to created_at is exactly the approximation the growth journal would otherwise
|
||||
// have had to make, and a pending row must stay NULL — nothing has been decided.
|
||||
func TestResolvedAtBackfill(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "old.db")
|
||||
d, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
|
||||
// Rewind to the state before 0012: drop the column and forget the migration.
|
||||
if _, err := d.Exec(`DROP INDEX idx_suggestions_resolved`); err != nil {
|
||||
t.Fatalf("rewind index: %v", err)
|
||||
}
|
||||
if _, err := d.Exec(`ALTER TABLE suggestions DROP COLUMN resolved_at`); err != nil {
|
||||
t.Fatalf("rewind schema: %v", err)
|
||||
}
|
||||
if _, err := d.Exec(`DELETE FROM schema_migrations WHERE name = '0012_suggestion_resolved_at'`); err != nil {
|
||||
t.Fatalf("rewind migration record: %v", err)
|
||||
}
|
||||
if _, err := d.Exec(`INSERT INTO documents (id, user_id) VALUES ('d1', ?)`, LocalUserID); err != nil {
|
||||
t.Fatalf("insert document: %v", err)
|
||||
}
|
||||
for _, s := range []struct{ id, status string }{
|
||||
{"s-old", "accepted"},
|
||||
{"s-open", "pending"},
|
||||
} {
|
||||
if _, err := d.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at)
|
||||
VALUES (?, 'd1', 0, 3, 'teh', 'the', 'x', 'grammar', ?, '2026-01-02 03:04:05')`,
|
||||
s.id, s.status,
|
||||
); err != nil {
|
||||
t.Fatalf("seed %s: %v", s.id, err)
|
||||
}
|
||||
}
|
||||
d.Close()
|
||||
|
||||
d2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen (migrate): %v", err)
|
||||
}
|
||||
defer d2.Close()
|
||||
|
||||
// Compared against created_at read back the same way: the driver renders a
|
||||
// DATETIME column itself, so the assertion is "the same instant", not a
|
||||
// particular text format.
|
||||
var settled, created *string
|
||||
if err := d2.QueryRow(
|
||||
`SELECT resolved_at, created_at FROM suggestions WHERE id = 's-old'`,
|
||||
).Scan(&settled, &created); err != nil {
|
||||
t.Fatalf("read settled row: %v", err)
|
||||
}
|
||||
if settled == nil || created == nil || *settled != *created {
|
||||
t.Errorf("resolved_at = %v, want it backfilled from created_at (%v)", settled, created)
|
||||
}
|
||||
|
||||
var pending *string
|
||||
if err := d2.QueryRow(`SELECT resolved_at FROM suggestions WHERE id = 's-open'`).Scan(&pending); err != nil {
|
||||
t.Fatalf("read pending row: %v", err)
|
||||
}
|
||||
if pending != nil {
|
||||
t.Errorf("pending row got resolved_at = %v, want NULL — nothing was decided", *pending)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSuggestionSourceBackfill runs migration 0013 against a database that
|
||||
// predates it — the shape the live box is actually in. `source` is the column
|
||||
// that lets the offline rule pack and the model share the collocation family
|
||||
// without deleting each other's rows, and it can only do that if the existing
|
||||
// rows are labelled correctly on the way in: everything the old deterministic
|
||||
// pass wrote is local, and everything else came from a model.
|
||||
func TestSuggestionSourceBackfill(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "old.db")
|
||||
d, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
|
||||
// Rewind to the state before 0013.
|
||||
if _, err := d.Exec(`ALTER TABLE suggestions DROP COLUMN source`); err != nil {
|
||||
t.Fatalf("rewind schema: %v", err)
|
||||
}
|
||||
if _, err := d.Exec(`DELETE FROM schema_migrations WHERE name = '0013_suggestion_source'`); err != nil {
|
||||
t.Fatalf("rewind migration record: %v", err)
|
||||
}
|
||||
if _, err := d.Exec(`INSERT INTO documents (id, user_id) VALUES ('d1', ?)`, LocalUserID); err != nil {
|
||||
t.Fatalf("insert document: %v", err)
|
||||
}
|
||||
for _, s := range []struct{ id, typ string }{
|
||||
{"s-mech", SuggestionTypeMechanics},
|
||||
{"s-gram", SuggestionTypeGrammar},
|
||||
{"s-coll", SuggestionTypeCollocation},
|
||||
} {
|
||||
if _, err := d.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES (?, 'd1', 0, 3, 'teh', 'the', 'x', ?)`,
|
||||
s.id, s.typ,
|
||||
); err != nil {
|
||||
t.Fatalf("seed %s: %v", s.id, err)
|
||||
}
|
||||
}
|
||||
d.Close()
|
||||
|
||||
d2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen (migrate): %v", err)
|
||||
}
|
||||
defer d2.Close()
|
||||
|
||||
// A pre-0013 collocation row can only have come from the coach — the offline
|
||||
// miscollocation list did not exist yet — so it must NOT be claimed as local.
|
||||
for id, want := range map[string]string{
|
||||
"s-mech": SuggestionSourceLocal,
|
||||
"s-gram": SuggestionSourceLLM,
|
||||
"s-coll": SuggestionSourceLLM,
|
||||
} {
|
||||
var got string
|
||||
if err := d2.QueryRow(`SELECT source FROM suggestions WHERE id = ?`, id).Scan(&got); err != nil {
|
||||
t.Fatalf("read %s: %v", id, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("%s: source = %q, want %q", id, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// And a row written after the migration defaults to the model, so a code path
|
||||
// that forgets to name a source can never silently claim to be offline.
|
||||
if _, err := d2.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES ('s-new', 'd1', 0, 3, 'teh', 'the', 'x', 'grammar')`,
|
||||
); err != nil {
|
||||
t.Fatalf("insert new row: %v", err)
|
||||
}
|
||||
var fresh string
|
||||
if err := d2.QueryRow(`SELECT source FROM suggestions WHERE id = 's-new'`).Scan(&fresh); err != nil {
|
||||
t.Fatalf("read new row: %v", err)
|
||||
}
|
||||
if fresh != SuggestionSourceLLM {
|
||||
t.Errorf("default source = %q, want %q", fresh, SuggestionSourceLLM)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTranslateTypeMigrationPreservesRows runs migration 0015 against a database
|
||||
// that predates it. Unlike the two backfills above, 0015 *rebuilds the table* —
|
||||
// SQLite can't ALTER a CHECK constraint — so it copies every row across by hand,
|
||||
// and a column left out of that copy list silently loses her data. Every test
|
||||
// elsewhere starts from a fresh database and would never notice; the live box has
|
||||
// years of rows in it.
|
||||
func TestTranslateTypeMigrationPreservesRows(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "old.db")
|
||||
d, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
|
||||
// Rewind to the pre-0015 table: the same shape, minus 'translate' in the CHECK.
|
||||
if _, err := d.Exec(`
|
||||
CREATE TABLE suggestions_old (
|
||||
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
||||
doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
from_pos INTEGER NOT NULL,
|
||||
to_pos INTEGER NOT NULL,
|
||||
original TEXT NOT NULL,
|
||||
replacement TEXT NOT NULL,
|
||||
explanation TEXT NOT NULL,
|
||||
type TEXT NOT NULL CHECK(type IN ('grammar','phrasing','idiom','clarity','voice','collocation','mechanics')),
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','accepted','rejected')),
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
resolved_at DATETIME,
|
||||
source TEXT NOT NULL DEFAULT 'llm',
|
||||
chunk_hash TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
DROP TABLE suggestions;
|
||||
ALTER TABLE suggestions_old RENAME TO suggestions;
|
||||
CREATE INDEX idx_suggestions_doc_id ON suggestions(doc_id);
|
||||
CREATE INDEX idx_suggestions_resolved ON suggestions(status, resolved_at);
|
||||
DELETE FROM schema_migrations WHERE name = '0015_translate_suggestion_type';
|
||||
`); err != nil {
|
||||
t.Fatalf("rewind schema: %v", err)
|
||||
}
|
||||
|
||||
if _, err := d.Exec(`INSERT INTO documents (id, user_id) VALUES ('d1', ?)`, LocalUserID); err != nil {
|
||||
t.Fatalf("insert document: %v", err)
|
||||
}
|
||||
// One row with every column carrying a distinguishable value, so a dropped
|
||||
// column shows up as a changed value rather than as a passing test.
|
||||
if _, err := d.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash)
|
||||
VALUES ('s-1', 'd1', 7, 11, 'by foots', 'on foot', 'idiom advice she has read', 'idiom', 'accepted', '2026-01-02 03:04:05', '2026-01-02 03:05:00', 'local', 'abc123')`,
|
||||
); err != nil {
|
||||
t.Fatalf("seed row: %v", err)
|
||||
}
|
||||
d.Close()
|
||||
|
||||
d2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen (migrate): %v", err)
|
||||
}
|
||||
defer d2.Close()
|
||||
|
||||
var (
|
||||
docID, original, replacement, explanation string
|
||||
typ, status, source, chunkHash string
|
||||
from, to int
|
||||
// Scanned as instants, not strings: the driver renders a DATETIME column in
|
||||
// its own format, so the claim is "the same moment", not the same text.
|
||||
createdAt, resolvedAt time.Time
|
||||
)
|
||||
if err := d2.QueryRow(
|
||||
`SELECT doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash
|
||||
FROM suggestions WHERE id = 's-1'`,
|
||||
).Scan(&docID, &from, &to, &original, &replacement, &explanation,
|
||||
&typ, &status, &createdAt, &resolvedAt, &source, &chunkHash); err != nil {
|
||||
t.Fatalf("read migrated row: %v", err)
|
||||
}
|
||||
for _, c := range []struct{ name, got, want string }{
|
||||
{"doc_id", docID, "d1"},
|
||||
{"original", original, "by foots"},
|
||||
{"replacement", replacement, "on foot"},
|
||||
{"explanation", explanation, "idiom advice she has read"},
|
||||
{"type", typ, SuggestionTypeIdiom},
|
||||
{"status", status, SuggestionStatusAccepted},
|
||||
{"source", source, SuggestionSourceLocal},
|
||||
{"chunk_hash", chunkHash, "abc123"},
|
||||
} {
|
||||
if c.got != c.want {
|
||||
t.Errorf("%s = %q, want %q", c.name, c.got, c.want)
|
||||
}
|
||||
}
|
||||
if from != 7 || to != 11 {
|
||||
t.Errorf("offsets = (%d, %d), want (7, 11)", from, to)
|
||||
}
|
||||
// created_at and resolved_at must survive: the rail's arrival chime keys on
|
||||
// created_at, and the growth journal counts by resolved_at. A rebuild that
|
||||
// reset either would re-chime her whole document and rewrite her history.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
got time.Time
|
||||
want string
|
||||
}{
|
||||
{"created_at", createdAt, "2026-01-02 03:04:05"},
|
||||
{"resolved_at", resolvedAt, "2026-01-02 03:05:00"},
|
||||
} {
|
||||
want, err := time.Parse("2006-01-02 15:04:05", c.want)
|
||||
if err != nil {
|
||||
t.Fatalf("parse want: %v", err)
|
||||
}
|
||||
if !c.got.Equal(want) {
|
||||
t.Errorf("%s = %v, want the original instant %v", c.name, c.got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The point of the rebuild: the new type is now insertable, and a bogus one
|
||||
// still isn't.
|
||||
if _, err := d2.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES ('s-2', 'd1', 0, 3, '苹果', 'apple', 'x', ?)`, SuggestionTypeTranslate,
|
||||
); err != nil {
|
||||
t.Fatalf("insert translate row: %v", err)
|
||||
}
|
||||
if _, err := d2.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES ('s-3', 'd1', 0, 3, 'x', 'y', 'x', 'nonsense')`,
|
||||
); err == nil {
|
||||
t.Error("CHECK constraint accepted an unknown type after the rebuild")
|
||||
}
|
||||
|
||||
// Both indexes must come back, or every document load starts table-scanning.
|
||||
for _, idx := range []string{"idx_suggestions_doc_id", "idx_suggestions_resolved"} {
|
||||
var name string
|
||||
if err := d2.QueryRow(
|
||||
`SELECT name FROM sqlite_master WHERE type = 'index' AND name = ?`, idx,
|
||||
).Scan(&name); err != nil {
|
||||
t.Errorf("index %s missing after rebuild: %v", idx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+36
-1
@@ -15,6 +15,19 @@ type User struct {
|
||||
// today, "pt-PT"/"fr"/"es" once the langpacks land. It selects the UI copy
|
||||
// and dictionary set, not the language they may type in.
|
||||
PairLang string `json:"pair_lang"`
|
||||
|
||||
// Direction says which half of the pair is being *learned*. Every pair until
|
||||
// now assumed one answer: the writer is native in X and practising English,
|
||||
// so hanzi is never tokenized and English is what gets underlined. Turn it
|
||||
// around — a native English speaker learning Chinese — and the same pair
|
||||
// wants the opposite of nearly every default.
|
||||
//
|
||||
// It is a separate column from PairLang rather than a second pair code
|
||||
// ("zh-learner") because it is a genuinely separate question: the pair says
|
||||
// *which two languages*, this says *which way round*. Keeping them apart is
|
||||
// what lets fr, es and pt-PT inherit the learner direction later without a
|
||||
// second langpack each.
|
||||
Direction string `json:"direction"`
|
||||
}
|
||||
|
||||
// Document is a single piece of writing. `Content` is the Tiptap JSON document
|
||||
@@ -28,6 +41,11 @@ type Document struct {
|
||||
ContentText string `json:"content_text"` // plain text for the LLM
|
||||
Tone string `json:"tone"` // target writing tone; steers LLM advice
|
||||
WordCount int `json:"word_count"`
|
||||
// DocLang is which language this document is written in — '' | 'en' | 'pair'
|
||||
// (migration 0017), written by the checkpoint pass and never by the client.
|
||||
// It reaches the client read-only, for the one decision the client has to
|
||||
// make on its own: which voice reads a selection aloud. '' means English.
|
||||
DocLang string `json:"doc_lang"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
|
||||
@@ -104,8 +122,12 @@ type Suggestion struct {
|
||||
Original string `json:"original"`
|
||||
Replacement string `json:"replacement"`
|
||||
Explanation string `json:"explanation"`
|
||||
Type string `json:"type"` // grammar | phrasing | idiom | clarity | voice | collocation
|
||||
Type string `json:"type"` // grammar | phrasing | idiom | clarity | translate | voice | collocation
|
||||
Status string `json:"status"` // pending | accepted | rejected
|
||||
// Source names the engine that proposed the edit, not its family: an offline
|
||||
// rule and the model can both propose a collocation, and the writer is never
|
||||
// told which one spoke. It exists so each pass can replace its own rows.
|
||||
Source string `json:"source"` // llm | local
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
@@ -115,10 +137,23 @@ const (
|
||||
SuggestionTypePhrasing = "phrasing"
|
||||
SuggestionTypeIdiom = "idiom"
|
||||
SuggestionTypeClarity = "clarity"
|
||||
// A span she wrote in her own language, rendered into English. Not a
|
||||
// correction — nothing was wrong with it — which is why it is its own type
|
||||
// rather than a clarity fix: the card is the pair model's flagship moment
|
||||
// (SUGGESTIONS §1), and labelling it "Clarity" reads as a tidy-up of her
|
||||
// first language. The model isn't asked for this label; it is derived from the
|
||||
// span itself (see suggestions/language.go), so it can't drift.
|
||||
SuggestionTypeTranslate = "translate"
|
||||
SuggestionTypeVoice = "voice"
|
||||
SuggestionTypeCollocation = "collocation"
|
||||
SuggestionTypeMechanics = "mechanics" // deterministic rule-based pass (no LLM)
|
||||
|
||||
// Who proposed it. The offline rule pack ('local') runs on every edit inside
|
||||
// the browser and survives a VPN-down box; the model ('llm') adds the long
|
||||
// tail when it is reachable.
|
||||
SuggestionSourceLLM = "llm"
|
||||
SuggestionSourceLocal = "local"
|
||||
|
||||
SuggestionStatusPending = "pending"
|
||||
SuggestionStatusAccepted = "accepted"
|
||||
SuggestionStatusRejected = "rejected"
|
||||
|
||||
+71
-4
@@ -287,7 +287,14 @@ func mdBlock(n pmNode, depth int) string {
|
||||
return "---"
|
||||
case "image":
|
||||
alt := n.attrStr("alt")
|
||||
return fmt.Sprintf("", alt, n.attrStr("src"))
|
||||
src := safeURL(n.attrStr("src"))
|
||||
if src == "" {
|
||||
// Nowhere safe to point. Keep the alt text as plain prose — it is
|
||||
// the part that carries meaning — rather than emitting an image
|
||||
// whose destination was rejected. See safeURL.
|
||||
return alt
|
||||
}
|
||||
return fmt.Sprintf("", alt, src)
|
||||
case "table":
|
||||
return mdTable(n)
|
||||
case "bulletList", "orderedList":
|
||||
@@ -396,7 +403,9 @@ func applyMdMarks(n pmNode) string {
|
||||
if n.hasMark("underline") {
|
||||
t = "<u>" + t + "</u>"
|
||||
}
|
||||
if href := n.markAttr("link", "href"); href != "" {
|
||||
// Same rule as the HTML export: plenty of Markdown renderers pass a
|
||||
// `javascript:` destination straight through into an <a href>. See safeURL.
|
||||
if href := safeURL(n.markAttr("link", "href")); href != "" {
|
||||
t = "[" + t + "](" + href + ")"
|
||||
}
|
||||
return t
|
||||
@@ -518,7 +527,16 @@ func htmlBlock(n pmNode) string {
|
||||
return "<hr>\n"
|
||||
case "image":
|
||||
alt := htmlEscape(n.attrStr("alt"))
|
||||
return fmt.Sprintf("<p><img src=\"%s\" alt=\"%s\"></p>\n", htmlEscape(n.attrStr("src")), alt)
|
||||
src := safeURL(n.attrStr("src"))
|
||||
if src == "" {
|
||||
// Nowhere safe to point: keep the alt text, which is the part that
|
||||
// carries meaning, rather than emitting a broken image.
|
||||
if alt == "" {
|
||||
return ""
|
||||
}
|
||||
return "<p>" + alt + "</p>\n"
|
||||
}
|
||||
return fmt.Sprintf("<p><img src=\"%s\" alt=\"%s\"></p>\n", htmlEscape(src), alt)
|
||||
case "table":
|
||||
return htmlTable(n)
|
||||
case "bulletList", "orderedList":
|
||||
@@ -617,7 +635,9 @@ func applyHTMLMarks(n pmNode) string {
|
||||
if n.hasMark("highlight") {
|
||||
t = "<mark>" + t + "</mark>"
|
||||
}
|
||||
if href := n.markAttr("link", "href"); href != "" {
|
||||
// An unsafe href is dropped, not the link: the words stay, they just stop
|
||||
// being clickable. See safeURL.
|
||||
if href := safeURL(n.markAttr("link", "href")); href != "" {
|
||||
t = fmt.Sprintf("<a href=\"%s\">%s</a>", htmlEscape(href), t)
|
||||
}
|
||||
return t
|
||||
@@ -858,6 +878,53 @@ func htmlEscape(s string) string {
|
||||
return r.Replace(s)
|
||||
}
|
||||
|
||||
// safeURLSchemes are the schemes an exported document may point at. Escaping
|
||||
// makes a URL safe to sit inside an attribute; it says nothing about what
|
||||
// happens when the attribute is followed, and `javascript:` survives it
|
||||
// untouched.
|
||||
//
|
||||
// The toolbar can't produce one — it prefixes anything it doesn't recognise
|
||||
// with https:// — but the toolbar is not the only way in: PUT /api/docs/{id}
|
||||
// stores whatever Tiptap JSON it is given. And an export is the one artifact
|
||||
// here that is *meant* to leave: the passport and the .html backup are files a
|
||||
// writer hands to a teacher or an editor, opened on a machine that has no
|
||||
// reason to trust them. A link that runs code when clicked is not something to
|
||||
// ship inside one.
|
||||
//
|
||||
// Relative and fragment links pass through: they're how a document refers to
|
||||
// its own headings, and they can't reach anything.
|
||||
var safeURLSchemes = map[string]bool{
|
||||
"http": true, "https": true, "mailto": true, "tel": true, "ftp": true,
|
||||
}
|
||||
|
||||
// safeURL returns u if it is safe to follow from an exported file, and "" if it
|
||||
// isn't. A dropped href leaves the link text in place — the reader loses a
|
||||
// destination, never the writing.
|
||||
func safeURL(u string) string {
|
||||
trimmed := strings.TrimSpace(u)
|
||||
if trimmed == "" {
|
||||
return ""
|
||||
}
|
||||
// A scheme is everything before the first ':', but only when no '/', '?' or
|
||||
// '#' comes first — otherwise "notes/a:b" would read as the "notes/a" scheme.
|
||||
// Nothing before a colon means a relative or fragment link, which is fine.
|
||||
if i := strings.IndexAny(trimmed, ":/?#"); i >= 0 && trimmed[i] == ':' {
|
||||
// Control characters and whitespace are stripped by browsers *before*
|
||||
// the scheme is read, so "java\nscript:" is javascript:. Fold them out
|
||||
// before deciding rather than after.
|
||||
scheme := strings.Map(func(r rune) rune {
|
||||
if r <= ' ' || r == 0x7f {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, trimmed[:i])
|
||||
if !safeURLSchemes[strings.ToLower(scheme)] {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
func xmlEscape(s string) string {
|
||||
r := strings.NewReplacer("&", "&", "<", "<", ">", ">", `"`, """)
|
||||
return r.Replace(s)
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// richDocJSON is a Tiptap document exercising headings, marks, and a list —
|
||||
@@ -243,3 +245,73 @@ func TestExportUnsupportedFormat(t *testing.T) {
|
||||
t.Fatalf("expected 400 for unsupported format, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Escaping makes a URL safe to sit inside an attribute; it says nothing about
|
||||
// what happens when the attribute is followed. An export is the one artifact
|
||||
// here meant to leave — the file handed to a teacher, opened on a machine with
|
||||
// no reason to trust it — so a destination that runs code is dropped.
|
||||
func TestExportDropsUnsafeLinkSchemes(t *testing.T) {
|
||||
unsafe := []string{
|
||||
"javascript:alert(1)",
|
||||
"JaVaScRiPt:alert(1)",
|
||||
"java\nscript:alert(1)", // browsers strip control characters first
|
||||
" javascript:alert(1)",
|
||||
"data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==",
|
||||
"vbscript:msgbox(1)",
|
||||
}
|
||||
for _, href := range unsafe {
|
||||
if got := safeURL(href); got != "" {
|
||||
t.Errorf("safeURL(%q) = %q, want it dropped", href, got)
|
||||
}
|
||||
}
|
||||
|
||||
safe := []string{
|
||||
"https://example.com/a?b=1#c",
|
||||
"http://example.com",
|
||||
"mailto:her@example.com",
|
||||
"/api/images/abc.png",
|
||||
"#a-heading",
|
||||
"notes/chapter:one.md", // a colon that isn't a scheme
|
||||
}
|
||||
for _, href := range safe {
|
||||
if got := safeURL(href); got != href {
|
||||
t.Errorf("safeURL(%q) = %q, want it kept", href, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// End to end through the renderers: an unsafe href loses its destination, never
|
||||
// its words.
|
||||
func TestRenderedExportsCarryNoScriptURLs(t *testing.T) {
|
||||
doc := db.Document{
|
||||
Title: "Notes",
|
||||
Content: `{"type":"doc","content":[{"type":"paragraph","content":[
|
||||
{"type":"text","text":"click me","marks":[{"type":"link","attrs":{"href":"javascript:alert(1)"}}]}]},
|
||||
{"type":"image","attrs":{"src":"javascript:alert(2)","alt":"a drawing"}}]}`,
|
||||
}
|
||||
|
||||
html, err := renderHTMLFile(doc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(string(html)), "javascript:") {
|
||||
t.Fatalf("html export carried a javascript: URL:\n%s", html)
|
||||
}
|
||||
if !strings.Contains(string(html), "click me") {
|
||||
t.Fatal("html export dropped the link text along with the href")
|
||||
}
|
||||
if !strings.Contains(string(html), "a drawing") {
|
||||
t.Fatal("html export dropped the alt text of the rejected image")
|
||||
}
|
||||
|
||||
md, err := renderMarkdown(doc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(string(md)), "javascript:") {
|
||||
t.Fatalf("markdown export carried a javascript: URL:\n%s", md)
|
||||
}
|
||||
if !strings.Contains(string(md), "click me") {
|
||||
t.Fatal("markdown export dropped the link text along with the href")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -225,13 +225,14 @@ func (h *Handler) fetch(userID, id string) (db.Document, error) {
|
||||
var doc db.Document
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT id, user_id, title, content, content_text, tone, word_count,
|
||||
created_at, updated_at, preserve_history
|
||||
created_at, updated_at, preserve_history, doc_lang
|
||||
FROM documents
|
||||
WHERE id = ? AND user_id = ?`,
|
||||
id, userID,
|
||||
).Scan(
|
||||
&doc.ID, &doc.UserID, &doc.Title, &doc.Content, &doc.ContentText,
|
||||
&doc.Tone, &doc.WordCount, &doc.CreatedAt, &doc.UpdatedAt, &doc.PreserveHistory,
|
||||
&doc.DocLang,
|
||||
)
|
||||
return doc, err
|
||||
}
|
||||
|
||||
@@ -35,3 +35,18 @@ func ServerError(w http.ResponseWriter, err error) {
|
||||
log.Printf("internal error: %v", err)
|
||||
ErrorJSON(w, http.StatusInternalServerError, "something went wrong")
|
||||
}
|
||||
|
||||
// UpstreamError is ServerError's counterpart for a dependency Petal calls out
|
||||
// to — the model, chiefly. Same discipline, and for a sharper reason: a dial
|
||||
// failure's error text contains the endpoint it failed to dial, so relaying it
|
||||
// hands anyone who can reach Petal the address of the inference box on the far
|
||||
// side of the VPN, along with which backend is running there.
|
||||
//
|
||||
// `what` names the pass for the operator's log ("checkpoint", "chat"). The
|
||||
// browser is told only that the helper is unreachable, which is all the client
|
||||
// ever did anything with: every LLM route's 502 renders as the same warm
|
||||
// "小助手在休息 · Petal's helper is resting".
|
||||
func UpstreamError(w http.ResponseWriter, what string, err error) {
|
||||
log.Printf("upstream error (%s): %v", what, err)
|
||||
ErrorJSON(w, http.StatusBadGateway, "Petal's helper is out of reach right now")
|
||||
}
|
||||
|
||||
@@ -37,6 +37,17 @@ import (
|
||||
// small enough to keep a careless paste from filling the disk.
|
||||
const maxUploadBytes = 10 << 20
|
||||
|
||||
// maxUserBytes caps what one account may keep stored, at 1 GiB. The per-upload
|
||||
// limit bounds a single careless paste; nothing bounded ten thousand of them,
|
||||
// and Petal's data directory is an 8 GiB encrypted volume shared with the
|
||||
// database, the backups and the TTS cache — the disk filling is the database
|
||||
// losing writes, not just images failing.
|
||||
//
|
||||
// A tenth of the volume per writer is far past any real use: a heavily
|
||||
// illustrated journal is tens of megabytes. It is a runaway backstop, and it is
|
||||
// deliberately generous enough that nobody writing normally will ever meet it.
|
||||
const maxUserBytes = 1 << 30
|
||||
|
||||
// extByContentType maps the image types we accept to a canonical extension. The
|
||||
// allowlist doubles as validation: anything not here is rejected.
|
||||
var extByContentType = map[string]string{
|
||||
@@ -177,6 +188,19 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
name := hex.EncodeToString(sum[:])[:32] + ext
|
||||
path := filepath.Join(h.dir, name)
|
||||
|
||||
userID := auth.UserID(r.Context())
|
||||
within, err := h.withinQuota(userID, name, int64(len(data)))
|
||||
if err != nil {
|
||||
log.Printf("images: quota check failed for %s: %v", userID, err)
|
||||
http.Error(w, "could not store image", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !within {
|
||||
http.Error(w, "you've filled Petal's picture store — delete a few images and try again",
|
||||
http.StatusInsufficientStorage)
|
||||
return
|
||||
}
|
||||
|
||||
// Skip the write if this exact content is already stored.
|
||||
if _, statErr := os.Stat(path); errors.Is(statErr, os.ErrNotExist) {
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
@@ -190,7 +214,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := h.db.Exec(
|
||||
`INSERT INTO images (name, user_id, content_type, size) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (name, user_id) DO NOTHING`,
|
||||
name, auth.UserID(r.Context()), ct, len(data),
|
||||
name, userID, ct, len(data),
|
||||
); err != nil {
|
||||
log.Printf("images: could not record ownership of %s: %v", name, err)
|
||||
http.Error(w, "could not store image", http.StatusInternalServerError)
|
||||
@@ -221,6 +245,20 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
// Private: a shared cache must never hand one writer's image to another.
|
||||
w.Header().Set("Cache-Control", "private, max-age=31536000, immutable")
|
||||
|
||||
// SVG is a document format wearing an image's name: it can carry <script>,
|
||||
// and this route serves it from Petal's own origin. Rendered through an
|
||||
// <img> — the only way the editor ever shows one — that script never runs.
|
||||
// Navigated to directly, which is one "open image in new tab" away, it does,
|
||||
// and it runs with the API of whoever opened it.
|
||||
//
|
||||
// So every stored image answers with a CSP that permits nothing at all
|
||||
// except the inline styles an illustration legitimately carries. It costs
|
||||
// pasted SVGs nothing (an <img> was already a script-free context) and
|
||||
// leaves the direct-navigation case inert. nosniff is set at the edge, but
|
||||
// repeated here so the guarantee doesn't depend on Traefik's config.
|
||||
w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
http.ServeFile(w, r, path)
|
||||
}
|
||||
|
||||
@@ -261,6 +299,29 @@ func (h *Handler) remove(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// withinQuota reports whether userID may store one more image of size bytes.
|
||||
//
|
||||
// An image the caller already owns is free: content addressing means re-pasting
|
||||
// the same picture stores nothing new, and charging for it would let a document
|
||||
// that merely repeats one illustration walk into the limit. Deduplication
|
||||
// across *accounts* is not credited the same way — two people each keep their
|
||||
// own claim on a shared file, because either of them deleting it must not
|
||||
// depend on what the other did.
|
||||
func (h *Handler) withinQuota(userID, name string, size int64) (bool, error) {
|
||||
var used, already sql.NullInt64
|
||||
if err := h.db.QueryRow(
|
||||
`SELECT (SELECT COALESCE(SUM(size), 0) FROM images WHERE user_id = ?),
|
||||
(SELECT size FROM images WHERE user_id = ? AND name = ?)`,
|
||||
userID, userID, name,
|
||||
).Scan(&used, &already); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if already.Valid {
|
||||
return true, nil // already stored for this account — costs nothing more
|
||||
}
|
||||
return used.Int64+size <= maxUserBytes, nil
|
||||
}
|
||||
|
||||
// owns reports whether userID has a claim on a stored image.
|
||||
func (h *Handler) owns(name, userID string) bool {
|
||||
var ok bool
|
||||
|
||||
@@ -240,3 +240,85 @@ func TestServeMissing(t *testing.T) {
|
||||
t.Fatalf("expected 404, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// An SVG is a document, not a picture: it can carry <script>, and this route
|
||||
// serves it from Petal's own origin. Rendered through an <img> that script
|
||||
// never runs, but "open image in new tab" is one click away, and there it
|
||||
// would — with the API of whoever opened it. Every stored image therefore
|
||||
// answers with a CSP that permits nothing.
|
||||
func TestStoredImagesAreServedInert(t *testing.T) {
|
||||
_, alice, _ := newStore(t)
|
||||
|
||||
svg := []byte(`<svg xmlns="http://www.w3.org/2000/svg"><script>fetch('/api/docs')</script></svg>`)
|
||||
rec := httptest.NewRecorder()
|
||||
alice.ServeHTTP(rec, uploadReq(t, "image", svg))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("svg upload code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
var resp struct{ URL string }
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := strings.TrimPrefix(resp.URL, "/api/images/")
|
||||
|
||||
got := get(t, alice, name)
|
||||
if got.Code != http.StatusOK {
|
||||
t.Fatalf("serve code=%d", got.Code)
|
||||
}
|
||||
csp := got.Header().Get("Content-Security-Policy")
|
||||
if !strings.Contains(csp, "default-src 'none'") || !strings.Contains(csp, "sandbox") {
|
||||
t.Fatalf("CSP %q does not neutralize the response", csp)
|
||||
}
|
||||
if got.Header().Get("X-Content-Type-Options") != "nosniff" {
|
||||
t.Fatal("stored images must be served nosniff")
|
||||
}
|
||||
}
|
||||
|
||||
// A per-upload cap bounds one careless paste; nothing bounded ten thousand of
|
||||
// them, on the same volume the database lives on.
|
||||
func TestUploadQuota(t *testing.T) {
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
if _, err := database.Exec(
|
||||
`INSERT INTO users (id, email, display_name) VALUES (?, ?, ?)`,
|
||||
"bob", "bob@petal.local", "Bob",
|
||||
); err != nil {
|
||||
t.Fatalf("seed second user: %v", err)
|
||||
}
|
||||
h, err := New(t.TempDir(), database.DB, db.LocalUserID)
|
||||
if err != nil {
|
||||
t.Fatalf("new store: %v", err)
|
||||
}
|
||||
alice := auth.Middleware(auth.StaticResolver(db.LocalUserID))(h.Routes())
|
||||
bob := auth.Middleware(auth.StaticResolver("bob"))(h.Routes())
|
||||
|
||||
// Fill Alice's allowance by hand — uploading a gibibyte in a test would be
|
||||
// absurd, and what's under test is the accounting, not the arithmetic.
|
||||
name := upload(t, alice, pngBytes)
|
||||
if _, err := database.Exec(
|
||||
`UPDATE images SET size = ? WHERE user_id = ? AND name = ?`,
|
||||
int64(maxUserBytes), db.LocalUserID, name,
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Re-storing something she already has costs nothing, so it still works.
|
||||
if again := upload(t, alice, pngBytes); again != name {
|
||||
t.Fatalf("a re-upload of an owned image should dedupe, got %q", again)
|
||||
}
|
||||
|
||||
// Anything new does not.
|
||||
rec := httptest.NewRecorder()
|
||||
alice.ServeHTTP(rec, uploadReq(t, "image", otherPNG))
|
||||
if rec.Code != http.StatusInsufficientStorage {
|
||||
t.Fatalf("over-quota upload code=%d, want 507", rec.Code)
|
||||
}
|
||||
|
||||
// And it is *her* allowance, not the store's: Bob is unaffected.
|
||||
if got := upload(t, bob, otherPNG); got == "" {
|
||||
t.Fatal("one writer's quota must not stop another writing")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,3 +36,13 @@ var glossGz []byte
|
||||
//
|
||||
//go:embed data/phonetic.json.gz
|
||||
var phoneticGz []byte
|
||||
|
||||
// hanziGz is the gzipped Chinese→English map: simplified headword → [[pinyin,
|
||||
// senses], …]. Built from CC-CEDICT (scripts/build_cedict.py), unfiltered — the
|
||||
// word a learner stops on is the one they do not know, so this is the one
|
||||
// dataset here with no frequency gate. Loaded on its own sync.Once (see
|
||||
// hanzi.go), not with the four above, because only a learner-direction account
|
||||
// ever asks for it.
|
||||
//
|
||||
//go:embed data/hanzi.json.gz
|
||||
var hanziGz []byte
|
||||
|
||||
Binary file not shown.
@@ -2,8 +2,10 @@ package lexicon
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
@@ -56,9 +58,34 @@ func (dd *DreamDict) Close() error {
|
||||
return dd.d.Close()
|
||||
}
|
||||
|
||||
// Langs returns the language codes dict.db was built with, so startup can log
|
||||
// what it actually got rather than what it hoped for.
|
||||
func (dd *DreamDict) Langs() []string { return dictionary.Langs() }
|
||||
// Contents reports how many words the open dict.db holds per language, so
|
||||
// startup can log what it actually got.
|
||||
//
|
||||
// It counts rows rather than returning DreamDict's list of supported languages.
|
||||
// Those are not the same thing and the difference is the whole point: a
|
||||
// database built before Spanish existed still *supports* Spanish, and a log
|
||||
// line naming the supported set would have said so cheerfully while every
|
||||
// Spanish lookup came back empty. Counting rows is the question worth asking of
|
||||
// a file somebody had to copy onto the box by hand.
|
||||
func (dd *DreamDict) Contents() string {
|
||||
counts, err := dd.d.WordCount()
|
||||
if err != nil {
|
||||
return "unreadable: " + err.Error()
|
||||
}
|
||||
langs := make([]string, 0, len(counts))
|
||||
for lang := range counts {
|
||||
langs = append(langs, lang)
|
||||
}
|
||||
sort.Strings(langs)
|
||||
parts := make([]string, 0, len(langs))
|
||||
for _, lang := range langs {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", lang, counts[lang]))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "no words"
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// dreamProvider serves one writer: English lookups from dict.db, glossed into
|
||||
// native. The struct is a value, created per request by [Set.For] — it holds no
|
||||
@@ -147,9 +174,63 @@ func (p dreamProvider) Lookup(word string) (Result, error) {
|
||||
}
|
||||
res.Etymology = trimEtymology(ety)
|
||||
|
||||
if res.Reverse, err = p.reverse(norm); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// reverse reads the token as a word of the writer's own language, and returns
|
||||
// nil when it isn't one — which is the answer for almost every word she looks
|
||||
// up, since she is writing English.
|
||||
//
|
||||
// The English de-inflection walk is deliberately *not* applied here. [candidates]
|
||||
// knows about -s, -ed and -ing; running it over Portuguese would turn "vinhas"
|
||||
// into "vinha" by an English rule that happens to be right and "cantava" into
|
||||
// nothing by rules that are simply irrelevant. dict.db stores headwords, so an
|
||||
// inflected Portuguese form finds nothing and the card shows only the English
|
||||
// reading — the same outcome as today, rather than a confidently wrong one.
|
||||
func (p dreamProvider) reverse(norm string) (*Reverse, error) {
|
||||
back, err := p.dict.d.Equivalents(norm, p.native, langEN)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defs, err := p.dict.d.Define(norm, p.native)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(back) == 0 && len(defs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rev := &Reverse{Lang: p.native}
|
||||
if len(back) > maxGlossSenses {
|
||||
back = back[:maxGlossSenses]
|
||||
}
|
||||
rev.Gloss = strings.Join(back, "; ")
|
||||
for _, d := range defs {
|
||||
rev.Definitions = append(rev.Definitions, Meaning{PartOfSpeech: d.POS, Definition: d.Gloss})
|
||||
if len(rev.Definitions) >= maxReverseDefinitions {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
prons, err := p.dict.d.Pronunciation(norm, p.native)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rev.Phonetic = pickIPA(prons)
|
||||
|
||||
return rev, nil
|
||||
}
|
||||
|
||||
// maxReverseDefinitions is smaller than [maxDefinitions]: the reverse reading is
|
||||
// the second half of a card that already has an English one, and it is there to
|
||||
// say "this is also a Portuguese word, and here is what it means" rather than to
|
||||
// be a dictionary entry in its own right.
|
||||
const maxReverseDefinitions = 2
|
||||
|
||||
// Gloss returns the writer's-language translation alone — the hover tooltip's
|
||||
// fast path, one indexed query per candidate form and nothing else.
|
||||
func (p dreamProvider) Gloss(word string) (GlossResult, error) {
|
||||
@@ -161,7 +242,21 @@ func (p dreamProvider) Gloss(word string) (GlossResult, error) {
|
||||
if err != nil {
|
||||
return GlossResult{}, err
|
||||
}
|
||||
return GlossResult{Word: word, Gloss: gloss}, nil
|
||||
res := GlossResult{Word: word, Gloss: gloss}
|
||||
|
||||
// The tooltip carries only the reverse *gloss*, not the whole reading: it is
|
||||
// a one-line bubble under a resting pointer, and the popover is one click
|
||||
// away for anyone who wants the rest.
|
||||
back, err := p.dict.d.Equivalents(norm, p.native, langEN)
|
||||
if err != nil {
|
||||
return GlossResult{}, err
|
||||
}
|
||||
if len(back) > maxGlossSenses {
|
||||
back = back[:maxGlossSenses]
|
||||
}
|
||||
res.Reverse = strings.Join(back, "; ")
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// maxGlossSenses caps how many translations are strung together. One is often
|
||||
@@ -180,8 +275,8 @@ const maxGlossSenses = 3
|
||||
//
|
||||
// A language dict.db was built without simply has no rows, so this returns "" —
|
||||
// which is exactly what an unglossed word returns, and the popover already
|
||||
// renders that case. Spanish today is precisely this: supported by DreamDict,
|
||||
// absent from the deployed database until it is rebuilt.
|
||||
// renders that case. Spanish was precisely this until the database was rebuilt
|
||||
// with it on 2026-07-27; the code path did not change, the file did.
|
||||
func (p dreamProvider) translate(norm string) (string, error) {
|
||||
for _, c := range candidates(norm) {
|
||||
trs, err := p.dict.d.Equivalents(c, langEN, p.native)
|
||||
|
||||
@@ -96,6 +96,24 @@ func writeFixture(t *testing.T, seeded bool) string {
|
||||
exec(`INSERT INTO word_synsets (word_id, synset_id, source) VALUES
|
||||
(5, 1, 'wordnet'), (6, 1, 'omw'), (7, 1, 'omw')`)
|
||||
|
||||
// "data" is the collision the Latin pairs create and the zh pair never did:
|
||||
// a real English word and a real Portuguese one, spelled identically and
|
||||
// meaning different things. There is no honest way to look at it in a mixed
|
||||
// document and know which was meant, so Petal shows both readings.
|
||||
exec(`INSERT INTO words (id, word, lang, pos, frequency) VALUES
|
||||
(8, 'data', 'en', 'noun', 800),
|
||||
(9, 'data', 'pt-PT', 'noun', 700),
|
||||
(10, 'date', 'en', 'noun', 750)`)
|
||||
exec(`INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES
|
||||
(8, 'noun', 'facts collected for reference', 'wordnet', 10),
|
||||
(9, 'noun', 'dia do mês', 'wiktionary', 20),
|
||||
(9, 'noun', 'momento no tempo', 'wiktionary', 30),
|
||||
(9, 'noun', 'um terceiro sentido', 'wiktionary', 40)`)
|
||||
exec(`INSERT INTO translations (word_id, translation, target_lang, source) VALUES
|
||||
(9, 'date', 'en', 'kaikki')`)
|
||||
exec(`INSERT INTO pronunciations (word_id, format, value, source) VALUES
|
||||
(9, 'ipa', '/ˈdatɐ/', 'wiktionary')`)
|
||||
|
||||
exec(`INSERT INTO etymology (word_id, text, source) VALUES
|
||||
(1, 'From Medieval Latin ephemerus, from Ancient Greek ἐφήμερος (ephḗmeros, "lasting only a day"), from ἐπί (epí, "upon") and ἡμέρα (hēméra, "day"). The sense of transience is attested in English from the late sixteenth century onwards.', 'wiktionary')`)
|
||||
|
||||
@@ -519,3 +537,128 @@ func TestHandlerDecodesPunctuatedWords(t *testing.T) {
|
||||
t.Errorf("Word = %q, want the decoded word", res.Word)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentsCountsRowsNotSupportedLanguages(t *testing.T) {
|
||||
// The fixture is seeded with English and pt-PT only. DreamDict *supports*
|
||||
// French, Spanish and Chinese too — and a startup line that reported the
|
||||
// supported set would have named all five while every French lookup came
|
||||
// back empty. That is the failure this log line exists to catch, so it must
|
||||
// count rows.
|
||||
got := NewSet(openFixture(t))
|
||||
summary := got.Contents()
|
||||
if !strings.Contains(summary, "en=") || !strings.Contains(summary, "pt-PT=") {
|
||||
t.Errorf("Contents = %q, want the languages the fixture actually holds", summary)
|
||||
}
|
||||
for _, absent := range []string{"fr=", "es=", "zh="} {
|
||||
if strings.Contains(summary, absent) {
|
||||
t.Errorf("Contents = %q, must not name %q — no rows exist for it", summary, absent)
|
||||
}
|
||||
}
|
||||
// No dictionary at all still has to answer something printable.
|
||||
if s := NewSet(nil).Contents(); s == "" {
|
||||
t.Error("Contents with no dictionary must still say something")
|
||||
}
|
||||
}
|
||||
|
||||
// The Latin+Latin wrinkle (SUGGESTIONS.md §3a). An English+Chinese pair never
|
||||
// had to decide which language a word was in — the script decided. An
|
||||
// English+Portuguese pair has no script boundary, and "data", "sale", "comum"
|
||||
// and "tarde" are real words on both sides of it. Petal asks both directions
|
||||
// and shows whatever answers, which needs no language detector and therefore
|
||||
// cannot be wrong about somebody's writing.
|
||||
func TestDreamLookupShowsBothReadingsOnACollision(t *testing.T) {
|
||||
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||
res, err := p.Lookup("data")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
|
||||
// The English reading is unchanged and still leads.
|
||||
if len(res.Definitions) == 0 || res.Definitions[0].Definition != "facts collected for reference" {
|
||||
t.Fatalf("Definitions = %v, want the English sense first", res.Definitions)
|
||||
}
|
||||
|
||||
if res.Reverse == nil {
|
||||
t.Fatal("Reverse = nil; a word that exists in both languages must carry both readings")
|
||||
}
|
||||
if res.Reverse.Lang != "pt-PT" {
|
||||
t.Errorf("Reverse.Lang = %q, want the writer's language", res.Reverse.Lang)
|
||||
}
|
||||
if res.Reverse.Gloss != "date" {
|
||||
t.Errorf("Reverse.Gloss = %q, want the English meaning of the Portuguese word", res.Reverse.Gloss)
|
||||
}
|
||||
if res.Reverse.Phonetic != "ˈdatɐ" {
|
||||
t.Errorf("Reverse.Phonetic = %q, want the Portuguese IPA without slashes", res.Reverse.Phonetic)
|
||||
}
|
||||
// The reverse reading is a footnote on a card that already has an English
|
||||
// half, so it is capped harder than the main entry.
|
||||
if len(res.Reverse.Definitions) != maxReverseDefinitions {
|
||||
t.Fatalf("Reverse.Definitions = %d, want %d", len(res.Reverse.Definitions), maxReverseDefinitions)
|
||||
}
|
||||
if res.Reverse.Definitions[0].Definition != "dia do mês" {
|
||||
t.Errorf("Reverse.Definitions[0] = %q, want the Portuguese sense",
|
||||
res.Reverse.Definitions[0].Definition)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDreamLookupHasNoReverseForAnEnglishOnlyWord(t *testing.T) {
|
||||
// Which is almost every word she looks up: she is writing English. A
|
||||
// second block under every card would make the collision case invisible.
|
||||
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||
res, err := p.Lookup("ephemeral")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if res.Reverse != nil {
|
||||
t.Fatalf("Reverse = %+v, want none for a word that is only English", res.Reverse)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDreamGlossCarriesTheReverseReading(t *testing.T) {
|
||||
// The hover tooltip takes the same both-directions rule in one line less
|
||||
// space: the reverse *gloss* only, never the definitions.
|
||||
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||
|
||||
g, err := p.Gloss("data")
|
||||
if err != nil {
|
||||
t.Fatalf("Gloss: %v", err)
|
||||
}
|
||||
if g.Reverse != "date" {
|
||||
t.Errorf("Gloss.Reverse = %q, want the English meaning of the Portuguese word", g.Reverse)
|
||||
}
|
||||
|
||||
g, err = p.Gloss("ephemeral")
|
||||
if err != nil {
|
||||
t.Fatalf("Gloss: %v", err)
|
||||
}
|
||||
if g.Reverse != "" {
|
||||
t.Errorf("Gloss.Reverse = %q, want none for an English-only word", g.Reverse)
|
||||
}
|
||||
if g.Gloss != "efémero; passageiro" {
|
||||
t.Errorf("Gloss = %q, want the forward gloss untouched", g.Gloss)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReverseIsSilentForTheEmbeddedProviders(t *testing.T) {
|
||||
// The zh pair has no collisions and no DreamDict, and a writer with no
|
||||
// dict.db at all falls through to `glossless`. Neither may start emitting a
|
||||
// reverse block: the card would then claim a Chinese reading of an English
|
||||
// word, which is worse than saying nothing.
|
||||
set := NewSet(nil)
|
||||
|
||||
res, err := set.For(LangZh).Lookup("river")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if res.Reverse != nil {
|
||||
t.Errorf("embedded Reverse = %+v, want none", res.Reverse)
|
||||
}
|
||||
|
||||
res, err = set.For("pt-PT").Lookup("river")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if res.Reverse != nil {
|
||||
t.Errorf("glossless Reverse = %+v, want none", res.Reverse)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
@@ -41,6 +42,45 @@ func (h *Handler) GlossRoutes() chi.Router {
|
||||
return r
|
||||
}
|
||||
|
||||
// HanziRoutes returns the router mounted at /api/hanzi — a Chinese word to its
|
||||
// pinyin and English senses, for a writer going the other way through the zh
|
||||
// pair (`users.direction = 'learning_pair'`).
|
||||
//
|
||||
// It does not go through [Handler.providerFor], and that is not an oversight.
|
||||
// providerFor picks a dictionary by the writer's *pair*, to answer "what does
|
||||
// this English word mean in her language" — a question whose answer differs per
|
||||
// pair. This endpoint asks the opposite question of exactly one language: it
|
||||
// reads hanzi, and hanzi are Chinese whoever is looking them up. Routing it
|
||||
// through the pair would add a database read per hover to choose between one
|
||||
// option and itself.
|
||||
//
|
||||
// What no longer holds is the reason this used to give — that
|
||||
// [auth.SupportsLearnerDirection] guarantees the caller is on the zh pair. Since
|
||||
// Portuguese joined `learnerPairs` a learning_pair account may be Portuguese, so
|
||||
// the guarantee now comes from the *caller*: the client only ever asks this
|
||||
// route about a token its Chinese segmenter found, and that segmenter is loaded
|
||||
// only for the zh pair (see useSegmenter in App.tsx). A stray lookup is still
|
||||
// answered safely — a word the Chinese dictionary has never heard of is a 200
|
||||
// with empty lists, exactly like any other miss.
|
||||
func (h *Handler) HanziRoutes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/{word}", h.hanzi)
|
||||
return r
|
||||
}
|
||||
|
||||
// hanzi answers a Chinese word lookup. Like the other two, a miss is a 200 with
|
||||
// empty lists — a hover that lands on a word the dictionary has never heard of
|
||||
// is an ordinary thing to happen while reading, and the tooltip simply doesn't
|
||||
// open.
|
||||
func (h *Handler) hanzi(w http.ResponseWriter, r *http.Request) {
|
||||
res, err := h.Set.Hanzi(pathWord(r))
|
||||
if err != nil {
|
||||
writeLookupErr(w, err)
|
||||
return
|
||||
}
|
||||
writeLookup(w, res)
|
||||
}
|
||||
|
||||
// providerFor returns the provider for the caller's language pair.
|
||||
//
|
||||
// The pair language is read here rather than threaded down because a word
|
||||
@@ -95,10 +135,15 @@ func (h *Handler) gloss(w http.ResponseWriter, r *http.Request) {
|
||||
writeLookup(w, res)
|
||||
}
|
||||
|
||||
// writeLookupErr answers a failed lookup. The real error is a dictionary or
|
||||
// database fault — a file path, a SQLite message — and belongs in the log, not
|
||||
// in a tooltip. The client treats any non-200 the same way, so nothing is lost
|
||||
// by saying less.
|
||||
func writeLookupErr(w http.ResponseWriter, err error) {
|
||||
log.Printf("lexicon: lookup failed: %v", err)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": "lookup failed"})
|
||||
}
|
||||
|
||||
func writeLookup(w http.ResponseWriter, v any) {
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// The Chinese half of the lexicon: a word written in hanzi to its pinyin and
|
||||
// English senses. This is the mirror image of `gloss` — that one reads English
|
||||
// and answers in Chinese, for a Mandarin native practising English; this one
|
||||
// reads Chinese and answers in English, for the other direction of the same
|
||||
// pair (`users.direction = 'learning_pair'`).
|
||||
//
|
||||
// It is deliberately not folded into [Lexicon.load]. That method reads four
|
||||
// datasets on the first lookup of any kind, and this one is 3.1 MB gzipped that
|
||||
// only a learner-direction account will ever ask for — every other writer would
|
||||
// pay the decompression and the resident memory for a map they never touch. Its
|
||||
// own sync.Once means the cost lands on the first Chinese hover and nowhere
|
||||
// else.
|
||||
|
||||
// HanziReading is one pronunciation of a word and the senses it carries in that
|
||||
// pronunciation. A word usually has one; the ones that have two are why this is
|
||||
// a list rather than a pair of strings. 得 is dé, "to obtain", *and* de, the
|
||||
// particle that makes 说得很好 mean "speaks well" — a learner shown only the
|
||||
// first has been told something false about the sentence in front of them.
|
||||
type HanziReading struct {
|
||||
Pinyin string `json:"pinyin"`
|
||||
Senses string `json:"senses"`
|
||||
}
|
||||
|
||||
// HanziChar is one character of a word that the dictionary could not answer as
|
||||
// a whole. See [Lexicon.Hanzi].
|
||||
type HanziChar struct {
|
||||
Char string `json:"char"`
|
||||
Pinyin string `json:"pinyin"`
|
||||
Senses string `json:"senses"`
|
||||
}
|
||||
|
||||
// HanziResult is what a Chinese word lookup answers. Readings is empty for a
|
||||
// word the dictionary does not have, in which case Chars may carry the
|
||||
// character-by-character reading instead.
|
||||
type HanziResult struct {
|
||||
Word string `json:"word"`
|
||||
Readings []HanziReading `json:"readings"`
|
||||
Chars []HanziChar `json:"chars"`
|
||||
}
|
||||
|
||||
type hanziStore struct {
|
||||
once sync.Once
|
||||
err error
|
||||
// word → [[pinyin, senses], …], exactly as scripts/build_cedict.py writes it.
|
||||
entries map[string][][]string
|
||||
}
|
||||
|
||||
var hanzi hanziStore
|
||||
|
||||
func (h *hanziStore) load() {
|
||||
h.once.Do(func() {
|
||||
if err := gunzipJSON(hanziGz, &h.entries); err != nil {
|
||||
h.err = fmt.Errorf("load hanzi: %w", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// maxHanziChars caps the per-character fallback. A run longer than this is
|
||||
// almost certainly a phrase the segmenter split badly rather than a word, and
|
||||
// spelling out eight characters one at a time is a wall, not a hint.
|
||||
const maxHanziChars = 6
|
||||
|
||||
// Hanzi returns the pinyin and English senses of a Chinese word.
|
||||
//
|
||||
// There is no de-inflection walk here, and its absence is a fact about the
|
||||
// language rather than an omission: Chinese words do not inflect, so the
|
||||
// candidate forms [lookupGloss] tries for "running" → "run" have no analogue.
|
||||
// A lookup either hits the headword or it does not.
|
||||
//
|
||||
// What it does instead is fall back to the characters. The segmentation word
|
||||
// list is a superset of this dictionary — every glossable word can be
|
||||
// segmented, but jieba knows ordinary compounds CC-CEDICT has no entry for — so
|
||||
// a hover really can land on a word with nothing to say about it. Chinese
|
||||
// compounds are usually transparent from their parts (电脑 is "electric brain"),
|
||||
// which makes the character reading a genuinely useful second answer rather
|
||||
// than a consolation prize. It is returned as its own field so the surface can
|
||||
// say which of the two it is showing; a caller that only wants whole words can
|
||||
// ignore it.
|
||||
func (l *Lexicon) Hanzi(word string) (HanziResult, error) {
|
||||
hanzi.load()
|
||||
if hanzi.err != nil {
|
||||
return HanziResult{}, hanzi.err
|
||||
}
|
||||
|
||||
norm := strings.TrimSpace(word)
|
||||
res := HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}
|
||||
if norm == "" {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
if rows, ok := hanzi.entries[norm]; ok {
|
||||
res.Readings = toReadings(rows)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
chars := []rune(norm)
|
||||
if len(chars) < 2 || len(chars) > maxHanziChars {
|
||||
// A single character that missed has no parts to fall back to, and a long
|
||||
// run is not a word. Either way the honest answer is nothing.
|
||||
return res, nil
|
||||
}
|
||||
for _, r := range chars {
|
||||
if !unicode.Is(unicode.Han, r) {
|
||||
// Mixed input (a stray letter or digit inside the run) is not something
|
||||
// the character reading can explain, and guessing at the hanzi parts of
|
||||
// it would be worse than silence.
|
||||
return HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}, nil
|
||||
}
|
||||
rows, ok := hanzi.entries[string(r)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
first := toReadings(rows)
|
||||
if len(first) == 0 {
|
||||
continue
|
||||
}
|
||||
res.Chars = append(res.Chars, HanziChar{
|
||||
Char: string(r),
|
||||
Pinyin: first[0].Pinyin,
|
||||
Senses: first[0].Senses,
|
||||
})
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func toReadings(rows [][]string) []HanziReading {
|
||||
out := make([]HanziReading, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if len(row) < 2 {
|
||||
continue
|
||||
}
|
||||
out = append(out, HanziReading{Pinyin: row[0], Senses: row[1]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
// The Chinese direction of the lexicon, against the real embedded asset — not a
|
||||
// fixture. The dataset is built by scripts/build_cedict.py, which asserts its
|
||||
// own invariants at build time; what these assert is that the *lookup* over it
|
||||
// behaves, including on the entries the build script goes out of its way to keep.
|
||||
|
||||
func TestHanziLookup(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
res, err := l.Hanzi("公园")
|
||||
if err != nil {
|
||||
t.Fatalf("lookup 公园: %v", err)
|
||||
}
|
||||
if len(res.Readings) == 0 {
|
||||
t.Fatal("公园 has no readings")
|
||||
}
|
||||
// Tone marks, not the numbered pinyin CC-CEDICT stores. The number is the
|
||||
// storage format; the marks are what a learner reads.
|
||||
if got := res.Readings[0].Pinyin; got != "gōngyuán" {
|
||||
t.Errorf("公园 pinyin = %q, want gōngyuán", got)
|
||||
}
|
||||
if !strings.Contains(res.Readings[0].Senses, "park") {
|
||||
t.Errorf("公园 senses = %q, want something about a park", res.Readings[0].Senses)
|
||||
}
|
||||
// A word answered whole says nothing about its characters — the fallback is
|
||||
// the other branch, and sending both would double the payload of the common
|
||||
// case to no purpose.
|
||||
if len(res.Chars) != 0 {
|
||||
t.Errorf("a whole-word hit also returned %d characters", len(res.Chars))
|
||||
}
|
||||
}
|
||||
|
||||
// 得 is the reason readings are a list. Answered with only dé "to obtain", a
|
||||
// learner hovering it in 说得很好 has been told something false about the
|
||||
// sentence they are looking at.
|
||||
func TestHanziParticleCarriesItsGrammaticalReading(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
for _, particle := range []string{"的", "地", "得"} {
|
||||
res, err := l.Hanzi(particle)
|
||||
if err != nil {
|
||||
t.Fatalf("lookup %s: %v", particle, err)
|
||||
}
|
||||
var found bool
|
||||
for _, r := range res.Readings {
|
||||
if r.Pinyin == "de" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s never reads as neutral \"de\": %+v", particle, res.Readings)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The fallback the segmentation gap makes necessary: jieba knows ordinary
|
||||
// compounds CC-CEDICT has no headword for, so a hover can land on a real word
|
||||
// with no entry. Chinese compounds are usually transparent from their parts, so
|
||||
// the characters are a real second answer.
|
||||
func TestHanziFallsBackToCharacters(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
// Constructed rather than borrowed from the corpus: a word that CC-CEDICT
|
||||
// *does* carry would test the other branch, and which compounds it happens to
|
||||
// omit is not something a test should pin.
|
||||
const made = "猫书"
|
||||
if _, ok := hanzi.entries[made]; ok {
|
||||
t.Skipf("%s has become a real headword; pick another compound", made)
|
||||
}
|
||||
res, err := l.Hanzi(made)
|
||||
if err != nil {
|
||||
t.Fatalf("lookup %s: %v", made, err)
|
||||
}
|
||||
if len(res.Readings) != 0 {
|
||||
t.Fatalf("%s answered as a whole word: %+v", made, res.Readings)
|
||||
}
|
||||
if len(res.Chars) != 2 {
|
||||
t.Fatalf("character fallback gave %d entries, want 2: %+v", len(res.Chars), res.Chars)
|
||||
}
|
||||
if res.Chars[0].Char != "猫" || !strings.Contains(res.Chars[0].Senses, "cat") {
|
||||
t.Errorf("first character = %+v, want 猫 ~ cat", res.Chars[0])
|
||||
}
|
||||
if res.Chars[0].Pinyin != "māo" {
|
||||
t.Errorf("猫 pinyin = %q, want māo", res.Chars[0].Pinyin)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHanziMisses(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
for name, word := range map[string]string{
|
||||
// A single character with no entry has no parts to fall back to.
|
||||
"lone unknown character": "龥",
|
||||
"empty": "",
|
||||
"whitespace": " ",
|
||||
// Not Chinese at all: the English tokenizer owns these, and answering
|
||||
// would mean guessing.
|
||||
"english": "hello",
|
||||
"mixed": "猫cat",
|
||||
// Longer than a word: a bad segmentation, not something to spell out
|
||||
// character by character.
|
||||
"a whole clause": "我今天早上去公园跑步了",
|
||||
} {
|
||||
res, err := l.Hanzi(word)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if len(res.Readings) != 0 || len(res.Chars) != 0 {
|
||||
t.Errorf("%s (%q) answered with %+v / %+v", name, word, res.Readings, res.Chars)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHanziEndpoint(t *testing.T) {
|
||||
h := NewHandler(nil, NewSet(nil))
|
||||
r := chi.NewRouter()
|
||||
r.Mount("/hanzi", h.HanziRoutes())
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/"+"跑步", nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d", w.Code)
|
||||
}
|
||||
var got HanziResult
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.Word != "跑步" || len(got.Readings) == 0 || got.Readings[0].Pinyin != "pǎobù" {
|
||||
t.Fatalf("response = %+v", got)
|
||||
}
|
||||
|
||||
// A miss is a 200 with empty lists, like the other two lookups — the tooltip
|
||||
// quietly doesn't open rather than showing an error over her writing.
|
||||
w = httptest.NewRecorder()
|
||||
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/zzz", nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("miss: status = %d, want 200", w.Code)
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,35 @@ type Result struct {
|
||||
// Latin roots with English — "ephemeral" is much easier to keep once you
|
||||
// have seen efémero next to it.
|
||||
Etymology string `json:"etymology"`
|
||||
|
||||
// Reverse is the same token read as a word of the writer's own language,
|
||||
// present only when it is one. Absent for every writer whose pair is not
|
||||
// Latin-script, and for the overwhelming majority of words in one that is.
|
||||
Reverse *Reverse `json:"reverse,omitempty"`
|
||||
}
|
||||
|
||||
// Reverse is a lookup in the other direction: the token treated as a word of the
|
||||
// writer's language, translated into English.
|
||||
//
|
||||
// It exists because a Latin-script pair has no script boundary to tell the two
|
||||
// halves apart. In English+Chinese, "which language is this word?" answers
|
||||
// itself. In English+Portuguese it does not: *sale*, *casa*, *comum*, *tarde*
|
||||
// and *ali* are all real words on both sides, and *chat* and *pain* are the
|
||||
// French versions of the same trap.
|
||||
//
|
||||
// Petal does not guess. It asks both directions and shows whatever comes back,
|
||||
// which needs no detector, cannot be wrong about someone's writing, and — for a
|
||||
// learner — is more interesting than a correct guess would have been.
|
||||
type Reverse struct {
|
||||
// Lang is the language this reading is in, so the card can label it.
|
||||
Lang string `json:"lang"`
|
||||
// Gloss is the English meaning of the native-language word.
|
||||
Gloss string `json:"gloss"`
|
||||
// Definitions are the word's senses as written in the writer's own
|
||||
// language — the monolingual half, for when the English gloss isn't enough.
|
||||
Definitions []Meaning `json:"definitions,omitempty"`
|
||||
// Phonetic is IPA for the native-language pronunciation; "" when absent.
|
||||
Phonetic string `json:"phonetic,omitempty"`
|
||||
}
|
||||
|
||||
// unknownDifficulty is the [Result.Difficulty] value meaning "no score",
|
||||
@@ -59,6 +88,10 @@ const unknownDifficulty = -1
|
||||
type GlossResult struct {
|
||||
Word string `json:"word"`
|
||||
Gloss string `json:"gloss"`
|
||||
// Reverse is the English meaning of the word read as one of the writer's
|
||||
// own language — the tooltip's half of the both-directions rule (see
|
||||
// [Reverse]). Empty unless the token is a word in her language too.
|
||||
Reverse string `json:"reverse,omitempty"`
|
||||
}
|
||||
|
||||
// maxSynonyms caps how many synonyms we hand the popover, even though the dataset
|
||||
|
||||
@@ -53,6 +53,16 @@ func NewSet(dream *DreamDict) *Set {
|
||||
// usable.
|
||||
func (s *Set) HasDreamDict() bool { return s.dream != nil }
|
||||
|
||||
// Contents describes what the open dict.db actually holds, for the startup log.
|
||||
// With no dictionary it says so rather than returning an empty string, because
|
||||
// a blank in a log line is indistinguishable from a bug in the log line.
|
||||
func (s *Set) Contents() string {
|
||||
if s.dream == nil {
|
||||
return "no dict.db — embedded datasets only"
|
||||
}
|
||||
return s.dream.Contents()
|
||||
}
|
||||
|
||||
// For returns the provider that should answer lookups for a writer whose pair
|
||||
// language is lang.
|
||||
//
|
||||
@@ -99,3 +109,13 @@ func (g glossless) Lookup(word string) (Result, error) {
|
||||
func (g glossless) Gloss(word string) (GlossResult, error) {
|
||||
return GlossResult{Word: word}, nil
|
||||
}
|
||||
|
||||
// Hanzi answers a Chinese-word lookup from the embedded CC-CEDICT map.
|
||||
//
|
||||
// It is on the Set rather than on [Provider] because it is not the same
|
||||
// question the other two ask. Lookup and Gloss vary by pair — which is why they
|
||||
// are behind an interface with two implementations — while this one is asked of
|
||||
// Chinese or not at all: the learner direction exists for exactly one pair (see
|
||||
// auth.learnerPairs), and DreamDict's own CC-CEDICT would be a second copy of
|
||||
// the same dictionary, chosen by a rule with one branch.
|
||||
func (s *Set) Hanzi(word string) (HanziResult, error) { return s.embedded.Hanzi(word) }
|
||||
|
||||
@@ -41,9 +41,9 @@ type checkpointResponse struct {
|
||||
// RunCheckpoint sends the grammar checkpoint and parses the JSON result. It
|
||||
// applies the latency-guard truncation and the checkpoint sampling parameters
|
||||
// from the spec.
|
||||
func RunCheckpoint(ctx context.Context, client LLMClient, contentText, tone string, _ Lang) ([]RawSuggestion, error) {
|
||||
func RunCheckpoint(ctx context.Context, client LLMClient, contentText, tone string, t Target) ([]RawSuggestion, error) {
|
||||
raw, err := client.Complete(ctx, CompletionRequest{
|
||||
Messages: CheckpointMessages(TruncateDoc(contentText), tone),
|
||||
Messages: CheckpointMessages(TruncateDoc(contentText), tone, t),
|
||||
MaxTokens: checkpointMaxTokens,
|
||||
Temperature: 0.3,
|
||||
RepetitionPenalty: 1.15,
|
||||
|
||||
@@ -20,9 +20,9 @@ const CollocationInterval = 25 * time.Second
|
||||
// The tone argument is accepted for a uniform pass signature and passed through
|
||||
// to the prompt so a hint can prefer a register-appropriate pairing. `lang` is
|
||||
// the writer's pair language — the one each hint's short gloss is written in.
|
||||
func RunCollocation(ctx context.Context, client LLMClient, contentText, tone string, lang Lang) ([]RawSuggestion, error) {
|
||||
func RunCollocation(ctx context.Context, client LLMClient, contentText, tone string, t Target) ([]RawSuggestion, error) {
|
||||
raw, err := client.Complete(ctx, CompletionRequest{
|
||||
Messages: CollocationMessages(contentText, tone, lang),
|
||||
Messages: CollocationMessages(contentText, tone, t),
|
||||
MaxTokens: 2048,
|
||||
Temperature: 0.3,
|
||||
RepetitionPenalty: 1.15,
|
||||
|
||||
@@ -31,7 +31,7 @@ func TestLangForFallsBackToDefault(t *testing.T) {
|
||||
func TestPromptsNameTheWritersLanguage(t *testing.T) {
|
||||
pt := LangFor("pt-PT")
|
||||
|
||||
collocation := CollocationMessages("The rain was strong.", "casual", pt)[0].Content
|
||||
collocation := CollocationMessages("The rain was strong.", "casual", EnglishTarget(pt))[0].Content
|
||||
if !strings.Contains(collocation, "European Portuguese") {
|
||||
t.Fatalf("collocation prompt doesn't ask for a pt-PT gloss:\n%s", collocation)
|
||||
}
|
||||
@@ -44,7 +44,7 @@ func TestPromptsNameTheWritersLanguage(t *testing.T) {
|
||||
t.Fatalf("collocation prompt lost its tone guidance:\n%s", collocation)
|
||||
}
|
||||
|
||||
translate := TranslateMessages("Try a shorter sentence here.", pt)[0].Content
|
||||
translate := TranslateMessages("Try a shorter sentence here.", English, pt)[0].Content
|
||||
if !strings.Contains(translate, "European Portuguese") || strings.Contains(translate, "Chinese") {
|
||||
t.Fatalf("translate prompt targets the wrong language:\n%s", translate)
|
||||
}
|
||||
@@ -73,13 +73,50 @@ func TestPromptsNameTheWritersLanguage(t *testing.T) {
|
||||
func TestDefaultPairStillReadsAsBefore(t *testing.T) {
|
||||
zh := LangFor("zh")
|
||||
|
||||
if got := CollocationMessages("x", "", zh)[0].Content; !strings.Contains(got, "Simplified Chinese (Mandarin) gloss in parentheses") {
|
||||
if got := CollocationMessages("x", "", EnglishTarget(zh))[0].Content; !strings.Contains(got, "Simplified Chinese (Mandarin) gloss in parentheses") {
|
||||
t.Fatalf("zh collocation gloss changed:\n%s", got)
|
||||
}
|
||||
if got := TranslateMessages("x", zh)[0].Content; !strings.Contains(got, "natural, friendly Simplified Chinese (Mandarin)") {
|
||||
if got := TranslateMessages("x", English, zh)[0].Content; !strings.Contains(got, "natural, friendly Simplified Chinese (Mandarin)") {
|
||||
t.Fatalf("zh translate target changed:\n%s", got)
|
||||
}
|
||||
if got := AskPetalSystemPrompt("a", "b", "c", "d", "e", zh); !strings.Contains(got, "为什么") {
|
||||
t.Fatalf("zh ask-petal lost its Mandarin \"why\":\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// UX item 6: the Ask Petal answer is bilingual, pair language first, halves
|
||||
// separated by one blank line. That separator is not a stylistic preference —
|
||||
// AskPetal.tsx splits on it to render the two halves the way the companion
|
||||
// renders its two lines — so the instruction has to survive prompt edits.
|
||||
//
|
||||
// The direction the writer is learning in is deliberately not encoded: the pair
|
||||
// is (English + X), and an English speaker learning French needs the same two
|
||||
// halves a Mandarin speaker learning English does. The prompt asks for both and
|
||||
// lets the reader choose, so there is nothing here that names one half the
|
||||
// answer and the other a courtesy.
|
||||
func TestAskPetalAnswersInBothLanguages(t *testing.T) {
|
||||
for _, code := range []string{"zh", "pt-PT", "fr", "es"} {
|
||||
lang := LangFor(code)
|
||||
ask := AskPetalSystemPrompt("a", "b", "grammar", "d", "e", lang)
|
||||
|
||||
if !strings.Contains(ask, "BOTH languages") {
|
||||
t.Fatalf("%s: ask-petal no longer asks for both languages:\n%s", code, ask)
|
||||
}
|
||||
if !strings.Contains(ask, "single blank line") {
|
||||
t.Fatalf("%s: ask-petal lost the blank-line separator the client splits on:\n%s", code, ask)
|
||||
}
|
||||
// Order matters to the rendering: the pair language is the prominent
|
||||
// half, English the muted one beneath it.
|
||||
if !strings.Contains(ask, "first the whole answer in "+lang.Name) {
|
||||
t.Fatalf("%s: ask-petal doesn't put %s first:\n%s", code, lang.Name, ask)
|
||||
}
|
||||
// The instruction it replaced. Left in place it directly contradicts the
|
||||
// new one, and a model given both will pick one at random.
|
||||
if strings.Contains(ask, "Never mix languages") {
|
||||
t.Fatalf("%s: ask-petal still forbids the bilingual reply it now asks for:\n%s", code, ask)
|
||||
}
|
||||
if strings.Contains(ask, "%!") {
|
||||
t.Fatalf("%s: ask-petal prompt has a formatting error:\n%s", code, ask)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+143
-25
@@ -45,11 +45,55 @@ func toneGuidance(tone string) string {
|
||||
"be improved, prefer suggestions that fit that tone, and gently flag wording that clashes with it."
|
||||
}
|
||||
|
||||
// pairCheckpointSystemPrompt is the grammar checkpoint for a document written in
|
||||
// the writer's own language rather than in English.
|
||||
//
|
||||
// It is a separate constant rather than a language clause appended to
|
||||
// checkpointSystemPrompt, because that prompt opens by naming the reader as an
|
||||
// ESL learner and asks for "common ESL patterns" — appending "and explain in
|
||||
// Portuguese" would hand the model two contradictory framings. Only the framing
|
||||
// differs; the JSON contract and the tone clause below it are the same
|
||||
// instructions in the same order, so the two prompts stay comparable.
|
||||
//
|
||||
// The "never translate" line is the one the model most wants to disobey: asked
|
||||
// to improve Portuguese while being an English writing assistant by training, it
|
||||
// will happily hand back an English rendering, which is a translation card
|
||||
// (Phase 25's `isTranslation`) and not a correction.
|
||||
const pairCheckpointSystemPrompt = `You are a warm, encouraging writing assistant. The person you are helping is ` +
|
||||
`writing in %[1]s, and the text below is %[1]s. ` +
|
||||
`Analyze it and identify up to 5 issues: grammar errors, unnatural phrasing, ` +
|
||||
`incorrect idiom usage, or unclear sentences.
|
||||
|
||||
Both "original" and "replacement" must be written in %[1]s. You are improving their %[1]s writing — ` +
|
||||
`never translate it into English, and never suggest they write in English instead.
|
||||
Write every "explanation" in %[2]s.
|
||||
|
||||
Be specific, friendly, and explain WHY each suggestion improves the writing.%[3]s
|
||||
|
||||
Respond ONLY with valid JSON. No preamble, no markdown fences. Format:
|
||||
{
|
||||
"suggestions": [
|
||||
{
|
||||
"original": "exact text from the document that needs fixing",
|
||||
"replacement": "corrected version",
|
||||
"explanation": "friendly one-sentence explanation",
|
||||
"type": "grammar|phrasing|idiom|clarity"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
If the writing looks good, return: {"suggestions": []}`
|
||||
|
||||
// CheckpointMessages builds the message array for a grammar checkpoint over the
|
||||
// given (already-truncated) document text, steered toward the document's tone.
|
||||
func CheckpointMessages(contentText, tone string) []Message {
|
||||
// given (already-truncated) document text, steered toward the document's tone
|
||||
// and aimed at the language the document is actually written in.
|
||||
func CheckpointMessages(contentText, tone string, t Target) []Message {
|
||||
system := fmt.Sprintf(checkpointSystemPrompt, toneGuidance(tone))
|
||||
if t.Flipped() {
|
||||
system = fmt.Sprintf(pairCheckpointSystemPrompt, t.Correct.Name, t.Explain.Name, toneGuidance(tone))
|
||||
}
|
||||
return []Message{
|
||||
{Role: "system", Content: fmt.Sprintf(checkpointSystemPrompt, toneGuidance(tone))},
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: contentText},
|
||||
}
|
||||
}
|
||||
@@ -82,12 +126,48 @@ Respond ONLY with valid JSON. No preamble, no markdown fences. Format:
|
||||
|
||||
If the voice is consistent throughout, return: {"suggestions": []}`
|
||||
|
||||
// pairVoiceSystemPrompt is the voice pass for a document in the writer's own
|
||||
// language. Voice consistency is the one pass that transfers across languages
|
||||
// unchanged — a paragraph that reads as pasted from elsewhere reads that way in
|
||||
// any language — so only the framing and the explanation language move.
|
||||
const pairVoiceSystemPrompt = `You are a warm, encouraging writing assistant. The person you are helping is writing ` +
|
||||
`in %[1]s. You are reviewing a COMPLETE %[1]s document for VOICE CONSISTENCY only — not grammar.
|
||||
|
||||
Read the whole document to learn the writer's natural voice, then identify any passages (2 or more sentences) ` +
|
||||
`that feel tonally inconsistent with the surrounding writing — unusually formal, unusually polished, or phrased ` +
|
||||
`in a way that differs from the writer's established voice elsewhere in the document. These often signal text ` +
|
||||
`that was paraphrased too closely from another source. Do not flag the first paragraph (there is no baseline yet). ` +
|
||||
`Do not flag grammar or spelling mistakes — only voice.
|
||||
|
||||
Quote each passage exactly as it appears, in %[1]s. Write every "explanation" in %[2]s.
|
||||
|
||||
Respond ONLY with valid JSON. No preamble, no markdown fences. Format:
|
||||
{
|
||||
"suggestions": [
|
||||
{
|
||||
"original": "exact passage from the document that feels inconsistent",
|
||||
"replacement": null,
|
||||
"explanation": "friendly one-sentence note about why this passage sounds unlike the rest",
|
||||
"type": "voice"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
If the voice is consistent throughout, return: {"suggestions": []}`
|
||||
|
||||
// VoiceMessages builds the message array for a voice-consistency pass. Unlike
|
||||
// the checkpoint, the caller passes the WHOLE document (no truncation) — voice
|
||||
// consistency is judged against the established voice everywhere else.
|
||||
func VoiceMessages(contentText string) []Message {
|
||||
//
|
||||
// The pass had no language argument at all before Phase 28, which was the same
|
||||
// English assumption the checkpoint made, just unstated.
|
||||
func VoiceMessages(contentText string, t Target) []Message {
|
||||
system := voiceSystemPrompt
|
||||
if t.Flipped() {
|
||||
system = fmt.Sprintf(pairVoiceSystemPrompt, t.Correct.Name, t.Explain.Name)
|
||||
}
|
||||
return []Message{
|
||||
{Role: "system", Content: voiceSystemPrompt},
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: contentText},
|
||||
}
|
||||
}
|
||||
@@ -133,10 +213,17 @@ If every pairing already sounds natural, return: {"suggestions": []}`
|
||||
// CollocationMessages builds the message array for a collocation pass over the
|
||||
// WHOLE document (no truncation), gently steered toward the document's tone so a
|
||||
// hint can prefer a register-appropriate pairing. The parenthetical gloss is
|
||||
// written in the writer's own language.
|
||||
func CollocationMessages(contentText, tone string, lang Lang) []Message {
|
||||
// written in the writer's own language — `Pair`, not `Explain`: the gloss is
|
||||
// addressed to her rather than to the document.
|
||||
//
|
||||
// The coach itself remains English-only. Collocation lists are the one thing
|
||||
// here that is genuinely per-language knowledge rather than framing, and
|
||||
// "natives usually say" for Portuguese is a claim this prompt has no grounds to
|
||||
// make yet; a flipped document simply gets the pass it always got. (Phase 28
|
||||
// moved the checkpoint and the voice pass; this one waits for evidence.)
|
||||
func CollocationMessages(contentText, tone string, t Target) []Message {
|
||||
return []Message{
|
||||
{Role: "system", Content: fmt.Sprintf(collocationSystemPrompt, toneGuidance(tone), lang.Name)},
|
||||
{Role: "system", Content: fmt.Sprintf(collocationSystemPrompt, toneGuidance(tone), t.Pair.Name)},
|
||||
{Role: "user", Content: contentText},
|
||||
}
|
||||
}
|
||||
@@ -144,6 +231,25 @@ func CollocationMessages(contentText, tone string, lang Lang) []Message {
|
||||
// askPetalSystemTemplate is the Ask Petal tutor prompt. The suggestion context
|
||||
// is interpolated in; the user's own messages are appended after this system
|
||||
// turn by the caller.
|
||||
//
|
||||
// The reply is bilingual, the pair language first. Until UX item 6 it mirrored
|
||||
// the language of the question instead — self-consistent, but it meant asking in
|
||||
// one language cost you the other, and the writer doesn't always know which one
|
||||
// the answer will be clearer in. Which half is the safety net and which is the
|
||||
// lesson depends on who is writing: the pair is (English + X) either way, and an
|
||||
// English speaker learning French wants the French half for the same reason a
|
||||
// Mandarin speaker learning English wants the English one. Petal cannot tell
|
||||
// them apart from a chat message, and doesn't need to — every other explanation
|
||||
// surface already gives both (the card's English body, the seeded bubble in the
|
||||
// pair language). The answer that goes deepest into the "why" was the one place
|
||||
// that didn't.
|
||||
//
|
||||
// The blank line between the halves is a contract with the client: AskPetal.tsx
|
||||
// splits on the first one to render her language prominently and the English
|
||||
// beneath it, mirroring the companion's bubble. A model that ignores the
|
||||
// instruction and writes one language degrades to a single plain block — the
|
||||
// answer is still readable, which is why the split is a rendering nicety and
|
||||
// never a parse the reply depends on.
|
||||
const askPetalSystemTemplate = `You are Petal, a warm and patient English writing tutor helping someone who is learning English ` +
|
||||
`as a second language. You are currently discussing a specific writing suggestion.
|
||||
|
||||
@@ -154,15 +260,21 @@ Suggestion context:
|
||||
- Initial explanation: "%[4]s"
|
||||
- Surrounding paragraph: "%[5]s"
|
||||
|
||||
The user wants to understand this suggestion better. Detect the language of the user's message ` +
|
||||
`and respond in that same language. If they write in %[6]s, respond entirely in ` +
|
||||
`%[6]s. If they write in English, respond in English. Never mix languages in a single response.
|
||||
The user wants to understand this suggestion better. Answer in BOTH languages, every time, ` +
|
||||
`whichever language they asked their question in: first the whole answer in %[6]s, then the ` +
|
||||
`same answer again in English. Separate the two with a single blank line. Do not label them, ` +
|
||||
`do not use a blank line anywhere else, and do not mix the two languages within one half — ` +
|
||||
`each half is complete on its own.
|
||||
|
||||
One of those two languages is the one they are surest in and the other is the one they are ` +
|
||||
`working in — you do not know which way round, so give both and let them choose. Both halves ` +
|
||||
`say the same thing: do not put a point in one that is missing from the other.
|
||||
|
||||
Explain clearly and kindly. Use simple language appropriate to the user's message. Give examples ` +
|
||||
`when helpful. If they ask "why" (or "%[7]s"), explain the grammar rule or idiom behind it. ` +
|
||||
`If they suggest an alternative phrasing, evaluate it honestly.
|
||||
|
||||
Keep responses concise (2-4 sentences). This is a chat, not an essay. Be encouraging — ` +
|
||||
Keep each half concise (2-3 sentences). This is a chat, not an essay. Be encouraging — ` +
|
||||
`learning a language is hard and they're doing great.`
|
||||
|
||||
// AskPetalSystemPrompt fills the tutor prompt with one suggestion's context and
|
||||
@@ -217,23 +329,29 @@ func RewriteMessages(text, style string) []Message {
|
||||
}
|
||||
|
||||
// translateSystemPrompt drives the explanation translator: it renders a
|
||||
// suggestion's English explanation into the writer's own language so an ESL
|
||||
// reader sees the "why" in her first language. Strict about returning ONLY the
|
||||
// translation (no quotes, no romanisation, no English echo) so it can drop
|
||||
// straight into the chat bubble. Kept warm and plain — these are short, friendly
|
||||
// one-liners.
|
||||
const translateSystemPrompt = `You are Petal, a warm writing assistant. Translate the English text the user ` +
|
||||
`sends into natural, friendly %[1]s. It is a short explanation of a writing ` +
|
||||
`suggestion, written for a native %[1]s speaker learning English.
|
||||
// suggestion's explanation into the half of the pair the explanation is not
|
||||
// already in, so the "why" is readable from both sides. Strict about returning
|
||||
// ONLY the translation (no quotes, no romanisation, no echo of the source) so it
|
||||
// can drop straight into the chat bubble. Kept warm and plain — these are short,
|
||||
// friendly one-liners.
|
||||
//
|
||||
// Both languages are parameters because neither end is a constant. Until Phase
|
||||
// 28 the source was always English and the destination always hers; a document
|
||||
// written in her own language is explained in her own language, and then the tap
|
||||
// runs the other way, into the English she is practising.
|
||||
const translateSystemPrompt = `You are Petal, a warm writing assistant. Translate the %[1]s text the user ` +
|
||||
`sends into natural, friendly %[2]s. It is a short explanation of a writing ` +
|
||||
`suggestion, written for someone who is learning one of %[1]s and %[2]s and reads the other most easily.
|
||||
|
||||
Respond with ONLY the %[1]s translation. No quotation marks, no romanisation, no English, no preamble — ` +
|
||||
Respond with ONLY the %[2]s translation. No quotation marks, no romanisation, no %[1]s, no preamble — ` +
|
||||
`just the translated sentence.`
|
||||
|
||||
// TranslateMessages builds the message array for translating one short English
|
||||
// explanation into the writer's own language.
|
||||
func TranslateMessages(text string, lang Lang) []Message {
|
||||
// TranslateMessages builds the message array for rendering one short
|
||||
// explanation out of the language it arrived in and into the other half of the
|
||||
// writer's pair.
|
||||
func TranslateMessages(text string, from, to Lang) []Message {
|
||||
return []Message{
|
||||
{Role: "system", Content: fmt.Sprintf(translateSystemPrompt, lang.Name)},
|
||||
{Role: "system", Content: fmt.Sprintf(translateSystemPrompt, from.Name, to.Name)},
|
||||
{Role: "user", Content: text},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package llm
|
||||
|
||||
// Which language a pass corrects, and which language it explains in.
|
||||
//
|
||||
// Until Phase 28 there was no question to answer: every prompt was written
|
||||
// around English prose explained in English, and the pair language reached her
|
||||
// only when she asked for it (Ask Petal, the explanation translator). That is
|
||||
// the right default for a writer practising English and the wrong one for a
|
||||
// document she wrote in her own language, where Petal would read Portuguese,
|
||||
// say nothing about it, and file a mechanics note about the one English
|
||||
// sentence at the end.
|
||||
//
|
||||
// The two fields are two different decisions reading two different pieces of
|
||||
// state, and collapsing them would be the bug:
|
||||
//
|
||||
// - Correct follows the DOCUMENT. Portuguese prose gets Portuguese
|
||||
// corrections; that is the whole point.
|
||||
// - Explain follows the WRITER — the half of her pair she is *not* learning
|
||||
// (users.direction), because an explanation is teaching, and teaching lands
|
||||
// in the language she reads most easily.
|
||||
//
|
||||
// Today those two coincide for every account that exists: `learnerPairs` is
|
||||
// {"zh"}, so fr, es and pt-PT writers are all `learning_en` and their
|
||||
// non-learned half *is* the pair language. That equality is a fact about
|
||||
// today's roster, not about the design — the same shape of assumption that had
|
||||
// to be unpicked from `pair_lang` in migration 0016. Keep them apart.
|
||||
type Target struct {
|
||||
// Correct is the language the writing is in, and so the language both
|
||||
// `original` and `replacement` must be written in.
|
||||
Correct Lang
|
||||
// Explain is the language each explanation is written in.
|
||||
Explain Lang
|
||||
// Pair is the writer's pair language regardless of what this document is
|
||||
// written in. The collocation coach's parenthetical gloss is addressed to
|
||||
// her rather than to the document, so it reads this and not Correct.
|
||||
Pair Lang
|
||||
}
|
||||
|
||||
// English as the prompts name it. Not in `langs`: that map answers "which
|
||||
// language is the writer's half of the pair", and English is the constant on
|
||||
// the other side of every pair Petal supports.
|
||||
var English = Lang{Code: "en", Name: "English", Why: "why"}
|
||||
|
||||
// EnglishTarget is the pre-Phase-28 behaviour, made explicit: an English
|
||||
// document, corrected and explained in English, for a writer whose pair
|
||||
// language is `pair`. Every existing user is on this path and the prompt it
|
||||
// produces is byte-identical to the one that shipped before this phase.
|
||||
func EnglishTarget(pair Lang) Target {
|
||||
return Target{Correct: English, Explain: English, Pair: pair}
|
||||
}
|
||||
|
||||
// Flipped reports whether this document is in the pair language rather than in
|
||||
// English — i.e. whether the pass is reading her own language.
|
||||
func (t Target) Flipped() bool { return t.Correct.Code != English.Code }
|
||||
@@ -0,0 +1,115 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The prompt every account is on today, written out in full.
|
||||
//
|
||||
// Phase 28 gave the checkpoint a second framing for documents in the writer's
|
||||
// own language, and the risk of that change is not that the new prompt is wrong
|
||||
// — it is that the old one moved by a word while nobody was looking. Every user
|
||||
// who exists is a Mandarin native writing English, so this string is the one
|
||||
// Petal actually sends, all day. It is duplicated here on purpose: a golden
|
||||
// copied from the constant it guards guards nothing.
|
||||
const goldenEnglishCheckpointPrompt = `You are a warm, encouraging writing assistant helping someone who speaks English as a second language. Analyze the text below and identify up to 5 issues: grammar errors, unnatural phrasing, incorrect idiom usage, or unclear sentences that are common ESL patterns.
|
||||
|
||||
Be specific, friendly, and explain WHY each suggestion improves the writing.
|
||||
|
||||
Respond ONLY with valid JSON. No preamble, no markdown fences. Format:
|
||||
{
|
||||
"suggestions": [
|
||||
{
|
||||
"original": "exact text from the document that needs fixing",
|
||||
"replacement": "corrected version",
|
||||
"explanation": "friendly one-sentence explanation",
|
||||
"type": "grammar|phrasing|idiom|clarity"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
If the writing looks good, return: {"suggestions": []}`
|
||||
|
||||
func TestEnglishDocumentPromptIsUnchanged(t *testing.T) {
|
||||
msgs := CheckpointMessages("I has two apple.", "", EnglishTarget(LangFor("zh")))
|
||||
if got := msgs[0].Content; got != goldenEnglishCheckpointPrompt {
|
||||
t.Fatalf("the English checkpoint prompt moved:\n--- got ---\n%s\n--- want ---\n%s", got, goldenEnglishCheckpointPrompt)
|
||||
}
|
||||
if msgs[1].Content != "I has two apple." {
|
||||
t.Fatalf("document text mangled: %q", msgs[1].Content)
|
||||
}
|
||||
|
||||
// The tone clause still lands, in the same place it always did.
|
||||
toned := CheckpointMessages("x", "academic", EnglishTarget(LangFor("zh")))[0].Content
|
||||
if !strings.Contains(toned, "formal, academic, and objective") {
|
||||
t.Fatalf("English checkpoint lost its tone guidance:\n%s", toned)
|
||||
}
|
||||
}
|
||||
|
||||
// A document in her own language gets a prompt that names that language, keeps
|
||||
// the corrections inside it, and drops the framing that only makes sense when
|
||||
// the thing being written is English.
|
||||
func TestFlippedCheckpointPrompt(t *testing.T) {
|
||||
pt := LangFor("pt-PT")
|
||||
system := CheckpointMessages("Hoje foi um dia bom.", "casual", Target{Correct: pt, Explain: pt, Pair: pt})[0].Content
|
||||
|
||||
if !strings.Contains(system, "European Portuguese") {
|
||||
t.Fatalf("flipped checkpoint doesn't name the language:\n%s", system)
|
||||
}
|
||||
if strings.Contains(system, "second language") || strings.Contains(system, "ESL") {
|
||||
t.Fatalf("flipped checkpoint kept the ESL framing:\n%s", system)
|
||||
}
|
||||
if !strings.Contains(system, "never translate it into English") {
|
||||
t.Fatalf("flipped checkpoint doesn't forbid translating:\n%s", system)
|
||||
}
|
||||
// The shared contract below the framing has to survive the split.
|
||||
for _, want := range []string{`"suggestions"`, `"replacement"`, "grammar|phrasing|idiom|clarity", "relaxed, friendly, and conversational"} {
|
||||
if !strings.Contains(system, want) {
|
||||
t.Fatalf("flipped checkpoint dropped %q:\n%s", want, system)
|
||||
}
|
||||
}
|
||||
if strings.Contains(system, "%!") {
|
||||
t.Fatalf("flipped checkpoint has a formatting error:\n%s", system)
|
||||
}
|
||||
}
|
||||
|
||||
// The two decisions are separate arguments and must reach the prompt separately:
|
||||
// corrections in the document's language, the explanation in the language she
|
||||
// reads most easily. Only the zh pair can be travelled both ways today, so it is
|
||||
// the only one that can prove they haven't been quietly collapsed into one.
|
||||
func TestFlippedPromptsExplainInTheirOwnLanguage(t *testing.T) {
|
||||
zh := LangFor("zh")
|
||||
|
||||
// Native Mandarin, practising English, writing Chinese: both halves Chinese.
|
||||
both := CheckpointMessages("今天天气很好。", "", Target{Correct: zh, Explain: zh, Pair: zh})[0].Content
|
||||
if strings.Count(both, "Simplified Chinese (Mandarin)") < 2 {
|
||||
t.Fatalf("expected corrections and explanations both in Chinese:\n%s", both)
|
||||
}
|
||||
if strings.Contains(both, "explanation"+`" in English`) {
|
||||
t.Fatalf("explanation language leaked to English:\n%s", both)
|
||||
}
|
||||
|
||||
// Native English, learning Chinese, writing Chinese: Chinese corrections,
|
||||
// English explanations.
|
||||
split := CheckpointMessages("今天天气很好。", "", Target{Correct: zh, Explain: English, Pair: zh})[0].Content
|
||||
if !strings.Contains(split, `Write every "explanation" in English.`) {
|
||||
t.Fatalf("learner direction didn't get English explanations:\n%s", split)
|
||||
}
|
||||
if !strings.Contains(split, "writing in Simplified Chinese (Mandarin)") {
|
||||
t.Fatalf("learner direction lost its Chinese corrections:\n%s", split)
|
||||
}
|
||||
|
||||
// Same for the voice pass, which had no language at all before this phase.
|
||||
voice := VoiceMessages("今天天气很好。", Target{Correct: zh, Explain: English, Pair: zh})[0].Content
|
||||
if !strings.Contains(voice, `Write every "explanation" in English.`) || !strings.Contains(voice, "Simplified Chinese") {
|
||||
t.Fatalf("flipped voice prompt got its languages wrong:\n%s", voice)
|
||||
}
|
||||
if strings.Contains(voice, "second language") {
|
||||
t.Fatalf("flipped voice prompt kept the ESL framing:\n%s", voice)
|
||||
}
|
||||
// An English document still gets exactly the voice prompt it always got.
|
||||
if got := VoiceMessages("x", EnglishTarget(zh))[0].Content; got != voiceSystemPrompt {
|
||||
t.Fatalf("English voice prompt moved:\n%s", got)
|
||||
}
|
||||
}
|
||||
@@ -4,13 +4,14 @@ import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// RunTranslate renders a short English explanation into the writer's own
|
||||
// language. It is a one-shot Complete (the result seeds the Ask Petal bubble),
|
||||
// kept at a low temperature so the translation is faithful rather than creative. Output is
|
||||
// trimmed of any stray surrounding quotes the model may add.
|
||||
func RunTranslate(ctx context.Context, client LLMClient, text string, lang Lang) (string, error) {
|
||||
// RunTranslate renders a short explanation out of the language it was written
|
||||
// in and into the other half of the writer's pair. It is a one-shot Complete
|
||||
// (the result seeds the Ask Petal bubble), kept at a low temperature so the
|
||||
// translation is faithful rather than creative. Output is trimmed of any stray
|
||||
// surrounding quotes the model may add.
|
||||
func RunTranslate(ctx context.Context, client LLMClient, text string, from, to Lang) (string, error) {
|
||||
out, err := client.Complete(ctx, CompletionRequest{
|
||||
Messages: TranslateMessages(text, lang),
|
||||
Messages: TranslateMessages(text, from, to),
|
||||
MaxTokens: 512,
|
||||
Temperature: 0.2,
|
||||
TopP: 0.9,
|
||||
|
||||
@@ -19,9 +19,9 @@ const VoiceInterval = 20 * time.Second
|
||||
// The tone argument is accepted for a uniform pass signature but ignored: voice
|
||||
// consistency is judged against the document's own established voice, not an
|
||||
// externally-chosen register.
|
||||
func RunVoice(ctx context.Context, client LLMClient, contentText, _ string, _ Lang) ([]RawSuggestion, error) {
|
||||
func RunVoice(ctx context.Context, client LLMClient, contentText, _ string, t Target) ([]RawSuggestion, error) {
|
||||
raw, err := client.Complete(ctx, CompletionRequest{
|
||||
Messages: VoiceMessages(contentText),
|
||||
Messages: VoiceMessages(contentText, t),
|
||||
MaxTokens: 2048,
|
||||
Temperature: 0.3,
|
||||
RepetitionPenalty: 1.15,
|
||||
|
||||
@@ -8,8 +8,18 @@
|
||||
// so it belongs to her account rather than to a browser profile.
|
||||
//
|
||||
// Everything here is scoped by `lang` as well as by user. That is the language
|
||||
// of the *dictionary* that flagged the word, not the writer's own language: an
|
||||
// en-US personal word must not silence a pt-PT flag once the second pair ships.
|
||||
// of the *dictionary* the word was accepted against, not the writer's own.
|
||||
//
|
||||
// Phase 18 justified that key by saying an en-US personal word must not silence
|
||||
// a pt-PT flag once the second pair shipped. Phase 21 shipped it and the
|
||||
// justification did not survive: under the both-dictionaries rule
|
||||
// (SUGGESTIONS.md §3a) a word is only ever flagged when *every* loaded
|
||||
// dictionary rejected it, so there is no such thing as a pt-PT flag an English
|
||||
// exception could silence. What the key is actually good for is narrower and
|
||||
// still worth having — the rows say which dictionary each acceptance was made
|
||||
// against, so a pair that later loses or gains a dictionary keeps a truthful
|
||||
// record instead of one merged list of unknown provenance. The browser writes a
|
||||
// row per loaded dictionary when she accepts a word; see useSpellChecker.
|
||||
package spell
|
||||
|
||||
import (
|
||||
@@ -24,8 +34,9 @@ import (
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
)
|
||||
|
||||
// DefaultLang is the dictionary assumed when a caller doesn't name one. Only
|
||||
// en-US ships today; pt-PT arrives with the first Latin pair.
|
||||
// DefaultLang is the dictionary assumed when a caller doesn't name one. English
|
||||
// is in every pair, so it is the safe assumption; pt-PT is named explicitly by
|
||||
// the pt-PT pair's second dictionary.
|
||||
const DefaultLang = "en"
|
||||
|
||||
// MaxWordLen bounds a single entry. A personal dictionary holds words, and a
|
||||
|
||||
@@ -75,7 +75,7 @@ func (h *Handler) chat(w http.ResponseWriter, r *http.Request) {
|
||||
if err != nil {
|
||||
// The stream never opened (e.g. LLM unreachable) — a normal JSON error is
|
||||
// still appropriate since we haven't written SSE headers yet.
|
||||
httputil.ErrorJSON(w, http.StatusBadGateway, "chat failed: "+err.Error())
|
||||
httputil.UpstreamError(w, "chat", err)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Chunking splits a document into sentence-sized units so a re-check can ask the
|
||||
// model only about the sentences that actually changed. Accepting one edit used
|
||||
// to re-run the whole document: every card vanished, came back with a new id and
|
||||
// a freshly-worded explanation, and spans re-merged into different shapes. The
|
||||
// sentences she didn't touch have nothing new to say about themselves, so their
|
||||
// suggestions are simply kept (see reconcilePending).
|
||||
//
|
||||
// A chunk's identity is its hash, not its position — she inserts a paragraph at
|
||||
// the top and every sentence below keeps its suggestions.
|
||||
|
||||
// chunk is one sentence of the document, with the hash that identifies it.
|
||||
type chunk struct {
|
||||
text string
|
||||
hash string
|
||||
}
|
||||
|
||||
// asciiTerminators end a sentence only when whitespace (or the end of the text)
|
||||
// follows, so "3.5" and "Ms." don't split mid-word — a wrong split costs only a
|
||||
// slightly smaller chunk, but a split inside a number would churn its hash on
|
||||
// every keystroke around it.
|
||||
const asciiTerminators = ".!?"
|
||||
|
||||
// cjkTerminators end a sentence outright: Chinese runs sentences together with
|
||||
// no space after 。, and she writes in both languages in one document.
|
||||
const cjkTerminators = "。!?"
|
||||
|
||||
// closers are swallowed into the sentence they close, so the quote mark travels
|
||||
// with the sentence rather than opening the next one.
|
||||
const closers = `)]}"'’”」』`
|
||||
|
||||
// splitChunks divides text into sentences, dropping whitespace-only runs.
|
||||
// Newlines always break a chunk, so a list or a line of dialogue is its own unit.
|
||||
//
|
||||
// `salt` distinguishes two *readings* of the same sentence. The grammar
|
||||
// checkpoint's advice depends on the document's tone — the same line gets
|
||||
// different notes as an academic essay than as a journal entry — so switching
|
||||
// tone must re-open every sentence rather than serve back advice written for the
|
||||
// old register.
|
||||
func splitChunks(text, salt string) []chunk {
|
||||
var out []chunk
|
||||
runes := []rune(text)
|
||||
start := 0
|
||||
add := func(end int) {
|
||||
if s := string(runes[start:end]); strings.TrimSpace(s) != "" {
|
||||
out = append(out, chunk{text: s, hash: hashChunk(s, salt)})
|
||||
}
|
||||
start = end
|
||||
}
|
||||
|
||||
for i := 0; i < len(runes); i++ {
|
||||
r := runes[i]
|
||||
if r == '\n' {
|
||||
add(i + 1)
|
||||
continue
|
||||
}
|
||||
cjk := strings.ContainsRune(cjkTerminators, r)
|
||||
if !cjk && !strings.ContainsRune(asciiTerminators, r) {
|
||||
continue
|
||||
}
|
||||
// Swallow a run of terminators ("?!", "…") and any closing punctuation.
|
||||
j := i + 1
|
||||
for j < len(runes) && (strings.ContainsRune(asciiTerminators+cjkTerminators+closers, runes[j])) {
|
||||
j++
|
||||
}
|
||||
if cjk || j >= len(runes) || unicode.IsSpace(runes[j]) {
|
||||
add(j)
|
||||
i = j - 1
|
||||
}
|
||||
}
|
||||
if start < len(runes) {
|
||||
add(len(runes))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hashChunk identifies a sentence by its content under the same normalization
|
||||
// the suppression logic uses: quote style and whitespace runs churn constantly
|
||||
// (the editor rewrites quotes as she types, a paragraph reflows) and none of
|
||||
// that changes what the sentence says, so none of it should cost a re-check.
|
||||
func hashChunk(s, salt string) string {
|
||||
sum := sha256.Sum256([]byte(salt + "\x00" + normalizeForDedup(s)))
|
||||
return hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
|
||||
// hashSet indexes chunks by hash — "is this sentence in the document?"
|
||||
func hashSet(chunks []chunk) map[string]bool {
|
||||
out := make(map[string]bool, len(chunks))
|
||||
for _, c := range chunks {
|
||||
out[c.hash] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changedChunks returns the chunks whose hash wasn't in the last checked set,
|
||||
// in document order and deduplicated — a sentence repeated verbatim is one
|
||||
// question, not two.
|
||||
func changedChunks(chunks []chunk, checked map[string]bool) []chunk {
|
||||
seen := make(map[string]bool, len(chunks))
|
||||
var out []chunk
|
||||
for _, c := range chunks {
|
||||
if checked[c.hash] || seen[c.hash] {
|
||||
continue
|
||||
}
|
||||
seen[c.hash] = true
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// joinChunks renders a chunk set as the text to hand the model: one sentence per
|
||||
// line, so two sentences pulled from opposite ends of the document don't read as
|
||||
// one run-on.
|
||||
func joinChunks(chunks []chunk) string {
|
||||
parts := make([]string, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
parts = append(parts, strings.TrimSpace(c.text))
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
// chunkFor names the sentence a suggestion belongs to: the first chunk whose
|
||||
// text contains the flagged span. Returns "" when the span straddles a sentence
|
||||
// boundary or the model paraphrased what it quoted — such a row is re-examined
|
||||
// on every pass rather than cached, which is the safe direction.
|
||||
func chunkFor(original string, chunks []chunk) string {
|
||||
o := normalizeForDedup(original)
|
||||
if o == "" {
|
||||
return ""
|
||||
}
|
||||
for _, c := range chunks {
|
||||
if strings.Contains(normalizeForDedup(c.text), o) {
|
||||
return c.hash
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func texts(chunks []chunk) []string {
|
||||
out := make([]string, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
out = append(out, strings.TrimSpace(c.text))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestSplitChunks(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "plain sentences",
|
||||
in: "I has two apple. She go to market yesterday! Why?",
|
||||
want: []string{"I has two apple.", "She go to market yesterday!", "Why?"},
|
||||
},
|
||||
{
|
||||
// A decimal must not split, or the sentence's identity would churn
|
||||
// while she types the number.
|
||||
name: "decimals stay whole",
|
||||
in: "It costs 3.50 today. Tomorrow, more.",
|
||||
want: []string{"It costs 3.50 today.", "Tomorrow, more."},
|
||||
},
|
||||
{
|
||||
name: "closing quote travels with its sentence",
|
||||
in: `He said "early," and left. She stayed.`,
|
||||
want: []string{`He said "early," and left.`, "She stayed."},
|
||||
},
|
||||
{
|
||||
// Chinese runs sentences together with no space after 。 — she writes
|
||||
// in both languages in one document.
|
||||
name: "cjk terminators split without a space",
|
||||
in: "我想说这句话。但是不知道用英语怎么说。",
|
||||
want: []string{"我想说这句话。", "但是不知道用英语怎么说。"},
|
||||
},
|
||||
{
|
||||
name: "newlines break chunks",
|
||||
in: "A list item\nAnother item\n",
|
||||
want: []string{"A list item", "Another item"},
|
||||
},
|
||||
{
|
||||
name: "blank runs are dropped",
|
||||
in: "\n\n \nOnly this.\n\n",
|
||||
want: []string{"Only this."},
|
||||
},
|
||||
{
|
||||
name: "trailing fragment is its own chunk",
|
||||
in: "Done. Still writing",
|
||||
want: []string{"Done.", "Still writing"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := texts(splitChunks(tc.in, ""))
|
||||
if len(got) != len(tc.want) {
|
||||
t.Fatalf("want %q, got %q", tc.want, got)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tc.want[i] {
|
||||
t.Fatalf("chunk %d: want %q, got %q", i, tc.want[i], got[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A sentence's identity survives the churn that doesn't change what it says:
|
||||
// the editor rewrites quotes as she types, and a paragraph reflows.
|
||||
func TestChunkIdentityIgnoresCosmeticChurn(t *testing.T) {
|
||||
a := splitChunks(`She said "hello" softly.`, "")
|
||||
b := splitChunks("She said “hello” softly.", "")
|
||||
if len(a) != 1 || len(b) != 1 {
|
||||
t.Fatalf("want one chunk each, got %d and %d", len(a), len(b))
|
||||
}
|
||||
if a[0].hash != b[0].hash {
|
||||
t.Fatalf("quote/whitespace churn changed the sentence's identity")
|
||||
}
|
||||
if same := splitChunks(`She said "hello" softly.`, "academic"); same[0].hash == a[0].hash {
|
||||
t.Fatalf("a different tone must be a different reading of the sentence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangedChunksAndLookup(t *testing.T) {
|
||||
chunks := splitChunks("One thing. Another thing. One thing.", "")
|
||||
if len(chunks) != 3 {
|
||||
t.Fatalf("want 3 chunks, got %d", len(chunks))
|
||||
}
|
||||
|
||||
// A repeated sentence is one question, not two.
|
||||
if got := changedChunks(chunks, nil); len(got) != 2 {
|
||||
t.Fatalf("want 2 distinct changed chunks, got %d", len(got))
|
||||
}
|
||||
|
||||
checked := hashSet(chunks[:1])
|
||||
changed := changedChunks(chunks, checked)
|
||||
if len(changed) != 1 || strings.TrimSpace(changed[0].text) != "Another thing." {
|
||||
t.Fatalf("want only the unread sentence, got %q", texts(changed))
|
||||
}
|
||||
|
||||
if chunkFor("Another", chunks) != chunks[1].hash {
|
||||
t.Fatalf("span was attributed to the wrong sentence")
|
||||
}
|
||||
// A span the document doesn't contain has no sentence, so it is never cached.
|
||||
if chunkFor("nowhere in here", chunks) != "" {
|
||||
t.Fatalf("unanchorable span should have no chunk")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package suggestions
|
||||
|
||||
import "strings"
|
||||
|
||||
// What language is this DOCUMENT in — as opposed to this span.
|
||||
//
|
||||
// `readsAsPairLang` next door answers a span-level question for the purpose of
|
||||
// labelling one card, and it is written to under-claim: two marker words, or
|
||||
// nothing. A whole document needs the opposite temperament. A proportion, not a
|
||||
// presence — one Portuguese quotation inside an English essay must not flip the
|
||||
// entire pass into Portuguese, and one English sentence at the end of a
|
||||
// Portuguese journal must not keep it in English.
|
||||
//
|
||||
// Three properties, in the order they bite:
|
||||
//
|
||||
// - Decided over the WHOLE document, never a chunk. The grammar checkpoint
|
||||
// sends only the sentences that changed, so a verdict computed from what it
|
||||
// asked about would put an English card in a Portuguese journal the moment
|
||||
// she edits its one English line. The caller passes content_text, always.
|
||||
//
|
||||
// - Hysteresis. A bilingual paragraph sits near whatever single threshold we
|
||||
// pick, and a document crossing it every few keystrokes would alternate card
|
||||
// languages between passes — the same instability the mascot needed a band
|
||||
// for. Flip to the pair at 70% and back only below 40%; in between, whatever
|
||||
// it was last time stands. The band is the feature, not a rounding
|
||||
// tolerance.
|
||||
//
|
||||
// - Plain code, no model call. The house rule is that the LLM is garnish,
|
||||
// never a gatekeeper: a document must not become uncheckable because the
|
||||
// inference box is down.
|
||||
const (
|
||||
docLangEnglish = "en"
|
||||
docLangPair = "pair"
|
||||
)
|
||||
|
||||
// The band. Deliberately wide: the cost of an unnecessary flip is every card in
|
||||
// the document changing language, which is far more startling than a paragraph
|
||||
// of mixed writing being read as whichever language it was a minute ago.
|
||||
const (
|
||||
flipToPairAt = 0.70
|
||||
flipToEnglishBelow = 0.40
|
||||
)
|
||||
|
||||
// documentLang returns the language verdict for a document, given the verdict it
|
||||
// carried before. `prev` is "" for a document that has never been read.
|
||||
//
|
||||
// The result is one of docLangEnglish / docLangPair — not a language code. Which
|
||||
// language "pair" means is the writer's `pair_lang`, and keeping the stored
|
||||
// verdict relative to her pair means changing her pair doesn't strand a stale
|
||||
// language name on every document she owns.
|
||||
func documentLang(contentText, pairLang, prev string) string {
|
||||
if prev != docLangPair {
|
||||
prev = docLangEnglish
|
||||
}
|
||||
p := normalizePairLang(pairLang)
|
||||
if !hasLangTest(p) {
|
||||
// No test for this pair: say English, which is what every surface did
|
||||
// before this phase. A wrong flip is louder than a missing one.
|
||||
return docLangEnglish
|
||||
}
|
||||
|
||||
var pair, english int
|
||||
for _, c := range splitChunks(contentText, "") {
|
||||
switch sentenceLang(c.text, p) {
|
||||
case docLangPair:
|
||||
pair++
|
||||
case docLangEnglish:
|
||||
english++
|
||||
}
|
||||
}
|
||||
decided := pair + english
|
||||
if decided == 0 {
|
||||
// Nothing to go on — an empty document, a list of numbers, a title. Hold
|
||||
// the previous verdict rather than resetting a Portuguese journal to
|
||||
// English because she cleared it to start again.
|
||||
return prev
|
||||
}
|
||||
|
||||
ratio := float64(pair) / float64(decided)
|
||||
switch {
|
||||
case ratio >= flipToPairAt && corroborated(contentText, p):
|
||||
return docLangPair
|
||||
case ratio < flipToEnglishBelow:
|
||||
return docLangEnglish
|
||||
default:
|
||||
return prev
|
||||
}
|
||||
}
|
||||
|
||||
// sentenceLang classifies one sentence as pair-language, English, or neither.
|
||||
//
|
||||
// Neither is a real answer and carries weight: a sentence with no evidence
|
||||
// either way ("Bom dia.", "OK.", a heading) is left out of the ratio entirely
|
||||
// rather than counted for the language it isn't. Counting the undecided as
|
||||
// English is what would keep a Portuguese document in English forever, since
|
||||
// short sentences carry no markers.
|
||||
func sentenceLang(s, p string) string {
|
||||
if p == "zh" {
|
||||
han, latin := scriptCounts(s)
|
||||
switch {
|
||||
case han >= 2 && han > latin:
|
||||
return docLangPair
|
||||
case latin >= 3 && latin > han:
|
||||
return docLangEnglish
|
||||
}
|
||||
return ""
|
||||
}
|
||||
// A Latin pair shares its alphabet with English, so both sides are counted
|
||||
// the same way and the larger pile of evidence wins. A tie — including no
|
||||
// evidence at all — is no answer, which is why the English list below is
|
||||
// curated as carefully against the pair languages as theirs is against
|
||||
// English.
|
||||
pair := distinctMarkers(s, latinMarkers[p])
|
||||
eng := distinctMarkers(s, englishMarkers)
|
||||
switch {
|
||||
case pair > eng:
|
||||
return docLangPair
|
||||
case eng > pair:
|
||||
return docLangEnglish
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// corroborated requires the document as a whole to carry real evidence of the
|
||||
// pair language before the pass flips into it. A two-sentence document of
|
||||
// "Sim." / "Não." would otherwise reach 100% on almost nothing; a flip changes
|
||||
// every card in the document, so it has to be earned document-wide and not only
|
||||
// in proportion.
|
||||
func corroborated(contentText, p string) bool {
|
||||
if p == "zh" {
|
||||
han, _ := scriptCounts(contentText)
|
||||
return han >= 8
|
||||
}
|
||||
return distinctMarkers(contentText, latinMarkers[p]) >= 3
|
||||
}
|
||||
|
||||
// hasLangTest reports whether readsAsPairLang / sentenceLang know how to test
|
||||
// this pair at all. Kept beside the tests it describes so a new pair that adds
|
||||
// markers without adding itself here fails loudly in review rather than quietly
|
||||
// at runtime.
|
||||
func hasLangTest(p string) bool {
|
||||
if p == "zh" {
|
||||
return true
|
||||
}
|
||||
return len(latinMarkers[p]) > 0
|
||||
}
|
||||
|
||||
// English function words, curated against the pair languages exactly as
|
||||
// `latinMarkers` is curated against English.
|
||||
//
|
||||
// Every word here is one a Portuguese, French or Spanish sentence has no reason
|
||||
// to contain. Deliberately absent, each a false English vote waiting to happen
|
||||
// in someone's own language: "on" and "son" (French), "as", "a", "o", "e", "no",
|
||||
// "os" (Portuguese), "no", "para", "sin" (Spanish), and "is"-alikes that are
|
||||
// really other languages' words. The list is short on purpose — it does not need
|
||||
// coverage, only a reliable vote in the sentences where the pair list is silent.
|
||||
var englishMarkers = words(
|
||||
"the", "and", "is", "are", "was", "were", "be", "been", "being",
|
||||
"of", "to", "that", "this", "these", "those", "with", "from", "for",
|
||||
"have", "has", "had", "they", "them", "their", "there", "then", "than",
|
||||
"what", "which", "when", "where", "why", "how", "who",
|
||||
"will", "would", "should", "could", "can", "about", "because",
|
||||
"into", "some", "such", "only", "very", "much", "many", "other",
|
||||
"our", "your", "its", "it", "he", "she", "we", "you", "but", "not",
|
||||
"just", "like", "also", "most", "over", "after", "before", "between",
|
||||
"through", "said", "says", "get", "got", "make", "made", "know", "think",
|
||||
"thing", "things", "time", "people", "here", "always", "never", "something",
|
||||
"want", "need", "feel", "day", "today", "good", "really", "still", "even",
|
||||
)
|
||||
|
||||
// normalizeDocLang folds a stored verdict into the two values the rest of the
|
||||
// code reasons about: the column holds "" for a document nothing has read yet,
|
||||
// and that means English, which is what every surface did before this phase.
|
||||
func normalizeDocLang(v string) string {
|
||||
if strings.TrimSpace(v) == docLangPair {
|
||||
return docLangPair
|
||||
}
|
||||
return docLangEnglish
|
||||
}
|
||||
@@ -0,0 +1,536 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
)
|
||||
|
||||
// A monolingual document in either language has to be read as that language, and
|
||||
// the mixed cases in between are where the whole design lives: one quotation
|
||||
// must not move a document, and one leftover English line must not hold a
|
||||
// journal in English.
|
||||
func TestDocumentLangReadsWholeDocuments(t *testing.T) {
|
||||
const ptJournal = "Hoje foi um dia muito bom. Eu gosto de escrever aqui todas as noites. " +
|
||||
"A minha irmã também quer aprender. Não sei porque isso é tão difícil para mim."
|
||||
const enEssay = "The weather was very cold this morning. I think that the bus was late again. " +
|
||||
"She told me about the meeting, but I could not hear what they said."
|
||||
const zhJournal = "今天天气很好。我和妹妹一起去公园散步。我们看到很多花。"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
pairLang string
|
||||
prev string
|
||||
want string
|
||||
}{
|
||||
{"portuguese journal", ptJournal, "pt-PT", "", docLangPair},
|
||||
{"english essay", enEssay, "pt-PT", "", docLangEnglish},
|
||||
{"chinese journal", zhJournal, "zh", "", docLangPair},
|
||||
{"english essay, zh writer", enEssay, "zh", "", docLangEnglish},
|
||||
|
||||
// One English sentence at the end of a Portuguese journal is the case that
|
||||
// motivated the whole phase: the pass must stay in Portuguese.
|
||||
{
|
||||
"portuguese with one english line",
|
||||
ptJournal + " I will write more tomorrow.",
|
||||
"pt-PT", "", docLangPair,
|
||||
},
|
||||
// And the mirror: an English essay quoting a line of Portuguese is still an
|
||||
// English essay.
|
||||
{
|
||||
"english quoting portuguese",
|
||||
enEssay + " She wrote: \"Eu não sei o que dizer.\"",
|
||||
"pt-PT", "", docLangEnglish,
|
||||
},
|
||||
// A pair Petal has no test for cannot flip anything. Saying English is what
|
||||
// every surface did before this phase.
|
||||
{"untested pair", ptJournal, "de", "", docLangEnglish},
|
||||
// Nothing to go on holds the previous answer rather than resetting a
|
||||
// journal because she cleared it to start again.
|
||||
{"emptied portuguese journal", "", "pt-PT", docLangPair, docLangPair},
|
||||
{"emptied english essay", " \n ", "pt-PT", docLangEnglish, docLangEnglish},
|
||||
// Proportion, not presence: a couple of Portuguese words are not a
|
||||
// Portuguese document even though readsAsPairLang would label that span.
|
||||
{
|
||||
"english with a portuguese phrase",
|
||||
enEssay + " The sign said pão com manteiga.",
|
||||
"pt-PT", "", docLangEnglish,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := documentLang(tc.text, tc.pairLang, tc.prev); got != tc.want {
|
||||
t.Fatalf("documentLang = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The band, from both directions. A document sitting inside it keeps whatever it
|
||||
// was, and that is the point: without it, a bilingual paragraph would alternate
|
||||
// its cards' language every few keystrokes as she typed across the threshold.
|
||||
func TestDocumentLangHysteresis(t *testing.T) {
|
||||
// Half and half: two Portuguese sentences, two English ones. Inside the band
|
||||
// from either side.
|
||||
const mixed = "Eu gosto muito de escrever aqui. A minha irmã não sabe porque é difícil. " +
|
||||
"The weather was very cold this morning. I think that they said the same thing."
|
||||
|
||||
if got := documentLang(mixed, "pt-PT", docLangEnglish); got != docLangEnglish {
|
||||
t.Fatalf("mixed document from english = %q, want it to stay %q", got, docLangEnglish)
|
||||
}
|
||||
if got := documentLang(mixed, "pt-PT", docLangPair); got != docLangPair {
|
||||
t.Fatalf("mixed document from pair = %q, want it to stay %q", got, docLangPair)
|
||||
}
|
||||
|
||||
// Above the upper threshold it flips regardless of where it came from; below
|
||||
// the lower one it flips back regardless.
|
||||
const mostlyPT = "Eu gosto muito de escrever aqui. A minha irmã não sabe porque é difícil. " +
|
||||
"Hoje foi um dia bom para mim. Amanhã também quero escrever mais uma coisa. " +
|
||||
"I think so too."
|
||||
if got := documentLang(mostlyPT, "pt-PT", docLangEnglish); got != docLangPair {
|
||||
t.Fatalf("mostly-portuguese from english = %q, want %q", got, docLangPair)
|
||||
}
|
||||
const mostlyEN = "The weather was very cold this morning. I think that they said the same thing. " +
|
||||
"She could not hear what the other people were saying about it. Eu não sei."
|
||||
if got := documentLang(mostlyEN, "pt-PT", docLangPair); got != docLangEnglish {
|
||||
t.Fatalf("mostly-english from pair = %q, want %q", got, docLangEnglish)
|
||||
}
|
||||
}
|
||||
|
||||
// Corroboration: a proportion computed over almost nothing is not evidence. Two
|
||||
// bare words at 100% must not flip a document, because a flip rewrites every
|
||||
// card in it.
|
||||
func TestDocumentLangNeedsCorroboration(t *testing.T) {
|
||||
if got := documentLang("Não. Eu.", "pt-PT", docLangEnglish); got != docLangEnglish {
|
||||
t.Fatalf("two bare words flipped the document: %q", got)
|
||||
}
|
||||
if got := documentLang("我。", "zh", docLangEnglish); got != docLangEnglish {
|
||||
t.Fatalf("two Han runes flipped the document: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The Portuguese half of the same rule, and the bug it was reported as: "the
|
||||
// Portuguese option isn't translating the advice in English — it's just
|
||||
// reprinting Portuguese."
|
||||
//
|
||||
// Nothing was wrong with targetFor when that was reported. It was reading a
|
||||
// direction the account could not leave: `learnerPairs` held only zh, so every
|
||||
// pt-PT writer was learning_en by force and this function correctly explained a
|
||||
// Portuguese document in Portuguese. Pinned here rather than only in the auth
|
||||
// package because this is where the consequence actually lands — the language
|
||||
// the writer reads her advice in.
|
||||
func TestTargetExplainsPortugueseInEnglishForALearner(t *testing.T) {
|
||||
learner := targetFor("pt-PT", auth.DirectionLearningPair, docLangPair)
|
||||
if learner.Correct.Code != "pt-PT" {
|
||||
t.Fatalf("corrected in %s, want the document's own Portuguese", learner.Correct.Code)
|
||||
}
|
||||
if learner.Explain.Code != "en" {
|
||||
t.Fatalf("explained in %s, want English", learner.Explain.Code)
|
||||
}
|
||||
|
||||
// And the native Portuguese speaker practising English is untouched: her
|
||||
// Portuguese is still explained in Portuguese.
|
||||
native := targetFor("pt-PT", auth.DirectionLearningEn, docLangPair)
|
||||
if native.Correct.Code != "pt-PT" || native.Explain.Code != "pt-PT" {
|
||||
t.Fatalf("learning_en on a Portuguese document: correct=%s explain=%s", native.Correct.Code, native.Explain.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// The two language decisions are genuinely independent, and zh was the first
|
||||
// pair that could prove it — the first that could be travelled in both
|
||||
// directions.
|
||||
//
|
||||
// A Mandarin native practising English who writes Chinese wants Chinese
|
||||
// corrections explained in Chinese. An English native learning Chinese who writes
|
||||
// Chinese wants the same Chinese corrections explained in English. Same document,
|
||||
// same Correct, different Explain.
|
||||
func TestTargetSeparatesCorrectedFromExplained(t *testing.T) {
|
||||
learningEn := targetFor("zh", auth.DirectionLearningEn, docLangPair)
|
||||
if learningEn.Correct.Code != "zh" || learningEn.Explain.Code != "zh" {
|
||||
t.Fatalf("learning_en on a Chinese document: correct=%s explain=%s", learningEn.Correct.Code, learningEn.Explain.Code)
|
||||
}
|
||||
|
||||
learningPair := targetFor("zh", auth.DirectionLearningPair, docLangPair)
|
||||
if learningPair.Correct.Code != "zh" {
|
||||
t.Fatalf("learner direction changed what gets corrected: %s", learningPair.Correct.Code)
|
||||
}
|
||||
if learningPair.Explain.Code != "en" {
|
||||
t.Fatalf("learner direction explained in %s, want English", learningPair.Explain.Code)
|
||||
}
|
||||
|
||||
// An English document is the path every account is on today, in either
|
||||
// direction: English corrections, English explanations, her language still on
|
||||
// the Ask Petal and translate taps.
|
||||
for _, dir := range []string{auth.DirectionLearningEn, auth.DirectionLearningPair} {
|
||||
got := targetFor("zh", dir, docLangEnglish)
|
||||
if got.Flipped() || got.Explain.Code != "en" {
|
||||
t.Fatalf("english document with direction %s: %+v", dir, got)
|
||||
}
|
||||
if got.Pair.Code != "zh" {
|
||||
t.Fatalf("english document lost the writer's pair: %+v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// newDirectedServer seeds one writer on a given pair and direction, with a
|
||||
// document of her own. Like newPairServer, but the direction is the variable.
|
||||
func newDirectedServer(t *testing.T, client llm.LLMClient, pairLang, direction, text string) (http.Handler, string, *Handler) {
|
||||
t.Helper()
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "doclang.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
|
||||
const userID = "writer-directed"
|
||||
if _, err := database.Exec(
|
||||
`INSERT INTO users (id, email, display_name, pair_lang, direction) VALUES (?, ?, ?, ?, ?)`,
|
||||
userID, "d@example.com", "Writer", pairLang, direction,
|
||||
); err != nil {
|
||||
t.Fatalf("seed user: %v", err)
|
||||
}
|
||||
|
||||
var docID string
|
||||
if err := database.QueryRow(
|
||||
`INSERT INTO documents (user_id, content_text) VALUES (?, ?) RETURNING id`,
|
||||
userID, text,
|
||||
).Scan(&docID); err != nil {
|
||||
t.Fatalf("seed doc: %v", err)
|
||||
}
|
||||
|
||||
h := New(database, client)
|
||||
h.Limit = llm.NewRateLimiter(0)
|
||||
h.VoiceLimit = llm.NewRateLimiter(0)
|
||||
r := chi.NewRouter()
|
||||
r.Route("/docs", func(dr chi.Router) { h.RegisterDocRoutes(dr) })
|
||||
r.Mount("/suggestions", h.Routes())
|
||||
return auth.Middleware(auth.StaticResolver(userID))(r), docID, h
|
||||
}
|
||||
|
||||
const ptDocument = "Hoje foi um dia muito bom. Eu gosto de escrever aqui todas as noites. " +
|
||||
"A minha irmã também quer aprender comigo. Não sei porque isso é tão difícil para mim."
|
||||
|
||||
// End to end: a Portuguese document reaches the model as a Portuguese
|
||||
// checkpoint. This is the observed bug from 2026-07-28 — two pt-PT sentences
|
||||
// drew no cards at all, because Petal was reading them as bad English.
|
||||
func TestCheckpointFollowsTheDocumentLanguage(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[]}`}
|
||||
srv, docID, _ := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)
|
||||
|
||||
if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
|
||||
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if !strings.Contains(client.lastPrompt, "European Portuguese") {
|
||||
t.Fatalf("checkpoint didn't follow the document into Portuguese:\n%s", client.lastPrompt)
|
||||
}
|
||||
if strings.Contains(client.lastPrompt, "second language") {
|
||||
t.Fatalf("checkpoint kept the ESL framing on a Portuguese document:\n%s", client.lastPrompt)
|
||||
}
|
||||
|
||||
// And the voice pass, which had no language argument at all before this phase.
|
||||
if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/voice", ""); rec.Code != http.StatusOK {
|
||||
t.Fatalf("voice: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if !strings.Contains(client.lastPrompt, "European Portuguese") {
|
||||
t.Fatalf("voice pass didn't follow the document:\n%s", client.lastPrompt)
|
||||
}
|
||||
}
|
||||
|
||||
// The verdict is persisted, because hysteresis needs a yesterday.
|
||||
func TestDocumentLangIsRemembered(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[]}`}
|
||||
srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)
|
||||
|
||||
if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
|
||||
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
var stored string
|
||||
if err := h.DB.QueryRow(`SELECT doc_lang FROM documents WHERE id = ?`, docID).Scan(&stored); err != nil {
|
||||
t.Fatalf("read doc_lang: %v", err)
|
||||
}
|
||||
if stored != docLangPair {
|
||||
t.Fatalf("doc_lang = %q, want %q", stored, docLangPair)
|
||||
}
|
||||
}
|
||||
|
||||
// A document that changes language re-opens every sentence. Without the verdict
|
||||
// in the chunk salt, the sentences she didn't touch would keep serving cards
|
||||
// written in the language the document no longer speaks.
|
||||
func TestLanguageFlipReopensCheckedSentences(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[]}`}
|
||||
const enStart = "The weather was very cold this morning."
|
||||
srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, enStart)
|
||||
|
||||
if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
|
||||
t.Fatalf("first check: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
first := client.calls
|
||||
|
||||
// She rewrites the document in Portuguese, keeping the first sentence.
|
||||
setDocText(t, h, docID, enStart+" "+ptDocument)
|
||||
if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
|
||||
t.Fatalf("second check: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if client.calls == first {
|
||||
t.Fatal("the flipped document was never sent to the model")
|
||||
}
|
||||
if !strings.Contains(client.lastPrompt, "The weather was very cold") {
|
||||
t.Fatalf("the already-checked sentence was not re-opened by the flip:\n%s", client.lastPrompt)
|
||||
}
|
||||
}
|
||||
|
||||
// The translate card, pointed the other way. She is writing her journal in
|
||||
// Portuguese and drops in the one English sentence she knows; Petal renders it
|
||||
// into Portuguese, and that card is a translation — not a correction to prose
|
||||
// that was never wrong.
|
||||
func TestEnglishSpanBecomesATranslateCardInAPortugueseDocument(t *testing.T) {
|
||||
const english = "I want to say this but I don't know how to say it."
|
||||
// The model volunteers "clarity", as it did for the zh case. Not consulted.
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"` + english + `","replacement":"Eu quero dizer isto mas não sei como o dizer.","explanation":"Aqui está em português.","type":"clarity"}
|
||||
]}`}
|
||||
srv, docID, _ := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument+" "+english)
|
||||
|
||||
var out []db.Suggestion
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("want 1 card, got %d: %+v", len(out), out)
|
||||
}
|
||||
if out[0].Type != db.SuggestionTypeTranslate {
|
||||
t.Fatalf("card type = %q, want %q", out[0].Type, db.SuggestionTypeTranslate)
|
||||
}
|
||||
}
|
||||
|
||||
// And the half that keeps it honest: a genuine Portuguese correction in the same
|
||||
// document stays a correction. Reading the English-document test backwards would
|
||||
// have called this a translation, because every Portuguese sentence also "reads
|
||||
// as English" by that test's deliberately low bar.
|
||||
func TestPortugueseCorrectionKeepsItsTypeInAPortugueseDocument(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"Não sei porque isso é tão difícil para mim.","replacement":"Não sei porque isto é tão difícil para mim.","explanation":"Aqui usa-se isto.","type":"grammar"}
|
||||
]}`}
|
||||
srv, docID, _ := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)
|
||||
|
||||
var out []db.Suggestion
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("want 1 card, got %d: %+v", len(out), out)
|
||||
}
|
||||
if out[0].Type == db.SuggestionTypeTranslate {
|
||||
t.Fatal("a Portuguese correction inside a Portuguese document was labelled a translation")
|
||||
}
|
||||
}
|
||||
|
||||
// setDocLang writes a document's language verdict directly, so a test of the
|
||||
// tap-through doesn't have to run a checkpoint through the same stub client to
|
||||
// get one.
|
||||
func setDocLang(t *testing.T, h *Handler, docID, lang string) {
|
||||
t.Helper()
|
||||
if _, err := h.DB.Exec(`UPDATE documents SET doc_lang = ? WHERE id = ?`, lang, docID); err != nil {
|
||||
t.Fatalf("set doc_lang: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// seedExplanation files one card carrying a given explanation and returns its
|
||||
// id — the shape the translate tap-through needs, where only the explanation and
|
||||
// the document it hangs off matter.
|
||||
func seedExplanation(t *testing.T, h *Handler, docID, explanation string) string {
|
||||
t.Helper()
|
||||
var sugID string
|
||||
if err := h.DB.QueryRow(
|
||||
`INSERT INTO suggestions (doc_id, original, replacement, explanation, type, from_pos, to_pos)
|
||||
VALUES (?, ?, ?, ?, ?, 0, 5) RETURNING id`,
|
||||
docID, "isso", "isto", explanation, "grammar",
|
||||
).Scan(&sugID); err != nil {
|
||||
t.Fatalf("seed suggestion: %v", err)
|
||||
}
|
||||
return sugID
|
||||
}
|
||||
|
||||
// The tap-through has to read the same decision the card was written under. On a
|
||||
// Portuguese document by a Portuguese writer the explanation already arrived in
|
||||
// Portuguese, so the destination is the other half of the pair — the English she
|
||||
// is practising — and never Portuguese into Portuguese again.
|
||||
//
|
||||
// This pins the report that "Petal presents the Ask Petal advice in both
|
||||
// sections as Portuguese": the endpoint used to answer "" here, which left the
|
||||
// card Portuguese, the bubble beneath it the same Portuguese, and no English on
|
||||
// the card at all for a writer whose whole reason for the pair is English.
|
||||
func TestTranslateRendersHerExplanationIntoTheEnglishSheIsLearning(t *testing.T) {
|
||||
client := &stubClient{response: "Use this one here."}
|
||||
srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)
|
||||
setDocLang(t, h, docID, docLangPair)
|
||||
sugID := seedExplanation(t, h, docID, "Aqui usa-se isto.")
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/suggestions/"+sugID+"/translate", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("translate: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
var out translateResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if out.Translation == "" {
|
||||
t.Fatal("a Portuguese explanation left the tap with nowhere to go")
|
||||
}
|
||||
if !strings.Contains(client.lastPrompt, "into natural, friendly English") {
|
||||
t.Fatalf("translate didn't render into English:\n%s", client.lastPrompt)
|
||||
}
|
||||
if strings.Contains(client.lastPrompt, "into natural, friendly European Portuguese") {
|
||||
t.Fatalf("the model was asked to render Portuguese into Portuguese:\n%s", client.lastPrompt)
|
||||
}
|
||||
}
|
||||
|
||||
// The learner travelling the other way is the case that proves the endpoint
|
||||
// derives its destination rather than skipping whenever a document is flipped: a
|
||||
// native English speaker learning Chinese, writing Chinese, gets her
|
||||
// explanations in English — and the tap still has somewhere to go.
|
||||
func TestTranslateStillRendersForALearnersEnglishExplanation(t *testing.T) {
|
||||
const zhDocument = "今天天气很好。我早上去公园散步。下午我在家里写作业。晚上我和朋友一起吃饭。"
|
||||
client := &stubClient{response: "这里应该用这个。"}
|
||||
srv, docID, h := newDirectedServer(t, client, "zh", auth.DirectionLearningPair, zhDocument)
|
||||
setDocLang(t, h, docID, docLangPair)
|
||||
sugID := seedExplanation(t, h, docID, "This measure word doesn't fit here.")
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/suggestions/"+sugID+"/translate", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("translate: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
var out translateResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if out.Translation == "" {
|
||||
t.Fatal("a learner's English explanation was left untranslated")
|
||||
}
|
||||
if !strings.Contains(client.lastPrompt, "Simplified Chinese") {
|
||||
t.Fatalf("translate didn't render into the pair language:\n%s", client.lastPrompt)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPassAnnouncesItsVerdict pins the header the client reads. Storing the
|
||||
// verdict on the document row is not enough on its own: the editor sees that row
|
||||
// only when the document is opened or saved, and the pass that decides the
|
||||
// verdict runs *after* a save — so the client would always be one save behind,
|
||||
// and read-aloud is reached for exactly when she has stopped typing and no
|
||||
// further save is coming. Caught in a browser: a Portuguese paragraph read in an
|
||||
// American voice, twice, until another keystroke went in.
|
||||
//
|
||||
// Asserted on both endpoints that can flip it, and on the empty-document early
|
||||
// return, which answers without ever reaching the model.
|
||||
func TestPassAnnouncesItsVerdict(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[]}`}
|
||||
srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)
|
||||
|
||||
for _, path := range []string{"/check", "/voice"} {
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+path, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: code=%d body=%s", path, rec.Code, rec.Body)
|
||||
}
|
||||
if got := rec.Header().Get("X-Petal-Doc-Lang"); got != docLangPair {
|
||||
t.Fatalf("%s: X-Petal-Doc-Lang = %q, want %q", path, got, docLangPair)
|
||||
}
|
||||
}
|
||||
|
||||
// An English document says so rather than saying nothing — the client has to
|
||||
// be able to hear a flip back, not just a flip away.
|
||||
if _, err := h.DB.Exec(
|
||||
`UPDATE documents SET content_text = ?, doc_lang = '' WHERE id = ?`,
|
||||
"The weather was very cold this morning. I walked to the shop and bought some bread.", docID,
|
||||
); err != nil {
|
||||
t.Fatalf("rewrite doc: %v", err)
|
||||
}
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if got := rec.Header().Get("X-Petal-Doc-Lang"); got != docLangEnglish {
|
||||
t.Fatalf("English document: X-Petal-Doc-Lang = %q, want %q", got, docLangEnglish)
|
||||
}
|
||||
|
||||
// The empty-document path returns before the model call, and still answers.
|
||||
if _, err := h.DB.Exec(`UPDATE documents SET content_text = '' WHERE id = ?`, docID); err != nil {
|
||||
t.Fatalf("empty doc: %v", err)
|
||||
}
|
||||
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("empty check: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if got := rec.Header().Get("X-Petal-Doc-Lang"); got == "" {
|
||||
t.Fatal("empty document answered with no verdict header at all")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOrdinaryProseIsEnoughEvidence is the regression for what the marker lists
|
||||
// were caught doing on 2026-07-29, live, in a browser: unremarkable Portuguese
|
||||
// read as English, because the list was curated against English so tightly that
|
||||
// it had also been curated against ordinary writing. The document below scored
|
||||
// two pair markers and zero English ones, and two is below the corroboration
|
||||
// floor — so a paragraph with no evidence of English in it at all came back
|
||||
// English, and was corrected and read aloud as English.
|
||||
//
|
||||
// Every sample here is prose a person might actually write, not prose chosen to
|
||||
// contain markers. That is the whole point of the test: the failure was invisible
|
||||
// to a suite whose fixtures all argued their own case.
|
||||
func TestOrdinaryProseIsEnoughEvidence(t *testing.T) {
|
||||
samples := []struct{ name, text string }{
|
||||
{"the one seen live", "Esta manhã acordei cedo e fui correr ao longo da marginal. O ar estava fresco e havia poucas pessoas na rua. Depois comprei um jornal e li-o sentado num banco ao sol."},
|
||||
{"an afternoon out", "Hoje o céu estava limpo e fomos até ao jardim junto ao rio. A minha mãe trouxe uma manta velha e sentámos-nos debaixo de uma árvore."},
|
||||
{"plans", "Amanhã vamos ao cinema depois do trabalho. Ontem estava demasiado cansada para sair de casa."},
|
||||
}
|
||||
for _, s := range samples {
|
||||
if got := documentLang(s.text, "pt-PT", ""); got != docLangPair {
|
||||
t.Errorf("%s: documentLang = %q, want %q — ordinary Portuguese must not read as English\n%s",
|
||||
s.name, got, docLangPair, s.text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEnglishDidNotGetEasierToMistake is the other half, and the reason the
|
||||
// additions were held to "a word an English sentence has no reason to contain".
|
||||
// Widening a marker list is only safe if it widens in one direction: these are
|
||||
// English documents, including ones about Portugal and ones quoting Portuguese,
|
||||
// and every one of them must still come back English.
|
||||
func TestEnglishDidNotGetEasierToMistake(t *testing.T) {
|
||||
samples := []struct{ name, text string }{
|
||||
{"plain English", "This morning I woke up early and went for a run along the seafront. The air was fresh and there were few people about. Afterwards I bought a newspaper and read it on a bench."},
|
||||
{"English about Portugal", "We spent a week in Lisbon last summer. The trams were crowded but the food was wonderful, and we walked up to the castle every evening."},
|
||||
{"English quoting her", "My mother always says \"até amanhã\" when she leaves, never goodbye. I asked her why once and she said it sounded less final to her."},
|
||||
{"an English diary", "Today was long. I had two meetings before lunch and another one after, and by the time I got home I could not think straight. Tomorrow should be quieter."},
|
||||
}
|
||||
for _, s := range samples {
|
||||
if got := documentLang(s.text, "pt-PT", ""); got != docLangEnglish {
|
||||
t.Errorf("%s: documentLang = %q, want %q — the widened list must not pull English across\n%s",
|
||||
s.name, got, docLangEnglish, s.text)
|
||||
}
|
||||
}
|
||||
// And the same document must not flip once it is already sitting in English:
|
||||
// the hysteresis band is only a safety net if the low side holds too.
|
||||
for _, s := range samples {
|
||||
if got := documentLang(s.text, "pt-PT", docLangEnglish); got != docLangEnglish {
|
||||
t.Errorf("%s: held verdict flipped to %q", s.name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
"gitea.parodia.dev/drwily/petal/internal/vocab"
|
||||
)
|
||||
|
||||
// The growth journal.
|
||||
//
|
||||
// The suggestions table already records everything this needs — it is purely a
|
||||
// read-side view, with no new capture and no model call. Two framing rules
|
||||
// decide what may appear here, and they are enforced in the SQL rather than left
|
||||
// to the copy:
|
||||
//
|
||||
// 1. It reports growth, never an error tally. Nothing counts what she got
|
||||
// wrong this month; the signals are things that *stopped* happening and
|
||||
// phrasing that *stuck*.
|
||||
// 2. It only ever compares the writer to her own past self. There is no
|
||||
// target, no average, no other user anywhere in these queries.
|
||||
//
|
||||
// A quiet month is quiet: every signal below is omitted rather than softened
|
||||
// when the data isn't there, because an invented milestone is worse than none.
|
||||
|
||||
// Journal is one writer's growth over the recent windows.
|
||||
type Journal struct {
|
||||
// Kept / KeptBefore are edits she took on board in the last 30 days and in
|
||||
// the 30 before that — her own past self, the only comparison offered.
|
||||
Kept int `json:"kept"`
|
||||
KeptBefore int `json:"kept_before"`
|
||||
// Stuck: phrasing she was given that now turns up across her own writing.
|
||||
Stuck []Chunk `json:"stuck"`
|
||||
// Faded: things she used to need fixing and hasn't, recently.
|
||||
Faded []Fade `json:"faded"`
|
||||
}
|
||||
|
||||
// Chunk is a phrase that has stuck: it appears in Docs of her documents now.
|
||||
type Chunk struct {
|
||||
Phrase string `json:"phrase"`
|
||||
Docs int `json:"docs"`
|
||||
}
|
||||
|
||||
// Fade is a pattern that has stopped appearing. Times is how often it came up
|
||||
// during the earlier window — context for "and not since", never a scoreboard.
|
||||
type Fade struct {
|
||||
Pattern string `json:"pattern"`
|
||||
Times int `json:"times"`
|
||||
}
|
||||
|
||||
// Journal windows, in days. `recent` is the month being reported on; `history`
|
||||
// reaches back far enough that a pattern's absence means something (one quiet
|
||||
// fortnight doesn't).
|
||||
const (
|
||||
recentDays = 30
|
||||
historyDays = 120
|
||||
maxSignals = 3 // per list: a journal is a couple of warm lines, not a report
|
||||
)
|
||||
|
||||
// growth serves GET /api/suggestions/growth.
|
||||
func (h *Handler) growth(w http.ResponseWriter, r *http.Request) {
|
||||
userID := auth.UserID(r.Context())
|
||||
j := Journal{Stuck: []Chunk{}, Faded: []Fade{}}
|
||||
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT
|
||||
sum(CASE WHEN s.resolved_at >= datetime('now', '-30 days') THEN 1 ELSE 0 END),
|
||||
sum(CASE WHEN s.resolved_at < datetime('now', '-30 days')
|
||||
AND s.resolved_at >= datetime('now', '-60 days') THEN 1 ELSE 0 END)
|
||||
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||
WHERE d.user_id = ? AND s.status = 'accepted' AND s.resolved_at IS NOT NULL`,
|
||||
userID,
|
||||
).Scan(&nullInt{&j.Kept}, &nullInt{&j.KeptBefore})
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
stuck, err := h.stuck(userID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
j.Stuck = stuck
|
||||
|
||||
faded, err := h.faded(userID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
j.Faded = faded
|
||||
|
||||
httputil.WriteJSON(w, http.StatusOK, j)
|
||||
}
|
||||
|
||||
// stuck finds accepted phrasing that now appears in more than one of her own
|
||||
// documents. One document is just the edit itself, still sitting where it was
|
||||
// applied; a second is her reaching for the phrase on her own, which is the
|
||||
// whole claim the line makes.
|
||||
func (h *Handler) stuck(userID string) ([]Chunk, error) {
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT DISTINCT s.replacement
|
||||
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||
WHERE d.user_id = ? AND s.status = 'accepted'
|
||||
AND s.resolved_at >= datetime('now', '-120 days')
|
||||
AND trim(s.replacement) != ''
|
||||
ORDER BY s.resolved_at DESC
|
||||
LIMIT 40`,
|
||||
userID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// vocab.PhraseKey is the same definition of "a learnable chunk" the garden
|
||||
// plants, so the journal and the garden can never disagree about what counts.
|
||||
var phrases []string
|
||||
for rows.Next() {
|
||||
var replacement string
|
||||
if err := rows.Scan(&replacement); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if key := vocab.PhraseKey(replacement); key != "" {
|
||||
phrases = append(phrases, key)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := []Chunk{}
|
||||
for _, p := range phrases {
|
||||
var docs int
|
||||
if err := h.DB.QueryRow(
|
||||
`SELECT count(*) FROM documents WHERE user_id = ? AND instr(lower(content_text), ?) > 0`,
|
||||
userID, p,
|
||||
).Scan(&docs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if docs >= 2 {
|
||||
out = append(out, Chunk{Phrase: p, Docs: docs})
|
||||
}
|
||||
}
|
||||
sortDesc(out, func(c Chunk) int { return c.Docs })
|
||||
return trim(out, maxSignals), nil
|
||||
}
|
||||
|
||||
// faded finds patterns she used to be corrected on during the earlier part of
|
||||
// the history window and hasn't been since.
|
||||
//
|
||||
// The guard that makes this honest: it says nothing at all unless she has
|
||||
// actually been writing lately. Without it, a month away from Petal would be
|
||||
// reported back to her as progress, which is the one way this feature could lie.
|
||||
func (h *Handler) faded(userID string) ([]Fade, error) {
|
||||
var wroteRecently int
|
||||
if err := h.DB.QueryRow(
|
||||
`SELECT count(*) FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||
WHERE d.user_id = ? AND s.resolved_at >= datetime('now', '-30 days')`,
|
||||
userID,
|
||||
).Scan(&wroteRecently); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if wroteRecently == 0 {
|
||||
return []Fade{}, nil
|
||||
}
|
||||
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT lower(trim(s.original)) AS pattern, count(*) AS times
|
||||
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||
WHERE d.user_id = ? AND s.status = 'accepted'
|
||||
AND s.resolved_at < datetime('now', '-30 days')
|
||||
AND s.resolved_at >= datetime('now', '-120 days')
|
||||
AND trim(s.original) != ''
|
||||
AND pattern NOT IN (
|
||||
SELECT lower(trim(s2.original))
|
||||
FROM suggestions s2 JOIN documents d2 ON d2.id = s2.doc_id
|
||||
WHERE d2.user_id = ? AND s2.status = 'accepted'
|
||||
AND s2.resolved_at >= datetime('now', '-30 days'))
|
||||
GROUP BY pattern
|
||||
HAVING times >= 2
|
||||
ORDER BY times DESC
|
||||
LIMIT 3`,
|
||||
userID, userID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []Fade{}
|
||||
for rows.Next() {
|
||||
var f Fade
|
||||
if err := rows.Scan(&f.Pattern, &f.Times); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// nullInt scans a possibly-NULL aggregate into an int (SUM over no rows is
|
||||
// NULL, which is a zero here, not an error).
|
||||
type nullInt struct{ dst *int }
|
||||
|
||||
func (n *nullInt) Scan(v any) error {
|
||||
switch t := v.(type) {
|
||||
case int64:
|
||||
*n.dst = int(t)
|
||||
case nil:
|
||||
*n.dst = 0
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortDesc[T any](s []T, key func(T) int) {
|
||||
for i := 1; i < len(s); i++ {
|
||||
for j := i; j > 0 && key(s[j]) > key(s[j-1]); j-- {
|
||||
s[j], s[j-1] = s[j-1], s[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func trim[T any](s []T, n int) []T {
|
||||
if len(s) > n {
|
||||
return s[:n]
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// resolved seeds one already-settled suggestion, dated `daysAgo` at the moment
|
||||
// she decided it (the journal reads decisions, not proposals).
|
||||
func resolved(t *testing.T, h *Handler, docID, status, original, replacement string, daysAgo int) {
|
||||
t.Helper()
|
||||
_, err := h.DB.Exec(
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at)
|
||||
VALUES (?, 0, 0, ?, ?, '', 'collocation', ?, datetime('now', ?), datetime('now', ?))`,
|
||||
docID, original, replacement, status,
|
||||
"-"+strconv.Itoa(daysAgo)+" days", "-"+strconv.Itoa(daysAgo)+" days",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("seed resolved suggestion: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func seedDoc(t *testing.T, h *Handler, userID, text string) string {
|
||||
t.Helper()
|
||||
var id string
|
||||
if err := h.DB.QueryRow(
|
||||
`INSERT INTO documents (user_id, content_text) VALUES (?, ?) RETURNING id`, userID, text,
|
||||
).Scan(&id); err != nil {
|
||||
t.Fatalf("seed doc: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func readJournal(t *testing.T, srv http.Handler) Journal {
|
||||
t.Helper()
|
||||
rec := do(t, srv, http.MethodGet, "/suggestions/growth", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("growth: got %d, want 200 (body %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
var j Journal
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &j); err != nil {
|
||||
t.Fatalf("decode journal: %v", err)
|
||||
}
|
||||
return j
|
||||
}
|
||||
|
||||
// TestJournalIsEmptyForANewWriter: nothing to report reports nothing. Empty
|
||||
// lists, not nulls, so the frontend never has to guess.
|
||||
func TestJournalIsEmptyForANewWriter(t *testing.T) {
|
||||
srv, _, _ := newTestServer(t, &stubClient{})
|
||||
j := readJournal(t, srv)
|
||||
if j.Kept != 0 || j.KeptBefore != 0 || len(j.Stuck) != 0 || len(j.Faded) != 0 {
|
||||
t.Fatalf("new writer got a journal: %+v", j)
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeptComparesHerToHerOwnPastSelf.
|
||||
func TestKeptCountsTwoWindows(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
for i := 0; i < 3; i++ {
|
||||
resolved(t, h, docID, "accepted", "do a decision", "make a decision", 5)
|
||||
}
|
||||
resolved(t, h, docID, "accepted", "big rain", "heavy rain", 40)
|
||||
resolved(t, h, docID, "rejected", "no thanks", "no, thank you", 5) // decisions kept only
|
||||
resolved(t, h, docID, "accepted", "long ago", "long since", 200) // outside both windows
|
||||
|
||||
j := readJournal(t, srv)
|
||||
if j.Kept != 3 {
|
||||
t.Errorf("Kept = %d, want 3", j.Kept)
|
||||
}
|
||||
if j.KeptBefore != 1 {
|
||||
t.Errorf("KeptBefore = %d, want 1", j.KeptBefore)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStuckNeedsASecondDocument: a phrase sitting in the one document it was
|
||||
// applied to has not stuck — it's just the edit, where she left it. A second
|
||||
// document is her reaching for it herself, which is the claim the line makes.
|
||||
func TestStuckNeedsASecondDocument(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
if _, err := h.DB.Exec(`UPDATE documents SET content_text = ? WHERE id = ?`,
|
||||
"I had to make a decision.", docID); err != nil {
|
||||
t.Fatalf("set content: %v", err)
|
||||
}
|
||||
resolved(t, h, docID, "accepted", "do a decision", "make a decision", 10)
|
||||
resolved(t, h, docID, "accepted", "do a photo", "take a photo", 10)
|
||||
|
||||
if j := readJournal(t, srv); len(j.Stuck) != 0 {
|
||||
t.Fatalf("one document counted as sticking: %+v", j.Stuck)
|
||||
}
|
||||
|
||||
// She uses it again, elsewhere, on her own.
|
||||
seedDoc(t, h, db.LocalUserID, "Later I had to Make A Decision about the flat.")
|
||||
j := readJournal(t, srv)
|
||||
if len(j.Stuck) != 1 {
|
||||
t.Fatalf("Stuck = %+v, want just the phrase she reused", j.Stuck)
|
||||
}
|
||||
if j.Stuck[0].Phrase != "make a decision" || j.Stuck[0].Docs != 2 {
|
||||
t.Errorf("Stuck[0] = %+v, want {make a decision 2} (case-insensitive)", j.Stuck[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestFadedNeedsRecentWriting is the guard that keeps this feature honest: a
|
||||
// month away from Petal must never be reported back as progress.
|
||||
func TestFadedNeedsRecentWriting(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 60)
|
||||
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 55)
|
||||
|
||||
if j := readJournal(t, srv); len(j.Faded) != 0 {
|
||||
t.Fatalf("silence reported as growth: %+v", j.Faded)
|
||||
}
|
||||
|
||||
// She has been writing again this month — now the absence means something.
|
||||
resolved(t, h, docID, "accepted", "big rain", "heavy rain", 3)
|
||||
j := readJournal(t, srv)
|
||||
if len(j.Faded) != 1 || j.Faded[0].Pattern != "在 the morning" || j.Faded[0].Times != 2 {
|
||||
t.Fatalf("Faded = %+v, want the pattern she stopped needing (twice, back then)", j.Faded)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFadedExcludesWhatStillHappens: a pattern corrected again this month has
|
||||
// not faded, however often it came up before.
|
||||
func TestFadedExcludesWhatStillHappens(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 60)
|
||||
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 55)
|
||||
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 2)
|
||||
|
||||
if j := readJournal(t, srv); len(j.Faded) != 0 {
|
||||
t.Fatalf("Faded = %+v, want empty — it still happens", j.Faded)
|
||||
}
|
||||
}
|
||||
|
||||
// TestJournalIsPerWriter: another account's learning is never anyone else's
|
||||
// journal, and the only comparison Petal draws is with her own past self.
|
||||
func TestJournalIsPerWriter(t *testing.T) {
|
||||
srv, _, h := newTestServer(t, &stubClient{})
|
||||
if _, err := h.DB.Exec(`INSERT INTO users (id, email) VALUES ('bob', 'bob@example.com')`); err != nil {
|
||||
t.Fatalf("seed user: %v", err)
|
||||
}
|
||||
bobDoc := seedDoc(t, h, "bob", "Bob had to make a decision.")
|
||||
seedDoc(t, h, "bob", "Bob will make a decision again.")
|
||||
resolved(t, h, bobDoc, "accepted", "do a decision", "make a decision", 5)
|
||||
resolved(t, h, bobDoc, "accepted", "big rain", "heavy rain", 60)
|
||||
resolved(t, h, bobDoc, "accepted", "big rain", "heavy rain", 55)
|
||||
|
||||
j := readJournal(t, srv)
|
||||
if j.Kept != 0 || j.KeptBefore != 0 || len(j.Stuck) != 0 || len(j.Faded) != 0 {
|
||||
t.Fatalf("bob's learning leaked into the local user's journal: %+v", j)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
"gitea.parodia.dev/drwily/petal/internal/vocab"
|
||||
)
|
||||
|
||||
// Handler holds the dependencies for the checkpoint + suggestion routes. The
|
||||
@@ -54,12 +56,17 @@ func (h *Handler) RegisterDocRoutes(r chi.Router) {
|
||||
r.Post("/{id}/collocation", h.collocation)
|
||||
r.Post("/{id}/rewrite", h.rewrite)
|
||||
r.Get("/{id}/suggestions", h.listForDoc)
|
||||
r.Get("/{id}/settled", h.listSettled)
|
||||
}
|
||||
|
||||
// Routes returns the router mounted at /api/suggestions for per-suggestion
|
||||
// actions.
|
||||
func (h *Handler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
// The growth journal reads the same table these actions write, so it lives
|
||||
// here rather than growing its own mount. A literal segment, so it can never
|
||||
// be shadowed by an id.
|
||||
r.Get("/growth", h.growth)
|
||||
r.Post("/{id}/accept", h.accept)
|
||||
r.Post("/{id}/dismiss", h.dismiss)
|
||||
r.Post("/{id}/chat", h.chat)
|
||||
@@ -84,6 +91,22 @@ type mechanicsFinding struct {
|
||||
Original string `json:"original"`
|
||||
Replacement string `json:"replacement"`
|
||||
Explanation string `json:"explanation"`
|
||||
// Which family this offline finding belongs to. Empty (the historical shape)
|
||||
// means mechanics; the miscollocation rules send 'collocation' so a chunk the
|
||||
// rule pack caught is indistinguishable from one the coach caught — same
|
||||
// family, same rail, and the same planting into the garden on accept.
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
// localType maps a client-supplied family onto the two an offline rule may claim.
|
||||
// Anything else — including the empty string older clients send — is mechanics,
|
||||
// so a stray label can never smuggle a row into an LLM family and survive that
|
||||
// pass's DELETE.
|
||||
func localType(t string) string {
|
||||
if strings.ToLower(strings.TrimSpace(t)) == db.SuggestionTypeCollocation {
|
||||
return db.SuggestionTypeCollocation
|
||||
}
|
||||
return db.SuggestionTypeMechanics
|
||||
}
|
||||
|
||||
// maxMechanicsFindings caps a single submission so a runaway client can't flood
|
||||
@@ -138,10 +161,21 @@ func (h *Handler) mechanics(w http.ResponseWriter, r *http.Request) {
|
||||
httputil.WriteJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// replaceMechanics swaps the document's pending mechanics rows for the supplied
|
||||
// findings in one transaction, leaving the LLM families and actioned rows
|
||||
// untouched. Findings the user already accepted or dismissed are suppressed (the
|
||||
// detector has no memory between runs), and malformed spans are skipped.
|
||||
// replaceMechanics brings the document's pending offline rows in line with the
|
||||
// supplied findings in one transaction, leaving the LLM families and actioned
|
||||
// rows untouched. Findings the user already accepted or dismissed are suppressed
|
||||
// (the detector has no memory between runs), and malformed spans are skipped.
|
||||
//
|
||||
// A finding the detector still reports keeps its existing row — same id, same
|
||||
// created_at — and only its offsets move. This pass fires 250 ms after a
|
||||
// keystroke, so deleting and re-inserting the family would hand every card a new
|
||||
// identity several times a sentence: the rail would remount, a card expanded for
|
||||
// Ask Petal would collapse under her, and the arrival chime would re-fire.
|
||||
//
|
||||
// The scope is *source*, not type: the rule pack owns both the mechanics family
|
||||
// and its share of the collocation family, and every run is a full recompute of
|
||||
// the document. Scoping by type instead would strand offline collocations the
|
||||
// current text no longer warrants — the one row nobody would ever replace.
|
||||
func (h *Handler) replaceMechanics(docID string, findings []mechanicsFinding) error {
|
||||
tx, err := h.DB.Begin()
|
||||
if err != nil {
|
||||
@@ -149,18 +183,18 @@ func (h *Handler) replaceMechanics(docID string, findings []mechanicsFinding) er
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
if _, err := tx.Exec(
|
||||
`DELETE FROM suggestions WHERE doc_id = ? AND status = ? AND type = ?`,
|
||||
docID, db.SuggestionStatusPending, db.SuggestionTypeMechanics,
|
||||
); err != nil {
|
||||
existing, err := loadPending(tx, docID, "source = '"+db.SuggestionSourceLocal+"'")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
index := indexByEdit(existing)
|
||||
|
||||
sup, err := buildSuppressor(tx, docID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
kept := make(map[string]bool, len(existing))
|
||||
for _, f := range findings {
|
||||
if f.From < 0 || f.To <= f.From || strings.TrimSpace(f.Original) == "" {
|
||||
continue // malformed span — the client re-anchors by string anyway
|
||||
@@ -168,15 +202,34 @@ func (h *Handler) replaceMechanics(docID string, findings []mechanicsFinding) er
|
||||
if sup.suppressed(f.Original, f.Replacement) {
|
||||
continue
|
||||
}
|
||||
typ := localType(f.Type)
|
||||
if row, ok := index.take(f.Original, f.Replacement, f.From); ok {
|
||||
kept[row.id] = true
|
||||
if err := reposition(tx, row, f.From, f.To, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
docID, f.From, f.To, f.Original, f.Replacement, f.Explanation, db.SuggestionTypeMechanics,
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, source)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
docID, f.From, f.To, f.Original, f.Replacement, f.Explanation,
|
||||
typ, db.SuggestionSourceLocal,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Whatever the detector no longer reports, she has fixed.
|
||||
for _, row := range existing {
|
||||
if kept[row.id] {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, row.id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@@ -194,12 +247,45 @@ func (h *Handler) collocation(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// pass is the signature shared by the grammar checkpoint and the voice pass:
|
||||
// given the document text, the document's tone and the writer's pair language it
|
||||
// returns the model's raw suggestions. The voice pass ignores both extras (see
|
||||
// llm.RunVoice) and the checkpoint ignores the language — only the collocation
|
||||
// coach writes a word of it — but one signature keeps runPass free of special
|
||||
// cases.
|
||||
type pass func(ctx context.Context, client llm.LLMClient, contentText, tone string, lang llm.Lang) ([]llm.RawSuggestion, error)
|
||||
// given the document text, the document's tone and the languages this document
|
||||
// is to be corrected and explained in, it returns the model's raw suggestions.
|
||||
// The voice pass ignores the tone (see llm.RunVoice) and the collocation coach
|
||||
// reads only the writer's pair language, but one signature keeps runPass free of
|
||||
// special cases.
|
||||
type pass func(ctx context.Context, client llm.LLMClient, contentText, tone string, t llm.Target) ([]llm.RawSuggestion, error)
|
||||
|
||||
// targetFor resolves the two language decisions for one pass over one document.
|
||||
//
|
||||
// They read different state on purpose. What gets *corrected* follows the
|
||||
// document, because Portuguese prose wants Portuguese corrections. What language
|
||||
// the correction is *explained* in follows the writer — the half of her pair she
|
||||
// is not learning — because an explanation is teaching, and teaching lands in the
|
||||
// language she reads most easily. A native Portuguese speaker practising English
|
||||
// gets Portuguese explained in Portuguese; a native English speaker learning
|
||||
// French gets French explained in English. Neither is trapped: the other language
|
||||
// stays one tap away, in both directions.
|
||||
//
|
||||
// An English document keeps the pre-Phase-28 behaviour exactly — explained in
|
||||
// English, with her language on the Ask Petal / translate taps — which is the
|
||||
// path every account today is on.
|
||||
//
|
||||
// The direction lookup costs nothing today: `learnerPairs` is {"zh"}, so fr, es
|
||||
// and pt-PT accounts are all learning_en and their non-learned half *is* the pair
|
||||
// language. This rule therefore produces "explain in the document's language" for
|
||||
// every writer who currently exists. It is written out anyway to stop the
|
||||
// coincidence being baked into the prompts, the way "English is the language
|
||||
// being learned" was baked into pair_lang before migration 0016.
|
||||
func targetFor(pairLang, direction, docLang string) llm.Target {
|
||||
pair := llm.LangFor(pairLang)
|
||||
if normalizeDocLang(docLang) != docLangPair {
|
||||
return llm.EnglishTarget(pair)
|
||||
}
|
||||
explain := pair
|
||||
if direction == auth.DirectionLearningPair {
|
||||
explain = llm.English
|
||||
}
|
||||
return llm.Target{Correct: pair, Explain: explain, Pair: pair}
|
||||
}
|
||||
|
||||
// runPass is the shared body for both LLM passes. It loads the document text,
|
||||
// enforces the pass's per-document rate limit, runs the model, swaps in the
|
||||
@@ -209,17 +295,19 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
||||
docID := chi.URLParam(r, "id")
|
||||
userID := auth.UserID(r.Context())
|
||||
|
||||
// The writer's pair language rides along with the document rather than in a
|
||||
// second query: it is read from the same row-scoped lookup that already
|
||||
// proves she owns this document.
|
||||
var contentText, tone, pairLang string
|
||||
// The writer's pair language and direction ride along with the document
|
||||
// rather than in a second query: they are read from the same row-scoped
|
||||
// lookup that already proves she owns this document. `doc_lang` is the
|
||||
// previous language verdict, which the new one needs (hysteresis).
|
||||
var contentText, tone, pairLang, direction, prevLang string
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT d.content_text, d.tone, COALESCE(u.pair_lang, '')
|
||||
`SELECT d.content_text, d.tone, d.doc_lang,
|
||||
COALESCE(u.pair_lang, ''), COALESCE(u.direction, '')
|
||||
FROM documents d
|
||||
JOIN users u ON u.id = d.user_id
|
||||
WHERE d.id = ? AND d.user_id = ?`,
|
||||
docID, userID,
|
||||
).Scan(&contentText, &tone, &pairLang)
|
||||
).Scan(&contentText, &tone, &prevLang, &pairLang, &direction)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusNotFound, "document not found")
|
||||
return
|
||||
@@ -229,11 +317,103 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
||||
return
|
||||
}
|
||||
|
||||
// Nothing to analyze on an empty document — skip the LLM round-trip.
|
||||
// What language is this document in, and so what language should its cards be
|
||||
// written in? Computed from the whole content_text — never from `askText`,
|
||||
// which on a chunked pass is only the sentences that changed, and would put an
|
||||
// English card in a Portuguese journal the moment she edits its one English
|
||||
// line.
|
||||
//
|
||||
// Decided before the empty-document exit so every reconcile below is told the
|
||||
// same verdict. An emptied document has nothing to go on and holds whatever it
|
||||
// said last (see documentLang), which is what keeps a Portuguese journal
|
||||
// Portuguese while she clears it to start the entry again.
|
||||
docLang := documentLang(contentText, pairLang, prevLang)
|
||||
|
||||
// Announce the verdict on every answer this pass gives, including the early
|
||||
// ones below. This pass is the only thing that decides the value, so it is
|
||||
// the only moment the client can learn it promptly — and the client needs it
|
||||
// promptly for read-aloud, which is reached for exactly when she has stopped
|
||||
// typing and no further save is coming. Carrying it back on the document row
|
||||
// alone means the editor is always one save behind the truth, and a paragraph
|
||||
// of Portuguese read in an American voice is how that sounds.
|
||||
//
|
||||
// A header rather than a wider body: /check and /voice answer with a bare
|
||||
// array of the unified pending set, and every caller of both endpoints reads
|
||||
// it as one. A verdict is metadata about the pass, not another suggestion.
|
||||
w.Header().Set("X-Petal-Doc-Lang", docLang)
|
||||
|
||||
// Nothing to analyze on an empty document — skip the LLM round-trip. The
|
||||
// family's rows go with the text they were about.
|
||||
if strings.TrimSpace(contentText) == "" {
|
||||
httputil.WriteJSON(w, http.StatusOK, []db.Suggestion{})
|
||||
if err := h.reconcilePending(docID, contentText, pairLang, docLang, nil, scope, nil, nil, false); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
out, err := h.fetchPending(userID, docID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, out)
|
||||
return
|
||||
}
|
||||
|
||||
if docLang != normalizeDocLang(prevLang) {
|
||||
if _, err := h.DB.Exec(
|
||||
`UPDATE documents SET doc_lang = ? WHERE id = ? AND user_id = ?`,
|
||||
docLang, docID, userID,
|
||||
); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
target := targetFor(pairLang, direction, docLang)
|
||||
|
||||
// Decide what to ask about before spending anything: a chunked pass asks only
|
||||
// about the sentences that changed since it last read the document, and when
|
||||
// none did it doesn't call the model at all — nor consume its rate-limit slot,
|
||||
// so the next real edit isn't throttled by a check that had nothing to do.
|
||||
//
|
||||
// Only a chunked pass consults that record, so only it needs the tone folded
|
||||
// into a sentence's identity — and, next to it, the language verdict. A
|
||||
// document that flips language changes every sentence's identity, so its
|
||||
// old-language cards are re-checked rather than left sitting there in a
|
||||
// language the rest of the document no longer speaks.
|
||||
salt := ""
|
||||
if scope.chunked {
|
||||
salt = tone + "\x00" + docLang
|
||||
}
|
||||
chunks := splitChunks(contentText, salt)
|
||||
askText, fresh := contentText, chunks
|
||||
if scope.chunked {
|
||||
checked, err := h.checkedChunks(docID, scope.family)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
changed := changedChunks(chunks, checked)
|
||||
if len(changed) == 0 {
|
||||
// Every sentence has already been read. Drop the rows whose sentence is
|
||||
// gone, keep the rest exactly as they are, and answer immediately.
|
||||
if err := h.reconcilePending(docID, contentText, pairLang, docLang, nil, scope, chunks, nil, false); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
out, err := h.fetchPending(userID, docID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, out)
|
||||
return
|
||||
}
|
||||
// When every sentence is new — a first pass, a paste, a tone switch — hand
|
||||
// over the document verbatim so the model reads it with its paragraphing
|
||||
// intact. Otherwise send just the delta, one sentence per line.
|
||||
if len(changed) < len(hashSet(chunks)) {
|
||||
askText, fresh = joinChunks(changed), changed
|
||||
}
|
||||
}
|
||||
|
||||
ok, _, slotAt := limiter.Allow(docID)
|
||||
if !ok {
|
||||
@@ -248,17 +428,19 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
||||
return
|
||||
}
|
||||
|
||||
raw, err := run(r.Context(), h.Client, contentText, tone, llm.LangFor(pairLang))
|
||||
raw, err := run(r.Context(), h.Client, askText, tone, target)
|
||||
if err != nil {
|
||||
// Allow ran before the model call, so a failed pass would otherwise hold
|
||||
// the per-document slot for the full interval — stranding the frontend's
|
||||
// auto-retry on the throttle path. Release it so a retry can re-run.
|
||||
limiter.Release(docID, slotAt)
|
||||
httputil.ErrorJSON(w, http.StatusBadGateway, "llm pass failed: "+err.Error())
|
||||
httputil.UpstreamError(w, "pass", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.replacePending(docID, contentText, raw, scope); err != nil {
|
||||
// A whole-document pass re-read everything, so every one of its rows is up for
|
||||
// re-proposal; a chunked pass only puts the sentences it asked about in play.
|
||||
if err := h.reconcilePending(docID, contentText, pairLang, docLang, raw, scope, chunks, fresh, !scope.chunked); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
@@ -279,73 +461,49 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
||||
// inserts. The grammar checkpoint and voice pass each own a disjoint family, so
|
||||
// running one never disturbs the other's pending flags.
|
||||
type pendingScope struct {
|
||||
deleteWhere string // extra WHERE clause scoping the DELETE to this family
|
||||
deleteWhere string // extra WHERE clause scoping this pass to its own family
|
||||
forceType string // if set, every inserted row gets this type; else normalizeType
|
||||
// family keys the sentences this pass has already read (see checked_chunks).
|
||||
family string
|
||||
// chunked passes re-read only the sentences that changed. True for the typing-
|
||||
// cadence grammar checkpoint, which fires constantly and must feel still;
|
||||
// false for the explicit whole-document passes, where she pressed a button
|
||||
// asking for a fresh read of everything.
|
||||
chunked bool
|
||||
}
|
||||
|
||||
// Every scope below is confined to source='llm'. The offline rule pack owns its
|
||||
// own rows and recomputes them on each edit (see replaceMechanics); its findings
|
||||
// must survive all three model passes — including the collocation coach, which
|
||||
// now shares the collocation family with it.
|
||||
var (
|
||||
// grammarScope owns the grammar/phrasing/idiom/clarity flags — everything but
|
||||
// the other self-owned families (voice, collocation, mechanics), which run on
|
||||
// their own cadence/pass and must survive a grammar checkpoint. Notably the
|
||||
// deterministic mechanics pass writes its rows in the same /check request just
|
||||
// before this DELETE runs, so excluding it here is what keeps them alive.
|
||||
grammarScope = pendingScope{deleteWhere: "type NOT IN ('voice','collocation','mechanics')", forceType: ""}
|
||||
// voiceScope owns the voice flags only.
|
||||
voiceScope = pendingScope{deleteWhere: "type = 'voice'", forceType: db.SuggestionTypeVoice}
|
||||
// collocationScope owns the collocation flags only.
|
||||
collocationScope = pendingScope{deleteWhere: "type = 'collocation'", forceType: db.SuggestionTypeCollocation}
|
||||
// the other self-owned families (voice, collocation), which run on their own
|
||||
// cadence/pass and must survive a grammar checkpoint. Notably the offline pass
|
||||
// writes its rows in the same /check request just before this pass reconciles,
|
||||
// so the source clause is also what keeps them alive.
|
||||
grammarScope = pendingScope{
|
||||
deleteWhere: "source = 'llm' AND type NOT IN ('voice','collocation')",
|
||||
family: "grammar",
|
||||
chunked: true,
|
||||
}
|
||||
// voiceScope owns the model's voice flags only. Voice is a property of the
|
||||
// document as a whole — a sentence isn't inconsistent with itself — so this
|
||||
// pass always reads everything.
|
||||
voiceScope = pendingScope{
|
||||
deleteWhere: "source = 'llm' AND type = 'voice'",
|
||||
forceType: db.SuggestionTypeVoice,
|
||||
family: "voice",
|
||||
}
|
||||
// collocationScope owns the model's collocation flags only — the rule pack's
|
||||
// share of the same family is left standing.
|
||||
collocationScope = pendingScope{
|
||||
deleteWhere: "source = 'llm' AND type = 'collocation'",
|
||||
forceType: db.SuggestionTypeCollocation,
|
||||
family: "collocation",
|
||||
}
|
||||
)
|
||||
|
||||
// replacePending swaps a document's pending suggestions within one family for a
|
||||
// fresh batch in a single transaction. Accepted/rejected suggestions and the
|
||||
// other family's pending rows are left untouched.
|
||||
//
|
||||
// Suggestions touching a sentence the user already settled are suppressed from
|
||||
// the fresh batch (see suppressor): not just the identical edit re-proposed, but
|
||||
// reversals and re-polishing of the model's own just-accepted output — the
|
||||
// "fickle, keeps going back and forth on a few sentences" behavior. The model has
|
||||
// no memory between passes, so without this it re-opens resolved sentences every
|
||||
// checkpoint.
|
||||
func (h *Handler) replacePending(docID, contentText string, raw []llm.RawSuggestion, scope pendingScope) error {
|
||||
tx, err := h.DB.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
if _, err := tx.Exec(
|
||||
`DELETE FROM suggestions WHERE doc_id = ? AND status = ? AND `+scope.deleteWhere,
|
||||
docID, db.SuggestionStatusPending,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sup, err := buildSuppressor(tx, docID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, s := range raw {
|
||||
if sup.suppressed(s.Original, s.Replacement) {
|
||||
continue
|
||||
}
|
||||
typ := scope.forceType
|
||||
if typ == "" {
|
||||
typ = normalizeType(s.Type)
|
||||
}
|
||||
from, to := locate(contentText, s.Original)
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
docID, from, to, s.Original, s.Replacement, s.Explanation, typ,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// dedupQuoteReplacer folds every straight/curly single- and double-quote variant
|
||||
// (and backtick/acute accent) onto one canonical character. The editor and the
|
||||
// model both rewrite quotes between passes — a sentence accepted with "…" comes
|
||||
@@ -484,6 +642,76 @@ func (h *Handler) listForDoc(w http.ResponseWriter, r *http.Request) {
|
||||
httputil.WriteJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// listSettled returns the normalized originals of every edit the user has
|
||||
// already accepted or dismissed on this document — the same spans buildSuppressor
|
||||
// drops on the server, handed to the client so its instant rule-pack pass can
|
||||
// drop them too.
|
||||
//
|
||||
// Without this the offline half of the loop has no memory. The rule pack detects
|
||||
// from the text alone and re-runs 250 ms after a keystroke, so a dismissed "the
|
||||
// the" comes straight back the moment she types anywhere in the document; the
|
||||
// server's reply then removes it again. That flicker is the visible symptom, but
|
||||
// the real one is worse: with the server unreachable — the case the rule pack
|
||||
// exists for — the reply never comes and a card she dismissed simply stays.
|
||||
//
|
||||
// Only the originals are sent. Replacements are the model's words, not hers, and
|
||||
// the client only needs to answer "has she settled this span?"
|
||||
func (h *Handler) listSettled(w http.ResponseWriter, r *http.Request) {
|
||||
out, err := h.fetchSettled(auth.UserID(r.Context()), chi.URLParam(r, "id"))
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, settledResponse{Originals: out})
|
||||
}
|
||||
|
||||
// settledResponse wraps the list so the endpoint can grow a second field without
|
||||
// breaking a client that reads a bare array.
|
||||
type settledResponse struct {
|
||||
Originals []string `json:"originals"`
|
||||
}
|
||||
|
||||
// fetchSettled loads the distinct normalized originals of the document's actioned
|
||||
// rows. Scoped through documents for the same reason fetchPending is: an original
|
||||
// is a quotation of her writing.
|
||||
func (h *Handler) fetchSettled(userID, docID string) ([]string, error) {
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT DISTINCT s.original
|
||||
FROM suggestions s
|
||||
JOIN documents d ON d.id = s.doc_id
|
||||
WHERE s.doc_id = ? AND d.user_id = ? AND s.status IN (?, ?)`,
|
||||
docID, userID, db.SuggestionStatusAccepted, db.SuggestionStatusRejected,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// DISTINCT is on the raw text; normalizing can collapse two rows into one, so
|
||||
// dedupe again on this side to keep the payload honest.
|
||||
seen := map[string]struct{}{}
|
||||
out := []string{}
|
||||
for rows.Next() {
|
||||
var original string
|
||||
if err := rows.Scan(&original); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
norm := normalizeForDedup(original)
|
||||
if norm == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[norm]; dup {
|
||||
continue
|
||||
}
|
||||
seen[norm] = struct{}{}
|
||||
out = append(out, norm)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// fetchPending loads a document's pending suggestions, joined through documents
|
||||
// so the rows are only reachable by the document's owner. A suggestion quotes the
|
||||
// sentence it corrects, so an unscoped read here would leak document text to
|
||||
@@ -491,7 +719,7 @@ func (h *Handler) listForDoc(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *Handler) fetchPending(userID, docID string) ([]db.Suggestion, error) {
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT s.id, s.doc_id, s.from_pos, s.to_pos, s.original, s.replacement,
|
||||
s.explanation, s.type, s.status, s.created_at
|
||||
s.explanation, s.type, s.status, s.source, s.created_at
|
||||
FROM suggestions s
|
||||
JOIN documents d ON d.id = s.doc_id
|
||||
WHERE s.doc_id = ? AND d.user_id = ? AND s.status = ?
|
||||
@@ -508,7 +736,7 @@ func (h *Handler) fetchPending(userID, docID string) ([]db.Suggestion, error) {
|
||||
var s db.Suggestion
|
||||
if err := rows.Scan(
|
||||
&s.ID, &s.DocID, &s.FromPos, &s.ToPos, &s.Original, &s.Replacement,
|
||||
&s.Explanation, &s.Type, &s.Status, &s.CreatedAt,
|
||||
&s.Explanation, &s.Type, &s.Status, &s.Source, &s.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -520,11 +748,14 @@ func (h *Handler) fetchPending(userID, docID string) ([]db.Suggestion, error) {
|
||||
return dedupeSpans(out), nil
|
||||
}
|
||||
|
||||
// dedupeSpans resolves collisions between the deterministic mechanics family and
|
||||
// the LLM families: when a mechanics finding and an LLM suggestion fight over the
|
||||
// same characters, mechanics wins and the LLM card is dropped. Its span is exact
|
||||
// (the detector matched it), whereas the LLM positions are only advisory
|
||||
// (re-anchored by string at render), so the precise fix should own the span.
|
||||
// dedupeSpans resolves collisions between the offline rule pack and the model:
|
||||
// when a local finding and an LLM suggestion fight over the same characters, the
|
||||
// local one wins and the LLM card is dropped. Its span is exact (the detector
|
||||
// matched it), whereas the LLM positions are only advisory (re-anchored by string
|
||||
// at render), so the precise fix should own the span. This is why the split is by
|
||||
// source rather than by type — an offline miscollocation is as exact as an
|
||||
// offline comma, and the coach's fuzzy version of the same chunk shouldn't
|
||||
// double up next to it.
|
||||
//
|
||||
// This deliberately does NOT dedupe LLM-vs-LLM overlaps: voice (awareness-only,
|
||||
// no replacement) and collocation legitimately co-occupy the same span, and that
|
||||
@@ -534,7 +765,7 @@ func dedupeSpans(in []db.Suggestion) []db.Suggestion {
|
||||
type span struct{ from, to int }
|
||||
var claimed []span
|
||||
for _, s := range in {
|
||||
if s.Type == db.SuggestionTypeMechanics && s.FromPos >= 0 {
|
||||
if s.Source == db.SuggestionSourceLocal && s.FromPos >= 0 {
|
||||
claimed = append(claimed, span{s.FromPos, s.ToPos})
|
||||
}
|
||||
}
|
||||
@@ -544,7 +775,7 @@ func dedupeSpans(in []db.Suggestion) []db.Suggestion {
|
||||
|
||||
out := make([]db.Suggestion, 0, len(in))
|
||||
for _, s := range in {
|
||||
if s.Type != db.SuggestionTypeMechanics && s.FromPos >= 0 {
|
||||
if s.Source != db.SuggestionSourceLocal && s.FromPos >= 0 {
|
||||
overlaps := false
|
||||
for _, sp := range claimed {
|
||||
if s.FromPos < sp.to && sp.from < s.ToPos {
|
||||
@@ -553,7 +784,7 @@ func dedupeSpans(in []db.Suggestion) []db.Suggestion {
|
||||
}
|
||||
}
|
||||
if overlaps {
|
||||
continue // an exact mechanics fix owns these characters
|
||||
continue // an exact offline fix owns these characters
|
||||
}
|
||||
}
|
||||
out = append(out, s)
|
||||
@@ -577,7 +808,7 @@ func (h *Handler) dismiss(w http.ResponseWriter, r *http.Request) {
|
||||
// no rows and surfaces as a 404.
|
||||
func (h *Handler) setStatus(w http.ResponseWriter, r *http.Request, status string) {
|
||||
res, err := h.DB.Exec(
|
||||
`UPDATE suggestions SET status = ?
|
||||
`UPDATE suggestions SET status = ?, resolved_at = datetime('now')
|
||||
WHERE id = ? AND status = ?
|
||||
AND doc_id IN (SELECT id FROM documents WHERE user_id = ?)`,
|
||||
status, chi.URLParam(r, "id"), db.SuggestionStatusPending,
|
||||
@@ -591,9 +822,75 @@ func (h *Handler) setStatus(w http.ResponseWriter, r *http.Request, status strin
|
||||
httputil.ErrorJSON(w, http.StatusNotFound, "pending suggestion not found")
|
||||
return
|
||||
}
|
||||
if status == db.SuggestionStatusAccepted {
|
||||
h.plant(chi.URLParam(r, "id"), auth.UserID(r.Context()))
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// plant grows an accepted collocation into a vocabulary-garden phrase card. It
|
||||
// runs after the status write and swallows its own errors: accepting an edit is
|
||||
// the thing the writer asked for, and it must not fail — or even feel slower —
|
||||
// because a flashcard couldn't be made.
|
||||
//
|
||||
// Only collocations are planted. The other families correct *this* sentence
|
||||
// ("their" → "there", a comma, a clearer clause); a collocation is the one that
|
||||
// hands over a reusable chunk, which is the only thing worth reviewing in a week.
|
||||
func (h *Handler) plant(id, userID string) {
|
||||
var s db.Suggestion
|
||||
var contentText, docLang string
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT s.type, s.original, s.replacement, s.explanation, s.doc_id, d.content_text, d.doc_lang
|
||||
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||
WHERE s.id = ? AND d.user_id = ?`,
|
||||
id, userID,
|
||||
).Scan(&s.Type, &s.Original, &s.Replacement, &s.Explanation, &s.DocID, &contentText, &docLang)
|
||||
if err != nil {
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
log.Printf("suggestions: could not read %s for planting: %v", id, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if s.Type != db.SuggestionTypeCollocation || strings.TrimSpace(s.Replacement) == "" {
|
||||
return
|
||||
}
|
||||
// The stored text is still the pre-accept draft — the client applies the
|
||||
// replacement in the editor. Correct the sentence here so the flashcard
|
||||
// quizzes the phrasing she is keeping, not the one she just left behind.
|
||||
docID := s.DocID
|
||||
if _, err := vocab.Plant(h.DB, userID, vocab.Phrase{
|
||||
Text: s.Replacement,
|
||||
Meaning: s.Explanation,
|
||||
Example: correctedSentence(contentText, s.Original, s.Replacement),
|
||||
DocID: &docID,
|
||||
// The chunk is her own sentence, corrected — so it is in the document's
|
||||
// language, whatever the collocation pass was framed in.
|
||||
Lang: docLang,
|
||||
}); err != nil {
|
||||
log.Printf("suggestions: could not plant %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// correctedSentence returns the sentence of contentText containing original,
|
||||
// with original swapped for replacement. Returns "" when the original isn't
|
||||
// found (the draft moved on) — a card with no example still reviews, just
|
||||
// without the cloze, so there's nothing to fall back to and nothing to guess.
|
||||
func correctedSentence(contentText, original, replacement string) string {
|
||||
idx := strings.Index(contentText, original)
|
||||
if original == "" || idx < 0 {
|
||||
return ""
|
||||
}
|
||||
start := strings.LastIndexAny(contentText[:idx], ".!?\n")
|
||||
end := strings.IndexAny(contentText[idx+len(original):], ".!?\n")
|
||||
if end < 0 {
|
||||
end = len(contentText)
|
||||
} else {
|
||||
end += idx + len(original) + 1 // keep the terminator
|
||||
}
|
||||
sentence := strings.TrimSpace(contentText[start+1 : end])
|
||||
return strings.Replace(sentence, original, replacement, 1)
|
||||
}
|
||||
|
||||
// locate finds the plaintext offsets of original within contentText. Returns
|
||||
// (-1, -1) when not found; the frontend anchors by string regardless, so a miss
|
||||
// here is non-fatal.
|
||||
@@ -607,6 +904,11 @@ func locate(contentText, original string) (int, int) {
|
||||
|
||||
// normalizeType maps the model's type string onto a valid suggestion type,
|
||||
// defaulting unknown values to grammar so a stray label never trips the CHECK.
|
||||
//
|
||||
// 'translate' is absent on purpose, and stays absent even though the type now
|
||||
// exists: it is decided from the span (see language.go), never taken from the
|
||||
// model. A model that volunteers the label anyway lands on grammar here and is
|
||||
// then promoted — or not — on the evidence.
|
||||
func normalizeType(t string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(t)) {
|
||||
case db.SuggestionTypeGrammar, db.SuggestionTypePhrasing, db.SuggestionTypeIdiom, db.SuggestionTypeClarity, db.SuggestionTypeCollocation:
|
||||
|
||||
@@ -20,10 +20,17 @@ import (
|
||||
type stubClient struct {
|
||||
response string
|
||||
calls int
|
||||
// The full prompt of the most recent call, so a test can assert which
|
||||
// sentences a chunked pass actually asked about.
|
||||
lastPrompt string
|
||||
}
|
||||
|
||||
func (s *stubClient) Complete(_ context.Context, _ llm.CompletionRequest) (string, error) {
|
||||
func (s *stubClient) Complete(_ context.Context, req llm.CompletionRequest) (string, error) {
|
||||
s.calls++
|
||||
s.lastPrompt = ""
|
||||
for _, m := range req.Messages {
|
||||
s.lastPrompt += m.Content + "\n"
|
||||
}
|
||||
return s.response, nil
|
||||
}
|
||||
|
||||
@@ -64,6 +71,17 @@ func newTestServer(t *testing.T, client llm.LLMClient) (http.Handler, string, *H
|
||||
return authed, docID, h
|
||||
}
|
||||
|
||||
// setDocText rewrites the seeded document, standing in for the writer editing.
|
||||
// The grammar checkpoint only asks the model about sentences that changed since
|
||||
// it last read the document, so a test that wants a second real pass has to
|
||||
// change something first — as she always has.
|
||||
func setDocText(t *testing.T, h *Handler, docID, text string) {
|
||||
t.Helper()
|
||||
if _, err := h.DB.Exec(`UPDATE documents SET content_text = ? WHERE id = ?`, text, docID); err != nil {
|
||||
t.Fatalf("update doc text: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func do(t *testing.T, srv http.Handler, method, path, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
var r *http.Request
|
||||
@@ -183,6 +201,7 @@ func TestFickleEditsSuppressed(t *testing.T) {
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
h.Limit = llm.NewRateLimiter(0)
|
||||
setDocText(t, h, docID, `He left "early," because of the rain. The cat always have a calm face.`)
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var got []db.Suggestion
|
||||
@@ -194,6 +213,10 @@ func TestFickleEditsSuppressed(t *testing.T) {
|
||||
do(t, srv, http.MethodPost, "/suggestions/"+s.ID+"/accept", "")
|
||||
}
|
||||
|
||||
// Both edits are now in the document, which is what re-opens those sentences
|
||||
// for a second reading.
|
||||
setDocText(t, h, docID, `He left "early," due to the rain. The cat always has a calm face.`)
|
||||
|
||||
// Reversal of the first accept (note the " → ' quote churn) and a re-polish of
|
||||
// the second accept must both be dropped; only the unrelated edit survives.
|
||||
client.response = `{"suggestions":[
|
||||
@@ -314,7 +337,9 @@ func TestCollocationPassCoexists(t *testing.T) {
|
||||
t.Fatalf("collocation response should carry all three families, got %+v", got)
|
||||
}
|
||||
|
||||
// A grammar checkpoint must NOT wipe the voice or collocation flags.
|
||||
// A grammar checkpoint must NOT wipe the voice or collocation flags. She fixes
|
||||
// the flagged sentence, so its own grammar row goes and nothing replaces it.
|
||||
setDocText(t, h, docID, "I have two apples.")
|
||||
client.response = `{"suggestions":[]}`
|
||||
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
|
||||
@@ -116,4 +116,14 @@ func TestSuggestionIsolation(t *testing.T) {
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("owner accept = %d, want 204 (body: %s)", rec.Code, rec.Body)
|
||||
}
|
||||
|
||||
// That accept created a settled span, which is the other read of this table.
|
||||
// It carries originals only — but an original is a verbatim quotation of her
|
||||
// sentence, so it is the same leak as the pending list through a smaller hole.
|
||||
if got := getSettled(t, owner, docID); len(got) != 1 {
|
||||
t.Fatalf("owner should see their own settled span, got %v", got)
|
||||
}
|
||||
if got := getSettled(t, stranger, docID); len(got) != 0 {
|
||||
t.Fatalf("stranger read %d settled span(s) (leaking %q)", len(got), got[0])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Telling her language from English, well enough to label a card.
|
||||
//
|
||||
// When the checkpoint quotes a span she wrote in her own language and hands back
|
||||
// an English rendering, that is not a correction — nothing was wrong with what
|
||||
// she wrote — and it should not be filed under 'clarity'. The label is derived
|
||||
// here rather than asked of the model: a type is structural, and a model that
|
||||
// re-reasons every pass would drift between labels for the same sentence.
|
||||
//
|
||||
// The failure mode is deliberately cheap. Getting this wrong changes a card's
|
||||
// coloured pill and nothing else — the replacement, the explanation and the
|
||||
// Accept button are identical either way — so a heuristic is the right tool. It
|
||||
// is written to under-claim: a span it isn't sure about stays whatever the model
|
||||
// called it.
|
||||
//
|
||||
// The two pair families need genuinely different tests, and pretending otherwise
|
||||
// would be the bug:
|
||||
//
|
||||
// - zh is a different script. Counting Han runes is close to certain.
|
||||
// - pt-PT, fr and es share the Latin alphabet with English, where no such
|
||||
// signal exists. Those fall back to function words — the short, extremely
|
||||
// common words a sentence in that language can hardly avoid and an English
|
||||
// sentence has no reason to contain.
|
||||
|
||||
// isTranslation reports whether this edit is a rendering of one language into
|
||||
// the other, rather than a correction. Both halves must hold: the quoted span
|
||||
// reads as one language, and what Petal offers back reads as the other. The
|
||||
// second half matters — a Chinese span rewritten into different Chinese is
|
||||
// something else entirely, and Petal has no business calling it a translation.
|
||||
//
|
||||
// Which way it points follows the document (Phase 28). In an English document
|
||||
// the translate card is her language rendered into English — she reached for a
|
||||
// sentence she couldn't say yet, and Petal said it for her. In a document she
|
||||
// wrote in her own language the useful card is the mirror image: an English
|
||||
// sentence she dropped into her Portuguese, rendered into Portuguese. Asking the
|
||||
// English-document question there would label nothing, and the card would file
|
||||
// as a correction to prose that was never wrong.
|
||||
//
|
||||
// The flipped direction cannot be the same test read backwards. `readsAsEnglish`
|
||||
// is a low bar on purpose — Latin letters, not swamped by another script — which
|
||||
// every Portuguese sentence also clears, so using it on the *original* would
|
||||
// call every genuine Portuguese correction a translation. The flipped test
|
||||
// instead uses the sentence-level vote from doclang.go, where English has its
|
||||
// own marker list and has to out-evidence the pair language to win.
|
||||
func isTranslation(original, replacement, pairLang, docLang string) bool {
|
||||
if strings.TrimSpace(original) == "" || strings.TrimSpace(replacement) == "" {
|
||||
return false
|
||||
}
|
||||
if normalizeDocLang(docLang) == docLangPair {
|
||||
p := normalizePairLang(pairLang)
|
||||
return sentenceLang(original, p) == docLangEnglish &&
|
||||
sentenceLang(replacement, p) == docLangPair
|
||||
}
|
||||
return readsAsPairLang(original, pairLang) && readsAsEnglish(replacement)
|
||||
}
|
||||
|
||||
// readsAsPairLang reports whether s is predominantly in the writer's language.
|
||||
func readsAsPairLang(s, pairLang string) bool {
|
||||
switch normalizePairLang(pairLang) {
|
||||
case "zh":
|
||||
han, latin := scriptCounts(s)
|
||||
// Predominantly, not merely partly: one Chinese word inside an English
|
||||
// sentence is a vocabulary question, and the sentence around it is still
|
||||
// English prose with its own grammar to correct. Two runes is the floor
|
||||
// because a single Han character is as likely to be a stray keystroke.
|
||||
return han >= 2 && han > latin
|
||||
case "pt-PT", "fr", "es":
|
||||
return distinctMarkers(s, latinMarkers[normalizePairLang(pairLang)]) >= 2
|
||||
}
|
||||
// A pair Petal has no test for. Say no: an unlabelled card is a card that
|
||||
// reads as it did yesterday, and a wrongly-labelled one is a new defect.
|
||||
return false
|
||||
}
|
||||
|
||||
// readsAsEnglish reports whether s is English prose rather than more of her own
|
||||
// language. It is not a language identifier — it only has to separate "English"
|
||||
// from "the pair language", and it is only ever asked about text Petal itself
|
||||
// generated, so the bar is low on purpose: Latin letters present, and not
|
||||
// swamped by another script.
|
||||
func readsAsEnglish(s string) bool {
|
||||
han, latin := scriptCounts(s)
|
||||
return latin > 0 && latin > han
|
||||
}
|
||||
|
||||
// normalizePairLang folds the stored `users.pair_lang` into the codes below.
|
||||
// Empty (a document whose owner has no pair recorded) falls through to no test.
|
||||
func normalizePairLang(pairLang string) string {
|
||||
switch p := strings.ToLower(strings.TrimSpace(pairLang)); p {
|
||||
case "zh", "zh-cn", "zh-hans":
|
||||
return "zh"
|
||||
case "pt", "pt-pt":
|
||||
return "pt-PT"
|
||||
case "fr", "fr-fr":
|
||||
return "fr"
|
||||
case "es", "es-es":
|
||||
return "es"
|
||||
default:
|
||||
return p
|
||||
}
|
||||
}
|
||||
|
||||
// scriptCounts counts Han runes and ASCII letters. Everything else — digits,
|
||||
// punctuation, spaces, emoji — is ignored, so trailing 。or a stray comma
|
||||
// changes nothing.
|
||||
func scriptCounts(s string) (han, latin int) {
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case unicode.Is(unicode.Han, r):
|
||||
han++
|
||||
case r < unicode.MaxASCII && unicode.IsLetter(r):
|
||||
latin++
|
||||
}
|
||||
}
|
||||
return han, latin
|
||||
}
|
||||
|
||||
// distinctMarkers counts how many *different* marker words appear in s. Distinct
|
||||
// rather than total: "que ... que" is one writer's habit, while "eu quero" is two
|
||||
// independent pieces of evidence.
|
||||
func distinctMarkers(s string, markers map[string]bool) int {
|
||||
if len(markers) == 0 {
|
||||
return 0
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
|
||||
// Split on anything that isn't a letter, so punctuation and digits are
|
||||
// separators. Apostrophes included: French elision (j'ai, n'est) should
|
||||
// yield its parts.
|
||||
return !unicode.IsLetter(r)
|
||||
}) {
|
||||
if markers[w] {
|
||||
seen[w] = true
|
||||
}
|
||||
}
|
||||
return len(seen)
|
||||
}
|
||||
|
||||
// Function words that a sentence in each Latin pair can hardly avoid.
|
||||
//
|
||||
// Curated against English, not for coverage: every entry here is a word an
|
||||
// English sentence has essentially no reason to contain, which is why the lists
|
||||
// omit plenty of far more common words. Deliberately absent — each of them a
|
||||
// false positive waiting to happen — is anything that is *also* an English word:
|
||||
// the pan-Romance shorts (a, o, e, as, no, on, en, de, se, na, mi, son, era,
|
||||
// plus, pour, si, ma, ce, ne), Portuguese "do", Spanish "con", "ya" and "todo".
|
||||
// Dropping "con" costs the Spanish list one of its commonest words, and that is
|
||||
// the right trade — a marker that fires on English corroborates the wrong
|
||||
// answer, which is worse than a sentence Petal declines to label.
|
||||
//
|
||||
// A single marker is not enough (see readsAsPairLang), so these lists are read
|
||||
// as evidence to be corroborated rather than as a decision.
|
||||
//
|
||||
// **Curated against English is not the same as curated thinly**, and the first
|
||||
// version of these lists confused the two. Seen live 2026-07-29: "Esta manhã
|
||||
// acordei cedo e fui correr ao longo da marginal. O ar estava fresco e havia
|
||||
// poucas pessoas na rua." — unremarkable Portuguese, two marker hits, *zero*
|
||||
// English hits, and a verdict of English, because the document-level floor wants
|
||||
// three. The list was missing the ordinary machinery of the language: the
|
||||
// contractions (ao, à, num), the past tenses a diary is written in (estava,
|
||||
// havia, fomos), and the words that join two clauses (até, depois, então,
|
||||
// onde). Every one of them clears the bar above — an English sentence has no
|
||||
// reason to contain them — so their absence bought nothing and cost the verdict.
|
||||
// The floor stays at three; what changed is that three is now reachable by
|
||||
// prose rather than only by a paragraph that happens to argue with itself.
|
||||
var latinMarkers = map[string]map[string]bool{
|
||||
"fr": words(
|
||||
"je", "tu", "il", "elle", "ils", "elles", "nous", "vous", "est", "sont",
|
||||
"était", "étais", "une", "des", "les", "du", "dans", "avec", "que", "qui",
|
||||
"mais", "très", "être", "avoir", "pas", "cette", "cet", "ces", "mon",
|
||||
"mes", "notre", "votre", "leur", "aussi", "alors", "parce", "comme",
|
||||
"beaucoup", "toujours", "jamais", "quand", "bien", "chose", "temps",
|
||||
"moi", "toi", "lui", "peux", "veux", "sais", "faire", "dit", "aujourd",
|
||||
"hui", "quelque", "chez", "tout", "tous", "rien", "déjà", "encore",
|
||||
// The same gap the pt-PT list was caught with, closed by analogy rather
|
||||
// than by observation — no fr account exists yet to catch it live.
|
||||
"aux", "après", "où", "avait", "étaient", "depuis", "jusqu", "chaque",
|
||||
"autre", "même", "hier", "demain", "matin", "soir", "nôtre", "leurs",
|
||||
),
|
||||
"pt-PT": words(
|
||||
"eu", "você", "ele", "ela", "eles", "elas", "nós", "são", "uma", "os",
|
||||
"da", "dos", "das", "com", "que", "mas", "muito", "não", "meu",
|
||||
"minha", "seu", "sua", "isso", "este", "esta", "está", "estou", "quero",
|
||||
"também", "quando", "porque", "coisa", "tempo", "fazer", "sempre",
|
||||
"nunca", "bem", "obrigado", "obrigada", "gosto", "tenho", "tem", "foi",
|
||||
"ser", "ter", "mais", "já", "ainda", "aqui", "ali", "nada", "tudo",
|
||||
"todos", "para", "pela", "pelo", "sobre", "assim",
|
||||
// The contractions, which no English sentence has any use for.
|
||||
"ao", "aos", "à", "às", "num", "numa", "dum", "duma", "pelos", "pelas",
|
||||
"neste", "nesta", "disso", "deste", "desta",
|
||||
// The tenses a journal is actually written in.
|
||||
"estava", "estavam", "estão", "estamos", "havia", "houve", "era", "eram",
|
||||
"fui", "fomos", "foram", "vai", "vamos", "tinha", "tinham",
|
||||
// The joins between two clauses.
|
||||
"até", "depois", "antes", "onde", "então", "enquanto", "embora",
|
||||
"sem", "quem", "entre",
|
||||
// And the everyday determiners and time words a diary can hardly avoid.
|
||||
"nosso", "nossa", "outro", "outra", "mesmo", "mesma", "tão",
|
||||
"muitos", "muitas", "poucos", "poucas", "hoje", "ontem", "amanhã",
|
||||
),
|
||||
"es": words(
|
||||
"yo", "él", "ella", "ellos", "ellas", "nosotros", "una", "los", "las",
|
||||
"del", "que", "pero", "muy", "esto", "esta", "este", "está",
|
||||
"estoy", "quiero", "también", "cuando", "porque", "cosa", "tiempo",
|
||||
"hacer", "siempre", "nunca", "bien", "gracias", "tengo", "tiene", "fue",
|
||||
"ser", "tener", "más", "aquí", "allí", "nada", "todos",
|
||||
"para", "sobre", "así", "hola", "señor", "usted", "muchas",
|
||||
// Likewise by analogy: no es account exists yet either. "sin" and "tan"
|
||||
// stay out — both are English words, which is the one disqualification.
|
||||
"al", "después", "antes", "donde", "entonces", "mientras", "aunque",
|
||||
"estaba", "estaban", "están", "había", "hubo", "fuimos", "fueron",
|
||||
"nuestro", "nuestra", "otro", "otra", "mismo", "misma", "quién", "quien",
|
||||
"muchos", "pocas", "pocos", "hoy", "ayer", "mañana",
|
||||
),
|
||||
}
|
||||
|
||||
func words(list ...string) map[string]bool {
|
||||
out := make(map[string]bool, len(list))
|
||||
for _, w := range list {
|
||||
out[w] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
package suggestions
|
||||
|
||||
import "testing"
|
||||
|
||||
// The flagship case, and the ones next to it that must NOT become translations.
|
||||
//
|
||||
// Every case here is an ENGLISH document — the path every account was on before
|
||||
// Phase 28 — so `docLang` is left at "". The mirror image lives in
|
||||
// TestIsTranslationInAPairLanguageDocument below.
|
||||
func TestIsTranslation(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
original string
|
||||
replacement string
|
||||
pairLang string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
// The sentence from the UX review, verbatim.
|
||||
name: "whole Chinese sentence rendered into English",
|
||||
original: "我想说这句话但是不知道用英语怎么说。",
|
||||
replacement: "I want to say this but I don't know how to say it in English.",
|
||||
pairLang: "zh",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "ordinary English correction is not a translation",
|
||||
original: "She goes to market yesterday",
|
||||
replacement: "She went to the market yesterday",
|
||||
pairLang: "zh",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// One Chinese word inside English prose. The sentence around it is
|
||||
// still English with its own grammar to fix, and calling the card a
|
||||
// translation would mislabel a grammar fix.
|
||||
name: "single Chinese word inside an English sentence",
|
||||
original: "I bought a 苹果 at the store",
|
||||
replacement: "I bought an apple at the store",
|
||||
pairLang: "zh",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "a lone stray Han rune is not a sentence",
|
||||
original: "的",
|
||||
replacement: "of",
|
||||
pairLang: "zh",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// Chinese in, Chinese out: whatever this is, Petal is not translating.
|
||||
name: "Chinese rewritten as Chinese",
|
||||
original: "我想说这句话",
|
||||
replacement: "我要说这句话",
|
||||
pairLang: "zh",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// The same Chinese span, but the writer is on the French pair. Petal
|
||||
// has no business offering to translate a language she never claimed.
|
||||
name: "Chinese span on a non-zh pair",
|
||||
original: "我想说这句话但是不知道用英语怎么说。",
|
||||
replacement: "I want to say this in English.",
|
||||
pairLang: "fr",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "French sentence rendered into English",
|
||||
original: "Je ne sais pas comment le dire en anglais.",
|
||||
replacement: "I don't know how to say it in English.",
|
||||
pairLang: "fr",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "Portuguese sentence rendered into English",
|
||||
original: "Eu quero dizer isso mas não sei como.",
|
||||
replacement: "I want to say this but I don't know how.",
|
||||
pairLang: "pt-PT",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "Spanish sentence rendered into English",
|
||||
original: "Yo quiero decir esto pero no sé cómo.",
|
||||
replacement: "I want to say this but I don't know how.",
|
||||
pairLang: "es",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// A single marker is not evidence. "Que" appears in English writing
|
||||
// about other languages, in names, in quoted phrases.
|
||||
name: "one Latin marker is not enough",
|
||||
original: "The word que confused me",
|
||||
replacement: "The word que confuses me",
|
||||
pairLang: "pt-PT",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// The words most likely to sink this heuristic: English function words
|
||||
// that are also Romance function words. They are kept out of the lists
|
||||
// precisely so this sentence stays a grammar fix.
|
||||
name: "English full of pan-Romance lookalikes",
|
||||
original: "I do not know if a con man on the plus side as no era",
|
||||
replacement: "I do not know whether a con man, on the plus side, is no era",
|
||||
pairLang: "es",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "English with a borrowed French phrase stays English",
|
||||
original: "It was a pas de deux, more or less",
|
||||
replacement: "It was a pas de deux, more or less.",
|
||||
pairLang: "fr",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "empty replacement (an awareness-only finding)",
|
||||
original: "我想说这句话但是不知道用英语怎么说。",
|
||||
replacement: "",
|
||||
pairLang: "zh",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// A document whose owner has no pair recorded. No test, no label.
|
||||
name: "no pair language",
|
||||
original: "我想说这句话但是不知道用英语怎么说。",
|
||||
replacement: "I want to say this in English.",
|
||||
pairLang: "",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// An unshipped pair. Same rule: decline rather than guess.
|
||||
name: "unknown pair language",
|
||||
original: "Ich weiß nicht wie man das sagt.",
|
||||
replacement: "I don't know how to say that.",
|
||||
pairLang: "de",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := isTranslation(c.original, c.replacement, c.pairLang, ""); got != c.want {
|
||||
t.Errorf("isTranslation(%q, %q, %q, en) = %v, want %v",
|
||||
c.original, c.replacement, c.pairLang, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The mirror image (Phase 28): in a document she wrote in her own language, the
|
||||
// translate card is the English sentence rendered into her language — and the
|
||||
// English-document question, asked here, would label nothing.
|
||||
//
|
||||
// The case this file exists to pin is the third one: a genuine Portuguese
|
||||
// correction inside a Portuguese document. Reading the English-document test
|
||||
// backwards would call it a translation, because `readsAsEnglish` is a low bar
|
||||
// that Portuguese clears too. It has to stay a correction.
|
||||
func TestIsTranslationInAPairLanguageDocument(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
original string
|
||||
replacement string
|
||||
pairLang string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "English sentence rendered into Portuguese",
|
||||
original: "I want to say this but I don't know how to say it.",
|
||||
replacement: "Eu quero dizer isso mas não sei como o dizer.",
|
||||
pairLang: "pt-PT",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "English sentence rendered into Chinese",
|
||||
original: "I don't know how to say this in Chinese.",
|
||||
replacement: "我不知道这句话用中文怎么说。",
|
||||
pairLang: "zh",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// The one that matters. Portuguese in, Portuguese out, inside a
|
||||
// Portuguese document: a correction, and nothing else.
|
||||
name: "Portuguese corrected as Portuguese",
|
||||
original: "Eu quero dizer isso mas não sei como.",
|
||||
replacement: "Eu quero dizer isto mas não sei como.",
|
||||
pairLang: "pt-PT",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// And its Chinese twin, which the script test already caught.
|
||||
name: "Chinese corrected as Chinese",
|
||||
original: "我想说这句话",
|
||||
replacement: "我要说这句话",
|
||||
pairLang: "zh",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// The old direction, asked in the new document. She quoted English in
|
||||
// her Portuguese and Petal rendered it into Portuguese — which IS a
|
||||
// translation, and is the case above. This is its reverse: Portuguese
|
||||
// out of an English document that isn't one. No label.
|
||||
name: "Portuguese rendered into English is not this document's translation",
|
||||
original: "Eu quero dizer isso mas não sei como.",
|
||||
replacement: "I want to say this but I don't know how.",
|
||||
pairLang: "pt-PT",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// English prose without enough evidence to vote. Silence, not a guess.
|
||||
name: "too short to read as English",
|
||||
original: "OK",
|
||||
replacement: "Está bem, muito obrigado.",
|
||||
pairLang: "pt-PT",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "unknown pair language declines in both directions",
|
||||
original: "I don't know how to say that.",
|
||||
replacement: "Ich weiß nicht wie man das sagt.",
|
||||
pairLang: "de",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := isTranslation(c.original, c.replacement, c.pairLang, docLangPair); got != c.want {
|
||||
t.Errorf("isTranslation(%q, %q, %q, pair) = %v, want %v",
|
||||
c.original, c.replacement, c.pairLang, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// pair_lang is stored as the pack code, but a stored value has drifted before
|
||||
// (see the picker's history), so the fold is tested rather than assumed.
|
||||
func TestNormalizePairLang(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"zh": "zh", "zh-CN": "zh", "ZH": "zh",
|
||||
"pt": "pt-PT", "pt-PT": "pt-PT", "pt-pt": "pt-PT",
|
||||
"fr": "fr", "fr-FR": "fr",
|
||||
"es": "es", "es-ES": "es",
|
||||
" zh ": "zh",
|
||||
"": "",
|
||||
"de": "de",
|
||||
} {
|
||||
if got := normalizePairLang(in); got != want {
|
||||
t.Errorf("normalizePairLang(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// French elision must yield its parts, or "j'ai" and "n'est" — two of the
|
||||
// commonest shapes in the language — count for nothing.
|
||||
func TestElisionYieldsMarkers(t *testing.T) {
|
||||
if n := distinctMarkers("Je n'est pas", latinMarkers["fr"]); n < 3 {
|
||||
t.Errorf("elided French: got %d markers, want >= 3 (je, est, pas)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Distinct, not total: one word repeated is one piece of evidence.
|
||||
func TestRepeatedMarkerCountsOnce(t *testing.T) {
|
||||
if n := distinctMarkers("que que que", latinMarkers["pt-PT"]); n != 1 {
|
||||
t.Errorf("repeated marker: got %d, want 1", n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// The offline rule pack and the LLM now share the collocation family, which is
|
||||
// the point: the writer sees one rail and is never told which engine spoke. What
|
||||
// makes that safe is `source` — each pass replaces only its own rows. These tests
|
||||
// pin the two ways that could go wrong, both of which the old type-scoped DELETEs
|
||||
// would have hit.
|
||||
|
||||
// pendingOfType counts the pending rows of one family in a response body.
|
||||
func pendingOfType(got []db.Suggestion, typ string) []db.Suggestion {
|
||||
var out []db.Suggestion
|
||||
for _, s := range got {
|
||||
if s.Type == typ {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestOfflineCollocationFilesAsCollocation proves a miscollocation the rule pack
|
||||
// found is stored in the collocation family (so accepting it plants a garden
|
||||
// card, exactly as the coach's would) while still being marked as locally found.
|
||||
func TestOfflineCollocationFilesAsCollocation(t *testing.T) {
|
||||
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":0,"to":13,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"},
|
||||
{"from":20,"to":27,"original":"the the","replacement":"the","explanation":"doubled word","type":"mechanics"}
|
||||
]`)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want both findings, got %+v", got)
|
||||
}
|
||||
coll := pendingOfType(got, db.SuggestionTypeCollocation)
|
||||
if len(coll) != 1 {
|
||||
t.Fatalf("want 1 collocation, got %+v", got)
|
||||
}
|
||||
if coll[0].Source != db.SuggestionSourceLocal {
|
||||
t.Errorf("offline finding should be source=local, got %q", coll[0].Source)
|
||||
}
|
||||
if mech := pendingOfType(got, db.SuggestionTypeMechanics); len(mech) != 1 {
|
||||
t.Fatalf("want 1 mechanics finding, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnknownLocalTypeFallsBackToMechanics: a family the offline pass isn't
|
||||
// allowed to claim (or an older client sending none at all) must land in
|
||||
// mechanics. Otherwise a stray label would smuggle a row into an LLM family,
|
||||
// where nothing would ever replace it.
|
||||
func TestUnknownLocalTypeFallsBackToMechanics(t *testing.T) {
|
||||
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":0,"to":5,"original":"aaaaa","replacement":"bbbbb","explanation":"x","type":"voice"},
|
||||
{"from":6,"to":11,"original":"ccccc","replacement":"ddddd","explanation":"y"}
|
||||
]`)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want 2 findings, got %+v", got)
|
||||
}
|
||||
for _, s := range got {
|
||||
if s.Type != db.SuggestionTypeMechanics {
|
||||
t.Errorf("offline finding claimed family %q; only mechanics/collocation are allowed", s.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCoachDoesNotWipeOfflineCollocations is the collision the source column
|
||||
// exists for: the LLM collocation pass replaces the collocation family, and the
|
||||
// rule pack's share of that family has to survive it. Before `source`, running
|
||||
// the coach silently deleted every offline chunk on the page.
|
||||
func TestCoachDoesNotWipeOfflineCollocations(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"apple","replacement":"an apple","explanation":"article","type":"collocation"}
|
||||
]}`}
|
||||
srv, docID, _ := newTestServer(t, client)
|
||||
|
||||
// The seeded doc is "I has two apple." — the coach's flag anchors on "apple"
|
||||
// at [10,15], so the offline finding is given a span well clear of it. Two
|
||||
// findings fighting over the same characters is a different rule (see
|
||||
// TestOfflineCardWinsSpanCollision); this test is about the DELETE.
|
||||
postMechanics(t, srv, docID, `[
|
||||
{"from":0,"to":5,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||
]`)
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/collocation", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("collocation pass: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
var got []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
|
||||
var local, llm int
|
||||
for _, s := range pendingOfType(got, db.SuggestionTypeCollocation) {
|
||||
if s.Source == db.SuggestionSourceLocal {
|
||||
local++
|
||||
} else {
|
||||
llm++
|
||||
}
|
||||
}
|
||||
if local != 1 {
|
||||
t.Errorf("the coach wiped the offline collocation: local=%d, got %+v", local, got)
|
||||
}
|
||||
if llm != 1 {
|
||||
t.Errorf("want the coach's own flag alongside it: llm=%d, got %+v", llm, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOfflinePassReplacesItsOwnCollocations is the mirror: the rule pack
|
||||
// recomputes the whole document every run, so a chunk the current text no longer
|
||||
// warrants must go — and the coach's flags must stay. Scoping the offline DELETE
|
||||
// by type instead of source would have stranded the first row forever.
|
||||
func TestOfflinePassReplacesItsOwnCollocations(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"apple","replacement":"an apple","explanation":"article","type":"collocation"}
|
||||
]}`}
|
||||
srv, docID, _ := newTestServer(t, client)
|
||||
|
||||
// A coach flag, then an offline chunk, then a rerun that no longer finds it.
|
||||
do(t, srv, http.MethodPost, "/docs/"+docID+"/collocation", "")
|
||||
postMechanics(t, srv, docID, `[
|
||||
{"from":0,"to":13,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||
]`)
|
||||
got := postMechanics(t, srv, docID, `[]`)
|
||||
|
||||
for _, s := range got {
|
||||
if s.Source == db.SuggestionSourceLocal {
|
||||
t.Errorf("stale offline finding survived a recompute: %+v", s)
|
||||
}
|
||||
}
|
||||
if len(pendingOfType(got, db.SuggestionTypeCollocation)) != 1 {
|
||||
t.Fatalf("the coach's own flag should be untouched, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOfflineCollocationPlantsOnAccept closes the loop the family split was for:
|
||||
// a chunk the rule pack found, accepted, becomes a vocabulary-garden card — with
|
||||
// no model involved anywhere in the path.
|
||||
func TestOfflineCollocationPlantsOnAccept(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
if _, err := h.DB.Exec(
|
||||
`UPDATE documents SET content_text = ? WHERE id = ?`,
|
||||
"I had to do a decision about the job.", docID,
|
||||
); err != nil {
|
||||
t.Fatalf("set content: %v", err)
|
||||
}
|
||||
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":9,"to":22,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||
]`)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want the offline chunk, got %+v", got)
|
||||
}
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+got[0].ID+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
|
||||
cards := gardenCards(t, h)
|
||||
if len(cards) != 1 || cards[0].word != "make a decision" {
|
||||
t.Fatalf("want a planted phrase card, got %+v", cards)
|
||||
}
|
||||
// The example is the corrected sentence — the phrasing she kept, not the one
|
||||
// she just left behind.
|
||||
if cards[0].example != "I had to make a decision about the job." {
|
||||
t.Errorf("example should be the corrected sentence, got %q", cards[0].example)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOfflineCardWinsSpanCollision: the tiebreak is by engine, not by family. An
|
||||
// offline miscollocation has an exact span; the coach's overlapping flag is only
|
||||
// advisory, so it is the one that goes.
|
||||
func TestOfflineCardWinsSpanCollision(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"do a decision about","replacement":"decide about","explanation":"wordy","type":"collocation"}
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
if _, err := h.DB.Exec(
|
||||
`UPDATE documents SET content_text = ? WHERE id = ?`,
|
||||
"I had to do a decision about the job.", docID,
|
||||
); err != nil {
|
||||
t.Fatalf("set content: %v", err)
|
||||
}
|
||||
|
||||
do(t, srv, http.MethodPost, "/docs/"+docID+"/collocation", "")
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":9,"to":22,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||
]`)
|
||||
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want the overlapping coach flag dropped, got %+v", got)
|
||||
}
|
||||
if got[0].Source != db.SuggestionSourceLocal {
|
||||
t.Errorf("the exact offline card should own the span, got %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestOfflineHanziFindingStaysMechanics: a 错别字 the Chinese rule pack found —
|
||||
// both halves written in hanzi — files as an ordinary mechanics row.
|
||||
//
|
||||
// The check is worth its own test because there is a rule one layer over that
|
||||
// would plausibly claim it. `isTranslation` re-labels an edit whose original
|
||||
// reads as the writer's language and whose replacement reads as English, which
|
||||
// is exactly how a zh-pair writer's quoted Chinese becomes a 'translate' card.
|
||||
// A wrong-character fix looks like the first half of that and nothing like the
|
||||
// second: 己经 → 已经 never leaves Chinese. It must stay a tidy-up in her own
|
||||
// sentence, on the same rail as a doubled word, with no rendering-into-English
|
||||
// implied anywhere.
|
||||
func TestOfflineHanziFindingStaysMechanics(t *testing.T) {
|
||||
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":1,"to":3,"original":"己经","replacement":"已经","explanation":"已经 (already) takes 已","type":"mechanics"}
|
||||
]`)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want the one finding, got %+v", got)
|
||||
}
|
||||
if got[0].Type != db.SuggestionTypeMechanics {
|
||||
t.Errorf("hanzi fix filed as %q, want %q", got[0].Type, db.SuggestionTypeMechanics)
|
||||
}
|
||||
if got[0].Source != db.SuggestionSourceLocal {
|
||||
t.Errorf("source = %q, want %q", got[0].Source, db.SuggestionSourceLocal)
|
||||
}
|
||||
// The characters survive the round trip intact — a mangled span here would
|
||||
// replace the wrong characters in her document.
|
||||
if got[0].Original != "己经" || got[0].Replacement != "已经" {
|
||||
t.Errorf("round-tripped as %q → %q", got[0].Original, got[0].Replacement)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// seedSuggestion writes one pending suggestion against the seeded doc, after
|
||||
// replacing the doc's text so the sentence around `original` is under the test's
|
||||
// control.
|
||||
func seedSuggestion(t *testing.T, h *Handler, docID, text, sType, original, replacement, explanation string) string {
|
||||
t.Helper()
|
||||
if _, err := h.DB.Exec(`UPDATE documents SET content_text = ? WHERE id = ?`, text, docID); err != nil {
|
||||
t.Fatalf("set content: %v", err)
|
||||
}
|
||||
var id string
|
||||
err := h.DB.QueryRow(
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, status)
|
||||
VALUES (?, 0, 0, ?, ?, ?, ?, 'pending') RETURNING id`,
|
||||
docID, original, replacement, explanation, sType,
|
||||
).Scan(&id)
|
||||
if err != nil {
|
||||
t.Fatalf("seed suggestion: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
type card struct {
|
||||
word, definition, example string
|
||||
interval int
|
||||
}
|
||||
|
||||
func gardenCards(t *testing.T, h *Handler) []card {
|
||||
t.Helper()
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT word, definition, example, interval_days FROM vocab_words WHERE user_id = ? ORDER BY word`,
|
||||
db.LocalUserID,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("read garden: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []card
|
||||
for rows.Next() {
|
||||
var c card
|
||||
if err := rows.Scan(&c.word, &c.definition, &c.example, &c.interval); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestAcceptedCollocationIsPlanted walks the whole hand-over: a collocation the
|
||||
// writer accepts becomes a phrase card whose example is the *corrected*
|
||||
// sentence, so the flashcard quizzes the phrasing she kept.
|
||||
func TestAcceptedCollocationIsPlanted(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
id := seedSuggestion(t, h, docID,
|
||||
"Yesterday was hard. I had to do a decision about the job. Then I slept.",
|
||||
db.SuggestionTypeCollocation, "do a decision", "make a decision",
|
||||
"English pairs “make” with “decision”.")
|
||||
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept: got %d, want 204", rec.Code)
|
||||
}
|
||||
|
||||
cards := gardenCards(t, h)
|
||||
if len(cards) != 1 {
|
||||
t.Fatalf("garden has %d cards, want 1: %+v", len(cards), cards)
|
||||
}
|
||||
got := cards[0]
|
||||
if got.word != "make a decision" {
|
||||
t.Errorf("word = %q, want %q", got.word, "make a decision")
|
||||
}
|
||||
if got.example != "I had to make a decision about the job." {
|
||||
t.Errorf("example = %q — want the corrected sentence, bounded to its own sentence", got.example)
|
||||
}
|
||||
if got.definition != "English pairs “make” with “decision”." {
|
||||
t.Errorf("definition = %q, want the explanation", got.definition)
|
||||
}
|
||||
if got.interval != 1 {
|
||||
t.Errorf("interval_days = %d, want 1 (due tomorrow, like a fresh capture)", got.interval)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOnlyCollocationsArePlanted: the other families correct this sentence and
|
||||
// hand over nothing reusable. A dismissed collocation is not a lesson either.
|
||||
func TestOnlyCollocationsArePlanted(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
|
||||
grammar := seedSuggestion(t, h, docID, "I has two apples.",
|
||||
db.SuggestionTypeGrammar, "I has", "I have", "Subject–verb agreement.")
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+grammar+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept grammar: got %d", rec.Code)
|
||||
}
|
||||
|
||||
dismissed := seedSuggestion(t, h, docID, "We must take a photo of it.",
|
||||
db.SuggestionTypeCollocation, "do a photo", "take a photo", "Photos are taken.")
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+dismissed+"/dismiss", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("dismiss: got %d", rec.Code)
|
||||
}
|
||||
|
||||
if cards := gardenCards(t, h); len(cards) != 0 {
|
||||
t.Fatalf("garden grew %d card(s) from a grammar fix and a dismissal: %+v", len(cards), cards)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPlantingIsIdempotentAndNeverResets: accepting the same chunk again is
|
||||
// evidence it's still being learned — the worst possible response is to wipe the
|
||||
// card's first context and the schedule it has been climbing.
|
||||
func TestPlantingIsIdempotentAndNeverResets(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
first := seedSuggestion(t, h, docID, "I had to do a decision.",
|
||||
db.SuggestionTypeCollocation, "do a decision", "make a decision", "First explanation.")
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+first+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept: got %d", rec.Code)
|
||||
}
|
||||
// The card climbs a little.
|
||||
if _, err := h.DB.Exec(
|
||||
`UPDATE vocab_words SET reps = 3, interval_days = 7 WHERE user_id = ? AND word = 'make a decision'`,
|
||||
db.LocalUserID,
|
||||
); err != nil {
|
||||
t.Fatalf("advance card: %v", err)
|
||||
}
|
||||
|
||||
second := seedSuggestion(t, h, docID, "Later I must do a decision again.",
|
||||
db.SuggestionTypeCollocation, "do a decision", "make a decision", "Second explanation.")
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+second+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept again: got %d", rec.Code)
|
||||
}
|
||||
|
||||
cards := gardenCards(t, h)
|
||||
if len(cards) != 1 {
|
||||
t.Fatalf("garden has %d cards, want 1 (one chunk, one card)", len(cards))
|
||||
}
|
||||
if cards[0].definition != "First explanation." {
|
||||
t.Errorf("definition = %q — the existing card should win", cards[0].definition)
|
||||
}
|
||||
if cards[0].example != "I had to make a decision." {
|
||||
t.Errorf("example = %q — the first context should survive", cards[0].example)
|
||||
}
|
||||
if cards[0].interval != 7 {
|
||||
t.Errorf("interval_days = %d, want 7 — progress must not be reset", cards[0].interval)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSentenceRewriteIsNotAPhraseCard: a "collocation" long enough to be a
|
||||
// rewritten sentence makes a miserable flashcard, so it is dropped rather than
|
||||
// planted — and the accept still succeeds.
|
||||
func TestSentenceRewriteIsNotAPhraseCard(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
long := "I would like to take this opportunity to thank you for everything"
|
||||
id := seedSuggestion(t, h, docID, "I want thank you for everything.",
|
||||
db.SuggestionTypeCollocation, "I want thank you for everything", long, "More natural.")
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept: got %d, want 204 — a skipped card must never fail the accept", rec.Code)
|
||||
}
|
||||
if cards := gardenCards(t, h); len(cards) != 0 {
|
||||
t.Fatalf("planted a sentence as a phrase card: %+v", cards)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectedSentence(t *testing.T) {
|
||||
const text = "One thing. I had to do a decision fast! Another thing."
|
||||
cases := []struct {
|
||||
name, original, replacement, want string
|
||||
}{
|
||||
{"bounded to its sentence", "do a decision", "make a decision", "I had to make a decision fast!"},
|
||||
{"original no longer present", "do a choice", "make a choice", ""},
|
||||
{"empty original", "", "make a decision", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := correctedSentence(text, tc.original, tc.replacement); got != tc.want {
|
||||
t.Errorf("correctedSentence = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A document with no terminator at all is one sentence, and still works.
|
||||
if got := correctedSentence("i had to do a decision", "do a decision", "make a decision"); got != "i had to make a decision" {
|
||||
t.Errorf("unterminated doc: got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPlantedPhraseCarriesTheDocumentLanguage: a chunk planted out of a document
|
||||
// written in her own language is a card in that language. The collocation pass
|
||||
// itself deliberately did not flip in Phase 28 — its prompt is per-language
|
||||
// knowledge, not framing — but the phrase it hands over is still lifted from her
|
||||
// prose, so the card has to know what language that prose was in or the garden
|
||||
// will read it aloud in the wrong voice.
|
||||
func TestPlantedPhraseCarriesTheDocumentLanguage(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
if _, err := h.DB.Exec(`UPDATE documents SET doc_lang = 'pair' WHERE id = ?`, docID); err != nil {
|
||||
t.Fatalf("set doc_lang: %v", err)
|
||||
}
|
||||
id := seedSuggestion(t, h, docID,
|
||||
"Ontem foi difícil. Tive de tomar uma decisão sobre o trabalho.",
|
||||
db.SuggestionTypeCollocation, "tomar uma decisão", "tomar uma decisão",
|
||||
"Em português diz-se “tomar” uma decisão.")
|
||||
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept: got %d, want 204", rec.Code)
|
||||
}
|
||||
|
||||
var lang string
|
||||
if err := h.DB.QueryRow(
|
||||
`SELECT lang FROM vocab_words WHERE user_id = ? AND word = ?`,
|
||||
db.LocalUserID, "tomar uma decisão",
|
||||
).Scan(&lang); err != nil {
|
||||
t.Fatalf("read planted card: %v", err)
|
||||
}
|
||||
if lang != "pair" {
|
||||
t.Fatalf("planted card lang = %q, want pair", lang)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPlantedPhraseOnAnEnglishDocumentIsUnchanged is the other half, and the one
|
||||
// every account is on today: an English document plants an English card, and the
|
||||
// empty backfill and 'en' both mean that.
|
||||
func TestPlantedPhraseOnAnEnglishDocumentIsUnchanged(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
id := seedSuggestion(t, h, docID, "I had to do a decision about the job.",
|
||||
db.SuggestionTypeCollocation, "do a decision", "make a decision",
|
||||
"English pairs “make” with “decision”.")
|
||||
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept: got %d, want 204", rec.Code)
|
||||
}
|
||||
|
||||
var lang string
|
||||
if err := h.DB.QueryRow(
|
||||
`SELECT lang FROM vocab_words WHERE user_id = ? AND word = ?`,
|
||||
db.LocalUserID, "make a decision",
|
||||
).Scan(&lang); err != nil {
|
||||
t.Fatalf("read planted card: %v", err)
|
||||
}
|
||||
if lang == "pair" {
|
||||
t.Fatalf("planted card lang = %q on an English document, want en or empty", lang)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
)
|
||||
|
||||
// Reconciliation replaces the old "delete the family, insert the new batch"
|
||||
// shape of every pass. A suggestion the pass proposes again is the *same*
|
||||
// suggestion: it keeps its row, and therefore its id, its created_at and — most
|
||||
// visibly — the explanation it was first given. The model re-words its reasoning
|
||||
// every time it is asked, so re-inserting meant one unchanged mistake carried
|
||||
// three different explanations in a single sitting.
|
||||
//
|
||||
// The id is what the frontend keys its cards on, so a stable id is also what
|
||||
// keeps the rail from emptying and refilling, a card from collapsing mid-read,
|
||||
// and the arrival chime from re-firing for advice she has already seen.
|
||||
|
||||
// pendingRow is the part of an existing pending suggestion reconciliation cares
|
||||
// about.
|
||||
type pendingRow struct {
|
||||
id string
|
||||
original string
|
||||
replacement string
|
||||
chunkHash string
|
||||
from int
|
||||
}
|
||||
|
||||
// loadPending reads the pending rows a pass owns. `where` is the pass's own
|
||||
// scoping clause (by source, and for the model passes by family) — the same
|
||||
// fragment that used to scope its DELETE.
|
||||
func loadPending(tx *sql.Tx, docID, where string) ([]pendingRow, error) {
|
||||
rows, err := tx.Query(
|
||||
`SELECT id, original, replacement, chunk_hash, from_pos FROM suggestions
|
||||
WHERE doc_id = ? AND status = ? AND `+where,
|
||||
docID, db.SuggestionStatusPending,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []pendingRow
|
||||
for rows.Next() {
|
||||
var r pendingRow
|
||||
if err := rows.Scan(&r.id, &r.original, &r.replacement, &r.chunkHash, &r.from); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// editKey identifies an edit by what it proposes, not where: "this exact change
|
||||
// to this exact text". Normalized like the suppression comparisons, so the
|
||||
// editor's quote rewriting and a reflowed paragraph don't read as a new edit.
|
||||
func editKey(original, replacement string) string {
|
||||
return normalizeForDedup(original) + "\x00" + normalizeForDedup(replacement)
|
||||
}
|
||||
|
||||
// editIndex matches freshly proposed edits against the rows already standing.
|
||||
type editIndex struct {
|
||||
rows []pendingRow
|
||||
used []bool
|
||||
byKey map[string][]int
|
||||
}
|
||||
|
||||
func indexByEdit(rows []pendingRow) *editIndex {
|
||||
idx := &editIndex{rows: rows, used: make([]bool, len(rows)), byKey: map[string][]int{}}
|
||||
for i, r := range rows {
|
||||
k := editKey(r.original, r.replacement)
|
||||
idx.byKey[k] = append(idx.byKey[k], i)
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// take claims the standing row for this edit, if there is one. When a document
|
||||
// repeats the same mistake, `near` (the fresh span's start) picks the closest
|
||||
// standing row, so two identical cards keep their own identities instead of
|
||||
// trading them whenever the text between them grows.
|
||||
func (i *editIndex) take(original, replacement string, near int) (pendingRow, bool) {
|
||||
best, bestDist := -1, 0
|
||||
for _, n := range i.byKey[editKey(original, replacement)] {
|
||||
if i.used[n] {
|
||||
continue
|
||||
}
|
||||
d := i.rows[n].from - near
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
if best < 0 || d < bestDist {
|
||||
best, bestDist = n, d
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return pendingRow{}, false
|
||||
}
|
||||
i.used[best] = true
|
||||
return i.rows[best], true
|
||||
}
|
||||
|
||||
// reposition updates the advisory offsets (and the sentence a row belongs to)
|
||||
// without touching anything the writer can see. The frontend re-anchors by
|
||||
// string at render time, so these only matter for the local-vs-model span
|
||||
// arbitration in dedupeSpans.
|
||||
func reposition(tx *sql.Tx, row pendingRow, from, to int, chunkHash string) error {
|
||||
if row.from == from && row.chunkHash == chunkHash {
|
||||
return nil
|
||||
}
|
||||
_, err := tx.Exec(
|
||||
`UPDATE suggestions SET from_pos = ?, to_pos = ?, chunk_hash = ? WHERE id = ?`,
|
||||
from, to, chunkHash, row.id,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// reconcilePending brings a model pass's family in line with what it just
|
||||
// proposed, sentence by sentence:
|
||||
//
|
||||
// - A row on a sentence this pass didn't ask about is kept untouched — that
|
||||
// is the whole point of chunking. Only its offsets are refreshed.
|
||||
// - A row on a sentence that no longer exists in the document is dropped: she
|
||||
// rewrote or deleted it.
|
||||
// - A row on a sentence the pass *did* ask about survives only if the model
|
||||
// proposed the same edit again, in which case it keeps its identity.
|
||||
//
|
||||
// `fresh` names the sentences the model was asked about (nil when it wasn't
|
||||
// called at all). inPlayAll marks the whole-document passes — voice and the
|
||||
// collocation coach — where every row is up for re-proposal because the model
|
||||
// just re-read everything.
|
||||
//
|
||||
// `pairLang` is the writer's own language and `docLang` this document's language
|
||||
// verdict; between them they type a finding that turns out to be one language
|
||||
// rendered into the other, in whichever direction this document makes useful
|
||||
// (see language.go).
|
||||
func (h *Handler) reconcilePending(
|
||||
docID, contentText, pairLang, docLang string,
|
||||
raw []llm.RawSuggestion,
|
||||
scope pendingScope,
|
||||
chunks, fresh []chunk,
|
||||
inPlayAll bool,
|
||||
) error {
|
||||
tx, err := h.DB.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
existing, err := loadPending(tx, docID, scope.deleteWhere)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
present := hashSet(chunks)
|
||||
asked := hashSet(fresh)
|
||||
modelRan := inPlayAll || fresh != nil
|
||||
|
||||
// Sentences to hand back to the model next time, because a row we were
|
||||
// caching on them turned out to be unanchorable (see below).
|
||||
reopen := map[string]bool{}
|
||||
|
||||
var inPlay []pendingRow
|
||||
for _, r := range existing {
|
||||
switch {
|
||||
// A row whose sentence we can't name is never cached — it is re-examined
|
||||
// whenever the model speaks, and left alone when it doesn't.
|
||||
case inPlayAll, r.chunkHash == "" && modelRan, asked[r.chunkHash]:
|
||||
inPlay = append(inPlay, r)
|
||||
case r.chunkHash != "" && !present[r.chunkHash]:
|
||||
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, r.id); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
// Untouched sentence: keep the card exactly as she last saw it.
|
||||
from, to := locate(contentText, r.original)
|
||||
if from < 0 {
|
||||
// The sentence is unchanged in substance but the quoted span no
|
||||
// longer matches byte for byte — a quote mark the editor rewrote
|
||||
// inside it, say. The frontend anchors by that string, so this card
|
||||
// can't be shown; drop it and let the sentence be read again rather
|
||||
// than cache advice nobody can see.
|
||||
reopen[r.chunkHash] = true
|
||||
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, r.id); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := reposition(tx, r, from, to, r.chunkHash); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
for h := range reopen {
|
||||
delete(present, h)
|
||||
}
|
||||
|
||||
sup, err := buildSuppressor(tx, docID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
index := indexByEdit(inPlay)
|
||||
kept := make(map[string]bool, len(inPlay))
|
||||
for _, s := range raw {
|
||||
if sup.suppressed(s.Original, s.Replacement) {
|
||||
continue
|
||||
}
|
||||
from, to := locate(contentText, s.Original)
|
||||
// Attribute the finding to a sentence the model was actually shown before
|
||||
// falling back to the whole document: a short span ("the the") can occur in
|
||||
// two sentences, and crediting it to the cached one would drop it as advice
|
||||
// we already have.
|
||||
hash := chunkFor(s.Original, fresh)
|
||||
if hash == "" {
|
||||
hash = chunkFor(s.Original, chunks)
|
||||
}
|
||||
// A sentence we didn't ask about already has whatever advice it deserves.
|
||||
// The model can't normally quote one — it was only shown the delta — but if
|
||||
// it wanders there anyway, the cached card stands rather than gaining a
|
||||
// twin.
|
||||
if !inPlayAll && hash != "" && present[hash] && !asked[hash] {
|
||||
continue
|
||||
}
|
||||
if row, ok := index.take(s.Original, s.Replacement, from); ok {
|
||||
kept[row.id] = true
|
||||
if err := reposition(tx, row, from, to, hash); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
// A pass with a forced type owns its family outright and is never asked
|
||||
// about translation: voice reads whole paragraphs for tone, and the
|
||||
// collocation coach is about English word pairings. Only the open-typed
|
||||
// grammar checkpoint can turn out to have been handed her own language.
|
||||
typ := scope.forceType
|
||||
if typ == "" {
|
||||
typ = normalizeType(s.Type)
|
||||
if isTranslation(s.Original, s.Replacement, pairLang, docLang) {
|
||||
typ = db.SuggestionTypeTranslate
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, source, chunk_hash)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
docID, from, to, s.Original, s.Replacement, s.Explanation, typ, db.SuggestionSourceLLM, hash,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Asked about and not proposed again: the model has changed its mind, or she
|
||||
// has fixed it.
|
||||
for _, r := range inPlay {
|
||||
if kept[r.id] {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, r.id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Record the sentences this family has now read. Every sentence still in the
|
||||
// document has been read by *some* pass: the ones just asked about now, the
|
||||
// rest in an earlier round.
|
||||
if scope.chunked {
|
||||
if _, err := tx.Exec(
|
||||
`DELETE FROM checked_chunks WHERE doc_id = ? AND family = ?`, docID, scope.family,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
for h := range present {
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO checked_chunks (doc_id, family, hash) VALUES (?, ?, ?)`,
|
||||
docID, scope.family, h,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// checkedChunks loads the sentences a family read on its last pass.
|
||||
func (h *Handler) checkedChunks(docID, family string) (map[string]bool, error) {
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT hash FROM checked_chunks WHERE doc_id = ? AND family = ?`, docID, family,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var hash string
|
||||
if err := rows.Scan(&hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[hash] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -69,7 +69,7 @@ func (h *Handler) rewrite(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
out, err := llm.RunRewrite(r.Context(), h.Client, text, body.Style)
|
||||
if err != nil {
|
||||
httputil.ErrorJSON(w, http.StatusBadGateway, "rewrite failed: "+err.Error())
|
||||
httputil.UpstreamError(w, "rewrite", err)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// The settled endpoint exists for the offline half of the loop. The rule pack
|
||||
// detects from the text alone, 250 ms after a keystroke, and has no memory
|
||||
// between runs — so without the document's record of what she has already
|
||||
// answered, a dismissed finding is re-detected and re-rendered on the next
|
||||
// keystroke, and stays there for as long as the server can't be reached.
|
||||
|
||||
func getSettled(t *testing.T, srv http.Handler, docID string) []string {
|
||||
t.Helper()
|
||||
rec := do(t, srv, http.MethodGet, "/docs/"+docID+"/settled", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("settled: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
var out struct {
|
||||
Originals []string `json:"originals"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
return out.Originals
|
||||
}
|
||||
|
||||
func contains(list []string, want string) bool {
|
||||
for _, s := range list {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TestSettledListsActionedSpans proves the endpoint reports exactly the spans the
|
||||
// suppressor would drop: accepted and dismissed, never pending. A pending row
|
||||
// leaking in would be the damaging direction — the client would hide a card she
|
||||
// has never been shown an answer to.
|
||||
func TestSettledListsActionedSpans(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"},
|
||||
{"original":"two apple","replacement":"two apples","explanation":"plural","type":"grammar"}
|
||||
]}`}
|
||||
srv, docID, _ := newTestServer(t, client)
|
||||
|
||||
if got := getSettled(t, srv, docID); len(got) != 0 {
|
||||
t.Fatalf("nothing actioned yet, got %v", got)
|
||||
}
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var got []db.Suggestion
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &got)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("first pass: want 2, got %d", len(got))
|
||||
}
|
||||
|
||||
// One accepted, one still pending: only the accepted span is settled.
|
||||
do(t, srv, http.MethodPost, "/suggestions/"+got[0].ID+"/accept", "")
|
||||
settled := getSettled(t, srv, docID)
|
||||
if len(settled) != 1 || settled[0] != got[0].Original {
|
||||
t.Fatalf("want just %q settled, got %v", got[0].Original, settled)
|
||||
}
|
||||
|
||||
// A dismissal settles a span just as an accept does — the whole point of the
|
||||
// item: "you already decided about this one" doesn't mean "you agreed".
|
||||
do(t, srv, http.MethodPost, "/suggestions/"+got[1].ID+"/dismiss", "")
|
||||
settled = getSettled(t, srv, docID)
|
||||
if len(settled) != 2 || !contains(settled, got[1].Original) {
|
||||
t.Fatalf("dismissed span missing from %v", settled)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSettledNormalizesAndDedupes proves the payload is normalized server-side
|
||||
// and collapsed. The client compares its freshly-detected findings against these
|
||||
// strings, so the two sides have to agree on what "the same span" is — the
|
||||
// editor's quote churn is the case that breaks a byte-exact match, and it is why
|
||||
// normalizeForDedup exists at all.
|
||||
func TestSettledNormalizesAndDedupes(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
|
||||
// The same span twice, differing only in quote style and line breaks — one
|
||||
// accepted, one dismissed. Distinct rows; one settled span.
|
||||
a := seedSuggestion(t, h, docID, "text", db.SuggestionTypeGrammar,
|
||||
"She said \"hello\"\n to me", "She said 'hello' to me", "quotes")
|
||||
b := seedSuggestion(t, h, docID, "text", db.SuggestionTypeGrammar,
|
||||
"She said “hello” to me", "She said 'hello' to me", "quotes")
|
||||
do(t, srv, http.MethodPost, "/suggestions/"+a+"/accept", "")
|
||||
do(t, srv, http.MethodPost, "/suggestions/"+b+"/dismiss", "")
|
||||
|
||||
settled := getSettled(t, srv, docID)
|
||||
if len(settled) != 1 {
|
||||
t.Fatalf("two spellings of one span should collapse to one, got %v", settled)
|
||||
}
|
||||
if want := "She said 'hello' to me"; settled[0] != want {
|
||||
t.Fatalf("settled[0] = %q, want normalized %q", settled[0], want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNormalizeMatchesTheClient is the Go half of a pair. Every case here also
|
||||
// appears in web/src/lib/settled.test.ts, asserted against the TypeScript
|
||||
// reimplementation of this function. The two are compared across a network
|
||||
// boundary — the server normalizes what it sends, the client normalizes what it
|
||||
// checks against it — so they have to fold the same characters the same way, and
|
||||
// nothing but a shared list of cases can say so. Add to both or neither.
|
||||
func TestNormalizeMatchesTheClient(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"She said “hello”", "She said 'hello'"},
|
||||
{"She said \"hello\"", "She said 'hello'"},
|
||||
{"it‘s", "it's"},
|
||||
{"it’s", "it's"},
|
||||
{"`code´", "'code'"},
|
||||
{" a apple\n here ", "a apple here"},
|
||||
{"a\tapple", "a apple"},
|
||||
{" \n ", ""},
|
||||
{"我想说这句话", "我想说这句话"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := normalizeForDedup(c.in); got != c.want {
|
||||
t.Errorf("normalizeForDedup(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSettledEmptyIsAList guards the shape rather than the content: the client
|
||||
// spreads this array into its settled set, and a null would throw there. Go
|
||||
// marshals a nil slice as null, so this is one `[]string{}` away from breaking.
|
||||
func TestSettledEmptyIsAList(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[]}`}
|
||||
srv, docID, _ := newTestServer(t, client)
|
||||
|
||||
rec := do(t, srv, http.MethodGet, "/docs/"+docID+"/settled", "")
|
||||
if body := rec.Body.String(); body != "{\"originals\":[]}\n" && body != "{\"originals\":[]}" {
|
||||
t.Fatalf("empty settled body = %q, want an empty list", body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
)
|
||||
|
||||
// byOriginal indexes a pending set by the text each card flags.
|
||||
func byOriginal(in []db.Suggestion) map[string]db.Suggestion {
|
||||
out := map[string]db.Suggestion{}
|
||||
for _, s := range in {
|
||||
out[s.Original] = s
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestUntouchedSentencesKeepTheirCards is the heart of the stability work: she
|
||||
// edits one sentence, and the cards on every other sentence stay exactly as they
|
||||
// were — same id (so the rail keeps the card instead of remounting it), same
|
||||
// explanation (the model re-words its reasoning every time it is asked, and one
|
||||
// unchanged mistake used to carry three different explanations in a sitting).
|
||||
// The model is only asked about the sentence that changed.
|
||||
func TestUntouchedSentencesKeepTheirCards(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"I has two apple","replacement":"I have two apples","explanation":"first wording","type":"grammar"},
|
||||
{"original":"She go to market","replacement":"She goes to market","explanation":"agreement","type":"grammar"}
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
h.Limit = llm.NewRateLimiter(0)
|
||||
setDocText(t, h, docID, "I has two apple. She go to market yesterday.")
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var first []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &first); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(first) != 2 {
|
||||
t.Fatalf("first pass: want 2, got %d: %+v", len(first), first)
|
||||
}
|
||||
kept := byOriginal(first)["I has two apple"]
|
||||
|
||||
// She fixes only the second sentence. The model, asked again, re-words its
|
||||
// reasoning about the first — which it must never get the chance to do.
|
||||
setDocText(t, h, docID, "I has two apple. She goes to market yesterday.")
|
||||
client.response = `{"suggestions":[
|
||||
{"original":"I has two apple","replacement":"I have two apples","explanation":"REWORDED","type":"grammar"}
|
||||
]}`
|
||||
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var second []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &second); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
|
||||
if strings.Contains(client.lastPrompt, "I has two apple") {
|
||||
t.Fatalf("untouched sentence was sent to the model:\n%s", client.lastPrompt)
|
||||
}
|
||||
if !strings.Contains(client.lastPrompt, "She goes to market") {
|
||||
t.Fatalf("edited sentence was not sent to the model:\n%s", client.lastPrompt)
|
||||
}
|
||||
|
||||
now := byOriginal(second)["I has two apple"]
|
||||
if now.ID != kept.ID {
|
||||
t.Fatalf("card was remounted: id %q became %q", kept.ID, now.ID)
|
||||
}
|
||||
if now.Explanation != "first wording" {
|
||||
t.Fatalf("explanation drifted: %q", now.Explanation)
|
||||
}
|
||||
// The fixed sentence's card is gone, and the model's stray re-proposal for the
|
||||
// cached sentence did not become a second card.
|
||||
if len(second) != 1 {
|
||||
t.Fatalf("want exactly one card left, got %d: %+v", len(second), second)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnchangedDocumentSkipsTheModel proves a check with nothing new to read
|
||||
// costs nothing: no model call, and every card left standing untouched. This is
|
||||
// the doc-open and tone-less re-check path.
|
||||
func TestUnchangedDocumentSkipsTheModel(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"}
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
h.Limit = llm.NewRateLimiter(0)
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var first []db.Suggestion
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &first)
|
||||
if len(first) != 1 || client.calls != 1 {
|
||||
t.Fatalf("first pass: %d cards, %d calls", len(first), client.calls)
|
||||
}
|
||||
|
||||
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var second []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &second); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if client.calls != 1 {
|
||||
t.Fatalf("re-checking an unedited document called the model %d times", client.calls)
|
||||
}
|
||||
if len(second) != 1 || second[0].ID != first[0].ID {
|
||||
t.Fatalf("card did not survive an idle re-check: %+v", second)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeletedSentenceDropsItsCard covers the other half of the skip path: she
|
||||
// removes a flagged sentence outright, so nothing changed that the model could
|
||||
// be asked about — but its card must still go.
|
||||
func TestDeletedSentenceDropsItsCard(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"}
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
h.Limit = llm.NewRateLimiter(0)
|
||||
|
||||
do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
setDocText(t, h, docID, "")
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var got []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("card outlived its sentence: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestToneChangeReopensEverySentence: the checkpoint's advice is written for the
|
||||
// document's tone, so switching from a journal to an academic essay has to
|
||||
// re-read sentences that haven't changed a character.
|
||||
func TestToneChangeReopensEverySentence(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"}
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
h.Limit = llm.NewRateLimiter(0)
|
||||
|
||||
do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if _, err := h.DB.Exec(`UPDATE documents SET tone = 'academic' WHERE id = ?`, docID); err != nil {
|
||||
t.Fatalf("set tone: %v", err)
|
||||
}
|
||||
do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
|
||||
if client.calls != 2 {
|
||||
t.Fatalf("tone change did not re-read the document: %d model calls", client.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMechanicsFindingsKeepTheirRows: the rule pack re-runs 250 ms after every
|
||||
// keystroke. A finding it still reports must keep its row, or the rail would
|
||||
// remount several times a sentence — collapsing a card she has open, and
|
||||
// re-firing the arrival chime for advice she is already reading.
|
||||
func TestMechanicsFindingsKeepTheirRows(t *testing.T) {
|
||||
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
body := `{"findings":[
|
||||
{"from":0,"to":5,"original":"I has","replacement":"I have","explanation":"agreement"},
|
||||
{"from":6,"to":15,"original":"two apple","replacement":"two apples","explanation":"plural"}
|
||||
]}`
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/mechanics", body)
|
||||
var first []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &first); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(first) != 2 {
|
||||
t.Fatalf("want 2 rows, got %d", len(first))
|
||||
}
|
||||
|
||||
// She types elsewhere: same findings, shifted spans, one of them now fixed.
|
||||
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/mechanics", `{"findings":[
|
||||
{"from":20,"to":25,"original":"I has","replacement":"I have","explanation":"agreement"}
|
||||
]}`)
|
||||
var second []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &second); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(second) != 1 {
|
||||
t.Fatalf("want 1 row, got %d: %+v", len(second), second)
|
||||
}
|
||||
if second[0].ID != byOriginal(first)["I has"].ID {
|
||||
t.Fatalf("surviving finding was given a new identity: %+v", second[0])
|
||||
}
|
||||
if second[0].FromPos != 20 {
|
||||
t.Fatalf("span did not follow the text: %+v", second[0])
|
||||
}
|
||||
}
|
||||
@@ -17,23 +17,44 @@ type translateResponse struct {
|
||||
Translation string `json:"translation"`
|
||||
}
|
||||
|
||||
// translate renders a suggestion's English explanation into Simplified Chinese
|
||||
// for the Ask Petal bubble, so the ESL reader sees the "why" in her first
|
||||
// language instead of a second copy of the same English text. The explanation is
|
||||
// loaded server-side from the suggestion id (scoped to the local user) and never
|
||||
// trusted from the client, mirroring chat (spec Note #10).
|
||||
// translate renders a suggestion's explanation into the other half of the
|
||||
// writer's pair for the Ask Petal bubble, so she sees the "why" in the language
|
||||
// she reads most easily instead of a second copy of the same text. The
|
||||
// explanation is loaded server-side from the suggestion id (scoped to the
|
||||
// caller) and never trusted from the client, mirroring chat (spec Note #10).
|
||||
//
|
||||
// Which language it renders into cannot be assumed (Phase 28). Before that phase
|
||||
// every explanation was English and every rendering went into her language, so
|
||||
// "the pair language" was a safe constant. Now the explanation's language is a
|
||||
// decision — `targetFor`, from the document's verdict and her direction — and
|
||||
// this endpoint has to read the same decision back, or it round-trips Portuguese
|
||||
// into Portuguese and calls it a translation.
|
||||
//
|
||||
// So: render into whichever half the explanation is NOT already in — and that
|
||||
// is the whole rule, in both directions. When it first shipped this endpoint
|
||||
// answered "" for a Portuguese explanation on the reasoning that an English
|
||||
// rendering she hadn't asked for was noise. It was reported as the opposite: a
|
||||
// writer whose pair is Portuguese and English, learning English, met a
|
||||
// Portuguese card with a Portuguese bubble under it and no English anywhere on
|
||||
// the card. The tap is the one place the other language was promised, and the
|
||||
// half she is *practising* is exactly the half worth a tap.
|
||||
//
|
||||
// The bubble sits directly beneath the explanation inside the card, so between
|
||||
// the two the writer always has both languages, whichever way round the document
|
||||
// put them.
|
||||
func (h *Handler) translate(w http.ResponseWriter, r *http.Request) {
|
||||
sugID := chi.URLParam(r, "id")
|
||||
|
||||
var explanation, pairLang string
|
||||
var explanation, pairLang, direction, docLang string
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT s.explanation, COALESCE(u.pair_lang, '')
|
||||
`SELECT s.explanation, COALESCE(u.pair_lang, ''),
|
||||
COALESCE(u.direction, ''), d.doc_lang
|
||||
FROM suggestions s
|
||||
JOIN documents d ON d.id = s.doc_id
|
||||
JOIN users u ON u.id = d.user_id
|
||||
WHERE s.id = ? AND d.user_id = ?`,
|
||||
sugID, auth.UserID(r.Context()),
|
||||
).Scan(&explanation, &pairLang)
|
||||
).Scan(&explanation, &pairLang, &direction, &docLang)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusNotFound, "suggestion not found")
|
||||
return
|
||||
@@ -49,9 +70,22 @@ func (h *Handler) translate(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
out, err := llm.RunTranslate(r.Context(), h.Client, explanation, llm.LangFor(pairLang))
|
||||
// The explanation's own language, recovered from the same rule that chose it
|
||||
// when the card was written. A card written before this phase — or on a
|
||||
// document whose verdict has since flipped — is read as whatever the rule says
|
||||
// today; the alternative is a language column on every suggestion row, and the
|
||||
// cost of being wrong is one bubble seeded in the language it was already in.
|
||||
target := targetFor(pairLang, direction, docLang)
|
||||
from, to := target.Explain, target.Pair
|
||||
if from.Code == to.Code {
|
||||
// The explanation is already in her language, so the half this tap has to
|
||||
// reach is the other one: the English she is practising.
|
||||
to = llm.English
|
||||
}
|
||||
|
||||
out, err := llm.RunTranslate(r.Context(), h.Client, explanation, from, to)
|
||||
if err != nil {
|
||||
httputil.ErrorJSON(w, http.StatusBadGateway, "translate failed: "+err.Error())
|
||||
httputil.UpstreamError(w, "translate", err)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// The pair model's flagship moment, end to end: she reaches for a sentence in
|
||||
// her own language mid-document, and the card that comes back is labelled as a
|
||||
// translation rather than as a tidy-up of her Chinese.
|
||||
//
|
||||
// The label is asserted through the real /check path rather than against
|
||||
// isTranslation directly, because the point of the item was never the detector —
|
||||
// Petal already found these spans and already rendered them into English. What
|
||||
// was wrong was the type that reached the rail.
|
||||
func TestChineseSpanBecomesATranslateCard(t *testing.T) {
|
||||
// Note the model calls it "clarity", as the live build did. The type it
|
||||
// volunteers is not consulted.
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"我想说这句话但是不知道用英语怎么说。","replacement":"I want to say this but I don't know how to say it in English.","explanation":"这是英文说法 · Here is how to say it in English","type":"clarity"}
|
||||
]}`}
|
||||
srv, docID, database := newPairServer(t, client, "zh")
|
||||
setDocTextDB(t, database, docID, "My weekend was good. 我想说这句话但是不知道用英语怎么说。")
|
||||
|
||||
var out []db.Suggestion
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("want 1 card, got %d: %+v", len(out), out)
|
||||
}
|
||||
if out[0].Type != db.SuggestionTypeTranslate {
|
||||
t.Fatalf("card type = %q, want %q", out[0].Type, db.SuggestionTypeTranslate)
|
||||
}
|
||||
// The rendering and the reasoning are the model's, untouched — only the label
|
||||
// is Petal's.
|
||||
if out[0].Replacement != "I want to say this but I don't know how to say it in English." {
|
||||
t.Fatalf("replacement was rewritten: %q", out[0].Replacement)
|
||||
}
|
||||
}
|
||||
|
||||
// The other half of the same claim: an ordinary English correction on the same
|
||||
// writer's document keeps the type the model gave it. A relabel that fired on
|
||||
// everything would be no better than the label it replaced.
|
||||
func TestEnglishCorrectionKeepsItsType(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"My weekend was very good","replacement":"My weekend was wonderful","explanation":"stronger wording","type":"phrasing"}
|
||||
]}`}
|
||||
srv, docID, database := newPairServer(t, client, "zh")
|
||||
setDocTextDB(t, database, docID, "My weekend was very good.")
|
||||
|
||||
var out []db.Suggestion
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("want 1 card, got %d: %+v", len(out), out)
|
||||
}
|
||||
if out[0].Type != db.SuggestionTypePhrasing {
|
||||
t.Fatalf("card type = %q, want %q", out[0].Type, db.SuggestionTypePhrasing)
|
||||
}
|
||||
}
|
||||
|
||||
// The voice pass reads whole paragraphs for tone and stamps its own family. A
|
||||
// Chinese paragraph must not be able to smuggle a translate row into it — voice
|
||||
// rows carry no replacement to accept, so a "translation" there would be a card
|
||||
// offering nothing.
|
||||
func TestVoicePassCannotProduceATranslateCard(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"我想说这句话但是不知道用英语怎么说。","replacement":"I want to say this in English.","explanation":"tone","type":"clarity"}
|
||||
]}`}
|
||||
srv, docID, database := newPairServer(t, client, "zh")
|
||||
setDocTextDB(t, database, docID, "A first paragraph.\n\n我想说这句话但是不知道用英语怎么说。")
|
||||
|
||||
var out []db.Suggestion
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/voice", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("voice: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
for _, s := range out {
|
||||
if s.Type == db.SuggestionTypeTranslate {
|
||||
t.Fatalf("voice pass produced a translate card: %+v", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// setDocTextDB is setDocText for the pair harness, which hands back the DB
|
||||
// rather than the Handler.
|
||||
func setDocTextDB(t *testing.T, database *db.DB, docID, text string) {
|
||||
t.Helper()
|
||||
if _, err := database.Exec(
|
||||
`UPDATE documents SET content_text = ? WHERE id = ?`, text, docID,
|
||||
); err != nil {
|
||||
t.Fatalf("update doc text: %v", err)
|
||||
}
|
||||
}
|
||||
+122
-17
@@ -19,7 +19,9 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
@@ -36,6 +38,12 @@ const maxTextBytes = 4000
|
||||
// with the words, mirroring the old utterance.rate = 0.95. Higher = slower.
|
||||
const lengthScale = 1.1
|
||||
|
||||
// slowLengthScale is the "say it slower" replay (SUGGESTIONS §5e): roughly 0.75×
|
||||
// the normal pace, which is the speed listening drills have used for decades.
|
||||
// Piper stretches durations rather than resampling, so the voice keeps its pitch
|
||||
// instead of turning into a slowed tape.
|
||||
const slowLengthScale = lengthScale / 0.75
|
||||
|
||||
// audioFormat describes one output encoding: the cache-file extension, the
|
||||
// response Content-Type, and the ffmpeg args that turn Piper's WAV (on stdin)
|
||||
// into this format (on stdout). A nil ffmpegArgs means "serve the WAV as-is".
|
||||
@@ -90,6 +98,7 @@ type Handler struct {
|
||||
cacheDir string
|
||||
format audioFormat
|
||||
client *http.Client
|
||||
writes atomic.Uint64 // cache writes since boot; drives the prune throttle
|
||||
}
|
||||
|
||||
// New builds a Handler from config. It returns (nil, false) when TTS_ENDPOINT is
|
||||
@@ -105,16 +114,14 @@ func New(cfg *config.Config) (*Handler, bool) {
|
||||
format = formats["mp3"]
|
||||
}
|
||||
|
||||
// Map by base language so en-US, en-GB, etc. all resolve to the English
|
||||
// instance (the client sends BCP-47 tags like the old Web Speech path did).
|
||||
// A language is only routable when both its endpoint and voice are set;
|
||||
// otherwise the client falls back to Web Speech for that language.
|
||||
// Keyed by base language so en-US, en-GB — and pt-PT, pt-BR, bare pt —
|
||||
// resolve to the one instance that has that language's model loaded (the
|
||||
// client sends BCP-47 tags, as the old Web Speech path did). Config has
|
||||
// already dropped any language configured by halves, so an unroutable
|
||||
// language reaches the client as a 404 and falls back to Web Speech.
|
||||
routes := map[string]route{}
|
||||
if cfg.TTSVoiceEN != "" {
|
||||
routes["en"] = route{strings.TrimRight(cfg.TTSEndpoint, "/"), cfg.TTSVoiceEN}
|
||||
}
|
||||
if cfg.TTSEndpointZH != "" && cfg.TTSVoiceZH != "" {
|
||||
routes["zh"] = route{strings.TrimRight(cfg.TTSEndpointZH, "/"), cfg.TTSVoiceZH}
|
||||
for lang, v := range cfg.TTSVoices {
|
||||
routes[lang] = route{endpoint: v.Endpoint, voice: v.Voice}
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(cfg.TTSCacheDir, 0o755); err != nil {
|
||||
@@ -132,6 +139,19 @@ func New(cfg *config.Config) (*Handler, bool) {
|
||||
}, true
|
||||
}
|
||||
|
||||
// Languages lists the base language tags this handler can synthesize, sorted,
|
||||
// each with the voice serving it — for the startup line, so a deployment says
|
||||
// which sidecars it actually reached rather than which ones it was configured
|
||||
// to want.
|
||||
func (h *Handler) Languages() []string {
|
||||
out := make([]string, 0, len(h.routes))
|
||||
for lang, rt := range h.routes {
|
||||
out = append(out, lang+"="+rt.voice)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Routes mounts the synthesis endpoint. Mount under "/tts" so the full path is
|
||||
// POST /api/tts.
|
||||
func (h *Handler) Routes() chi.Router {
|
||||
@@ -140,11 +160,13 @@ func (h *Handler) Routes() chi.Router {
|
||||
return r
|
||||
}
|
||||
|
||||
// synthRequest is the body the editor posts: a passage and the BCP-47 language
|
||||
// tag it's written in (e.g. "en-US", "zh-CN").
|
||||
// synthRequest is the body the editor posts: a passage, the BCP-47 language tag
|
||||
// it's written in (e.g. "en-US", "zh-CN", "pt-PT"), and whether to say it slowly
|
||||
// — the replay a learner reaches for when the sentence went past too fast.
|
||||
type synthRequest struct {
|
||||
Text string `json:"text"`
|
||||
Lang string `json:"lang"`
|
||||
Slow bool `json:"slow"`
|
||||
}
|
||||
|
||||
// synth resolves a voice for the requested language, returns cached audio when
|
||||
@@ -180,9 +202,17 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Content-addressed: identical (voice, text) → identical clip. The format
|
||||
// extension keeps encodings from colliding in the same dir.
|
||||
sum := sha256.Sum256([]byte(rt.voice + "\n" + text))
|
||||
scale := lengthScale
|
||||
if req.Slow {
|
||||
scale = slowLengthScale
|
||||
}
|
||||
|
||||
// Content-addressed: identical (voice, pace, text) → identical clip. The pace
|
||||
// belongs in the key — without it the slow replay of a word already heard at
|
||||
// normal speed would be served from cache at normal speed, which is the one
|
||||
// request where the difference is the whole point. The format extension keeps
|
||||
// encodings from colliding in the same dir.
|
||||
sum := sha256.Sum256([]byte(fmt.Sprintf("%s\n%.3f\n%s", rt.voice, scale, text)))
|
||||
name := hex.EncodeToString(sum[:])[:32] + h.format.ext
|
||||
path := filepath.Join(h.cacheDir, name)
|
||||
|
||||
@@ -191,7 +221,7 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
audio, err := h.synthesize(r.Context(), rt, text)
|
||||
audio, err := h.synthesize(r.Context(), rt, text, scale)
|
||||
if err != nil {
|
||||
http.Error(w, "synthesis failed", http.StatusBadGateway)
|
||||
fmt.Fprintf(os.Stderr, "tts: synthesize: %v\n", err)
|
||||
@@ -205,6 +235,7 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
||||
if err := os.WriteFile(tmp, audio, 0o644); err == nil {
|
||||
_ = os.Rename(tmp, path)
|
||||
}
|
||||
h.pruneCache()
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", h.format.contentType)
|
||||
@@ -212,6 +243,80 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write(audio)
|
||||
}
|
||||
|
||||
// maxCacheBytes bounds the whole clip cache at 512 MiB.
|
||||
//
|
||||
// Each clip is small, so nothing about ordinary reading approaches this — a
|
||||
// year of tapping words is tens of megabytes. What it bounds is the shape of
|
||||
// the endpoint: the cache key is the *text*, so a client asking for four
|
||||
// thousand distinct characters at a time writes a new file every request, for
|
||||
// as long as it cares to. That is an authenticated writer filling the same
|
||||
// encrypted volume the database lives on, and a full disk is SQLite failing to
|
||||
// write, not merely read-aloud getting slower.
|
||||
const maxCacheBytes = 512 << 20
|
||||
|
||||
// pruneEvery throttles the sweep: checking the directory on every synthesis
|
||||
// would stat the whole cache for each new word. Synthesis is already the slow
|
||||
// path and misses are rare once a writer settles, so one sweep per this many
|
||||
// cache writes keeps the cost invisible while still converging long before the
|
||||
// limit means anything.
|
||||
const pruneEvery = 64
|
||||
|
||||
// pruneCache trims the cache back under maxCacheBytes, oldest-first, and is a
|
||||
// no-op the great majority of the time it is called.
|
||||
//
|
||||
// Oldest by modification time is a fair approximation of least-recently-useful
|
||||
// here: a clip is written once and only ever read afterwards, so its age is how
|
||||
// long ago someone wanted it. Evicting one costs a re-synthesis, never data —
|
||||
// which is why this can be as approximate as it likes, and why every error
|
||||
// along the way is simply given up on.
|
||||
func (h *Handler) pruneCache() {
|
||||
if n := h.writes.Add(1); n%pruneEvery != 0 {
|
||||
return
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(h.cacheDir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
type clip struct {
|
||||
path string
|
||||
size int64
|
||||
mod time.Time
|
||||
}
|
||||
var clips []clip
|
||||
var total int64
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
clips = append(clips, clip{filepath.Join(h.cacheDir, e.Name()), info.Size(), info.ModTime()})
|
||||
total += info.Size()
|
||||
}
|
||||
if total <= maxCacheBytes {
|
||||
return
|
||||
}
|
||||
|
||||
sort.Slice(clips, func(i, j int) bool { return clips[i].mod.Before(clips[j].mod) })
|
||||
// Drop to 80% rather than exactly to the line, so the next few hundred
|
||||
// clips don't each trigger another sweep.
|
||||
target := int64(maxCacheBytes / 100 * 80)
|
||||
removed := 0
|
||||
for _, c := range clips {
|
||||
if total <= target {
|
||||
break
|
||||
}
|
||||
if os.Remove(c.path) == nil {
|
||||
total -= c.size
|
||||
removed++
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "tts: cache over %d bytes — evicted %d oldest clip(s)\n", int64(maxCacheBytes), removed)
|
||||
}
|
||||
|
||||
// serve streams a cached clip with a long-lived immutable cache header (the URL
|
||||
// is content-addressed, so the bytes never change for a given request).
|
||||
func (h *Handler) serve(w http.ResponseWriter, r *http.Request, path string) {
|
||||
@@ -222,11 +327,11 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, path string) {
|
||||
|
||||
// synthesize POSTs to the route's Piper instance, then transcodes the returned
|
||||
// WAV when the configured format calls for it.
|
||||
func (h *Handler) synthesize(ctx context.Context, rt route, text string) ([]byte, error) {
|
||||
func (h *Handler) synthesize(ctx context.Context, rt route, text string, scale float64) ([]byte, error) {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"text": text,
|
||||
"voice": rt.voice,
|
||||
"length_scale": lengthScale,
|
||||
"length_scale": scale,
|
||||
})
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, rt.endpoint+h.synthURI, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
|
||||
@@ -22,6 +22,7 @@ func newStubPiper(t *testing.T, body []byte) (*httptest.Server, *int32, *synthEc
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
last.voice, _ = req["voice"].(string)
|
||||
last.text, _ = req["text"].(string)
|
||||
last.scale, _ = req["length_scale"].(float64)
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
_, _ = w.Write(body)
|
||||
}))
|
||||
@@ -29,7 +30,10 @@ func newStubPiper(t *testing.T, body []byte) (*httptest.Server, *int32, *synthEc
|
||||
return srv, &calls, last
|
||||
}
|
||||
|
||||
type synthEcho struct{ voice, text string }
|
||||
type synthEcho struct {
|
||||
voice, text string
|
||||
scale float64
|
||||
}
|
||||
|
||||
// newHandler builds a wav-format handler (no ffmpeg) pointed at a stub server.
|
||||
func newHandler(t *testing.T, endpoint string) *Handler {
|
||||
@@ -88,7 +92,12 @@ func TestSynthPathNormalisation(t *testing.T) {
|
||||
|
||||
func post(t *testing.T, h *Handler, text, lang string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
b, _ := json.Marshal(synthRequest{Text: text, Lang: lang})
|
||||
return postReq(t, h, synthRequest{Text: text, Lang: lang})
|
||||
}
|
||||
|
||||
func postReq(t *testing.T, h *Handler, body synthRequest) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
b, _ := json.Marshal(body)
|
||||
req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(b))
|
||||
rr := httptest.NewRecorder()
|
||||
h.synth(rr, req)
|
||||
@@ -192,8 +201,87 @@ func TestTextIsCapped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The slow replay is the whole of SUGGESTIONS §5e: same text, same voice, more
|
||||
// time per phoneme.
|
||||
func TestSlowRequestStretchesTheVoice(t *testing.T) {
|
||||
srv, _, last := newStubPiper(t, []byte("RIFF....fake-wav"))
|
||||
h := newHandler(t, srv.URL)
|
||||
|
||||
if rr := postReq(t, h, synthRequest{Text: "reception", Lang: "en-US"}); rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
if last.scale != lengthScale {
|
||||
t.Fatalf("normal length_scale = %v, want %v", last.scale, lengthScale)
|
||||
}
|
||||
|
||||
if rr := postReq(t, h, synthRequest{Text: "reception", Lang: "en-US", Slow: true}); rr.Code != http.StatusOK {
|
||||
t.Fatalf("slow status = %d, want 200", rr.Code)
|
||||
}
|
||||
if last.scale != slowLengthScale {
|
||||
t.Fatalf("slow length_scale = %v, want %v", last.scale, slowLengthScale)
|
||||
}
|
||||
if slowLengthScale <= lengthScale {
|
||||
t.Fatalf("slowLengthScale %v is not slower than %v", slowLengthScale, lengthScale)
|
||||
}
|
||||
}
|
||||
|
||||
// The pace has to be part of the cache key. Without it, asking for the slow
|
||||
// replay of a word already heard at normal speed serves the normal clip — the
|
||||
// one request where hearing the difference is the entire point.
|
||||
func TestSlowClipIsNotServedFromTheNormalCache(t *testing.T) {
|
||||
srv, calls, last := newStubPiper(t, []byte("RIFF....fake-wav"))
|
||||
h := newHandler(t, srv.URL)
|
||||
|
||||
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US"})
|
||||
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US", Slow: true})
|
||||
if *calls != 2 {
|
||||
t.Fatalf("piper calls = %d, want 2 (the slow clip is a different clip)", *calls)
|
||||
}
|
||||
if last.scale != slowLengthScale {
|
||||
t.Fatalf("second call length_scale = %v, want the slow one", last.scale)
|
||||
}
|
||||
|
||||
// …and each pace still caches on its own.
|
||||
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US", Slow: true})
|
||||
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US"})
|
||||
if *calls != 2 {
|
||||
t.Fatalf("piper calls = %d, want 2 (both paces now cached)", *calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A Portuguese request must reach the Portuguese instance on the base tag alone:
|
||||
// env var names cannot hold the hyphen in pt-PT, so config keys the map on "pt"
|
||||
// and the handler has to meet it there. pt-BR resolves to the same instance
|
||||
// because there is only one Portuguese voice loaded — and it is the European one.
|
||||
func TestPortugueseRoutesOnTheBaseTag(t *testing.T) {
|
||||
enSrv, enCalls, _ := newStubPiper(t, []byte("EN-wav"))
|
||||
ptSrv, ptCalls, ptLast := newStubPiper(t, []byte("PT-wav"))
|
||||
h := &Handler{
|
||||
routes: map[string]route{
|
||||
"en": {strings.TrimRight(enSrv.URL, "/"), "en_US-amy-medium"},
|
||||
"pt": {strings.TrimRight(ptSrv.URL, "/"), "pt_PT-tugão-medium"},
|
||||
},
|
||||
cacheDir: t.TempDir(),
|
||||
format: formats["wav"],
|
||||
client: http.DefaultClient,
|
||||
}
|
||||
|
||||
// Distinct text per tag, so a cache hit can't stand in for a route.
|
||||
for i, tag := range []string{"pt-PT", "pt", "pt-BR"} {
|
||||
if rr := post(t, h, strings.Repeat("receção ", i+1), tag); rr.Code != http.StatusOK {
|
||||
t.Fatalf("%s status = %d, want 200", tag, rr.Code)
|
||||
}
|
||||
}
|
||||
if *ptCalls != 3 || *enCalls != 0 {
|
||||
t.Fatalf("calls en=%d pt=%d, want en=0 pt=3", *enCalls, *ptCalls)
|
||||
}
|
||||
if ptLast.voice != "pt_PT-tugão-medium" {
|
||||
t.Fatalf("pt voice = %q, want the European Portuguese voice", ptLast.voice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBaseLang(t *testing.T) {
|
||||
cases := map[string]string{"en-US": "en", "EN_gb": "en", "zh-CN": "zh", "en": "en", "": ""}
|
||||
cases := map[string]string{"en-US": "en", "EN_gb": "en", "zh-CN": "zh", "pt-PT": "pt", "en": "en", "": ""}
|
||||
for in, want := range cases {
|
||||
if got := baseLang(in); got != want {
|
||||
t.Errorf("baseLang(%q) = %q, want %q", in, got, want)
|
||||
|
||||
@@ -26,6 +26,11 @@ type Word struct {
|
||||
Phonetic string `json:"phonetic"`
|
||||
Example string `json:"example"`
|
||||
DocID *string `json:"doc_id"`
|
||||
// Lang is '' | 'en' | 'pair' — the language of the document the word was met
|
||||
// in (see migration 0018). '' reads as English, like everywhere else this
|
||||
// vocabulary appears. The client needs it to pick a read-aloud voice: "comum"
|
||||
// is unguessable from its letters, so the card has to carry the answer.
|
||||
Lang string `json:"lang"`
|
||||
DueAt time.Time `json:"due_at"`
|
||||
IntervalDays int `json:"interval_days"`
|
||||
Ease float64 `json:"ease"`
|
||||
@@ -54,7 +59,7 @@ func (h *Handler) Routes() chi.Router {
|
||||
return r
|
||||
}
|
||||
|
||||
const vocabColumns = `id, word, gloss, definition, phonetic, example, doc_id,
|
||||
const vocabColumns = `id, word, gloss, definition, phonetic, example, doc_id, lang,
|
||||
due_at, interval_days, ease, reps, lapses, last_reviewed, created_at`
|
||||
|
||||
func scanWord(s interface {
|
||||
@@ -62,7 +67,7 @@ func scanWord(s interface {
|
||||
}) (Word, error) {
|
||||
var w Word
|
||||
err := s.Scan(
|
||||
&w.ID, &w.Word, &w.Gloss, &w.Definition, &w.Phonetic, &w.Example, &w.DocID,
|
||||
&w.ID, &w.Word, &w.Gloss, &w.Definition, &w.Phonetic, &w.Example, &w.DocID, &w.Lang,
|
||||
&w.DueAt, &w.IntervalDays, &w.Ease, &w.Reps, &w.Lapses, &w.LastReviewed, &w.CreatedAt,
|
||||
)
|
||||
return w, err
|
||||
@@ -165,15 +170,22 @@ func (h *Handler) capture(w http.ResponseWriter, r *http.Request) {
|
||||
// would hit the foreign key and leak a raw "FOREIGN KEY constraint" 500
|
||||
// instead of a clean 400 (and, once auth lands, would let a word be attached
|
||||
// to another user's document).
|
||||
//
|
||||
// The same row-scoped lookup answers what language the card is in: a word is
|
||||
// met inside a document, so the document's verdict is the word's language.
|
||||
// Asking the document rather than trusting a `lang` in the request body is
|
||||
// the same choice `runPass` makes — the client never gets to name a language
|
||||
// the server can already read. A word with no document is '', which reads as
|
||||
// English.
|
||||
lang := ""
|
||||
if req.DocID != nil {
|
||||
if strings.TrimSpace(*req.DocID) == "" {
|
||||
req.DocID = nil
|
||||
} else {
|
||||
var ok int
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT 1 FROM documents WHERE id = ? AND user_id = ?`,
|
||||
`SELECT doc_lang FROM documents WHERE id = ? AND user_id = ?`,
|
||||
*req.DocID, userID,
|
||||
).Scan(&ok)
|
||||
).Scan(&lang)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusBadRequest, "unknown doc_id")
|
||||
return
|
||||
@@ -189,15 +201,19 @@ func (h *Handler) capture(w http.ResponseWriter, r *http.Request) {
|
||||
// schedule (due_at/reps/interval/ease) alone so re-looking-up a word never
|
||||
// resets its progress.
|
||||
_, err := h.DB.Exec(
|
||||
`INSERT INTO vocab_words (user_id, word, gloss, definition, phonetic, example, doc_id, due_at, interval_days)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now', '+1 day'), 1)
|
||||
`INSERT INTO vocab_words (user_id, word, gloss, definition, phonetic, example, doc_id, lang, due_at, interval_days)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now', '+1 day'), 1)
|
||||
ON CONFLICT(user_id, word) DO UPDATE SET
|
||||
gloss = excluded.gloss,
|
||||
definition = excluded.definition,
|
||||
phonetic = excluded.phonetic,
|
||||
example = CASE WHEN excluded.example != '' THEN excluded.example ELSE vocab_words.example END,
|
||||
doc_id = COALESCE(excluded.doc_id, vocab_words.doc_id)`,
|
||||
userID, word, req.Gloss, req.Definition, req.Phonetic, req.Example, req.DocID,
|
||||
doc_id = COALESCE(excluded.doc_id, vocab_words.doc_id),
|
||||
-- lang travels with doc_id, and for the same reason: it is the new
|
||||
-- context or it is nothing. A lookup made outside any document must
|
||||
-- not relabel a card that was captured inside one.
|
||||
lang = CASE WHEN excluded.doc_id IS NOT NULL THEN excluded.lang ELSE vocab_words.lang END`,
|
||||
userID, word, req.Gloss, req.Definition, req.Phonetic, req.Example, req.DocID, lang,
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
|
||||
@@ -253,3 +253,89 @@ func TestDocLinkSurvivesDocDelete(t *testing.T) {
|
||||
t.Fatalf("doc_id should be nulled after doc delete, got %v", *all[0].DocID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCaptureTakesLanguageFromItsDocument is the garden's half of Phase 28: a
|
||||
// word met inside a document written in her own language is a card in that
|
||||
// language, and the server reads that off the document rather than being told.
|
||||
//
|
||||
// The three cases are the three the client can actually produce: a lookup inside
|
||||
// a flipped document, a lookup inside an English one, and a lookup with no
|
||||
// document at all (the search box) — the last of which is '', which reads as
|
||||
// English everywhere this value is used.
|
||||
func TestCaptureTakesLanguageFromItsDocument(t *testing.T) {
|
||||
srv, database := newTestServer(t)
|
||||
|
||||
seed := func(lang string) string {
|
||||
t.Helper()
|
||||
var id string
|
||||
if err := database.QueryRow(
|
||||
`INSERT INTO documents (user_id, content_text, doc_lang) VALUES (?, 'hi', ?) RETURNING id`,
|
||||
db.LocalUserID, lang,
|
||||
).Scan(&id); err != nil {
|
||||
t.Fatalf("seed doc: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
pairDoc, enDoc := seed("pair"), seed("en")
|
||||
|
||||
capture := func(word, body string) Word {
|
||||
t.Helper()
|
||||
rec := do(t, srv, http.MethodPost, "/vocab", body)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("capture %s: code=%d body=%s", word, rec.Code, rec.Body)
|
||||
}
|
||||
var w Word
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &w); err != nil {
|
||||
t.Fatalf("decode %s: %v", word, err)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
if got := capture("comum", `{"word":"comum","doc_id":"`+pairDoc+`"}`); got.Lang != "pair" {
|
||||
t.Fatalf("word from a flipped document: lang=%q, want pair", got.Lang)
|
||||
}
|
||||
if got := capture("reception", `{"word":"reception","doc_id":"`+enDoc+`"}`); got.Lang != "en" {
|
||||
t.Fatalf("word from an English document: lang=%q, want en", got.Lang)
|
||||
}
|
||||
if got := capture("orphan", `{"word":"orphan"}`); got.Lang != "" {
|
||||
t.Fatalf("word with no document: lang=%q, want empty", got.Lang)
|
||||
}
|
||||
|
||||
// A client that names a language is ignored: the document is the authority,
|
||||
// the same way runPass never lets the request pick its own target.
|
||||
if got := capture("comum", `{"word":"comum","lang":"en","doc_id":"`+pairDoc+`"}`); got.Lang != "pair" {
|
||||
t.Fatalf("client-supplied lang should not win: lang=%q, want pair", got.Lang)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecaptureOutsideADocumentKeepsItsLanguage pins the one asymmetry in the
|
||||
// upsert. Re-looking-up a word refreshes its context, but a lookup made with no
|
||||
// document carries no verdict — and relabelling a Portuguese card English
|
||||
// because she checked the word again from the search box would silently move it
|
||||
// to the wrong voice. lang travels with doc_id, or it doesn't travel.
|
||||
func TestRecaptureOutsideADocumentKeepsItsLanguage(t *testing.T) {
|
||||
srv, database := newTestServer(t)
|
||||
var docID string
|
||||
if err := database.QueryRow(
|
||||
`INSERT INTO documents (user_id, content_text, doc_lang) VALUES (?, 'olá', 'pair') RETURNING id`,
|
||||
db.LocalUserID,
|
||||
).Scan(&docID); err != nil {
|
||||
t.Fatalf("seed doc: %v", err)
|
||||
}
|
||||
if rec := do(t, srv, http.MethodPost, "/vocab",
|
||||
`{"word":"saudade","gloss":"","doc_id":"`+docID+`"}`); rec.Code != http.StatusCreated {
|
||||
t.Fatalf("capture: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
rec := do(t, srv, http.MethodPost, "/vocab", `{"word":"saudade","gloss":"longing"}`)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("recapture: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
var w Word
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &w)
|
||||
if w.Lang != "pair" {
|
||||
t.Fatalf("recapture outside a document: lang=%q, want pair held", w.Lang)
|
||||
}
|
||||
if w.Gloss != "longing" {
|
||||
t.Fatalf("recapture should still refresh the gloss, got %q", w.Gloss)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package vocab
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Planting: the garden's second source.
|
||||
//
|
||||
// Capture (handlers.go) records words the writer *sought out*. Planting records
|
||||
// phrasing she was gently *given* — an accepted collocation like "make a
|
||||
// decision" is a learnable chunk exactly like a looked-up word, and the SM-2-lite
|
||||
// scheduler doesn't care that it's three words rather than one. Together the two
|
||||
// halves make the garden a record of both sides of learning.
|
||||
//
|
||||
// Everything here is best-effort by design: planting hangs off accepting a
|
||||
// suggestion, and that accept must succeed whether or not a card comes of it.
|
||||
|
||||
// Execer is the slice of *sql.DB (or *sql.Tx) that planting needs.
|
||||
type Execer interface {
|
||||
Exec(query string, args ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
// Phrase is one chunk to plant.
|
||||
type Phrase struct {
|
||||
Text string // the corrected phrasing, e.g. "make a decision"
|
||||
Meaning string // why it's better — the suggestion's explanation
|
||||
Example string // the sentence she met it in, already corrected
|
||||
DocID *string // where, so "where did I see this?" stays one tap
|
||||
// Lang is the document's verdict ('' | 'en' | 'pair'), because the chunk is
|
||||
// lifted out of her own prose and is therefore in whatever language that
|
||||
// prose is. See migration 0018.
|
||||
Lang string
|
||||
}
|
||||
|
||||
// Phrase-card caps. A collocation is a short chunk; anything longer is a
|
||||
// rewritten sentence wearing a collocation's label, and a sentence makes a
|
||||
// miserable flashcard. Both bounds are deliberately tight — the cost of
|
||||
// skipping a real chunk is one missing card, the cost of planting a sentence is
|
||||
// a garden the writer stops trusting.
|
||||
const (
|
||||
maxPhraseRunes = 60
|
||||
maxPhraseWords = 6
|
||||
minPhraseWords = 2
|
||||
)
|
||||
|
||||
// PhraseKey normalizes a replacement into a garden key, or returns "" when the
|
||||
// text isn't a plantable chunk.
|
||||
//
|
||||
// Lowercasing matches capture's normalization, so a phrase and a looked-up word
|
||||
// share one UNIQUE(user_id, word) namespace rather than colliding sideways.
|
||||
// Single words are rejected on purpose: a one-word fix is word choice, and word
|
||||
// choice already reaches the garden through lookup — planting it here would give
|
||||
// it a card with no gloss and no phonetic, which reviews badly.
|
||||
func PhraseKey(text string) string {
|
||||
// Collapse all whitespace (a replacement can carry a newline from the
|
||||
// editor) so the key is stable and the word count is honest.
|
||||
s := strings.Join(strings.Fields(strings.ToLower(text)), " ")
|
||||
// Trim the punctuation a phrase picks up from the sentence around it, but
|
||||
// leave inner marks alone: "can't afford" and "in one's own time" are chunks.
|
||||
s = strings.Trim(s, `.,;:!?…"'“”‘’()[]`)
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || len([]rune(s)) > maxPhraseRunes {
|
||||
return ""
|
||||
}
|
||||
n := len(strings.Fields(s))
|
||||
if n < minPhraseWords || n > maxPhraseWords {
|
||||
return ""
|
||||
}
|
||||
// A chunk of pure digits or symbols ("12 000", "-- --") isn't vocabulary.
|
||||
if !strings.ContainsFunc(s, unicode.IsLetter) {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Plant adds a phrase card to the garden, due tomorrow like any fresh capture.
|
||||
// It reports whether a new card was created.
|
||||
//
|
||||
// ON CONFLICT DO NOTHING, unlike capture's refresh-the-context upsert: accepting
|
||||
// the same collocation again months later is evidence the chunk is still being
|
||||
// learned, and the last thing that should do is overwrite the card's first
|
||||
// context or disturb a schedule it has been climbing. An existing card wins.
|
||||
func Plant(ex Execer, userID string, p Phrase) (bool, error) {
|
||||
key := PhraseKey(p.Text)
|
||||
if key == "" {
|
||||
return false, nil
|
||||
}
|
||||
res, err := ex.Exec(
|
||||
`INSERT INTO vocab_words (user_id, word, gloss, definition, phonetic, example, doc_id, lang, due_at, interval_days)
|
||||
VALUES (?, ?, '', ?, '', ?, ?, ?, datetime('now', '+1 day'), 1)
|
||||
ON CONFLICT(user_id, word) DO NOTHING`,
|
||||
userID, key,
|
||||
clamp(strings.TrimSpace(p.Meaning), maxDefinitionLen),
|
||||
clamp(strings.TrimSpace(p.Example), maxExampleLen),
|
||||
p.DocID,
|
||||
p.Lang,
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
return n > 0, err
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package vocab
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
func TestPhraseKey(t *testing.T) {
|
||||
cases := []struct {
|
||||
in, want string
|
||||
}{
|
||||
{"make a decision", "make a decision"},
|
||||
{"Make A Decision", "make a decision"}, // shares one namespace with lookups
|
||||
{" make a\ndecision ", "make a decision"}, // the editor's whitespace
|
||||
{"“make a decision.”", "make a decision"}, // punctuation from the sentence around it
|
||||
{"can’t afford it", "can’t afford it"}, // inner marks are part of the chunk
|
||||
{"decision", ""}, // word choice, not a chunk — lookup's job
|
||||
{"", ""}, //
|
||||
{"...", ""}, //
|
||||
{"12 000", ""}, // digits aren't vocabulary
|
||||
{"a b c d e f g", ""}, // a clause wearing a chunk's label
|
||||
{"in one’s own good time again", "in one’s own good time again"}, // six words is still a chunk
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := PhraseKey(tc.in); got != tc.want {
|
||||
t.Errorf("PhraseKey(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
// The length cap counts runes, not bytes — otherwise a Portuguese chunk well
|
||||
// inside the limit would be dropped for being accented.
|
||||
accented := "ãããããã ãããããã ãããããã ãããããã ãããããã" // 34 runes, 64 bytes
|
||||
if got := PhraseKey(accented); got != accented {
|
||||
t.Errorf("PhraseKey(%d runes / %d bytes) = %q, want it kept", len([]rune(accented)), len(accented), got)
|
||||
}
|
||||
long := "ãããããããããããã ãããããããããããã ãããããããããããã ãããããããããããã ãããããããããããã ãããããããããããã"
|
||||
if got := PhraseKey(long); got != "" {
|
||||
t.Errorf("PhraseKey(%d runes) = %q, want \"\"", len([]rune(long)), got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlantCreatesOnceAndReportsIt(t *testing.T) {
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
|
||||
p := Phrase{Text: "make a decision", Meaning: "why", Example: "I had to make a decision."}
|
||||
created, err := Plant(database, db.LocalUserID, p)
|
||||
if err != nil || !created {
|
||||
t.Fatalf("first plant: created=%v err=%v", created, err)
|
||||
}
|
||||
created, err = Plant(database, db.LocalUserID, p)
|
||||
if err != nil || created {
|
||||
t.Fatalf("second plant: created=%v err=%v, want false", created, err)
|
||||
}
|
||||
|
||||
// A card that isn't plantable is a silent no-op, not an error: planting hangs
|
||||
// off accepting an edit, and that accept must never fail for a flashcard.
|
||||
created, err = Plant(database, db.LocalUserID, Phrase{Text: "decision"})
|
||||
if err != nil || created {
|
||||
t.Fatalf("unplantable: created=%v err=%v", created, err)
|
||||
}
|
||||
|
||||
var n int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM vocab_words WHERE user_id = ?`, db.LocalUserID).Scan(&n); err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("garden has %d cards, want 1", n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the two Chinese assets the learner direction of the zh pair needs.
|
||||
|
||||
Why two, and why they are split the way they are
|
||||
------------------------------------------------
|
||||
Every other pair Petal ships needs one asset: a word list the browser loads so
|
||||
it can underline. Chinese needs two, because the browser and the server want
|
||||
different halves of the same dictionary and for different reasons.
|
||||
|
||||
* **The browser needs a word list, and it needs it offline.** Chinese is
|
||||
written without spaces, so there is no such thing as "the word under the
|
||||
cursor" until something segments the sentence. Every ESL surface Petal
|
||||
already has — the hover gloss, the right-click lookup, Ctrl/Cmd+D, the
|
||||
vocabulary garden capture — is built on `wordAt`, and `wordAt` is a regex
|
||||
over Latin letters. Segmentation is what replaces that regex, it runs on
|
||||
every hover, and a round-trip per hover is not a hover. So the word list
|
||||
ships to the browser: `web/public/dictionaries/zh/words.txt`.
|
||||
|
||||
* **The server holds the whole dictionary.** Pinyin and English senses are
|
||||
only ever wanted one word at a time, in answer to a hover or a click, which
|
||||
is exactly what `/api/gloss/{word}` already does for the other direction. So
|
||||
the readings stay in the binary — `internal/lexicon/data/hanzi.json.gz` —
|
||||
where their size costs a browser nothing.
|
||||
|
||||
That split is what makes the coverage decisions below come out *opposite* to
|
||||
each other, and both are deliberate.
|
||||
|
||||
Two sources, because neither one has both halves
|
||||
------------------------------------------------
|
||||
* **CC-CEDICT** (CC BY-SA 4.0, https://www.mdbg.net/) has the headwords,
|
||||
pinyin and English senses, and no frequency information at all.
|
||||
* **jieba's `dict.txt`** (MIT, https://github.com/fxsjy/jieba) has ~349k
|
||||
headwords with corpus frequencies, and no definitions.
|
||||
|
||||
Segmentation needs the frequencies: the standard algorithm is a shortest-path
|
||||
walk over log-probabilities, not longest-match, and without frequencies the
|
||||
classic ambiguities go the wrong way. The client list therefore carries
|
||||
`word freq` per line; the gloss map carries readings.
|
||||
|
||||
The size decision is the client list, and it is a size decision only
|
||||
--------------------------------------------------------------------
|
||||
Measured on ordinary learner prose, the segmentation produced by the full jieba
|
||||
dictionary (381,886 hanzi headwords once CC-CEDICT is unioned in) and by a
|
||||
frequency-gated one is **identical**, including on the textbook ambiguities
|
||||
(研究生命的起源, 乒乓球拍卖完了, 南京市长江大桥). What the long tail contains is
|
||||
rare proper nouns, and the max-probability walk almost never chooses one: a
|
||||
freq-3 name loses to two common words every time. The cases where a missing word
|
||||
does change the answer degrade *gracefully* — the sentence splits into smaller
|
||||
real words, which is a slightly clumsier gloss, not a wrong underline.
|
||||
|
||||
So the gate is set where the size is, at **freq >= 5**: 188,522 words, ~0.97 MB
|
||||
gzipped over the wire, in line with fr (1.19 MB) and es (1.74 MB) rather than in
|
||||
excess of them. Every CC-CEDICT headword is unioned back in regardless of
|
||||
frequency, so the segmenter can always see a word the server can explain.
|
||||
|
||||
The gloss map is gated by nothing, for the opposite reason
|
||||
-----------------------------------------------------------
|
||||
The es phase settled that a *spelling* dictionary should hold the union of every
|
||||
variety, because its only power is to underline and it must not underline
|
||||
correct writing. This asset's only power is to **explain**, and the word a
|
||||
learner stops on is precisely the one they do not know — which is to say, the
|
||||
rare one. Trimming this by frequency would remove exactly the entries it exists
|
||||
for. All 113,637 glossable headwords ship, ~3.1 MB gzipped, which is less than
|
||||
half of what `synonyms.json.gz` has embedded since Phase 9.
|
||||
|
||||
Simplified only, and said out loud
|
||||
-----------------------------------
|
||||
The zh langpack is written in simplified characters and jieba's frequencies are
|
||||
counted over simplified text, so the traditional headword in each CC-CEDICT line
|
||||
is dropped and simplified is what both assets are keyed by. Glossing traditional
|
||||
would be nearly free *here* and useless in the app: nothing would segment it, so
|
||||
nothing would ever ask. Traditional support is a real feature and it starts with
|
||||
a traditional word list, not with this file.
|
||||
|
||||
Usage:
|
||||
curl -sL https://www.mdbg.net/chinese/export/cedict/cedict_1_0_ts_utf-8_mdbg.txt.gz | gunzip > cedict.txt
|
||||
curl -sL https://raw.githubusercontent.com/fxsjy/jieba/master/jieba/dict.txt -o jieba.txt
|
||||
python3 scripts/build_cedict.py cedict.txt jieba.txt \
|
||||
web/public/dictionaries/zh/words.txt.gz \
|
||||
internal/lexicon/data/hanzi.json.gz
|
||||
"""
|
||||
import gzip
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
# Frequency gate for the *client* list only (see the module docstring). Words
|
||||
# below it survive if CC-CEDICT knows them, so "segmentable" is always a superset
|
||||
# of "glossable" and a hover can never land on a word the server cannot explain.
|
||||
MIN_FREQ = 5
|
||||
|
||||
# A CC-CEDICT headword we keep must be nothing but han characters. This drops the
|
||||
# entries that are really English or numerals with a Chinese gloss attached
|
||||
# ("AA制", "PM2.5", "11区"): the segmenter walks runs of hanzi, so a mixed
|
||||
# headword can never be matched anyway, and a Latin one would collide with the
|
||||
# English tokenizer that is still running on the same paragraph.
|
||||
HANZI_ONLY = re.compile(r'^[一-鿿]+$')
|
||||
|
||||
CEDICT_LINE = re.compile(r'^(\S+) (\S+) \[(.*?)\] /(.*)/$')
|
||||
|
||||
# At most this many readings per word, and this many senses per reading. Two
|
||||
# readings is not an arbitrary cap: it is what the particles need. 得 is dé "to
|
||||
# obtain" *and* de, the complement marker — and a learner who hovers 得 in
|
||||
# 说得很好 and is told only "to obtain" has been actively misinformed. Beyond two
|
||||
# the tail is dialect and surnames, which crowd out the sense actually wanted.
|
||||
MAX_READINGS = 2
|
||||
MAX_SENSES = 3
|
||||
MAX_SENSE_CHARS = 110
|
||||
|
||||
# Senses that describe the *dictionary* rather than the word. A learner hovering
|
||||
# a word wants to know what it means, not that it is an orthographic variant of
|
||||
# another headword they also do not know.
|
||||
SKIP_SENSE_PREFIXES = ('variant of', 'old variant', 'see ', 'used in', 'abbr. for')
|
||||
|
||||
# ── pinyin: numbered syllables to tone marks ────────────────────────────────
|
||||
# CC-CEDICT stores "gong1 yuan2". A learner reading their own writing back wants
|
||||
# gōngyuán: the tone mark is the part that is hard to remember and the part that
|
||||
# changes the word. The placement rule is the standard one — a/o/e take the mark
|
||||
# if present, otherwise the last vowel of the final — and it is small enough to
|
||||
# do here rather than to take a dependency for.
|
||||
TONE_VOWELS = {
|
||||
'a': 'āáǎà',
|
||||
'e': 'ēéěè',
|
||||
'i': 'īíǐì',
|
||||
'o': 'ōóǒò',
|
||||
'u': 'ūúǔù',
|
||||
'ü': 'ǖǘǚǜ',
|
||||
}
|
||||
|
||||
SYLLABLE = re.compile(r'^([a-zA-Zü:]+)([1-5])$')
|
||||
|
||||
|
||||
def tone_mark(syllable: str) -> str:
|
||||
"""One numbered pinyin syllable to its tone-marked form."""
|
||||
m = SYLLABLE.match(syllable)
|
||||
if not m:
|
||||
# Punctuation, a bare letter (CC-CEDICT writes "X" for unknown), or an
|
||||
# already-marked syllable: pass it through rather than mangling it.
|
||||
return syllable
|
||||
body, tone = m.group(1), int(m.group(2))
|
||||
# CC-CEDICT writes ü as "u:" and, in a few entries, as "v".
|
||||
body = body.replace('u:', 'ü').replace('U:', 'Ü').replace('v', 'ü').replace('V', 'Ü')
|
||||
if tone == 5: # neutral tone carries no mark
|
||||
return body
|
||||
low = body.lower()
|
||||
idx = -1
|
||||
for vowel in ('a', 'o', 'e'):
|
||||
idx = low.find(vowel)
|
||||
if idx >= 0:
|
||||
break
|
||||
if idx < 0:
|
||||
# No a/o/e: the mark goes on the last of i/u/ü (liú, guǐ, nǚ).
|
||||
idx = max(low.rfind('i'), low.rfind('u'), low.rfind('ü'))
|
||||
if idx < 0:
|
||||
return body
|
||||
marked = TONE_VOWELS[low[idx]][tone - 1]
|
||||
if body[idx].isupper():
|
||||
marked = marked.upper()
|
||||
return body[:idx] + marked + body[idx + 1:]
|
||||
|
||||
|
||||
def pinyin(numbered: str) -> str:
|
||||
"""A whole CC-CEDICT pinyin field to tone marks, syllables joined up.
|
||||
|
||||
Joined rather than spaced because that is how a word is written when it is
|
||||
being read as a word (gōngyuán, not gōng yuán); the spaces in the source are
|
||||
a storage convention, not orthography.
|
||||
"""
|
||||
return ''.join(tone_mark(s) for s in numbered.split())
|
||||
|
||||
|
||||
def clean_senses(raw: list[str]) -> list[str]:
|
||||
"""Strip the apparatus CC-CEDICT carries for lexicographers, not learners."""
|
||||
out = []
|
||||
for sense in raw:
|
||||
# "CL:座[zuo4]" is the measure-word field, useful and not a definition.
|
||||
sense = re.sub(r'\s*CL:.*$', '', sense).strip()
|
||||
# Bracketed pinyin cross-references ("abbr. for 的士[di1 shi4]").
|
||||
sense = re.sub(r'\[[a-zA-Z0-9: ]+\]', '', sense).strip()
|
||||
# Both edits cut inside parentheses — "cat (CL:只)" loses its closing
|
||||
# bracket and leaves "cat (" on the card. Drop a dangling opener rather
|
||||
# than trying to rebalance: what it introduced is gone.
|
||||
if sense.count('(') > sense.count(')'):
|
||||
sense = re.sub(r'\s*\([^()]*$', '', sense).strip()
|
||||
if not sense or sense.startswith(SKIP_SENSE_PREFIXES):
|
||||
continue
|
||||
out.append(sense)
|
||||
return out
|
||||
|
||||
|
||||
def read_cedict(path: str) -> dict[str, list[tuple[str, list[str]]]]:
|
||||
entries: dict[str, list[tuple[str, list[str]]]] = {}
|
||||
for line in open(path, encoding='utf-8'):
|
||||
if line.startswith('#'):
|
||||
continue
|
||||
m = CEDICT_LINE.match(line.strip())
|
||||
if not m:
|
||||
continue
|
||||
_traditional, simplified, py, defs = m.groups()
|
||||
if not HANZI_ONLY.match(simplified):
|
||||
continue
|
||||
entries.setdefault(simplified, []).append((py, defs.split('/')))
|
||||
return entries
|
||||
|
||||
|
||||
def read_jieba(path: str) -> dict[str, int]:
|
||||
freqs: dict[str, int] = {}
|
||||
for line in open(path, encoding='utf-8'):
|
||||
parts = line.split()
|
||||
if len(parts) >= 2 and HANZI_ONLY.match(parts[0]):
|
||||
freqs[parts[0]] = int(parts[1])
|
||||
return freqs
|
||||
|
||||
|
||||
# ── the assertions ──────────────────────────────────────────────────────────
|
||||
# The es phase's lesson, in the place it applies here: a check that every
|
||||
# plausible input would pass is not a check. The Spanish MUST_ACCEPT list
|
||||
# asserted vocabulary that all twenty-four builds carried, so it could not tell
|
||||
# them apart. These assert the things that actually go wrong in *this* build —
|
||||
# a mis-parsed pinyin field, a missing particle reading, a word list gated so
|
||||
# hard the segmenter can no longer see a word the server can explain.
|
||||
|
||||
# Tone marking, including the three cases the placement rule exists for.
|
||||
MUST_MARK = {
|
||||
'gong1 yuan2': 'gōngyuán', # a/o/e rule, first syllable
|
||||
'pao3 bu4': 'pǎobù',
|
||||
'liu2': 'liú', # no a/o/e: mark the *last* of i/u
|
||||
'gui3': 'guǐ',
|
||||
'nu:3': 'nǚ', # u: is ü
|
||||
'lu:e4': 'lüè', # ü and an e in the same syllable: e wins
|
||||
'de5': 'de', # neutral tone takes no mark at all
|
||||
'Zhong1 wen2': 'Zhōngwén', # capitalised headword keeps its capital
|
||||
}
|
||||
|
||||
# The particles the 错别字 rules are about must each carry the *grammatical*
|
||||
# reading, not only the lexical one. 的/地/得 are the single most confused triple
|
||||
# in written Chinese and all three are neutral-tone "de" in the use that matters;
|
||||
# an entry that only knows 得 as dé is worse than no entry.
|
||||
MUST_READ_DE = ('的', '地', '得')
|
||||
|
||||
# Words the segmenter must be able to see. 图书馆 and 乒乓球 are ordinary
|
||||
# vocabulary; 我 and 的 are the two commonest words in the language and a gate
|
||||
# that dropped either would be visibly broken; 的士 is a CC-CEDICT headword rare
|
||||
# enough to fall below the frequency gate, and is here to prove the union.
|
||||
MUST_SEGMENT = ('我', '的', '图书馆', '乒乓球', '公园', '的士')
|
||||
|
||||
|
||||
def check(words: dict[str, int], gloss: dict[str, list[list[str]]]) -> None:
|
||||
for numbered, want in MUST_MARK.items():
|
||||
got = pinyin(numbered)
|
||||
assert got == want, f'pinyin({numbered!r}) = {got!r}, want {want!r}'
|
||||
|
||||
for particle in MUST_READ_DE:
|
||||
readings = gloss.get(particle)
|
||||
assert readings, f'{particle} has no gloss entry at all'
|
||||
assert any(r[0] == 'de' for r in readings), \
|
||||
f'{particle} never reads as neutral "de": {readings}'
|
||||
|
||||
for word in MUST_SEGMENT:
|
||||
assert word in words, f'{word} missing from the segmentation list'
|
||||
|
||||
# The invariant the two gates exist to keep: everything the server can
|
||||
# explain, the browser can find.
|
||||
missing = [w for w in gloss if w not in words]
|
||||
assert not missing, f'{len(missing)} glossable words are unsegmentable, e.g. {missing[:5]}'
|
||||
|
||||
# Nothing Latin leaked into either asset (see HANZI_ONLY).
|
||||
for name, keys in (('words', words), ('gloss', gloss)):
|
||||
bad = [k for k in keys if not HANZI_ONLY.match(k)]
|
||||
assert not bad, f'non-hanzi headwords in {name}: {bad[:5]}'
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) != 5:
|
||||
sys.exit(__doc__.strip().rsplit('Usage:', 1)[-1].strip())
|
||||
cedict_path, jieba_path, words_out, gloss_out = sys.argv[1:]
|
||||
|
||||
entries = read_cedict(cedict_path)
|
||||
freqs = read_jieba(jieba_path)
|
||||
|
||||
# The client list: frequency-gated, then unioned with every glossable word.
|
||||
# A CC-CEDICT word jieba has never seen gets frequency 1 — real, and rare
|
||||
# enough that the max-probability walk will only choose it when nothing else
|
||||
# fits, which is exactly the standing it should have.
|
||||
words = {w: f for w, f in freqs.items() if f >= MIN_FREQ}
|
||||
for w in entries:
|
||||
words.setdefault(w, 1)
|
||||
|
||||
gloss: dict[str, list[list[str]]] = {}
|
||||
for word, rows in entries.items():
|
||||
readings: list[list[str]] = []
|
||||
for numbered, defs in rows:
|
||||
senses = clean_senses(defs)
|
||||
if not senses:
|
||||
continue
|
||||
readings.append([pinyin(numbered), '; '.join(senses[:MAX_SENSES])[:MAX_SENSE_CHARS]])
|
||||
if len(readings) == MAX_READINGS:
|
||||
break
|
||||
if readings:
|
||||
gloss[word] = readings
|
||||
|
||||
check(words, gloss)
|
||||
|
||||
# Gzipped on disk, like the pt-PT/fr/es word lists: the browser inflates it
|
||||
# with DecompressionStream (see useSpellChecker.fetchText), which costs no
|
||||
# bundle bytes, and 0.97 MB over the wire rather than 2.23 MB is the whole
|
||||
# difference between this and the biggest asset Petal ships.
|
||||
body = ('\n'.join(f'{w} {words[w]}' for w in sorted(words)) + '\n').encode('utf-8')
|
||||
with gzip.open(words_out, 'wb', compresslevel=9) as fh:
|
||||
fh.write(body)
|
||||
|
||||
payload = json.dumps(gloss, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
|
||||
with gzip.open(gloss_out, 'wb', compresslevel=9) as fh:
|
||||
fh.write(payload)
|
||||
|
||||
print(f'{words_out}: {len(words)} words, {len(body) / 1e6:.2f} MB raw, '
|
||||
f'{len(gzip.compress(body, 9)) / 1e6:.2f} MB gzipped')
|
||||
print(f'{gloss_out}: {len(gloss)} entries, {len(gzip.compress(payload, 9)) / 1e6:.2f} MB gzipped')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,543 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build one of Petal's browser spelling dictionaries from a Hunspell one.
|
||||
|
||||
Why this script exists at all
|
||||
----------------------------
|
||||
English is vendored the obvious way: `dictionary-en`'s `en.aff` + `en.dic` go
|
||||
into web/public/dictionaries/en and nspell reads them in the browser. The plan
|
||||
for Phase 21 said "Hunspell pt-PT vendored like en-US", and that turns out not to
|
||||
work, for a measured reason.
|
||||
|
||||
nspell expands affixes **eagerly at load time** — it materialises every surface
|
||||
form into a hash the moment you construct it. English gets away with this: ~50k
|
||||
stems and a small rule set. European Portuguese does not. `pt_PT.aff` carries
|
||||
1,340 affix rules (the full verb paradigm: six persons x a dozen tenses, plus
|
||||
diminutives, plus productive prefixes) over 44,257 stems. Measured on this
|
||||
machine, nspell needed ~340 MB of heap for the first 12,000 entries alone and had
|
||||
not returned after three minutes on the whole file; extrapolated, it wants well
|
||||
over a gigabyte. That is not something to hand a browser, still less a tablet.
|
||||
French is larger again: 5,600 affix rules over 84,140 stems.
|
||||
|
||||
So the expansion happens **here**, once, at build time, and the browser gets a
|
||||
flat word list it can load with no affix machinery at all. The runtime code path
|
||||
is then *identical* to English — same nspell, same interface — which is the real
|
||||
prize. The aff shipped alongside keeps only the suggestion-shaping directives
|
||||
(TRY/KEY/REP/MAP), so corrections still know that "cao" wants "ção" and that a
|
||||
missing acute accent is a near miss.
|
||||
|
||||
What Phase 24 had to add
|
||||
------------------------
|
||||
The build plan recorded that the pt-PT version of this script "generalizes" to
|
||||
French. It did not. It handled single-character flags and plain PFX/SFX and
|
||||
stopped on anything else — the right call, because `fr.aff` uses four of the
|
||||
things it stopped on, and getting any of them wrong changes which words are
|
||||
accepted:
|
||||
|
||||
* **`FLAG long`** — French flags are *two characters* (`S.`, `L'`, `Um`). The
|
||||
pt-PT reader took `set(flagstr)`, one flag per character, which on a French
|
||||
entry yields a bag of unrelated single letters: every entry would have been
|
||||
expanded through the wrong paradigm. This is the one that fails silently.
|
||||
* **Continuation flags** — pt-PT's affixes append plain text, so that script
|
||||
dropped anything after a `/` and asserted the drop was safe. French really
|
||||
does affix an affixed form: `PFX Um 0 0/S.` says the prefixed form then takes
|
||||
the plural suffix, and the elision prefixes arrive the same way from the
|
||||
other side (`SFX ... ait/n'q'l'm't's'`).
|
||||
* **`NEEDAFFIX`** — French marks thousands of stems "not a word on its own"
|
||||
(`Allemagne/S.()`), the bare form arriving instead through a zero-append
|
||||
rule. Ignoring the flag accepts stems the real dictionary rejects.
|
||||
* **`FULLSTRIP`** — a rule may strip the whole stem.
|
||||
|
||||
`CIRCUMFIX` and `FORBIDDENWORD` are *declared* in `fr.aff` and used by nothing,
|
||||
which this script asserts rather than assumes: an upstream release that started
|
||||
using either would otherwise change what is accepted without changing this file.
|
||||
`KEEPCASE` and `NOSUGGEST` are honoured by being ignored on purpose — they shape
|
||||
casing and suggestions, not membership, and a NOSUGGEST word is still a word.
|
||||
|
||||
Elision is handled at lookup, not here — and that is the size decision
|
||||
----------------------------------------------------------------------
|
||||
Most of French's affix machinery by volume is elision: `l'`, `d'`, `qu'`, `j'`,
|
||||
`n'`, `s'`, `jusqu'`, `puisqu'`. Hunspell treats `l'arbre` as one word, so a
|
||||
faithful expansion carries much of the language thirty-four times over — and
|
||||
Petal's tokenizer keeps internal apostrophes, so `l'arbre` really does arrive at
|
||||
the dictionary as one token and really would be underlined if it were absent.
|
||||
|
||||
Both halves were built and measured. Keeping the elided forms: **3,159,832 forms,
|
||||
8.25 MB gzipped**, ~45 MB of text for nspell to hash on a tablet. Dropping them:
|
||||
**473,326 forms, 1.19 MB gzipped**. The elided seven-eighths are not new words —
|
||||
they are thirteen little words glued to words already in the list — so the third
|
||||
option is the one taken: rules whose append carries an apostrophe are skipped
|
||||
here (the count is printed), and `withElision` in `useSpellChecker.ts` splits a
|
||||
token at a *known clitic* and checks the remainder. `l'arbre` costs one extra
|
||||
lookup instead of seven megabytes, and `zzz'arbre` is still flagged because
|
||||
`zzz` is not one of the thirteen.
|
||||
|
||||
Stems that carry an apostrophe of their own — `aujourd'hui`, `quelqu'un`,
|
||||
`presqu'île`, `prud'homme` — are dictionary entries rather than affixed forms,
|
||||
so they are kept verbatim and matched directly. `entr'aide` and `grand'mère` are
|
||||
absent for the same reason they are absent from Dicollecte: modern French spells
|
||||
them `entraide` and `grand-mère`.
|
||||
|
||||
Choosing the source
|
||||
-------------------
|
||||
Both languages have a trap here, and they are different traps.
|
||||
|
||||
**pt-PT: the wrong country.** npm's `dictionary-pt` is not European Portuguese.
|
||||
Both it and `dictionary-pt-br` package VERO ("Verificador Ortográfico Livre",
|
||||
Brasil), so vendoring the obvious npm name would have shipped Brazilian spellings
|
||||
under a pt-PT label — the pt-BR drift SUGGESTIONS.md §3 warns about, arriving
|
||||
through the packaging rather than through the model. The authentic dictionary is
|
||||
the Projecto Natura one (Universidade do Minho) that LibreOffice ships and Debian
|
||||
packages as `hunspell-pt-pt`; its aff declares `LANG pt_PT`.
|
||||
|
||||
**es: the wrong country again, hidden one layer further down.** Spanish looked
|
||||
like it would repeat the pt trap — `hunspell-es` installs twenty country codes,
|
||||
`es_AR` through `es_VE` — and then looked like it did not, because every one of
|
||||
them is a symlink to a single `es_ES.aff`/`es_ES.dic`. Both readings were wrong.
|
||||
Debian collapses the twenty because it ships **one** of upstream's builds, and
|
||||
the one it ships is the **peninsular** `es_ES`. RLA (Santiago Bosio's project,
|
||||
`sbosio/rla-es`) publishes twenty-four dictionaries per release: one per country,
|
||||
plus a **generic `es`** that is the union of all of them. Debian packages neither
|
||||
the generic one nor a choice — it packages Spain, under a name that reads like
|
||||
"Spanish".
|
||||
|
||||
Measured against the v2.9 release: Debian's file is 659,085 expanded forms and
|
||||
upstream `es_ES` is 659,018; the generic `es` is **717,640**. The 58,622-form
|
||||
difference is almost entirely **voseo** — `vení`, `tenés`, `querés`, `sabés`,
|
||||
`andá` — the present tense of most of Latin America, which Debian's package
|
||||
rejects as misspellings. Petal ships the **generic** build.
|
||||
|
||||
**Vocabulary cannot detect this and morphology can.** The first version of the es
|
||||
profile asserted the pan-Hispanic lexicon — *computadora* and *ordenador*, *papa*
|
||||
and *patata* — and passed happily on the peninsular file, because **every** RLA
|
||||
variant carries the full pan-Hispanic vocabulary; only the verb paradigms are
|
||||
localised. The `REP` table is no help either: its `ll`↔`y` and `ás`↔`az` entries
|
||||
look like evidence of yeísmo and seseo, but they are shared by all twenty-four
|
||||
builds. What separates them is exactly two things, and the profile now demands
|
||||
both at once: **voseo** (absent from `es_ES`) and **vosotros** (largely absent
|
||||
from `es_MX`). Only the generic build has both, so only the generic build passes.
|
||||
|
||||
This is the same decision fr made between `-classical` and `-revised`, arriving
|
||||
by a different road. The only thing this dictionary can do is underline
|
||||
something, and *tienes* and *tenés* are both correct Spanish taught in different
|
||||
countries — so Petal takes the build that accepts every variety rather than one
|
||||
that makes a writer wrong for where she is from. Nothing is generated to get
|
||||
there: the forms come from a real upstream package, which is what lets the
|
||||
MUST_ACCEPT list prove which package it was.
|
||||
|
||||
Licensing note: RLA is tri-licensed GPL-3+ / LGPL-3+ / MPL-1.1+; Petal
|
||||
redistributes under the MPL. The upstream README and LICENSE are vendored beside
|
||||
the output.
|
||||
|
||||
**fr: the wrong side of an argument the French have not settled.** The regional
|
||||
question turns out to be a non-question — Debian's `fr_FR`, `fr_CA`, `fr_BE`,
|
||||
`fr_CH`, `fr_LU` and `fr_MC` are all symlinks to one `fr.dic`, so unlike pt there
|
||||
is no country here to get wrong. What there is instead is the 1990 spelling
|
||||
reform, packaged three ways: `hunspell-fr-classical` (traditional), `-revised`
|
||||
(reform only) and `-comprehensive` (both). Petal ships **comprehensive**, because
|
||||
Petal never corrects her French — the only thing this dictionary can do is
|
||||
underline something. *coût* and *cout* are both correct French, taught in
|
||||
different decades to different people, and a writing companion has no business
|
||||
underlining one of them to take a side. The `fr` MUST_ACCEPT list is written to
|
||||
*prove* which package was used: classical rejects `cout`, revised rejects `coût`,
|
||||
and only comprehensive accepts both.
|
||||
|
||||
Licensing: pt-PT is GPL-2 or LGPL-2.1 or MPL-1.1, (c) José João de Almeida, Rui
|
||||
Vilela, Alberto Simões. fr is MPL-2.0, (c) 2007-2018 the Dicollecte contributors
|
||||
(grammalecte.net). The upstream copyright file is vendored beside each output.
|
||||
|
||||
Usage
|
||||
-----
|
||||
apt-get download hunspell-fr-comprehensive # or hunspell-pt-pt
|
||||
dpkg-deb -x hunspell-fr-comprehensive_*.deb src
|
||||
python3 scripts/build_hunspell_dictionary.py fr \\
|
||||
src/usr/share/hunspell/fr.aff \\
|
||||
src/usr/share/hunspell/fr.dic \\
|
||||
web/public/dictionaries/fr
|
||||
|
||||
Spanish does not come from Debian — see below; `hunspell-es` is the peninsular
|
||||
build. Take the generic dictionary from an upstream release instead:
|
||||
|
||||
curl -LO https://github.com/sbosio/rla-es/releases/download/v2.9/es.oxt
|
||||
unzip -d src es.oxt # an .oxt is a zip
|
||||
python3 scripts/build_hunspell_dictionary.py es \\
|
||||
src/es.aff src/es.dic web/public/dictionaries/es
|
||||
"""
|
||||
import gzip
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import unicodedata
|
||||
|
||||
# Directives worth keeping in the shipped aff. These shape *suggestions*, not
|
||||
# membership: TRY orders the alphabet the corrector tries, KEY knows which keys
|
||||
# are adjacent, REP holds the language's own confusions (cao/ção, ss/ç), and MAP
|
||||
# says an accented vowel and its bare form are the same letter for scoring —
|
||||
# which is most of what an ESL writer gets wrong in either language.
|
||||
KEEP_DIRECTIVES = ("SET", "TRY", "KEY", "REP", "MAP", "WORDCHARS")
|
||||
|
||||
# Directives that would change which words are *accepted* and that this expander
|
||||
# does not implement. If a future upstream release starts using one, the output
|
||||
# would silently disagree with the real dictionary, so the build stops instead.
|
||||
UNSUPPORTED = (
|
||||
"COMPOUNDFLAG", "COMPOUNDMIN", "COMPOUNDRULE", "COMPOUNDBEGIN",
|
||||
"ONLYINCOMPOUND", "PSEUDOROOT", "AF", "AM",
|
||||
)
|
||||
|
||||
APOSTROPHES = "'’"
|
||||
|
||||
|
||||
class Aff:
|
||||
"""The parts of an .aff file that decide which words exist."""
|
||||
|
||||
def __init__(self):
|
||||
self.pfx = {} # flag -> [(strip, append, condition, continuation)]
|
||||
self.sfx = {}
|
||||
# Cross-product, per flag — and kept per table, because PFX and SFX are
|
||||
# separate flag namespaces in hunspell: the same flag may name a prefix
|
||||
# table and a suffix table, with different cross-product settings. One
|
||||
# shared dict let the second block silently overwrite the first.
|
||||
self.cross_pfx = {} # flag -> bool
|
||||
self.cross_sfx = {}
|
||||
self.flag_kind = "char"
|
||||
self.needaffix = None
|
||||
self.circumfix = None
|
||||
self.forbidden = None
|
||||
self.dropped_apostrophe_rules = 0
|
||||
|
||||
|
||||
def parse_flags(raw, kind):
|
||||
"""Split a flag string into flags, per the aff's FLAG declaration."""
|
||||
raw = raw.strip()
|
||||
if not raw:
|
||||
return set()
|
||||
if kind == "long":
|
||||
# Two characters per flag, exactly. An odd length is a malformed flag
|
||||
# string, and silently dropping the trailing character would quietly
|
||||
# expand an entry through the wrong paradigm — the failure mode this
|
||||
# whole FLAG-aware rewrite exists to avoid.
|
||||
if len(raw) % 2:
|
||||
raise SystemExit(f"odd-length long flag string {raw!r}")
|
||||
return {raw[i:i + 2] for i in range(0, len(raw), 2)}
|
||||
if kind == "num":
|
||||
return {f for f in raw.split(",") if f}
|
||||
return set(raw)
|
||||
|
||||
|
||||
def parse_aff(path):
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
lines = fh.read().splitlines()
|
||||
|
||||
aff = Aff()
|
||||
|
||||
# FLAG has to be known before anything containing a flag is read, and it can
|
||||
# sit anywhere in the file. So: header pass first, rules second.
|
||||
for line in lines:
|
||||
parts = line.split()
|
||||
if not parts:
|
||||
continue
|
||||
head = parts[0]
|
||||
if head in UNSUPPORTED:
|
||||
raise SystemExit(
|
||||
f"{path}: unsupported directive {head!r} — this expander handles "
|
||||
"PFX/SFX affixation with continuation flags, and honouring "
|
||||
f"{head} would change which words are accepted. Extend the "
|
||||
"script before shipping."
|
||||
)
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
if head == "FLAG":
|
||||
aff.flag_kind = parts[1]
|
||||
if aff.flag_kind not in ("long", "num", "UTF-8"):
|
||||
raise SystemExit(f"{path}: unknown FLAG type {aff.flag_kind!r}")
|
||||
elif head == "NEEDAFFIX":
|
||||
aff.needaffix = parts[1]
|
||||
elif head == "CIRCUMFIX":
|
||||
aff.circumfix = parts[1]
|
||||
elif head == "FORBIDDENWORD":
|
||||
aff.forbidden = parts[1]
|
||||
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
parts = lines[i].split()
|
||||
if parts and parts[0] in ("PFX", "SFX"):
|
||||
kind, flag, cross_flag, count = parts[0], parts[1], parts[2], int(parts[3])
|
||||
table = aff.pfx if kind == "PFX" else aff.sfx
|
||||
cross = aff.cross_pfx if kind == "PFX" else aff.cross_sfx
|
||||
cross[flag] = cross_flag == "Y"
|
||||
rules = table.setdefault(flag, [])
|
||||
for j in range(1, count + 1):
|
||||
p = lines[i + j].split()
|
||||
strip = "" if p[2] == "0" else p[2]
|
||||
append, _, cont_raw = p[3].partition("/")
|
||||
if append == "0":
|
||||
append = ""
|
||||
# Elision. See the header: these forms are `l'` and its twelve
|
||||
# siblings glued to words already in the list, they multiply the
|
||||
# download by seven, and `withElision` reconstructs them at
|
||||
# lookup for the cost of one extra hash probe.
|
||||
if any(a in append for a in APOSTROPHES):
|
||||
aff.dropped_apostrophe_rules += 1
|
||||
continue
|
||||
cont = parse_flags(cont_raw, aff.flag_kind)
|
||||
if aff.circumfix and aff.circumfix in cont:
|
||||
raise SystemExit(
|
||||
f"{path}: CIRCUMFIX is used by a {kind} {flag} rule. It "
|
||||
"was declared-but-unused when this expander was written "
|
||||
"and is not implemented; honouring it would change which "
|
||||
"words are accepted."
|
||||
)
|
||||
cond = p[4] if len(p) > 4 else "."
|
||||
anchored = ("^" + cond) if kind == "PFX" else (cond + "$")
|
||||
rules.append((strip, append, re.compile(anchored), cont))
|
||||
i += count + 1
|
||||
continue
|
||||
i += 1
|
||||
return aff
|
||||
|
||||
|
||||
def apply_suffix(word, rules):
|
||||
"""Every (form, continuation flags) a suffix table yields for `word`."""
|
||||
out = []
|
||||
for strip, append, cond, cont in rules:
|
||||
if strip and not word.endswith(strip):
|
||||
continue
|
||||
if not cond.search(word):
|
||||
continue
|
||||
stem = word[: len(word) - len(strip)] if strip else word
|
||||
out.append((stem + append, cont))
|
||||
return out
|
||||
|
||||
|
||||
def apply_prefix(word, rules):
|
||||
out = []
|
||||
for strip, append, cond, cont in rules:
|
||||
if strip and not word.startswith(strip):
|
||||
continue
|
||||
if not cond.search(word):
|
||||
continue
|
||||
stem = word[len(strip):] if strip else word
|
||||
out.append((append + stem, cont))
|
||||
return out
|
||||
|
||||
|
||||
def expand_entry(word, flags, aff, out):
|
||||
"""Add every surface form of one dictionary entry to `out`.
|
||||
|
||||
Hunspell's model without compounding: a form is the stem plus at most one
|
||||
prefix and at most one suffix. A flag reaches an affix either from the stem's
|
||||
own flags or from the continuation flags of the affix applied on the other
|
||||
side; when both sides apply, both rules must be declared cross-product.
|
||||
|
||||
NEEDAFFIX is why the bare form is not simply added: the flag says *this* form
|
||||
is not a word, only whatever can be built from it — and it arrives both on
|
||||
stems and on continuations.
|
||||
"""
|
||||
def is_word(carried):
|
||||
return not (aff.needaffix and aff.needaffix in carried)
|
||||
|
||||
if is_word(flags):
|
||||
out.add(word)
|
||||
|
||||
suffixed = [] # (form, flag, continuation flags)
|
||||
for f in flags:
|
||||
if f in aff.sfx:
|
||||
for form, cont in apply_suffix(word, aff.sfx[f]):
|
||||
suffixed.append((form, f, cont))
|
||||
if is_word(cont):
|
||||
out.add(form)
|
||||
|
||||
prefixed = []
|
||||
for f in flags:
|
||||
if f in aff.pfx:
|
||||
for form, cont in apply_prefix(word, aff.pfx[f]):
|
||||
prefixed.append((form, f, cont))
|
||||
if is_word(cont):
|
||||
out.add(form)
|
||||
|
||||
# Prefix then suffix. The suffix flag may come from the stem or from the
|
||||
# prefix's own continuation (`PFX Um 0 0/S.`), and the suffix condition is
|
||||
# matched against the whole prefixed word, which is what hunspell does.
|
||||
# NEEDAFFIX is checked here too, exactly as on the single-affix paths above:
|
||||
# a doubly-affixed form whose last continuation still carries the flag is
|
||||
# "not a word on its own", and without compounding there is no third affix
|
||||
# left to make it one.
|
||||
for form, pf, pcont in prefixed:
|
||||
if not aff.cross_pfx.get(pf):
|
||||
continue
|
||||
for f in flags | pcont:
|
||||
if f in aff.sfx and aff.cross_sfx.get(f):
|
||||
for full, fcont in apply_suffix(form, aff.sfx[f]):
|
||||
if is_word(fcont):
|
||||
out.add(full)
|
||||
|
||||
# Suffix then prefix — the same pair reached from the other side, which is
|
||||
# how the elision prefixes arrive in French. Only the flags the suffix hands
|
||||
# forward are new here; the stem's own were covered above.
|
||||
for form, sf, scont in suffixed:
|
||||
if not aff.cross_sfx.get(sf):
|
||||
continue
|
||||
for f in scont:
|
||||
if f in aff.pfx and aff.cross_pfx.get(f):
|
||||
for full, fcont in apply_prefix(form, aff.pfx[f]):
|
||||
if is_word(fcont):
|
||||
out.add(full)
|
||||
|
||||
|
||||
def expand(aff_path, dic_path):
|
||||
aff = parse_aff(aff_path)
|
||||
forms = set()
|
||||
needaffix_stems = 0
|
||||
with open(dic_path, encoding="utf-8") as fh:
|
||||
fh.readline() # leading entry count, not a word
|
||||
for raw in fh:
|
||||
# Morphological fields (po:nom is:fem) follow the entry, separated by
|
||||
# a tab in pt-PT and by a space in fr.
|
||||
entry = raw.strip().split("\t")[0].split(" ")[0]
|
||||
if not entry:
|
||||
continue
|
||||
word, _, flagstr = entry.partition("/")
|
||||
word = word.strip()
|
||||
if not word:
|
||||
continue
|
||||
flags = parse_flags(flagstr, aff.flag_kind)
|
||||
if aff.forbidden and aff.forbidden in flags:
|
||||
raise SystemExit(
|
||||
f"{dic_path}: FORBIDDENWORD is in use ({word!r}). It was "
|
||||
"declared-but-unused when this expander was written; the "
|
||||
"forms it removes would be wrongly accepted."
|
||||
)
|
||||
if aff.needaffix and aff.needaffix in flags:
|
||||
needaffix_stems += 1
|
||||
expand_entry(word, flags, aff, forms)
|
||||
|
||||
# NFC, because the aff's own ICONV table normalises decomposed accents on the
|
||||
# way in and the browser hands nspell whatever the keyboard produced.
|
||||
forms = {unicodedata.normalize("NFC", f) for f in forms}
|
||||
return forms, aff, needaffix_stems
|
||||
|
||||
|
||||
def shipped_aff(aff_path):
|
||||
keep = []
|
||||
for line in open(aff_path, encoding="utf-8").read().splitlines():
|
||||
head = line.split()[0] if line.split() else ""
|
||||
if head in KEEP_DIRECTIVES:
|
||||
keep.append(line)
|
||||
return "\n".join(keep) + "\n"
|
||||
|
||||
|
||||
# Words the built list must accept, and must reject, before it is written. Each
|
||||
# set is chosen to fail loudly on the *specific* wrong source that language has a
|
||||
# packaged, plausible way of reaching — not to spot-check spelling in general.
|
||||
PROFILES = {
|
||||
# The pt-PT/pt-BR fault lines: post-Acordo spellings, the European lexicon,
|
||||
# and the first-person-plural preterite accent that only pt-PT writes.
|
||||
"pt-PT": {
|
||||
"accept": ("receção", "húmido", "telemóvel", "autocarro", "comboio",
|
||||
"ótimo", "pensámos", "escrevêssemos", "jardim"),
|
||||
"reject": ("recepção", "úmido", "ônibus", "óptimo"),
|
||||
"wrong": "this does not look like European Portuguese",
|
||||
},
|
||||
# Which of the three 1990-reform packagings this is. `coût`/`cout` and
|
||||
# `paraître`/`paraitre` are each accepted by exactly one of classical and
|
||||
# revised, so a build accepting all four is comprehensive and one that drops
|
||||
# any of them is not. `Allemagne` is a NEEDAFFIX stem reachable only through
|
||||
# a zero-append rule and `km` only through a prefix continuation, so between
|
||||
# them they also check that this expander honoured the two features the
|
||||
# pt-PT one refused.
|
||||
"fr": {
|
||||
"accept": ("coût", "cout", "paraître", "paraitre", "nénuphar", "nénufar",
|
||||
"oignon", "ognon", "événement", "évènement", "jardin",
|
||||
"Allemagne", "écrivissions", "km"),
|
||||
"reject": ("jardinn", "écrivaitz", "xyzzyque"),
|
||||
"wrong": "this does not look like the comprehensive French dictionary",
|
||||
},
|
||||
# The generic RLA build, and the accept list is written to reject the four
|
||||
# neighbouring builds rather than to describe this one.
|
||||
#
|
||||
# The first version of this profile demanded *computadora* and *ordenador*,
|
||||
# *papa* and *patata*, and passed — on the peninsular file, because **every**
|
||||
# RLA variant carries the whole pan-Hispanic vocabulary. Vocabulary does not
|
||||
# discriminate here at all; only morphology does, and it discriminates
|
||||
# completely:
|
||||
#
|
||||
# * **voseo** (`vení`, `tenés`, `querés`) is in `es` and `es_AR` and not in
|
||||
# `es_ES` or Debian's package. Demanding it rejects the peninsular build.
|
||||
# * **vosotros** (`tenéis`, `escribid`) is in `es`, `es_AR` and `es_ES`, and
|
||||
# largely absent from `es_MX`. Demanding it rejects the Mexican build.
|
||||
#
|
||||
# Requiring both at once leaves exactly one package standing: the generic
|
||||
# `es`, which is the only one that accepts every variety of Spanish. That is
|
||||
# the same reason fr ships `-comprehensive` — the only thing this dictionary
|
||||
# can do is underline something, and *tienes* and *tenés* are both correct
|
||||
# Spanish taught in different countries.
|
||||
#
|
||||
# The rest are shape checks: `escribiésemos` is the -se imperfect subjunctive,
|
||||
# `dámelo` proves the enclitic pronoun rules ran, and `jardín`/`niño` prove
|
||||
# FLAG UTF-8 was read as characters rather than bytes.
|
||||
"es": {
|
||||
"accept": (
|
||||
# Rejects es_ES and Debian's hunspell-es.
|
||||
"vení", "tenés", "querés", "sabés", "andá",
|
||||
# Rejects es_MX.
|
||||
"tenéis", "escribid",
|
||||
# Rejects es_AR, which has both voseo and vosotros and would
|
||||
# otherwise pass. Caribbean and Andean everyday words: the generic
|
||||
# build is the union of all twenty-four, so it is the only one that
|
||||
# holds another region's vocabulary as well as its own.
|
||||
"arepa", "chévere", "bacán",
|
||||
# Pan-Hispanic vocabulary. These pass on every RLA build, so they
|
||||
# prove nothing on their own — kept because a source that stopped
|
||||
# being RLA at all would fail them.
|
||||
"computadora", "ordenador", "papa", "patata", "jugo", "zumo",
|
||||
# Morphology and encoding.
|
||||
"escribiéramos", "escribiésemos", "escríbeme", "dámelo",
|
||||
"jardín", "niño", "corazón",
|
||||
),
|
||||
"reject": ("jardinn", "escribiz", "xyzzyque", "haiga"),
|
||||
"wrong": "this is not the generic RLA build (a per-country one accepts "
|
||||
"only some of these)",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main(lang, aff_path, dic_path, out_dir):
|
||||
profile = PROFILES.get(lang)
|
||||
if profile is None:
|
||||
raise SystemExit(f"no profile for {lang!r}; known: {', '.join(PROFILES)}")
|
||||
|
||||
forms, aff, needaffix_stems = expand(aff_path, dic_path)
|
||||
|
||||
missing = [w for w in profile["accept"] if w not in forms]
|
||||
present = [w for w in profile["reject"] if w in forms]
|
||||
if missing or present:
|
||||
raise SystemExit(
|
||||
f"{profile['wrong']}: missing {missing}, unexpectedly present {present}"
|
||||
)
|
||||
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
ordered = sorted(forms)
|
||||
body = f"{len(ordered)}\n" + "\n".join(ordered) + "\n"
|
||||
|
||||
dic_out = os.path.join(out_dir, f"{lang}.dic.gz")
|
||||
# mtime=0 so rebuilding identical input produces an identical file — a
|
||||
# vendored asset that changes on every build is noise in the diff.
|
||||
with gzip.GzipFile(dic_out, "wb", compresslevel=9, mtime=0) as fh:
|
||||
fh.write(body.encode("utf-8"))
|
||||
|
||||
aff_out = os.path.join(out_dir, f"{lang}.aff")
|
||||
with open(aff_out, "w", encoding="utf-8") as fh:
|
||||
fh.write(shipped_aff(aff_path))
|
||||
|
||||
print(f"{len(ordered)} forms -> {dic_out} "
|
||||
f"({os.path.getsize(dic_out) / 1e6:.2f} MB gzipped); "
|
||||
f"{needaffix_stems} NEEDAFFIX stems, "
|
||||
f"{aff.dropped_apostrophe_rules} elision rules skipped", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 5:
|
||||
raise SystemExit(
|
||||
"usage: build_hunspell_dictionary.py <lang> <aff> <dic> <out-dir>\n"
|
||||
f" lang is one of: {', '.join(PROFILES)}"
|
||||
)
|
||||
main(*sys.argv[1:5])
|
||||
Generated
+28
-10
@@ -30,6 +30,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/vite": "^4.0.0",
|
||||
"@types/node": "^26.1.2",
|
||||
"@types/react": "^19.1.0",
|
||||
"@types/react-dom": "^19.1.0",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
@@ -2107,6 +2108,16 @@
|
||||
"integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "26.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.2.tgz",
|
||||
"integrity": "sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~8.3.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/react": {
|
||||
"version": "19.2.17",
|
||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz",
|
||||
@@ -2928,9 +2939,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/linkify-it": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.1.tgz",
|
||||
"integrity": "sha512-wVoTjP4Q6R0NW5hiZkVJaFZPWgtXfoGF+6LucL3/FtiNjmcHhYjEr5f1Kqjirc1nBW07J/ZuRFumqr2oqccEWg==",
|
||||
"version": "5.0.2",
|
||||
"resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.2.tgz",
|
||||
"integrity": "sha512-ONTm2jCMAVZjgQa/Fy1kScXsuOoF5NPTsoFBdE1KVIZ2vAh/r9+Bqo+0jINCBYnavTPQZz38QzFTme79ENoN3Q==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@@ -3019,9 +3030,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/nanoid": {
|
||||
"version": "3.3.15",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz",
|
||||
"integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==",
|
||||
"version": "3.3.16",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz",
|
||||
"integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -3104,9 +3115,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/postcss": {
|
||||
"version": "8.5.15",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz",
|
||||
"integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==",
|
||||
"version": "8.5.23",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz",
|
||||
"integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -3124,7 +3135,7 @@
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"nanoid": "^3.3.12",
|
||||
"nanoid": "^3.3.16",
|
||||
"picocolors": "^1.1.1",
|
||||
"source-map-js": "^1.2.1"
|
||||
},
|
||||
@@ -3559,6 +3570,13 @@
|
||||
"integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "8.3.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
|
||||
"integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/update-browserslist-db": {
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz",
|
||||
|
||||
@@ -33,6 +33,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/vite": "^4.0.0",
|
||||
"@types/node": "^26.1.2",
|
||||
"@types/react": "^19.1.0",
|
||||
"@types/react-dom": "^19.1.0",
|
||||
"@vitejs/plugin-react": "^4.3.4",
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
Spanish spelling dictionary
|
||||
===========================
|
||||
|
||||
The word list in `es.dic.gz` and the suggestion directives in `es.aff` are
|
||||
derived from the **generic** Spanish Hunspell dictionary published by the RLA-ES
|
||||
project ("Recursos Lingüísticos Abiertos del Español"), release v2.9.
|
||||
|
||||
Copyright (C) Santiago Bosio and the RLA-ES contributors
|
||||
|
||||
License: GPL-3+ or LGPL-3+ or MPL-1.1+
|
||||
Tri-licensed; you may choose freely among the three. Petal
|
||||
redistributes under the MPL. Full texts:
|
||||
https://www.gnu.org/licenses/gpl-3.0.en.html
|
||||
https://www.gnu.org/licenses/lgpl-3.0.en.html
|
||||
https://www.mozilla.org/en-US/MPL/1.1/
|
||||
|
||||
Upstream: https://github.com/sbosio/rla-es
|
||||
Source: https://github.com/sbosio/rla-es/releases/download/v2.9/es.oxt
|
||||
(an .oxt is a zip; es.aff and es.dic are at its root)
|
||||
|
||||
Not the Debian package, and that is the point
|
||||
---------------------------------------------
|
||||
`hunspell-es` looks like the obvious source and is the wrong one. It installs
|
||||
twenty country codes — `es_AR` through `es_VE` — all symlinked to a single file,
|
||||
which reads like "one pan-Hispanic dictionary". It is not. RLA publishes
|
||||
twenty-four dictionaries per release: one per country, plus a **generic `es`**
|
||||
that is the union of all of them, and Debian ships the **peninsular `es_ES`**
|
||||
build under the collapsed name.
|
||||
|
||||
Measured against v2.9, expanded to surface forms:
|
||||
|
||||
Debian hunspell-es 659,085 forms voseo: no vosotros: yes
|
||||
upstream es_ES 659,018 forms voseo: no vosotros: yes
|
||||
upstream es_MX 554,923 forms voseo: no vosotros: no
|
||||
upstream es_AR 669,605 forms voseo: yes vosotros: yes
|
||||
upstream es (generic) 717,640 forms voseo: yes vosotros: yes <-- this
|
||||
|
||||
The 58,622-form gap between Debian's file and the generic one is essentially the
|
||||
**voseo** paradigm — `vení`, `tenés`, `querés`, `sabés`, `andá` — the ordinary
|
||||
present tense of Argentina, Uruguay, Paraguay and much of Central America. Under
|
||||
the Debian package, a writer using it would have had her own verbs underlined as
|
||||
misspellings.
|
||||
|
||||
Why the generic build rather than one country
|
||||
---------------------------------------------
|
||||
The only thing this dictionary can do is underline something. *Tienes* and
|
||||
*tenés* are both correct Spanish, taught in different countries, and a writing
|
||||
companion has no business marking one of them wrong — the same reasoning that
|
||||
makes the French dictionary here the `-comprehensive` packaging rather than
|
||||
`-classical` or `-revised`. The generic build accepts every variety, so Petal
|
||||
underlines only what no Spanish speaker anywhere would write.
|
||||
|
||||
How the build proves it got this file
|
||||
-------------------------------------
|
||||
Vocabulary cannot tell these builds apart: *every* RLA variant carries the full
|
||||
pan-Hispanic lexicon, so *computadora* alongside *ordenador* passes on the
|
||||
peninsular file too. (The `REP` table is likewise no evidence — its `ll`/`y` and
|
||||
`ás`/`az` entries look like yeísmo and seseo but are shared by all builds.) Only
|
||||
the verb paradigms are localised, so the `es` profile in
|
||||
`scripts/build_hunspell_dictionary.py` demands, all at once:
|
||||
|
||||
* **voseo** (`vení`, `tenés`, `querés`) — rejects `es_ES` and Debian's package;
|
||||
* **vosotros** (`tenéis`, `escribid`) — rejects `es_MX`;
|
||||
* **another region's everyday words** (`arepa`, `chévere`, `bacán`) — rejects
|
||||
`es_AR`, which has both paradigms and would otherwise pass.
|
||||
|
||||
Only the generic build satisfies all three. Each of the four neighbouring builds
|
||||
was run through the profile and confirmed to fail.
|
||||
@@ -0,0 +1,29 @@
|
||||
SET UTF-8
|
||||
TRY aeroinsctldumpbgfvhzóíjáqéñxyúükwAEROINSCTLDUMPBGFVHZÓÍJÁQÉÑXYÚÜKW
|
||||
REP 19
|
||||
REP ás az
|
||||
REP az ás
|
||||
REP cc x
|
||||
REP és ez
|
||||
REP ez és
|
||||
REP güe hue
|
||||
REP güi hui
|
||||
REP hue güe
|
||||
REP hui güi
|
||||
REP ís iz
|
||||
REP ío ido
|
||||
REP ke que
|
||||
REP ki qui
|
||||
REP ll y
|
||||
REP mb nv
|
||||
REP nv mb
|
||||
REP seci cesi
|
||||
REP x cc
|
||||
REP y ll
|
||||
MAP 6
|
||||
MAP aáAÁ
|
||||
MAP eéEÉ
|
||||
MAP iíIÍ
|
||||
MAP oóOÓ
|
||||
MAP uúüUÚÜ
|
||||
MAP nñNÑ
|
||||
Binary file not shown.
@@ -0,0 +1,49 @@
|
||||
French spelling dictionary
|
||||
==========================
|
||||
|
||||
The word list in `fr.dic.gz` and the suggestion directives in `fr.aff` are
|
||||
derived from the Dicollecte / Grammalecte Hunspell dictionary for French
|
||||
(`fr.aff` / `fr.dic`), as packaged by Debian/Ubuntu in
|
||||
`hunspell-fr-comprehensive`.
|
||||
|
||||
Copyright (C) 2007-2018 the Dicollecte contributors
|
||||
(full list at
|
||||
https://grammalecte.net/members.php?prj=fr)
|
||||
Dictionary author: Olivier R.
|
||||
|
||||
License: MPL-2.0
|
||||
This Source Code Form is subject to the terms of the Mozilla
|
||||
Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
with this file, You can obtain one at
|
||||
http://mozilla.org/MPL/2.0/.
|
||||
|
||||
Upstream: https://grammalecte.net/home.php?prj=fr
|
||||
|
||||
Which of the three
|
||||
------------------
|
||||
|
||||
Debian packages this dictionary three ways, by how it treats the 1990 spelling
|
||||
reform: `hunspell-fr-classical` (traditional spellings), `hunspell-fr-revised`
|
||||
(reform spellings) and `hunspell-fr-comprehensive` (both). Petal ships the
|
||||
**comprehensive** one, because Petal never corrects her French — the only thing
|
||||
this dictionary can do is underline something, and *coût* and *cout* are both
|
||||
correct French. The regional packages (`fr_FR`, `fr_CA`, `fr_BE`, `fr_CH`,
|
||||
`fr_LU`, `fr_MC`) are all symlinks to the same word list, so there is no
|
||||
regional choice being made here.
|
||||
|
||||
What Petal changed
|
||||
------------------
|
||||
|
||||
`scripts/build_hunspell_dictionary.py` applies the upstream affix rules ahead of
|
||||
time — Hunspell's PFX/SFX expansion run once at build time instead of once per
|
||||
browser — and writes the resulting 473,326 surface forms as a flat word list.
|
||||
The shipped `.aff` keeps only upstream's TRY/KEY/REP/MAP/WORDCHARS lines, which
|
||||
shape *corrections* rather than membership.
|
||||
|
||||
One thing about membership did change, and it is reversible at lookup rather
|
||||
than lost: the elided forms (`l'arbre`, `qu'elle`, `jusqu'ici`) are **not** in
|
||||
the word list. Expanding them costs 8.25 MB gzipped against 1.19 MB, and they
|
||||
are not new words — they are thirteen clitics glued to words already present —
|
||||
so `withElision` in `web/src/hooks/useSpellChecker.ts` splits the token and
|
||||
checks the remainder instead. Stems that carry an apostrophe of their own
|
||||
(`aujourd'hui`, `quelqu'un`, `presqu'île`, `prud'homme`) are kept verbatim.
|
||||
@@ -0,0 +1,141 @@
|
||||
SET UTF-8
|
||||
WORDCHARS -’'1234567890.
|
||||
TRY esntiarulodcpmévqfgbhàxèjyêMILzACçôîPâùJFSûBVœRDGNETHXkïOwKWYUëQÉZŒüãÎáöóÈíæÅñäśńÿ
|
||||
MAP 25
|
||||
MAP aàâäAÀÂÄ
|
||||
MAP eéèêëEÉÈÊË
|
||||
MAP iîïyIÎÏY
|
||||
MAP oôöOÔÖ
|
||||
MAP uùûüUÙÛÜ
|
||||
MAP cçCÇ
|
||||
MAP bB
|
||||
MAP dD
|
||||
MAP fF
|
||||
MAP gG
|
||||
MAP hH
|
||||
MAP jJ
|
||||
MAP kK
|
||||
MAP lL
|
||||
MAP mM
|
||||
MAP nN
|
||||
MAP pP
|
||||
MAP qQ
|
||||
MAP rR
|
||||
MAP sS
|
||||
MAP tT
|
||||
MAP vV
|
||||
MAP wW
|
||||
MAP xX
|
||||
MAP zZ
|
||||
REP 110
|
||||
REP a â
|
||||
REP â a
|
||||
REP e é
|
||||
REP é e
|
||||
REP e ê
|
||||
REP ê e
|
||||
REP e è
|
||||
REP è e
|
||||
REP i î
|
||||
REP î i
|
||||
REP o ô
|
||||
REP ô o
|
||||
REP u û
|
||||
REP û u
|
||||
REP A Â
|
||||
REP Â A
|
||||
REP E É
|
||||
REP É E
|
||||
REP E Ê
|
||||
REP Ê E
|
||||
REP E È
|
||||
REP È E
|
||||
REP I Î
|
||||
REP Î I
|
||||
REP O Ô
|
||||
REP Ô O
|
||||
REP U Û
|
||||
REP Û U
|
||||
REP ^Ca$ Ça
|
||||
REP ^l l'
|
||||
REP ^d d'
|
||||
REP ^n n'
|
||||
REP ^s s'
|
||||
REP ^j j'
|
||||
REP ^m m'
|
||||
REP ^t t'
|
||||
REP ^c c'
|
||||
REP f ph
|
||||
REP ph f
|
||||
REP c qu
|
||||
REP qu c
|
||||
REP k qu
|
||||
REP qu k
|
||||
REP x ct
|
||||
REP ct x
|
||||
REP bb b
|
||||
REP b bb
|
||||
REP cc c
|
||||
REP c cc
|
||||
REP ff f
|
||||
REP f ff
|
||||
REP ll l
|
||||
REP l ll
|
||||
REP mm m
|
||||
REP m mm
|
||||
REP nn n
|
||||
REP n nn
|
||||
REP pp p
|
||||
REP p pp
|
||||
REP rr r
|
||||
REP r rr
|
||||
REP ss s
|
||||
REP s ss
|
||||
REP ss c
|
||||
REP c ss
|
||||
REP ss ç
|
||||
REP ç ss
|
||||
REP tt t
|
||||
REP t tt
|
||||
REP œ oe
|
||||
REP oe œ
|
||||
REP æ ae
|
||||
REP ae æ
|
||||
REP ai é
|
||||
REP é ai
|
||||
REP ai è
|
||||
REP è ai
|
||||
REP ai ê
|
||||
REP ê ai
|
||||
REP ei é
|
||||
REP é ei
|
||||
REP ei è
|
||||
REP è ei
|
||||
REP ei ê
|
||||
REP ê ei
|
||||
REP o au
|
||||
REP au o
|
||||
REP o eau
|
||||
REP eau o
|
||||
REP ett èt
|
||||
REP èt ett
|
||||
REP ell èl
|
||||
REP èl ell
|
||||
REP t th
|
||||
REP th t
|
||||
REP ième$ e
|
||||
REP ème$ e
|
||||
REP è$ e
|
||||
REP mn$ min
|
||||
REP ogue$ ogiste
|
||||
REP ogiste$ ogue
|
||||
REP disez$ dites
|
||||
REP fesez$ faites
|
||||
REP faisez$ faites
|
||||
REP puit puits
|
||||
REP sanctionnable punissable
|
||||
REP questionnable discutable
|
||||
REP antitartre détartrant
|
||||
REP email courriel
|
||||
REP construirent construisirent
|
||||
KEY azertyuiop|qsdfghjklmù|wxcvbn|aéz|yèu|iço|oàp|aqz|zse|edr|rft|tgy|yhu|uji|iko|olpm|qws|sxd|dcf|fvg|gbh|hnj
|
||||
Binary file not shown.
@@ -0,0 +1,32 @@
|
||||
European Portuguese spelling dictionary
|
||||
=======================================
|
||||
|
||||
The word list in `pt-PT.dic.gz` and the suggestion directives in `pt-PT.aff` are
|
||||
derived from the LibreOffice/Projecto Natura Hunspell dictionary for European
|
||||
Portuguese (`pt_PT.aff` / `pt_PT.dic`), as packaged by Debian/Ubuntu in
|
||||
`hunspell-pt-pt`.
|
||||
|
||||
Copyright (C) 2006-2012 José João de Almeida <jj@di.uminho.pt>
|
||||
Rui Vilela <ruivilela@di.uminho.pt>
|
||||
Alberto Simões <ambs@di.uminho.pt>
|
||||
Universidade do Minho — Projecto Natura
|
||||
|
||||
License: GPL-2 or LGPL-2.1 or MPL-1.1
|
||||
(Petal redistributes it under the MPL-1.1 option.)
|
||||
|
||||
Upstream: https://natura.di.uminho.pt/ — via
|
||||
https://git.libreoffice.org/dictionaries/+/refs/heads/master/pt_PT
|
||||
|
||||
What Petal changed
|
||||
------------------
|
||||
|
||||
Nothing about which words are correct. `scripts/build_hunspell_dictionary.py`
|
||||
applies the upstream affix rules ahead of time — Hunspell's PFX/SFX expansion
|
||||
run once at build time instead of once per browser — and writes the resulting
|
||||
1,039,058 surface forms as a flat word list. The shipped `.aff` keeps only
|
||||
upstream's TRY/KEY/REP/MAP/WORDCHARS lines, which shape *corrections* rather
|
||||
than membership. See that script's header for why the dictionary could not be
|
||||
vendored in its original form.
|
||||
|
||||
Note that npm's `dictionary-pt` is *not* this dictionary: both it and
|
||||
`dictionary-pt-br` package the Brazilian VERO word list.
|
||||
@@ -0,0 +1,42 @@
|
||||
SET UTF-8
|
||||
TRY aerisontcdmlupvgbfzáhçqjíxãóéêâúõACMPSBTELGRIFVDkHJONôywUKXZWQÁYÍÉàÓèÂÚ
|
||||
KEY qwertyuiop|asdfghjkl|zxcvbnm
|
||||
WORDCHARS -
|
||||
REP 25
|
||||
REP por pro
|
||||
REP pre per
|
||||
REP damente mente
|
||||
REP mente damente
|
||||
REP iz íz
|
||||
REP cao ção
|
||||
REP ç ss
|
||||
REP ss ç
|
||||
REP c ss
|
||||
REP ss c
|
||||
REP ch x
|
||||
REP x ch
|
||||
REP cç x
|
||||
REP x cç
|
||||
REP k qu
|
||||
REP íti ití
|
||||
REP ití íti
|
||||
REP issí íssi
|
||||
REP ilí íli
|
||||
REP íli ilí
|
||||
REP ífi ifí
|
||||
REP ifí ífi
|
||||
REP nume mune
|
||||
REP coen quen
|
||||
REP concerteza com_certeza
|
||||
MAP 11
|
||||
MAP aá
|
||||
MAP aã
|
||||
MAP aâ
|
||||
MAP eé
|
||||
MAP eê
|
||||
MAP ií
|
||||
MAP cç
|
||||
MAP oó
|
||||
MAP oô
|
||||
MAP oõ
|
||||
MAP uú
|
||||
Binary file not shown.
@@ -0,0 +1,59 @@
|
||||
Chinese word list (segmentation)
|
||||
================================
|
||||
|
||||
`words.txt.gz` is not a spelling dictionary — Chinese has no spelling to check
|
||||
in the Hunspell sense. It is the word list Petal's segmenter walks, so that a
|
||||
sentence written without spaces has words in it to hover, look up and capture.
|
||||
Each line is `word frequency`. See scripts/build_cedict.py for how it is built
|
||||
and why it is gated where it is.
|
||||
|
||||
It is derived from two upstream sources, both redistributable, both credited
|
||||
here because the file itself has no room for a header.
|
||||
|
||||
|
||||
CC-CEDICT — the headwords
|
||||
-------------------------
|
||||
Community maintained free Chinese-English dictionary, published by MDBG.
|
||||
https://www.mdbg.net/chinese/dictionary?page=cedict
|
||||
|
||||
Licensed under the Creative Commons Attribution-ShareAlike 4.0 International
|
||||
License — https://creativecommons.org/licenses/by-sa/4.0/
|
||||
|
||||
Referenced works:
|
||||
CEDICT — Copyright (C) 1997, 1998 Paul Andrew Denisowski
|
||||
|
||||
CC-CEDICT is also the source of `internal/lexicon/data/hanzi.json.gz`, the
|
||||
pinyin and English senses embedded in the Petal binary. The same attribution and
|
||||
the same ShareAlike terms apply to that file; it is named here because it has
|
||||
nowhere of its own to say so.
|
||||
|
||||
|
||||
jieba — the frequencies
|
||||
-----------------------
|
||||
"结巴" Chinese word segmentation, by Sun Junyi.
|
||||
https://github.com/fxsjy/jieba
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2013 Sun Junyi
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
Only the word/frequency columns are used; jieba's part-of-speech tags and its
|
||||
algorithm are not (Petal's segmenter is its own, in web/src/lib/segment.ts).
|
||||
Binary file not shown.
+158
-27
@@ -1,12 +1,23 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { api, type DocSummary, type DocUpdate, type Document, type Suggestion, type Tag, type TagColor } from './api/client'
|
||||
import {
|
||||
api,
|
||||
onDocLang,
|
||||
type DocSummary,
|
||||
type DocUpdate,
|
||||
type Document,
|
||||
type Suggestion,
|
||||
type Tag,
|
||||
type TagColor,
|
||||
} from './api/client'
|
||||
import { useAutoSave } from './hooks/useAutoSave'
|
||||
import { useCheckpoint } from './hooks/useCheckpoint'
|
||||
import { findingKey, useCheckpoint } from './hooks/useCheckpoint'
|
||||
import { useSpellChecker } from './hooks/useSpellChecker'
|
||||
import { useSegmenter } from './hooks/useSegmenter'
|
||||
import { useTags } from './hooks/useTags'
|
||||
import { DocList } from './components/DocList/DocList'
|
||||
import { EditorCore, type EditorChange } from './components/Editor/EditorCore'
|
||||
import { ToneSelect } from './components/Editor/ToneSelect'
|
||||
import { ChromeStrip } from './components/Editor/ChromeStrip'
|
||||
import { ExportMenu } from './components/Export/ExportMenu'
|
||||
import { HistoryPanel } from './components/History/HistoryPanel'
|
||||
import { GardenPanel } from './components/Garden/GardenPanel'
|
||||
@@ -21,6 +32,7 @@ import { PetalFall } from './effects/PetalFall'
|
||||
import { usePack } from './i18n'
|
||||
import { useNightMode } from './hooks/useNightMode'
|
||||
import { playSuggestionSound } from './audio/sounds'
|
||||
import { fromIME } from './lib/ime'
|
||||
|
||||
export default function App() {
|
||||
const updateAvailable = useVersionWatch()
|
||||
@@ -30,7 +42,7 @@ export default function App() {
|
||||
const night = useNightMode()
|
||||
// Who's writing, and whether the server still recognises them. `signedOut`
|
||||
// flips the moment any call comes back 401.
|
||||
const { me, signedOut } = useSession()
|
||||
const { me, signedOut, setDirection, setPair } = useSession()
|
||||
const t = usePack()
|
||||
// A real account to sign out of, as opposed to the hardcoded local user a
|
||||
// build without auth configured runs as.
|
||||
@@ -78,7 +90,42 @@ export default function App() {
|
||||
return wordCountRef.current === 0 && (t === '' || t === 'Untitled')
|
||||
}, [])
|
||||
|
||||
const { status, schedule, saveNow } = useAutoSave(currentDoc?.id ?? null)
|
||||
// The pass announces its verdict the moment it decides one, which is the only
|
||||
// moment that is prompt enough: read-aloud is reached for when she has stopped
|
||||
// typing, so waiting for the next save means waiting for a save that isn't
|
||||
// coming. Registered once, and it updates the same one field the save path
|
||||
// does — whichever arrives first wins, and they agree.
|
||||
useEffect(() => {
|
||||
onDocLang((docId, lang) =>
|
||||
setCurrentDoc((prev) => (prev && prev.id === docId && prev.doc_lang !== lang ? { ...prev, doc_lang: lang } : prev)),
|
||||
)
|
||||
}, [])
|
||||
|
||||
// Only `doc_lang` is lifted out of the save response, and only when it moved.
|
||||
// It is the one field the server decides on its own — the checkpoint pass reads
|
||||
// the whole document and writes back whether it is English or hers — so it is
|
||||
// the one field that would otherwise go stale under her while she writes. Read
|
||||
// -aloud is what notices: a Portuguese paragraph read in an American voice.
|
||||
// Everything else in the response is what the client just sent, and copying it
|
||||
// back mid-keystroke would be a way to lose a character, not to gain one.
|
||||
const { status, schedule, saveNow } = useAutoSave(currentDoc?.id ?? null, (saved) =>
|
||||
setCurrentDoc((prev) =>
|
||||
prev && prev.id === saved.id && prev.doc_lang !== saved.doc_lang ? { ...prev, doc_lang: saved.doc_lang } : prev,
|
||||
),
|
||||
)
|
||||
// The Chinese word list, for a writer going the other way through the zh pair.
|
||||
// Gated on the account's own setting rather than on anything in the text: a
|
||||
// Mandarin native drafting English quotes Chinese constantly, and none of that
|
||||
// is what segmentation is for. Declared above the checkpoint because the
|
||||
// offline 错别字 pass reads it.
|
||||
//
|
||||
// Both halves of the gate matter now that Chinese is not the only pair with a
|
||||
// learner direction. `learning_pair` alone used to imply zh; a writer learning
|
||||
// Portuguese is also learning_pair and has no use for a megabyte of Chinese
|
||||
// word list — nor for the hanzi hover it turns on, which would ask /api/hanzi
|
||||
// about Portuguese words.
|
||||
const segmenter = useSegmenter(me?.direction === 'learning_pair' && me?.pair_lang === 'zh')
|
||||
|
||||
const {
|
||||
suggestions,
|
||||
checking,
|
||||
@@ -89,7 +136,8 @@ export default function App() {
|
||||
runVoice,
|
||||
runCollocation,
|
||||
removeSuggestion,
|
||||
} = useCheckpoint(currentDoc?.id ?? null)
|
||||
resolveServerId,
|
||||
} = useCheckpoint(currentDoc?.id ?? null, segmenter)
|
||||
// Browser-side spell checker — loads the en-US dictionary once per session.
|
||||
const { checker: spellChecker, addWord } = useSpellChecker()
|
||||
// The tag roster (with counts). Assignments live on the doc summaries below.
|
||||
@@ -307,16 +355,39 @@ export default function App() {
|
||||
[currentDoc, patchSummary, schedule],
|
||||
)
|
||||
|
||||
// She took the kitten up on its daily invitation. The prompt becomes the
|
||||
// blank page's title, so the question she agreed to answer stays in front of
|
||||
// her while she answers it — rather than being said once and then gone the
|
||||
// moment the bubble fades.
|
||||
const handleAcceptInvitation = useCallback(
|
||||
(prompt: string) => {
|
||||
if (!currentDoc) return
|
||||
handleTitleChange(prompt)
|
||||
},
|
||||
[currentDoc, handleTitleChange],
|
||||
)
|
||||
|
||||
const handleEditorChange = useCallback(
|
||||
(change: EditorChange) => {
|
||||
const { composing, ...patch } = change
|
||||
setWordCount(change.word_count)
|
||||
setDocText(change.content_text)
|
||||
setEditTick((n) => n + 1)
|
||||
if (currentDoc) {
|
||||
patchSummary(currentDoc.id, { word_count: change.word_count })
|
||||
schedule(change)
|
||||
scheduleCheckpoint(change.content_text)
|
||||
// The save is never held: see EditorChange.composing. The flag itself
|
||||
// stays out of the patch — it describes the keyboard, not the document,
|
||||
// and the stashed draft a signed-out save leaves behind should be the
|
||||
// document alone.
|
||||
schedule(patch)
|
||||
}
|
||||
// Everything below reads the text as prose. While an IME composition is
|
||||
// in flight it is not prose yet — it is the pinyin she is converting — so
|
||||
// the checkpoint, the rule pack and the companion all wait for the word
|
||||
// to commit. EditorCore emits one more change the moment it does, so
|
||||
// nothing is skipped, only deferred by the length of a word.
|
||||
if (composing) return
|
||||
setDocText(change.content_text)
|
||||
setEditTick((n) => n + 1)
|
||||
if (currentDoc) scheduleCheckpoint(change.content_text)
|
||||
},
|
||||
[currentDoc, patchSummary, schedule, scheduleCheckpoint],
|
||||
)
|
||||
@@ -337,17 +408,44 @@ export default function App() {
|
||||
|
||||
// Accept applies the replacement in the editor (handled in EditorCore) and
|
||||
// marks the suggestion accepted; dismiss just rejects it. Both drop it locally.
|
||||
// A rule-pack card can be accepted before its row exists — the edit has already
|
||||
// landed either way, so a missing id just means there's nothing to file.
|
||||
const handleAccept = useCallback(
|
||||
async (s: Suggestion) => {
|
||||
removeSuggestion(s.id)
|
||||
setAcceptTick((n) => n + 1)
|
||||
try {
|
||||
await api.acceptSuggestion(s.id)
|
||||
const id = await resolveServerId(s)
|
||||
if (id) await api.acceptSuggestion(id)
|
||||
} catch (err) {
|
||||
console.error('accept failed', err)
|
||||
}
|
||||
},
|
||||
[removeSuggestion],
|
||||
[removeSuggestion, resolveServerId],
|
||||
)
|
||||
|
||||
// Accept-all: EditorCore has already applied the whole category in one editor
|
||||
// transaction, so this is only the bookkeeping. Each row is filed individually
|
||||
// (there's no batch endpoint, and each accept plants its own word in the
|
||||
// garden), but the kitten cheers once — five cheers for one click would read as
|
||||
// five separate congratulations for a decision she made once.
|
||||
const handleAcceptMany = useCallback(
|
||||
async (list: Suggestion[]) => {
|
||||
if (list.length === 0) return
|
||||
for (const s of list) removeSuggestion(s.id)
|
||||
setAcceptTick((n) => n + 1)
|
||||
await Promise.all(
|
||||
list.map(async (s) => {
|
||||
try {
|
||||
const id = await resolveServerId(s)
|
||||
if (id) await api.acceptSuggestion(id)
|
||||
} catch (err) {
|
||||
console.error('accept failed', err)
|
||||
}
|
||||
}),
|
||||
)
|
||||
},
|
||||
[removeSuggestion, resolveServerId],
|
||||
)
|
||||
|
||||
// After restoring a version, swap the restored doc into the editor. Bumping
|
||||
@@ -365,11 +463,14 @@ export default function App() {
|
||||
[patchSummary],
|
||||
)
|
||||
|
||||
// Escape always restores the sidebar while in distraction-free mode.
|
||||
// Escape always restores the sidebar while in distraction-free mode — unless
|
||||
// it belongs to an IME, where it cancels a candidate and never reaches Petal
|
||||
// at all. This is the writer typing Chinese in the very mode built for
|
||||
// uninterrupted writing, so it is the one worth getting right.
|
||||
useEffect(() => {
|
||||
if (!focusMode) return
|
||||
const onKey = (e: KeyboardEvent) => {
|
||||
if (e.key === 'Escape') setFocusMode(false)
|
||||
if (e.key === 'Escape' && !fromIME(e)) setFocusMode(false)
|
||||
}
|
||||
window.addEventListener('keydown', onKey)
|
||||
return () => window.removeEventListener('keydown', onKey)
|
||||
@@ -385,12 +486,13 @@ export default function App() {
|
||||
async (s: Suggestion) => {
|
||||
removeSuggestion(s.id)
|
||||
try {
|
||||
await api.dismissSuggestion(s.id)
|
||||
const id = await resolveServerId(s)
|
||||
if (id) await api.dismissSuggestion(id)
|
||||
} catch (err) {
|
||||
console.error('dismiss failed', err)
|
||||
}
|
||||
},
|
||||
[removeSuggestion],
|
||||
[removeSuggestion, resolveServerId],
|
||||
)
|
||||
|
||||
// Play a soft sound when freshly-checked suggestions arrive — one per distinct
|
||||
@@ -398,10 +500,14 @@ export default function App() {
|
||||
// a pile-up. We track which ids we've already chimed for, and only chime for
|
||||
// recently-created suggestions so opening a doc with old pending advice stays
|
||||
// silent (the existing set was created in a past session).
|
||||
// Rule-pack findings are chimed by their wording, not their id: the same fix
|
||||
// appears first as a provisional card and then as its persisted row, and the
|
||||
// writer should hear it once.
|
||||
const chimedRef = useRef<Set<string>>(new Set())
|
||||
useEffect(() => {
|
||||
const fresh = suggestions.filter((s) => !chimedRef.current.has(s.id))
|
||||
fresh.forEach((s) => chimedRef.current.add(s.id))
|
||||
const key = (s: Suggestion) => (s.source === 'local' ? `local:${findingKey(s)}` : s.id)
|
||||
const fresh = suggestions.filter((s) => !chimedRef.current.has(key(s)))
|
||||
fresh.forEach((s) => chimedRef.current.add(key(s)))
|
||||
const justMade = fresh.filter(
|
||||
(s) => Date.now() - new Date(s.created_at).getTime() < 12_000,
|
||||
)
|
||||
@@ -423,7 +529,7 @@ export default function App() {
|
||||
<PetalFall night={night} />
|
||||
<header
|
||||
onMouseDown={handleChromeDown}
|
||||
className="petal-no-print flex h-12 shrink-0 items-center gap-2 px-5"
|
||||
className="petal-no-print flex h-12 shrink-0 items-center gap-2 px-3 md:px-5"
|
||||
style={{ borderBottom: '1px solid var(--color-border)' }}
|
||||
>
|
||||
<button
|
||||
@@ -439,7 +545,15 @@ export default function App() {
|
||||
☰
|
||||
</button>
|
||||
<span className="text-xl">🌸</span>
|
||||
<span className="text-lg font-extrabold text-plum">Petal</span>
|
||||
{/* The wordmark is the first thing to go on a small phone. The header
|
||||
holds a hamburger, a name and the garden button, and the garden
|
||||
button's label is a langpack string: 词汇花园 · Garden is 130px
|
||||
where Jardim de palavras · Garden is nearly 200, which is the
|
||||
difference between fitting a 320px screen and scrolling the whole
|
||||
app sideways. Of everything in this row, the one that can be spared
|
||||
is the app's own name — she is already inside the app, and the
|
||||
blossom stays. */}
|
||||
<span className="petal-wordmark text-lg font-extrabold text-plum">Petal</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setGardenOpen(true)}
|
||||
@@ -474,6 +588,9 @@ export default function App() {
|
||||
onToggleTag={handleToggleTag}
|
||||
onCreateTag={handleCreateTag}
|
||||
account={account}
|
||||
direction={me?.direction}
|
||||
onDirection={setDirection}
|
||||
onPair={setPair}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -488,27 +605,36 @@ export default function App() {
|
||||
<>
|
||||
<div
|
||||
onMouseDown={handleChromeDown}
|
||||
className="flex flex-1 flex-col overflow-y-auto px-6 py-8"
|
||||
// `petal-scrollport` marks this as the editor's scrolling
|
||||
// ancestor; EditorCore measures it to pin the text column while
|
||||
// the suggestion rail's overhang is scrolled.
|
||||
className="petal-scrollport flex flex-1 flex-col overflow-y-auto px-6 py-8"
|
||||
>
|
||||
<div ref={canvasRef} className="mx-auto flex w-full max-w-[720px] flex-1 flex-col">
|
||||
<div className="mb-5 flex items-center gap-3">
|
||||
{/* Title, then the three chrome pills. Their labels are
|
||||
bilingual and don't shrink, so how much width this row
|
||||
wants is a property of the langpack — 历史 is two glyphs
|
||||
where Historique is ten — and on a phone no language's
|
||||
version of it fits. The pills therefore live in a strip
|
||||
that scrolls itself; the title takes its own line below
|
||||
the drawer breakpoint so it keeps its full width. */}
|
||||
<div className="mb-5 flex flex-wrap items-center gap-2 md:gap-3">
|
||||
<input
|
||||
value={title}
|
||||
onChange={(e) => handleTitleChange(e.target.value)}
|
||||
placeholder="Untitled"
|
||||
aria-label="Document title"
|
||||
className="min-w-0 flex-1 bg-transparent text-3xl font-extrabold text-plum focus:outline-none"
|
||||
className="min-w-0 flex-1 basis-full bg-transparent text-3xl font-extrabold text-plum focus:outline-none md:basis-0"
|
||||
style={{ fontFamily: 'var(--font-ui)' }}
|
||||
/>
|
||||
<div className="petal-no-print shrink-0">
|
||||
<ChromeStrip className="petal-no-print flex items-center gap-2 py-0.5 md:gap-3">
|
||||
<ToneSelect value={tone} onChange={handleToneChange} />
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setHistoryOpen(true)}
|
||||
aria-label="Version history"
|
||||
title="Browse and restore earlier versions"
|
||||
className="petal-no-print inline-flex h-9 shrink-0 items-center gap-1.5 whitespace-nowrap px-3 text-sm font-bold"
|
||||
className="inline-flex h-9 shrink-0 items-center gap-1.5 whitespace-nowrap px-3 text-sm font-bold"
|
||||
style={{
|
||||
borderRadius: 'var(--radius-pill)',
|
||||
background: 'var(--color-surface)',
|
||||
@@ -520,17 +646,19 @@ export default function App() {
|
||||
<span>{t.app.history}</span>
|
||||
<span style={{ color: 'var(--color-muted)' }}>· History</span>
|
||||
</button>
|
||||
<div className="petal-no-print">
|
||||
<ExportMenu docId={currentDoc.id} />
|
||||
</div>
|
||||
</ChromeStrip>
|
||||
</div>
|
||||
<EditorCore
|
||||
key={`${currentDoc.id}:${editorEpoch}`}
|
||||
docId={currentDoc.id}
|
||||
docLang={currentDoc.doc_lang}
|
||||
initialContent={currentDoc.content}
|
||||
onChange={handleEditorChange}
|
||||
segmenter={segmenter}
|
||||
suggestions={suggestions}
|
||||
onAccept={handleAccept}
|
||||
onAcceptMany={handleAcceptMany}
|
||||
onDismiss={handleDismiss}
|
||||
onVoiceCheck={runVoice}
|
||||
voicing={voicing}
|
||||
@@ -551,6 +679,7 @@ export default function App() {
|
||||
voicing={voicing}
|
||||
collocating={collocating}
|
||||
llmDown={llmDown}
|
||||
suggestionCount={suggestions.length}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
@@ -597,6 +726,8 @@ export default function App() {
|
||||
editTick={editTick}
|
||||
acceptTick={acceptTick}
|
||||
text={docText}
|
||||
blankPage={wordCount === 0 && docText.trim() === ''}
|
||||
onAcceptInvitation={handleAcceptInvitation}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
+142
-7
@@ -44,8 +44,18 @@ export interface Document {
|
||||
// When true, this document's automatic snapshots are never pruned, so its
|
||||
// full writing trail survives as authorship evidence (see the passport).
|
||||
preserve_history: boolean
|
||||
// Which language this document is written in, as decided server-side by the
|
||||
// checkpoint pass: '' | 'en' | 'pair' ('' reads as English). Read-only — the
|
||||
// editor never sends it. It is here so read-aloud can use the right voice on a
|
||||
// document written in her own language.
|
||||
doc_lang: DocLang
|
||||
}
|
||||
|
||||
// The document-language verdict, shared by documents and garden cards. 'pair'
|
||||
// names the writer's own language rather than a language code, so changing her
|
||||
// pair re-reads her documents instead of stranding a stale name on them.
|
||||
export type DocLang = '' | 'en' | 'pair'
|
||||
|
||||
// Fields the editor sends on auto-save. All optional so a rename can send title
|
||||
// alone; the editor sends the full set.
|
||||
export interface DocUpdate {
|
||||
@@ -78,15 +88,41 @@ export interface WordInfo {
|
||||
frequency: number
|
||||
difficulty: number
|
||||
etymology: string // free-form, already trimmed to a line by the server; '' when absent
|
||||
// The same token read as a word of the writer's own language, when it is one.
|
||||
// Absent for a zh-pair writer and for almost every word in a Latin pair — see
|
||||
// lexicon.Reverse for why Petal asks both directions instead of guessing.
|
||||
reverse?: WordReverse
|
||||
}
|
||||
|
||||
// A word looked up in the other direction: the writer's language -> English.
|
||||
export interface WordReverse {
|
||||
lang: string
|
||||
gloss: string
|
||||
definitions?: WordMeaning[]
|
||||
phonetic?: string
|
||||
}
|
||||
|
||||
// The lightweight Chinese-only gloss behind the inline hover/select tooltip.
|
||||
export interface Gloss {
|
||||
word: string
|
||||
gloss: string
|
||||
// The English meaning of the token read as a word of her own language.
|
||||
// Present only on a collision (Portuguese *sale*, French *chat*).
|
||||
reverse?: string
|
||||
}
|
||||
|
||||
export type SuggestionType = 'grammar' | 'phrasing' | 'idiom' | 'clarity' | 'voice' | 'collocation' | 'mechanics'
|
||||
// 'translate' is a span she wrote in her own language, rendered into English —
|
||||
// not a correction. The server decides the label from the span itself, never from
|
||||
// the model, so the client can trust it (see suggestions/language.go).
|
||||
export type SuggestionType =
|
||||
| 'grammar'
|
||||
| 'phrasing'
|
||||
| 'idiom'
|
||||
| 'clarity'
|
||||
| 'translate'
|
||||
| 'voice'
|
||||
| 'collocation'
|
||||
| 'mechanics'
|
||||
|
||||
// One word in the vocabulary garden: a looked-up word with its gloss/phonetic,
|
||||
// the sentence it was met in, and its spaced-repetition state. `reps` drives how
|
||||
@@ -99,6 +135,9 @@ export interface VocabWord {
|
||||
phonetic: string
|
||||
example: string
|
||||
doc_id: string | null
|
||||
// The language of the document this word was met in — the card's own language
|
||||
// (migration 0018). Read-aloud needs it: "comum" is unguessable from letters.
|
||||
lang: DocLang
|
||||
due_at: string
|
||||
interval_days: number
|
||||
ease: number
|
||||
@@ -142,18 +181,39 @@ export interface Suggestion {
|
||||
explanation: string
|
||||
type: SuggestionType
|
||||
status: 'pending' | 'accepted' | 'rejected'
|
||||
// Which engine proposed it — the offline rule pack or the model. The rail
|
||||
// deliberately renders both identically; this is here because the wire format
|
||||
// carries it, not because the writer is ever shown it.
|
||||
source?: 'llm' | 'local'
|
||||
created_at: string
|
||||
}
|
||||
|
||||
// The growth journal (GET /api/suggestions/growth). `kept`/`kept_before` are
|
||||
// the last thirty days and the thirty before them — the only comparison Petal
|
||||
// draws is with her own past self. `stuck` is phrasing she was given that now
|
||||
// turns up across her own documents; `faded` is what she used to be corrected
|
||||
// on and hasn't been lately. Both lists are empty when the data isn't there:
|
||||
// nothing here is padded to fill a page.
|
||||
export interface GrowthJournal {
|
||||
kept: number
|
||||
kept_before: number
|
||||
stuck: { phrase: string; docs: number }[]
|
||||
faded: { pattern: string; times: number }[]
|
||||
}
|
||||
|
||||
// A deterministic, rule-based fix detected client-side (see Companion/prose.ts).
|
||||
// The frontend owns mechanics detection; the backend only persists these as the
|
||||
// 'mechanics' suggestion family. Spans are exact plaintext offsets.
|
||||
// The frontend owns offline detection; the backend only persists these. Spans
|
||||
// are exact plaintext offsets. `type` names the family the finding belongs to:
|
||||
// 'mechanics' for a fix to this sentence, 'collocation' for the miscollocation
|
||||
// rules, whose findings are chunks worth keeping and are filed — and planted in
|
||||
// the garden on accept — exactly like the LLM coach's.
|
||||
export interface MechanicsFinding {
|
||||
from: number
|
||||
to: number
|
||||
original: string
|
||||
replacement: string
|
||||
explanation: string
|
||||
type: 'mechanics' | 'collocation'
|
||||
}
|
||||
|
||||
// One dictionary's worth of personal words — the ones she's excused from
|
||||
@@ -163,6 +223,21 @@ export interface PersonalWords {
|
||||
words: string[]
|
||||
}
|
||||
|
||||
// One pronunciation of a Chinese word, and what it means in that pronunciation.
|
||||
// A list, because 得 is dé "to obtain" and also the particle in 说得很好.
|
||||
export interface HanziReading {
|
||||
pinyin: string
|
||||
senses: string
|
||||
}
|
||||
|
||||
// A Chinese word lookup. `readings` is empty for a word with no headword, in
|
||||
// which case `chars` may carry the character-by-character reading.
|
||||
export interface HanziInfo {
|
||||
word: string
|
||||
readings: HanziReading[]
|
||||
chars: { char: string; pinyin: string; senses: string }[]
|
||||
}
|
||||
|
||||
// Who's writing. Mirrors the backend db.User.
|
||||
export interface Me {
|
||||
id: string
|
||||
@@ -170,6 +245,11 @@ export interface Me {
|
||||
display_name: string
|
||||
created_at: string
|
||||
pair_lang: string
|
||||
// Which half of the pair is being learned: 'learning_en' (the writer is
|
||||
// native in pair_lang and practising English) or 'learning_pair' (the other
|
||||
// way round). Mirrors users.direction; the server refuses 'learning_pair' for
|
||||
// a pair it has no word list for.
|
||||
direction: string
|
||||
}
|
||||
|
||||
// Thrown when the server says the session is gone. Callers can tell it apart
|
||||
@@ -196,11 +276,34 @@ function signedOut(): UnauthorizedError {
|
||||
return new UnauthorizedError()
|
||||
}
|
||||
|
||||
async function req<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
// The document-language verdict is decided by the checkpoint pass, so the pass's
|
||||
// own response is the first moment the client can know it. It rides on a header
|
||||
// (the pass answers with a bare array of suggestions, and every caller reads it
|
||||
// as one), and reaches the app through a handler registered here — the same
|
||||
// shape onUnauthorized already uses, for the same reason: it is one fact from
|
||||
// deep inside a request that a component several layers up needs.
|
||||
//
|
||||
// Without it the editor learns the verdict only from a document save, which is
|
||||
// always one save behind the pass — and read-aloud is reached for precisely when
|
||||
// she has stopped typing and no further save is coming.
|
||||
let docLangHandler: ((docId: string, lang: DocLang) => void) | null = null
|
||||
|
||||
export function onDocLang(handler: (docId: string, lang: DocLang) => void) {
|
||||
docLangHandler = handler
|
||||
}
|
||||
|
||||
// `verdictFor` names the document whose language this response may announce.
|
||||
// Only the three pass endpoints pass it; everything else has no verdict to carry
|
||||
// and never touches the handler.
|
||||
async function req<T>(path: string, init?: RequestInit, verdictFor?: string): Promise<T> {
|
||||
const res = await fetch(`/api${path}`, {
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
...init,
|
||||
})
|
||||
if (verdictFor && res.ok) {
|
||||
const lang = res.headers.get('X-Petal-Doc-Lang')
|
||||
if (lang === '' || lang === 'en' || lang === 'pair') docLangHandler?.(verdictFor, lang)
|
||||
}
|
||||
if (res.status === 401) throw signedOut()
|
||||
if (!res.ok) {
|
||||
const detail = await res.text().catch(() => '')
|
||||
@@ -215,6 +318,22 @@ export const api = {
|
||||
// the hardcoded local user, so the frontend needs no separate mode for it.
|
||||
me: () => req<Me>('/me'),
|
||||
|
||||
// Move to another (English + X) pair. Answers with the whole updated user, so
|
||||
// the caller re-reads the pair from the server rather than assuming its own
|
||||
// request took — a code the server won't ship comes back 400 and the app is
|
||||
// still on a language it can render.
|
||||
setPairLang: (lang: string) =>
|
||||
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ pair_lang: lang }) }),
|
||||
|
||||
// Turn the pair around. Same endpoint, same contract, and deliberately a
|
||||
// separate call: the two fields are validated together server-side, so a
|
||||
// client that wants to change both says both in one request rather than
|
||||
// sending two that each pass on their own.
|
||||
setDirection: (direction: string) =>
|
||||
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ direction }) }),
|
||||
setPair: (lang: string, direction: string) =>
|
||||
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ pair_lang: lang, direction }) }),
|
||||
|
||||
listDocs: () => req<DocSummary[]>('/docs'),
|
||||
createDoc: () => req<Document>('/docs', { method: 'POST' }),
|
||||
getDoc: (id: string) => req<Document>(`/docs/${id}`),
|
||||
@@ -226,14 +345,14 @@ export const api = {
|
||||
// Rate-limited per document server-side (returns the existing set if too soon).
|
||||
// Both passes return the UNIFIED pending set (grammar + voice), so the client
|
||||
// never drops one family's highlights when the other refreshes.
|
||||
checkDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/check`, { method: 'POST' }),
|
||||
checkDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/check`, { method: 'POST' }, id),
|
||||
// Voice-consistency pass: whole-document, explicit-action, slower. Returns the
|
||||
// unified pending set too. Rate-limited per document server-side.
|
||||
voiceDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/voice`, { method: 'POST' }),
|
||||
voiceDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/voice`, { method: 'POST' }, id),
|
||||
// Collocation coach: whole-document, explicit-action pass flagging non-native
|
||||
// word pairings ("do a decision" → "make a decision"). Returns the unified
|
||||
// pending set too. Rate-limited per document server-side.
|
||||
collocationDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/collocation`, { method: 'POST' }),
|
||||
collocationDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/collocation`, { method: 'POST' }, id),
|
||||
// Mechanics pass: persist the client-detected deterministic fixes as the
|
||||
// 'mechanics' family and return the unified pending set. Not rate-limited (it's
|
||||
// free, local detection); runs alongside the grammar checkpoint.
|
||||
@@ -244,6 +363,12 @@ export const api = {
|
||||
}),
|
||||
// Pending suggestions for a doc, loaded when the editor opens it.
|
||||
listSuggestions: (id: string) => req<Suggestion[]>(`/docs/${id}/suggestions`),
|
||||
// The spans she has already accepted or dismissed on this doc, normalized. The
|
||||
// server suppresses these itself; the client needs them so the instant rule-pack
|
||||
// pass doesn't hand back a dismissed card before the server can say otherwise —
|
||||
// or, with the server unreachable, at all. See lib/settled.ts.
|
||||
listSettled: (id: string) =>
|
||||
req<{ originals: string[] }>(`/docs/${id}/settled`),
|
||||
acceptSuggestion: (id: string) =>
|
||||
req<void>(`/suggestions/${id}/accept`, { method: 'POST' }),
|
||||
dismissSuggestion: (id: string) =>
|
||||
@@ -252,6 +377,10 @@ export const api = {
|
||||
// opening bubble (the explanation itself stays English in the card body).
|
||||
translateSuggestion: (id: string) =>
|
||||
req<{ translation: string }>(`/suggestions/${id}/translate`, { method: 'POST' }),
|
||||
// The growth journal: her own accepted edits read back as patterns. Purely a
|
||||
// read-side view of a table Petal already keeps, computed locally with no
|
||||
// model call, so it costs nothing and leaves nothing.
|
||||
growth: () => req<GrowthJournal>('/suggestions/growth'),
|
||||
|
||||
// Version history. listVersions returns metadata only (no bodies); getVersion
|
||||
// loads one full snapshot for preview; snapshotDoc takes an explicit restore
|
||||
@@ -284,6 +413,12 @@ export const api = {
|
||||
// Lightweight Chinese-only gloss for the inline hover/select tooltip — instant
|
||||
// and offline, so it fires on hover without spinning up the heavier lookup.
|
||||
glossWord: (word: string) => req<Gloss>(`/gloss/${encodeURIComponent(word)}`),
|
||||
// The same lookup pointing the other way: a Chinese word to its pinyin and
|
||||
// English senses, for an account learning the pair language rather than
|
||||
// English. A word the dictionary has no headword for comes back with empty
|
||||
// readings and — when its characters are known — a per-character reading
|
||||
// instead, which is a real second answer for a compound.
|
||||
hanziWord: (word: string) => req<HanziInfo>(`/hanzi/${encodeURIComponent(word)}`),
|
||||
// Tone-rewrite: rewrites a selected passage in the given style ('natural',
|
||||
// 'academic', …) and returns the rewritten text for an in-editor preview. Not
|
||||
// persisted — the editor applies it directly on accept.
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { docLang, nativeLang, speak, stopSpeech } from './speech'
|
||||
import { resetPackForTests, setPackLang } from '../i18n'
|
||||
|
||||
// Read-aloud has two jobs beyond "make a sound": ask for the right pace, and ask
|
||||
// in the right language. Both are decided at the call site and travel in the
|
||||
// request body, so this checks the body — the part a component author can get
|
||||
// wrong without anything failing loudly.
|
||||
|
||||
let bodies: Array<Record<string, unknown>>
|
||||
|
||||
beforeEach(() => {
|
||||
bodies = []
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn((_url: string, init: RequestInit) => {
|
||||
bodies.push(JSON.parse(String(init.body)))
|
||||
// Never resolves to audio: the fallback path needs no window.Audio here,
|
||||
// and rejecting would run the Web Speech branch instead of the server one.
|
||||
return new Promise(() => {})
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
stopSpeech()
|
||||
vi.unstubAllGlobals()
|
||||
resetPackForTests()
|
||||
})
|
||||
|
||||
describe('speak', () => {
|
||||
it('asks for the normal pace by default', () => {
|
||||
speak('reception')
|
||||
expect(bodies).toHaveLength(1)
|
||||
expect(bodies[0]).toMatchObject({ text: 'reception', lang: 'en-US', slow: false })
|
||||
})
|
||||
|
||||
it('asks for the slow replay when the slow control is used', () => {
|
||||
speak('reception', undefined, true)
|
||||
expect(bodies[0]).toMatchObject({ text: 'reception', slow: true })
|
||||
})
|
||||
|
||||
it('still detects Chinese by script, so a zh selection is never read in English', () => {
|
||||
speak('你好世界')
|
||||
expect(bodies[0]).toMatchObject({ lang: 'zh-CN' })
|
||||
})
|
||||
|
||||
it('sends nothing for empty text', () => {
|
||||
speak(' ')
|
||||
expect(bodies).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('nativeLang', () => {
|
||||
// The voice for her own language comes from the pack, not from the letters.
|
||||
// "comum" is spelled the same in both halves of the pt pair, so a detector
|
||||
// would have to guess; the component that knows it is rendering her language
|
||||
// says so instead.
|
||||
it('follows the pair language', () => {
|
||||
setPackLang('zh')
|
||||
expect(nativeLang()).toBe('zh-CN')
|
||||
setPackLang('pt-PT')
|
||||
expect(nativeLang()).toBe('pt-PT')
|
||||
setPackLang('fr')
|
||||
expect(nativeLang()).toBe('fr-FR')
|
||||
})
|
||||
|
||||
it('names a European Portuguese voice, never a Brazilian one', () => {
|
||||
setPackLang('pt-PT')
|
||||
expect(nativeLang()).not.toBe('pt-BR')
|
||||
})
|
||||
|
||||
it('is what a Latin-pair lookup speaks the other reading in', () => {
|
||||
setPackLang('pt-PT')
|
||||
speak('comum', nativeLang())
|
||||
expect(bodies[0]).toMatchObject({ text: 'comum', lang: 'pt-PT' })
|
||||
})
|
||||
|
||||
it('speaks the French reading of a collision in French', () => {
|
||||
// "chat" is the sharpest case in the fr pair: an English word, a French
|
||||
// word, and the companion's own animal. Nothing about the letters says
|
||||
// which — only the pack does.
|
||||
setPackLang('fr')
|
||||
speak('chat', nativeLang())
|
||||
expect(bodies.at(-1)).toMatchObject({ text: 'chat', lang: 'fr-FR' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('docLang', () => {
|
||||
// The document's verdict is the answer for a passage lifted out of it, because
|
||||
// for a Latin pair there is no other answer available: an English sentence and
|
||||
// a Portuguese one are the same letters.
|
||||
it('reads a flipped document in her own language', () => {
|
||||
setPackLang('pt-PT')
|
||||
expect(docLang('Ontem foi difícil.', 'pair')).toBe('pt-PT')
|
||||
})
|
||||
|
||||
it('reads an English document in English, and treats the backfill as English', () => {
|
||||
setPackLang('pt-PT')
|
||||
expect(docLang('Yesterday was hard.', 'en')).toBe('en-US')
|
||||
expect(docLang('Yesterday was hard.', '')).toBe('en-US')
|
||||
})
|
||||
|
||||
it('lets the script win over the verdict, so quoted Chinese is never spelled out', () => {
|
||||
// An English document quoting Chinese is 'en' by verdict, and the English
|
||||
// voice reads Han characters one "Chinese letter" at a time — the one
|
||||
// failure worse than silence.
|
||||
setPackLang('zh')
|
||||
expect(docLang('你好世界', 'en')).toBe('zh-CN')
|
||||
})
|
||||
|
||||
it('is what the editor selection and the garden card ask for', () => {
|
||||
setPackLang('fr')
|
||||
speak('Le chat dort.', docLang('Le chat dort.', 'pair'))
|
||||
expect(bodies.at(-1)).toMatchObject({ text: 'Le chat dort.', lang: 'fr-FR' })
|
||||
})
|
||||
})
|
||||
+44
-9
@@ -6,6 +6,8 @@
|
||||
// (TTS disabled) or unreachable, we fall back to the browser's Web Speech API so
|
||||
// the buttons still do something. No model or network is strictly required.
|
||||
|
||||
import { pack } from '../i18n'
|
||||
|
||||
// speechSupported reports whether read-aloud can do anything at all. Audio
|
||||
// playback is universal, so as long as we can construct an Audio element OR the
|
||||
// Web Speech API exists, the buttons should show. The server path is tried at
|
||||
@@ -51,8 +53,10 @@ function pickVoice(lang: string): SpeechSynthesisVoice | undefined {
|
||||
}
|
||||
|
||||
// speakWebSpeech is the fallback: the browser's built-in synthesizer. A touch
|
||||
// slower than default so learners can follow along.
|
||||
function speakWebSpeech(text: string, lang: string): void {
|
||||
// slower than default so learners can follow along, and slower still when the
|
||||
// slow replay was asked for — the fallback should degrade in voice quality, not
|
||||
// in what the button does.
|
||||
function speakWebSpeech(text: string, lang: string, slow: boolean): void {
|
||||
if (!webSpeechSupported()) return
|
||||
const synth = window.speechSynthesis
|
||||
synth.cancel()
|
||||
@@ -60,7 +64,7 @@ function speakWebSpeech(text: string, lang: string): void {
|
||||
utterance.lang = lang
|
||||
const voice = pickVoice(lang)
|
||||
if (voice) utterance.voice = voice
|
||||
utterance.rate = 0.95
|
||||
utterance.rate = slow ? 0.7 : 0.95
|
||||
synth.speak(utterance)
|
||||
}
|
||||
|
||||
@@ -74,13 +78,44 @@ export function detectLang(text: string): string {
|
||||
return CJK.test(text) ? 'zh-CN' : 'en-US'
|
||||
}
|
||||
|
||||
// nativeLang is the locale of the writer's own language — the voice for the
|
||||
// *other* reading of a word that exists in both halves of a Latin pair.
|
||||
//
|
||||
// It is asked for explicitly rather than detected, and that is the point. A
|
||||
// script boundary can be detected (the CJK test above); "comum" cannot. So the
|
||||
// component that knows it is rendering her language says so, and everything
|
||||
// rendering English lets the default stand. No guess, therefore no wrong guess
|
||||
// about her writing — the same rule the both-directions gloss follows.
|
||||
export function nativeLang(): string {
|
||||
return pack().locale
|
||||
}
|
||||
|
||||
// docLang turns a document-language verdict ('' | 'en' | 'pair', decided
|
||||
// server-side — see internal/suggestions/doclang.go) into a locale for a passage
|
||||
// taken out of that document. It is what the editor's read-aloud and the garden's
|
||||
// review card ask instead of guessing.
|
||||
//
|
||||
// The script test still wins, and that is not redundant with the verdict. A
|
||||
// Chinese sentence quoted inside an English document is 'en' by verdict and
|
||||
// still has to be read by the Chinese voice: the English voice spells Han
|
||||
// characters out one "Chinese letter" at a time, which is the one failure loud
|
||||
// enough to be worse than no audio. In the other direction there is nothing to
|
||||
// test — an English sentence inside Portuguese prose looks exactly like the
|
||||
// Portuguese around it — so the document's verdict is the only answer available,
|
||||
// and it is the answer this phase decided on.
|
||||
export function docLang(text: string, verdict: string): string {
|
||||
if (CJK.test(text)) return 'zh-CN'
|
||||
return verdict === 'pair' ? pack().locale : 'en-US'
|
||||
}
|
||||
|
||||
// speak reads `text` aloud, cancelling anything already in flight so rapid taps
|
||||
// don't queue up. `lang` defaults to a guess from the text (Chinese vs English)
|
||||
// so callers can just pass the selection; pass an explicit locale to override.
|
||||
// It tries the server's neural voice first and silently falls back to the browser
|
||||
// voice if that's unavailable (route off, network error, or a 404 for a language
|
||||
// with no configured voice).
|
||||
export function speak(text: string, lang = detectLang(text)): void {
|
||||
// `slow` asks for the stretched replay (SUGGESTIONS §5e) — the second tap on a
|
||||
// sentence that went by too fast. It tries the server's neural voice first and
|
||||
// silently falls back to the browser voice if that's unavailable (route off,
|
||||
// network error, or a 404 for a language with no configured voice).
|
||||
export function speak(text: string, lang = detectLang(text), slow = false): void {
|
||||
if (!text.trim()) return
|
||||
stopSpeech()
|
||||
const seq = ++requestSeq
|
||||
@@ -88,7 +123,7 @@ export function speak(text: string, lang = detectLang(text)): void {
|
||||
fetch('/api/tts', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ text, lang }),
|
||||
body: JSON.stringify({ text, lang, slow }),
|
||||
})
|
||||
.then((res) => {
|
||||
if (!res.ok) throw new Error(`tts ${res.status}`)
|
||||
@@ -115,6 +150,6 @@ export function speak(text: string, lang = detectLang(text)): void {
|
||||
// Server TTS unavailable for this request — use the browser voice instead,
|
||||
// unless a newer tap has already superseded this one.
|
||||
if (seq !== requestSeq) return
|
||||
speakWebSpeech(text, lang)
|
||||
speakWebSpeech(text, lang, slow)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { SaveStatus } from '../../hooks/useAutoSave'
|
||||
import { useCompanion, type Mood } from './useCompanion'
|
||||
import { LottiePlayer } from './LottiePlayer'
|
||||
import { COMPANIONS, DEFAULT_COMPANION } from './companions'
|
||||
import { useCardOverlap } from './useCardOverlap'
|
||||
import { onPrefsScopeChange, readPref, writePref } from '../../lib/prefs'
|
||||
import { usePack } from '../../i18n'
|
||||
|
||||
@@ -13,6 +14,11 @@ interface Props {
|
||||
editTick: number
|
||||
acceptTick: number
|
||||
text: string
|
||||
// The open document is still empty — the one state the daily writing
|
||||
// invitation is offered in.
|
||||
blankPage: boolean
|
||||
// Called when she takes the invitation up, with the English prompt.
|
||||
onAcceptInvitation: (prompt: string) => void
|
||||
}
|
||||
|
||||
// Emoji placeholder per mood, used for any mood a companion has no Lottie for.
|
||||
@@ -30,17 +36,38 @@ const STORAGE_KEY = 'petal.companion'
|
||||
// useCompanion and shows a Mandarin-first speech bubble for cheers, tips, and
|
||||
// break reminders. Clicking the mascot opens a picker to switch companions
|
||||
// (the choice persists in localStorage).
|
||||
export function PetalCompanion({ wordCount, saveStatus, llmDown, editTick, acceptTick, text }: Props) {
|
||||
const t = usePack()
|
||||
const { mood, bubble, dismiss, holdBubble, releaseBubble } = useCompanion({
|
||||
export function PetalCompanion({
|
||||
wordCount,
|
||||
saveStatus,
|
||||
llmDown,
|
||||
editTick,
|
||||
acceptTick,
|
||||
text,
|
||||
blankPage,
|
||||
onAcceptInvitation,
|
||||
}: Props) {
|
||||
const t = usePack()
|
||||
const {
|
||||
mood,
|
||||
bubble,
|
||||
dismiss,
|
||||
holdBubble,
|
||||
releaseBubble,
|
||||
acceptInvite,
|
||||
declineInvite,
|
||||
setInviteHandler,
|
||||
} = useCompanion({
|
||||
wordCount,
|
||||
saveStatus,
|
||||
llmDown,
|
||||
editTick,
|
||||
acceptTick,
|
||||
text,
|
||||
blankPage,
|
||||
})
|
||||
|
||||
useEffect(() => setInviteHandler(onAcceptInvitation), [setInviteHandler, onAcceptInvitation])
|
||||
|
||||
const [companionId, setCompanionId] = useState<string>(
|
||||
() => readPref(STORAGE_KEY) || DEFAULT_COMPANION,
|
||||
)
|
||||
@@ -60,6 +87,20 @@ export function PetalCompanion({ wordCount, saveStatus, llmDown, editTick, accep
|
||||
const companion = COMPANIONS.find((c) => c.id === companionId) ?? COMPANIONS[0]
|
||||
const [pickerOpen, setPickerOpen] = useState(false)
|
||||
const rootRef = useRef<HTMLDivElement>(null)
|
||||
const badgeRef = useRef<HTMLButtonElement>(null)
|
||||
// Stand-in for the badge's corner, used only for measuring — see
|
||||
// useCardOverlap. It sits exactly where the badge sits but never bobs, shrinks
|
||||
// or hovers, so what the mascot yields to can't depend on whether it is
|
||||
// currently yielding.
|
||||
const probeRef = useRef<HTMLSpanElement>(null)
|
||||
|
||||
// When suggestion cards stack down into the corner, the kitten fades to
|
||||
// translucent and shrinks a step so the card stays readable and clickable.
|
||||
// It wakes back up whenever it has something to say (bubble) or is being
|
||||
// interacted with (picker open) — except under an open History or Garden
|
||||
// panel, where even a cheer would cover the controls she just reached for.
|
||||
const crowded = useCardOverlap(probeRef)
|
||||
const faded = crowded.modal || (crowded.cards && !pickerOpen && !bubble)
|
||||
|
||||
// Awake companions (no sleeping clip) don't visibly nap — when the engine
|
||||
// dozes them, keep their normal idle pose instead of a sleepy face. Only a
|
||||
@@ -99,7 +140,7 @@ export function PetalCompanion({ wordCount, saveStatus, llmDown, editTick, accep
|
||||
return (
|
||||
<div
|
||||
ref={rootRef}
|
||||
className="pointer-events-none fixed bottom-4 right-4 z-40 flex flex-col items-end gap-2"
|
||||
className="petal-corner pointer-events-none fixed bottom-4 right-4 z-40 flex flex-col items-end gap-2"
|
||||
>
|
||||
{pickerOpen && (
|
||||
<div
|
||||
@@ -150,7 +191,10 @@ export function PetalCompanion({ wordCount, saveStatus, llmDown, editTick, accep
|
||||
</div>
|
||||
)}
|
||||
|
||||
{bubble && !pickerOpen && (
|
||||
{/* The bubble is its own layer, so fading the badge doesn't hide it —
|
||||
hold it back explicitly while a panel is open. useCompanion keeps the
|
||||
bubble in state, so it reappears when she closes the panel. */}
|
||||
{bubble && !pickerOpen && !crowded.modal && (
|
||||
<div
|
||||
role="status"
|
||||
onClick={dismiss}
|
||||
@@ -179,15 +223,58 @@ export function PetalCompanion({ wordCount, saveStatus, llmDown, editTick, accep
|
||||
>
|
||||
{bubble.en}
|
||||
</p>
|
||||
|
||||
{/* The daily invitation's two answers. "Not today" is a real button
|
||||
sitting level with the other one, not a small grey escape — a no
|
||||
that has to be hunted for isn't much of a no. */}
|
||||
{bubble.invite && (
|
||||
<div className="mt-3 flex flex-wrap items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
acceptInvite(bubble.invite!.prompt)
|
||||
}}
|
||||
className="rounded-full px-3.5 py-1.5 text-sm font-bold"
|
||||
style={{ background: 'var(--color-accent)', color: 'var(--color-plum)' }}
|
||||
>
|
||||
{t.companion.inviteAccept}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
declineInvite()
|
||||
}}
|
||||
className="rounded-full px-3.5 py-1.5 text-sm font-semibold"
|
||||
style={{ background: 'var(--color-surface-alt)', color: 'var(--color-plum)' }}
|
||||
>
|
||||
{t.companion.inviteDecline}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<span
|
||||
ref={probeRef}
|
||||
aria-hidden
|
||||
className="pointer-events-none absolute bottom-0 right-0"
|
||||
style={{
|
||||
width: 'var(--petal-companion-size, 9rem)',
|
||||
height: 'var(--petal-companion-size, 9rem)',
|
||||
}}
|
||||
/>
|
||||
|
||||
<button
|
||||
ref={badgeRef}
|
||||
type="button"
|
||||
onClick={() => setPickerOpen((o) => !o)}
|
||||
title="Choose a companion"
|
||||
aria-label="Choose a companion"
|
||||
className={`petal-companion pointer-events-auto select-none ${napping ? 'petal-companion-sleep' : ''}`}
|
||||
className={`petal-companion select-none ${faded ? 'petal-companion-faded' : 'pointer-events-auto'}${
|
||||
napping ? ' petal-companion-sleep' : ''
|
||||
}`}
|
||||
style={{
|
||||
// Size scales with the viewport — see --petal-companion-size in index.css.
|
||||
width: 'var(--petal-companion-size)',
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { analyzeProse, mechanicsFindings } from './prose'
|
||||
import { resetPackForTests, setPackLang } from '../../i18n'
|
||||
|
||||
// Phase 22's offline half: the grammar-lite rule pack, the embedded
|
||||
// miscollocation list, the false-friend heads-up, and the per-pair L1 rules.
|
||||
//
|
||||
// The bar these tests enforce is the one the pack promises: **precision over
|
||||
// recall**. Every rule is pinned in two directions — the mistake it must catch,
|
||||
// and the correct English next to it that it must leave alone. A rule that
|
||||
// cannot be guarded that way was left out of the pack rather than tested
|
||||
// loosely here.
|
||||
|
||||
afterEach(() => resetPackForTests())
|
||||
|
||||
const rules = (text: string) => analyzeProse(text).map((h) => h.rule)
|
||||
const findings = (text: string) => mechanicsFindings(text)
|
||||
|
||||
// The one-click fix a given rule produced, if any.
|
||||
function fix(text: string, original: string) {
|
||||
return findings(text).find((f) => f.original.toLowerCase() === original.toLowerCase())
|
||||
}
|
||||
|
||||
// Every finding must be able to anchor: its span has to be exactly the text it
|
||||
// claims, or the editor applies the edit to the wrong characters.
|
||||
function expectExactSpans(text: string) {
|
||||
for (const f of findings(text)) {
|
||||
expect(text.slice(f.from, f.to), `span mismatch for "${f.original}"`).toBe(f.original)
|
||||
}
|
||||
return findings(text)
|
||||
}
|
||||
|
||||
describe('prepositions', () => {
|
||||
it('depend of → depend on, keeping the writer\'s own verb form', () => {
|
||||
expect(fix('It all depends of the weather on the day we travel.', 'depends of')?.replacement).toBe(
|
||||
'depends on',
|
||||
)
|
||||
expect(fix('Depending of the weather we will go to the beach today.', 'Depending of')?.replacement).toBe(
|
||||
'Depending on',
|
||||
)
|
||||
})
|
||||
|
||||
it('discuss about → discuss (the preposition simply goes)', () => {
|
||||
expect(fix('We discussed about the plan for a long time yesterday.', 'discussed about')?.replacement).toBe(
|
||||
'discussed',
|
||||
)
|
||||
})
|
||||
|
||||
it('explain me → explain to me', () => {
|
||||
expect(fix('Can you explain me the rules of this game again please.', 'explain me')?.replacement).toBe(
|
||||
'explain to me',
|
||||
)
|
||||
})
|
||||
|
||||
it('listen the radio → listen to the radio', () => {
|
||||
expect(fix('I listen the radio every morning while I make my coffee.', 'listen the')?.replacement).toBe(
|
||||
'listen to the',
|
||||
)
|
||||
})
|
||||
|
||||
// The pairings deliberately NOT in the list, because they are only usually
|
||||
// wrong. Each of these is correct English and must stay silent.
|
||||
it('leaves the correct prepositions alone', () => {
|
||||
expect(rules('It all depends on the weather on the day we travel.')).not.toContain('preposition')
|
||||
expect(rules('We discussed the plan for a long time yesterday afternoon.')).not.toContain('preposition')
|
||||
expect(rules('I listen to the radio every morning while I make coffee.')).not.toContain('preposition')
|
||||
// Left out of the pack on purpose — "married with children" is a phrase,
|
||||
// "arrive to" wants at or in, "different than" is ordinary American usage.
|
||||
expect(rules('She is married with children and lives near the old harbour.')).not.toContain('preposition')
|
||||
expect(rules('This result is different than the one we saw last week.')).not.toContain('preposition')
|
||||
})
|
||||
})
|
||||
|
||||
describe('doubled comparatives', () => {
|
||||
it('more better → better', () => {
|
||||
expect(fix('This one is more better than the other one we tried.', 'more better')?.replacement).toBe('better')
|
||||
})
|
||||
|
||||
it('most easiest → easiest', () => {
|
||||
expect(fix('That was the most easiest question on the whole exam paper.', 'most easiest')?.replacement).toBe(
|
||||
'easiest',
|
||||
)
|
||||
})
|
||||
|
||||
// The guard the generic /\w+er/ pattern would have failed: these are correct.
|
||||
it('leaves ordinary "more/most + adjective" alone', () => {
|
||||
expect(rules('She is more clever than anyone else in the whole class.')).not.toContain('doublecomp')
|
||||
expect(rules('He was the most eager student in the room that morning.')).not.toContain('doublecomp')
|
||||
expect(rules('This is the most beautiful garden I have ever seen here.')).not.toContain('doublecomp')
|
||||
})
|
||||
})
|
||||
|
||||
describe('people is', () => {
|
||||
it('people is → people are, and people has → people have', () => {
|
||||
expect(fix('Many people is waiting outside the hall in the rain.', 'people is')?.replacement).toBe('people are')
|
||||
expect(fix('Some people has never seen the sea in their whole life.', 'people has')?.replacement).toBe(
|
||||
'people have',
|
||||
)
|
||||
})
|
||||
|
||||
it('leaves the plural alone', () => {
|
||||
expect(rules('Many people are waiting outside the hall in the rain.')).not.toContain('peopleare')
|
||||
})
|
||||
})
|
||||
|
||||
describe('miscollocations', () => {
|
||||
// The whole point of the family: these file as 'collocation', not
|
||||
// 'mechanics', so an accepted chunk plants in the vocabulary garden exactly
|
||||
// as one the LLM coach proposed would.
|
||||
it('files as the collocation family, not as mechanics', () => {
|
||||
const f = fix('I had to do a decision about the job offer quickly.', 'do a decision')
|
||||
expect(f?.replacement).toBe('make a decision')
|
||||
expect(f?.type).toBe('collocation')
|
||||
})
|
||||
|
||||
it('mechanics fixes keep their own family', () => {
|
||||
expect(fix('I saw the the cat in the garden this morning.', 'the the')?.type).toBe('mechanics')
|
||||
})
|
||||
|
||||
it('agrees with the tense the writer was already using', () => {
|
||||
expect(fix('She did a mistake on the form and had to start again.', 'did a mistake')?.replacement).toBe(
|
||||
'made a mistake',
|
||||
)
|
||||
expect(fix('He is making his homework at the kitchen table right now.', 'making his homework')?.replacement).toBe(
|
||||
'doing his homework',
|
||||
)
|
||||
})
|
||||
|
||||
it('say me → tell me', () => {
|
||||
expect(fix('Please say me what happened at the meeting this afternoon.', 'say me')?.replacement).toBe('tell me')
|
||||
})
|
||||
|
||||
it('make a photo → take a photo', () => {
|
||||
expect(fix('We made a photo together in front of the old church.', 'made a photo')?.replacement).toBe(
|
||||
'took a photo',
|
||||
)
|
||||
})
|
||||
|
||||
it('strong rain → heavy rain', () => {
|
||||
expect(fix('There was strong rain all afternoon and we stayed inside.', 'strong rain')?.replacement).toBe(
|
||||
'heavy rain',
|
||||
)
|
||||
})
|
||||
|
||||
// "strong wind" is the correct pairing, and the rule that fixes "big wind"
|
||||
// must not propose it as a change to itself.
|
||||
it('never proposes a phrase identical to what she wrote', () => {
|
||||
expect(rules('There was a strong wind blowing across the open field today.')).not.toContain('collocation')
|
||||
expect(fix('There was a big wind blowing across the open field today.', 'big wind')?.replacement).toBe(
|
||||
'strong wind',
|
||||
)
|
||||
})
|
||||
|
||||
it('leaves the correct pairings alone', () => {
|
||||
expect(rules('I had to make a decision about the job offer quickly.')).not.toContain('collocation')
|
||||
expect(rules('She does her homework at the kitchen table every evening.')).not.toContain('collocation')
|
||||
expect(rules('We took a photo together in front of the old church.')).not.toContain('collocation')
|
||||
})
|
||||
})
|
||||
|
||||
describe('false friends', () => {
|
||||
it('flags a Portuguese false friend for the pt-PT pair, awareness-only', () => {
|
||||
setPackLang('pt-PT')
|
||||
const hints = analyzeProse('I will eventually finish the report before the end of the week.')
|
||||
const ff = hints.find((h) => h.rule === 'falsefriend')
|
||||
expect(ff).toBeDefined()
|
||||
// Never a card: the word may well be the one she meant, and a one-click
|
||||
// "fix" would be Petal deciding that for her.
|
||||
expect(ff?.fix).toBeUndefined()
|
||||
expect(findings('I will eventually finish the report before the end of the week.')).toEqual([])
|
||||
})
|
||||
|
||||
it('raises at most one per pass — a heads-up, not a sweep', () => {
|
||||
setPackLang('pt-PT')
|
||||
const hints = analyzeProse(
|
||||
'Actually I did not pretend to assist the lecture at the library this week.',
|
||||
)
|
||||
expect(hints.filter((h) => h.rule === 'falsefriend')).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('says nothing for the zh pair, which has no false friends at all', () => {
|
||||
setPackLang('zh')
|
||||
expect(rules('I will eventually finish the report before the end of the week.')).not.toContain('falsefriend')
|
||||
})
|
||||
})
|
||||
|
||||
describe('per-pair L1 interference', () => {
|
||||
it('pt-PT: "have 30 years" → "am 30 years old"', () => {
|
||||
setPackLang('pt-PT')
|
||||
expect(fix('My sister I have 30 years and she is older than me.', 'I have 30 years')?.replacement).toBe(
|
||||
'I am 30 years old',
|
||||
)
|
||||
// The subject and tense she wrote in are carried into the correction.
|
||||
expect(fix('When we met she had twenty years and I was still at school.', 'she had twenty years')?.replacement).toBe(
|
||||
'she was twenty years old',
|
||||
)
|
||||
})
|
||||
|
||||
it('pt-PT: "I am agree" → "I agree"', () => {
|
||||
setPackLang('pt-PT')
|
||||
expect(fix('I am agree with everything that was said at the meeting.', 'I am agree')?.replacement).toBe('I agree')
|
||||
})
|
||||
|
||||
it('pt-PT: "since three years" → "for three years"', () => {
|
||||
setPackLang('pt-PT')
|
||||
expect(fix('I have lived in this city since three years and I love it.', 'since three years')?.replacement).toBe(
|
||||
'for three years',
|
||||
)
|
||||
})
|
||||
|
||||
it('pt-PT: leaves "since" with a starting point alone', () => {
|
||||
setPackLang('pt-PT')
|
||||
expect(rules('I have lived in this city since 2020 and I still love it.')).not.toContain('since')
|
||||
})
|
||||
|
||||
it('zh: "very like" → "really like", and "open the light" → "turn on the light"', () => {
|
||||
setPackLang('zh')
|
||||
expect(fix('I very like the small garden behind my grandmother house.', 'very like')?.replacement).toBe(
|
||||
'really like',
|
||||
)
|
||||
expect(fix('Please open the light before you come into the dark room.', 'open the light')?.replacement).toBe(
|
||||
'turn on the light',
|
||||
)
|
||||
expect(fix('She closed the television and went straight to bed last night.', 'closed the television')?.replacement).toBe(
|
||||
'turned off the television',
|
||||
)
|
||||
})
|
||||
|
||||
it('zh: although…but is awareness-only — the "but" is too common to anchor a card to', () => {
|
||||
setPackLang('zh')
|
||||
const text = 'Although it was raining hard, but we still went to the park.'
|
||||
expect(rules(text)).toContain('althoughbut')
|
||||
expect(findings(text).some((f) => f.original.includes('but'))).toBe(false)
|
||||
})
|
||||
|
||||
it('zh: leaves "very + adjective" alone', () => {
|
||||
setPackLang('zh')
|
||||
expect(rules('I am very happy about the small garden behind the house.')).not.toContain('veryverb')
|
||||
})
|
||||
|
||||
// The gating is the reason these rules can be confident. A rule that is a
|
||||
// near-certainty for one L1 is only a guess for another, and a guess does not
|
||||
// belong in a rule pack that runs on every keystroke.
|
||||
it('does not run one pair\'s interference rules for the other pair', () => {
|
||||
setPackLang('zh')
|
||||
expect(rules('I have 30 years and I still live near the old harbour.')).not.toContain('haveyears')
|
||||
setPackLang('pt-PT')
|
||||
expect(rules('I very like the small garden behind my grandmother house.')).not.toContain('veryverb')
|
||||
})
|
||||
})
|
||||
|
||||
describe('spans stay exact across every new rule', () => {
|
||||
it('anchors each finding to the text it claims', () => {
|
||||
setPackLang('pt-PT')
|
||||
expectExactSpans(
|
||||
'I have 30 years and I am agree that we depends of the weather, and she did a mistake since three years.',
|
||||
)
|
||||
setPackLang('zh')
|
||||
expectExactSpans('I very like to open the light, and many people is more better at it than me.')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,141 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { gunzipSync } from 'node:zlib'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { CONFUSION_PAIRS, hanziFindings } from './hanzi'
|
||||
import { buildSegmenter } from '../../lib/segment'
|
||||
|
||||
// The 错别字 pack, held to the bar Phase 22 set for the English rule pack: every
|
||||
// rule pinned in *two* directions — the mistake it must catch, and the correct
|
||||
// writing next to it that it must leave alone.
|
||||
//
|
||||
// Here the second direction is the one that matters, and it is unusually easy to
|
||||
// get wrong. Chinese has no spaces, so every one of these rules is a substring
|
||||
// match on running text, and for most of them there exists an ordinary correct
|
||||
// sentence that contains the substring across a word boundary. Those sentences
|
||||
// are the real test.
|
||||
|
||||
const raw = gunzipSync(readFileSync(new URL('../../../public/dictionaries/zh/words.txt.gz', import.meta.url)))
|
||||
const seg = buildSegmenter(raw.toString('utf8'))
|
||||
|
||||
const flagged = (text: string) => hanziFindings(text, seg).map((f) => `${f.original}→${f.replacement}`)
|
||||
|
||||
describe('the gate that admits a rule', () => {
|
||||
// The pack's own claim about itself, checked against the shipped dictionary
|
||||
// rather than asserted in a comment. A pair whose wrong form is a real word
|
||||
// cannot be decided mechanically and does not belong here.
|
||||
it('every wrong form is not a word, and every right form is', () => {
|
||||
for (const { wrong, right } of CONFUSION_PAIRS) {
|
||||
expect(seg.has(wrong), `${wrong} is a dictionary word and must not be flagged`).toBe(false)
|
||||
expect(seg.has(right), `${right} is not a dictionary word`).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
// The errors this pack deliberately refuses, and why — each is a genuine
|
||||
// mistake by a modern standard whose wrong form is itself a headword. If a
|
||||
// dictionary rebuild ever drops one of these, this test fails and the pair
|
||||
// becomes admissible; that is the intended way to find out.
|
||||
it('refuses the well-known errors it cannot decide', () => {
|
||||
for (const undecidable of ['自已', '好象', '倒底', '帐号', '部份']) {
|
||||
expect(seg.has(undecidable), `${undecidable} is no longer a word — reconsider the rule`).toBe(true)
|
||||
expect(flagged(`这是${undecidable}的例子`)).toEqual([])
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('the mistakes it catches', () => {
|
||||
it('已 / 己 / 以', () => {
|
||||
expect(flagged('我己经写完了作业')).toEqual(['己经→已经'])
|
||||
expect(flagged('我以经吃过饭了')).toEqual(['以经→已经'])
|
||||
expect(flagged('下课已后我们去公园')).toEqual(['已后→以后'])
|
||||
})
|
||||
|
||||
it('在 / 再', () => {
|
||||
expect(flagged('明天在见')).toEqual(['在见→再见'])
|
||||
expect(flagged('他正再看书')).toEqual(['正再→正在'])
|
||||
expect(flagged('现再几点了')).toEqual(['现再→现在'])
|
||||
})
|
||||
|
||||
it('做 / 作', () => {
|
||||
expect(flagged('我的工做很忙')).toEqual(['工做→工作'])
|
||||
expect(flagged('老师给我们很多做业')).toEqual(['做业→作业'])
|
||||
expect(flagged('这本书的做者是谁')).toEqual(['做者→作者'])
|
||||
})
|
||||
|
||||
it('the rest', () => {
|
||||
expect(flagged('我觉的这个很好')).toEqual(['觉的→觉得'])
|
||||
expect(flagged('你因该早点睡')).toEqual(['因该→应该'])
|
||||
expect(flagged('即然你来了就坐下吧')).toEqual(['即然→既然'])
|
||||
expect(flagged('你知到吗')).toEqual(['知到→知道'])
|
||||
expect(flagged('请输入你的蜜码')).toEqual(['蜜码→密码'])
|
||||
})
|
||||
|
||||
it('reports an exact span, so the card replaces the right characters', () => {
|
||||
const text = '我己经到了'
|
||||
const [f] = hanziFindings(text, seg)
|
||||
expect(text.slice(f.from, f.to)).toBe('己经')
|
||||
expect(text.slice(0, f.from) + f.replacement + text.slice(f.to)).toBe('我已经到了')
|
||||
})
|
||||
|
||||
it('finds every occurrence, in document order', () => {
|
||||
expect(flagged('我己经吃了,他也己经吃了')).toEqual(['己经→已经', '己经→已经'])
|
||||
expect(flagged('我的工做很忙,所以我觉的很累')).toEqual(['工做→工作', '觉的→觉得'])
|
||||
})
|
||||
})
|
||||
|
||||
// ── the direction that matters ──────────────────────────────────────────────
|
||||
|
||||
describe('the correct writing it must not touch', () => {
|
||||
// Each of these is an ordinary sentence containing a flagged substring across
|
||||
// a word boundary. Without the boundary gate, every one would be corrupted —
|
||||
// and corrupted silently, into text that is still made of real characters.
|
||||
it('leaves two real words alone where they happen to abut', () => {
|
||||
// 自己 + 经常. The substring is 己经.
|
||||
expect(flagged('他自己经常做饭')).toEqual([])
|
||||
// 睡觉 + 的. The substring is 觉的.
|
||||
expect(flagged('睡觉的时候不要看手机')).toEqual([])
|
||||
// 感觉 + 的.
|
||||
expect(flagged('这是我感觉的方向')).toEqual([])
|
||||
// 不知 + 到底.
|
||||
expect(flagged('我不知到底该怎么办')).toEqual([])
|
||||
// 因 + 位置.
|
||||
expect(flagged('因位置不好我们换了座位')).toEqual([])
|
||||
// 已 + 后悔.
|
||||
expect(flagged('他已后悔了')).toEqual([])
|
||||
})
|
||||
|
||||
it('leaves ordinary correct prose entirely alone', () => {
|
||||
for (const good of [
|
||||
'我今天早上去公园跑步了',
|
||||
'他的中文说得很好',
|
||||
'我已经完成了我的作业',
|
||||
'现在几点了,我们再见面吧',
|
||||
'我觉得这个工作很有意思',
|
||||
'既然你已经知道了,就按照计划做',
|
||||
]) {
|
||||
expect(flagged(good), good).toEqual([])
|
||||
}
|
||||
})
|
||||
|
||||
// Where the gate costs the pack a real catch, and the trade it is making.
|
||||
// 不知 is itself a word, so 我不知到他在哪里 — which really is 知到 for 知道 —
|
||||
// reads to the segmenter as 不知 + 到 and is left alone. That is the gate
|
||||
// preferring a missed error to a corrupted sentence, which is the whole
|
||||
// premise: 我不知到底该怎么办 is the same three characters and is correct.
|
||||
it('declines a real error rather than risk the sentence beside it', () => {
|
||||
expect(flagged('我不知到他在哪里')).toEqual([])
|
||||
expect(flagged('你知到吗')).toEqual(['知到→知道'])
|
||||
})
|
||||
|
||||
it('says nothing about English, or about nothing', () => {
|
||||
expect(flagged('I already finished my homework')).toEqual([])
|
||||
expect(flagged('')).toEqual([])
|
||||
})
|
||||
|
||||
// The direction gate. The word list is loaded only for an account learning
|
||||
// Chinese, so without one this pack is silent — a writer practising English
|
||||
// must never be told her own quoted Chinese is wrong.
|
||||
it('is silent without a segmenter, which is how the direction gate works', () => {
|
||||
expect(hanziFindings('我己经写完了', null)).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,149 @@
|
||||
import type { MechanicsFinding } from '../../api/client'
|
||||
import type { Segmenter } from '../../lib/segment'
|
||||
|
||||
// 错别字 — wrong-character detection, the Chinese counterpart of the spell
|
||||
// checker, and a different problem from the one Hunspell solves.
|
||||
//
|
||||
// Chinese has no misspellings in the English sense: every character a writer can
|
||||
// type is a real character, correctly formed, and an IME will not offer one that
|
||||
// is not. What it *will* offer is the wrong one. Typing pinyin `yijing` and
|
||||
// taking the first candidate gives 已经 or 己经 depending on the moment, and both
|
||||
// are made of real characters. So the unit of error is not a malformed word but
|
||||
// a **substituted character inside a correct-looking one** — which is why this
|
||||
// is a rule pack over confusable pairs rather than a dictionary membership test.
|
||||
//
|
||||
// The discipline is Phase 22's, and the bar is the same: **precision over
|
||||
// recall**. A wrong nudge costs more trust than a missed one earns, and it costs
|
||||
// double here, because a learner has no way to know the tool is wrong. Two
|
||||
// mechanical gates enforce it, and both are checked in the tests rather than
|
||||
// asserted in prose.
|
||||
|
||||
// A confusable pair: `wrong` is never a word, `right` is what was meant.
|
||||
//
|
||||
// **Gate one — the pair must be decidable by the dictionary.** Each entry is
|
||||
// admitted only if `wrong` is absent from the 188k-word list *and* `right` is
|
||||
// present. That is what makes the correction a fact rather than a preference,
|
||||
// and it is checked against the shipped asset in hanzi.test.ts.
|
||||
//
|
||||
// It is also the gate that keeps out errors everyone knows are errors. 自已 for
|
||||
// 自己 is among the commonest slips in written Chinese, and 自已 is itself a
|
||||
// dictionary headword — so this pack does not flag it, exactly as Phase 22's
|
||||
// English pack left out `married with`. The same fate for 好象 (an older form of
|
||||
// 好像, still in the dictionary), 倒底, 帐号 and 部份: all real errors by a modern
|
||||
// standard, none of them decidable here.
|
||||
interface Confusion {
|
||||
wrong: string
|
||||
right: string
|
||||
// The note on the card. English, because this pack only ever runs for a writer
|
||||
// whose English is the language they think in — see the direction gate below.
|
||||
why: string
|
||||
}
|
||||
|
||||
const CONFUSIONS: Confusion[] = [
|
||||
// 已 / 己 / 以 — three characters that differ by one stroke and share a
|
||||
// syllable. The most productive source of 错别字 there is.
|
||||
{ wrong: '己经', right: '已经', why: '已经 (already) — 己 is the "self" character; the one you want is 已.' },
|
||||
{ wrong: '以经', right: '已经', why: '已经 (already) — 以 is a different word; 已 is the one that means "already".' },
|
||||
{ wrong: '已后', right: '以后', why: '以后 (afterwards) takes 以, not 已.' },
|
||||
|
||||
// 在 / 再 — same pinyin (zài), completely different jobs: one is location and
|
||||
// ongoing action, the other is repetition.
|
||||
{ wrong: '在见', right: '再见', why: '再见 (goodbye) — 再 is "again", which is what "see you again" needs.' },
|
||||
{ wrong: '正再', right: '正在', why: '正在 (in the middle of doing) takes 在, the one about being somewhere.' },
|
||||
{ wrong: '现再', right: '现在', why: '现在 (now) takes 在.' },
|
||||
|
||||
// 做 / 作 — both zuò, both "to do", and which one a compound takes is simply
|
||||
// fixed by convention. A learner cannot reason it out, which is what makes a
|
||||
// reminder worth having.
|
||||
{ wrong: '工做', right: '工作', why: '工作 (work) is written with 作.' },
|
||||
{ wrong: '做业', right: '作业', why: '作业 (homework) is written with 作.' },
|
||||
{ wrong: '做者', right: '作者', why: '作者 (author) is written with 作.' },
|
||||
{ wrong: '做文', right: '作文', why: '作文 (an essay) is written with 作.' },
|
||||
{ wrong: '做用', right: '作用', why: '作用 (effect, function) is written with 作.' },
|
||||
|
||||
// 得 / 的 — the pair everyone knows about. Only the fixed compound is flagged:
|
||||
// deciding 的 against 地 against 得 in the general case needs to know whether
|
||||
// the next word is a verb or a noun, which nothing here can tell.
|
||||
{ wrong: '觉的', right: '觉得', why: '觉得 (to feel, to think) ends in 得.' },
|
||||
|
||||
// 即 / 既 — one stroke apart, opposite meanings ("namely" against "since").
|
||||
{ wrong: '即然', right: '既然', why: '既然 (since, given that) takes 既.' },
|
||||
{ wrong: '既使', right: '即使', why: '即使 (even if) takes 即.' },
|
||||
|
||||
// The rest: ordinary IME slips where the wrong character is a homophone.
|
||||
{ wrong: '因该', right: '应该', why: '应该 (should) — 因 means "because"; the word you want starts with 应.' },
|
||||
{ wrong: '因位', right: '因为', why: '因为 (because) ends in 为.' },
|
||||
{ wrong: '知到', right: '知道', why: '知道 (to know) ends in 道.' },
|
||||
{ wrong: '安照', right: '按照', why: '按照 (according to) takes 按.' },
|
||||
{ wrong: '蜜码', right: '密码', why: '密码 (password) takes 密 — 蜜 is honey.' },
|
||||
{ wrong: '犹其', right: '尤其', why: '尤其 (especially) takes 尤.' },
|
||||
{ wrong: '甘净', right: '干净', why: '干净 (clean) takes 干.' },
|
||||
{ wrong: '什末', right: '什么', why: '什么 (what) ends in 么.' },
|
||||
{ wrong: '一像', right: '一样', why: '一样 (the same) ends in 样 — 像 is "to resemble".' },
|
||||
{ wrong: '必须品', right: '必需品', why: '必需品 (a necessity) takes 需. 必须 is "must", which is a different word.' },
|
||||
]
|
||||
|
||||
// **Gate two — the characters must not already belong to two different words.**
|
||||
//
|
||||
// This is the gate that stops the pack from destroying correct writing, and
|
||||
// without it every rule above is dangerous. 自己经常 ("oneself, often") contains
|
||||
// the string 己经. 睡觉的时候 ("when sleeping") contains 觉的. 不知到底 contains 知到.
|
||||
// A substring match would corrupt all three.
|
||||
//
|
||||
// The segmenter already knows the difference, so the test is: split the text,
|
||||
// and if the two characters land in different tokens *and* either token is a
|
||||
// real multi-character word, this is a word boundary and not an error. Two
|
||||
// adjacent single-character tokens is what the walk produces when it has nothing
|
||||
// better to offer — which is exactly what a mistyped compound looks like.
|
||||
function isWordBoundary(tokens: { word: string; from: number; to: number }[], at: number): boolean {
|
||||
const left = tokens.find((t) => at >= t.from && at < t.to)
|
||||
const right = tokens.find((t) => at + 1 >= t.from && at + 1 < t.to)
|
||||
if (!left || !right || left === right) return false
|
||||
return left.word.length > 1 || right.word.length > 1
|
||||
}
|
||||
|
||||
// hanziFindings returns the 错别字 in a piece of text, as ordinary mechanics
|
||||
// findings — the same shape, the same rail, the same cards, the same accept.
|
||||
//
|
||||
// It needs the segmenter and does nothing without one, which is also the
|
||||
// direction gate: the word list is loaded only for an account learning Chinese
|
||||
// (useSegmenter), so a writer practising English can never be told her quoted
|
||||
// Chinese is wrong. That is not a nicety. Petal deliberately never corrects the
|
||||
// pair language — the fr and es dictionaries are chosen to hold every variety
|
||||
// precisely so they cannot underline correct writing — and a Mandarin native
|
||||
// does not need her own language checked by a rule pack of two dozen entries.
|
||||
export function hanziFindings(text: string, segmenter: Segmenter | null): MechanicsFinding[] {
|
||||
if (!segmenter || !text) return []
|
||||
// One segmentation for the whole text, shared by every rule. The walk is
|
||||
// linear, but running it two dozen times over a long document would not be.
|
||||
const tokens = segmenter.segment(text)
|
||||
const found: MechanicsFinding[] = []
|
||||
|
||||
for (const c of CONFUSIONS) {
|
||||
let from = text.indexOf(c.wrong)
|
||||
while (from !== -1) {
|
||||
// The boundary test is asked at the seam the substitution sits on: the
|
||||
// gap between the first two characters, which is where a mistyped
|
||||
// compound and two adjacent words look different from each other.
|
||||
if (!isWordBoundary(tokens, from)) {
|
||||
found.push({
|
||||
from,
|
||||
to: from + c.wrong.length,
|
||||
original: c.wrong,
|
||||
replacement: c.right,
|
||||
explanation: c.why,
|
||||
type: 'mechanics',
|
||||
})
|
||||
}
|
||||
from = text.indexOf(c.wrong, from + 1)
|
||||
}
|
||||
}
|
||||
|
||||
// Document order, so the rail reads down the page rather than down this file.
|
||||
return found.sort((a, b) => a.from - b.from)
|
||||
}
|
||||
|
||||
// Exported for the tests, which check every pair against the shipped word list.
|
||||
// A pack whose own gate is only described in a comment is a pack whose gate can
|
||||
// rot; this is how the description is made to stay true.
|
||||
export const CONFUSION_PAIRS = CONFUSIONS.map((c) => ({ wrong: c.wrong, right: c.right }))
|
||||
@@ -0,0 +1,111 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { markInvited, mayInvite, todayKey } from './invitation'
|
||||
import { resetPackForTests, setPackLang } from '../../i18n'
|
||||
import { zh } from '../../i18n/packs/zh'
|
||||
import { ptPT } from '../../i18n/packs/pt-PT'
|
||||
import { declined, invitations } from './tips'
|
||||
|
||||
// The suite runs without a DOM, so localStorage is stubbed the same way
|
||||
// prefs.test.ts does it.
|
||||
function fakeStorage(): Storage {
|
||||
const map = new Map<string, string>()
|
||||
return {
|
||||
get length() {
|
||||
return map.size
|
||||
},
|
||||
key: (i: number) => [...map.keys()][i] ?? null,
|
||||
getItem: (k: string) => map.get(k) ?? null,
|
||||
setItem: (k: string, v: string) => void map.set(k, v),
|
||||
removeItem: (k: string) => void map.delete(k),
|
||||
clear: () => map.clear(),
|
||||
} as Storage
|
||||
}
|
||||
|
||||
beforeEach(() => vi.stubGlobal('localStorage', fakeStorage()))
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
resetPackForTests()
|
||||
})
|
||||
|
||||
describe('once a day', () => {
|
||||
it('offers, then does not offer again the same day', () => {
|
||||
expect(mayInvite()).toBe(true)
|
||||
markInvited()
|
||||
expect(mayInvite()).toBe(false)
|
||||
})
|
||||
|
||||
it('offers again the next day', () => {
|
||||
const monday = new Date(2026, 6, 27, 10, 0)
|
||||
const tuesday = new Date(2026, 6, 28, 9, 0)
|
||||
markInvited(monday)
|
||||
expect(mayInvite(monday)).toBe(false)
|
||||
expect(mayInvite(tuesday)).toBe(true)
|
||||
})
|
||||
|
||||
// The whole promise of §5c: nothing is counting. A month away has to look
|
||||
// exactly like a day away, because the alternative is a streak, and a streak
|
||||
// punishes exactly the person this feature is for.
|
||||
it('treats a month away the same as a day away', () => {
|
||||
const june = new Date(2026, 5, 1, 10, 0)
|
||||
const july = new Date(2026, 6, 27, 10, 0)
|
||||
markInvited(june)
|
||||
expect(mayInvite(july)).toBe(true)
|
||||
// And after being asked again, still only ever one stored value: a date.
|
||||
markInvited(july)
|
||||
expect(localStorage.getItem('petal.invited')).toBe(todayKey(july))
|
||||
// One stored value, and it is a date. Nothing accumulates.
|
||||
expect(localStorage.length).toBe(1)
|
||||
})
|
||||
|
||||
it('uses the writer\'s local day, so a late night and the small hours differ', () => {
|
||||
const lateNight = new Date(2026, 6, 27, 23, 30)
|
||||
const smallHours = new Date(2026, 6, 28, 1, 15)
|
||||
markInvited(lateNight)
|
||||
expect(mayInvite(smallHours)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('the invitation copy', () => {
|
||||
const PACKS = [
|
||||
{ name: 'zh', pack: zh },
|
||||
{ name: 'pt-PT', pack: ptPT },
|
||||
]
|
||||
|
||||
it('every pack offers something to write about, and a way to say no', () => {
|
||||
for (const { name, pack } of PACKS) {
|
||||
expect(pack.companion.invitations.length, name).toBeGreaterThan(3)
|
||||
expect(pack.companion.inviteAccept.trim(), name).not.toBe('')
|
||||
expect(pack.companion.inviteDecline.trim(), name).not.toBe('')
|
||||
expect(pack.companion.declined.native.trim(), name).not.toBe('')
|
||||
}
|
||||
})
|
||||
|
||||
// The copy is bound by the same rule as the timing: no streaks, no guilt, no
|
||||
// counting of days. This is the part a well-meaning future edit would undo —
|
||||
// "day 4 in a row!" is a natural thing to write and the wrong thing to say.
|
||||
it('never invokes a streak, a target, or a missed day', () => {
|
||||
const forbidden =
|
||||
/streak|in a row|every day|don't break|dont break|missed|behind|连续|打卡|坚持|seguidos|todos os dias|falhaste/i
|
||||
for (const { name, pack } of PACKS) {
|
||||
const copy = [
|
||||
...pack.companion.invitations.flatMap((l) => [l.native, l.en]),
|
||||
pack.companion.inviteAccept,
|
||||
pack.companion.inviteDecline,
|
||||
pack.companion.declined.native,
|
||||
pack.companion.declined.en,
|
||||
]
|
||||
for (const line of copy) {
|
||||
expect(line, `${name}: ${line}`).not.toMatch(forbidden)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('reads from the pair in force, like every other companion line', () => {
|
||||
setPackLang('pt-PT')
|
||||
expect(invitations()).toBe(ptPT.companion.invitations)
|
||||
expect(declined()).toBe(ptPT.companion.declined)
|
||||
setPackLang('zh')
|
||||
expect(invitations()).toBe(zh.companion.invitations)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,36 @@
|
||||
// When the companion may offer its daily invitation to write.
|
||||
//
|
||||
// A handful of lines, pulled out of the timing engine on purpose: this is the
|
||||
// part of §5c with the ethics in it, and it should be readable and testable on
|
||||
// its own rather than buried in a heartbeat.
|
||||
//
|
||||
// The rule is a *date*, not a count and not a run of days. Petal remembers the
|
||||
// last day it asked and nothing else — so there is no streak to break, no tally
|
||||
// of days missed, and nothing that gets worse for being away a week. Coming
|
||||
// back after a month looks exactly like coming back tomorrow, which is the only
|
||||
// version of this feature worth shipping to someone learning a language.
|
||||
//
|
||||
// Either answer spends the day's invitation. Being asked again after saying no
|
||||
// would turn "not today" into a negotiation.
|
||||
|
||||
import { readPref, writePref } from '../../lib/prefs'
|
||||
|
||||
const INVITED_KEY = 'petal.invited'
|
||||
|
||||
// The local calendar day. Local rather than UTC because "today" is the writer's
|
||||
// day: a nudge at 11pm and another at 1am would otherwise be two different days.
|
||||
export function todayKey(now = new Date()): string {
|
||||
const pad = (n: number) => String(n).padStart(2, '0')
|
||||
return `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`
|
||||
}
|
||||
|
||||
// mayInvite reports whether today's invitation is still unspent.
|
||||
export function mayInvite(now = new Date()): boolean {
|
||||
return readPref(INVITED_KEY) !== todayKey(now)
|
||||
}
|
||||
|
||||
// markInvited spends it — called when the invitation is *offered*, not when it
|
||||
// is accepted, because declining has to count too.
|
||||
export function markInvited(now = new Date()): void {
|
||||
writePref(INVITED_KEY, todayKey(now))
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const growth = vi.fn()
|
||||
vi.mock('../../api/client', () => ({ api: { growth: () => growth() } }))
|
||||
|
||||
import { personalCheer, resetPersonalCheersForTests, warmPersonalCheers } from './journalCheers'
|
||||
import { resetPackForTests, setPackLang } from '../../i18n'
|
||||
|
||||
const journal = {
|
||||
kept: 4,
|
||||
kept_before: 2,
|
||||
stuck: [{ phrase: 'make a decision', docs: 3 }],
|
||||
faded: [{ pattern: '在 the morning', times: 2 }],
|
||||
}
|
||||
|
||||
// Let the warm-up promise settle.
|
||||
const settle = () => new Promise((r) => setTimeout(r, 0))
|
||||
|
||||
describe('personal cheers', () => {
|
||||
beforeEach(() => {
|
||||
resetPersonalCheersForTests()
|
||||
resetPackForTests()
|
||||
growth.mockReset()
|
||||
})
|
||||
|
||||
it('says nothing before the journal has arrived — the cheer never waits', () => {
|
||||
growth.mockResolvedValue(journal)
|
||||
warmPersonalCheers()
|
||||
expect(personalCheer()).toBeNull()
|
||||
})
|
||||
|
||||
it('serves each personal line once, then falls silent', async () => {
|
||||
growth.mockResolvedValue(journal)
|
||||
warmPersonalCheers()
|
||||
await settle()
|
||||
|
||||
const first = personalCheer()
|
||||
const second = personalCheer()
|
||||
expect(first).not.toBeNull()
|
||||
expect(second).not.toBeNull()
|
||||
expect(first!.en).not.toBe(second!.en)
|
||||
// Both lines used: personal praise repeated is wallpaper, so the caller is
|
||||
// handed back to the generic pool instead.
|
||||
expect(personalCheer()).toBeNull()
|
||||
})
|
||||
|
||||
it('fetches once however often it is warmed', async () => {
|
||||
growth.mockResolvedValue(journal)
|
||||
warmPersonalCheers()
|
||||
warmPersonalCheers()
|
||||
await settle()
|
||||
warmPersonalCheers()
|
||||
expect(growth).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('is silent when the journal fails, rather than failing visibly', async () => {
|
||||
growth.mockRejectedValue(new Error('offline'))
|
||||
warmPersonalCheers()
|
||||
await settle()
|
||||
expect(personalCheer()).toBeNull()
|
||||
})
|
||||
|
||||
it('speaks the pair the writer is in, resolved at call time', async () => {
|
||||
growth.mockResolvedValue({ ...journal, faded: [] })
|
||||
warmPersonalCheers()
|
||||
await settle()
|
||||
|
||||
setPackLang('pt-PT')
|
||||
const line = personalCheer()
|
||||
expect(line).not.toBeNull()
|
||||
expect(line!.native).toContain('make a decision')
|
||||
expect(line!.native).not.toBe(line!.en)
|
||||
// The English half is the same sentence in every pack.
|
||||
expect(line!.en).toContain('make a decision')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,62 @@
|
||||
// Personal material for the companion, drawn from the growth journal.
|
||||
//
|
||||
// The kitten's cheers are warm but generic — they'd be the same words for
|
||||
// anybody. The journal already knows things that are true of *this* writer and
|
||||
// nobody else ("you're using 'make a decision' on your own now"), and that is a
|
||||
// far better thing to hear after accepting an edit. §5a's own example.
|
||||
//
|
||||
// Three rules keep it from wearing out:
|
||||
// * A given line is served once per session. Personal praise repeated is
|
||||
// wallpaper, and wallpaper is worse than the generic cheer it replaced.
|
||||
// * The journal is fetched lazily, on the first accept, and never awaited —
|
||||
// the first cheer of a session is generic, and that's fine.
|
||||
// * Lines are built at call time from the pack, like every other companion
|
||||
// line, so a bubble composed after /api/me is in the right language.
|
||||
|
||||
import { api, type GrowthJournal } from '../../api/client'
|
||||
import { pack, type Line } from '../../i18n'
|
||||
|
||||
let journal: GrowthJournal | null = null
|
||||
let inFlight = false
|
||||
let served = new Set<string>()
|
||||
|
||||
// warmPersonalCheers starts the one fetch this module ever needs. Safe to call
|
||||
// often; a failure is silent and simply leaves the companion with its generic
|
||||
// pool, which is exactly the pre-journal behaviour.
|
||||
export function warmPersonalCheers(): void {
|
||||
if (journal || inFlight) return
|
||||
inFlight = true
|
||||
api
|
||||
.growth()
|
||||
.then((j) => {
|
||||
journal = j
|
||||
})
|
||||
.catch(() => {
|
||||
/* no journal, no personal cheer — never a visible failure */
|
||||
})
|
||||
.finally(() => {
|
||||
inFlight = false
|
||||
})
|
||||
}
|
||||
|
||||
// personalCheer returns an unserved line about her own writing, or null when
|
||||
// there is none — the caller falls back to the generic pool.
|
||||
export function personalCheer(): Line | null {
|
||||
if (!journal) return null
|
||||
const t = pack()
|
||||
const candidates = [
|
||||
...journal.stuck.map((s) => ({ key: `stuck:${s.phrase}`, line: () => t.journal.cheerStuck(s.phrase) })),
|
||||
...journal.faded.map((f) => ({ key: `faded:${f.pattern}`, line: () => t.journal.cheerFaded(f.pattern) })),
|
||||
].filter((c) => !served.has(c.key))
|
||||
if (candidates.length === 0) return null
|
||||
const chosen = candidates[Math.floor(Math.random() * candidates.length)]
|
||||
served.add(chosen.key)
|
||||
return chosen.line()
|
||||
}
|
||||
|
||||
// Test seam, matching resetPackForTests: module state is per-session by design.
|
||||
export function resetPersonalCheersForTests(): void {
|
||||
journal = null
|
||||
inFlight = false
|
||||
served = new Set()
|
||||
}
|
||||
@@ -19,6 +19,7 @@
|
||||
// fix-bearing hints so the same span never appears as both a bubble and a card.
|
||||
|
||||
import { pack } from '../../i18n'
|
||||
import type { PairLang } from '../../i18n'
|
||||
import type { Line } from './tips'
|
||||
|
||||
// The pair's prose copy. Read per finding rather than captured once, so a rule
|
||||
@@ -41,8 +42,16 @@ export interface Fix {
|
||||
from: number
|
||||
to: number
|
||||
replacement: string
|
||||
// Which suggestion family the card belongs to. Omitted means 'mechanics' — a
|
||||
// fix to *this* sentence. The miscollocation rules set 'collocation', because
|
||||
// what they hand over is a reusable chunk: the writer sees the same rail and
|
||||
// the same warm phrasing as the LLM coach, and an accepted chunk plants in the
|
||||
// vocabulary garden exactly as the coach's would.
|
||||
family?: FindingFamily
|
||||
}
|
||||
|
||||
export type FindingFamily = 'mechanics' | 'collocation'
|
||||
|
||||
// A deterministic suggestion-card finding, derived from an applyable hint. Mirrors
|
||||
// the backend's card shape (original/replacement/explanation + span) so the card
|
||||
// pipeline can persist it as the 'mechanics' family. `explanation` is the English
|
||||
@@ -53,6 +62,9 @@ export interface MechanicsFinding {
|
||||
original: string
|
||||
replacement: string
|
||||
explanation: string
|
||||
// The family the server should file it under (see Fix.family). Always sent, so
|
||||
// the backend never has to infer it from the endpoint it arrived on.
|
||||
type: FindingFamily
|
||||
}
|
||||
|
||||
// ── small text helpers ──────────────────────────────────────────────────────
|
||||
@@ -630,19 +642,438 @@ function thanThen(text: string, out: ProseHint[]) {
|
||||
}
|
||||
}
|
||||
|
||||
// "people is" — people is already plural in English, and every language Petal
|
||||
// pairs with has a singular word for it (人 / a gente / les gens / la gente).
|
||||
// Universal rather than per-pair for exactly that reason.
|
||||
const PEOPLE_IS_RE = /\b(people)\s+(is|was|has)\b/gi
|
||||
const PEOPLE_PLURAL: Record<string, string> = { is: 'are', was: 'were', has: 'have' }
|
||||
|
||||
function peopleAre(text: string, out: ProseHint[]) {
|
||||
let m: RegExpExecArray | null
|
||||
PEOPLE_IS_RE.lastIndex = 0
|
||||
while ((m = PEOPLE_IS_RE.exec(text))) {
|
||||
const plural = PEOPLE_PLURAL[m[2].toLowerCase()]
|
||||
const fixed = `${m[1]} ${matchCase(m[2], plural)}`
|
||||
out.push({
|
||||
id: `peopleare:${m.index}`,
|
||||
rule: 'peopleare',
|
||||
native: P().peopleArePlural(plural),
|
||||
en: `“People” is plural in English: “people ${plural}”.`,
|
||||
fix: { from: m.index, to: m.index + m[0].length, replacement: fixed },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── preposition pairs ───────────────────────────────────────────────────────
|
||||
// The verbs and adjectives whose preposition English simply *decides* for you.
|
||||
// There is no rule to learn here — "depend" takes "on" and that is the end of
|
||||
// it — which is exactly what makes the offline pack the right place for them:
|
||||
// they are data, not judgement, and a lookup is instant.
|
||||
//
|
||||
// Every entry is a pairing that is wrong in essentially all contexts. The ones
|
||||
// that are only *usually* wrong were left out on purpose: "married with" is a
|
||||
// mistake until "married with children", "arrive to" wants at or in depending
|
||||
// on the noun, "different than" is ordinary American English. Precision over
|
||||
// recall — a confident wrong correction costs more than a quiet miss.
|
||||
interface PrepRule {
|
||||
// Matched case-insensitively, with \b at both ends. The first group is the
|
||||
// head word (kept, casing preserved), the rest is replaced wholesale.
|
||||
re: RegExp
|
||||
// The corrected phrase, with $1 standing for the captured head word.
|
||||
to: string
|
||||
}
|
||||
|
||||
const PREPOSITIONS: PrepRule[] = [
|
||||
{ re: /\b(depend|depends|depended|depending)\s+of\b/gi, to: '$1 on' },
|
||||
{ re: /\b(discuss|discusses|discussed|discussing)\s+about\b/gi, to: '$1' },
|
||||
{ re: /\b(participate|participates|participated|participating)\s+to\b/gi, to: '$1 in' },
|
||||
{ re: /\b(interested)\s+(?:about|for)\b/gi, to: '$1 in' },
|
||||
{ re: /\b(responsible)\s+of\b/gi, to: '$1 for' },
|
||||
{ re: /\b(capable)\s+to\b/gi, to: '$1 of' },
|
||||
{ re: /\b(afraid)\s+(?:from|of to)\b/gi, to: '$1 of' },
|
||||
{ re: /\b(according)\s+with\b/gi, to: '$1 to' },
|
||||
{ re: /\b(explain|explains|explained)\s+(me|us|him|her|them)\b/gi, to: '$1 to $2' },
|
||||
// "listen the radio" — the object of "listen" always arrives through "to".
|
||||
{ re: /\b(listen|listens|listened|listening)\s+(the|a|an|my|your|his|her|our|their|this|that|these|those|music|me|him|us|them)\b/gi, to: '$1 to $2' },
|
||||
]
|
||||
|
||||
function prepositions(text: string, out: ProseHint[]) {
|
||||
for (const rule of PREPOSITIONS) {
|
||||
let m: RegExpExecArray | null
|
||||
rule.re.lastIndex = 0
|
||||
while ((m = rule.re.exec(text))) {
|
||||
// Rebuild the corrected phrase from the captures so the head word keeps the
|
||||
// writer's own casing ("Depending of" → "Depending on").
|
||||
const fixed = rule.to.replace(/\$(\d)/g, (_, d: string) => m![Number(d)] ?? '')
|
||||
if (fixed === m[0]) continue
|
||||
out.push({
|
||||
id: `prep:${m.index}:${key(m[0])}`,
|
||||
rule: 'preposition',
|
||||
native: P().preposition(m[0].trim(), fixed),
|
||||
en: `In English it's “${fixed}”, not “${m[0].trim()}”.`,
|
||||
fix: { from: m.index, to: m.index + m[0].length, replacement: fixed },
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Doubled comparatives and superlatives — "more better", "most easiest". The
|
||||
// -er/-est ending already carries the comparison, so the "more"/"most" is the
|
||||
// part that goes. An explicit form list rather than a generic /\w+er/ pattern,
|
||||
// which would catch "more clever" and "most eager" (both perfectly correct).
|
||||
const COMPARATIVE_FORMS =
|
||||
'better|worse|greater|older|younger|bigger|smaller|larger|faster|slower|higher|' +
|
||||
'lower|cheaper|stronger|weaker|easier|harder|earlier|later|sooner|longer|' +
|
||||
'shorter|taller|richer|poorer|happier|safer|nicer|closer|warmer|colder'
|
||||
const SUPERLATIVE_FORMS =
|
||||
'best|worst|greatest|oldest|youngest|biggest|smallest|largest|fastest|slowest|' +
|
||||
'highest|lowest|cheapest|strongest|weakest|easiest|hardest|earliest|latest|' +
|
||||
'soonest|longest|shortest|tallest|richest|poorest|happiest|safest|nicest|' +
|
||||
'closest|warmest|coldest'
|
||||
const DOUBLE_COMPARATIVE_RE = new RegExp(
|
||||
`\\b(more)\\s+(${COMPARATIVE_FORMS})\\b|\\b(most)\\s+(${SUPERLATIVE_FORMS})\\b`,
|
||||
'gi',
|
||||
)
|
||||
|
||||
function doubleComparative(text: string, out: ProseHint[]) {
|
||||
let m: RegExpExecArray | null
|
||||
DOUBLE_COMPARATIVE_RE.lastIndex = 0
|
||||
while ((m = DOUBLE_COMPARATIVE_RE.exec(text))) {
|
||||
const lead = m[1] ?? m[3]
|
||||
const word = m[2] ?? m[4]
|
||||
out.push({
|
||||
id: `doublecomp:${m.index}`,
|
||||
rule: 'doublecomp',
|
||||
native: P().doubleComparative(lead, word),
|
||||
en: `“${word}” is already the comparison — “${lead}” isn't needed: just “${word}”.`,
|
||||
fix: { from: m.index, to: m.index + m[0].length, replacement: matchCase(lead, word) },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── miscollocations (the collocation family, offline half) ──────────────────
|
||||
// A few dozen entries of curated data doing what the collocation coach does
|
||||
// with a model behind a VPN. These are the do/make, say/tell, heavy-rain pairs
|
||||
// that fill every ESL collocation workbook: the writer's grammar is perfect and
|
||||
// the pairing is simply not the one English uses.
|
||||
//
|
||||
// They file as 'collocation', not 'mechanics', because that is what they are —
|
||||
// and it earns the writer the rest of the family's behaviour for free: the same
|
||||
// card, and a phrase card planted in the vocabulary garden when she accepts.
|
||||
// The writer never needs to know which engine spoke.
|
||||
//
|
||||
// Each entry is a whole-phrase swap so the card can anchor by string, and the
|
||||
// object is captured rather than listed, so "do a serious mistake" is caught
|
||||
// alongside "do a mistake".
|
||||
interface CollocationRule {
|
||||
re: RegExp
|
||||
to: string
|
||||
}
|
||||
|
||||
const MISCOLLOCATIONS: CollocationRule[] = [
|
||||
// do / make — the classic pair, in both directions.
|
||||
{ re: /\b(do|does|did|doing)\s+(a|an|the|my|your|his|her|our|their)\s+(decision|mistake|mistakes|effort|question|questions|progress|joke|jokes|favou?r)\b/gi, to: 'MAKE' },
|
||||
{ re: /\b(make|makes|made|making)\s+(my|your|his|her|our|their|the)\s+(homework|housework|laundry|dishes|shopping)\b/gi, to: 'DO' },
|
||||
{ re: /\b(make|makes|made|making)\s+(a|an|the|my|your|his|her|our|their)\s+(photo|photos|picture|pictures|shower|bath|walk|trip|nap|break|exam|exams|test|bus|taxi|train)\b/gi, to: 'TAKE' },
|
||||
{ re: /\b(make|makes|made|making)\s+(a|an|the)\s+(party|baby|good time|meeting)\b/gi, to: 'HAVE' },
|
||||
{ re: /\b(make|makes|made|making)\s+(a|an|the|my|your|his|her)\s+(question|questions)\b/gi, to: 'ASK' },
|
||||
{ re: /\b(make|makes|made|making|do|does|did|doing)\s+attention\b/gi, to: 'PAY_ATTENTION' },
|
||||
// say / tell — "say me" for "tell me" is near-universal among learners.
|
||||
{ re: /\b(say|says|said|saying)\s+(me|him|her|us|them)\b/gi, to: 'TELL' },
|
||||
{ re: /\b(say|says|said|saying)\s+(a|the)\s+(lie|lies|truth|joke|jokes|story|stories)\b/gi, to: 'TELL_A' },
|
||||
// Weather and intensity — English picks a different adjective per noun.
|
||||
{ re: /\b(strong|big|hard|huge)\s+(rain|snow|traffic|fog)\b/gi, to: 'HEAVY' },
|
||||
{ re: /\b(strong|heavy|big)\s+(wind|winds)\b/gi, to: 'STRONG_WIND' },
|
||||
]
|
||||
|
||||
// Rebuild the corrected phrase for one miscollocation match. Kept as code rather
|
||||
// than a `to` template because the verb has to agree with the writer's own tense
|
||||
// ("did a mistake" → "made a mistake"), and only the verb form knows that.
|
||||
const VERB_FORMS: Record<string, Record<string, string>> = {
|
||||
make: { base: 'make', s: 'makes', past: 'made', ing: 'making' },
|
||||
do: { base: 'do', s: 'does', past: 'did', ing: 'doing' },
|
||||
take: { base: 'take', s: 'takes', past: 'took', ing: 'taking' },
|
||||
have: { base: 'have', s: 'has', past: 'had', ing: 'having' },
|
||||
ask: { base: 'ask', s: 'asks', past: 'asked', ing: 'asking' },
|
||||
tell: { base: 'tell', s: 'tells', past: 'told', ing: 'telling' },
|
||||
pay: { base: 'pay', s: 'pays', past: 'paid', ing: 'paying' },
|
||||
}
|
||||
|
||||
// Which slot of VERB_FORMS the writer's own verb occupies, so the replacement
|
||||
// lands in the same tense she was writing in.
|
||||
function verbSlot(verb: string): string {
|
||||
const v = verb.toLowerCase()
|
||||
if (v.endsWith('ing')) return 'ing'
|
||||
if (v === 'did' || v === 'made' || v === 'took' || v === 'had' || v === 'told' || v === 'said' || v === 'paid' || v.endsWith('ed')) return 'past'
|
||||
if (v === 'does' || v === 'says' || v === 'has' || v.endsWith('s')) return 's'
|
||||
return 'base'
|
||||
}
|
||||
|
||||
function collocationFix(m: RegExpExecArray, to: string): string | null {
|
||||
const slot = verbSlot(m[1])
|
||||
const conj = (v: string) => matchCase(m[1], VERB_FORMS[v][slot])
|
||||
switch (to) {
|
||||
case 'MAKE':
|
||||
case 'DO':
|
||||
case 'TAKE':
|
||||
case 'HAVE':
|
||||
case 'ASK':
|
||||
return `${conj(to.toLowerCase())} ${m[2]} ${m[3]}`
|
||||
case 'TELL':
|
||||
return `${conj('tell')} ${m[2]}`
|
||||
case 'TELL_A':
|
||||
return `${conj('tell')} ${m[2]} ${m[3]}`
|
||||
case 'PAY_ATTENTION':
|
||||
return `${conj('pay')} attention`
|
||||
case 'HEAVY':
|
||||
return `${matchCase(m[1], 'heavy')} ${m[2]}`
|
||||
case 'STRONG_WIND':
|
||||
return m[1].toLowerCase() === 'strong' ? null : `${matchCase(m[1], 'strong')} ${m[2]}`
|
||||
default:
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function miscollocations(text: string, out: ProseHint[]) {
|
||||
for (const rule of MISCOLLOCATIONS) {
|
||||
let m: RegExpExecArray | null
|
||||
rule.re.lastIndex = 0
|
||||
while ((m = rule.re.exec(text))) {
|
||||
const fixed = collocationFix(m, rule.to)
|
||||
if (!fixed || fixed.toLowerCase() === m[0].toLowerCase()) continue
|
||||
out.push({
|
||||
id: `colloc:${m.index}:${key(m[0])}`,
|
||||
rule: 'collocation',
|
||||
native: P().collocation(m[0].trim(), fixed),
|
||||
en: `English usually pairs these differently: “${fixed}” rather than “${m[0].trim()}”.`,
|
||||
fix: { from: m.index, to: m.index + m[0].length, replacement: fixed, family: 'collocation' },
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── false friends ───────────────────────────────────────────────────────────
|
||||
// A word that looks like one of hers and means something else. This is the
|
||||
// mistake that makes a learner feel foolish rather than merely corrected, so
|
||||
// Petal only ever raises an eyebrow: awareness-only, one per pass, and never a
|
||||
// replacement. "Actually" really might be the word she wanted — the flag says
|
||||
// what it means in English and lets her decide.
|
||||
//
|
||||
// The list is per pair and lives in the langpack (a zh pair has none: the trap
|
||||
// needs a shared script to spring). See Pack.falseFriends.
|
||||
function falseFriends(text: string, out: ProseHint[]) {
|
||||
const list = pack().falseFriends
|
||||
const words = Object.keys(list)
|
||||
if (words.length === 0) return
|
||||
// Escaped, because these keys are pack data: a future author writing "e.g."
|
||||
// should get a heads-up, not a pattern that quietly matches everything.
|
||||
const re = new RegExp(`\\b(${words.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|')})\\b`, 'gi')
|
||||
let m: RegExpExecArray | null
|
||||
while ((m = re.exec(text))) {
|
||||
const entry = list[m[1].toLowerCase()]
|
||||
if (!entry) continue
|
||||
out.push({
|
||||
id: `falsefriend:${m[1].toLowerCase()}`,
|
||||
rule: 'falsefriend',
|
||||
native: entry.native,
|
||||
en: entry.en,
|
||||
})
|
||||
return // one per pass — a heads-up, not a sweep
|
||||
}
|
||||
}
|
||||
|
||||
// ── per-pair L1 interference ────────────────────────────────────────────────
|
||||
// Mistakes that are not "English mistakes" at all but the writer's own language
|
||||
// showing through: *ter 30 anos* becomes "have 30 years", 很喜欢 becomes "very
|
||||
// like", 开灯 becomes "open the light". They are gated by pair precisely so they
|
||||
// can be confident — a pattern that is a near-certainty for a Portuguese speaker
|
||||
// is only a guess for anybody else, and a guess doesn't belong in a rule pack.
|
||||
//
|
||||
// The rules a pair *doesn't* get are as deliberate as the ones it does. Mandarin
|
||||
// drops articles and slips he/she, both of which the plan names — and neither is
|
||||
// detectable from the text alone. "She said he was late" is a perfect sentence
|
||||
// whichever pronoun was meant, and no offline rule can tell a missing "the" from
|
||||
// a mass noun. Flagging them would mean correcting correct writing, which is the
|
||||
// one thing this pack promises not to do.
|
||||
|
||||
const NUMBER_WORD =
|
||||
'one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|' +
|
||||
'fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|thirty|forty|' +
|
||||
'fifty|sixty|seventy|eighty|ninety'
|
||||
|
||||
// *Ter X anos* / *avoir X ans* / *tener X años*: age is something you *have* in
|
||||
// every Romance language and something you *are* in English.
|
||||
const HAVE_YEARS_RE = new RegExp(
|
||||
`\\b(I|you|we|they|he|she)\\s+(have|has|had)\\s+(\\d{1,3}|${NUMBER_WORD})\\s+years(\\s+old)?\\b`,
|
||||
'gi',
|
||||
)
|
||||
|
||||
// The English "to be" that matches the subject and the tense she wrote in.
|
||||
function beFor(subject: string, verb: string): string {
|
||||
const past = verb.toLowerCase() === 'had'
|
||||
const s = subject.toLowerCase()
|
||||
if (s === 'i') return past ? 'was' : 'am'
|
||||
if (s === 'he' || s === 'she') return past ? 'was' : 'is'
|
||||
return past ? 'were' : 'are'
|
||||
}
|
||||
|
||||
function haveYears(text: string, out: ProseHint[]) {
|
||||
let m: RegExpExecArray | null
|
||||
HAVE_YEARS_RE.lastIndex = 0
|
||||
while ((m = HAVE_YEARS_RE.exec(text))) {
|
||||
const fixed = `${m[1]} ${beFor(m[1], m[2])} ${m[3]} years old`
|
||||
out.push({
|
||||
id: `haveyears:${m.index}`,
|
||||
rule: 'haveyears',
|
||||
native: P().ageIsNotHave(m[3]),
|
||||
en: `In English you *are* your age: “${fixed}”.`,
|
||||
fix: { from: m.index, to: m.index + m[0].length, replacement: fixed },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// *Estou de acordo* / *je suis d'accord*: agreement is a verb in English, so the
|
||||
// "to be" in front of it has nothing to do.
|
||||
const AM_AGREE_RE = /\b(I|you|we|they|he|she)\s+(am|are|is|was|were)\s+agree\b/gi
|
||||
|
||||
function amAgree(text: string, out: ProseHint[]) {
|
||||
let m: RegExpExecArray | null
|
||||
AM_AGREE_RE.lastIndex = 0
|
||||
while ((m = AM_AGREE_RE.exec(text))) {
|
||||
const past = /^(was|were)$/i.test(m[2])
|
||||
const verb = past ? 'agreed' : m[1].toLowerCase() === 'he' || m[1].toLowerCase() === 'she' ? 'agrees' : 'agree'
|
||||
const fixed = `${m[1]} ${verb}`
|
||||
out.push({
|
||||
id: `amagree:${m.index}`,
|
||||
rule: 'amagree',
|
||||
native: P().agreeIsAVerb,
|
||||
en: `“Agree” is already the verb: “${fixed}”, not “${m[0].trim()}”.`,
|
||||
fix: { from: m.index, to: m.index + m[0].length, replacement: fixed },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// *desde há três anos* / *depuis trois ans*: a stretch of time takes "for";
|
||||
// "since" wants the moment it started.
|
||||
const SINCE_DURATION_RE = new RegExp(
|
||||
`\\b(since)\\s+((?:\\d{1,3}|${NUMBER_WORD}|a few|several|many)\\s+(?:years|months|weeks|days|hours|minutes))\\b`,
|
||||
'gi',
|
||||
)
|
||||
|
||||
function sinceDuration(text: string, out: ProseHint[]) {
|
||||
let m: RegExpExecArray | null
|
||||
SINCE_DURATION_RE.lastIndex = 0
|
||||
while ((m = SINCE_DURATION_RE.exec(text))) {
|
||||
const fixed = `${matchCase(m[1], 'for')} ${m[2]}`
|
||||
out.push({
|
||||
id: `since:${m.index}`,
|
||||
rule: 'since',
|
||||
native: P().forNotSince(m[2]),
|
||||
en: `For a length of time use “for”: “${fixed}”. “Since” names when it started.`,
|
||||
fix: { from: m.index, to: m.index + m[0].length, replacement: fixed },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 很喜欢 — 很 modifies adjectives *and* verbs in Mandarin, so "very" arrives in
|
||||
// front of English verbs, where it cannot go.
|
||||
const VERY_VERB_RE =
|
||||
/\b(very)\s+(like|likes|liked|want|wants|wanted|enjoy|enjoys|enjoyed|hope|hopes|hoped|miss|misses|missed|need|needs|needed|love|loves|loved|agree|agrees|agreed|understand|understands)\b/gi
|
||||
|
||||
function veryVerb(text: string, out: ProseHint[]) {
|
||||
let m: RegExpExecArray | null
|
||||
VERY_VERB_RE.lastIndex = 0
|
||||
while ((m = VERY_VERB_RE.exec(text))) {
|
||||
const fixed = `${matchCase(m[1], 'really')} ${m[2]}`
|
||||
out.push({
|
||||
id: `veryverb:${m.index}`,
|
||||
rule: 'veryverb',
|
||||
native: P().veryBeforeVerb(m[2]),
|
||||
en: `“Very” goes with adjectives, not verbs — “${fixed}” (or “${m[2]} … very much”).`,
|
||||
fix: { from: m.index, to: m.index + m[0].length, replacement: fixed },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 开灯 / 关电视 — Mandarin opens and closes appliances; English turns them on
|
||||
// and off.
|
||||
const TURN_FORMS: Record<string, string> = {
|
||||
base: 'turn',
|
||||
s: 'turns',
|
||||
past: 'turned',
|
||||
ing: 'turning',
|
||||
}
|
||||
const OPEN_LIGHT_RE =
|
||||
/\b(open|opens|opened|close|closes|closed)\s+(the|a|my|your|his|her|our|their)\s+(light|lights|lamp|tv|television|radio|computer|fan|heater|air conditioner|air-conditioner)\b/gi
|
||||
|
||||
function openTheLight(text: string, out: ProseHint[]) {
|
||||
let m: RegExpExecArray | null
|
||||
OPEN_LIGHT_RE.lastIndex = 0
|
||||
while ((m = OPEN_LIGHT_RE.exec(text))) {
|
||||
const opening = m[1].toLowerCase().startsWith('open')
|
||||
const turn = matchCase(m[1], TURN_FORMS[verbSlot(m[1])] ?? 'turn')
|
||||
const fixed = `${turn} ${opening ? 'on' : 'off'} ${m[2]} ${m[3]}`
|
||||
out.push({
|
||||
id: `openlight:${m.index}`,
|
||||
rule: 'openlight',
|
||||
native: P().turnOnNotOpen(m[3], opening),
|
||||
en: `In English you turn a ${m[3]} ${opening ? 'on' : 'off'}: “${fixed}”.`,
|
||||
fix: { from: m.index, to: m.index + m[0].length, replacement: fixed },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 虽然…但是 is a matched pair in Mandarin; English takes one or the other, never
|
||||
// both. Awareness-only: which half to drop is the writer's call, and the "but"
|
||||
// on its own is far too common a word to anchor a card to.
|
||||
const ALTHOUGH_BUT_RE = /\b(although|though|even though)\b[^.!?]{0,120}?,?\s+but\b/gi
|
||||
|
||||
function althoughBut(text: string, out: ProseHint[]) {
|
||||
let m: RegExpExecArray | null
|
||||
ALTHOUGH_BUT_RE.lastIndex = 0
|
||||
while ((m = ALTHOUGH_BUT_RE.exec(text))) {
|
||||
out.push({
|
||||
id: `althoughbut:${key(m[0])}`,
|
||||
rule: 'althoughbut',
|
||||
native: P().althoughOrBut(m[1]),
|
||||
en: `English uses “${m[1]}” or “but”, not both — one of them can go.`,
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
type Rule = (text: string, out: ProseHint[]) => void
|
||||
|
||||
// Which interference rules belong to which pair. A pair with no entry simply
|
||||
// runs the shared pack, which is the correct behaviour for a language Petal has
|
||||
// not studied yet rather than a gap to fill with guesses.
|
||||
const L1_RULES: Partial<Record<PairLang, Rule[]>> = {
|
||||
zh: [veryVerb, openTheLight, althoughBut],
|
||||
'pt-PT': [haveYears, amAgree, sinceDuration],
|
||||
fr: [haveYears, amAgree, sinceDuration],
|
||||
es: [haveYears, amAgree, sinceDuration],
|
||||
}
|
||||
|
||||
// ── orchestration ───────────────────────────────────────────────────────────
|
||||
|
||||
// Rules run in priority order — the ones the writer cares most about first, so
|
||||
// that when several fire at once the companion leads with the weightiest note.
|
||||
// The false-friend heads-up sits near the top: of everything here it is the one
|
||||
// that costs her most to find out about later.
|
||||
const RULES: Array<(text: string, out: ProseHint[]) => void> = [
|
||||
runOns,
|
||||
commaSplices,
|
||||
falseFriends,
|
||||
antecedents,
|
||||
oxford,
|
||||
miscollocations,
|
||||
articles,
|
||||
uncountables,
|
||||
properCaps,
|
||||
subjectVerbAgreement,
|
||||
peopleAre,
|
||||
prepositions,
|
||||
doubleComparative,
|
||||
pluralAfterNumber,
|
||||
doubleDeterminer,
|
||||
thereIsPlural,
|
||||
@@ -656,6 +1087,13 @@ const RULES: Array<(text: string, out: ProseHint[]) => void> = [
|
||||
spaceAfterPunct,
|
||||
]
|
||||
|
||||
// Every rule that runs for this writer: the shared pack, plus the interference
|
||||
// rules belonging to her pair. Read per call rather than built once — the pair
|
||||
// isn't known until /api/me answers, and the checker runs long before and after.
|
||||
function rulesFor(): Rule[] {
|
||||
return [...RULES, ...(L1_RULES[pack().code] ?? [])]
|
||||
}
|
||||
|
||||
// analyzeProse returns context-aware hints, highest-priority first. It bails on
|
||||
// text too short to advise on (mid-thought drafts shouldn't get picked apart).
|
||||
// Hints that carry a `fix` are applyable (they also surface as suggestion cards);
|
||||
@@ -664,26 +1102,27 @@ export function analyzeProse(text: string): ProseHint[] {
|
||||
const englishWords = text.match(ENGLISH_WORD_RE)?.length ?? 0
|
||||
if (englishWords < 8) return []
|
||||
const out: ProseHint[] = []
|
||||
for (const rule of RULES) rule(text, out)
|
||||
for (const rule of rulesFor()) rule(text, out)
|
||||
return out
|
||||
}
|
||||
|
||||
// mechanicsFindings returns every applyable deterministic fix in the text, as
|
||||
// suggestion-card findings with exact spans. No word-count floor: a doubled word
|
||||
// or a stray lowercase “i” is worth fixing even in a short draft, the way a
|
||||
// spell-checker would. The card pipeline persists these as the 'mechanics'
|
||||
// family; collisions with the LLM cards are resolved server-side (mechanics
|
||||
// wins, since its span is exact).
|
||||
// spell-checker would. The card pipeline persists these under the family each
|
||||
// finding names (mechanics, or collocation for the miscollocation rules);
|
||||
// collisions with the LLM cards are resolved server-side (the offline card wins,
|
||||
// since its span is exact).
|
||||
export function mechanicsFindings(text: string): MechanicsFinding[] {
|
||||
const hints: ProseHint[] = []
|
||||
for (const rule of RULES) rule(text, hints)
|
||||
for (const rule of rulesFor()) rule(text, hints)
|
||||
const found: MechanicsFinding[] = []
|
||||
for (const h of hints) {
|
||||
if (!h.fix) continue
|
||||
const { from, to, replacement } = h.fix
|
||||
const { from, to, replacement, family } = h.fix
|
||||
const original = text.slice(from, to)
|
||||
if (!original || original === replacement) continue
|
||||
found.push({ from, to, original, replacement, explanation: h.en })
|
||||
found.push({ from, to, original, replacement, explanation: h.en, type: family ?? 'mechanics' })
|
||||
}
|
||||
// Two rules can occasionally claim overlapping spans (e.g. a doubled word that
|
||||
// also reads as stacked determiners). Resolve to one card per stretch of text:
|
||||
|
||||
@@ -18,6 +18,8 @@ export const errors = (): Line[] => pack().companion.errors
|
||||
export const greeting = (): Line => pack().companion.greeting
|
||||
export const welcomeBack = (): Line => pack().companion.welcomeBack
|
||||
export const milestoneLine = (n: number): Line => pack().companion.milestone(n)
|
||||
export const invitations = (): Line[] => pack().companion.invitations
|
||||
export const declined = (): Line => pack().companion.declined
|
||||
|
||||
// Word-count milestones worth a little cheer — every 100 words, on up. A count,
|
||||
// not copy: the same in every language.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user