Compare commits
69
Commits
336cae93e0
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d961a89bf8 | ||
|
|
7fa98d03c7 | ||
|
|
6bed33c27e | ||
|
|
7383bdb403 | ||
|
|
3cc23b8ea4 | ||
|
|
6026d98598 | ||
|
|
15398eab4d | ||
|
|
2c5b05b398 | ||
|
|
acb35108c0 | ||
|
|
e67f77eb05 | ||
|
|
c719effe1d | ||
|
|
1fdc206576 | ||
|
|
466055020f | ||
|
|
29eb2fe1fc | ||
|
|
76dede8856 | ||
|
|
db9cfb7abf | ||
|
|
f82f2b589d | ||
|
|
5cd3aeabde | ||
|
|
047f4ae67f | ||
|
|
f2dd30628a | ||
|
|
c6bf36bddf | ||
|
|
178cb7ae67 | ||
|
|
8f2ad34a10 | ||
|
|
5659312358 | ||
|
|
a216614c81 | ||
|
|
c348a9b8ae | ||
|
|
77f284f65c | ||
|
|
9224c44fff | ||
|
|
39d4e4770a | ||
|
|
bd92cdc9b6 | ||
|
|
1acc23244e | ||
|
|
40de65b3d1 | ||
|
|
b2d50e9136 | ||
|
|
1aa3a14030 | ||
|
|
978cb80642 | ||
|
|
f082a930cb | ||
|
|
ce5de68b9b | ||
|
|
b23c5a9a13 | ||
|
|
25e415daa2 | ||
|
|
3bcc967f51 | ||
|
|
963fc1754d | ||
|
|
de251ceae2 | ||
|
|
10e8aef86c | ||
|
|
c33de1175b | ||
|
|
ba06d904f0 | ||
|
|
ea14eb5e88 | ||
|
|
aac15b5ac5 | ||
|
|
be9aa13287 | ||
|
|
ec9fba9252 | ||
|
|
9c40a8ad3f | ||
|
|
ac1c6cddb0 | ||
|
|
3e714b6f00 | ||
|
|
69bf3ffde1 | ||
|
|
9a0edd6679 | ||
|
|
be1ab5cef7 | ||
|
|
071ea7b835 | ||
|
|
9a2e909b85 | ||
|
|
1f4ca4775a | ||
|
|
1bbc8fc8d3 | ||
|
|
e9b8595456 | ||
|
|
7b845644be | ||
|
|
3b714e297a | ||
|
|
24c3533e18 | ||
|
|
ccb43e5a4d | ||
|
|
4de83d0da5 | ||
|
|
74bf600593 | ||
|
|
86175f1559 | ||
|
|
3640ce9324 | ||
|
|
97e9c269ec |
+20
-1
@@ -10,6 +10,12 @@ DATABASE_PATH=./data/petal.db
|
|||||||
# On-disk store for images pasted/dropped/inserted in the editor
|
# On-disk store for images pasted/dropped/inserted in the editor
|
||||||
IMAGE_DIR=./data/images
|
IMAGE_DIR=./data/images
|
||||||
|
|
||||||
|
# DreamDict's built dictionary (French, European Portuguese, Spanish, Mandarin),
|
||||||
|
# opened read-only beside petal.db. Optional: with no file here, word lookups use
|
||||||
|
# the embedded English/Chinese datasets, which is how a laptop checkout runs.
|
||||||
|
# Build one with `go run ./cmd/dictimport` in the dreamdict repo.
|
||||||
|
DICT_PATH=./data/dict.db
|
||||||
|
|
||||||
# LLM
|
# LLM
|
||||||
LLM_BACKEND=vllm # vllm | ollama
|
LLM_BACKEND=vllm # vllm | ollama
|
||||||
LLM_ENDPOINT=http://localhost:8000 # vLLM :8000, Ollama :11434
|
LLM_ENDPOINT=http://localhost:8000 # vLLM :8000, Ollama :11434
|
||||||
@@ -38,6 +44,12 @@ TTS_AUDIO_FORMAT=mp3 # mp3 | opus | wav — mp3/opus transcode Pipe
|
|||||||
# them commented out for local development and Petal runs as the single
|
# them commented out for local development and Petal runs as the single
|
||||||
# hardcoded `local` user, exactly as it did before auth landed.
|
# hardcoded `local` user, exactly as it did before auth landed.
|
||||||
#
|
#
|
||||||
|
# That fallback is scoped to development on purpose. With a BASE_URL naming
|
||||||
|
# anything but localhost, Petal refuses to start rather than run open — see
|
||||||
|
# PETAL_REQUIRE_AUTH — because the fallback on a reachable host means every
|
||||||
|
# anonymous visitor is the `local` user, with full read and write over every
|
||||||
|
# document in the database.
|
||||||
|
#
|
||||||
# AUTHENTIK_URL is the issuer of the Petal provider in Authentik (the value of
|
# AUTHENTIK_URL is the issuer of the Petal provider in Authentik (the value of
|
||||||
# its "OpenID Configuration Issuer" field). The redirect URI to register there
|
# its "OpenID Configuration Issuer" field). The redirect URI to register there
|
||||||
# is BASE_URL + /auth/callback.
|
# is BASE_URL + /auth/callback.
|
||||||
@@ -46,8 +58,15 @@ TTS_AUDIO_FORMAT=mp3 # mp3 | opus | wav — mp3/opus transcode Pipe
|
|||||||
# AUTHENTIK_CLIENT_SECRET=
|
# AUTHENTIK_CLIENT_SECRET=
|
||||||
#
|
#
|
||||||
# Who may sign in: comma-separated OIDC subject ids and/or email addresses.
|
# Who may sign in: comma-separated OIDC subject ids and/or email addresses.
|
||||||
# Empty = anyone Authentik authenticates.
|
# Empty = anyone Authentik authenticates, which is right for a single-household
|
||||||
|
# instance and wrong the moment the IdP serves a wider audience than Petal. An
|
||||||
|
# empty list is warned about at every boot rather than assumed either way.
|
||||||
# PETAL_ALLOWED_SUBS=her@example.com,me@example.com
|
# PETAL_ALLOWED_SUBS=her@example.com,me@example.com
|
||||||
|
#
|
||||||
|
# Whether a missing OIDC configuration is fatal. Defaults to false for a
|
||||||
|
# loopback BASE_URL and true for anything else, so neither a laptop nor a
|
||||||
|
# deployment normally has to name it.
|
||||||
|
# PETAL_REQUIRE_AUTH=true
|
||||||
|
|
||||||
# --- Deferred (not wired in the local-dev build) ---
|
# --- Deferred (not wired in the local-dev build) ---
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,10 @@ web/dist/*
|
|||||||
!web/dist/.gitkeep
|
!web/dist/.gitkeep
|
||||||
*.log
|
*.log
|
||||||
|
|
||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
|
||||||
# Local env & data
|
# Local env & data
|
||||||
.env
|
.env
|
||||||
*.db
|
*.db
|
||||||
|
|||||||
+273
-22
File diff suppressed because one or more lines are too long
+6
-2
@@ -47,7 +47,10 @@ RUN apk add --no-cache ca-certificates ffmpeg tzdata \
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
COPY --from=build /out/petal /app/petal
|
COPY --from=build /out/petal /app/petal
|
||||||
|
|
||||||
# Mount point for petal.db (+ -wal/-shm), the image store and the TTS cache.
|
# Mount point for petal.db (+ -wal/-shm), the image store, the TTS cache and
|
||||||
|
# DreamDict's read-only dict.db. dict.db is deployed alongside rather than baked
|
||||||
|
# in: it is ~450 MB, changes a few times a year, and is shared with other
|
||||||
|
# services on the host — putting it in the image would multiply it by every tag.
|
||||||
RUN mkdir -p /data && chown -R petal:petal /data
|
RUN mkdir -p /data && chown -R petal:petal /data
|
||||||
VOLUME ["/data"]
|
VOLUME ["/data"]
|
||||||
|
|
||||||
@@ -57,7 +60,8 @@ EXPOSE 8080
|
|||||||
ENV PORT=8080 \
|
ENV PORT=8080 \
|
||||||
DATABASE_PATH=/data/petal.db \
|
DATABASE_PATH=/data/petal.db \
|
||||||
IMAGE_DIR=/data/images \
|
IMAGE_DIR=/data/images \
|
||||||
TTS_CACHE_DIR=/data/tts
|
TTS_CACHE_DIR=/data/tts \
|
||||||
|
DICT_PATH=/data/dict.db
|
||||||
|
|
||||||
# Same endpoint Traefik and the uptime probe use; needs no session by design.
|
# Same endpoint Traefik and the uptime probe use; needs no session by design.
|
||||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||||
|
|||||||
@@ -252,6 +252,17 @@ almost 1:1 onto `lexicon.Result`:
|
|||||||
It also carries data Petal has no equivalent for and could use: `Antonyms`,
|
It also carries data Petal has no equivalent for and could use: `Antonyms`,
|
||||||
`Frequency`, `Difficulty`, and `Etymology`.
|
`Frequency`, `Difficulty`, and `Etymology`.
|
||||||
|
|
||||||
|
> **Correction (2026-07-27, Phase 20).** The `Gloss` row of that table is wrong.
|
||||||
|
> `Translate(word, "en", L1)` reads Wiktionary's translation sections, which are
|
||||||
|
> thin in the en→X direction: measured on the real `dict.db`, it answers for
|
||||||
|
> **17%** of the 2,000 commonest English words into pt-PT and 16% into fr.
|
||||||
|
> Meaning has to come through shared WordNet synset ids instead (**61%**), which
|
||||||
|
> is what DreamDict's new `Equivalents(word, from, to)` does — falling back to
|
||||||
|
> the translations table, for 62% combined. The 1:1 mapping was assumed from the
|
||||||
|
> API surface and never checked against the data; it did not survive contact
|
||||||
|
> with it. Same measurement on zh reads 53% against ECDICT's near-total coverage
|
||||||
|
> of those words, which is why the zh pair did **not** converge.
|
||||||
|
|
||||||
So Phase D stops being gated on data and becomes an integration decision.
|
So Phase D stops being gated on data and becomes an integration decision.
|
||||||
|
|
||||||
### OPEN #6a (new): how to integrate — **SETTLED: Option 3** (ratified 2026-07-26)
|
### OPEN #6a (new): how to integrate — **SETTLED: Option 3** (ratified 2026-07-26)
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,72 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The baseline policy has to actually permit the frontend Vite builds. These
|
||||||
|
// are the allowances dist/index.html needs; if a future build starts emitting
|
||||||
|
// an inline script or pulling from a new host, that shows up here rather than
|
||||||
|
// as a blank page in production.
|
||||||
|
func TestBaselineCSPCoversTheBuiltFrontend(t *testing.T) {
|
||||||
|
required := []string{
|
||||||
|
"script-src 'self'", // Vite emits no inline script
|
||||||
|
"'unsafe-inline' https://fonts.googleapis.com", // React style={{…}} + the font link
|
||||||
|
"https://fonts.gstatic.com", // the font files themselves
|
||||||
|
"blob:", // read-aloud plays an object URL
|
||||||
|
"object-src 'none'",
|
||||||
|
"base-uri 'self'",
|
||||||
|
"frame-ancestors 'self'",
|
||||||
|
}
|
||||||
|
for _, want := range required {
|
||||||
|
if !strings.Contains(contentSecurityPolicy, want) {
|
||||||
|
t.Errorf("baseline CSP is missing %q:\n%s", want, contentSecurityPolicy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The middleware is a floor, not a ceiling: it runs *before* the handler
|
||||||
|
// precisely so a route serving untrusted bytes can overwrite the policy with a
|
||||||
|
// stricter one. This is the contract the image store depends on, and the reason
|
||||||
|
// the policy no longer lives in the Traefik labels — customresponseheaders
|
||||||
|
// would overwrite it in the other direction.
|
||||||
|
func TestRouteMayTightenTheBaselineCSP(t *testing.T) {
|
||||||
|
strict := "default-src 'none'; sandbox"
|
||||||
|
handler := securityHeaders(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.Header().Set("Content-Security-Policy", strict)
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/images/x.svg", nil))
|
||||||
|
|
||||||
|
if got := rec.Header().Get("Content-Security-Policy"); got != strict {
|
||||||
|
t.Fatalf("handler's policy was not honoured: got %q, want %q", got, strict)
|
||||||
|
}
|
||||||
|
// The headers it didn't touch still stand.
|
||||||
|
if rec.Header().Get("X-Content-Type-Options") != "nosniff" {
|
||||||
|
t.Error("baseline nosniff was lost")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every ordinary response carries the baseline.
|
||||||
|
func TestBaselineHeadersOnAPlainResponse(t *testing.T) {
|
||||||
|
handler := securityHeaders(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
||||||
|
|
||||||
|
for header, want := range map[string]string{
|
||||||
|
"Content-Security-Policy": contentSecurityPolicy,
|
||||||
|
"X-Content-Type-Options": "nosniff",
|
||||||
|
"Referrer-Policy": "same-origin",
|
||||||
|
} {
|
||||||
|
if got := rec.Header().Get(header); got != want {
|
||||||
|
t.Errorf("%s = %q, want %q", header, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+106
-7
@@ -64,30 +64,73 @@ func main() {
|
|||||||
sessions := auth.NewSessionStore(database.DB)
|
sessions := auth.NewSessionStore(database.DB)
|
||||||
users := auth.NewUserStore(database.DB)
|
users := auth.NewUserStore(database.DB)
|
||||||
|
|
||||||
|
// …and the fallback is exactly what must not happen quietly on a public
|
||||||
|
// host. Refuse to start rather than serve someone's journals to the open
|
||||||
|
// internet because one environment variable was misspelled. See
|
||||||
|
// config.RequireAuth for why this defaults on for any non-loopback BASE_URL.
|
||||||
|
if !cfg.AuthEnabled() && cfg.RequireAuth {
|
||||||
|
log.Fatalf("auth: refusing to start unauthenticated at %s.\n"+
|
||||||
|
" Petal would resolve every anonymous request to the single %q user, with full\n"+
|
||||||
|
" read and write over every document in the database.\n"+
|
||||||
|
" Set AUTHENTIK_URL, AUTHENTIK_CLIENT_ID and AUTHENTIK_CLIENT_SECRET, or set\n"+
|
||||||
|
" PETAL_REQUIRE_AUTH=false if this really is a trusted private network.",
|
||||||
|
cfg.BaseURL, db.LocalUserID)
|
||||||
|
}
|
||||||
|
|
||||||
var resolver auth.Resolver = auth.StaticResolver(db.LocalUserID)
|
var resolver auth.Resolver = auth.StaticResolver(db.LocalUserID)
|
||||||
var oidcClient *auth.OIDC
|
var oidcClient *auth.OIDC
|
||||||
if cfg.AuthEnabled() {
|
if cfg.AuthEnabled() {
|
||||||
|
allowed := auth.ParseAllowlist(cfg.AllowedSubs)
|
||||||
oidcClient = auth.NewOIDC(context.Background(), auth.Options{
|
oidcClient = auth.NewOIDC(context.Background(), auth.Options{
|
||||||
IssuerURL: cfg.AuthentikURL,
|
IssuerURL: cfg.AuthentikURL,
|
||||||
ClientID: cfg.AuthentikClientID,
|
ClientID: cfg.AuthentikClientID,
|
||||||
ClientSecret: cfg.AuthentikClientSecret,
|
ClientSecret: cfg.AuthentikClientSecret,
|
||||||
BaseURL: cfg.BaseURL,
|
BaseURL: cfg.BaseURL,
|
||||||
Allowed: auth.ParseAllowlist(cfg.AllowedSubs),
|
Allowed: allowed,
|
||||||
}, sessions, users)
|
}, sessions, users)
|
||||||
resolver = sessions
|
resolver = sessions
|
||||||
if n, err := sessions.Prune(); err == nil && n > 0 {
|
if n, err := sessions.Prune(); err == nil && n > 0 {
|
||||||
log.Printf("auth: pruned %d expired session(s)", n)
|
log.Printf("auth: pruned %d expired session(s)", n)
|
||||||
}
|
}
|
||||||
log.Printf("auth: OIDC enabled (issuer=%s, redirect=%s)", cfg.AuthentikURL, oidcClient.RedirectURI())
|
log.Printf("auth: OIDC enabled (issuer=%s, redirect=%s)", cfg.AuthentikURL, oidcClient.RedirectURI())
|
||||||
|
// An empty allowlist is a legitimate choice for a single-household
|
||||||
|
// instance and a wide-open door in front of an IdP that fronts anything
|
||||||
|
// else. Petal cannot tell which it is, so it says so every boot rather
|
||||||
|
// than assuming.
|
||||||
|
if len(allowed) == 0 {
|
||||||
|
log.Printf("auth: WARNING — PETAL_ALLOWED_SUBS is empty, so EVERY account %s "+
|
||||||
|
"authenticates may sign in and start writing here. Set it to the "+
|
||||||
|
"comma-separated emails (or subject ids) that belong in this Petal.",
|
||||||
|
cfg.AuthentikURL)
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
log.Printf("auth: OIDC not configured — running as the single %q user", db.LocalUserID)
|
log.Printf("auth: OIDC not configured — running as the single %q user", db.LocalUserID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The dictionary behind word lookups. dict.db is DreamDict's built database
|
||||||
|
// — French, European Portuguese, Spanish and Mandarin in one read-only file
|
||||||
|
// beside petal.db. It is optional on purpose: a laptop checkout has never
|
||||||
|
// had one, and the Chinese pair doesn't need one, so its absence downgrades
|
||||||
|
// lookups rather than stopping Petal. A file that is present but broken is
|
||||||
|
// a different matter and gets said out loud.
|
||||||
|
dict, err := lexicon.OpenDreamDict(cfg.DictPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("dictionary: %s unusable (%v) — falling back to the embedded datasets", cfg.DictPath, err)
|
||||||
|
}
|
||||||
|
defer dict.Close()
|
||||||
|
lexSet := lexicon.NewSet(dict)
|
||||||
|
if lexSet.HasDreamDict() {
|
||||||
|
log.Printf("dictionary: DreamDict open at %s (%s)", cfg.DictPath, lexSet.Contents())
|
||||||
|
} else {
|
||||||
|
log.Printf("dictionary: no dict.db at %s — English/Chinese only", cfg.DictPath)
|
||||||
|
}
|
||||||
|
|
||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
r.Use(middleware.RequestID)
|
r.Use(middleware.RequestID)
|
||||||
r.Use(middleware.RealIP)
|
r.Use(middleware.RealIP)
|
||||||
r.Use(middleware.Logger)
|
r.Use(middleware.Logger)
|
||||||
r.Use(middleware.Recoverer)
|
r.Use(middleware.Recoverer)
|
||||||
|
r.Use(securityHeaders)
|
||||||
|
|
||||||
// Build version: a hash of the embedded SPA shell. Vite rewrites index.html
|
// Build version: a hash of the embedded SPA shell. Vite rewrites index.html
|
||||||
// with content-hashed asset names on every build, so this string changes
|
// with content-hashed asset names on every build, so this string changes
|
||||||
@@ -131,6 +174,12 @@ func main() {
|
|||||||
// this id and shows the signed-in writer.
|
// this id and shows the signed-in writer.
|
||||||
pr.Get("/me", users.MeHandler())
|
pr.Get("/me", users.MeHandler())
|
||||||
|
|
||||||
|
// …and the one thing about herself she can change: which language
|
||||||
|
// Petal is her pair in. It lives here rather than under a /settings
|
||||||
|
// tree because there is exactly one setting and it is a property of
|
||||||
|
// the user row — the same row /me reads back.
|
||||||
|
pr.Patch("/me", users.UpdateMeHandler())
|
||||||
|
|
||||||
llmClient := llm.NewLLMClient(cfg)
|
llmClient := llm.NewLLMClient(cfg)
|
||||||
sug := suggestions.New(database, llmClient)
|
sug := suggestions.New(database, llmClient)
|
||||||
|
|
||||||
@@ -149,13 +198,18 @@ func main() {
|
|||||||
pr.Mount("/suggestions", sug.Routes())
|
pr.Mount("/suggestions", sug.Routes())
|
||||||
|
|
||||||
// Offline lexicon: full word lookups (gloss + definition + synonyms) for
|
// Offline lexicon: full word lookups (gloss + definition + synonyms) for
|
||||||
// the right-click popover, and the lightweight Chinese-only gloss for the
|
// the right-click popover, and the lightweight gloss-only lookup for the
|
||||||
// inline hover/select tooltip. One handler so the datasets load once.
|
// inline hover/select tooltip. One handler over one provider Set, so the
|
||||||
// The dataset is static and identical for everyone, but it stays behind
|
// embedded datasets and dict.db are each opened once. Which of them
|
||||||
// auth so the API surface has no unauthenticated read holes.
|
// answers depends on the caller's language pair — so unlike before, the
|
||||||
lex := lexicon.NewHandler()
|
// response is no longer identical for everyone, and it stays behind auth
|
||||||
|
// for that reason as much as for the API surface.
|
||||||
|
lex := lexicon.NewHandler(database.DB, lexSet)
|
||||||
pr.Mount("/word", lex.Routes())
|
pr.Mount("/word", lex.Routes())
|
||||||
pr.Mount("/gloss", lex.GlossRoutes())
|
pr.Mount("/gloss", lex.GlossRoutes())
|
||||||
|
// The same lookup pointing the other way: a Chinese word to its pinyin
|
||||||
|
// and English senses, for an account whose direction is learning_pair.
|
||||||
|
pr.Mount("/hanzi", lex.HanziRoutes())
|
||||||
|
|
||||||
// Vocabulary garden: words the writer looks up are captured here and
|
// Vocabulary garden: words the writer looks up are captured here and
|
||||||
// surfaced for gentle spaced-repetition review.
|
// surfaced for gentle spaced-repetition review.
|
||||||
@@ -180,7 +234,11 @@ func main() {
|
|||||||
// back to the browser's Web Speech API on its own.
|
// back to the browser's Web Speech API on its own.
|
||||||
if ttsHandler, ok := tts.New(cfg); ok {
|
if ttsHandler, ok := tts.New(cfg); ok {
|
||||||
pr.Mount("/tts", ttsHandler.Routes())
|
pr.Mount("/tts", ttsHandler.Routes())
|
||||||
log.Printf("read-aloud enabled (TTS endpoint=%s)", cfg.TTSEndpoint)
|
// Name the languages, not just the English endpoint: which
|
||||||
|
// voices a deployment actually reached is the thing worth
|
||||||
|
// seeing at boot, and a missing sidecar is silent otherwise
|
||||||
|
// (a 404 the client answers by quietly using Web Speech).
|
||||||
|
log.Printf("read-aloud enabled (voices: %s)", strings.Join(ttsHandler.Languages(), ", "))
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
@@ -201,6 +259,47 @@ func main() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// contentSecurityPolicy is the default policy for everything Petal serves.
|
||||||
|
//
|
||||||
|
// It lives here rather than in the Traefik labels, and that move is the point:
|
||||||
|
// Traefik's customResponseHeaders *sets* a header, overwriting whatever the
|
||||||
|
// application chose, so a policy declared at the edge silently replaces the
|
||||||
|
// stricter one an individual route needs. Stored images need exactly that (an
|
||||||
|
// uploaded SVG is a document that can carry script — see internal/images), and
|
||||||
|
// a rule the edge can quietly undo is not a rule.
|
||||||
|
//
|
||||||
|
// The allowances are what the built frontend actually uses, no more: script
|
||||||
|
// only from Petal itself (Vite emits no inline script — this policy is checked
|
||||||
|
// against dist/index.html), inline *styles* because React's style={{…}} props
|
||||||
|
// compile to style attributes, and Google's font hosts because index.html links
|
||||||
|
// them. object-src and base-uri close the two attribute-injection routes that
|
||||||
|
// survive HTML escaping.
|
||||||
|
const contentSecurityPolicy = "default-src 'self'; " +
|
||||||
|
"script-src 'self'; " +
|
||||||
|
"style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " +
|
||||||
|
"font-src 'self' data: https://fonts.gstatic.com; " +
|
||||||
|
"img-src 'self' data: blob:; " +
|
||||||
|
"media-src 'self' data: blob:; " +
|
||||||
|
"connect-src 'self'; " +
|
||||||
|
"object-src 'none'; " +
|
||||||
|
"base-uri 'self'; " +
|
||||||
|
"form-action 'self'; " +
|
||||||
|
"frame-ancestors 'self'"
|
||||||
|
|
||||||
|
// securityHeaders lays down the baseline response headers before the handler
|
||||||
|
// runs, so a route that needs something stricter — the image store — simply
|
||||||
|
// overwrites its own copy on the way past. Ordering is the mechanism: this is a
|
||||||
|
// floor, not a ceiling.
|
||||||
|
func securityHeaders(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
h := w.Header()
|
||||||
|
h.Set("Content-Security-Policy", contentSecurityPolicy)
|
||||||
|
h.Set("X-Content-Type-Options", "nosniff")
|
||||||
|
h.Set("Referrer-Policy", "same-origin")
|
||||||
|
next.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// maxAPIBodyBytes caps a JSON API request body at 2 MiB. That's far above any
|
// maxAPIBodyBytes caps a JSON API request body at 2 MiB. That's far above any
|
||||||
// real document save (the body is text plus lightweight marks; images upload
|
// real document save (the body is text plus lightweight marks; images upload
|
||||||
// separately by reference) while still bounding abuse. Exceeding it makes the
|
// separately by reference) while still bounding abuse. Exceeding it makes the
|
||||||
|
|||||||
+163
-7
@@ -23,11 +23,13 @@ runs them.
|
|||||||
|
|
||||||
- **petal** — the single Go binary with the frontend embedded. Publishes no host
|
- **petal** — the single Go binary with the frontend embedded. Publishes no host
|
||||||
port; Traefik is the only way in.
|
port; Traefik is the only way in.
|
||||||
- **piper-en** / **piper-zh** — read-aloud. Each Piper HTTP server loads exactly
|
- **piper-en** / **piper-zh** / **piper-pt** / **piper-fr** — read-aloud. Each
|
||||||
one voice, so English and Chinese are separate containers off one image, with
|
Piper HTTP server loads exactly one voice, so every language is its own
|
||||||
the models cached in a shared volume. They sit on an internal network with no
|
container off one image, with the models cached in a shared volume. They sit
|
||||||
published ports, so only Petal can reach them. Adding pt-PT in Phase 21 is a
|
on an internal network with no published ports, so only Petal can reach them.
|
||||||
fourth service, not a new image.
|
Adding pt-PT in Phase 21 was a third service and fr in Phase 24 a fourth —
|
||||||
|
never a new image, and since Phase 21 never any Go either (the languages are
|
||||||
|
discovered from `TTS_ENDPOINT_<LANG>`/`TTS_VOICE_<LANG>`).
|
||||||
|
|
||||||
They run as containers rather than the host systemd units millenia uses because
|
They run as containers rather than the host systemd units millenia uses because
|
||||||
Piper was never actually installed on the VPS, and the `reala` account has no
|
Piper was never actually installed on the VPS, and the `reala` account has no
|
||||||
@@ -231,8 +233,21 @@ unauthenticated `/api/health` router that existed only to escape it: every `/api
|
|||||||
route now answers 401 without a session, and the only thing an anonymous visitor
|
route now answers 401 without a session, and the only thing an anonymous visitor
|
||||||
gets is the app shell and a redirect to sign in.
|
gets is the app shell and a redirect to sign in.
|
||||||
|
|
||||||
If you ever run this stack *without* `AUTHENTIK_*` configured — Petal then falls
|
Removing that gate meant a missing `AUTHENTIK_*` variable stopped being a
|
||||||
back to the single `local` user — put the gate back before pointing DNS at it:
|
nuisance and became an exposure: Petal would fall back to the single `local`
|
||||||
|
user and hand every anonymous visitor full read and write over the database,
|
||||||
|
saying so only in a log line. So **it now refuses to start instead**:
|
||||||
|
|
||||||
|
```
|
||||||
|
auth: refusing to start unauthenticated at https://petal.parodia.dev.
|
||||||
|
Petal would resolve every anonymous request to the single "local" user, ...
|
||||||
|
```
|
||||||
|
|
||||||
|
The guard defaults on for any `BASE_URL` that isn't loopback, so a laptop
|
||||||
|
checkout still runs open and a deployment cannot. If you genuinely want an
|
||||||
|
unauthenticated instance on a trusted private network, say so out loud with
|
||||||
|
`PETAL_REQUIRE_AUTH=false` — and if it is reachable from anywhere else, put a
|
||||||
|
gate back in front of it first:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
traefik.http.routers.petal.middlewares: compression@file,petal-headers,petal-auth
|
traefik.http.routers.petal.middlewares: compression@file,petal-headers,petal-auth
|
||||||
@@ -241,6 +256,28 @@ traefik.http.middlewares.petal-auth.basicauth.users: ${PETAL_BASIC_AUTH:?}
|
|||||||
|
|
||||||
with `htpasswd -nbB petal 'your-password'` in `.env` as `PETAL_BASIC_AUTH`.
|
with `htpasswd -nbB petal 'your-password'` in `.env` as `PETAL_BASIC_AUTH`.
|
||||||
|
|
||||||
|
### Who gets in
|
||||||
|
|
||||||
|
`PETAL_ALLOWED_SUBS` is a comma-separated list of emails and/or OIDC subject
|
||||||
|
ids. **Set it.** Empty means everyone authentik authenticates, and authentik on
|
||||||
|
this host fronts several applications — a valid account there is not the same as
|
||||||
|
belonging in someone's private journal. An empty list is legal (a single-
|
||||||
|
household instance may want it) and warns at every boot:
|
||||||
|
|
||||||
|
```
|
||||||
|
auth: WARNING — PETAL_ALLOWED_SUBS is empty, so EVERY account ... may sign in
|
||||||
|
```
|
||||||
|
|
||||||
|
### Response headers
|
||||||
|
|
||||||
|
`Content-Security-Policy`, `X-Content-Type-Options` and `Referrer-Policy` are set
|
||||||
|
by the binary, not by the Traefik labels. Traefik's `customresponseheaders`
|
||||||
|
*overwrites*, which would silently replace the stricter policy an individual
|
||||||
|
route picks for itself — the image store serves stored uploads under
|
||||||
|
`default-src 'none'; sandbox` so that an SVG someone pasted into a document
|
||||||
|
cannot run as a page on Petal's own origin. Only HSTS stays at the edge, where
|
||||||
|
TLS is actually terminated.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 4a. Moving an account (`scripts/migrate_local_user.py`)
|
## 4a. Moving an account (`scripts/migrate_local_user.py`)
|
||||||
@@ -284,6 +321,89 @@ a migration took; the presence of the row itself says nothing.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## 4b. The dictionary (`dict.db`)
|
||||||
|
|
||||||
|
Word lookups for the French, European Portuguese and Spanish pairs come from
|
||||||
|
[DreamDict](https://github.com/prosolis/dreamdict)'s built database, which Petal
|
||||||
|
opens **read-only** beside `petal.db`. Petal imports DreamDict's `dictionary`
|
||||||
|
package directly — there is no DreamDict service to run and nothing to reach
|
||||||
|
over the VPN, which matters because a hover gloss must answer in milliseconds.
|
||||||
|
|
||||||
|
`dict.db` is **optional**. With no file at `DICT_PATH` Petal logs
|
||||||
|
|
||||||
|
```
|
||||||
|
dictionary: no dict.db at /data/dict.db — English/Chinese only
|
||||||
|
```
|
||||||
|
|
||||||
|
and serves lookups from the datasets compiled into the binary. The Chinese pair
|
||||||
|
is unaffected either way — it stays on ECDICT (see below) — and a non-Chinese
|
||||||
|
writer still gets English definitions, synonyms and pronunciation, losing only
|
||||||
|
the translation. **A dictionary that failed to deploy costs the gloss, not the
|
||||||
|
popover.** A file that is present but was never imported is a different matter
|
||||||
|
and is logged as an error.
|
||||||
|
|
||||||
|
### Installing it
|
||||||
|
|
||||||
|
The database is built by DreamDict's own import CLI from ~6 GB of source data;
|
||||||
|
it is not built on the VPS. Copy the built file into the data volume:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# on the machine holding a built dict.db (millenia: ~/dreamdict/data/dict.db)
|
||||||
|
scp ~/dreamdict/data/dict.db reala@100.64.0.1:/home/reala/petal/data/dict.db
|
||||||
|
# on parodia
|
||||||
|
chown "$(id -u):$(id -g)" /home/reala/petal/data/dict.db
|
||||||
|
docker compose restart petal # the handle is opened once, at startup
|
||||||
|
```
|
||||||
|
|
||||||
|
Expect ~450 MB. It sits inside the LUKS volume with everything else (§6). The
|
||||||
|
backups name `petal.db` explicitly rather than sweeping the data directory
|
||||||
|
(§5), so `dict.db` stays out of them — which is the right outcome and worth
|
||||||
|
keeping: it is rebuildable from public data and would otherwise dominate every
|
||||||
|
nightly snapshot. Petal never writes to it.
|
||||||
|
|
||||||
|
### Why Chinese doesn't use it
|
||||||
|
|
||||||
|
The zh pair stays on the embedded ECDICT gloss, deliberately. Measured on the
|
||||||
|
deployed database, DreamDict reaches a Chinese gloss for 53% of the 2,000
|
||||||
|
commonest English words; ECDICT covers essentially all of them and is in daily
|
||||||
|
use by a real writer. `lexicon.Set.For` is where that decision lives — one
|
||||||
|
`switch`, changed the day a comparison on her actual lookups says otherwise.
|
||||||
|
|
||||||
|
For pt-PT and French the same measurement reads 62% and 63%, which is why they
|
||||||
|
use DreamDict: there is no alternative source for them at all.
|
||||||
|
|
||||||
|
### Rebuilding it
|
||||||
|
|
||||||
|
Rebuilt 2026-07-27 to add Spanish (the previous file predated DreamDict's
|
||||||
|
Spanish support). The recipe, since it will be needed again:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# on millenia, from a clean checkout of dreamdict main
|
||||||
|
./scripts/download-dict-data.sh ~/dreamdict/data # idempotent; skips what's there
|
||||||
|
go run ./cmd/dictimport --data ~/dreamdict/data --db ./dict.db --clean
|
||||||
|
```
|
||||||
|
|
||||||
|
~6 minutes on 32 cores; the data directory is ~7 GB and mostly already
|
||||||
|
downloaded. **Build to a new path, never over a file in use** — then verify by
|
||||||
|
hash on both ends before swapping.
|
||||||
|
|
||||||
|
Two things worth knowing before trusting a rebuild:
|
||||||
|
|
||||||
|
- The SUBTLEX-US download fails (the source moved behind a manual export). It
|
||||||
|
does not matter: the loader falls back to `SUBTLEX-US.txt`, which is present,
|
||||||
|
and English "frequency" is mostly SCOWL's commonness bucket anyway —
|
||||||
|
1000/800/600/…/50, refined by SUBTLEX for only ~1,600 words. That is why the
|
||||||
|
word-difficulty chip reads `difficulty`, not `frequency`.
|
||||||
|
- Check the *other* languages' counts are unchanged before shipping. The 2026-07
|
||||||
|
rebuild came out byte-identical for en/fr/pt-PT/zh, which is what says it
|
||||||
|
added a language rather than quietly shifting the rest.
|
||||||
|
|
||||||
|
Gloss coverage of the 2,000 commonest English words, after the rebuild:
|
||||||
|
**es 68.6%**, fr 63.1%, pt-PT 62.1%, zh 53.2%. The startup line reports actual
|
||||||
|
per-language row counts, so a database missing a language says so.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## 5. Backups
|
## 5. Backups
|
||||||
|
|
||||||
### On the VPS — folded into `parodia-backup`
|
### On the VPS — folded into `parodia-backup`
|
||||||
@@ -496,6 +616,42 @@ Petal's env then carries `TTS_ENDPOINT=http://127.0.0.1:5005`,
|
|||||||
maps language → instance from config, so another language is another instance
|
maps language → instance from config, so another language is another instance
|
||||||
plus an env pair, no code change.
|
plus an env pair, no code change.
|
||||||
|
|
||||||
|
**Adding a language (Phase 21 made this literal).** Petal discovers its Piper
|
||||||
|
instances from the environment: English is the unsuffixed
|
||||||
|
`TTS_ENDPOINT`/`TTS_VOICE_EN`, and every other language is a
|
||||||
|
`TTS_ENDPOINT_<LANG>`/`TTS_VOICE_<LANG>` pair. `<LANG>` is the *base* tag —
|
||||||
|
`PT`, not `PT_PT`, because an environment variable name cannot hold a hyphen and
|
||||||
|
only one Portuguese model is loaded regardless. Both halves must be set: an
|
||||||
|
endpoint with no voice is dropped, so a half-finished language reads to the
|
||||||
|
browser as "no voice here, use Web Speech" instead of erroring on every tap. The
|
||||||
|
startup line names what it actually resolved:
|
||||||
|
|
||||||
|
```
|
||||||
|
read-aloud enabled (voices: en=en_US-amy-medium, pt=pt_PT-tugão-medium, zh=zh_CN-huayan-medium)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Portuguese: `pt_PT-tugão-medium` is the only European voice Piper ships.** The
|
||||||
|
other five `pt_*` models in the catalogue are all Brazilian, so the voice has to
|
||||||
|
be named explicitly for the same reason the Hunspell dictionary did (Phase 21):
|
||||||
|
the obvious default is the wrong country. Check what exists before assuming:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec petal-piper-en python -c "import urllib.request,json; \
|
||||||
|
d=json.load(urllib.request.urlopen('https://huggingface.co/rhasspy/piper-voices/resolve/main/voices.json')); \
|
||||||
|
print([k for k in d if k.startswith('pt')])"
|
||||||
|
```
|
||||||
|
|
||||||
|
**French: the opposite situation, and worth knowing it is.** Every `fr_*` voice
|
||||||
|
in the catalogue is `fr_FR`, so there is no wrong country to land on by default
|
||||||
|
and no Québec voice to choose instead; `fr_FR-siwis-medium` is picked to match
|
||||||
|
the register of the other three rather than to avoid anything. The name is also
|
||||||
|
plain ASCII, so the entrypoint's percent-encoded download fallback — which
|
||||||
|
exists only because `tugão` broke `piper.download_voices` — never fires here.
|
||||||
|
|
||||||
|
**Slow replay.** `POST /api/tts` takes `slow: true`, which raises Piper's
|
||||||
|
`length_scale` to about 4/3 (≈0.75× pace). It is a separate cache entry, not a
|
||||||
|
playback-rate trick, so the slow clip is synthesized once and then instant.
|
||||||
|
|
||||||
**Piper version note:** piper-tts moved synthesis from `POST /` to
|
**Piper version note:** piper-tts moved synthesis from `POST /` to
|
||||||
`POST /synthesize` in 1.6.0, with an identical request body. `TTS_PATH` selects
|
`POST /synthesize` in 1.6.0, with an identical request body. `TTS_PATH` selects
|
||||||
which — it defaults to `/`, and both the VPS compose and millenia's `start.sh`
|
which — it defaults to `/`, and both the VPS compose and millenia's `start.sh`
|
||||||
|
|||||||
+48
-16
@@ -20,13 +20,9 @@ TZ=Europe/Lisbon
|
|||||||
PETAL_UID=1001
|
PETAL_UID=1001
|
||||||
PETAL_GID=1001
|
PETAL_GID=1001
|
||||||
|
|
||||||
# --- Interim edge gate (delete when Phase 16 auth lands) ---------------------
|
# (The interim PETAL_BASIC_AUTH edge gate is gone: Petal authenticates for
|
||||||
# Petal has no authentication of its own yet — StaticResolver hands every
|
# itself now, and the auth block at the bottom of this file is what holds the
|
||||||
# request the same local user — so Traefik holds the door with basic auth until
|
# door. A second password in front of a real login is one more thing to lose.)
|
||||||
# the OIDC flow exists. user:bcrypt-hash, as produced by:
|
|
||||||
# htpasswd -nbB petal 'your-password'
|
|
||||||
# /api/health is deliberately exempt (its own router) so monitoring still works.
|
|
||||||
PETAL_BASIC_AUTH=
|
|
||||||
|
|
||||||
# --- LLM (millenia, over headscale) ------------------------------------------
|
# --- LLM (millenia, over headscale) ------------------------------------------
|
||||||
# The only cross-VPN dependency. Petal degrades warmly when it's unreachable:
|
# The only cross-VPN dependency. Petal degrades warmly when it's unreachable:
|
||||||
@@ -49,22 +45,58 @@ LLM_TIMEOUT=90s
|
|||||||
# --- Read-aloud (Piper sidecars) ---------------------------------------------
|
# --- Read-aloud (Piper sidecars) ---------------------------------------------
|
||||||
# Endpoints are wired in docker-compose.yml; these pick the voice each sidecar
|
# Endpoints are wired in docker-compose.yml; these pick the voice each sidecar
|
||||||
# loads. Changing one means recreating that container so it downloads the model.
|
# loads. Changing one means recreating that container so it downloads the model.
|
||||||
|
#
|
||||||
|
# A language is routable only when both halves are set — a TTS_ENDPOINT_XX with
|
||||||
|
# no TTS_VOICE_XX reads as "no voice for this language" and the browser's own
|
||||||
|
# synthesizer takes over, rather than as an instance that errors on every
|
||||||
|
# request. Adding es is a compose service plus a pair of lines here.
|
||||||
|
#
|
||||||
|
# pt_PT-tugão-medium is the only European Portuguese voice Piper ships; every
|
||||||
|
# other pt model in the catalogue is Brazilian. French has the opposite
|
||||||
|
# property — every fr voice in the catalogue is fr_FR — so there is no wrong
|
||||||
|
# country to land on and no non-ASCII name to trip the downloader.
|
||||||
TTS_VOICE_EN=en_US-amy-medium
|
TTS_VOICE_EN=en_US-amy-medium
|
||||||
TTS_VOICE_ZH=zh_CN-huayan-medium
|
TTS_VOICE_ZH=zh_CN-huayan-medium
|
||||||
|
TTS_VOICE_PT=pt_PT-tugão-medium
|
||||||
|
TTS_VOICE_FR=fr_FR-siwis-medium
|
||||||
|
# Mexican, not peninsular — the es pack is written in neutral Latin American
|
||||||
|
# Spanish, and es_ES-davefx-medium would read it in the accent it avoids.
|
||||||
|
TTS_VOICE_ES=es_MX-ald-medium
|
||||||
TTS_AUDIO_FORMAT=mp3
|
TTS_AUDIO_FORMAT=mp3
|
||||||
TTS_TIMEOUT=15s
|
TTS_TIMEOUT=15s
|
||||||
|
|
||||||
# --- Auth (Authentik OIDC) ---------------------------------------------------
|
# --- Auth (Authentik OIDC) ---------------------------------------------------
|
||||||
# Authentik already runs on this host. Set all three and Petal authenticates
|
# NOT OPTIONAL HERE. Authentik already runs on this host; set all three and
|
||||||
# for itself; leave any unset and it falls back to the single `local` user
|
# Petal authenticates for itself.
|
||||||
# (which on a public host means the Traefik basic-auth gate must stay).
|
#
|
||||||
|
# Leave any of them unset and Petal REFUSES TO START, because the alternative is
|
||||||
|
# worse: it would otherwise fall back to resolving every anonymous request to
|
||||||
|
# the single `local` user, handing the open internet full read and write over
|
||||||
|
# every document in the database. That fallback is right on a laptop and a
|
||||||
|
# catastrophe on this host, so the guard is on for any non-loopback BASE_URL.
|
||||||
|
# See PETAL_REQUIRE_AUTH below.
|
||||||
#
|
#
|
||||||
# AUTHENTIK_URL is the provider's issuer, and the redirect URI to register in
|
# AUTHENTIK_URL is the provider's issuer, and the redirect URI to register in
|
||||||
# Authentik is https://petal.parodia.dev/auth/callback.
|
# Authentik is https://petal.parodia.dev/auth/callback.
|
||||||
# AUTHENTIK_URL=https://auth.parodia.dev/application/o/petal/
|
AUTHENTIK_URL=https://auth.parodia.dev/application/o/petal/
|
||||||
# AUTHENTIK_CLIENT_ID=petal
|
AUTHENTIK_CLIENT_ID=petal
|
||||||
# AUTHENTIK_CLIENT_SECRET=
|
AUTHENTIK_CLIENT_SECRET=
|
||||||
|
|
||||||
|
# Who may sign in: comma-separated subject ids and/or emails.
|
||||||
#
|
#
|
||||||
# Who may sign in: comma-separated subject ids and/or emails. Empty = anyone
|
# SET THIS. Empty means everyone Authentik authenticates, and Authentik on this
|
||||||
# Authentik authenticates, which is wider than this instance wants.
|
# host fronts half a dozen applications — being a valid user there is not the
|
||||||
# PETAL_ALLOWED_SUBS=
|
# same as belonging in someone's private journal. An empty value is legal (a
|
||||||
|
# single-household instance may genuinely want it) and says so loudly in the
|
||||||
|
# startup log every boot.
|
||||||
|
#
|
||||||
|
# An email is knowable in advance; a subject id is an opaque uuid nobody can
|
||||||
|
# know before that person's first login. Use emails to invite, subject ids to
|
||||||
|
# pin.
|
||||||
|
PETAL_ALLOWED_SUBS=her@example.com,me@example.com
|
||||||
|
|
||||||
|
# The guard itself. Defaulted from BASE_URL — loopback origins run open, real
|
||||||
|
# ones demand a login — so it does not normally need setting. Set it to false
|
||||||
|
# only for a deployment genuinely reachable from nowhere but a trusted network,
|
||||||
|
# and understand that it means anyone who reaches Petal is the `local` user.
|
||||||
|
# PETAL_REQUIRE_AUTH=true
|
||||||
|
|||||||
@@ -10,7 +10,42 @@ port="${PIPER_PORT:-5000}"
|
|||||||
|
|
||||||
if [ ! -f "${data_dir}/${voice}.onnx" ]; then
|
if [ ! -f "${data_dir}/${voice}.onnx" ]; then
|
||||||
echo ">> downloading voice ${voice} into ${data_dir}"
|
echo ">> downloading voice ${voice} into ${data_dir}"
|
||||||
python -m piper.download_voices "${voice}" --data-dir "${data_dir}"
|
# piper.download_voices cannot fetch a voice whose name isn't ASCII, and the
|
||||||
|
# only European Portuguese voice in the catalogue is pt_PT-tugão-medium:
|
||||||
|
# the downloader pastes the name straight into the request line, and
|
||||||
|
# http.client encodes that as ASCII, so it dies with UnicodeEncodeError on
|
||||||
|
# the ã before a byte leaves the container. Every pt_BR voice downloads
|
||||||
|
# fine — the failure lands precisely on the voice the pt-PT pair needs.
|
||||||
|
#
|
||||||
|
# So: try the supported path, and fall back to fetching the two files
|
||||||
|
# ourselves with the URL percent-encoded, which is all the downloader was
|
||||||
|
# missing. Same host, same files, same destination names.
|
||||||
|
python -m piper.download_voices "${voice}" --data-dir "${data_dir}" || {
|
||||||
|
echo ">> download_voices failed for ${voice}; fetching directly (non-ASCII voice name)"
|
||||||
|
python - "${voice}" "${data_dir}" <<'PY'
|
||||||
|
import json, sys, urllib.parse, urllib.request
|
||||||
|
|
||||||
|
voice, data_dir = sys.argv[1], sys.argv[2]
|
||||||
|
BASE = "https://huggingface.co/rhasspy/piper-voices/resolve/main/"
|
||||||
|
|
||||||
|
catalogue = json.load(urllib.request.urlopen(BASE + "voices.json", timeout=120))
|
||||||
|
entry = catalogue.get(voice)
|
||||||
|
if entry is None:
|
||||||
|
sys.exit(f"no voice named {voice!r} in the catalogue")
|
||||||
|
|
||||||
|
# The catalogue keys the files by repo path; only the model and its config are
|
||||||
|
# needed to serve (MODEL_CARD is licence text).
|
||||||
|
for path in entry["files"]:
|
||||||
|
if not path.endswith((".onnx", ".onnx.json")):
|
||||||
|
continue
|
||||||
|
url = BASE + urllib.parse.quote(path)
|
||||||
|
dest = f"{data_dir}/{path.rsplit('/', 1)[-1]}"
|
||||||
|
print(f">> {url} -> {dest}", flush=True)
|
||||||
|
with urllib.request.urlopen(url, timeout=600) as r, open(dest, "wb") as out:
|
||||||
|
while chunk := r.read(1 << 20):
|
||||||
|
out.write(chunk)
|
||||||
|
PY
|
||||||
|
}
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo ">> serving ${voice} on :${port}"
|
echo ">> serving ${voice} on :${port}"
|
||||||
|
|||||||
+82
-4
@@ -38,10 +38,21 @@ services:
|
|||||||
DATABASE_PATH: /data/petal.db
|
DATABASE_PATH: /data/petal.db
|
||||||
IMAGE_DIR: /data/images
|
IMAGE_DIR: /data/images
|
||||||
TTS_CACHE_DIR: /data/tts
|
TTS_CACHE_DIR: /data/tts
|
||||||
|
# DreamDict's built dictionary, read-only, deployed into the data volume
|
||||||
|
# (see deploy/README.md). Absent it, word lookups fall back to the
|
||||||
|
# embedded English/Chinese datasets rather than failing.
|
||||||
|
DICT_PATH: /data/dict.db
|
||||||
# Piper sidecars. Each server loads one voice, so English and Chinese are
|
# Piper sidecars. Each server loads one voice, so English and Chinese are
|
||||||
# separate containers; the handler maps language → instance from config.
|
# separate containers; the handler maps language → instance from config.
|
||||||
TTS_ENDPOINT: http://piper-en:5000
|
TTS_ENDPOINT: http://piper-en:5000
|
||||||
TTS_ENDPOINT_ZH: http://piper-zh:5000
|
TTS_ENDPOINT_ZH: http://piper-zh:5000
|
||||||
|
# A language is discovered from the TTS_ENDPOINT_<LANG>/TTS_VOICE_<LANG>
|
||||||
|
# pair, so fr and es cost a service and two lines rather than a code
|
||||||
|
# change. <LANG> is the base tag — an env var name can't hold pt-PT's
|
||||||
|
# hyphen, and there is one Portuguese voice loaded either way.
|
||||||
|
TTS_ENDPOINT_PT: http://piper-pt:5000
|
||||||
|
TTS_ENDPOINT_FR: http://piper-fr:5000
|
||||||
|
TTS_ENDPOINT_ES: http://piper-es:5000
|
||||||
# The sidecars run piper-tts 1.6.0, which serves synthesis on
|
# The sidecars run piper-tts 1.6.0, which serves synthesis on
|
||||||
# /synthesize; millenia's older server keeps the default "/".
|
# /synthesize; millenia's older server keeps the default "/".
|
||||||
TTS_PATH: /synthesize
|
TTS_PATH: /synthesize
|
||||||
@@ -71,6 +82,7 @@ services:
|
|||||||
depends_on:
|
depends_on:
|
||||||
- piper-en
|
- piper-en
|
||||||
- piper-zh
|
- piper-zh
|
||||||
|
- piper-pt
|
||||||
labels:
|
labels:
|
||||||
traefik.enable: "true"
|
traefik.enable: "true"
|
||||||
traefik.docker.network: traefik
|
traefik.docker.network: traefik
|
||||||
@@ -87,11 +99,18 @@ services:
|
|||||||
# second password in front of a real login is just one more thing to lose.
|
# second password in front of a real login is just one more thing to lose.
|
||||||
traefik.http.routers.petal.middlewares: compression@file,petal-headers
|
traefik.http.routers.petal.middlewares: compression@file,petal-headers
|
||||||
traefik.http.services.petal.loadbalancer.server.port: "8080"
|
traefik.http.services.petal.loadbalancer.server.port: "8080"
|
||||||
# Petal is a private writing space: no framing, no sniffing, HSTS on.
|
# HSTS is the edge's business — it is a statement about the TLS
|
||||||
traefik.http.middlewares.petal-headers.headers.customresponseheaders.Content-Security-Policy: frame-ancestors 'self'
|
# termination, which happens here and not in the container.
|
||||||
|
#
|
||||||
|
# The Content-Security-Policy that used to sit alongside it has moved into
|
||||||
|
# the app (see securityHeaders in cmd/server/main.go). customresponseheaders
|
||||||
|
# *overwrites*, so a policy set here would silently replace the stricter
|
||||||
|
# one an individual route chooses for itself — which is exactly what the
|
||||||
|
# image store does to keep an uploaded SVG from running as a page. A rule
|
||||||
|
# the edge can quietly undo is not a rule. X-Content-Type-Options and
|
||||||
|
# Referrer-Policy moved with it for the same reason: one place to read,
|
||||||
|
# and no dependence on this file being deployed alongside the binary.
|
||||||
traefik.http.middlewares.petal-headers.headers.customresponseheaders.Strict-Transport-Security: max-age=31536000; includeSubDomains
|
traefik.http.middlewares.petal-headers.headers.customresponseheaders.Strict-Transport-Security: max-age=31536000; includeSubDomains
|
||||||
traefik.http.middlewares.petal-headers.headers.customresponseheaders.X-Content-Type-Options: nosniff
|
|
||||||
traefik.http.middlewares.petal-headers.headers.customresponseheaders.Referrer-Policy: same-origin
|
|
||||||
|
|
||||||
piper-en:
|
piper-en:
|
||||||
build:
|
build:
|
||||||
@@ -119,6 +138,65 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- internal
|
- internal
|
||||||
|
|
||||||
|
# European Portuguese, for the pt-PT pair. pt_PT-tugão-medium is the *only*
|
||||||
|
# European voice in Piper's catalogue — the other five Portuguese models are
|
||||||
|
# all pt_BR — so the default anyone reaches for is the Brazilian one, exactly
|
||||||
|
# as it was with the Hunspell dictionary in Phase 21. Named here rather than
|
||||||
|
# left to the image default for that reason.
|
||||||
|
piper-pt:
|
||||||
|
build:
|
||||||
|
context: deploy/piper
|
||||||
|
image: petal-piper:local
|
||||||
|
container_name: petal-piper-pt
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
PIPER_VOICE: ${TTS_VOICE_PT:-pt_PT-tugão-medium}
|
||||||
|
volumes:
|
||||||
|
- piper-voices:/voices
|
||||||
|
networks:
|
||||||
|
- internal
|
||||||
|
|
||||||
|
# French, for the fr pair. The opposite situation to Portuguese: every French
|
||||||
|
# voice Piper ships is fr_FR, so there is no wrong country to land on by
|
||||||
|
# default, and the name is plain ASCII so the entrypoint's percent-encoded
|
||||||
|
# fallback (added for tugão) never has to fire. siwis-medium to match the
|
||||||
|
# register of the other three.
|
||||||
|
piper-fr:
|
||||||
|
build:
|
||||||
|
context: deploy/piper
|
||||||
|
image: petal-piper:local
|
||||||
|
container_name: petal-piper-fr
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
PIPER_VOICE: ${TTS_VOICE_FR:-fr_FR-siwis-medium}
|
||||||
|
volumes:
|
||||||
|
- piper-voices:/voices
|
||||||
|
networks:
|
||||||
|
- internal
|
||||||
|
|
||||||
|
# Spanish, for the es pair — and the Portuguese trap rather than the French
|
||||||
|
# one. Piper's catalogue has nine Spanish voices, six of them es_ES, and the
|
||||||
|
# obvious pick (es_ES-davefx-medium, which the build plan itself named) is
|
||||||
|
# peninsular. The es pack is written in neutral Latin American Spanish, so a
|
||||||
|
# Castilian voice would read it aloud in the accent the copy was written to
|
||||||
|
# avoid — the same wrong-country default that pt-PT hit through packaging,
|
||||||
|
# arriving here through the voice list. Only two Latin American voices exist,
|
||||||
|
# es_AR-daniela-high and es_MX; Mexican is the neutral broadcast standard and
|
||||||
|
# ald-medium matches the register of the other four. ASCII, so the
|
||||||
|
# percent-encoded download fallback added for tugão never has to fire.
|
||||||
|
piper-es:
|
||||||
|
build:
|
||||||
|
context: deploy/piper
|
||||||
|
image: petal-piper:local
|
||||||
|
container_name: petal-piper-es
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
PIPER_VOICE: ${TTS_VOICE_ES:-es_MX-ald-medium}
|
||||||
|
volumes:
|
||||||
|
- piper-voices:/voices
|
||||||
|
networks:
|
||||||
|
- internal
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
# Created and owned by the host's Traefik stack.
|
# Created and owned by the host's Traefik stack.
|
||||||
traefik:
|
traefik:
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ require (
|
|||||||
github.com/coreos/go-oidc/v3 v3.20.0
|
github.com/coreos/go-oidc/v3 v3.20.0
|
||||||
github.com/go-chi/chi/v5 v5.3.0
|
github.com/go-chi/chi/v5 v5.3.0
|
||||||
github.com/go-jose/go-jose/v4 v4.1.4
|
github.com/go-jose/go-jose/v4 v4.1.4
|
||||||
|
github.com/prosolis/dreamdict v0.0.0-20260727163219-302a39d5c768
|
||||||
golang.org/x/oauth2 v0.36.0
|
golang.org/x/oauth2 v0.36.0
|
||||||
modernc.org/sqlite v1.53.0
|
modernc.org/sqlite v1.53.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -16,6 +16,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
|||||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||||
|
github.com/prosolis/dreamdict v0.0.0-20260727163219-302a39d5c768 h1:+7NP78QlAVcXMviA23cMBtxwvYwDhADnABa3JBhbApI=
|
||||||
|
github.com/prosolis/dreamdict v0.0.0-20260727163219-302a39d5c768/go.mod h1:s4D+Q++6Qjq8X7/ZXEMStGXA6tScm4GadFo0+HF4BHY=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||||
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
||||||
|
|||||||
@@ -129,7 +129,10 @@ func (o *OIDC) Routes() chi.Router {
|
|||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
r.Get("/login", o.login)
|
r.Get("/login", o.login)
|
||||||
r.Get("/callback", o.callback)
|
r.Get("/callback", o.callback)
|
||||||
r.Get("/logout", o.logout)
|
// POST only. Signing out is a state change, and SameSite=Lax deliberately
|
||||||
|
// *does* send the session cookie on a top-level cross-site GET — so a GET
|
||||||
|
// route here means any page on the internet can sign her out mid-draft by
|
||||||
|
// linking to it, or embedding it as an image. Small harm, free to remove.
|
||||||
r.Post("/logout", o.logout)
|
r.Post("/logout", o.logout)
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
@@ -275,8 +278,8 @@ func (o *OIDC) callback(w http.ResponseWriter, r *http.Request) {
|
|||||||
// matters: clearing only the cookie leaves a token that still works if it was
|
// matters: clearing only the cookie leaves a token that still works if it was
|
||||||
// ever captured.
|
// ever captured.
|
||||||
func (o *OIDC) logout(w http.ResponseWriter, r *http.Request) {
|
func (o *OIDC) logout(w http.ResponseWriter, r *http.Request) {
|
||||||
if c, err := r.Cookie(SessionCookie); err == nil && c.Value != "" {
|
if token := SessionToken(r); token != "" {
|
||||||
if err := o.sessions.Revoke(c.Value); err != nil {
|
if err := o.sessions.Revoke(token); err != nil {
|
||||||
log.Printf("auth: revoke failed: %v", err)
|
log.Printf("auth: revoke failed: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -234,7 +234,7 @@ func TestLoginRoundTrip(t *testing.T) {
|
|||||||
|
|
||||||
// Signing out revokes server-side, not just in the browser.
|
// Signing out revokes server-side, not just in the browser.
|
||||||
out := httptest.NewRecorder()
|
out := httptest.NewRecorder()
|
||||||
flow.ServeHTTP(out, jar.attach(httptest.NewRequest(http.MethodGet, "/logout", nil)))
|
flow.ServeHTTP(out, jar.attach(httptest.NewRequest(http.MethodPost, "/logout", nil)))
|
||||||
if out.Code != http.StatusFound {
|
if out.Code != http.StatusFound {
|
||||||
t.Fatalf("logout status=%d", out.Code)
|
t.Fatalf("logout status=%d", out.Code)
|
||||||
}
|
}
|
||||||
@@ -243,6 +243,19 @@ func TestLoginRoundTrip(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Signing out is a state change, so it must not be reachable by GET: with
|
||||||
|
// SameSite=Lax the session cookie *is* sent on a top-level cross-site
|
||||||
|
// navigation, which would let any page on the internet sign her out mid-draft.
|
||||||
|
func TestLogoutRejectsGET(t *testing.T) {
|
||||||
|
_, flow, _, _ := newFlow(t, nil)
|
||||||
|
|
||||||
|
out := httptest.NewRecorder()
|
||||||
|
flow.ServeHTTP(out, httptest.NewRequest(http.MethodGet, "/logout", nil))
|
||||||
|
if out.Code != http.StatusMethodNotAllowed {
|
||||||
|
t.Fatalf("GET /logout status=%d, want 405", out.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// A callback whose state doesn't match the cookie is a forged one.
|
// A callback whose state doesn't match the cookie is a forged one.
|
||||||
func TestCallbackRejectsBadState(t *testing.T) {
|
func TestCallbackRejectsBadState(t *testing.T) {
|
||||||
idp, flow, sessions, _ := newFlow(t, nil)
|
idp, flow, sessions, _ := newFlow(t, nil)
|
||||||
|
|||||||
@@ -0,0 +1,275 @@
|
|||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
// patchMe drives UpdateMeHandler as the given user would reach it: behind the
|
||||||
|
// middleware, which is the only thing that puts an id in the context.
|
||||||
|
func patchMe(t *testing.T, users *UserStore, id, body string) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
r := httptest.NewRequest(http.MethodPatch, "/me", strings.NewReader(body))
|
||||||
|
r = r.WithContext(WithUser(r.Context(), id))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
users.UpdateMeHandler()(w, r)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetPairLang(t *testing.T) {
|
||||||
|
_, users, _ := newStores(t)
|
||||||
|
|
||||||
|
if err := users.SetPair("bob", "pt-PT", DirectionLearningEn); err != nil {
|
||||||
|
t.Fatalf("set pt-PT: %v", err)
|
||||||
|
}
|
||||||
|
if u, _ := users.Get("bob"); u.PairLang != "pt-PT" {
|
||||||
|
t.Fatalf("pair_lang = %q, want pt-PT", u.PairLang)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every pair with a langpack, not just the first one: this list and the
|
||||||
|
// frontend's PACKS are two copies of the same fact, and the day they
|
||||||
|
// disagree is the day she can pick a pair the app cannot render.
|
||||||
|
if err := users.SetPair("bob", "fr", DirectionLearningEn); err != nil {
|
||||||
|
t.Fatalf("set fr: %v", err)
|
||||||
|
}
|
||||||
|
if u, _ := users.Get("bob"); u.PairLang != "fr" {
|
||||||
|
t.Fatalf("pair_lang = %q, want fr", u.PairLang)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := users.SetPair("bob", "es", DirectionLearningEn); err != nil {
|
||||||
|
t.Fatalf("set es: %v", err)
|
||||||
|
}
|
||||||
|
if u, _ := users.Get("bob"); u.PairLang != "es" {
|
||||||
|
t.Fatalf("pair_lang = %q, want es", u.PairLang)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And back — a writer who tries a pair and doesn't like it must be able to
|
||||||
|
// return, which is the whole reason the picker exists.
|
||||||
|
if err := users.SetPair("bob", "zh", DirectionLearningEn); err != nil {
|
||||||
|
t.Fatalf("set zh: %v", err)
|
||||||
|
}
|
||||||
|
if u, _ := users.Get("bob"); u.PairLang != "zh" {
|
||||||
|
t.Fatalf("pair_lang = %q, want zh", u.PairLang)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A pair the frontend has no langpack for must not be storable. Accepting it
|
||||||
|
// would leave her looking at Chinese copy with no way back except a lucky guess.
|
||||||
|
func TestSetPairLangRejectsUnshippedPairs(t *testing.T) {
|
||||||
|
_, users, _ := newStores(t)
|
||||||
|
|
||||||
|
// The near-misses are the ones that matter, and there are two of them now.
|
||||||
|
// "pt-BR" must not be quietly served European copy and a European voice;
|
||||||
|
// "es-ES" is the same mistake pointing the other way, because the es pack is
|
||||||
|
// deliberately Latin American and reads itself aloud in a Mexican voice. A
|
||||||
|
// regional code Petal has not decided about is refused rather than rounded
|
||||||
|
// to the nearest pack it happens to have.
|
||||||
|
for _, lang := range []string{"es-ES", "pt-BR", "fr-CA", "de", "klingon", "", " "} {
|
||||||
|
if err := users.SetPair("bob", lang, DirectionLearningEn); err == nil {
|
||||||
|
t.Fatalf("stored unshipped pair %q", lang)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if u, _ := users.Get("bob"); u.PairLang != "zh" {
|
||||||
|
t.Fatalf("a refused write still moved pair_lang to %q", u.PairLang)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetPairLangUnknownUser(t *testing.T) {
|
||||||
|
_, users, _ := newStores(t)
|
||||||
|
if err := users.SetPair("nobody", "pt-PT", DirectionLearningEn); err == nil {
|
||||||
|
t.Fatal("set a pair language on an account that does not exist")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateMeHandler(t *testing.T) {
|
||||||
|
_, users, _ := newStores(t)
|
||||||
|
|
||||||
|
w := patchMe(t, users, "bob", `{"pair_lang":"pt-PT"}`)
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
// The whole user comes back, so the client can re-read the pair from the
|
||||||
|
// server instead of assuming its request took.
|
||||||
|
var got db.User
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if got.ID != "bob" || got.PairLang != "pt-PT" {
|
||||||
|
t.Fatalf("response = %+v, want bob on pt-PT", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateMeHandlerRejects(t *testing.T) {
|
||||||
|
_, users, _ := newStores(t)
|
||||||
|
|
||||||
|
for name, body := range map[string]string{
|
||||||
|
"unshipped pair": `{"pair_lang":"es-ES"}`,
|
||||||
|
"unknown direction": `{"direction":"learning_klingon"}`,
|
||||||
|
"not json": `pt-PT`,
|
||||||
|
} {
|
||||||
|
if w := patchMe(t, users, "bob", body); w.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("%s: status = %d, want 400", name, w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if u, _ := users.Get("bob"); u.PairLang != "zh" {
|
||||||
|
t.Fatalf("a rejected request still moved pair_lang to %q", u.PairLang)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A caller the middleware never resolved (or whose row is gone) is a lapsed
|
||||||
|
// session, not a bad request — the client turns 401 into the sign-in overlay.
|
||||||
|
if w := patchMe(t, users, "nobody", `{"pair_lang":"pt-PT"}`); w.Code != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("unknown user: status = %d, want 401", w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An empty body used to be a 400, back when pair_lang was the only field and a
|
||||||
|
// request that named none of it could only be a client bug. With two optional
|
||||||
|
// fields it is an ordinary PATCH that changes nothing, and it has to be: the
|
||||||
|
// picker sends one field without knowing the other, and "omitted" has to mean
|
||||||
|
// "leave it alone" for that to be safe.
|
||||||
|
func TestUpdateMeHandlerEmptyBodyChangesNothing(t *testing.T) {
|
||||||
|
_, users, _ := newStores(t)
|
||||||
|
|
||||||
|
if err := users.SetPair("bob", "zh", DirectionLearningPair); err != nil {
|
||||||
|
t.Fatalf("set up: %v", err)
|
||||||
|
}
|
||||||
|
w := patchMe(t, users, "bob", `{}`)
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
u, _ := users.Get("bob")
|
||||||
|
if u.PairLang != "zh" || u.Direction != DirectionLearningPair {
|
||||||
|
t.Fatalf("empty PATCH moved the account to %q/%q", u.PairLang, u.Direction)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The direction axis: an account can be turned around and turned back, and the
|
||||||
|
// default every existing row already carries is the one it had before the column
|
||||||
|
// existed.
|
||||||
|
func TestDirectionRoundTrip(t *testing.T) {
|
||||||
|
_, users, _ := newStores(t)
|
||||||
|
|
||||||
|
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||||
|
t.Fatalf("a fresh account starts at %q, want %q", u.Direction, DirectionLearningEn)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`)
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("turn around: status = %d (%s)", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var got db.User
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
// The response carries the direction, not just the pair — the client reads
|
||||||
|
// its whole state back from here rather than assuming the write took.
|
||||||
|
if got.Direction != DirectionLearningPair || got.PairLang != "zh" {
|
||||||
|
t.Fatalf("response = %+v, want bob learning zh", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if w := patchMe(t, users, "bob", `{"direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("turn back: status = %d (%s)", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||||
|
t.Fatalf("direction = %q after turning back", u.Direction)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The refusal this axis exists to make: a pair with no learner-side data cannot
|
||||||
|
// be learned toward, however good its langpack is. fr and es have copy, voices
|
||||||
|
// and spelling dictionaries, and no `learner` block in their packs to offer the
|
||||||
|
// choice with — so the server keeps saying no until one is written.
|
||||||
|
//
|
||||||
|
// pt-PT is deliberately no longer in this list; see TestLearnerDirectionForPtPT
|
||||||
|
// below and the argument in `learnerPairs`.
|
||||||
|
func TestLearnerDirectionRefusedForPairsWithoutData(t *testing.T) {
|
||||||
|
_, users, _ := newStores(t)
|
||||||
|
|
||||||
|
for _, lang := range []string{"fr", "es"} {
|
||||||
|
if err := users.SetPair("bob", lang, DirectionLearningEn); err != nil {
|
||||||
|
t.Fatalf("set %s: %v", lang, err)
|
||||||
|
}
|
||||||
|
w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`)
|
||||||
|
if w.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("%s: status = %d, want 400", lang, w.Code)
|
||||||
|
}
|
||||||
|
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||||
|
t.Fatalf("%s: a refused write still moved direction to %q", lang, u.Direction)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The other direction of that same rule, and the one a native English speaker
|
||||||
|
// writing Portuguese depends on.
|
||||||
|
//
|
||||||
|
// This is not only a settings toggle: `direction` is what decides which language
|
||||||
|
// Petal *explains* in (see suggestions.targetFor), so an account that cannot
|
||||||
|
// reach learning_pair gets its Portuguese annotated in Portuguese with no way to
|
||||||
|
// ask for English. Pinned in both directions — the move must take, and it must
|
||||||
|
// still be there when the account is read back.
|
||||||
|
func TestLearnerDirectionForPtPT(t *testing.T) {
|
||||||
|
_, users, _ := newStores(t)
|
||||||
|
|
||||||
|
if err := users.SetPair("bob", "pt-PT", DirectionLearningEn); err != nil {
|
||||||
|
t.Fatalf("set pt-PT: %v", err)
|
||||||
|
}
|
||||||
|
if w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`); w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d (%s), want 200", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
u, _ := users.Get("bob")
|
||||||
|
if u.Direction != DirectionLearningPair || u.PairLang != "pt-PT" {
|
||||||
|
t.Fatalf("account = %+v, want pt-PT learning_pair", u)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And it can be turned back, the same as zh.
|
||||||
|
if w := patchMe(t, users, "bob", `{"direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("turn back: status = %d (%s)", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||||
|
t.Fatalf("direction = %q after turning back", u.Direction)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The two-field combination the handler validates as one decision. An account
|
||||||
|
// already learning Chinese that asks only to change pair is asking for a state
|
||||||
|
// neither field names on its own — French with segmentation — and it must not
|
||||||
|
// arrive by leaving one field out.
|
||||||
|
func TestPairChangeCannotStrandTheLearnerDirection(t *testing.T) {
|
||||||
|
_, users, _ := newStores(t)
|
||||||
|
|
||||||
|
if err := users.SetPair("bob", "zh", DirectionLearningPair); err != nil {
|
||||||
|
t.Fatalf("set up: %v", err)
|
||||||
|
}
|
||||||
|
if w := patchMe(t, users, "bob", `{"pair_lang":"fr"}`); w.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want 400", w.Code)
|
||||||
|
}
|
||||||
|
u, _ := users.Get("bob")
|
||||||
|
if u.PairLang != "zh" || u.Direction != DirectionLearningPair {
|
||||||
|
t.Fatalf("refused write left the account at %q/%q", u.PairLang, u.Direction)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Naming both at once is how that move is actually made, and it works.
|
||||||
|
if w := patchMe(t, users, "bob", `{"pair_lang":"fr","direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("both fields: status = %d (%s)", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if u, _ := users.Get("bob"); u.PairLang != "fr" || u.Direction != DirectionLearningEn {
|
||||||
|
t.Fatalf("account = %q/%q, want fr/learning_en", u.PairLang, u.Direction)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The CHECK constraint is the last line, below the handler and below SetPair:
|
||||||
|
// a direction that reaches the column by any other route is still refused.
|
||||||
|
func TestDirectionCheckConstraint(t *testing.T) {
|
||||||
|
_, users, database := newStores(t)
|
||||||
|
if _, err := database.Exec(`UPDATE users SET direction = 'sideways' WHERE id = 'bob'`); err == nil {
|
||||||
|
t.Fatal("the users.direction CHECK accepted 'sideways'")
|
||||||
|
}
|
||||||
|
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||||
|
t.Fatalf("direction = %q after a refused UPDATE", u.Direction)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,8 +11,31 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SessionCookie is the cookie carrying the opaque session token.
|
// The cookie carrying the opaque session token, in its two spellings.
|
||||||
const SessionCookie = "petal_session"
|
//
|
||||||
|
// Over https the name takes the __Host- prefix, which is not decoration: the
|
||||||
|
// browser will only accept such a cookie if it is Secure, Path=/, and carries
|
||||||
|
// no Domain attribute — and, crucially, refuses to let any other host set it.
|
||||||
|
// Without the prefix, anything that can write cookies for a sibling name under
|
||||||
|
// parodia.dev (another service on the box, a subdomain takeover) can plant a
|
||||||
|
// session cookie in her browser that Petal will then read as hers.
|
||||||
|
//
|
||||||
|
// The prefix is impossible over plain http, because it requires Secure and a
|
||||||
|
// browser drops a Secure cookie on an insecure origin. So local development
|
||||||
|
// keeps the bare name, and the name in use follows the same `secure` flag the
|
||||||
|
// rest of the cookie does.
|
||||||
|
const (
|
||||||
|
SessionCookie = "petal_session"
|
||||||
|
HostSessionCookie = "__Host-petal_session"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sessionCookieName is the name to *write* under this scheme.
|
||||||
|
func sessionCookieName(secure bool) string {
|
||||||
|
if secure {
|
||||||
|
return HostSessionCookie
|
||||||
|
}
|
||||||
|
return SessionCookie
|
||||||
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// sessionTTL is how long a session lives without use. Thirty days, sliding:
|
// sessionTTL is how long a session lives without use. Thirty days, sliding:
|
||||||
@@ -78,11 +101,28 @@ func (s *SessionStore) Create(userID, userAgent string) (string, error) {
|
|||||||
// Resolve implements [Resolver]: it reads the session cookie, validates it, and
|
// Resolve implements [Resolver]: it reads the session cookie, validates it, and
|
||||||
// returns the user it belongs to — extending the session's life while it does.
|
// returns the user it belongs to — extending the session's life while it does.
|
||||||
func (s *SessionStore) Resolve(r *http.Request) (string, error) {
|
func (s *SessionStore) Resolve(r *http.Request) (string, error) {
|
||||||
c, err := r.Cookie(SessionCookie)
|
token := SessionToken(r)
|
||||||
if err != nil || c.Value == "" {
|
if token == "" {
|
||||||
return "", ErrNoSession
|
return "", ErrNoSession
|
||||||
}
|
}
|
||||||
return s.userFor(c.Value)
|
return s.userFor(token)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionToken pulls the raw session token out of a request, preferring the
|
||||||
|
// __Host- spelling.
|
||||||
|
//
|
||||||
|
// Both are read because a deployment that was signing people in before the
|
||||||
|
// prefix existed has browsers holding the old name; those sessions stay valid
|
||||||
|
// and quietly re-issue under the new name at the next sign-in. The prefixed one
|
||||||
|
// wins where both are present, since it is the one another host could not have
|
||||||
|
// planted.
|
||||||
|
func SessionToken(r *http.Request) string {
|
||||||
|
for _, name := range []string{HostSessionCookie, SessionCookie} {
|
||||||
|
if c, err := r.Cookie(name); err == nil && c.Value != "" {
|
||||||
|
return c.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// userFor validates a raw token and slides its expiry forward.
|
// userFor validates a raw token and slides its expiry forward.
|
||||||
@@ -146,7 +186,7 @@ func hashToken(token string) string {
|
|||||||
// browser drop the cookie and silently break local login.
|
// browser drop the cookie and silently break local login.
|
||||||
func SetSessionCookie(w http.ResponseWriter, token string, secure bool) {
|
func SetSessionCookie(w http.ResponseWriter, token string, secure bool) {
|
||||||
http.SetCookie(w, &http.Cookie{
|
http.SetCookie(w, &http.Cookie{
|
||||||
Name: SessionCookie,
|
Name: sessionCookieName(secure),
|
||||||
Value: token,
|
Value: token,
|
||||||
Path: "/",
|
Path: "/",
|
||||||
HttpOnly: true,
|
HttpOnly: true,
|
||||||
@@ -158,9 +198,18 @@ func SetSessionCookie(w http.ResponseWriter, token string, secure bool) {
|
|||||||
|
|
||||||
// ClearSessionCookie expires the session cookie in the browser. The matching
|
// ClearSessionCookie expires the session cookie in the browser. The matching
|
||||||
// server-side row must be revoked separately — that's the half that counts.
|
// server-side row must be revoked separately — that's the half that counts.
|
||||||
|
//
|
||||||
|
// Both spellings are expired, not just the one currently written: a browser
|
||||||
|
// carrying a pre-prefix cookie must not be left holding it after signing out,
|
||||||
|
// which is precisely the case where "clear the cookie" is the part the user can
|
||||||
|
// see working.
|
||||||
func ClearSessionCookie(w http.ResponseWriter, secure bool) {
|
func ClearSessionCookie(w http.ResponseWriter, secure bool) {
|
||||||
|
for _, name := range []string{HostSessionCookie, SessionCookie} {
|
||||||
|
if name == HostSessionCookie && !secure {
|
||||||
|
continue // the browser would reject a non-Secure __Host- cookie
|
||||||
|
}
|
||||||
http.SetCookie(w, &http.Cookie{
|
http.SetCookie(w, &http.Cookie{
|
||||||
Name: SessionCookie,
|
Name: name,
|
||||||
Value: "",
|
Value: "",
|
||||||
Path: "/",
|
Path: "/",
|
||||||
HttpOnly: true,
|
HttpOnly: true,
|
||||||
@@ -169,3 +218,4 @@ func ClearSessionCookie(w http.ResponseWriter, secure bool) {
|
|||||||
MaxAge: -1,
|
MaxAge: -1,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -309,3 +309,67 @@ func TestAllowlist(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Over https the cookie takes the __Host- prefix, which the browser will only
|
||||||
|
// accept from the exact host that set it — closing the door on a sibling
|
||||||
|
// service under the same registrable domain planting a session in her browser.
|
||||||
|
// Over plain http it cannot: the prefix requires Secure, and a browser drops a
|
||||||
|
// Secure cookie on an insecure origin, so local development would silently stop
|
||||||
|
// logging in.
|
||||||
|
func TestSessionCookieNamePerScheme(t *testing.T) {
|
||||||
|
secure := httptest.NewRecorder()
|
||||||
|
SetSessionCookie(secure, "tok", true)
|
||||||
|
c := secure.Result().Cookies()[0]
|
||||||
|
if c.Name != HostSessionCookie {
|
||||||
|
t.Fatalf("https cookie name=%q, want %q", c.Name, HostSessionCookie)
|
||||||
|
}
|
||||||
|
// The prefix is a promise about these three attributes; a browser rejects
|
||||||
|
// the cookie outright if any is wrong.
|
||||||
|
if !c.Secure || c.Path != "/" || c.Domain != "" {
|
||||||
|
t.Fatalf("__Host- cookie violates its own contract: %+v", c)
|
||||||
|
}
|
||||||
|
|
||||||
|
insecure := httptest.NewRecorder()
|
||||||
|
SetSessionCookie(insecure, "tok", false)
|
||||||
|
if name := insecure.Result().Cookies()[0].Name; name != SessionCookie {
|
||||||
|
t.Fatalf("http cookie name=%q, want %q", name, SessionCookie)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A browser holding a cookie issued before the prefix existed must stay signed
|
||||||
|
// in — and start using the new name at its next sign-in, not be logged out to
|
||||||
|
// get there.
|
||||||
|
func TestResolveAcceptsEitherCookieName(t *testing.T) {
|
||||||
|
store, _, _ := newStores(t)
|
||||||
|
token, err := store.Create("bob", "test-agent")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range []string{SessionCookie, HostSessionCookie} {
|
||||||
|
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||||
|
r.AddCookie(&http.Cookie{Name: name, Value: token})
|
||||||
|
got, err := store.Resolve(r)
|
||||||
|
if err != nil || got != "bob" {
|
||||||
|
t.Fatalf("%s: resolved to %q (err=%v)", name, got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Signing out must not leave the browser holding either spelling.
|
||||||
|
func TestClearSessionCookieExpiresBothNames(t *testing.T) {
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
ClearSessionCookie(rec, true)
|
||||||
|
|
||||||
|
cleared := map[string]bool{}
|
||||||
|
for _, c := range rec.Result().Cookies() {
|
||||||
|
if c.MaxAge < 0 {
|
||||||
|
cleared[c.Name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, name := range []string{SessionCookie, HostSessionCookie} {
|
||||||
|
if !cleared[name] {
|
||||||
|
t.Errorf("%s was left in the browser after signing out", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+196
-2
@@ -2,6 +2,7 @@ package auth
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -48,9 +49,9 @@ func (u *UserStore) Upsert(sub, email, displayName string) error {
|
|||||||
func (u *UserStore) Get(id string) (db.User, error) {
|
func (u *UserStore) Get(id string) (db.User, error) {
|
||||||
var user db.User
|
var user db.User
|
||||||
err := u.db.QueryRow(
|
err := u.db.QueryRow(
|
||||||
`SELECT id, email, COALESCE(display_name, ''), created_at, pair_lang
|
`SELECT id, email, COALESCE(display_name, ''), created_at, pair_lang, direction
|
||||||
FROM users WHERE id = ?`, id,
|
FROM users WHERE id = ?`, id,
|
||||||
).Scan(&user.ID, &user.Email, &user.DisplayName, &user.CreatedAt, &user.PairLang)
|
).Scan(&user.ID, &user.Email, &user.DisplayName, &user.CreatedAt, &user.PairLang, &user.Direction)
|
||||||
return user, err
|
return user, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,6 +69,199 @@ func (u *UserStore) MeHandler() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The pairs a writer may actually choose, in the order the picker offers them.
|
||||||
|
//
|
||||||
|
// This is deliberately *not* internal/llm's list of languages. That one names
|
||||||
|
// every pair the prompts know how to talk about, which is a cheap thing to add;
|
||||||
|
// this one names the pairs Petal can render itself in, which requires a langpack
|
||||||
|
// on the frontend. Accepting a code with no pack would leave her looking at
|
||||||
|
// Chinese with no way back except another guess, so the server refuses it. es
|
||||||
|
// joined on the day its pack landed, not before.
|
||||||
|
//
|
||||||
|
// These four are now every pair PairLang names on the frontend, which makes the
|
||||||
|
// two lists look redundant. They are not: the next pair will exist in the type
|
||||||
|
// and in the prompts long before it has copy, and this list is the one that
|
||||||
|
// says a writer may actually be sent there.
|
||||||
|
var shippedPairs = []string{"zh", "pt-PT", "fr", "es"}
|
||||||
|
|
||||||
|
func pairIsShipped(lang string) bool {
|
||||||
|
for _, p := range shippedPairs {
|
||||||
|
if p == lang {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// The two directions a pair can be travelled in. `DirectionLearningEn` is the
|
||||||
|
// original assumption made explicit: the writer is native in X and practising
|
||||||
|
// English. `DirectionLearningPair` is the other way round.
|
||||||
|
const (
|
||||||
|
DirectionLearningEn = "learning_en"
|
||||||
|
DirectionLearningPair = "learning_pair"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The pairs whose *learner* direction Petal can actually serve, which is a
|
||||||
|
// narrower thing than a shipped pair and narrower again than a langpack.
|
||||||
|
//
|
||||||
|
// Turning a pair around needs data no langpack carries: a way to find word
|
||||||
|
// boundaries, and a dictionary that reads from the pair language into English. A
|
||||||
|
// pair missing either would leave a writer looking at an editor that silently
|
||||||
|
// does nothing when she hovers — worse than a missing pack, which at least reads
|
||||||
|
// as a bug rather than as an absence. So the server refuses, for the same reason
|
||||||
|
// and by the same mechanism as `shippedPairs`.
|
||||||
|
//
|
||||||
|
// Chinese has both as of Phase 26 (CC-CEDICT + jieba). Portuguese turns out to
|
||||||
|
// have both as well, and the original note here — "French, Spanish and
|
||||||
|
// Portuguese have neither" — was written one phase too early to see it:
|
||||||
|
//
|
||||||
|
// - Word boundaries are spaces. The megabyte word list jieba needs is a
|
||||||
|
// property of a writing system that doesn't use them, not a debt every pair
|
||||||
|
// owes; a Latin-script pair needs nothing loaded to be segmented.
|
||||||
|
// - The dictionary arrived with dict.db, which reads pt→en as readily as
|
||||||
|
// en→pt (see lexicon.dreamProvider.reverse). The reverse lookup the hover
|
||||||
|
// and the word card need is already there and already answering.
|
||||||
|
//
|
||||||
|
// So the pair a native English speaker learning Portuguese needs is real, and
|
||||||
|
// what was actually blocking it was this list. French and Spanish clear the same
|
||||||
|
// two bars through the same dict.db; they are held back only by their packs
|
||||||
|
// carrying no `learner` copy yet (see Pack.learner), which is a translation
|
||||||
|
// question rather than a data one.
|
||||||
|
//
|
||||||
|
// This list is still expected to grow one pair at a time and never to be
|
||||||
|
// inferred: segmentation is a property of a writing system, and there is no rule
|
||||||
|
// that derives "has a word list" from a language code.
|
||||||
|
var learnerPairs = []string{"zh", "pt-PT"}
|
||||||
|
|
||||||
|
// SupportsLearnerDirection reports whether a pair can be turned around.
|
||||||
|
func SupportsLearnerDirection(lang string) bool {
|
||||||
|
for _, p := range learnerPairs {
|
||||||
|
if p == lang {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func directionIsKnown(d string) bool {
|
||||||
|
return d == DirectionLearningEn || d == DirectionLearningPair
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetPair moves an account to another (English + X) pair, in a given direction.
|
||||||
|
//
|
||||||
|
// The two are written together because they constrain each other: a direction is
|
||||||
|
// only meaningful for a pair that can be travelled in it, and validating them a
|
||||||
|
// field at a time would let a two-step change pass through a state that neither
|
||||||
|
// step is allowed to leave behind.
|
||||||
|
func (u *UserStore) SetPair(id, lang, direction string) error {
|
||||||
|
if !pairIsShipped(lang) {
|
||||||
|
return errors.New("auth: unshipped pair language " + lang)
|
||||||
|
}
|
||||||
|
if !directionIsKnown(direction) {
|
||||||
|
return errors.New("auth: unknown direction " + direction)
|
||||||
|
}
|
||||||
|
if direction == DirectionLearningPair && !SupportsLearnerDirection(lang) {
|
||||||
|
return errors.New("auth: no learner direction for " + lang)
|
||||||
|
}
|
||||||
|
res, err := u.db.Exec(
|
||||||
|
`UPDATE users SET pair_lang = ?, direction = ? WHERE id = ?`, lang, direction, id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if n, err := res.RowsAffected(); err == nil && n == 0 {
|
||||||
|
return sql.ErrNoRows
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateMeHandler changes the caller's own settings: which language Petal
|
||||||
|
// speaks alongside her English, and which of the two she is learning.
|
||||||
|
//
|
||||||
|
// It answers with the whole updated user rather than an empty 204 so the client
|
||||||
|
// has one shape to trust: /api/me and this return the same thing, and the app
|
||||||
|
// re-reads the pair from the response instead of assuming its request took.
|
||||||
|
//
|
||||||
|
// The pair language reaches further than the UI copy — it picks her Hunspell
|
||||||
|
// dictionary, her read-aloud voice, which word-lookup provider answers, and the
|
||||||
|
// language the prompts ask the model to explain in. All of those read
|
||||||
|
// `users.pair_lang` at use time, so all of them follow from this one write.
|
||||||
|
//
|
||||||
|
// Both fields are optional and each defaults to what the account already has, so
|
||||||
|
// the picker can send one without knowing the other. That matters for the
|
||||||
|
// combination this endpoint exists to prevent: a client that sent only
|
||||||
|
// `pair_lang: "fr"` while the account sat on `learning_pair` would otherwise ask
|
||||||
|
// for French-with-segmentation, which does not exist. Here it is one decision
|
||||||
|
// with one validation, and the answer carries whatever actually landed.
|
||||||
|
func (u *UserStore) UpdateMeHandler() http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var body struct {
|
||||||
|
PairLang *string `json:"pair_lang"`
|
||||||
|
Direction *string `json:"direction"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||||
|
httputil.BadRequest(w, "invalid request body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
id := UserID(r.Context())
|
||||||
|
current, err := u.Get(id)
|
||||||
|
if err != nil {
|
||||||
|
// Only a missing row means "not signed in". A dictionary-file or
|
||||||
|
// SQLite fault answered as 401 would trip the client's session
|
||||||
|
// interceptor and throw a writer out of an app she is still signed
|
||||||
|
// in to — the same distinction SetPair's error branch makes below.
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
lang, direction := current.PairLang, current.Direction
|
||||||
|
if body.PairLang != nil {
|
||||||
|
lang = strings.TrimSpace(*body.PairLang)
|
||||||
|
}
|
||||||
|
if body.Direction != nil {
|
||||||
|
direction = strings.TrimSpace(*body.Direction)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !pairIsShipped(lang) {
|
||||||
|
// Name the ones that work. A writer who lands here has picked from a
|
||||||
|
// stale client, and "not a language" tells her nothing.
|
||||||
|
httputil.BadRequest(w, "unsupported language pair — Petal speaks "+strings.Join(shippedPairs, ", "))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !directionIsKnown(direction) {
|
||||||
|
httputil.BadRequest(w, "unknown direction — expected "+DirectionLearningEn+" or "+DirectionLearningPair)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if direction == DirectionLearningPair && !SupportsLearnerDirection(lang) {
|
||||||
|
// Refused rather than quietly downgraded to learning_en. A silent
|
||||||
|
// downgrade would leave the writer looking at an editor that behaves
|
||||||
|
// like the one she just tried to leave, with nothing to read as an
|
||||||
|
// explanation — and the caller cannot tell the two outcomes apart
|
||||||
|
// without diffing the response it was given.
|
||||||
|
httputil.BadRequest(w, "Petal can only be learned toward "+strings.Join(learnerPairs, ", ")+" so far")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := u.SetPair(id, lang, direction); err != nil {
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
user, err := u.Get(id)
|
||||||
|
if err != nil {
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, user)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Allowlist decides which of Authentik's users may write in this Petal.
|
// Allowlist decides which of Authentik's users may write in this Petal.
|
||||||
// Authentik fronts several applications; being a valid user there does not mean
|
// Authentik fronts several applications; being a valid user there does not mean
|
||||||
// being a user here.
|
// being a user here.
|
||||||
|
|||||||
+132
-8
@@ -1,7 +1,10 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -12,6 +15,12 @@ type Config struct {
|
|||||||
BaseURL string
|
BaseURL string
|
||||||
DatabasePath string
|
DatabasePath string
|
||||||
ImageDir string // on-disk store for editor image uploads
|
ImageDir string // on-disk store for editor image uploads
|
||||||
|
// DictPath is DreamDict's built dict.db, read-only, sitting beside
|
||||||
|
// petal.db. It is what gives Petal French, European Portuguese and Spanish
|
||||||
|
// word lookups; without it only the embedded English/Chinese datasets
|
||||||
|
// exist, which is exactly how a laptop checkout runs. A missing file is
|
||||||
|
// therefore not an error — see lexicon.OpenDreamDict.
|
||||||
|
DictPath string
|
||||||
|
|
||||||
// LLM
|
// LLM
|
||||||
LLMBackend string // "vllm" | "ollama"
|
LLMBackend string // "vllm" | "ollama"
|
||||||
@@ -23,10 +32,16 @@ type Config struct {
|
|||||||
// TTS (read-aloud). Off unless TTSEndpoint is set — when empty, the /api/tts
|
// TTS (read-aloud). Off unless TTSEndpoint is set — when empty, the /api/tts
|
||||||
// route isn't mounted and the frontend falls back to the browser's Web Speech
|
// route isn't mounted and the frontend falls back to the browser's Web Speech
|
||||||
// API. Endpoint points at a local Piper HTTP server.
|
// API. Endpoint points at a local Piper HTTP server.
|
||||||
TTSEndpoint string // Piper instance serving the English voice
|
TTSEndpoint string // Piper instance serving the English voice; also the on/off switch
|
||||||
TTSEndpointZH string // Piper instance serving the Chinese voice; empty = zh falls back to Web Speech
|
// TTSVoices is every language Petal can read aloud, keyed by base language
|
||||||
TTSVoiceEN string // Piper voice id for English (e.g. en_US-amy-medium)
|
// tag ("en", "zh", "pt", …). Each Piper server loads exactly one model, so
|
||||||
TTSVoiceZH string // Piper voice id for Chinese (e.g. zh_CN-huayan-medium)
|
// a language *is* an instance — and the instances are discovered from the
|
||||||
|
// environment rather than named in this struct: one
|
||||||
|
// TTS_ENDPOINT_<LANG>/TTS_VOICE_<LANG> pair per language, so the fr and es
|
||||||
|
// pairs cost a compose service and two lines of .env rather than a code
|
||||||
|
// change. English keeps the unsuffixed TTS_ENDPOINT/TTS_VOICE_EN it has
|
||||||
|
// always had.
|
||||||
|
TTSVoices map[string]TTSVoice
|
||||||
// TTSPath is the path Piper serves synthesis on. Piper moved it from "/" to
|
// TTSPath is the path Piper serves synthesis on. Piper moved it from "/" to
|
||||||
// "/synthesize" in 1.6.0 with an unchanged request body, so this is a
|
// "/synthesize" in 1.6.0 with an unchanged request body, so this is a
|
||||||
// version knob, not a feature: millenia's older server keeps the default,
|
// version knob, not a feature: millenia's older server keeps the default,
|
||||||
@@ -48,6 +63,28 @@ type Config struct {
|
|||||||
// authenticates — right for a single-household instance, wrong the moment
|
// authenticates — right for a single-household instance, wrong the moment
|
||||||
// the IdP serves an audience wider than Petal's.
|
// the IdP serves an audience wider than Petal's.
|
||||||
AllowedSubs string
|
AllowedSubs string
|
||||||
|
// RequireAuth refuses to start when OIDC isn't configured, instead of
|
||||||
|
// falling back to the single local user.
|
||||||
|
//
|
||||||
|
// The fallback is the right behaviour on a laptop and a catastrophe on a
|
||||||
|
// public host: a typo in AUTHENTIK_CLIENT_SECRET turns every anonymous
|
||||||
|
// visitor into the `local` user, with full read and write over someone's
|
||||||
|
// private journals, and says so only in a log line nobody is reading. The
|
||||||
|
// Traefik basic-auth gate that used to stand behind that mistake was
|
||||||
|
// removed when Petal learned to authenticate for itself, so nothing catches
|
||||||
|
// it now.
|
||||||
|
//
|
||||||
|
// Defaulted from BASE_URL rather than declared: a Petal that knows itself by
|
||||||
|
// a real public origin has no business running open, and one on localhost
|
||||||
|
// has no business demanding an IdP. Set PETAL_REQUIRE_AUTH explicitly to
|
||||||
|
// override in either direction.
|
||||||
|
RequireAuth bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// TTSVoice is one Piper instance and the single voice it has loaded.
|
||||||
|
type TTSVoice struct {
|
||||||
|
Endpoint string
|
||||||
|
Voice string
|
||||||
}
|
}
|
||||||
|
|
||||||
// AuthEnabled reports whether real logins are configured. When false, Petal
|
// AuthEnabled reports whether real logins are configured. When false, Petal
|
||||||
@@ -58,11 +95,13 @@ func (c *Config) AuthEnabled() bool {
|
|||||||
|
|
||||||
// Load reads configuration from the environment, applying sane local-dev defaults.
|
// Load reads configuration from the environment, applying sane local-dev defaults.
|
||||||
func Load() *Config {
|
func Load() *Config {
|
||||||
|
baseURL := env("BASE_URL", "http://localhost:8080")
|
||||||
return &Config{
|
return &Config{
|
||||||
Port: env("PORT", "8080"),
|
Port: env("PORT", "8080"),
|
||||||
BaseURL: env("BASE_URL", "http://localhost:8080"),
|
BaseURL: baseURL,
|
||||||
DatabasePath: env("DATABASE_PATH", "./data/petal.db"),
|
DatabasePath: env("DATABASE_PATH", "./data/petal.db"),
|
||||||
ImageDir: env("IMAGE_DIR", "./data/images"),
|
ImageDir: env("IMAGE_DIR", "./data/images"),
|
||||||
|
DictPath: env("DICT_PATH", "./data/dict.db"),
|
||||||
|
|
||||||
LLMBackend: env("LLM_BACKEND", "vllm"),
|
LLMBackend: env("LLM_BACKEND", "vllm"),
|
||||||
LLMEndpoint: env("LLM_ENDPOINT", "http://localhost:8000"),
|
LLMEndpoint: env("LLM_ENDPOINT", "http://localhost:8000"),
|
||||||
@@ -71,9 +110,7 @@ func Load() *Config {
|
|||||||
LLMTimeout: envDuration("LLM_TIMEOUT", 30*time.Second),
|
LLMTimeout: envDuration("LLM_TIMEOUT", 30*time.Second),
|
||||||
|
|
||||||
TTSEndpoint: env("TTS_ENDPOINT", ""),
|
TTSEndpoint: env("TTS_ENDPOINT", ""),
|
||||||
TTSEndpointZH: env("TTS_ENDPOINT_ZH", ""),
|
TTSVoices: ttsVoices(os.Environ()),
|
||||||
TTSVoiceEN: env("TTS_VOICE_EN", "en_US-amy-medium"),
|
|
||||||
TTSVoiceZH: env("TTS_VOICE_ZH", "zh_CN-huayan-medium"),
|
|
||||||
TTSPath: env("TTS_PATH", "/"),
|
TTSPath: env("TTS_PATH", "/"),
|
||||||
TTSCacheDir: env("TTS_CACHE_DIR", "./data/tts"),
|
TTSCacheDir: env("TTS_CACHE_DIR", "./data/tts"),
|
||||||
TTSTimeout: envDuration("TTS_TIMEOUT", 15*time.Second),
|
TTSTimeout: envDuration("TTS_TIMEOUT", 15*time.Second),
|
||||||
@@ -83,9 +120,84 @@ func Load() *Config {
|
|||||||
AuthentikClientID: env("AUTHENTIK_CLIENT_ID", ""),
|
AuthentikClientID: env("AUTHENTIK_CLIENT_ID", ""),
|
||||||
AuthentikClientSecret: env("AUTHENTIK_CLIENT_SECRET", ""),
|
AuthentikClientSecret: env("AUTHENTIK_CLIENT_SECRET", ""),
|
||||||
AllowedSubs: env("PETAL_ALLOWED_SUBS", ""),
|
AllowedSubs: env("PETAL_ALLOWED_SUBS", ""),
|
||||||
|
RequireAuth: envBool("PETAL_REQUIRE_AUTH", !isLoopbackOrigin(baseURL)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isLoopbackOrigin reports whether a base URL names this machine — the shape a
|
||||||
|
// development checkout has, and the only shape where running without a login is
|
||||||
|
// a reasonable default. Anything else (a hostname, a public origin) is a
|
||||||
|
// deployment, however small.
|
||||||
|
func isLoopbackOrigin(baseURL string) bool {
|
||||||
|
u, err := url.Parse(strings.TrimSpace(baseURL))
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch strings.ToLower(u.Hostname()) {
|
||||||
|
case "localhost", "127.0.0.1", "::1", "":
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ttsVoices reads the Piper instances out of an environment slice (as returned
|
||||||
|
// by os.Environ) into a map keyed by base language tag.
|
||||||
|
//
|
||||||
|
// English is the unsuffixed pair, TTS_ENDPOINT + TTS_VOICE_EN, because that is
|
||||||
|
// what every deployment already sets and read-aloud has always been English
|
||||||
|
// first. Every other language is a TTS_ENDPOINT_<LANG>/TTS_VOICE_<LANG> pair,
|
||||||
|
// discovered rather than enumerated — TTS_ENDPOINT_ZH is what millenia and the
|
||||||
|
// VPS already use, and TTS_ENDPOINT_PT is all the Portuguese pair needs.
|
||||||
|
//
|
||||||
|
// <LANG> is the *base* tag: an environment variable name cannot hold the hyphen
|
||||||
|
// in "pt-PT", and the handler routes on the base tag anyway (a request for
|
||||||
|
// pt-PT, pt-BR or bare pt reaches the same instance, because there is only one
|
||||||
|
// Portuguese voice loaded). A pair is ignored unless both halves are set: half
|
||||||
|
// a configuration should read as "no voice for this language" and fall back to
|
||||||
|
// the browser, not as an instance that answers every request with an error.
|
||||||
|
func ttsVoices(environ []string) map[string]TTSVoice {
|
||||||
|
vals := make(map[string]string, len(environ))
|
||||||
|
for _, kv := range environ {
|
||||||
|
if k, v, ok := strings.Cut(kv, "="); ok {
|
||||||
|
vals[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
voices := map[string]TTSVoice{}
|
||||||
|
add := func(lang, endpoint, voice string) {
|
||||||
|
endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/")
|
||||||
|
voice = strings.TrimSpace(voice)
|
||||||
|
if endpoint == "" || voice == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
voices[lang] = TTSVoice{Endpoint: endpoint, Voice: voice}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The two languages that shipped before this was a map keep their voice
|
||||||
|
// defaults, so an existing deployment that names only the endpoints (as
|
||||||
|
// millenia's unit does) sounds exactly as it did.
|
||||||
|
voiceOr := func(key, fallback string) string {
|
||||||
|
if v := strings.TrimSpace(vals[key]); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
add("en", vals["TTS_ENDPOINT"], voiceOr("TTS_VOICE_EN", "en_US-amy-medium"))
|
||||||
|
for k, endpoint := range vals {
|
||||||
|
suffix, ok := strings.CutPrefix(k, "TTS_ENDPOINT_")
|
||||||
|
if !ok || suffix == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
voice := vals["TTS_VOICE_"+suffix]
|
||||||
|
if suffix == "ZH" {
|
||||||
|
voice = voiceOr("TTS_VOICE_ZH", "zh_CN-huayan-medium")
|
||||||
|
}
|
||||||
|
add(strings.ToLower(suffix), endpoint, voice)
|
||||||
|
}
|
||||||
|
return voices
|
||||||
|
}
|
||||||
|
|
||||||
func env(key, fallback string) string {
|
func env(key, fallback string) string {
|
||||||
if v := os.Getenv(key); v != "" {
|
if v := os.Getenv(key); v != "" {
|
||||||
return v
|
return v
|
||||||
@@ -93,6 +205,18 @@ func env(key, fallback string) string {
|
|||||||
return fallback
|
return fallback
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// envBool reads a boolean knob. Anything unparseable keeps the default rather
|
||||||
|
// than silently reading as false — a mistyped PETAL_REQUIRE_AUTH must not be the
|
||||||
|
// thing that turns the guard off.
|
||||||
|
func envBool(key string, fallback bool) bool {
|
||||||
|
if v := strings.TrimSpace(os.Getenv(key)); v != "" {
|
||||||
|
if b, err := strconv.ParseBool(v); err == nil {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
func envDuration(key string, fallback time.Duration) time.Duration {
|
func envDuration(key string, fallback time.Duration) time.Duration {
|
||||||
if v := os.Getenv(key); v != "" {
|
if v := os.Getenv(key); v != "" {
|
||||||
if d, err := time.ParseDuration(v); err == nil {
|
if d, err := time.ParseDuration(v); err == nil {
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// The Piper instances are discovered from the environment rather than named in
|
||||||
|
// code, so that a new pair costs a compose service and two .env lines. These
|
||||||
|
// assert the discovery rule, including the two shapes that already exist in the
|
||||||
|
// wild (millenia's systemd unit and the VPS compose file).
|
||||||
|
func TestTTSVoicesDiscovery(t *testing.T) {
|
||||||
|
voices := ttsVoices([]string{
|
||||||
|
"TTS_ENDPOINT=http://piper-en:5000",
|
||||||
|
"TTS_VOICE_EN=en_US-amy-medium",
|
||||||
|
"TTS_ENDPOINT_ZH=http://piper-zh:5000/",
|
||||||
|
"TTS_VOICE_ZH=zh_CN-huayan-medium",
|
||||||
|
"TTS_ENDPOINT_PT=http://piper-pt:5000",
|
||||||
|
"TTS_VOICE_PT=pt_PT-tugão-medium",
|
||||||
|
"TTS_ENDPOINT_FR=http://piper-fr:5000",
|
||||||
|
"TTS_VOICE_FR=fr_FR-siwis-medium",
|
||||||
|
"TTS_ENDPOINT_ES=http://piper-es:5000",
|
||||||
|
"TTS_VOICE_ES=es_MX-ald-medium",
|
||||||
|
// Noise that must not become a language.
|
||||||
|
"TTS_PATH=/synthesize",
|
||||||
|
"PATH=/usr/bin",
|
||||||
|
})
|
||||||
|
|
||||||
|
want := map[string]TTSVoice{
|
||||||
|
"en": {"http://piper-en:5000", "en_US-amy-medium"},
|
||||||
|
// The trailing slash is trimmed here so the synthesis path concatenates
|
||||||
|
// cleanly rather than producing a double slash at every call site.
|
||||||
|
"zh": {"http://piper-zh:5000", "zh_CN-huayan-medium"},
|
||||||
|
"pt": {"http://piper-pt:5000", "pt_PT-tugão-medium"},
|
||||||
|
// Phase 24's whole TTS change: a fourth language costs two lines here
|
||||||
|
// and a compose service, and no Go at all.
|
||||||
|
"fr": {"http://piper-fr:5000", "fr_FR-siwis-medium"},
|
||||||
|
// And a fifth cost exactly the same, which is the claim actually being
|
||||||
|
// tested. The voice is Mexican on purpose: the es pack is Latin
|
||||||
|
// American, and es_ES-davefx-medium would read it in the wrong accent.
|
||||||
|
"es": {"http://piper-es:5000", "es_MX-ald-medium"},
|
||||||
|
}
|
||||||
|
if len(voices) != len(want) {
|
||||||
|
t.Fatalf("discovered %v, want %v", voices, want)
|
||||||
|
}
|
||||||
|
for lang, w := range want {
|
||||||
|
if voices[lang] != w {
|
||||||
|
t.Errorf("%s = %+v, want %+v", lang, voices[lang], w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Half a configuration is not a language. An endpoint with no voice (or the
|
||||||
|
// reverse) must read as "no voice for this language" — a 404 the client answers
|
||||||
|
// by falling back to Web Speech — rather than as an instance that exists and
|
||||||
|
// errors on every request.
|
||||||
|
func TestTTSVoicesIgnoresHalfConfiguredLanguages(t *testing.T) {
|
||||||
|
voices := ttsVoices([]string{
|
||||||
|
"TTS_ENDPOINT=http://piper-en:5000",
|
||||||
|
"TTS_VOICE_EN=en_US-amy-medium",
|
||||||
|
"TTS_ENDPOINT_FR=http://piper-fr:5000", // no TTS_VOICE_FR
|
||||||
|
"TTS_VOICE_ES=es_ES-davefx-medium", // no TTS_ENDPOINT_ES
|
||||||
|
})
|
||||||
|
if _, ok := voices["fr"]; ok {
|
||||||
|
t.Errorf("fr routed with no voice configured")
|
||||||
|
}
|
||||||
|
if _, ok := voices["es"]; ok {
|
||||||
|
t.Errorf("es routed with no endpoint configured")
|
||||||
|
}
|
||||||
|
if len(voices) != 1 {
|
||||||
|
t.Errorf("discovered %v, want English only", voices)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A deployment that predates the map names only the endpoints and relies on the
|
||||||
|
// voice defaults; it must sound exactly as it did.
|
||||||
|
func TestTTSVoicesKeepsTheOriginalDefaults(t *testing.T) {
|
||||||
|
voices := ttsVoices([]string{
|
||||||
|
"TTS_ENDPOINT=http://127.0.0.1:5005",
|
||||||
|
"TTS_ENDPOINT_ZH=http://127.0.0.1:5006",
|
||||||
|
})
|
||||||
|
if got := voices["en"].Voice; got != "en_US-amy-medium" {
|
||||||
|
t.Errorf("en voice = %q, want the default", got)
|
||||||
|
}
|
||||||
|
if got := voices["zh"].Voice; got != "zh_CN-huayan-medium" {
|
||||||
|
t.Errorf("zh voice = %q, want the default", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read-aloud is off when no English instance is configured; nothing else may
|
||||||
|
// switch it on. (tts.New gates on TTSEndpoint, so a stray TTS_ENDPOINT_PT with
|
||||||
|
// no English sibling must not produce a routable map that outlives that gate.)
|
||||||
|
func TestTTSVoicesEmptyWithoutEndpoints(t *testing.T) {
|
||||||
|
if voices := ttsVoices([]string{"TTS_VOICE_EN=en_US-amy-medium"}); len(voices) != 0 {
|
||||||
|
t.Errorf("discovered %v, want none", voices)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The fallback to the single local user is right on a laptop and a catastrophe
|
||||||
|
// on a public host, so it is defaulted from the origin Petal knows itself by
|
||||||
|
// rather than left to be remembered.
|
||||||
|
func TestRequireAuthDefaultsFromBaseURL(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
baseURL string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"http://localhost:8080", false},
|
||||||
|
{"http://127.0.0.1:8080", false},
|
||||||
|
{"http://[::1]:8080", false},
|
||||||
|
{"", false}, // no BASE_URL set at all: the local-dev default
|
||||||
|
{"https://petal.parodia.dev", true},
|
||||||
|
{"http://petal.parodia.dev", true},
|
||||||
|
{"https://petal.example.com/", true},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Setenv("BASE_URL", c.baseURL)
|
||||||
|
t.Setenv("PETAL_REQUIRE_AUTH", "")
|
||||||
|
if got := Load().RequireAuth; got != c.want {
|
||||||
|
t.Errorf("BASE_URL=%q: RequireAuth=%v, want %v", c.baseURL, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The default is a default, not a rule: a trusted private network is a real
|
||||||
|
// deployment shape, and so is wanting the guard on locally.
|
||||||
|
func TestRequireAuthExplicitOverride(t *testing.T) {
|
||||||
|
t.Setenv("BASE_URL", "https://petal.parodia.dev")
|
||||||
|
t.Setenv("PETAL_REQUIRE_AUTH", "false")
|
||||||
|
if Load().RequireAuth {
|
||||||
|
t.Error("an explicit false must be honoured on a public origin")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv("BASE_URL", "http://localhost:8080")
|
||||||
|
t.Setenv("PETAL_REQUIRE_AUTH", "true")
|
||||||
|
if !Load().RequireAuth {
|
||||||
|
t.Error("an explicit true must be honoured on localhost")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A typo must not be the thing that disables the guard.
|
||||||
|
t.Setenv("BASE_URL", "https://petal.parodia.dev")
|
||||||
|
t.Setenv("PETAL_REQUIRE_AUTH", "nope")
|
||||||
|
if !Load().RequireAuth {
|
||||||
|
t.Error("an unparseable value must keep the default, not read as false")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -459,6 +459,185 @@ CREATE TABLE personal_words (
|
|||||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||||
PRIMARY KEY (user_id, lang, word)
|
PRIMARY KEY (user_id, lang, word)
|
||||||
);
|
);
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// The growth journal reads the suggestions table as a record of what the
|
||||||
|
// writer has been learning, and that reading only works if a row is dated
|
||||||
|
// by *her decision* rather than by the model's proposal. `created_at` is
|
||||||
|
// when a checkpoint offered the edit; a suggestion offered in April and
|
||||||
|
// accepted in June is June's growth, not April's.
|
||||||
|
//
|
||||||
|
// Existing rows are backfilled to created_at — which is exactly the
|
||||||
|
// approximation the journal would have had to make anyway, and is very
|
||||||
|
// nearly right in practice since edits are settled minutes after a
|
||||||
|
// checkpoint. Only pending rows keep a NULL: nothing has been decided.
|
||||||
|
name: "0012_suggestion_resolved_at",
|
||||||
|
stmt: `
|
||||||
|
ALTER TABLE suggestions ADD COLUMN resolved_at DATETIME;
|
||||||
|
UPDATE suggestions SET resolved_at = created_at WHERE status != 'pending';
|
||||||
|
CREATE INDEX idx_suggestions_resolved ON suggestions(status, resolved_at);
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Which engine proposed a row. Until now `type` doubled as that answer —
|
||||||
|
// 'mechanics' meant "the offline rule pack found this" and everything else
|
||||||
|
// meant "the model did". That breaks the moment an offline rule proposes a
|
||||||
|
// *collocation*: the miscollocation list (SUGGESTIONS §6) is the same
|
||||||
|
// family, the same rail and the same warm phrasing as the LLM coach, and it
|
||||||
|
// must stay type='collocation' so an accepted chunk still plants in the
|
||||||
|
// garden and still counts in the journal. With type no longer naming the
|
||||||
|
// engine, the two passes could not scope their own DELETEs — the coach
|
||||||
|
// would wipe the offline flags, and the offline pass would leave the
|
||||||
|
// coach's behind to accumulate.
|
||||||
|
//
|
||||||
|
// Existing mechanics rows are local by definition; everything else came
|
||||||
|
// from a model.
|
||||||
|
name: "0013_suggestion_source",
|
||||||
|
stmt: `
|
||||||
|
ALTER TABLE suggestions ADD COLUMN source TEXT NOT NULL DEFAULT 'llm';
|
||||||
|
UPDATE suggestions SET source = 'local' WHERE type = 'mechanics';
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Sentence-level identity, so a re-check stops regenerating the world.
|
||||||
|
// Every pass used to delete its whole family and re-insert it, which
|
||||||
|
// meant accepting one edit gave every other card a new id and a newly
|
||||||
|
// worded explanation — the rail visibly emptied and refilled, and the
|
||||||
|
// model was asked again about sentences nobody had touched.
|
||||||
|
//
|
||||||
|
// `chunk_hash` records which sentence a suggestion belongs to, and
|
||||||
|
// checked_chunks records which sentences a family has already read. A
|
||||||
|
// re-check then asks only about the difference and keeps the rest of
|
||||||
|
// the rows exactly as they are, id and wording included.
|
||||||
|
//
|
||||||
|
// Existing rows get '' — "sentence unknown", which reads as in-play, so
|
||||||
|
// they are simply reconciled on the next pass like any fresh finding.
|
||||||
|
name: "0014_suggestion_chunk_hash",
|
||||||
|
stmt: `
|
||||||
|
ALTER TABLE suggestions ADD COLUMN chunk_hash TEXT NOT NULL DEFAULT '';
|
||||||
|
|
||||||
|
CREATE TABLE checked_chunks (
|
||||||
|
doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||||
|
family TEXT NOT NULL,
|
||||||
|
hash TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (doc_id, family, hash)
|
||||||
|
);
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// A sentence she wrote in her own language gets its own type. Petal already
|
||||||
|
// detected such spans and already rendered them into English — it just
|
||||||
|
// filed the result under 'clarity', so the pair model's flagship moment
|
||||||
|
// read as tidying up her Chinese. As with 0005 and 0008, the `type` CHECK
|
||||||
|
// can't be ALTERed in place, so rebuild the table with the extended
|
||||||
|
// constraint, copy every row across, and recreate both indexes.
|
||||||
|
//
|
||||||
|
// Existing rows are left on whatever type they have. A card she is already
|
||||||
|
// reading keeps the label she has already read (the same rule reconcile.go
|
||||||
|
// follows for a re-proposed edit); new findings get the new label.
|
||||||
|
name: "0015_translate_suggestion_type",
|
||||||
|
stmt: `
|
||||||
|
CREATE TABLE suggestions_new (
|
||||||
|
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
||||||
|
doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||||
|
from_pos INTEGER NOT NULL,
|
||||||
|
to_pos INTEGER NOT NULL,
|
||||||
|
original TEXT NOT NULL,
|
||||||
|
replacement TEXT NOT NULL,
|
||||||
|
explanation TEXT NOT NULL,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('grammar','phrasing','idiom','clarity','translate','voice','collocation','mechanics')),
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','accepted','rejected')),
|
||||||
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
resolved_at DATETIME,
|
||||||
|
source TEXT NOT NULL DEFAULT 'llm',
|
||||||
|
chunk_hash TEXT NOT NULL DEFAULT ''
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO suggestions_new (id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash)
|
||||||
|
SELECT id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash FROM suggestions;
|
||||||
|
|
||||||
|
DROP TABLE suggestions;
|
||||||
|
ALTER TABLE suggestions_new RENAME TO suggestions;
|
||||||
|
CREATE INDEX idx_suggestions_doc_id ON suggestions(doc_id);
|
||||||
|
CREATE INDEX idx_suggestions_resolved ON suggestions(status, resolved_at);
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Which half of the pair is being learned.
|
||||||
|
//
|
||||||
|
// `pair_lang` (0010) has always answered "which two languages", and every
|
||||||
|
// surface built on it assumed the answer to a second question nobody had
|
||||||
|
// asked: that English is the language being *learned*. That assumption is
|
||||||
|
// load-bearing in a dozen places — CJK is deliberately never tokenized,
|
||||||
|
// never spell-checked, never glossed; the prompts explain English in her
|
||||||
|
// language; the vocabulary garden captures English words. All correct for
|
||||||
|
// a Mandarin native practising English, and all backwards for an English
|
||||||
|
// native practising Mandarin.
|
||||||
|
//
|
||||||
|
// A second pair code ('zh-learner') was the cheaper option and is the
|
||||||
|
// wrong shape: it would make the two directions of one pair look like two
|
||||||
|
// unrelated languages to every query, and it would have to be repeated for
|
||||||
|
// fr, es and pt-PT before any of them could turn around. A column keeps
|
||||||
|
// the two questions separate, which is what they are.
|
||||||
|
//
|
||||||
|
// 'learning_en' is the default and is what every existing row means — the
|
||||||
|
// backfill is the DEFAULT itself, and it is right rather than merely
|
||||||
|
// convenient: all three accounts today are Mandarin natives writing
|
||||||
|
// English.
|
||||||
|
name: "0016_user_direction",
|
||||||
|
stmt: `
|
||||||
|
ALTER TABLE users ADD COLUMN direction TEXT NOT NULL DEFAULT 'learning_en'
|
||||||
|
CHECK(direction IN ('learning_en','learning_pair'));
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Which language this document is written in — 'en' or 'pair'.
|
||||||
|
//
|
||||||
|
// It is stored, rather than recomputed per pass and forgotten, for one
|
||||||
|
// reason: the verdict has hysteresis (see suggestions/doclang.go). A
|
||||||
|
// bilingual document sits between the two thresholds, and "whatever it
|
||||||
|
// was last time" is only an answer if last time was written down. Without
|
||||||
|
// the column a mixed paragraph would alternate its cards' language
|
||||||
|
// between passes.
|
||||||
|
//
|
||||||
|
// 'pair' rather than a language code, deliberately. Which language "pair"
|
||||||
|
// names is the owner's users.pair_lang, so changing her pair re-reads her
|
||||||
|
// documents instead of stranding a stale language name on every one of
|
||||||
|
// them.
|
||||||
|
//
|
||||||
|
// Empty is the backfill and means English: every document that exists
|
||||||
|
// today was written by a Mandarin native practising English, and English
|
||||||
|
// is what every surface assumed before this phase.
|
||||||
|
name: "0017_document_lang",
|
||||||
|
stmt: `
|
||||||
|
ALTER TABLE documents ADD COLUMN doc_lang TEXT NOT NULL DEFAULT ''
|
||||||
|
CHECK(doc_lang IN ('', 'en', 'pair'));
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Which language a garden card is in — the same '' | 'en' | 'pair'
|
||||||
|
// vocabulary as documents.doc_lang, and set from it: a word is captured
|
||||||
|
// (or a phrase planted) out of a document, so the document's verdict is
|
||||||
|
// the card's language. A card with no document keeps '', which reads as
|
||||||
|
// English like every other empty here.
|
||||||
|
//
|
||||||
|
// The garden needed this the moment a document could be written in her
|
||||||
|
// own language. Before Phase 28 every card was English by construction;
|
||||||
|
// now a Portuguese lookup lands beside an English one with nothing to
|
||||||
|
// tell them apart, and two surfaces get it wrong without the tag — the
|
||||||
|
// review card's read-aloud (which would say a Portuguese word in a US
|
||||||
|
// English voice) and the panel, where a mixed garden is illegible.
|
||||||
|
//
|
||||||
|
// Every card is reviewed regardless. Filtering the queue to the half she
|
||||||
|
// is learning was the alternative and is wrong for the writer this is
|
||||||
|
// for: the words she met while writing Portuguese are still words she
|
||||||
|
// met, and a garden that quietly drops them is a garden that stops being
|
||||||
|
// a record of her reading.
|
||||||
|
name: "0018_vocab_lang",
|
||||||
|
stmt: `
|
||||||
|
ALTER TABLE vocab_words ADD COLUMN lang TEXT NOT NULL DEFAULT ''
|
||||||
|
CHECK(lang IN ('', 'en', 'pair'));
|
||||||
`,
|
`,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package db
|
|||||||
import (
|
import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestOpenMigratesAndSeeds(t *testing.T) {
|
func TestOpenMigratesAndSeeds(t *testing.T) {
|
||||||
@@ -83,3 +84,283 @@ func TestOpenMigratesAndSeeds(t *testing.T) {
|
|||||||
t.Errorf("expected exactly 1 local user after reopen, got %d", users)
|
t.Errorf("expected exactly 1 local user after reopen, got %d", users)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestResolvedAtBackfill runs migration 0012 against a database that predates
|
||||||
|
// it, which is the only shape that matters: on the live box the suggestions
|
||||||
|
// table is years of settled edits with no resolved_at to their name. Backfilling
|
||||||
|
// to created_at is exactly the approximation the growth journal would otherwise
|
||||||
|
// have had to make, and a pending row must stay NULL — nothing has been decided.
|
||||||
|
func TestResolvedAtBackfill(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "old.db")
|
||||||
|
d, err := Open(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rewind to the state before 0012: drop the column and forget the migration.
|
||||||
|
if _, err := d.Exec(`DROP INDEX idx_suggestions_resolved`); err != nil {
|
||||||
|
t.Fatalf("rewind index: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := d.Exec(`ALTER TABLE suggestions DROP COLUMN resolved_at`); err != nil {
|
||||||
|
t.Fatalf("rewind schema: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := d.Exec(`DELETE FROM schema_migrations WHERE name = '0012_suggestion_resolved_at'`); err != nil {
|
||||||
|
t.Fatalf("rewind migration record: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := d.Exec(`INSERT INTO documents (id, user_id) VALUES ('d1', ?)`, LocalUserID); err != nil {
|
||||||
|
t.Fatalf("insert document: %v", err)
|
||||||
|
}
|
||||||
|
for _, s := range []struct{ id, status string }{
|
||||||
|
{"s-old", "accepted"},
|
||||||
|
{"s-open", "pending"},
|
||||||
|
} {
|
||||||
|
if _, err := d.Exec(
|
||||||
|
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at)
|
||||||
|
VALUES (?, 'd1', 0, 3, 'teh', 'the', 'x', 'grammar', ?, '2026-01-02 03:04:05')`,
|
||||||
|
s.id, s.status,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("seed %s: %v", s.id, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d.Close()
|
||||||
|
|
||||||
|
d2, err := Open(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reopen (migrate): %v", err)
|
||||||
|
}
|
||||||
|
defer d2.Close()
|
||||||
|
|
||||||
|
// Compared against created_at read back the same way: the driver renders a
|
||||||
|
// DATETIME column itself, so the assertion is "the same instant", not a
|
||||||
|
// particular text format.
|
||||||
|
var settled, created *string
|
||||||
|
if err := d2.QueryRow(
|
||||||
|
`SELECT resolved_at, created_at FROM suggestions WHERE id = 's-old'`,
|
||||||
|
).Scan(&settled, &created); err != nil {
|
||||||
|
t.Fatalf("read settled row: %v", err)
|
||||||
|
}
|
||||||
|
if settled == nil || created == nil || *settled != *created {
|
||||||
|
t.Errorf("resolved_at = %v, want it backfilled from created_at (%v)", settled, created)
|
||||||
|
}
|
||||||
|
|
||||||
|
var pending *string
|
||||||
|
if err := d2.QueryRow(`SELECT resolved_at FROM suggestions WHERE id = 's-open'`).Scan(&pending); err != nil {
|
||||||
|
t.Fatalf("read pending row: %v", err)
|
||||||
|
}
|
||||||
|
if pending != nil {
|
||||||
|
t.Errorf("pending row got resolved_at = %v, want NULL — nothing was decided", *pending)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSuggestionSourceBackfill runs migration 0013 against a database that
|
||||||
|
// predates it — the shape the live box is actually in. `source` is the column
|
||||||
|
// that lets the offline rule pack and the model share the collocation family
|
||||||
|
// without deleting each other's rows, and it can only do that if the existing
|
||||||
|
// rows are labelled correctly on the way in: everything the old deterministic
|
||||||
|
// pass wrote is local, and everything else came from a model.
|
||||||
|
func TestSuggestionSourceBackfill(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "old.db")
|
||||||
|
d, err := Open(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rewind to the state before 0013.
|
||||||
|
if _, err := d.Exec(`ALTER TABLE suggestions DROP COLUMN source`); err != nil {
|
||||||
|
t.Fatalf("rewind schema: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := d.Exec(`DELETE FROM schema_migrations WHERE name = '0013_suggestion_source'`); err != nil {
|
||||||
|
t.Fatalf("rewind migration record: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := d.Exec(`INSERT INTO documents (id, user_id) VALUES ('d1', ?)`, LocalUserID); err != nil {
|
||||||
|
t.Fatalf("insert document: %v", err)
|
||||||
|
}
|
||||||
|
for _, s := range []struct{ id, typ string }{
|
||||||
|
{"s-mech", SuggestionTypeMechanics},
|
||||||
|
{"s-gram", SuggestionTypeGrammar},
|
||||||
|
{"s-coll", SuggestionTypeCollocation},
|
||||||
|
} {
|
||||||
|
if _, err := d.Exec(
|
||||||
|
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||||
|
VALUES (?, 'd1', 0, 3, 'teh', 'the', 'x', ?)`,
|
||||||
|
s.id, s.typ,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("seed %s: %v", s.id, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d.Close()
|
||||||
|
|
||||||
|
d2, err := Open(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reopen (migrate): %v", err)
|
||||||
|
}
|
||||||
|
defer d2.Close()
|
||||||
|
|
||||||
|
// A pre-0013 collocation row can only have come from the coach — the offline
|
||||||
|
// miscollocation list did not exist yet — so it must NOT be claimed as local.
|
||||||
|
for id, want := range map[string]string{
|
||||||
|
"s-mech": SuggestionSourceLocal,
|
||||||
|
"s-gram": SuggestionSourceLLM,
|
||||||
|
"s-coll": SuggestionSourceLLM,
|
||||||
|
} {
|
||||||
|
var got string
|
||||||
|
if err := d2.QueryRow(`SELECT source FROM suggestions WHERE id = ?`, id).Scan(&got); err != nil {
|
||||||
|
t.Fatalf("read %s: %v", id, err)
|
||||||
|
}
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("%s: source = %q, want %q", id, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And a row written after the migration defaults to the model, so a code path
|
||||||
|
// that forgets to name a source can never silently claim to be offline.
|
||||||
|
if _, err := d2.Exec(
|
||||||
|
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||||
|
VALUES ('s-new', 'd1', 0, 3, 'teh', 'the', 'x', 'grammar')`,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("insert new row: %v", err)
|
||||||
|
}
|
||||||
|
var fresh string
|
||||||
|
if err := d2.QueryRow(`SELECT source FROM suggestions WHERE id = 's-new'`).Scan(&fresh); err != nil {
|
||||||
|
t.Fatalf("read new row: %v", err)
|
||||||
|
}
|
||||||
|
if fresh != SuggestionSourceLLM {
|
||||||
|
t.Errorf("default source = %q, want %q", fresh, SuggestionSourceLLM)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTranslateTypeMigrationPreservesRows runs migration 0015 against a database
|
||||||
|
// that predates it. Unlike the two backfills above, 0015 *rebuilds the table* —
|
||||||
|
// SQLite can't ALTER a CHECK constraint — so it copies every row across by hand,
|
||||||
|
// and a column left out of that copy list silently loses her data. Every test
|
||||||
|
// elsewhere starts from a fresh database and would never notice; the live box has
|
||||||
|
// years of rows in it.
|
||||||
|
func TestTranslateTypeMigrationPreservesRows(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "old.db")
|
||||||
|
d, err := Open(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rewind to the pre-0015 table: the same shape, minus 'translate' in the CHECK.
|
||||||
|
if _, err := d.Exec(`
|
||||||
|
CREATE TABLE suggestions_old (
|
||||||
|
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
||||||
|
doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||||
|
from_pos INTEGER NOT NULL,
|
||||||
|
to_pos INTEGER NOT NULL,
|
||||||
|
original TEXT NOT NULL,
|
||||||
|
replacement TEXT NOT NULL,
|
||||||
|
explanation TEXT NOT NULL,
|
||||||
|
type TEXT NOT NULL CHECK(type IN ('grammar','phrasing','idiom','clarity','voice','collocation','mechanics')),
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','accepted','rejected')),
|
||||||
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
resolved_at DATETIME,
|
||||||
|
source TEXT NOT NULL DEFAULT 'llm',
|
||||||
|
chunk_hash TEXT NOT NULL DEFAULT ''
|
||||||
|
);
|
||||||
|
DROP TABLE suggestions;
|
||||||
|
ALTER TABLE suggestions_old RENAME TO suggestions;
|
||||||
|
CREATE INDEX idx_suggestions_doc_id ON suggestions(doc_id);
|
||||||
|
CREATE INDEX idx_suggestions_resolved ON suggestions(status, resolved_at);
|
||||||
|
DELETE FROM schema_migrations WHERE name = '0015_translate_suggestion_type';
|
||||||
|
`); err != nil {
|
||||||
|
t.Fatalf("rewind schema: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := d.Exec(`INSERT INTO documents (id, user_id) VALUES ('d1', ?)`, LocalUserID); err != nil {
|
||||||
|
t.Fatalf("insert document: %v", err)
|
||||||
|
}
|
||||||
|
// One row with every column carrying a distinguishable value, so a dropped
|
||||||
|
// column shows up as a changed value rather than as a passing test.
|
||||||
|
if _, err := d.Exec(
|
||||||
|
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash)
|
||||||
|
VALUES ('s-1', 'd1', 7, 11, 'by foots', 'on foot', 'idiom advice she has read', 'idiom', 'accepted', '2026-01-02 03:04:05', '2026-01-02 03:05:00', 'local', 'abc123')`,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("seed row: %v", err)
|
||||||
|
}
|
||||||
|
d.Close()
|
||||||
|
|
||||||
|
d2, err := Open(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reopen (migrate): %v", err)
|
||||||
|
}
|
||||||
|
defer d2.Close()
|
||||||
|
|
||||||
|
var (
|
||||||
|
docID, original, replacement, explanation string
|
||||||
|
typ, status, source, chunkHash string
|
||||||
|
from, to int
|
||||||
|
// Scanned as instants, not strings: the driver renders a DATETIME column in
|
||||||
|
// its own format, so the claim is "the same moment", not the same text.
|
||||||
|
createdAt, resolvedAt time.Time
|
||||||
|
)
|
||||||
|
if err := d2.QueryRow(
|
||||||
|
`SELECT doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash
|
||||||
|
FROM suggestions WHERE id = 's-1'`,
|
||||||
|
).Scan(&docID, &from, &to, &original, &replacement, &explanation,
|
||||||
|
&typ, &status, &createdAt, &resolvedAt, &source, &chunkHash); err != nil {
|
||||||
|
t.Fatalf("read migrated row: %v", err)
|
||||||
|
}
|
||||||
|
for _, c := range []struct{ name, got, want string }{
|
||||||
|
{"doc_id", docID, "d1"},
|
||||||
|
{"original", original, "by foots"},
|
||||||
|
{"replacement", replacement, "on foot"},
|
||||||
|
{"explanation", explanation, "idiom advice she has read"},
|
||||||
|
{"type", typ, SuggestionTypeIdiom},
|
||||||
|
{"status", status, SuggestionStatusAccepted},
|
||||||
|
{"source", source, SuggestionSourceLocal},
|
||||||
|
{"chunk_hash", chunkHash, "abc123"},
|
||||||
|
} {
|
||||||
|
if c.got != c.want {
|
||||||
|
t.Errorf("%s = %q, want %q", c.name, c.got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if from != 7 || to != 11 {
|
||||||
|
t.Errorf("offsets = (%d, %d), want (7, 11)", from, to)
|
||||||
|
}
|
||||||
|
// created_at and resolved_at must survive: the rail's arrival chime keys on
|
||||||
|
// created_at, and the growth journal counts by resolved_at. A rebuild that
|
||||||
|
// reset either would re-chime her whole document and rewrite her history.
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
got time.Time
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"created_at", createdAt, "2026-01-02 03:04:05"},
|
||||||
|
{"resolved_at", resolvedAt, "2026-01-02 03:05:00"},
|
||||||
|
} {
|
||||||
|
want, err := time.Parse("2006-01-02 15:04:05", c.want)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse want: %v", err)
|
||||||
|
}
|
||||||
|
if !c.got.Equal(want) {
|
||||||
|
t.Errorf("%s = %v, want the original instant %v", c.name, c.got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The point of the rebuild: the new type is now insertable, and a bogus one
|
||||||
|
// still isn't.
|
||||||
|
if _, err := d2.Exec(
|
||||||
|
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||||
|
VALUES ('s-2', 'd1', 0, 3, '苹果', 'apple', 'x', ?)`, SuggestionTypeTranslate,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("insert translate row: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := d2.Exec(
|
||||||
|
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||||
|
VALUES ('s-3', 'd1', 0, 3, 'x', 'y', 'x', 'nonsense')`,
|
||||||
|
); err == nil {
|
||||||
|
t.Error("CHECK constraint accepted an unknown type after the rebuild")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Both indexes must come back, or every document load starts table-scanning.
|
||||||
|
for _, idx := range []string{"idx_suggestions_doc_id", "idx_suggestions_resolved"} {
|
||||||
|
var name string
|
||||||
|
if err := d2.QueryRow(
|
||||||
|
`SELECT name FROM sqlite_master WHERE type = 'index' AND name = ?`, idx,
|
||||||
|
).Scan(&name); err != nil {
|
||||||
|
t.Errorf("index %s missing after rebuild: %v", idx, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+36
-1
@@ -15,6 +15,19 @@ type User struct {
|
|||||||
// today, "pt-PT"/"fr"/"es" once the langpacks land. It selects the UI copy
|
// today, "pt-PT"/"fr"/"es" once the langpacks land. It selects the UI copy
|
||||||
// and dictionary set, not the language they may type in.
|
// and dictionary set, not the language they may type in.
|
||||||
PairLang string `json:"pair_lang"`
|
PairLang string `json:"pair_lang"`
|
||||||
|
|
||||||
|
// Direction says which half of the pair is being *learned*. Every pair until
|
||||||
|
// now assumed one answer: the writer is native in X and practising English,
|
||||||
|
// so hanzi is never tokenized and English is what gets underlined. Turn it
|
||||||
|
// around — a native English speaker learning Chinese — and the same pair
|
||||||
|
// wants the opposite of nearly every default.
|
||||||
|
//
|
||||||
|
// It is a separate column from PairLang rather than a second pair code
|
||||||
|
// ("zh-learner") because it is a genuinely separate question: the pair says
|
||||||
|
// *which two languages*, this says *which way round*. Keeping them apart is
|
||||||
|
// what lets fr, es and pt-PT inherit the learner direction later without a
|
||||||
|
// second langpack each.
|
||||||
|
Direction string `json:"direction"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Document is a single piece of writing. `Content` is the Tiptap JSON document
|
// Document is a single piece of writing. `Content` is the Tiptap JSON document
|
||||||
@@ -28,6 +41,11 @@ type Document struct {
|
|||||||
ContentText string `json:"content_text"` // plain text for the LLM
|
ContentText string `json:"content_text"` // plain text for the LLM
|
||||||
Tone string `json:"tone"` // target writing tone; steers LLM advice
|
Tone string `json:"tone"` // target writing tone; steers LLM advice
|
||||||
WordCount int `json:"word_count"`
|
WordCount int `json:"word_count"`
|
||||||
|
// DocLang is which language this document is written in — '' | 'en' | 'pair'
|
||||||
|
// (migration 0017), written by the checkpoint pass and never by the client.
|
||||||
|
// It reaches the client read-only, for the one decision the client has to
|
||||||
|
// make on its own: which voice reads a selection aloud. '' means English.
|
||||||
|
DocLang string `json:"doc_lang"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
UpdatedAt time.Time `json:"updated_at"`
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
|
||||||
@@ -104,8 +122,12 @@ type Suggestion struct {
|
|||||||
Original string `json:"original"`
|
Original string `json:"original"`
|
||||||
Replacement string `json:"replacement"`
|
Replacement string `json:"replacement"`
|
||||||
Explanation string `json:"explanation"`
|
Explanation string `json:"explanation"`
|
||||||
Type string `json:"type"` // grammar | phrasing | idiom | clarity | voice | collocation
|
Type string `json:"type"` // grammar | phrasing | idiom | clarity | translate | voice | collocation
|
||||||
Status string `json:"status"` // pending | accepted | rejected
|
Status string `json:"status"` // pending | accepted | rejected
|
||||||
|
// Source names the engine that proposed the edit, not its family: an offline
|
||||||
|
// rule and the model can both propose a collocation, and the writer is never
|
||||||
|
// told which one spoke. It exists so each pass can replace its own rows.
|
||||||
|
Source string `json:"source"` // llm | local
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -115,10 +137,23 @@ const (
|
|||||||
SuggestionTypePhrasing = "phrasing"
|
SuggestionTypePhrasing = "phrasing"
|
||||||
SuggestionTypeIdiom = "idiom"
|
SuggestionTypeIdiom = "idiom"
|
||||||
SuggestionTypeClarity = "clarity"
|
SuggestionTypeClarity = "clarity"
|
||||||
|
// A span she wrote in her own language, rendered into English. Not a
|
||||||
|
// correction — nothing was wrong with it — which is why it is its own type
|
||||||
|
// rather than a clarity fix: the card is the pair model's flagship moment
|
||||||
|
// (SUGGESTIONS §1), and labelling it "Clarity" reads as a tidy-up of her
|
||||||
|
// first language. The model isn't asked for this label; it is derived from the
|
||||||
|
// span itself (see suggestions/language.go), so it can't drift.
|
||||||
|
SuggestionTypeTranslate = "translate"
|
||||||
SuggestionTypeVoice = "voice"
|
SuggestionTypeVoice = "voice"
|
||||||
SuggestionTypeCollocation = "collocation"
|
SuggestionTypeCollocation = "collocation"
|
||||||
SuggestionTypeMechanics = "mechanics" // deterministic rule-based pass (no LLM)
|
SuggestionTypeMechanics = "mechanics" // deterministic rule-based pass (no LLM)
|
||||||
|
|
||||||
|
// Who proposed it. The offline rule pack ('local') runs on every edit inside
|
||||||
|
// the browser and survives a VPN-down box; the model ('llm') adds the long
|
||||||
|
// tail when it is reachable.
|
||||||
|
SuggestionSourceLLM = "llm"
|
||||||
|
SuggestionSourceLocal = "local"
|
||||||
|
|
||||||
SuggestionStatusPending = "pending"
|
SuggestionStatusPending = "pending"
|
||||||
SuggestionStatusAccepted = "accepted"
|
SuggestionStatusAccepted = "accepted"
|
||||||
SuggestionStatusRejected = "rejected"
|
SuggestionStatusRejected = "rejected"
|
||||||
|
|||||||
+71
-4
@@ -287,7 +287,14 @@ func mdBlock(n pmNode, depth int) string {
|
|||||||
return "---"
|
return "---"
|
||||||
case "image":
|
case "image":
|
||||||
alt := n.attrStr("alt")
|
alt := n.attrStr("alt")
|
||||||
return fmt.Sprintf("", alt, n.attrStr("src"))
|
src := safeURL(n.attrStr("src"))
|
||||||
|
if src == "" {
|
||||||
|
// Nowhere safe to point. Keep the alt text as plain prose — it is
|
||||||
|
// the part that carries meaning — rather than emitting an image
|
||||||
|
// whose destination was rejected. See safeURL.
|
||||||
|
return alt
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("", alt, src)
|
||||||
case "table":
|
case "table":
|
||||||
return mdTable(n)
|
return mdTable(n)
|
||||||
case "bulletList", "orderedList":
|
case "bulletList", "orderedList":
|
||||||
@@ -396,7 +403,9 @@ func applyMdMarks(n pmNode) string {
|
|||||||
if n.hasMark("underline") {
|
if n.hasMark("underline") {
|
||||||
t = "<u>" + t + "</u>"
|
t = "<u>" + t + "</u>"
|
||||||
}
|
}
|
||||||
if href := n.markAttr("link", "href"); href != "" {
|
// Same rule as the HTML export: plenty of Markdown renderers pass a
|
||||||
|
// `javascript:` destination straight through into an <a href>. See safeURL.
|
||||||
|
if href := safeURL(n.markAttr("link", "href")); href != "" {
|
||||||
t = "[" + t + "](" + href + ")"
|
t = "[" + t + "](" + href + ")"
|
||||||
}
|
}
|
||||||
return t
|
return t
|
||||||
@@ -518,7 +527,16 @@ func htmlBlock(n pmNode) string {
|
|||||||
return "<hr>\n"
|
return "<hr>\n"
|
||||||
case "image":
|
case "image":
|
||||||
alt := htmlEscape(n.attrStr("alt"))
|
alt := htmlEscape(n.attrStr("alt"))
|
||||||
return fmt.Sprintf("<p><img src=\"%s\" alt=\"%s\"></p>\n", htmlEscape(n.attrStr("src")), alt)
|
src := safeURL(n.attrStr("src"))
|
||||||
|
if src == "" {
|
||||||
|
// Nowhere safe to point: keep the alt text, which is the part that
|
||||||
|
// carries meaning, rather than emitting a broken image.
|
||||||
|
if alt == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return "<p>" + alt + "</p>\n"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("<p><img src=\"%s\" alt=\"%s\"></p>\n", htmlEscape(src), alt)
|
||||||
case "table":
|
case "table":
|
||||||
return htmlTable(n)
|
return htmlTable(n)
|
||||||
case "bulletList", "orderedList":
|
case "bulletList", "orderedList":
|
||||||
@@ -617,7 +635,9 @@ func applyHTMLMarks(n pmNode) string {
|
|||||||
if n.hasMark("highlight") {
|
if n.hasMark("highlight") {
|
||||||
t = "<mark>" + t + "</mark>"
|
t = "<mark>" + t + "</mark>"
|
||||||
}
|
}
|
||||||
if href := n.markAttr("link", "href"); href != "" {
|
// An unsafe href is dropped, not the link: the words stay, they just stop
|
||||||
|
// being clickable. See safeURL.
|
||||||
|
if href := safeURL(n.markAttr("link", "href")); href != "" {
|
||||||
t = fmt.Sprintf("<a href=\"%s\">%s</a>", htmlEscape(href), t)
|
t = fmt.Sprintf("<a href=\"%s\">%s</a>", htmlEscape(href), t)
|
||||||
}
|
}
|
||||||
return t
|
return t
|
||||||
@@ -858,6 +878,53 @@ func htmlEscape(s string) string {
|
|||||||
return r.Replace(s)
|
return r.Replace(s)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// safeURLSchemes are the schemes an exported document may point at. Escaping
|
||||||
|
// makes a URL safe to sit inside an attribute; it says nothing about what
|
||||||
|
// happens when the attribute is followed, and `javascript:` survives it
|
||||||
|
// untouched.
|
||||||
|
//
|
||||||
|
// The toolbar can't produce one — it prefixes anything it doesn't recognise
|
||||||
|
// with https:// — but the toolbar is not the only way in: PUT /api/docs/{id}
|
||||||
|
// stores whatever Tiptap JSON it is given. And an export is the one artifact
|
||||||
|
// here that is *meant* to leave: the passport and the .html backup are files a
|
||||||
|
// writer hands to a teacher or an editor, opened on a machine that has no
|
||||||
|
// reason to trust them. A link that runs code when clicked is not something to
|
||||||
|
// ship inside one.
|
||||||
|
//
|
||||||
|
// Relative and fragment links pass through: they're how a document refers to
|
||||||
|
// its own headings, and they can't reach anything.
|
||||||
|
var safeURLSchemes = map[string]bool{
|
||||||
|
"http": true, "https": true, "mailto": true, "tel": true, "ftp": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// safeURL returns u if it is safe to follow from an exported file, and "" if it
|
||||||
|
// isn't. A dropped href leaves the link text in place — the reader loses a
|
||||||
|
// destination, never the writing.
|
||||||
|
func safeURL(u string) string {
|
||||||
|
trimmed := strings.TrimSpace(u)
|
||||||
|
if trimmed == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
// A scheme is everything before the first ':', but only when no '/', '?' or
|
||||||
|
// '#' comes first — otherwise "notes/a:b" would read as the "notes/a" scheme.
|
||||||
|
// Nothing before a colon means a relative or fragment link, which is fine.
|
||||||
|
if i := strings.IndexAny(trimmed, ":/?#"); i >= 0 && trimmed[i] == ':' {
|
||||||
|
// Control characters and whitespace are stripped by browsers *before*
|
||||||
|
// the scheme is read, so "java\nscript:" is javascript:. Fold them out
|
||||||
|
// before deciding rather than after.
|
||||||
|
scheme := strings.Map(func(r rune) rune {
|
||||||
|
if r <= ' ' || r == 0x7f {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}, trimmed[:i])
|
||||||
|
if !safeURLSchemes[strings.ToLower(scheme)] {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return trimmed
|
||||||
|
}
|
||||||
|
|
||||||
func xmlEscape(s string) string {
|
func xmlEscape(s string) string {
|
||||||
r := strings.NewReplacer("&", "&", "<", "<", ">", ">", `"`, """)
|
r := strings.NewReplacer("&", "&", "<", "<", ">", ">", `"`, """)
|
||||||
return r.Replace(s)
|
return r.Replace(s)
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
)
|
)
|
||||||
|
|
||||||
// richDocJSON is a Tiptap document exercising headings, marks, and a list —
|
// richDocJSON is a Tiptap document exercising headings, marks, and a list —
|
||||||
@@ -243,3 +245,73 @@ func TestExportUnsupportedFormat(t *testing.T) {
|
|||||||
t.Fatalf("expected 400 for unsupported format, got %d", rec.Code)
|
t.Fatalf("expected 400 for unsupported format, got %d", rec.Code)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Escaping makes a URL safe to sit inside an attribute; it says nothing about
|
||||||
|
// what happens when the attribute is followed. An export is the one artifact
|
||||||
|
// here meant to leave — the file handed to a teacher, opened on a machine with
|
||||||
|
// no reason to trust it — so a destination that runs code is dropped.
|
||||||
|
func TestExportDropsUnsafeLinkSchemes(t *testing.T) {
|
||||||
|
unsafe := []string{
|
||||||
|
"javascript:alert(1)",
|
||||||
|
"JaVaScRiPt:alert(1)",
|
||||||
|
"java\nscript:alert(1)", // browsers strip control characters first
|
||||||
|
" javascript:alert(1)",
|
||||||
|
"data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==",
|
||||||
|
"vbscript:msgbox(1)",
|
||||||
|
}
|
||||||
|
for _, href := range unsafe {
|
||||||
|
if got := safeURL(href); got != "" {
|
||||||
|
t.Errorf("safeURL(%q) = %q, want it dropped", href, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
safe := []string{
|
||||||
|
"https://example.com/a?b=1#c",
|
||||||
|
"http://example.com",
|
||||||
|
"mailto:her@example.com",
|
||||||
|
"/api/images/abc.png",
|
||||||
|
"#a-heading",
|
||||||
|
"notes/chapter:one.md", // a colon that isn't a scheme
|
||||||
|
}
|
||||||
|
for _, href := range safe {
|
||||||
|
if got := safeURL(href); got != href {
|
||||||
|
t.Errorf("safeURL(%q) = %q, want it kept", href, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// End to end through the renderers: an unsafe href loses its destination, never
|
||||||
|
// its words.
|
||||||
|
func TestRenderedExportsCarryNoScriptURLs(t *testing.T) {
|
||||||
|
doc := db.Document{
|
||||||
|
Title: "Notes",
|
||||||
|
Content: `{"type":"doc","content":[{"type":"paragraph","content":[
|
||||||
|
{"type":"text","text":"click me","marks":[{"type":"link","attrs":{"href":"javascript:alert(1)"}}]}]},
|
||||||
|
{"type":"image","attrs":{"src":"javascript:alert(2)","alt":"a drawing"}}]}`,
|
||||||
|
}
|
||||||
|
|
||||||
|
html, err := renderHTMLFile(doc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(strings.ToLower(string(html)), "javascript:") {
|
||||||
|
t.Fatalf("html export carried a javascript: URL:\n%s", html)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(html), "click me") {
|
||||||
|
t.Fatal("html export dropped the link text along with the href")
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(html), "a drawing") {
|
||||||
|
t.Fatal("html export dropped the alt text of the rejected image")
|
||||||
|
}
|
||||||
|
|
||||||
|
md, err := renderMarkdown(doc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(strings.ToLower(string(md)), "javascript:") {
|
||||||
|
t.Fatalf("markdown export carried a javascript: URL:\n%s", md)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(md), "click me") {
|
||||||
|
t.Fatal("markdown export dropped the link text along with the href")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -225,13 +225,14 @@ func (h *Handler) fetch(userID, id string) (db.Document, error) {
|
|||||||
var doc db.Document
|
var doc db.Document
|
||||||
err := h.DB.QueryRow(
|
err := h.DB.QueryRow(
|
||||||
`SELECT id, user_id, title, content, content_text, tone, word_count,
|
`SELECT id, user_id, title, content, content_text, tone, word_count,
|
||||||
created_at, updated_at, preserve_history
|
created_at, updated_at, preserve_history, doc_lang
|
||||||
FROM documents
|
FROM documents
|
||||||
WHERE id = ? AND user_id = ?`,
|
WHERE id = ? AND user_id = ?`,
|
||||||
id, userID,
|
id, userID,
|
||||||
).Scan(
|
).Scan(
|
||||||
&doc.ID, &doc.UserID, &doc.Title, &doc.Content, &doc.ContentText,
|
&doc.ID, &doc.UserID, &doc.Title, &doc.Content, &doc.ContentText,
|
||||||
&doc.Tone, &doc.WordCount, &doc.CreatedAt, &doc.UpdatedAt, &doc.PreserveHistory,
|
&doc.Tone, &doc.WordCount, &doc.CreatedAt, &doc.UpdatedAt, &doc.PreserveHistory,
|
||||||
|
&doc.DocLang,
|
||||||
)
|
)
|
||||||
return doc, err
|
return doc, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,3 +35,18 @@ func ServerError(w http.ResponseWriter, err error) {
|
|||||||
log.Printf("internal error: %v", err)
|
log.Printf("internal error: %v", err)
|
||||||
ErrorJSON(w, http.StatusInternalServerError, "something went wrong")
|
ErrorJSON(w, http.StatusInternalServerError, "something went wrong")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UpstreamError is ServerError's counterpart for a dependency Petal calls out
|
||||||
|
// to — the model, chiefly. Same discipline, and for a sharper reason: a dial
|
||||||
|
// failure's error text contains the endpoint it failed to dial, so relaying it
|
||||||
|
// hands anyone who can reach Petal the address of the inference box on the far
|
||||||
|
// side of the VPN, along with which backend is running there.
|
||||||
|
//
|
||||||
|
// `what` names the pass for the operator's log ("checkpoint", "chat"). The
|
||||||
|
// browser is told only that the helper is unreachable, which is all the client
|
||||||
|
// ever did anything with: every LLM route's 502 renders as the same warm
|
||||||
|
// "小助手在休息 · Petal's helper is resting".
|
||||||
|
func UpstreamError(w http.ResponseWriter, what string, err error) {
|
||||||
|
log.Printf("upstream error (%s): %v", what, err)
|
||||||
|
ErrorJSON(w, http.StatusBadGateway, "Petal's helper is out of reach right now")
|
||||||
|
}
|
||||||
|
|||||||
@@ -37,6 +37,17 @@ import (
|
|||||||
// small enough to keep a careless paste from filling the disk.
|
// small enough to keep a careless paste from filling the disk.
|
||||||
const maxUploadBytes = 10 << 20
|
const maxUploadBytes = 10 << 20
|
||||||
|
|
||||||
|
// maxUserBytes caps what one account may keep stored, at 1 GiB. The per-upload
|
||||||
|
// limit bounds a single careless paste; nothing bounded ten thousand of them,
|
||||||
|
// and Petal's data directory is an 8 GiB encrypted volume shared with the
|
||||||
|
// database, the backups and the TTS cache — the disk filling is the database
|
||||||
|
// losing writes, not just images failing.
|
||||||
|
//
|
||||||
|
// A tenth of the volume per writer is far past any real use: a heavily
|
||||||
|
// illustrated journal is tens of megabytes. It is a runaway backstop, and it is
|
||||||
|
// deliberately generous enough that nobody writing normally will ever meet it.
|
||||||
|
const maxUserBytes = 1 << 30
|
||||||
|
|
||||||
// extByContentType maps the image types we accept to a canonical extension. The
|
// extByContentType maps the image types we accept to a canonical extension. The
|
||||||
// allowlist doubles as validation: anything not here is rejected.
|
// allowlist doubles as validation: anything not here is rejected.
|
||||||
var extByContentType = map[string]string{
|
var extByContentType = map[string]string{
|
||||||
@@ -177,6 +188,19 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
|||||||
name := hex.EncodeToString(sum[:])[:32] + ext
|
name := hex.EncodeToString(sum[:])[:32] + ext
|
||||||
path := filepath.Join(h.dir, name)
|
path := filepath.Join(h.dir, name)
|
||||||
|
|
||||||
|
userID := auth.UserID(r.Context())
|
||||||
|
within, err := h.withinQuota(userID, name, int64(len(data)))
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("images: quota check failed for %s: %v", userID, err)
|
||||||
|
http.Error(w, "could not store image", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !within {
|
||||||
|
http.Error(w, "you've filled Petal's picture store — delete a few images and try again",
|
||||||
|
http.StatusInsufficientStorage)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Skip the write if this exact content is already stored.
|
// Skip the write if this exact content is already stored.
|
||||||
if _, statErr := os.Stat(path); errors.Is(statErr, os.ErrNotExist) {
|
if _, statErr := os.Stat(path); errors.Is(statErr, os.ErrNotExist) {
|
||||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||||
@@ -190,7 +214,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
|||||||
if _, err := h.db.Exec(
|
if _, err := h.db.Exec(
|
||||||
`INSERT INTO images (name, user_id, content_type, size) VALUES (?, ?, ?, ?)
|
`INSERT INTO images (name, user_id, content_type, size) VALUES (?, ?, ?, ?)
|
||||||
ON CONFLICT (name, user_id) DO NOTHING`,
|
ON CONFLICT (name, user_id) DO NOTHING`,
|
||||||
name, auth.UserID(r.Context()), ct, len(data),
|
name, userID, ct, len(data),
|
||||||
); err != nil {
|
); err != nil {
|
||||||
log.Printf("images: could not record ownership of %s: %v", name, err)
|
log.Printf("images: could not record ownership of %s: %v", name, err)
|
||||||
http.Error(w, "could not store image", http.StatusInternalServerError)
|
http.Error(w, "could not store image", http.StatusInternalServerError)
|
||||||
@@ -221,6 +245,20 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
// Private: a shared cache must never hand one writer's image to another.
|
// Private: a shared cache must never hand one writer's image to another.
|
||||||
w.Header().Set("Cache-Control", "private, max-age=31536000, immutable")
|
w.Header().Set("Cache-Control", "private, max-age=31536000, immutable")
|
||||||
|
|
||||||
|
// SVG is a document format wearing an image's name: it can carry <script>,
|
||||||
|
// and this route serves it from Petal's own origin. Rendered through an
|
||||||
|
// <img> — the only way the editor ever shows one — that script never runs.
|
||||||
|
// Navigated to directly, which is one "open image in new tab" away, it does,
|
||||||
|
// and it runs with the API of whoever opened it.
|
||||||
|
//
|
||||||
|
// So every stored image answers with a CSP that permits nothing at all
|
||||||
|
// except the inline styles an illustration legitimately carries. It costs
|
||||||
|
// pasted SVGs nothing (an <img> was already a script-free context) and
|
||||||
|
// leaves the direct-navigation case inert. nosniff is set at the edge, but
|
||||||
|
// repeated here so the guarantee doesn't depend on Traefik's config.
|
||||||
|
w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox")
|
||||||
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||||
http.ServeFile(w, r, path)
|
http.ServeFile(w, r, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -261,6 +299,29 @@ func (h *Handler) remove(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withinQuota reports whether userID may store one more image of size bytes.
|
||||||
|
//
|
||||||
|
// An image the caller already owns is free: content addressing means re-pasting
|
||||||
|
// the same picture stores nothing new, and charging for it would let a document
|
||||||
|
// that merely repeats one illustration walk into the limit. Deduplication
|
||||||
|
// across *accounts* is not credited the same way — two people each keep their
|
||||||
|
// own claim on a shared file, because either of them deleting it must not
|
||||||
|
// depend on what the other did.
|
||||||
|
func (h *Handler) withinQuota(userID, name string, size int64) (bool, error) {
|
||||||
|
var used, already sql.NullInt64
|
||||||
|
if err := h.db.QueryRow(
|
||||||
|
`SELECT (SELECT COALESCE(SUM(size), 0) FROM images WHERE user_id = ?),
|
||||||
|
(SELECT size FROM images WHERE user_id = ? AND name = ?)`,
|
||||||
|
userID, userID, name,
|
||||||
|
).Scan(&used, &already); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if already.Valid {
|
||||||
|
return true, nil // already stored for this account — costs nothing more
|
||||||
|
}
|
||||||
|
return used.Int64+size <= maxUserBytes, nil
|
||||||
|
}
|
||||||
|
|
||||||
// owns reports whether userID has a claim on a stored image.
|
// owns reports whether userID has a claim on a stored image.
|
||||||
func (h *Handler) owns(name, userID string) bool {
|
func (h *Handler) owns(name, userID string) bool {
|
||||||
var ok bool
|
var ok bool
|
||||||
|
|||||||
@@ -240,3 +240,85 @@ func TestServeMissing(t *testing.T) {
|
|||||||
t.Fatalf("expected 404, got %d", rec.Code)
|
t.Fatalf("expected 404, got %d", rec.Code)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An SVG is a document, not a picture: it can carry <script>, and this route
|
||||||
|
// serves it from Petal's own origin. Rendered through an <img> that script
|
||||||
|
// never runs, but "open image in new tab" is one click away, and there it
|
||||||
|
// would — with the API of whoever opened it. Every stored image therefore
|
||||||
|
// answers with a CSP that permits nothing.
|
||||||
|
func TestStoredImagesAreServedInert(t *testing.T) {
|
||||||
|
_, alice, _ := newStore(t)
|
||||||
|
|
||||||
|
svg := []byte(`<svg xmlns="http://www.w3.org/2000/svg"><script>fetch('/api/docs')</script></svg>`)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
alice.ServeHTTP(rec, uploadReq(t, "image", svg))
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("svg upload code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
var resp struct{ URL string }
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
name := strings.TrimPrefix(resp.URL, "/api/images/")
|
||||||
|
|
||||||
|
got := get(t, alice, name)
|
||||||
|
if got.Code != http.StatusOK {
|
||||||
|
t.Fatalf("serve code=%d", got.Code)
|
||||||
|
}
|
||||||
|
csp := got.Header().Get("Content-Security-Policy")
|
||||||
|
if !strings.Contains(csp, "default-src 'none'") || !strings.Contains(csp, "sandbox") {
|
||||||
|
t.Fatalf("CSP %q does not neutralize the response", csp)
|
||||||
|
}
|
||||||
|
if got.Header().Get("X-Content-Type-Options") != "nosniff" {
|
||||||
|
t.Fatal("stored images must be served nosniff")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A per-upload cap bounds one careless paste; nothing bounded ten thousand of
|
||||||
|
// them, on the same volume the database lives on.
|
||||||
|
func TestUploadQuota(t *testing.T) {
|
||||||
|
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open db: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { database.Close() })
|
||||||
|
if _, err := database.Exec(
|
||||||
|
`INSERT INTO users (id, email, display_name) VALUES (?, ?, ?)`,
|
||||||
|
"bob", "bob@petal.local", "Bob",
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("seed second user: %v", err)
|
||||||
|
}
|
||||||
|
h, err := New(t.TempDir(), database.DB, db.LocalUserID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("new store: %v", err)
|
||||||
|
}
|
||||||
|
alice := auth.Middleware(auth.StaticResolver(db.LocalUserID))(h.Routes())
|
||||||
|
bob := auth.Middleware(auth.StaticResolver("bob"))(h.Routes())
|
||||||
|
|
||||||
|
// Fill Alice's allowance by hand — uploading a gibibyte in a test would be
|
||||||
|
// absurd, and what's under test is the accounting, not the arithmetic.
|
||||||
|
name := upload(t, alice, pngBytes)
|
||||||
|
if _, err := database.Exec(
|
||||||
|
`UPDATE images SET size = ? WHERE user_id = ? AND name = ?`,
|
||||||
|
int64(maxUserBytes), db.LocalUserID, name,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-storing something she already has costs nothing, so it still works.
|
||||||
|
if again := upload(t, alice, pngBytes); again != name {
|
||||||
|
t.Fatalf("a re-upload of an owned image should dedupe, got %q", again)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Anything new does not.
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
alice.ServeHTTP(rec, uploadReq(t, "image", otherPNG))
|
||||||
|
if rec.Code != http.StatusInsufficientStorage {
|
||||||
|
t.Fatalf("over-quota upload code=%d, want 507", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And it is *her* allowance, not the store's: Bob is unaffected.
|
||||||
|
if got := upload(t, bob, otherPNG); got == "" {
|
||||||
|
t.Fatal("one writer's quota must not stop another writing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -36,3 +36,13 @@ var glossGz []byte
|
|||||||
//
|
//
|
||||||
//go:embed data/phonetic.json.gz
|
//go:embed data/phonetic.json.gz
|
||||||
var phoneticGz []byte
|
var phoneticGz []byte
|
||||||
|
|
||||||
|
// hanziGz is the gzipped Chinese→English map: simplified headword → [[pinyin,
|
||||||
|
// senses], …]. Built from CC-CEDICT (scripts/build_cedict.py), unfiltered — the
|
||||||
|
// word a learner stops on is the one they do not know, so this is the one
|
||||||
|
// dataset here with no frequency gate. Loaded on its own sync.Once (see
|
||||||
|
// hanzi.go), not with the four above, because only a learner-direction account
|
||||||
|
// ever asks for it.
|
||||||
|
//
|
||||||
|
//go:embed data/hanzi.json.gz
|
||||||
|
var hanziGz []byte
|
||||||
|
|||||||
Binary file not shown.
@@ -0,0 +1,333 @@
|
|||||||
|
package lexicon
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"github.com/prosolis/dreamdict/dictionary"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DreamDict is a read-only handle on a built dict.db — one SQLite file holding
|
||||||
|
// English, French, European Portuguese, Spanish and Mandarin.
|
||||||
|
//
|
||||||
|
// It is a second database beside petal.db and is never written to: the file is
|
||||||
|
// built by DreamDict's own import CLI a few times a year, and Petal only reads
|
||||||
|
// it. That is what makes importing the package the right shape rather than
|
||||||
|
// running DreamDict as a service — a hover gloss should not depend on a second
|
||||||
|
// process being up, still less on one reachable across a VPN.
|
||||||
|
type DreamDict struct {
|
||||||
|
d *dictionary.Dictionary
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenDreamDict opens dict.db read-only.
|
||||||
|
//
|
||||||
|
// A missing file returns (nil, nil), not an error. Petal is expected to run
|
||||||
|
// without dict.db — a laptop checkout has never had one, and the zh pair does
|
||||||
|
// not need one — so "the file isn't there" is a deployment state the caller
|
||||||
|
// handles by carrying on. A file that is *present but unusable* (corrupt, or
|
||||||
|
// never imported) does return an error, because that one is a mistake someone
|
||||||
|
// should hear about.
|
||||||
|
func OpenDreamDict(path string) (*DreamDict, error) {
|
||||||
|
if strings.TrimSpace(path) == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
d, err := dictionary.NewReadOnly(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &DreamDict{d: d}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close releases the dict.db handle. Safe on a nil DreamDict, so a caller that
|
||||||
|
// never got one can defer it unconditionally.
|
||||||
|
func (dd *DreamDict) Close() error {
|
||||||
|
if dd == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return dd.d.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Contents reports how many words the open dict.db holds per language, so
|
||||||
|
// startup can log what it actually got.
|
||||||
|
//
|
||||||
|
// It counts rows rather than returning DreamDict's list of supported languages.
|
||||||
|
// Those are not the same thing and the difference is the whole point: a
|
||||||
|
// database built before Spanish existed still *supports* Spanish, and a log
|
||||||
|
// line naming the supported set would have said so cheerfully while every
|
||||||
|
// Spanish lookup came back empty. Counting rows is the question worth asking of
|
||||||
|
// a file somebody had to copy onto the box by hand.
|
||||||
|
func (dd *DreamDict) Contents() string {
|
||||||
|
counts, err := dd.d.WordCount()
|
||||||
|
if err != nil {
|
||||||
|
return "unreadable: " + err.Error()
|
||||||
|
}
|
||||||
|
langs := make([]string, 0, len(counts))
|
||||||
|
for lang := range counts {
|
||||||
|
langs = append(langs, lang)
|
||||||
|
}
|
||||||
|
sort.Strings(langs)
|
||||||
|
parts := make([]string, 0, len(langs))
|
||||||
|
for _, lang := range langs {
|
||||||
|
parts = append(parts, fmt.Sprintf("%s=%d", lang, counts[lang]))
|
||||||
|
}
|
||||||
|
if len(parts) == 0 {
|
||||||
|
return "no words"
|
||||||
|
}
|
||||||
|
return strings.Join(parts, " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// dreamProvider serves one writer: English lookups from dict.db, glossed into
|
||||||
|
// native. The struct is a value, created per request by [Set.For] — it holds no
|
||||||
|
// state beyond the shared handle and the language to translate into.
|
||||||
|
type dreamProvider struct {
|
||||||
|
dict *DreamDict
|
||||||
|
native string // the writer's language, e.g. "pt-PT"
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxEtymology caps the free-form Wiktionary etymology. It is the one field
|
||||||
|
// with no natural length: some entries are a clause, some are four paragraphs
|
||||||
|
// tracing a word through three dead languages. The popover wants a line.
|
||||||
|
const maxEtymology = 220
|
||||||
|
|
||||||
|
// Lookup fills a Result from dict.db.
|
||||||
|
//
|
||||||
|
// The word is de-inflected with the same [candidates] walk the embedded
|
||||||
|
// datasets use, because dict.db stores headwords: "running" has no definitions
|
||||||
|
// row of its own. The first candidate that *has* definitions becomes the
|
||||||
|
// headword every other field is then read from, so a single popover never
|
||||||
|
// mixes "running"'s frequency with "run"'s definitions.
|
||||||
|
//
|
||||||
|
// The gloss is walked separately. A word can be absent from the definitions
|
||||||
|
// table and still have a translation (and vice versa), and the hover tooltip
|
||||||
|
// asks for the gloss alone — so tying it to the definition headword would lose
|
||||||
|
// glosses for no benefit.
|
||||||
|
func (p dreamProvider) Lookup(word string) (Result, error) {
|
||||||
|
res := Result{Word: word, Definitions: []Meaning{}, Synonyms: []string{}, Difficulty: unknownDifficulty}
|
||||||
|
norm := strings.ToLower(strings.TrimSpace(word))
|
||||||
|
if norm == "" {
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
head := norm
|
||||||
|
for _, c := range candidates(norm) {
|
||||||
|
defs, err := p.dict.d.Define(c, langEN)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
if len(defs) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
head = c
|
||||||
|
for _, d := range defs {
|
||||||
|
// DreamDict orders by source priority, so the curated senses
|
||||||
|
// (WordNet, WOLF) are already ahead of the Wiktionary tail — taking
|
||||||
|
// the first few is taking the best few.
|
||||||
|
res.Definitions = append(res.Definitions, Meaning{PartOfSpeech: d.POS, Definition: d.Gloss})
|
||||||
|
if len(res.Definitions) >= maxDefinitions {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
gloss, err := p.translate(norm)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
res.Gloss = gloss
|
||||||
|
|
||||||
|
syns, err := p.dict.d.Synonyms(head, langEN)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
if len(syns) > maxSynonyms {
|
||||||
|
syns = syns[:maxSynonyms]
|
||||||
|
}
|
||||||
|
res.Synonyms = append(res.Synonyms, syns...)
|
||||||
|
|
||||||
|
prons, err := p.dict.d.Pronunciation(head, langEN)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
res.Phonetic = pickIPA(prons)
|
||||||
|
|
||||||
|
if res.Frequency, err = p.dict.d.Frequency(head, langEN); err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
if res.Difficulty, err = p.dict.d.Difficulty(head, langEN); err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
ety, err := p.dict.d.Etymology(head, langEN)
|
||||||
|
if err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
res.Etymology = trimEtymology(ety)
|
||||||
|
|
||||||
|
if res.Reverse, err = p.reverse(norm); err != nil {
|
||||||
|
return Result{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// reverse reads the token as a word of the writer's own language, and returns
|
||||||
|
// nil when it isn't one — which is the answer for almost every word she looks
|
||||||
|
// up, since she is writing English.
|
||||||
|
//
|
||||||
|
// The English de-inflection walk is deliberately *not* applied here. [candidates]
|
||||||
|
// knows about -s, -ed and -ing; running it over Portuguese would turn "vinhas"
|
||||||
|
// into "vinha" by an English rule that happens to be right and "cantava" into
|
||||||
|
// nothing by rules that are simply irrelevant. dict.db stores headwords, so an
|
||||||
|
// inflected Portuguese form finds nothing and the card shows only the English
|
||||||
|
// reading — the same outcome as today, rather than a confidently wrong one.
|
||||||
|
func (p dreamProvider) reverse(norm string) (*Reverse, error) {
|
||||||
|
back, err := p.dict.d.Equivalents(norm, p.native, langEN)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defs, err := p.dict.d.Define(norm, p.native)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(back) == 0 && len(defs) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
rev := &Reverse{Lang: p.native}
|
||||||
|
if len(back) > maxGlossSenses {
|
||||||
|
back = back[:maxGlossSenses]
|
||||||
|
}
|
||||||
|
rev.Gloss = strings.Join(back, "; ")
|
||||||
|
for _, d := range defs {
|
||||||
|
rev.Definitions = append(rev.Definitions, Meaning{PartOfSpeech: d.POS, Definition: d.Gloss})
|
||||||
|
if len(rev.Definitions) >= maxReverseDefinitions {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
prons, err := p.dict.d.Pronunciation(norm, p.native)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
rev.Phonetic = pickIPA(prons)
|
||||||
|
|
||||||
|
return rev, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxReverseDefinitions is smaller than [maxDefinitions]: the reverse reading is
|
||||||
|
// the second half of a card that already has an English one, and it is there to
|
||||||
|
// say "this is also a Portuguese word, and here is what it means" rather than to
|
||||||
|
// be a dictionary entry in its own right.
|
||||||
|
const maxReverseDefinitions = 2
|
||||||
|
|
||||||
|
// Gloss returns the writer's-language translation alone — the hover tooltip's
|
||||||
|
// fast path, one indexed query per candidate form and nothing else.
|
||||||
|
func (p dreamProvider) Gloss(word string) (GlossResult, error) {
|
||||||
|
norm := strings.ToLower(strings.TrimSpace(word))
|
||||||
|
if norm == "" {
|
||||||
|
return GlossResult{Word: word}, nil
|
||||||
|
}
|
||||||
|
gloss, err := p.translate(norm)
|
||||||
|
if err != nil {
|
||||||
|
return GlossResult{}, err
|
||||||
|
}
|
||||||
|
res := GlossResult{Word: word, Gloss: gloss}
|
||||||
|
|
||||||
|
// The tooltip carries only the reverse *gloss*, not the whole reading: it is
|
||||||
|
// a one-line bubble under a resting pointer, and the popover is one click
|
||||||
|
// away for anyone who wants the rest.
|
||||||
|
back, err := p.dict.d.Equivalents(norm, p.native, langEN)
|
||||||
|
if err != nil {
|
||||||
|
return GlossResult{}, err
|
||||||
|
}
|
||||||
|
if len(back) > maxGlossSenses {
|
||||||
|
back = back[:maxGlossSenses]
|
||||||
|
}
|
||||||
|
res.Reverse = strings.Join(back, "; ")
|
||||||
|
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxGlossSenses caps how many translations are strung together. One is often
|
||||||
|
// too thin to disambiguate; the whole list is a wall of words in a tooltip.
|
||||||
|
const maxGlossSenses = 3
|
||||||
|
|
||||||
|
// translate walks the candidate forms and returns the first that has an
|
||||||
|
// equivalent in the writer's language, joined into one line.
|
||||||
|
//
|
||||||
|
// It asks for Equivalents rather than Translate on the strength of measuring
|
||||||
|
// both against the real dict.db: Wiktionary's en→pt-PT translation table
|
||||||
|
// answers for 17% of the 2,000 commonest English words, and the shared-synset
|
||||||
|
// path answers for 62%. The plan assumed Translate would do — the database
|
||||||
|
// says otherwise, and a gloss that is absent five times out of six is not a
|
||||||
|
// gloss. Equivalents falls back to Translate internally, so nothing is lost.
|
||||||
|
//
|
||||||
|
// A language dict.db was built without simply has no rows, so this returns "" —
|
||||||
|
// which is exactly what an unglossed word returns, and the popover already
|
||||||
|
// renders that case. Spanish was precisely this until the database was rebuilt
|
||||||
|
// with it on 2026-07-27; the code path did not change, the file did.
|
||||||
|
func (p dreamProvider) translate(norm string) (string, error) {
|
||||||
|
for _, c := range candidates(norm) {
|
||||||
|
trs, err := p.dict.d.Equivalents(c, langEN, p.native)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if len(trs) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(trs) > maxGlossSenses {
|
||||||
|
trs = trs[:maxGlossSenses]
|
||||||
|
}
|
||||||
|
return strings.Join(trs, "; "), nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// pickIPA chooses what to show beside the read-aloud button. IPA is the only
|
||||||
|
// form worth showing a learner — CMU's "IH0 F EH1 M ER0 AH0 L" is a machine
|
||||||
|
// format, and printing it would be noise dressed up as help. If there's no IPA,
|
||||||
|
// there's no phonetic line.
|
||||||
|
func pickIPA(prons []dictionary.Pronunciation) string {
|
||||||
|
for _, p := range prons {
|
||||||
|
if strings.EqualFold(p.Format, "ipa") && strings.TrimSpace(p.Value) != "" {
|
||||||
|
return strings.Trim(strings.TrimSpace(p.Value), "/[]")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// trimEtymology cuts Wiktionary's prose down to a line, preferring to stop at a
|
||||||
|
// sentence boundary so the result reads as a finished thought rather than a
|
||||||
|
// truncation.
|
||||||
|
func trimEtymology(text string) string {
|
||||||
|
text = strings.Join(strings.Fields(text), " ")
|
||||||
|
if utf8.RuneCountInString(text) <= maxEtymology {
|
||||||
|
return text
|
||||||
|
}
|
||||||
|
// Counted and cut in runes, not bytes. An etymology is the one field that
|
||||||
|
// is *mostly* not English — ἐφήμερος, ephemerus, 短暫 — and a byte slice
|
||||||
|
// through the middle of one of those characters is invalid UTF-8 in the
|
||||||
|
// JSON response.
|
||||||
|
cut := string([]rune(text)[:maxEtymology-1])
|
||||||
|
// Stop at a sentence when one ends late enough to be worth keeping. An
|
||||||
|
// early full stop ("From Latin. …") is not a summary, it's a discarded
|
||||||
|
// paragraph, so that case falls through to the word-boundary cut.
|
||||||
|
if i := strings.LastIndex(cut, ". "); i > len(cut)/2 {
|
||||||
|
return cut[:i+1]
|
||||||
|
}
|
||||||
|
if i := strings.LastIndex(cut, " "); i > 0 {
|
||||||
|
cut = cut[:i]
|
||||||
|
}
|
||||||
|
return cut + "…"
|
||||||
|
}
|
||||||
@@ -0,0 +1,664 @@
|
|||||||
|
package lexicon
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
"github.com/prosolis/dreamdict/dictionary"
|
||||||
|
_ "modernc.org/sqlite"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The fixture is a real dict.db on disk rather than an in-memory handle, so
|
||||||
|
// these tests exercise the path production takes: stat the file, open it
|
||||||
|
// read-only, find it seeded. A fake would have skipped every one of those.
|
||||||
|
//
|
||||||
|
// "ephemeral" is the worked example throughout: it has definitions only under
|
||||||
|
// its own headword, translations into two languages, IPA alongside a CMU
|
||||||
|
// pronunciation Petal must not show, a frequency, a difficulty and an etymology
|
||||||
|
// long enough to need trimming.
|
||||||
|
func writeFixture(t *testing.T, seeded bool) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(t.TempDir(), "dict.db")
|
||||||
|
sqldb, err := sql.Open("sqlite", path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open fixture: %v", err)
|
||||||
|
}
|
||||||
|
defer sqldb.Close()
|
||||||
|
if err := dictionary.BootstrapSchema(sqldb); err != nil {
|
||||||
|
t.Fatalf("bootstrap: %v", err)
|
||||||
|
}
|
||||||
|
if !seeded {
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
exec := func(q string, args ...any) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := sqldb.Exec(q, args...); err != nil {
|
||||||
|
t.Fatalf("seed %q: %v", q, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
exec(`INSERT INTO meta (key, value) VALUES ('schema_version', '2')`)
|
||||||
|
|
||||||
|
exec(`INSERT INTO words (id, word, lang, pos, frequency, difficulty) VALUES
|
||||||
|
(1, 'ephemeral', 'en', 'adjective', 50, 0.72),
|
||||||
|
(2, 'run', 'en', 'verb', 900, 0.05),
|
||||||
|
(3, 'plain', 'en', 'adjective', 0, NULL),
|
||||||
|
(4, 'efémero', 'pt-PT', 'adjective', 12, 0.6)`)
|
||||||
|
|
||||||
|
exec(`INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES
|
||||||
|
(1, 'adjective', 'lasting a very short time', 'wordnet', 10),
|
||||||
|
(1, 'adjective', 'short-lived', 'wiktionary', 99),
|
||||||
|
(1, 'adjective', 'transitory', 'wiktionary', 99),
|
||||||
|
(1, 'adjective', 'fleeting', 'wiktionary', 99),
|
||||||
|
(1, 'adjective', 'evanescent', 'wiktionary', 99),
|
||||||
|
(2, 'verb', 'move fast on foot', 'wordnet', 10),
|
||||||
|
(3, 'adjective', 'without decoration', 'wordnet', 10)`)
|
||||||
|
|
||||||
|
exec(`INSERT INTO synonyms (word_id, synonym, source) VALUES
|
||||||
|
(1, 'fleeting', 'wordnet'), (1, 'transient', 'wordnet'),
|
||||||
|
(2, 'sprint', 'wordnet')`)
|
||||||
|
|
||||||
|
exec(`INSERT INTO translations (word_id, translation, target_lang, source) VALUES
|
||||||
|
(1, 'efémero', 'pt-PT', 'kaikki'),
|
||||||
|
(1, 'passageiro','pt-PT', 'kaikki'),
|
||||||
|
(1, 'éphémère', 'fr', 'kaikki'),
|
||||||
|
(1, '短暂的', 'zh', 'cedict'),
|
||||||
|
(2, 'correr', 'pt-PT', 'kaikki')`)
|
||||||
|
|
||||||
|
// CMU is listed first deliberately: picking the first row would show a
|
||||||
|
// learner "IH0 F EH1 M ER0 AH0 L", which is a machine format, not help.
|
||||||
|
exec(`INSERT INTO pronunciations (word_id, format, value, source) VALUES
|
||||||
|
(1, 'cmu', 'IH0 F EH1 M ER0 AH0 L', 'cmudict'),
|
||||||
|
(1, 'ipa', '/ɪˈfɛm.ər.əl/', 'wiktionary')`)
|
||||||
|
|
||||||
|
// "brief" carries no translation row at all — only a shared WordNet synset
|
||||||
|
// with two pt-PT words. On the real database that is the *usual* case, not
|
||||||
|
// the exotic one, so Petal must reach a gloss this way or the pt-PT pair
|
||||||
|
// has almost no glosses. "breve" is the commoner of the two and leads.
|
||||||
|
exec(`INSERT INTO words (id, word, lang, pos, frequency) VALUES
|
||||||
|
(5, 'brief', 'en', 'adjective', 400),
|
||||||
|
(6, 'breve', 'pt-PT', 'adjective', 300),
|
||||||
|
(7, 'sucinto', 'pt-PT', 'adjective', 20)`)
|
||||||
|
exec(`INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES
|
||||||
|
(5, 'adjective', 'of short duration', 'wordnet', 10)`)
|
||||||
|
exec(`INSERT INTO synsets (id, synset_id, pos) VALUES (1, '00751145-a', 'adjective')`)
|
||||||
|
exec(`INSERT INTO word_synsets (word_id, synset_id, source) VALUES
|
||||||
|
(5, 1, 'wordnet'), (6, 1, 'omw'), (7, 1, 'omw')`)
|
||||||
|
|
||||||
|
// "data" is the collision the Latin pairs create and the zh pair never did:
|
||||||
|
// a real English word and a real Portuguese one, spelled identically and
|
||||||
|
// meaning different things. There is no honest way to look at it in a mixed
|
||||||
|
// document and know which was meant, so Petal shows both readings.
|
||||||
|
exec(`INSERT INTO words (id, word, lang, pos, frequency) VALUES
|
||||||
|
(8, 'data', 'en', 'noun', 800),
|
||||||
|
(9, 'data', 'pt-PT', 'noun', 700),
|
||||||
|
(10, 'date', 'en', 'noun', 750)`)
|
||||||
|
exec(`INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES
|
||||||
|
(8, 'noun', 'facts collected for reference', 'wordnet', 10),
|
||||||
|
(9, 'noun', 'dia do mês', 'wiktionary', 20),
|
||||||
|
(9, 'noun', 'momento no tempo', 'wiktionary', 30),
|
||||||
|
(9, 'noun', 'um terceiro sentido', 'wiktionary', 40)`)
|
||||||
|
exec(`INSERT INTO translations (word_id, translation, target_lang, source) VALUES
|
||||||
|
(9, 'date', 'en', 'kaikki')`)
|
||||||
|
exec(`INSERT INTO pronunciations (word_id, format, value, source) VALUES
|
||||||
|
(9, 'ipa', '/ˈdatɐ/', 'wiktionary')`)
|
||||||
|
|
||||||
|
exec(`INSERT INTO etymology (word_id, text, source) VALUES
|
||||||
|
(1, 'From Medieval Latin ephemerus, from Ancient Greek ἐφήμερος (ephḗmeros, "lasting only a day"), from ἐπί (epí, "upon") and ἡμέρα (hēméra, "day"). The sense of transience is attested in English from the late sixteenth century onwards.', 'wiktionary')`)
|
||||||
|
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func openFixture(t *testing.T) *DreamDict {
|
||||||
|
t.Helper()
|
||||||
|
dd, err := OpenDreamDict(writeFixture(t, true))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("OpenDreamDict: %v", err)
|
||||||
|
}
|
||||||
|
if dd == nil {
|
||||||
|
t.Fatal("OpenDreamDict returned no dictionary for a seeded file")
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { dd.Close() })
|
||||||
|
return dd
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenMissingFileIsNotAnError(t *testing.T) {
|
||||||
|
dd, err := OpenDreamDict(filepath.Join(t.TempDir(), "absent.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a missing dict.db must not be an error: %v", err)
|
||||||
|
}
|
||||||
|
if dd != nil {
|
||||||
|
t.Fatal("a missing dict.db must yield no dictionary")
|
||||||
|
}
|
||||||
|
// An unset path is the laptop default and must behave the same way.
|
||||||
|
if dd, err := OpenDreamDict(""); err != nil || dd != nil {
|
||||||
|
t.Fatalf(`OpenDreamDict("") = %v, %v; want nil, nil`, dd, err)
|
||||||
|
}
|
||||||
|
// Close on the nil handle is what main.go defers unconditionally.
|
||||||
|
if err := dd.Close(); err != nil {
|
||||||
|
t.Fatalf("Close on absent dictionary: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenPresentButUnseededIsAnError(t *testing.T) {
|
||||||
|
// A file that exists but was never imported is somebody's mistake — a
|
||||||
|
// half-finished deploy — and must be loud, unlike a file that isn't there.
|
||||||
|
dd, err := OpenDreamDict(writeFixture(t, false))
|
||||||
|
if err == nil {
|
||||||
|
dd.Close()
|
||||||
|
t.Fatal("an unseeded dict.db must report an error")
|
||||||
|
}
|
||||||
|
if dd != nil {
|
||||||
|
t.Fatal("an unseeded dict.db must not yield a usable dictionary")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenUnreadablePathIsAnError(t *testing.T) {
|
||||||
|
// Not a missing file: a directory where dict.db should be. Distinguishing
|
||||||
|
// this from ErrNotExist is the whole point of the stat.
|
||||||
|
dir := filepath.Join(t.TempDir(), "dict.db")
|
||||||
|
if err := os.Mkdir(dir, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := OpenDreamDict(dir); err == nil {
|
||||||
|
t.Fatal("a directory in place of dict.db must report an error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDreamLookupFillsEveryField(t *testing.T) {
|
||||||
|
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||||
|
res, err := p.Lookup("ephemeral")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Lookup: %v", err)
|
||||||
|
}
|
||||||
|
if res.Gloss != "efémero; passageiro" {
|
||||||
|
t.Errorf("Gloss = %q, want the pt-PT translations joined", res.Gloss)
|
||||||
|
}
|
||||||
|
if res.Phonetic != "ɪˈfɛm.ər.əl" {
|
||||||
|
t.Errorf("Phonetic = %q, want the IPA without its slashes", res.Phonetic)
|
||||||
|
}
|
||||||
|
if len(res.Definitions) != maxDefinitions {
|
||||||
|
t.Fatalf("Definitions = %d, want them capped at %d", len(res.Definitions), maxDefinitions)
|
||||||
|
}
|
||||||
|
if res.Definitions[0].Definition != "lasting a very short time" {
|
||||||
|
t.Errorf("first definition = %q, want the curated (wordnet) sense first",
|
||||||
|
res.Definitions[0].Definition)
|
||||||
|
}
|
||||||
|
if res.Definitions[0].PartOfSpeech != "adjective" {
|
||||||
|
t.Errorf("part of speech = %q, want adjective", res.Definitions[0].PartOfSpeech)
|
||||||
|
}
|
||||||
|
if len(res.Synonyms) != 2 {
|
||||||
|
t.Errorf("Synonyms = %v, want both", res.Synonyms)
|
||||||
|
}
|
||||||
|
if res.Frequency != 50 {
|
||||||
|
t.Errorf("Frequency = %d, want 50", res.Frequency)
|
||||||
|
}
|
||||||
|
if res.Difficulty != 0.72 {
|
||||||
|
t.Errorf("Difficulty = %v, want 0.72", res.Difficulty)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(res.Etymology, "From Medieval Latin ephemerus") {
|
||||||
|
t.Errorf("Etymology = %q, want the Wiktionary text", res.Etymology)
|
||||||
|
}
|
||||||
|
if len(res.Etymology) > maxEtymology {
|
||||||
|
t.Errorf("Etymology not trimmed: %d chars", len(res.Etymology))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDreamGlossFollowsTheWriterNotTheWord(t *testing.T) {
|
||||||
|
dd := openFixture(t)
|
||||||
|
for lang, want := range map[string]string{
|
||||||
|
"pt-PT": "efémero; passageiro",
|
||||||
|
"fr": "éphémère",
|
||||||
|
"es": "", // DreamDict supports Spanish; this database wasn't built with it
|
||||||
|
"de": "", // never a Petal pair, and must not silently borrow another's
|
||||||
|
} {
|
||||||
|
got, err := dreamProvider{dict: dd, native: lang}.Gloss("ephemeral")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Gloss(%s): %v", lang, err)
|
||||||
|
}
|
||||||
|
if got.Gloss != want {
|
||||||
|
t.Errorf("Gloss for %s = %q, want %q", lang, got.Gloss, want)
|
||||||
|
}
|
||||||
|
if got.Word != "ephemeral" {
|
||||||
|
t.Errorf("Word = %q, want the word as asked", got.Word)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDreamGlossesThroughSharedSynsets(t *testing.T) {
|
||||||
|
// The measurement that drove this: on the real dict.db, Wiktionary's
|
||||||
|
// en→pt-PT translation table answers for 17% of the 2,000 commonest English
|
||||||
|
// words and the shared-synset path answers for 62%. A word with no
|
||||||
|
// translation row must still get a gloss, commonest sense first.
|
||||||
|
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||||
|
res, err := p.Lookup("brief")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Lookup: %v", err)
|
||||||
|
}
|
||||||
|
if res.Gloss != "breve; sucinto" {
|
||||||
|
t.Errorf("Gloss = %q, want the synset equivalents, commonest first", res.Gloss)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDreamDeinflectsToTheHeadword(t *testing.T) {
|
||||||
|
// dict.db stores headwords: "running" has no row of its own. The candidate
|
||||||
|
// walk is what makes a right-click on real prose work at all.
|
||||||
|
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||||
|
res, err := p.Lookup("running")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Lookup: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Definitions) == 0 || res.Definitions[0].Definition != "move fast on foot" {
|
||||||
|
t.Fatalf("Definitions = %+v, want run's", res.Definitions)
|
||||||
|
}
|
||||||
|
// Every other field must come from the same headword — a popover that mixed
|
||||||
|
// "running"'s (absent) frequency with "run"'s definitions would be lying.
|
||||||
|
if res.Frequency != 900 {
|
||||||
|
t.Errorf("Frequency = %d, want run's 900", res.Frequency)
|
||||||
|
}
|
||||||
|
if res.Difficulty != 0.05 {
|
||||||
|
t.Errorf("Difficulty = %v, want run's 0.05", res.Difficulty)
|
||||||
|
}
|
||||||
|
if res.Synonyms[0] != "sprint" {
|
||||||
|
t.Errorf("Synonyms = %v, want run's", res.Synonyms)
|
||||||
|
}
|
||||||
|
if res.Gloss != "correr" {
|
||||||
|
t.Errorf("Gloss = %q, want run's", res.Gloss)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDreamMissIsAnEmptyResultNotAnError(t *testing.T) {
|
||||||
|
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||||
|
res, err := p.Lookup("zzzxqqq")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Lookup: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Definitions) != 0 || len(res.Synonyms) != 0 || res.Gloss != "" {
|
||||||
|
t.Errorf("expected an empty result, got %+v", res)
|
||||||
|
}
|
||||||
|
// The frontend renders [] and never null.
|
||||||
|
if res.Definitions == nil || res.Synonyms == nil {
|
||||||
|
t.Errorf("empty slices must be non-nil: %+v", res)
|
||||||
|
}
|
||||||
|
if res.Difficulty != unknownDifficulty {
|
||||||
|
t.Errorf("Difficulty = %v, want the unknown sentinel", res.Difficulty)
|
||||||
|
}
|
||||||
|
// Empty input is a miss, not a crash.
|
||||||
|
if res, err := p.Lookup(" "); err != nil || res.Gloss != "" {
|
||||||
|
t.Errorf("Lookup(blank) = %+v, %v", res, err)
|
||||||
|
}
|
||||||
|
if res, err := p.Gloss(""); err != nil || res.Gloss != "" {
|
||||||
|
t.Errorf("Gloss(empty) = %+v, %v", res, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDreamUnscoredWordKeepsTheUnknownSentinel(t *testing.T) {
|
||||||
|
// "plain" is in the database with no frequency and a NULL difficulty. The
|
||||||
|
// popover must be able to tell that apart from "difficulty 0.0, the easiest
|
||||||
|
// word there is" — which is why the sentinel is -1 and not omitempty.
|
||||||
|
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||||
|
res, err := p.Lookup("plain")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Lookup: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Definitions) == 0 {
|
||||||
|
t.Fatal("expected plain to be found")
|
||||||
|
}
|
||||||
|
if res.Difficulty != unknownDifficulty {
|
||||||
|
t.Errorf("Difficulty = %v, want the unknown sentinel for a NULL score", res.Difficulty)
|
||||||
|
}
|
||||||
|
if res.Frequency != 0 {
|
||||||
|
t.Errorf("Frequency = %d, want 0 for no count", res.Frequency)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPickIPASkipsMachineFormats(t *testing.T) {
|
||||||
|
if got := pickIPA([]dictionary.Pronunciation{{Format: "cmu", Value: "K AE1 T"}}); got != "" {
|
||||||
|
t.Errorf("pickIPA on CMU alone = %q, want empty — CMU is not for a reader", got)
|
||||||
|
}
|
||||||
|
if got := pickIPA(nil); got != "" {
|
||||||
|
t.Errorf("pickIPA(nil) = %q", got)
|
||||||
|
}
|
||||||
|
if got := pickIPA([]dictionary.Pronunciation{{Format: "IPA", Value: " [kæt] "}}); got != "kæt" {
|
||||||
|
t.Errorf("pickIPA = %q, want the bare IPA regardless of case or brackets", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTrimEtymologyPrefersASentence(t *testing.T) {
|
||||||
|
// A sentence that ends past halfway is the good cut: keep it, drop the rest.
|
||||||
|
long := strings.Repeat("padding word ", 14) + "end. " + strings.Repeat("more ", 40)
|
||||||
|
got := trimEtymology(long)
|
||||||
|
if utf8.RuneCountInString(got) > maxEtymology {
|
||||||
|
t.Errorf("not trimmed: %d runes", utf8.RuneCountInString(got))
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(got, "end.") {
|
||||||
|
t.Errorf("trimEtymology = %q, want it to stop at the sentence", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An early full stop is not a summary — cutting there would throw away
|
||||||
|
// almost the whole line — so this falls through to a word boundary.
|
||||||
|
got = trimEtymology("From Latin. " + strings.Repeat("padding word ", 40))
|
||||||
|
if strings.HasSuffix(got, "Latin.") {
|
||||||
|
t.Errorf("trimEtymology = %q, want more than the first four words", got)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(got, "…") {
|
||||||
|
t.Errorf("trimEtymology = %q, want an ellipsis when cut mid-thought", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Multi-byte text must be cut on rune boundaries: a byte slice through
|
||||||
|
// ἐφήμερος would put invalid UTF-8 in the JSON.
|
||||||
|
greek := trimEtymology(strings.Repeat("ἐφήμερος ", 60))
|
||||||
|
if !utf8.ValidString(greek) {
|
||||||
|
t.Errorf("trimEtymology produced invalid UTF-8: %q", greek)
|
||||||
|
}
|
||||||
|
if n := utf8.RuneCountInString(greek); n > maxEtymology {
|
||||||
|
t.Errorf("trimmed to %d runes, want at most %d", n, maxEtymology)
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(got, " ") || strings.Contains(trimEtymology("a\n b"), "\n") {
|
||||||
|
t.Error("whitespace should be collapsed to a single line")
|
||||||
|
}
|
||||||
|
// Short text passes through untouched.
|
||||||
|
if got := trimEtymology("From Old English."); got != "From Old English." {
|
||||||
|
t.Errorf("trimEtymology = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- the Set: which provider answers, and what happens when dict.db is absent
|
||||||
|
|
||||||
|
func TestSetRoutesByPairLanguage(t *testing.T) {
|
||||||
|
set := NewSet(openFixture(t))
|
||||||
|
if !set.HasDreamDict() {
|
||||||
|
t.Fatal("HasDreamDict = false with a dictionary open")
|
||||||
|
}
|
||||||
|
// zh — and an empty column, which is what a pre-auth row reads as — stays
|
||||||
|
// on the embedded datasets until the two have been compared on real
|
||||||
|
// lookups. This test is the guard on that decision.
|
||||||
|
for _, lang := range []string{"", LangZh} {
|
||||||
|
if _, ok := set.For(lang).(*Lexicon); !ok {
|
||||||
|
t.Errorf("For(%q) = %T, want the embedded Lexicon", lang, set.For(lang))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, lang := range []string{"pt-PT", "fr", "es"} {
|
||||||
|
p, ok := set.For(lang).(dreamProvider)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("For(%q) = %T, want DreamDict", lang, set.For(lang))
|
||||||
|
}
|
||||||
|
if p.native != lang {
|
||||||
|
t.Errorf("For(%q) glosses into %q", lang, p.native)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetWithoutDictKeepsTheEnglishHalf(t *testing.T) {
|
||||||
|
// The interesting degradation: dict.db never got deployed. A pt-PT writer
|
||||||
|
// should still get definitions, synonyms and phonetics — all compiled into
|
||||||
|
// the binary and all correct for her — and lose only the translation.
|
||||||
|
set := NewSet(nil)
|
||||||
|
if set.HasDreamDict() {
|
||||||
|
t.Fatal("HasDreamDict = true with no dictionary")
|
||||||
|
}
|
||||||
|
p := set.For("pt-PT")
|
||||||
|
res, err := p.Lookup("happy")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Lookup: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Definitions) == 0 || len(res.Synonyms) == 0 {
|
||||||
|
t.Error("expected the embedded English half to survive a missing dict.db")
|
||||||
|
}
|
||||||
|
if res.Gloss != "" {
|
||||||
|
t.Errorf("Gloss = %q — a pt-PT writer must never be handed the Chinese gloss", res.Gloss)
|
||||||
|
}
|
||||||
|
g, err := p.Gloss("happy")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Gloss: %v", err)
|
||||||
|
}
|
||||||
|
if g.Gloss != "" {
|
||||||
|
t.Errorf("Gloss = %q, want empty", g.Gloss)
|
||||||
|
}
|
||||||
|
if g.Word != "happy" {
|
||||||
|
t.Errorf("Word = %q, want the word as asked", g.Word)
|
||||||
|
}
|
||||||
|
// The zh writer is untouched by any of this.
|
||||||
|
zh, err := set.For(LangZh).Lookup("happy")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Lookup(zh): %v", err)
|
||||||
|
}
|
||||||
|
if zh.Gloss == "" {
|
||||||
|
t.Error("the zh pair must keep its embedded gloss with no dict.db")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- the handler: the pair language is read per request, from the caller's row
|
||||||
|
|
||||||
|
func mountLexicon(t *testing.T, set *Set) (*chi.Mux, *db.DB) {
|
||||||
|
t.Helper()
|
||||||
|
database, err := db.Open(filepath.Join(t.TempDir(), "petal.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("db.Open: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { database.Close() })
|
||||||
|
|
||||||
|
for _, u := range []struct{ id, lang string }{
|
||||||
|
{"alice", LangZh}, {"bob", "pt-PT"},
|
||||||
|
} {
|
||||||
|
if _, err := database.Exec(
|
||||||
|
`INSERT INTO users (id, email, display_name, pair_lang) VALUES (?, ?, ?, ?)`,
|
||||||
|
u.id, u.id+"@example.com", u.id, u.lang,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("seed user: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
h := NewHandler(database.DB, set)
|
||||||
|
r := chi.NewMux()
|
||||||
|
r.Mount("/word", h.Routes())
|
||||||
|
r.Mount("/gloss", h.GlossRoutes())
|
||||||
|
return r, database
|
||||||
|
}
|
||||||
|
|
||||||
|
func getAs(t *testing.T, r http.Handler, userID, path string) Result {
|
||||||
|
t.Helper()
|
||||||
|
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||||
|
req = req.WithContext(auth.WithUser(req.Context(), userID))
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
r.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("GET %s as %s = %d: %s", path, userID, rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
var res Result
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &res); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerGlossesInTheCallersLanguage(t *testing.T) {
|
||||||
|
r, _ := mountLexicon(t, NewSet(openFixture(t)))
|
||||||
|
|
||||||
|
// Same URL, two writers, two languages. This is why the response is no
|
||||||
|
// longer cacheable as `public`.
|
||||||
|
bob := getAs(t, r, "bob", "/word/ephemeral")
|
||||||
|
if bob.Gloss != "efémero; passageiro" {
|
||||||
|
t.Errorf("bob's gloss = %q, want pt-PT", bob.Gloss)
|
||||||
|
}
|
||||||
|
alice := getAs(t, r, "alice", "/word/ephemeral")
|
||||||
|
if !strings.ContainsAny(alice.Gloss, "短暂的") && alice.Gloss != "" {
|
||||||
|
// alice is on the embedded ECDICT dataset, not the fixture's zh row —
|
||||||
|
// what matters is that she is *not* served bob's Portuguese.
|
||||||
|
t.Logf("alice's embedded gloss: %q", alice.Gloss)
|
||||||
|
}
|
||||||
|
if alice.Gloss == bob.Gloss && bob.Gloss != "" {
|
||||||
|
t.Error("the zh writer was served the pt-PT gloss")
|
||||||
|
}
|
||||||
|
if strings.Contains(alice.Gloss, "efémero") {
|
||||||
|
t.Errorf("alice's gloss = %q, want the embedded Chinese one", alice.Gloss)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerUnknownCallerFallsBackRatherThanFailing(t *testing.T) {
|
||||||
|
// No session, or a user row that has gone: the lookup still answers, from
|
||||||
|
// the embedded datasets. A dictionary that fails closed would be worse than
|
||||||
|
// one that answers in the wrong language, because nothing at all is not a
|
||||||
|
// dictionary.
|
||||||
|
r, _ := mountLexicon(t, NewSet(openFixture(t)))
|
||||||
|
res := getAs(t, r, "nobody", "/word/happy")
|
||||||
|
if len(res.Definitions) == 0 {
|
||||||
|
t.Error("expected the embedded fallback to answer for an unknown caller")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerCachesPrivately(t *testing.T) {
|
||||||
|
r, _ := mountLexicon(t, NewSet(nil))
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/gloss/happy", nil)
|
||||||
|
req = req.WithContext(auth.WithUser(req.Context(), "alice"))
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
r.ServeHTTP(rec, req)
|
||||||
|
if got := rec.Header().Get("Cache-Control"); !strings.HasPrefix(got, "private") {
|
||||||
|
t.Errorf("Cache-Control = %q — a per-writer gloss must not go in a shared cache", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerDecodesPunctuatedWords(t *testing.T) {
|
||||||
|
r, _ := mountLexicon(t, NewSet(openFixture(t)))
|
||||||
|
res := getAs(t, r, "bob", "/word/"+"caf%C3%A9")
|
||||||
|
if res.Word != "café" {
|
||||||
|
t.Errorf("Word = %q, want the decoded word", res.Word)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContentsCountsRowsNotSupportedLanguages(t *testing.T) {
|
||||||
|
// The fixture is seeded with English and pt-PT only. DreamDict *supports*
|
||||||
|
// French, Spanish and Chinese too — and a startup line that reported the
|
||||||
|
// supported set would have named all five while every French lookup came
|
||||||
|
// back empty. That is the failure this log line exists to catch, so it must
|
||||||
|
// count rows.
|
||||||
|
got := NewSet(openFixture(t))
|
||||||
|
summary := got.Contents()
|
||||||
|
if !strings.Contains(summary, "en=") || !strings.Contains(summary, "pt-PT=") {
|
||||||
|
t.Errorf("Contents = %q, want the languages the fixture actually holds", summary)
|
||||||
|
}
|
||||||
|
for _, absent := range []string{"fr=", "es=", "zh="} {
|
||||||
|
if strings.Contains(summary, absent) {
|
||||||
|
t.Errorf("Contents = %q, must not name %q — no rows exist for it", summary, absent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// No dictionary at all still has to answer something printable.
|
||||||
|
if s := NewSet(nil).Contents(); s == "" {
|
||||||
|
t.Error("Contents with no dictionary must still say something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The Latin+Latin wrinkle (SUGGESTIONS.md §3a). An English+Chinese pair never
|
||||||
|
// had to decide which language a word was in — the script decided. An
|
||||||
|
// English+Portuguese pair has no script boundary, and "data", "sale", "comum"
|
||||||
|
// and "tarde" are real words on both sides of it. Petal asks both directions
|
||||||
|
// and shows whatever answers, which needs no language detector and therefore
|
||||||
|
// cannot be wrong about somebody's writing.
|
||||||
|
func TestDreamLookupShowsBothReadingsOnACollision(t *testing.T) {
|
||||||
|
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||||
|
res, err := p.Lookup("data")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Lookup: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The English reading is unchanged and still leads.
|
||||||
|
if len(res.Definitions) == 0 || res.Definitions[0].Definition != "facts collected for reference" {
|
||||||
|
t.Fatalf("Definitions = %v, want the English sense first", res.Definitions)
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.Reverse == nil {
|
||||||
|
t.Fatal("Reverse = nil; a word that exists in both languages must carry both readings")
|
||||||
|
}
|
||||||
|
if res.Reverse.Lang != "pt-PT" {
|
||||||
|
t.Errorf("Reverse.Lang = %q, want the writer's language", res.Reverse.Lang)
|
||||||
|
}
|
||||||
|
if res.Reverse.Gloss != "date" {
|
||||||
|
t.Errorf("Reverse.Gloss = %q, want the English meaning of the Portuguese word", res.Reverse.Gloss)
|
||||||
|
}
|
||||||
|
if res.Reverse.Phonetic != "ˈdatɐ" {
|
||||||
|
t.Errorf("Reverse.Phonetic = %q, want the Portuguese IPA without slashes", res.Reverse.Phonetic)
|
||||||
|
}
|
||||||
|
// The reverse reading is a footnote on a card that already has an English
|
||||||
|
// half, so it is capped harder than the main entry.
|
||||||
|
if len(res.Reverse.Definitions) != maxReverseDefinitions {
|
||||||
|
t.Fatalf("Reverse.Definitions = %d, want %d", len(res.Reverse.Definitions), maxReverseDefinitions)
|
||||||
|
}
|
||||||
|
if res.Reverse.Definitions[0].Definition != "dia do mês" {
|
||||||
|
t.Errorf("Reverse.Definitions[0] = %q, want the Portuguese sense",
|
||||||
|
res.Reverse.Definitions[0].Definition)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDreamLookupHasNoReverseForAnEnglishOnlyWord(t *testing.T) {
|
||||||
|
// Which is almost every word she looks up: she is writing English. A
|
||||||
|
// second block under every card would make the collision case invisible.
|
||||||
|
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||||
|
res, err := p.Lookup("ephemeral")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Lookup: %v", err)
|
||||||
|
}
|
||||||
|
if res.Reverse != nil {
|
||||||
|
t.Fatalf("Reverse = %+v, want none for a word that is only English", res.Reverse)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDreamGlossCarriesTheReverseReading(t *testing.T) {
|
||||||
|
// The hover tooltip takes the same both-directions rule in one line less
|
||||||
|
// space: the reverse *gloss* only, never the definitions.
|
||||||
|
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||||
|
|
||||||
|
g, err := p.Gloss("data")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Gloss: %v", err)
|
||||||
|
}
|
||||||
|
if g.Reverse != "date" {
|
||||||
|
t.Errorf("Gloss.Reverse = %q, want the English meaning of the Portuguese word", g.Reverse)
|
||||||
|
}
|
||||||
|
|
||||||
|
g, err = p.Gloss("ephemeral")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Gloss: %v", err)
|
||||||
|
}
|
||||||
|
if g.Reverse != "" {
|
||||||
|
t.Errorf("Gloss.Reverse = %q, want none for an English-only word", g.Reverse)
|
||||||
|
}
|
||||||
|
if g.Gloss != "efémero; passageiro" {
|
||||||
|
t.Errorf("Gloss = %q, want the forward gloss untouched", g.Gloss)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReverseIsSilentForTheEmbeddedProviders(t *testing.T) {
|
||||||
|
// The zh pair has no collisions and no DreamDict, and a writer with no
|
||||||
|
// dict.db at all falls through to `glossless`. Neither may start emitting a
|
||||||
|
// reverse block: the card would then claim a Chinese reading of an English
|
||||||
|
// word, which is worse than saying nothing.
|
||||||
|
set := NewSet(nil)
|
||||||
|
|
||||||
|
res, err := set.For(LangZh).Lookup("river")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Lookup: %v", err)
|
||||||
|
}
|
||||||
|
if res.Reverse != nil {
|
||||||
|
t.Errorf("embedded Reverse = %+v, want none", res.Reverse)
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err = set.For("pt-PT").Lookup("river")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Lookup: %v", err)
|
||||||
|
}
|
||||||
|
if res.Reverse != nil {
|
||||||
|
t.Errorf("glossless Reverse = %+v, want none", res.Reverse)
|
||||||
|
}
|
||||||
|
}
|
||||||
+115
-38
@@ -1,20 +1,28 @@
|
|||||||
package lexicon
|
package lexicon
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Handler serves the word-lookup endpoint backed by a single shared Lexicon.
|
// Handler serves the word-lookup endpoints. It holds the shared provider Set
|
||||||
|
// and the database, because which provider answers depends on who is asking.
|
||||||
type Handler struct {
|
type Handler struct {
|
||||||
Lex *Lexicon
|
Set *Set
|
||||||
|
db *sql.DB
|
||||||
}
|
}
|
||||||
|
|
||||||
// New constructs a Handler with a fresh (lazily-loaded) Lexicon.
|
// NewHandler constructs a Handler over a provider Set. db is used for one
|
||||||
func NewHandler() *Handler { return &Handler{Lex: New()} }
|
// thing: reading the caller's pair language.
|
||||||
|
func NewHandler(db *sql.DB, set *Set) *Handler { return &Handler{Set: set, db: db} }
|
||||||
|
|
||||||
// Routes returns the router mounted at /api/word. The word is a path segment so
|
// Routes returns the router mounted at /api/word. The word is a path segment so
|
||||||
// "/api/word/happy" reads naturally; it's URL-decoded to tolerate the rare
|
// "/api/word/happy" reads naturally; it's URL-decoded to tolerate the rare
|
||||||
@@ -26,56 +34,125 @@ func (h *Handler) Routes() chi.Router {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GlossRoutes returns the router mounted at /api/gloss — the lightweight
|
// GlossRoutes returns the router mounted at /api/gloss — the lightweight
|
||||||
// Chinese-only lookup behind the inline hover/select gloss. It shares the
|
// translation-only lookup behind the inline hover/select gloss. It shares the
|
||||||
// Handler's Lexicon, so the datasets still load just once.
|
// Handler's Set, so the embedded datasets and dict.db are still opened once.
|
||||||
func (h *Handler) GlossRoutes() chi.Router {
|
func (h *Handler) GlossRoutes() chi.Router {
|
||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
r.Get("/{word}", h.gloss)
|
r.Get("/{word}", h.gloss)
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
// lookup returns the definition + synonyms for one word. A word found in neither
|
// HanziRoutes returns the router mounted at /api/hanzi — a Chinese word to its
|
||||||
// dataset still returns 200 with empty lists, so the popover can show a friendly
|
// pinyin and English senses, for a writer going the other way through the zh
|
||||||
// "nothing found" rather than an error state.
|
// pair (`users.direction = 'learning_pair'`).
|
||||||
|
//
|
||||||
|
// It does not go through [Handler.providerFor], and that is not an oversight.
|
||||||
|
// providerFor picks a dictionary by the writer's *pair*, to answer "what does
|
||||||
|
// this English word mean in her language" — a question whose answer differs per
|
||||||
|
// pair. This endpoint asks the opposite question of exactly one language: it
|
||||||
|
// reads hanzi, and hanzi are Chinese whoever is looking them up. Routing it
|
||||||
|
// through the pair would add a database read per hover to choose between one
|
||||||
|
// option and itself.
|
||||||
|
//
|
||||||
|
// What no longer holds is the reason this used to give — that
|
||||||
|
// [auth.SupportsLearnerDirection] guarantees the caller is on the zh pair. Since
|
||||||
|
// Portuguese joined `learnerPairs` a learning_pair account may be Portuguese, so
|
||||||
|
// the guarantee now comes from the *caller*: the client only ever asks this
|
||||||
|
// route about a token its Chinese segmenter found, and that segmenter is loaded
|
||||||
|
// only for the zh pair (see useSegmenter in App.tsx). A stray lookup is still
|
||||||
|
// answered safely — a word the Chinese dictionary has never heard of is a 200
|
||||||
|
// with empty lists, exactly like any other miss.
|
||||||
|
func (h *Handler) HanziRoutes() chi.Router {
|
||||||
|
r := chi.NewRouter()
|
||||||
|
r.Get("/{word}", h.hanzi)
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// hanzi answers a Chinese word lookup. Like the other two, a miss is a 200 with
|
||||||
|
// empty lists — a hover that lands on a word the dictionary has never heard of
|
||||||
|
// is an ordinary thing to happen while reading, and the tooltip simply doesn't
|
||||||
|
// open.
|
||||||
|
func (h *Handler) hanzi(w http.ResponseWriter, r *http.Request) {
|
||||||
|
res, err := h.Set.Hanzi(pathWord(r))
|
||||||
|
if err != nil {
|
||||||
|
writeLookupErr(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeLookup(w, res)
|
||||||
|
}
|
||||||
|
|
||||||
|
// providerFor returns the provider for the caller's language pair.
|
||||||
|
//
|
||||||
|
// The pair language is read here rather than threaded down because a word
|
||||||
|
// lookup has no other query to piggyback on — unlike the document handlers,
|
||||||
|
// which take pair_lang from the row-scoped query that already proves
|
||||||
|
// ownership. It is one indexed primary-key read against a local SQLite file,
|
||||||
|
// which costs less than encoding the response it feeds.
|
||||||
|
//
|
||||||
|
// A read that fails, or a caller with no user row, resolves to the empty
|
||||||
|
// language, and [Set.For] maps that to today's embedded behaviour. Falling back
|
||||||
|
// to a working dictionary beats failing the lookup.
|
||||||
|
func (h *Handler) providerFor(ctx context.Context) Provider {
|
||||||
|
var lang string
|
||||||
|
if h.db != nil {
|
||||||
|
_ = h.db.QueryRowContext(ctx,
|
||||||
|
`SELECT COALESCE(pair_lang, '') FROM users WHERE id = ?`,
|
||||||
|
auth.UserID(ctx),
|
||||||
|
).Scan(&lang)
|
||||||
|
}
|
||||||
|
return h.Set.For(lang)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pathWord reads the {word} segment, URL-decoded.
|
||||||
|
func pathWord(r *http.Request) string {
|
||||||
|
word := chi.URLParam(r, "word")
|
||||||
|
if decoded, err := url.PathUnescape(word); err == nil {
|
||||||
|
word = decoded
|
||||||
|
}
|
||||||
|
return word
|
||||||
|
}
|
||||||
|
|
||||||
|
// lookup returns the definition + synonyms for one word. A word found in no
|
||||||
|
// dataset still returns 200 with empty lists, so the popover can show a
|
||||||
|
// friendly "nothing found" rather than an error state.
|
||||||
func (h *Handler) lookup(w http.ResponseWriter, r *http.Request) {
|
func (h *Handler) lookup(w http.ResponseWriter, r *http.Request) {
|
||||||
word := chi.URLParam(r, "word")
|
res, err := h.providerFor(r.Context()).Lookup(pathWord(r))
|
||||||
if decoded, err := url.PathUnescape(word); err == nil {
|
|
||||||
word = decoded
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := h.Lex.Lookup(word)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
writeLookupErr(w, err)
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
writeLookup(w, res)
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
// Word lookups are static for the life of the build; let the browser cache
|
|
||||||
// them so repeated right-clicks on the same word are instant.
|
|
||||||
w.Header().Set("Cache-Control", "public, max-age=86400")
|
|
||||||
_ = json.NewEncoder(w).Encode(res)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// gloss returns just the Chinese translation for one word. Like lookup, a miss
|
// gloss returns just the translation for one word. Like lookup, a miss is a 200
|
||||||
// is a 200 with an empty gloss so the hover tooltip can quietly skip rather than
|
// with an empty gloss so the hover tooltip can quietly skip rather than error.
|
||||||
// error.
|
|
||||||
func (h *Handler) gloss(w http.ResponseWriter, r *http.Request) {
|
func (h *Handler) gloss(w http.ResponseWriter, r *http.Request) {
|
||||||
word := chi.URLParam(r, "word")
|
res, err := h.providerFor(r.Context()).Gloss(pathWord(r))
|
||||||
if decoded, err := url.PathUnescape(word); err == nil {
|
|
||||||
word = decoded
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := h.Lex.Gloss(word)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
writeLookupErr(w, err)
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
writeLookup(w, res)
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeLookupErr answers a failed lookup. The real error is a dictionary or
|
||||||
|
// database fault — a file path, a SQLite message — and belongs in the log, not
|
||||||
|
// in a tooltip. The client treats any non-200 the same way, so nothing is lost
|
||||||
|
// by saying less.
|
||||||
|
func writeLookupErr(w http.ResponseWriter, err error) {
|
||||||
|
log.Printf("lexicon: lookup failed: %v", err)
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
w.Header().Set("Cache-Control", "public, max-age=86400")
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
_ = json.NewEncoder(w).Encode(res)
|
_ = json.NewEncoder(w).Encode(map[string]string{"error": "lookup failed"})
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeLookup(w http.ResponseWriter, v any) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
// A lookup is stable for the life of the deployment, so let the browser
|
||||||
|
// keep it — repeated right-clicks on the same word are then instant. It is
|
||||||
|
// `private` rather than `public` because the gloss is now in *her*
|
||||||
|
// language: a shared cache keyed on the URL alone would hand one writer
|
||||||
|
// another writer's language.
|
||||||
|
w.Header().Set("Cache-Control", "private, max-age=86400")
|
||||||
|
_ = json.NewEncoder(w).Encode(v)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,144 @@
|
|||||||
|
package lexicon
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"unicode"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The Chinese half of the lexicon: a word written in hanzi to its pinyin and
|
||||||
|
// English senses. This is the mirror image of `gloss` — that one reads English
|
||||||
|
// and answers in Chinese, for a Mandarin native practising English; this one
|
||||||
|
// reads Chinese and answers in English, for the other direction of the same
|
||||||
|
// pair (`users.direction = 'learning_pair'`).
|
||||||
|
//
|
||||||
|
// It is deliberately not folded into [Lexicon.load]. That method reads four
|
||||||
|
// datasets on the first lookup of any kind, and this one is 3.1 MB gzipped that
|
||||||
|
// only a learner-direction account will ever ask for — every other writer would
|
||||||
|
// pay the decompression and the resident memory for a map they never touch. Its
|
||||||
|
// own sync.Once means the cost lands on the first Chinese hover and nowhere
|
||||||
|
// else.
|
||||||
|
|
||||||
|
// HanziReading is one pronunciation of a word and the senses it carries in that
|
||||||
|
// pronunciation. A word usually has one; the ones that have two are why this is
|
||||||
|
// a list rather than a pair of strings. 得 is dé, "to obtain", *and* de, the
|
||||||
|
// particle that makes 说得很好 mean "speaks well" — a learner shown only the
|
||||||
|
// first has been told something false about the sentence in front of them.
|
||||||
|
type HanziReading struct {
|
||||||
|
Pinyin string `json:"pinyin"`
|
||||||
|
Senses string `json:"senses"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HanziChar is one character of a word that the dictionary could not answer as
|
||||||
|
// a whole. See [Lexicon.Hanzi].
|
||||||
|
type HanziChar struct {
|
||||||
|
Char string `json:"char"`
|
||||||
|
Pinyin string `json:"pinyin"`
|
||||||
|
Senses string `json:"senses"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HanziResult is what a Chinese word lookup answers. Readings is empty for a
|
||||||
|
// word the dictionary does not have, in which case Chars may carry the
|
||||||
|
// character-by-character reading instead.
|
||||||
|
type HanziResult struct {
|
||||||
|
Word string `json:"word"`
|
||||||
|
Readings []HanziReading `json:"readings"`
|
||||||
|
Chars []HanziChar `json:"chars"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type hanziStore struct {
|
||||||
|
once sync.Once
|
||||||
|
err error
|
||||||
|
// word → [[pinyin, senses], …], exactly as scripts/build_cedict.py writes it.
|
||||||
|
entries map[string][][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
var hanzi hanziStore
|
||||||
|
|
||||||
|
func (h *hanziStore) load() {
|
||||||
|
h.once.Do(func() {
|
||||||
|
if err := gunzipJSON(hanziGz, &h.entries); err != nil {
|
||||||
|
h.err = fmt.Errorf("load hanzi: %w", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxHanziChars caps the per-character fallback. A run longer than this is
|
||||||
|
// almost certainly a phrase the segmenter split badly rather than a word, and
|
||||||
|
// spelling out eight characters one at a time is a wall, not a hint.
|
||||||
|
const maxHanziChars = 6
|
||||||
|
|
||||||
|
// Hanzi returns the pinyin and English senses of a Chinese word.
|
||||||
|
//
|
||||||
|
// There is no de-inflection walk here, and its absence is a fact about the
|
||||||
|
// language rather than an omission: Chinese words do not inflect, so the
|
||||||
|
// candidate forms [lookupGloss] tries for "running" → "run" have no analogue.
|
||||||
|
// A lookup either hits the headword or it does not.
|
||||||
|
//
|
||||||
|
// What it does instead is fall back to the characters. The segmentation word
|
||||||
|
// list is a superset of this dictionary — every glossable word can be
|
||||||
|
// segmented, but jieba knows ordinary compounds CC-CEDICT has no entry for — so
|
||||||
|
// a hover really can land on a word with nothing to say about it. Chinese
|
||||||
|
// compounds are usually transparent from their parts (电脑 is "electric brain"),
|
||||||
|
// which makes the character reading a genuinely useful second answer rather
|
||||||
|
// than a consolation prize. It is returned as its own field so the surface can
|
||||||
|
// say which of the two it is showing; a caller that only wants whole words can
|
||||||
|
// ignore it.
|
||||||
|
func (l *Lexicon) Hanzi(word string) (HanziResult, error) {
|
||||||
|
hanzi.load()
|
||||||
|
if hanzi.err != nil {
|
||||||
|
return HanziResult{}, hanzi.err
|
||||||
|
}
|
||||||
|
|
||||||
|
norm := strings.TrimSpace(word)
|
||||||
|
res := HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}
|
||||||
|
if norm == "" {
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if rows, ok := hanzi.entries[norm]; ok {
|
||||||
|
res.Readings = toReadings(rows)
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
chars := []rune(norm)
|
||||||
|
if len(chars) < 2 || len(chars) > maxHanziChars {
|
||||||
|
// A single character that missed has no parts to fall back to, and a long
|
||||||
|
// run is not a word. Either way the honest answer is nothing.
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
for _, r := range chars {
|
||||||
|
if !unicode.Is(unicode.Han, r) {
|
||||||
|
// Mixed input (a stray letter or digit inside the run) is not something
|
||||||
|
// the character reading can explain, and guessing at the hanzi parts of
|
||||||
|
// it would be worse than silence.
|
||||||
|
return HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}, nil
|
||||||
|
}
|
||||||
|
rows, ok := hanzi.entries[string(r)]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
first := toReadings(rows)
|
||||||
|
if len(first) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
res.Chars = append(res.Chars, HanziChar{
|
||||||
|
Char: string(r),
|
||||||
|
Pinyin: first[0].Pinyin,
|
||||||
|
Senses: first[0].Senses,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func toReadings(rows [][]string) []HanziReading {
|
||||||
|
out := make([]HanziReading, 0, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
if len(row) < 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, HanziReading{Pinyin: row[0], Senses: row[1]})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
package lexicon
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The Chinese direction of the lexicon, against the real embedded asset — not a
|
||||||
|
// fixture. The dataset is built by scripts/build_cedict.py, which asserts its
|
||||||
|
// own invariants at build time; what these assert is that the *lookup* over it
|
||||||
|
// behaves, including on the entries the build script goes out of its way to keep.
|
||||||
|
|
||||||
|
func TestHanziLookup(t *testing.T) {
|
||||||
|
l := New()
|
||||||
|
|
||||||
|
res, err := l.Hanzi("公园")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("lookup 公园: %v", err)
|
||||||
|
}
|
||||||
|
if len(res.Readings) == 0 {
|
||||||
|
t.Fatal("公园 has no readings")
|
||||||
|
}
|
||||||
|
// Tone marks, not the numbered pinyin CC-CEDICT stores. The number is the
|
||||||
|
// storage format; the marks are what a learner reads.
|
||||||
|
if got := res.Readings[0].Pinyin; got != "gōngyuán" {
|
||||||
|
t.Errorf("公园 pinyin = %q, want gōngyuán", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(res.Readings[0].Senses, "park") {
|
||||||
|
t.Errorf("公园 senses = %q, want something about a park", res.Readings[0].Senses)
|
||||||
|
}
|
||||||
|
// A word answered whole says nothing about its characters — the fallback is
|
||||||
|
// the other branch, and sending both would double the payload of the common
|
||||||
|
// case to no purpose.
|
||||||
|
if len(res.Chars) != 0 {
|
||||||
|
t.Errorf("a whole-word hit also returned %d characters", len(res.Chars))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 得 is the reason readings are a list. Answered with only dé "to obtain", a
|
||||||
|
// learner hovering it in 说得很好 has been told something false about the
|
||||||
|
// sentence they are looking at.
|
||||||
|
func TestHanziParticleCarriesItsGrammaticalReading(t *testing.T) {
|
||||||
|
l := New()
|
||||||
|
|
||||||
|
for _, particle := range []string{"的", "地", "得"} {
|
||||||
|
res, err := l.Hanzi(particle)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("lookup %s: %v", particle, err)
|
||||||
|
}
|
||||||
|
var found bool
|
||||||
|
for _, r := range res.Readings {
|
||||||
|
if r.Pinyin == "de" {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Errorf("%s never reads as neutral \"de\": %+v", particle, res.Readings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The fallback the segmentation gap makes necessary: jieba knows ordinary
|
||||||
|
// compounds CC-CEDICT has no headword for, so a hover can land on a real word
|
||||||
|
// with no entry. Chinese compounds are usually transparent from their parts, so
|
||||||
|
// the characters are a real second answer.
|
||||||
|
func TestHanziFallsBackToCharacters(t *testing.T) {
|
||||||
|
l := New()
|
||||||
|
|
||||||
|
// Constructed rather than borrowed from the corpus: a word that CC-CEDICT
|
||||||
|
// *does* carry would test the other branch, and which compounds it happens to
|
||||||
|
// omit is not something a test should pin.
|
||||||
|
const made = "猫书"
|
||||||
|
if _, ok := hanzi.entries[made]; ok {
|
||||||
|
t.Skipf("%s has become a real headword; pick another compound", made)
|
||||||
|
}
|
||||||
|
res, err := l.Hanzi(made)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("lookup %s: %v", made, err)
|
||||||
|
}
|
||||||
|
if len(res.Readings) != 0 {
|
||||||
|
t.Fatalf("%s answered as a whole word: %+v", made, res.Readings)
|
||||||
|
}
|
||||||
|
if len(res.Chars) != 2 {
|
||||||
|
t.Fatalf("character fallback gave %d entries, want 2: %+v", len(res.Chars), res.Chars)
|
||||||
|
}
|
||||||
|
if res.Chars[0].Char != "猫" || !strings.Contains(res.Chars[0].Senses, "cat") {
|
||||||
|
t.Errorf("first character = %+v, want 猫 ~ cat", res.Chars[0])
|
||||||
|
}
|
||||||
|
if res.Chars[0].Pinyin != "māo" {
|
||||||
|
t.Errorf("猫 pinyin = %q, want māo", res.Chars[0].Pinyin)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHanziMisses(t *testing.T) {
|
||||||
|
l := New()
|
||||||
|
|
||||||
|
for name, word := range map[string]string{
|
||||||
|
// A single character with no entry has no parts to fall back to.
|
||||||
|
"lone unknown character": "龥",
|
||||||
|
"empty": "",
|
||||||
|
"whitespace": " ",
|
||||||
|
// Not Chinese at all: the English tokenizer owns these, and answering
|
||||||
|
// would mean guessing.
|
||||||
|
"english": "hello",
|
||||||
|
"mixed": "猫cat",
|
||||||
|
// Longer than a word: a bad segmentation, not something to spell out
|
||||||
|
// character by character.
|
||||||
|
"a whole clause": "我今天早上去公园跑步了",
|
||||||
|
} {
|
||||||
|
res, err := l.Hanzi(word)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", name, err)
|
||||||
|
}
|
||||||
|
if len(res.Readings) != 0 || len(res.Chars) != 0 {
|
||||||
|
t.Errorf("%s (%q) answered with %+v / %+v", name, word, res.Readings, res.Chars)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHanziEndpoint(t *testing.T) {
|
||||||
|
h := NewHandler(nil, NewSet(nil))
|
||||||
|
r := chi.NewRouter()
|
||||||
|
r.Mount("/hanzi", h.HanziRoutes())
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/"+"跑步", nil))
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d", w.Code)
|
||||||
|
}
|
||||||
|
var got HanziResult
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if got.Word != "跑步" || len(got.Readings) == 0 || got.Readings[0].Pinyin != "pǎobù" {
|
||||||
|
t.Fatalf("response = %+v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A miss is a 200 with empty lists, like the other two lookups — the tooltip
|
||||||
|
// quietly doesn't open rather than showing an error over her writing.
|
||||||
|
w = httptest.NewRecorder()
|
||||||
|
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/zzz", nil))
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("miss: status = %d, want 200", w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -29,14 +29,69 @@ type Result struct {
|
|||||||
Phonetic string `json:"phonetic"` // IPA for the English word; "" when absent
|
Phonetic string `json:"phonetic"` // IPA for the English word; "" when absent
|
||||||
Definitions []Meaning `json:"definitions"`
|
Definitions []Meaning `json:"definitions"`
|
||||||
Synonyms []string `json:"synonyms"`
|
Synonyms []string `json:"synonyms"`
|
||||||
|
|
||||||
|
// The fields below only ever come from DreamDict; the embedded datasets
|
||||||
|
// leave them at their unknown values, and the popover hides them.
|
||||||
|
|
||||||
|
// Frequency is how common the word is (higher = more common). 0 means
|
||||||
|
// unknown, which is DreamDict's own convention — a word it carries but has
|
||||||
|
// no corpus count for is indistinguishable from a word it doesn't carry,
|
||||||
|
// and the popover treats both the same way.
|
||||||
|
Frequency int `json:"frequency"`
|
||||||
|
// Difficulty runs 0.0 (easiest) to 1.0 (hardest); -1 means unknown. It is a
|
||||||
|
// sentinel rather than an omitted field because 0.0 is a real, meaningful
|
||||||
|
// score and `omitempty` would erase it.
|
||||||
|
Difficulty float64 `json:"difficulty"`
|
||||||
|
// Etymology is free-form Wiktionary prose, trimmed to a line. Where the
|
||||||
|
// word came from is a real hook for a writer whose own language shares
|
||||||
|
// Latin roots with English — "ephemeral" is much easier to keep once you
|
||||||
|
// have seen efémero next to it.
|
||||||
|
Etymology string `json:"etymology"`
|
||||||
|
|
||||||
|
// Reverse is the same token read as a word of the writer's own language,
|
||||||
|
// present only when it is one. Absent for every writer whose pair is not
|
||||||
|
// Latin-script, and for the overwhelming majority of words in one that is.
|
||||||
|
Reverse *Reverse `json:"reverse,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Reverse is a lookup in the other direction: the token treated as a word of the
|
||||||
|
// writer's language, translated into English.
|
||||||
|
//
|
||||||
|
// It exists because a Latin-script pair has no script boundary to tell the two
|
||||||
|
// halves apart. In English+Chinese, "which language is this word?" answers
|
||||||
|
// itself. In English+Portuguese it does not: *sale*, *casa*, *comum*, *tarde*
|
||||||
|
// and *ali* are all real words on both sides, and *chat* and *pain* are the
|
||||||
|
// French versions of the same trap.
|
||||||
|
//
|
||||||
|
// Petal does not guess. It asks both directions and shows whatever comes back,
|
||||||
|
// which needs no detector, cannot be wrong about someone's writing, and — for a
|
||||||
|
// learner — is more interesting than a correct guess would have been.
|
||||||
|
type Reverse struct {
|
||||||
|
// Lang is the language this reading is in, so the card can label it.
|
||||||
|
Lang string `json:"lang"`
|
||||||
|
// Gloss is the English meaning of the native-language word.
|
||||||
|
Gloss string `json:"gloss"`
|
||||||
|
// Definitions are the word's senses as written in the writer's own
|
||||||
|
// language — the monolingual half, for when the English gloss isn't enough.
|
||||||
|
Definitions []Meaning `json:"definitions,omitempty"`
|
||||||
|
// Phonetic is IPA for the native-language pronunciation; "" when absent.
|
||||||
|
Phonetic string `json:"phonetic,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// unknownDifficulty is the [Result.Difficulty] value meaning "no score",
|
||||||
|
// matching DreamDict's own -1 return.
|
||||||
|
const unknownDifficulty = -1
|
||||||
|
|
||||||
// GlossResult is the lightweight payload for the inline hover/select gloss: just
|
// GlossResult is the lightweight payload for the inline hover/select gloss: just
|
||||||
// the word and its Chinese translation, no definitions or synonyms. Kept small
|
// the word and its Chinese translation, no definitions or synonyms. Kept small
|
||||||
// so the hover tooltip is instant and trivially cacheable.
|
// so the hover tooltip is instant and trivially cacheable.
|
||||||
type GlossResult struct {
|
type GlossResult struct {
|
||||||
Word string `json:"word"`
|
Word string `json:"word"`
|
||||||
Gloss string `json:"gloss"`
|
Gloss string `json:"gloss"`
|
||||||
|
// Reverse is the English meaning of the word read as one of the writer's
|
||||||
|
// own language — the tooltip's half of the both-directions rule (see
|
||||||
|
// [Reverse]). Empty unless the token is a word in her language too.
|
||||||
|
Reverse string `json:"reverse,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// maxSynonyms caps how many synonyms we hand the popover, even though the dataset
|
// maxSynonyms caps how many synonyms we hand the popover, even though the dataset
|
||||||
@@ -95,7 +150,7 @@ func (l *Lexicon) Lookup(word string) (Result, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
norm := strings.ToLower(strings.TrimSpace(word))
|
norm := strings.ToLower(strings.TrimSpace(word))
|
||||||
res := Result{Word: word, Definitions: []Meaning{}, Synonyms: []string{}}
|
res := Result{Word: word, Definitions: []Meaning{}, Synonyms: []string{}, Difficulty: unknownDifficulty}
|
||||||
if norm == "" {
|
if norm == "" {
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
package lexicon
|
||||||
|
|
||||||
|
// A word lookup used to mean exactly one thing: the embedded datasets, which
|
||||||
|
// speak English and Mandarin and nothing else. That was fine while Petal had
|
||||||
|
// one writer. It stops being fine the moment a pt-PT writer right-clicks a
|
||||||
|
// word and gets a Chinese gloss.
|
||||||
|
//
|
||||||
|
// So the lookup becomes a seam. A [Provider] answers the same two questions the
|
||||||
|
// popover and the hover tooltip have always asked; which provider answers them
|
||||||
|
// depends on the writer's language pair, and [Set.For] is the only place that
|
||||||
|
// decision is made.
|
||||||
|
|
||||||
|
// Provider answers word lookups for one writer. The embedded datasets and
|
||||||
|
// DreamDict both satisfy it, and both treat a word they don't carry as an empty
|
||||||
|
// result rather than an error — a miss is an ordinary outcome of looking a word
|
||||||
|
// up, not a failure.
|
||||||
|
type Provider interface {
|
||||||
|
// Lookup returns the full popover payload: gloss, phonetic, definitions,
|
||||||
|
// synonyms, and whatever extras the provider carries.
|
||||||
|
Lookup(word string) (Result, error)
|
||||||
|
// Gloss returns just the writer's-language translation. It is the hover
|
||||||
|
// tooltip's fast path and skips everything else.
|
||||||
|
Gloss(word string) (GlossResult, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LangZh is the one pair language still served by the embedded datasets. Every
|
||||||
|
// other pair goes to DreamDict — see [Set.For] for why zh is held back.
|
||||||
|
const LangZh = "zh"
|
||||||
|
|
||||||
|
// langEN is the language DreamDict is asked about for definitions, synonyms and
|
||||||
|
// pronunciation. English is always the *target* language of the pair — what
|
||||||
|
// varies is the language the gloss is written in.
|
||||||
|
const langEN = "en"
|
||||||
|
|
||||||
|
// Set holds every provider Petal can serve a lookup from and picks between them
|
||||||
|
// by pair language. One Set is shared by the whole process: the embedded
|
||||||
|
// datasets load once, and dict.db is one read-only handle.
|
||||||
|
type Set struct {
|
||||||
|
embedded *Lexicon
|
||||||
|
// dream is nil when dict.db was not deployed. That is a supported state,
|
||||||
|
// not an error — see [Set.For].
|
||||||
|
dream *DreamDict
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSet returns a Set backed by the embedded datasets and, when dream is
|
||||||
|
// non-nil, DreamDict. Passing a nil dream is how Petal runs without dict.db.
|
||||||
|
func NewSet(dream *DreamDict) *Set {
|
||||||
|
return &Set{embedded: New(), dream: dream}
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasDreamDict reports whether a dict.db is open. Only startup logging and
|
||||||
|
// tests care; a handler never asks, because [Set.For] always returns something
|
||||||
|
// usable.
|
||||||
|
func (s *Set) HasDreamDict() bool { return s.dream != nil }
|
||||||
|
|
||||||
|
// Contents describes what the open dict.db actually holds, for the startup log.
|
||||||
|
// With no dictionary it says so rather than returning an empty string, because
|
||||||
|
// a blank in a log line is indistinguishable from a bug in the log line.
|
||||||
|
func (s *Set) Contents() string {
|
||||||
|
if s.dream == nil {
|
||||||
|
return "no dict.db — embedded datasets only"
|
||||||
|
}
|
||||||
|
return s.dream.Contents()
|
||||||
|
}
|
||||||
|
|
||||||
|
// For returns the provider that should answer lookups for a writer whose pair
|
||||||
|
// language is lang.
|
||||||
|
//
|
||||||
|
// Three rules, in order:
|
||||||
|
//
|
||||||
|
// zh — and an empty code, which is what a pre-Phase-16 row reads as — stays on
|
||||||
|
// the embedded ECDICT gloss. Not because DreamDict lacks Chinese (it has
|
||||||
|
// CC-CEDICT), but because that path is in daily use by a real writer and the
|
||||||
|
// two have not yet been compared on her actual lookups. Switching it is a
|
||||||
|
// quality decision, and it hasn't been made.
|
||||||
|
//
|
||||||
|
// Any other pair goes to DreamDict, which is the only source that has pt-PT,
|
||||||
|
// French or Spanish at all.
|
||||||
|
//
|
||||||
|
// If dict.db was never deployed, a non-zh writer falls back to the embedded
|
||||||
|
// datasets with the gloss suppressed. This is the interesting case: the naive
|
||||||
|
// "no data" answer would blank the popover entirely, when in fact the English
|
||||||
|
// half of it — definitions, synonyms, phonetic — is compiled into the binary
|
||||||
|
// and perfectly correct for her. Only the translation is missing, so only the
|
||||||
|
// translation goes missing. A failed dictionary deploy costs her the gloss, not
|
||||||
|
// the dictionary.
|
||||||
|
func (s *Set) For(lang string) Provider {
|
||||||
|
if lang == "" || lang == LangZh {
|
||||||
|
return s.embedded
|
||||||
|
}
|
||||||
|
if s.dream != nil {
|
||||||
|
return dreamProvider{dict: s.dream, native: lang}
|
||||||
|
}
|
||||||
|
return glossless{s.embedded}
|
||||||
|
}
|
||||||
|
|
||||||
|
// glossless serves the embedded datasets with the Chinese gloss stripped, for a
|
||||||
|
// writer who does not read Chinese. Handing her the zh gloss would be worse
|
||||||
|
// than handing her nothing: an empty field reads as "not found", where the
|
||||||
|
// wrong language reads as Petal being broken.
|
||||||
|
type glossless struct{ inner Provider }
|
||||||
|
|
||||||
|
func (g glossless) Lookup(word string) (Result, error) {
|
||||||
|
res, err := g.inner.Lookup(word)
|
||||||
|
res.Gloss = ""
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g glossless) Gloss(word string) (GlossResult, error) {
|
||||||
|
return GlossResult{Word: word}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hanzi answers a Chinese-word lookup from the embedded CC-CEDICT map.
|
||||||
|
//
|
||||||
|
// It is on the Set rather than on [Provider] because it is not the same
|
||||||
|
// question the other two ask. Lookup and Gloss vary by pair — which is why they
|
||||||
|
// are behind an interface with two implementations — while this one is asked of
|
||||||
|
// Chinese or not at all: the learner direction exists for exactly one pair (see
|
||||||
|
// auth.learnerPairs), and DreamDict's own CC-CEDICT would be a second copy of
|
||||||
|
// the same dictionary, chosen by a rule with one branch.
|
||||||
|
func (s *Set) Hanzi(word string) (HanziResult, error) { return s.embedded.Hanzi(word) }
|
||||||
@@ -41,9 +41,9 @@ type checkpointResponse struct {
|
|||||||
// RunCheckpoint sends the grammar checkpoint and parses the JSON result. It
|
// RunCheckpoint sends the grammar checkpoint and parses the JSON result. It
|
||||||
// applies the latency-guard truncation and the checkpoint sampling parameters
|
// applies the latency-guard truncation and the checkpoint sampling parameters
|
||||||
// from the spec.
|
// from the spec.
|
||||||
func RunCheckpoint(ctx context.Context, client LLMClient, contentText, tone string, _ Lang) ([]RawSuggestion, error) {
|
func RunCheckpoint(ctx context.Context, client LLMClient, contentText, tone string, t Target) ([]RawSuggestion, error) {
|
||||||
raw, err := client.Complete(ctx, CompletionRequest{
|
raw, err := client.Complete(ctx, CompletionRequest{
|
||||||
Messages: CheckpointMessages(TruncateDoc(contentText), tone),
|
Messages: CheckpointMessages(TruncateDoc(contentText), tone, t),
|
||||||
MaxTokens: checkpointMaxTokens,
|
MaxTokens: checkpointMaxTokens,
|
||||||
Temperature: 0.3,
|
Temperature: 0.3,
|
||||||
RepetitionPenalty: 1.15,
|
RepetitionPenalty: 1.15,
|
||||||
|
|||||||
@@ -20,9 +20,9 @@ const CollocationInterval = 25 * time.Second
|
|||||||
// The tone argument is accepted for a uniform pass signature and passed through
|
// The tone argument is accepted for a uniform pass signature and passed through
|
||||||
// to the prompt so a hint can prefer a register-appropriate pairing. `lang` is
|
// to the prompt so a hint can prefer a register-appropriate pairing. `lang` is
|
||||||
// the writer's pair language — the one each hint's short gloss is written in.
|
// the writer's pair language — the one each hint's short gloss is written in.
|
||||||
func RunCollocation(ctx context.Context, client LLMClient, contentText, tone string, lang Lang) ([]RawSuggestion, error) {
|
func RunCollocation(ctx context.Context, client LLMClient, contentText, tone string, t Target) ([]RawSuggestion, error) {
|
||||||
raw, err := client.Complete(ctx, CompletionRequest{
|
raw, err := client.Complete(ctx, CompletionRequest{
|
||||||
Messages: CollocationMessages(contentText, tone, lang),
|
Messages: CollocationMessages(contentText, tone, t),
|
||||||
MaxTokens: 2048,
|
MaxTokens: 2048,
|
||||||
Temperature: 0.3,
|
Temperature: 0.3,
|
||||||
RepetitionPenalty: 1.15,
|
RepetitionPenalty: 1.15,
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ func TestLangForFallsBackToDefault(t *testing.T) {
|
|||||||
func TestPromptsNameTheWritersLanguage(t *testing.T) {
|
func TestPromptsNameTheWritersLanguage(t *testing.T) {
|
||||||
pt := LangFor("pt-PT")
|
pt := LangFor("pt-PT")
|
||||||
|
|
||||||
collocation := CollocationMessages("The rain was strong.", "casual", pt)[0].Content
|
collocation := CollocationMessages("The rain was strong.", "casual", EnglishTarget(pt))[0].Content
|
||||||
if !strings.Contains(collocation, "European Portuguese") {
|
if !strings.Contains(collocation, "European Portuguese") {
|
||||||
t.Fatalf("collocation prompt doesn't ask for a pt-PT gloss:\n%s", collocation)
|
t.Fatalf("collocation prompt doesn't ask for a pt-PT gloss:\n%s", collocation)
|
||||||
}
|
}
|
||||||
@@ -44,7 +44,7 @@ func TestPromptsNameTheWritersLanguage(t *testing.T) {
|
|||||||
t.Fatalf("collocation prompt lost its tone guidance:\n%s", collocation)
|
t.Fatalf("collocation prompt lost its tone guidance:\n%s", collocation)
|
||||||
}
|
}
|
||||||
|
|
||||||
translate := TranslateMessages("Try a shorter sentence here.", pt)[0].Content
|
translate := TranslateMessages("Try a shorter sentence here.", English, pt)[0].Content
|
||||||
if !strings.Contains(translate, "European Portuguese") || strings.Contains(translate, "Chinese") {
|
if !strings.Contains(translate, "European Portuguese") || strings.Contains(translate, "Chinese") {
|
||||||
t.Fatalf("translate prompt targets the wrong language:\n%s", translate)
|
t.Fatalf("translate prompt targets the wrong language:\n%s", translate)
|
||||||
}
|
}
|
||||||
@@ -73,13 +73,50 @@ func TestPromptsNameTheWritersLanguage(t *testing.T) {
|
|||||||
func TestDefaultPairStillReadsAsBefore(t *testing.T) {
|
func TestDefaultPairStillReadsAsBefore(t *testing.T) {
|
||||||
zh := LangFor("zh")
|
zh := LangFor("zh")
|
||||||
|
|
||||||
if got := CollocationMessages("x", "", zh)[0].Content; !strings.Contains(got, "Simplified Chinese (Mandarin) gloss in parentheses") {
|
if got := CollocationMessages("x", "", EnglishTarget(zh))[0].Content; !strings.Contains(got, "Simplified Chinese (Mandarin) gloss in parentheses") {
|
||||||
t.Fatalf("zh collocation gloss changed:\n%s", got)
|
t.Fatalf("zh collocation gloss changed:\n%s", got)
|
||||||
}
|
}
|
||||||
if got := TranslateMessages("x", zh)[0].Content; !strings.Contains(got, "natural, friendly Simplified Chinese (Mandarin)") {
|
if got := TranslateMessages("x", English, zh)[0].Content; !strings.Contains(got, "natural, friendly Simplified Chinese (Mandarin)") {
|
||||||
t.Fatalf("zh translate target changed:\n%s", got)
|
t.Fatalf("zh translate target changed:\n%s", got)
|
||||||
}
|
}
|
||||||
if got := AskPetalSystemPrompt("a", "b", "c", "d", "e", zh); !strings.Contains(got, "为什么") {
|
if got := AskPetalSystemPrompt("a", "b", "c", "d", "e", zh); !strings.Contains(got, "为什么") {
|
||||||
t.Fatalf("zh ask-petal lost its Mandarin \"why\":\n%s", got)
|
t.Fatalf("zh ask-petal lost its Mandarin \"why\":\n%s", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UX item 6: the Ask Petal answer is bilingual, pair language first, halves
|
||||||
|
// separated by one blank line. That separator is not a stylistic preference —
|
||||||
|
// AskPetal.tsx splits on it to render the two halves the way the companion
|
||||||
|
// renders its two lines — so the instruction has to survive prompt edits.
|
||||||
|
//
|
||||||
|
// The direction the writer is learning in is deliberately not encoded: the pair
|
||||||
|
// is (English + X), and an English speaker learning French needs the same two
|
||||||
|
// halves a Mandarin speaker learning English does. The prompt asks for both and
|
||||||
|
// lets the reader choose, so there is nothing here that names one half the
|
||||||
|
// answer and the other a courtesy.
|
||||||
|
func TestAskPetalAnswersInBothLanguages(t *testing.T) {
|
||||||
|
for _, code := range []string{"zh", "pt-PT", "fr", "es"} {
|
||||||
|
lang := LangFor(code)
|
||||||
|
ask := AskPetalSystemPrompt("a", "b", "grammar", "d", "e", lang)
|
||||||
|
|
||||||
|
if !strings.Contains(ask, "BOTH languages") {
|
||||||
|
t.Fatalf("%s: ask-petal no longer asks for both languages:\n%s", code, ask)
|
||||||
|
}
|
||||||
|
if !strings.Contains(ask, "single blank line") {
|
||||||
|
t.Fatalf("%s: ask-petal lost the blank-line separator the client splits on:\n%s", code, ask)
|
||||||
|
}
|
||||||
|
// Order matters to the rendering: the pair language is the prominent
|
||||||
|
// half, English the muted one beneath it.
|
||||||
|
if !strings.Contains(ask, "first the whole answer in "+lang.Name) {
|
||||||
|
t.Fatalf("%s: ask-petal doesn't put %s first:\n%s", code, lang.Name, ask)
|
||||||
|
}
|
||||||
|
// The instruction it replaced. Left in place it directly contradicts the
|
||||||
|
// new one, and a model given both will pick one at random.
|
||||||
|
if strings.Contains(ask, "Never mix languages") {
|
||||||
|
t.Fatalf("%s: ask-petal still forbids the bilingual reply it now asks for:\n%s", code, ask)
|
||||||
|
}
|
||||||
|
if strings.Contains(ask, "%!") {
|
||||||
|
t.Fatalf("%s: ask-petal prompt has a formatting error:\n%s", code, ask)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+143
-25
@@ -45,11 +45,55 @@ func toneGuidance(tone string) string {
|
|||||||
"be improved, prefer suggestions that fit that tone, and gently flag wording that clashes with it."
|
"be improved, prefer suggestions that fit that tone, and gently flag wording that clashes with it."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pairCheckpointSystemPrompt is the grammar checkpoint for a document written in
|
||||||
|
// the writer's own language rather than in English.
|
||||||
|
//
|
||||||
|
// It is a separate constant rather than a language clause appended to
|
||||||
|
// checkpointSystemPrompt, because that prompt opens by naming the reader as an
|
||||||
|
// ESL learner and asks for "common ESL patterns" — appending "and explain in
|
||||||
|
// Portuguese" would hand the model two contradictory framings. Only the framing
|
||||||
|
// differs; the JSON contract and the tone clause below it are the same
|
||||||
|
// instructions in the same order, so the two prompts stay comparable.
|
||||||
|
//
|
||||||
|
// The "never translate" line is the one the model most wants to disobey: asked
|
||||||
|
// to improve Portuguese while being an English writing assistant by training, it
|
||||||
|
// will happily hand back an English rendering, which is a translation card
|
||||||
|
// (Phase 25's `isTranslation`) and not a correction.
|
||||||
|
const pairCheckpointSystemPrompt = `You are a warm, encouraging writing assistant. The person you are helping is ` +
|
||||||
|
`writing in %[1]s, and the text below is %[1]s. ` +
|
||||||
|
`Analyze it and identify up to 5 issues: grammar errors, unnatural phrasing, ` +
|
||||||
|
`incorrect idiom usage, or unclear sentences.
|
||||||
|
|
||||||
|
Both "original" and "replacement" must be written in %[1]s. You are improving their %[1]s writing — ` +
|
||||||
|
`never translate it into English, and never suggest they write in English instead.
|
||||||
|
Write every "explanation" in %[2]s.
|
||||||
|
|
||||||
|
Be specific, friendly, and explain WHY each suggestion improves the writing.%[3]s
|
||||||
|
|
||||||
|
Respond ONLY with valid JSON. No preamble, no markdown fences. Format:
|
||||||
|
{
|
||||||
|
"suggestions": [
|
||||||
|
{
|
||||||
|
"original": "exact text from the document that needs fixing",
|
||||||
|
"replacement": "corrected version",
|
||||||
|
"explanation": "friendly one-sentence explanation",
|
||||||
|
"type": "grammar|phrasing|idiom|clarity"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
If the writing looks good, return: {"suggestions": []}`
|
||||||
|
|
||||||
// CheckpointMessages builds the message array for a grammar checkpoint over the
|
// CheckpointMessages builds the message array for a grammar checkpoint over the
|
||||||
// given (already-truncated) document text, steered toward the document's tone.
|
// given (already-truncated) document text, steered toward the document's tone
|
||||||
func CheckpointMessages(contentText, tone string) []Message {
|
// and aimed at the language the document is actually written in.
|
||||||
|
func CheckpointMessages(contentText, tone string, t Target) []Message {
|
||||||
|
system := fmt.Sprintf(checkpointSystemPrompt, toneGuidance(tone))
|
||||||
|
if t.Flipped() {
|
||||||
|
system = fmt.Sprintf(pairCheckpointSystemPrompt, t.Correct.Name, t.Explain.Name, toneGuidance(tone))
|
||||||
|
}
|
||||||
return []Message{
|
return []Message{
|
||||||
{Role: "system", Content: fmt.Sprintf(checkpointSystemPrompt, toneGuidance(tone))},
|
{Role: "system", Content: system},
|
||||||
{Role: "user", Content: contentText},
|
{Role: "user", Content: contentText},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -82,12 +126,48 @@ Respond ONLY with valid JSON. No preamble, no markdown fences. Format:
|
|||||||
|
|
||||||
If the voice is consistent throughout, return: {"suggestions": []}`
|
If the voice is consistent throughout, return: {"suggestions": []}`
|
||||||
|
|
||||||
|
// pairVoiceSystemPrompt is the voice pass for a document in the writer's own
|
||||||
|
// language. Voice consistency is the one pass that transfers across languages
|
||||||
|
// unchanged — a paragraph that reads as pasted from elsewhere reads that way in
|
||||||
|
// any language — so only the framing and the explanation language move.
|
||||||
|
const pairVoiceSystemPrompt = `You are a warm, encouraging writing assistant. The person you are helping is writing ` +
|
||||||
|
`in %[1]s. You are reviewing a COMPLETE %[1]s document for VOICE CONSISTENCY only — not grammar.
|
||||||
|
|
||||||
|
Read the whole document to learn the writer's natural voice, then identify any passages (2 or more sentences) ` +
|
||||||
|
`that feel tonally inconsistent with the surrounding writing — unusually formal, unusually polished, or phrased ` +
|
||||||
|
`in a way that differs from the writer's established voice elsewhere in the document. These often signal text ` +
|
||||||
|
`that was paraphrased too closely from another source. Do not flag the first paragraph (there is no baseline yet). ` +
|
||||||
|
`Do not flag grammar or spelling mistakes — only voice.
|
||||||
|
|
||||||
|
Quote each passage exactly as it appears, in %[1]s. Write every "explanation" in %[2]s.
|
||||||
|
|
||||||
|
Respond ONLY with valid JSON. No preamble, no markdown fences. Format:
|
||||||
|
{
|
||||||
|
"suggestions": [
|
||||||
|
{
|
||||||
|
"original": "exact passage from the document that feels inconsistent",
|
||||||
|
"replacement": null,
|
||||||
|
"explanation": "friendly one-sentence note about why this passage sounds unlike the rest",
|
||||||
|
"type": "voice"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
If the voice is consistent throughout, return: {"suggestions": []}`
|
||||||
|
|
||||||
// VoiceMessages builds the message array for a voice-consistency pass. Unlike
|
// VoiceMessages builds the message array for a voice-consistency pass. Unlike
|
||||||
// the checkpoint, the caller passes the WHOLE document (no truncation) — voice
|
// the checkpoint, the caller passes the WHOLE document (no truncation) — voice
|
||||||
// consistency is judged against the established voice everywhere else.
|
// consistency is judged against the established voice everywhere else.
|
||||||
func VoiceMessages(contentText string) []Message {
|
//
|
||||||
|
// The pass had no language argument at all before Phase 28, which was the same
|
||||||
|
// English assumption the checkpoint made, just unstated.
|
||||||
|
func VoiceMessages(contentText string, t Target) []Message {
|
||||||
|
system := voiceSystemPrompt
|
||||||
|
if t.Flipped() {
|
||||||
|
system = fmt.Sprintf(pairVoiceSystemPrompt, t.Correct.Name, t.Explain.Name)
|
||||||
|
}
|
||||||
return []Message{
|
return []Message{
|
||||||
{Role: "system", Content: voiceSystemPrompt},
|
{Role: "system", Content: system},
|
||||||
{Role: "user", Content: contentText},
|
{Role: "user", Content: contentText},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -133,10 +213,17 @@ If every pairing already sounds natural, return: {"suggestions": []}`
|
|||||||
// CollocationMessages builds the message array for a collocation pass over the
|
// CollocationMessages builds the message array for a collocation pass over the
|
||||||
// WHOLE document (no truncation), gently steered toward the document's tone so a
|
// WHOLE document (no truncation), gently steered toward the document's tone so a
|
||||||
// hint can prefer a register-appropriate pairing. The parenthetical gloss is
|
// hint can prefer a register-appropriate pairing. The parenthetical gloss is
|
||||||
// written in the writer's own language.
|
// written in the writer's own language — `Pair`, not `Explain`: the gloss is
|
||||||
func CollocationMessages(contentText, tone string, lang Lang) []Message {
|
// addressed to her rather than to the document.
|
||||||
|
//
|
||||||
|
// The coach itself remains English-only. Collocation lists are the one thing
|
||||||
|
// here that is genuinely per-language knowledge rather than framing, and
|
||||||
|
// "natives usually say" for Portuguese is a claim this prompt has no grounds to
|
||||||
|
// make yet; a flipped document simply gets the pass it always got. (Phase 28
|
||||||
|
// moved the checkpoint and the voice pass; this one waits for evidence.)
|
||||||
|
func CollocationMessages(contentText, tone string, t Target) []Message {
|
||||||
return []Message{
|
return []Message{
|
||||||
{Role: "system", Content: fmt.Sprintf(collocationSystemPrompt, toneGuidance(tone), lang.Name)},
|
{Role: "system", Content: fmt.Sprintf(collocationSystemPrompt, toneGuidance(tone), t.Pair.Name)},
|
||||||
{Role: "user", Content: contentText},
|
{Role: "user", Content: contentText},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -144,6 +231,25 @@ func CollocationMessages(contentText, tone string, lang Lang) []Message {
|
|||||||
// askPetalSystemTemplate is the Ask Petal tutor prompt. The suggestion context
|
// askPetalSystemTemplate is the Ask Petal tutor prompt. The suggestion context
|
||||||
// is interpolated in; the user's own messages are appended after this system
|
// is interpolated in; the user's own messages are appended after this system
|
||||||
// turn by the caller.
|
// turn by the caller.
|
||||||
|
//
|
||||||
|
// The reply is bilingual, the pair language first. Until UX item 6 it mirrored
|
||||||
|
// the language of the question instead — self-consistent, but it meant asking in
|
||||||
|
// one language cost you the other, and the writer doesn't always know which one
|
||||||
|
// the answer will be clearer in. Which half is the safety net and which is the
|
||||||
|
// lesson depends on who is writing: the pair is (English + X) either way, and an
|
||||||
|
// English speaker learning French wants the French half for the same reason a
|
||||||
|
// Mandarin speaker learning English wants the English one. Petal cannot tell
|
||||||
|
// them apart from a chat message, and doesn't need to — every other explanation
|
||||||
|
// surface already gives both (the card's English body, the seeded bubble in the
|
||||||
|
// pair language). The answer that goes deepest into the "why" was the one place
|
||||||
|
// that didn't.
|
||||||
|
//
|
||||||
|
// The blank line between the halves is a contract with the client: AskPetal.tsx
|
||||||
|
// splits on the first one to render her language prominently and the English
|
||||||
|
// beneath it, mirroring the companion's bubble. A model that ignores the
|
||||||
|
// instruction and writes one language degrades to a single plain block — the
|
||||||
|
// answer is still readable, which is why the split is a rendering nicety and
|
||||||
|
// never a parse the reply depends on.
|
||||||
const askPetalSystemTemplate = `You are Petal, a warm and patient English writing tutor helping someone who is learning English ` +
|
const askPetalSystemTemplate = `You are Petal, a warm and patient English writing tutor helping someone who is learning English ` +
|
||||||
`as a second language. You are currently discussing a specific writing suggestion.
|
`as a second language. You are currently discussing a specific writing suggestion.
|
||||||
|
|
||||||
@@ -154,15 +260,21 @@ Suggestion context:
|
|||||||
- Initial explanation: "%[4]s"
|
- Initial explanation: "%[4]s"
|
||||||
- Surrounding paragraph: "%[5]s"
|
- Surrounding paragraph: "%[5]s"
|
||||||
|
|
||||||
The user wants to understand this suggestion better. Detect the language of the user's message ` +
|
The user wants to understand this suggestion better. Answer in BOTH languages, every time, ` +
|
||||||
`and respond in that same language. If they write in %[6]s, respond entirely in ` +
|
`whichever language they asked their question in: first the whole answer in %[6]s, then the ` +
|
||||||
`%[6]s. If they write in English, respond in English. Never mix languages in a single response.
|
`same answer again in English. Separate the two with a single blank line. Do not label them, ` +
|
||||||
|
`do not use a blank line anywhere else, and do not mix the two languages within one half — ` +
|
||||||
|
`each half is complete on its own.
|
||||||
|
|
||||||
|
One of those two languages is the one they are surest in and the other is the one they are ` +
|
||||||
|
`working in — you do not know which way round, so give both and let them choose. Both halves ` +
|
||||||
|
`say the same thing: do not put a point in one that is missing from the other.
|
||||||
|
|
||||||
Explain clearly and kindly. Use simple language appropriate to the user's message. Give examples ` +
|
Explain clearly and kindly. Use simple language appropriate to the user's message. Give examples ` +
|
||||||
`when helpful. If they ask "why" (or "%[7]s"), explain the grammar rule or idiom behind it. ` +
|
`when helpful. If they ask "why" (or "%[7]s"), explain the grammar rule or idiom behind it. ` +
|
||||||
`If they suggest an alternative phrasing, evaluate it honestly.
|
`If they suggest an alternative phrasing, evaluate it honestly.
|
||||||
|
|
||||||
Keep responses concise (2-4 sentences). This is a chat, not an essay. Be encouraging — ` +
|
Keep each half concise (2-3 sentences). This is a chat, not an essay. Be encouraging — ` +
|
||||||
`learning a language is hard and they're doing great.`
|
`learning a language is hard and they're doing great.`
|
||||||
|
|
||||||
// AskPetalSystemPrompt fills the tutor prompt with one suggestion's context and
|
// AskPetalSystemPrompt fills the tutor prompt with one suggestion's context and
|
||||||
@@ -217,23 +329,29 @@ func RewriteMessages(text, style string) []Message {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// translateSystemPrompt drives the explanation translator: it renders a
|
// translateSystemPrompt drives the explanation translator: it renders a
|
||||||
// suggestion's English explanation into the writer's own language so an ESL
|
// suggestion's explanation into the half of the pair the explanation is not
|
||||||
// reader sees the "why" in her first language. Strict about returning ONLY the
|
// already in, so the "why" is readable from both sides. Strict about returning
|
||||||
// translation (no quotes, no romanisation, no English echo) so it can drop
|
// ONLY the translation (no quotes, no romanisation, no echo of the source) so it
|
||||||
// straight into the chat bubble. Kept warm and plain — these are short, friendly
|
// can drop straight into the chat bubble. Kept warm and plain — these are short,
|
||||||
// one-liners.
|
// friendly one-liners.
|
||||||
const translateSystemPrompt = `You are Petal, a warm writing assistant. Translate the English text the user ` +
|
//
|
||||||
`sends into natural, friendly %[1]s. It is a short explanation of a writing ` +
|
// Both languages are parameters because neither end is a constant. Until Phase
|
||||||
`suggestion, written for a native %[1]s speaker learning English.
|
// 28 the source was always English and the destination always hers; a document
|
||||||
|
// written in her own language is explained in her own language, and then the tap
|
||||||
|
// runs the other way, into the English she is practising.
|
||||||
|
const translateSystemPrompt = `You are Petal, a warm writing assistant. Translate the %[1]s text the user ` +
|
||||||
|
`sends into natural, friendly %[2]s. It is a short explanation of a writing ` +
|
||||||
|
`suggestion, written for someone who is learning one of %[1]s and %[2]s and reads the other most easily.
|
||||||
|
|
||||||
Respond with ONLY the %[1]s translation. No quotation marks, no romanisation, no English, no preamble — ` +
|
Respond with ONLY the %[2]s translation. No quotation marks, no romanisation, no %[1]s, no preamble — ` +
|
||||||
`just the translated sentence.`
|
`just the translated sentence.`
|
||||||
|
|
||||||
// TranslateMessages builds the message array for translating one short English
|
// TranslateMessages builds the message array for rendering one short
|
||||||
// explanation into the writer's own language.
|
// explanation out of the language it arrived in and into the other half of the
|
||||||
func TranslateMessages(text string, lang Lang) []Message {
|
// writer's pair.
|
||||||
|
func TranslateMessages(text string, from, to Lang) []Message {
|
||||||
return []Message{
|
return []Message{
|
||||||
{Role: "system", Content: fmt.Sprintf(translateSystemPrompt, lang.Name)},
|
{Role: "system", Content: fmt.Sprintf(translateSystemPrompt, from.Name, to.Name)},
|
||||||
{Role: "user", Content: text},
|
{Role: "user", Content: text},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
// Which language a pass corrects, and which language it explains in.
|
||||||
|
//
|
||||||
|
// Until Phase 28 there was no question to answer: every prompt was written
|
||||||
|
// around English prose explained in English, and the pair language reached her
|
||||||
|
// only when she asked for it (Ask Petal, the explanation translator). That is
|
||||||
|
// the right default for a writer practising English and the wrong one for a
|
||||||
|
// document she wrote in her own language, where Petal would read Portuguese,
|
||||||
|
// say nothing about it, and file a mechanics note about the one English
|
||||||
|
// sentence at the end.
|
||||||
|
//
|
||||||
|
// The two fields are two different decisions reading two different pieces of
|
||||||
|
// state, and collapsing them would be the bug:
|
||||||
|
//
|
||||||
|
// - Correct follows the DOCUMENT. Portuguese prose gets Portuguese
|
||||||
|
// corrections; that is the whole point.
|
||||||
|
// - Explain follows the WRITER — the half of her pair she is *not* learning
|
||||||
|
// (users.direction), because an explanation is teaching, and teaching lands
|
||||||
|
// in the language she reads most easily.
|
||||||
|
//
|
||||||
|
// Today those two coincide for every account that exists: `learnerPairs` is
|
||||||
|
// {"zh"}, so fr, es and pt-PT writers are all `learning_en` and their
|
||||||
|
// non-learned half *is* the pair language. That equality is a fact about
|
||||||
|
// today's roster, not about the design — the same shape of assumption that had
|
||||||
|
// to be unpicked from `pair_lang` in migration 0016. Keep them apart.
|
||||||
|
type Target struct {
|
||||||
|
// Correct is the language the writing is in, and so the language both
|
||||||
|
// `original` and `replacement` must be written in.
|
||||||
|
Correct Lang
|
||||||
|
// Explain is the language each explanation is written in.
|
||||||
|
Explain Lang
|
||||||
|
// Pair is the writer's pair language regardless of what this document is
|
||||||
|
// written in. The collocation coach's parenthetical gloss is addressed to
|
||||||
|
// her rather than to the document, so it reads this and not Correct.
|
||||||
|
Pair Lang
|
||||||
|
}
|
||||||
|
|
||||||
|
// English as the prompts name it. Not in `langs`: that map answers "which
|
||||||
|
// language is the writer's half of the pair", and English is the constant on
|
||||||
|
// the other side of every pair Petal supports.
|
||||||
|
var English = Lang{Code: "en", Name: "English", Why: "why"}
|
||||||
|
|
||||||
|
// EnglishTarget is the pre-Phase-28 behaviour, made explicit: an English
|
||||||
|
// document, corrected and explained in English, for a writer whose pair
|
||||||
|
// language is `pair`. Every existing user is on this path and the prompt it
|
||||||
|
// produces is byte-identical to the one that shipped before this phase.
|
||||||
|
func EnglishTarget(pair Lang) Target {
|
||||||
|
return Target{Correct: English, Explain: English, Pair: pair}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flipped reports whether this document is in the pair language rather than in
|
||||||
|
// English — i.e. whether the pass is reading her own language.
|
||||||
|
func (t Target) Flipped() bool { return t.Correct.Code != English.Code }
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The prompt every account is on today, written out in full.
|
||||||
|
//
|
||||||
|
// Phase 28 gave the checkpoint a second framing for documents in the writer's
|
||||||
|
// own language, and the risk of that change is not that the new prompt is wrong
|
||||||
|
// — it is that the old one moved by a word while nobody was looking. Every user
|
||||||
|
// who exists is a Mandarin native writing English, so this string is the one
|
||||||
|
// Petal actually sends, all day. It is duplicated here on purpose: a golden
|
||||||
|
// copied from the constant it guards guards nothing.
|
||||||
|
const goldenEnglishCheckpointPrompt = `You are a warm, encouraging writing assistant helping someone who speaks English as a second language. Analyze the text below and identify up to 5 issues: grammar errors, unnatural phrasing, incorrect idiom usage, or unclear sentences that are common ESL patterns.
|
||||||
|
|
||||||
|
Be specific, friendly, and explain WHY each suggestion improves the writing.
|
||||||
|
|
||||||
|
Respond ONLY with valid JSON. No preamble, no markdown fences. Format:
|
||||||
|
{
|
||||||
|
"suggestions": [
|
||||||
|
{
|
||||||
|
"original": "exact text from the document that needs fixing",
|
||||||
|
"replacement": "corrected version",
|
||||||
|
"explanation": "friendly one-sentence explanation",
|
||||||
|
"type": "grammar|phrasing|idiom|clarity"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
If the writing looks good, return: {"suggestions": []}`
|
||||||
|
|
||||||
|
func TestEnglishDocumentPromptIsUnchanged(t *testing.T) {
|
||||||
|
msgs := CheckpointMessages("I has two apple.", "", EnglishTarget(LangFor("zh")))
|
||||||
|
if got := msgs[0].Content; got != goldenEnglishCheckpointPrompt {
|
||||||
|
t.Fatalf("the English checkpoint prompt moved:\n--- got ---\n%s\n--- want ---\n%s", got, goldenEnglishCheckpointPrompt)
|
||||||
|
}
|
||||||
|
if msgs[1].Content != "I has two apple." {
|
||||||
|
t.Fatalf("document text mangled: %q", msgs[1].Content)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tone clause still lands, in the same place it always did.
|
||||||
|
toned := CheckpointMessages("x", "academic", EnglishTarget(LangFor("zh")))[0].Content
|
||||||
|
if !strings.Contains(toned, "formal, academic, and objective") {
|
||||||
|
t.Fatalf("English checkpoint lost its tone guidance:\n%s", toned)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A document in her own language gets a prompt that names that language, keeps
|
||||||
|
// the corrections inside it, and drops the framing that only makes sense when
|
||||||
|
// the thing being written is English.
|
||||||
|
func TestFlippedCheckpointPrompt(t *testing.T) {
|
||||||
|
pt := LangFor("pt-PT")
|
||||||
|
system := CheckpointMessages("Hoje foi um dia bom.", "casual", Target{Correct: pt, Explain: pt, Pair: pt})[0].Content
|
||||||
|
|
||||||
|
if !strings.Contains(system, "European Portuguese") {
|
||||||
|
t.Fatalf("flipped checkpoint doesn't name the language:\n%s", system)
|
||||||
|
}
|
||||||
|
if strings.Contains(system, "second language") || strings.Contains(system, "ESL") {
|
||||||
|
t.Fatalf("flipped checkpoint kept the ESL framing:\n%s", system)
|
||||||
|
}
|
||||||
|
if !strings.Contains(system, "never translate it into English") {
|
||||||
|
t.Fatalf("flipped checkpoint doesn't forbid translating:\n%s", system)
|
||||||
|
}
|
||||||
|
// The shared contract below the framing has to survive the split.
|
||||||
|
for _, want := range []string{`"suggestions"`, `"replacement"`, "grammar|phrasing|idiom|clarity", "relaxed, friendly, and conversational"} {
|
||||||
|
if !strings.Contains(system, want) {
|
||||||
|
t.Fatalf("flipped checkpoint dropped %q:\n%s", want, system)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(system, "%!") {
|
||||||
|
t.Fatalf("flipped checkpoint has a formatting error:\n%s", system)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The two decisions are separate arguments and must reach the prompt separately:
|
||||||
|
// corrections in the document's language, the explanation in the language she
|
||||||
|
// reads most easily. Only the zh pair can be travelled both ways today, so it is
|
||||||
|
// the only one that can prove they haven't been quietly collapsed into one.
|
||||||
|
func TestFlippedPromptsExplainInTheirOwnLanguage(t *testing.T) {
|
||||||
|
zh := LangFor("zh")
|
||||||
|
|
||||||
|
// Native Mandarin, practising English, writing Chinese: both halves Chinese.
|
||||||
|
both := CheckpointMessages("今天天气很好。", "", Target{Correct: zh, Explain: zh, Pair: zh})[0].Content
|
||||||
|
if strings.Count(both, "Simplified Chinese (Mandarin)") < 2 {
|
||||||
|
t.Fatalf("expected corrections and explanations both in Chinese:\n%s", both)
|
||||||
|
}
|
||||||
|
if strings.Contains(both, "explanation"+`" in English`) {
|
||||||
|
t.Fatalf("explanation language leaked to English:\n%s", both)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Native English, learning Chinese, writing Chinese: Chinese corrections,
|
||||||
|
// English explanations.
|
||||||
|
split := CheckpointMessages("今天天气很好。", "", Target{Correct: zh, Explain: English, Pair: zh})[0].Content
|
||||||
|
if !strings.Contains(split, `Write every "explanation" in English.`) {
|
||||||
|
t.Fatalf("learner direction didn't get English explanations:\n%s", split)
|
||||||
|
}
|
||||||
|
if !strings.Contains(split, "writing in Simplified Chinese (Mandarin)") {
|
||||||
|
t.Fatalf("learner direction lost its Chinese corrections:\n%s", split)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same for the voice pass, which had no language at all before this phase.
|
||||||
|
voice := VoiceMessages("今天天气很好。", Target{Correct: zh, Explain: English, Pair: zh})[0].Content
|
||||||
|
if !strings.Contains(voice, `Write every "explanation" in English.`) || !strings.Contains(voice, "Simplified Chinese") {
|
||||||
|
t.Fatalf("flipped voice prompt got its languages wrong:\n%s", voice)
|
||||||
|
}
|
||||||
|
if strings.Contains(voice, "second language") {
|
||||||
|
t.Fatalf("flipped voice prompt kept the ESL framing:\n%s", voice)
|
||||||
|
}
|
||||||
|
// An English document still gets exactly the voice prompt it always got.
|
||||||
|
if got := VoiceMessages("x", EnglishTarget(zh))[0].Content; got != voiceSystemPrompt {
|
||||||
|
t.Fatalf("English voice prompt moved:\n%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,13 +4,14 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RunTranslate renders a short English explanation into the writer's own
|
// RunTranslate renders a short explanation out of the language it was written
|
||||||
// language. It is a one-shot Complete (the result seeds the Ask Petal bubble),
|
// in and into the other half of the writer's pair. It is a one-shot Complete
|
||||||
// kept at a low temperature so the translation is faithful rather than creative. Output is
|
// (the result seeds the Ask Petal bubble), kept at a low temperature so the
|
||||||
// trimmed of any stray surrounding quotes the model may add.
|
// translation is faithful rather than creative. Output is trimmed of any stray
|
||||||
func RunTranslate(ctx context.Context, client LLMClient, text string, lang Lang) (string, error) {
|
// surrounding quotes the model may add.
|
||||||
|
func RunTranslate(ctx context.Context, client LLMClient, text string, from, to Lang) (string, error) {
|
||||||
out, err := client.Complete(ctx, CompletionRequest{
|
out, err := client.Complete(ctx, CompletionRequest{
|
||||||
Messages: TranslateMessages(text, lang),
|
Messages: TranslateMessages(text, from, to),
|
||||||
MaxTokens: 512,
|
MaxTokens: 512,
|
||||||
Temperature: 0.2,
|
Temperature: 0.2,
|
||||||
TopP: 0.9,
|
TopP: 0.9,
|
||||||
|
|||||||
@@ -19,9 +19,9 @@ const VoiceInterval = 20 * time.Second
|
|||||||
// The tone argument is accepted for a uniform pass signature but ignored: voice
|
// The tone argument is accepted for a uniform pass signature but ignored: voice
|
||||||
// consistency is judged against the document's own established voice, not an
|
// consistency is judged against the document's own established voice, not an
|
||||||
// externally-chosen register.
|
// externally-chosen register.
|
||||||
func RunVoice(ctx context.Context, client LLMClient, contentText, _ string, _ Lang) ([]RawSuggestion, error) {
|
func RunVoice(ctx context.Context, client LLMClient, contentText, _ string, t Target) ([]RawSuggestion, error) {
|
||||||
raw, err := client.Complete(ctx, CompletionRequest{
|
raw, err := client.Complete(ctx, CompletionRequest{
|
||||||
Messages: VoiceMessages(contentText),
|
Messages: VoiceMessages(contentText, t),
|
||||||
MaxTokens: 2048,
|
MaxTokens: 2048,
|
||||||
Temperature: 0.3,
|
Temperature: 0.3,
|
||||||
RepetitionPenalty: 1.15,
|
RepetitionPenalty: 1.15,
|
||||||
|
|||||||
@@ -8,8 +8,18 @@
|
|||||||
// so it belongs to her account rather than to a browser profile.
|
// so it belongs to her account rather than to a browser profile.
|
||||||
//
|
//
|
||||||
// Everything here is scoped by `lang` as well as by user. That is the language
|
// Everything here is scoped by `lang` as well as by user. That is the language
|
||||||
// of the *dictionary* that flagged the word, not the writer's own language: an
|
// of the *dictionary* the word was accepted against, not the writer's own.
|
||||||
// en-US personal word must not silence a pt-PT flag once the second pair ships.
|
//
|
||||||
|
// Phase 18 justified that key by saying an en-US personal word must not silence
|
||||||
|
// a pt-PT flag once the second pair shipped. Phase 21 shipped it and the
|
||||||
|
// justification did not survive: under the both-dictionaries rule
|
||||||
|
// (SUGGESTIONS.md §3a) a word is only ever flagged when *every* loaded
|
||||||
|
// dictionary rejected it, so there is no such thing as a pt-PT flag an English
|
||||||
|
// exception could silence. What the key is actually good for is narrower and
|
||||||
|
// still worth having — the rows say which dictionary each acceptance was made
|
||||||
|
// against, so a pair that later loses or gains a dictionary keeps a truthful
|
||||||
|
// record instead of one merged list of unknown provenance. The browser writes a
|
||||||
|
// row per loaded dictionary when she accepts a word; see useSpellChecker.
|
||||||
package spell
|
package spell
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -24,8 +34,9 @@ import (
|
|||||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DefaultLang is the dictionary assumed when a caller doesn't name one. Only
|
// DefaultLang is the dictionary assumed when a caller doesn't name one. English
|
||||||
// en-US ships today; pt-PT arrives with the first Latin pair.
|
// is in every pair, so it is the safe assumption; pt-PT is named explicitly by
|
||||||
|
// the pt-PT pair's second dictionary.
|
||||||
const DefaultLang = "en"
|
const DefaultLang = "en"
|
||||||
|
|
||||||
// MaxWordLen bounds a single entry. A personal dictionary holds words, and a
|
// MaxWordLen bounds a single entry. A personal dictionary holds words, and a
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ func (h *Handler) chat(w http.ResponseWriter, r *http.Request) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
// The stream never opened (e.g. LLM unreachable) — a normal JSON error is
|
// The stream never opened (e.g. LLM unreachable) — a normal JSON error is
|
||||||
// still appropriate since we haven't written SSE headers yet.
|
// still appropriate since we haven't written SSE headers yet.
|
||||||
httputil.ErrorJSON(w, http.StatusBadGateway, "chat failed: "+err.Error())
|
httputil.UpstreamError(w, "chat", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,145 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Chunking splits a document into sentence-sized units so a re-check can ask the
|
||||||
|
// model only about the sentences that actually changed. Accepting one edit used
|
||||||
|
// to re-run the whole document: every card vanished, came back with a new id and
|
||||||
|
// a freshly-worded explanation, and spans re-merged into different shapes. The
|
||||||
|
// sentences she didn't touch have nothing new to say about themselves, so their
|
||||||
|
// suggestions are simply kept (see reconcilePending).
|
||||||
|
//
|
||||||
|
// A chunk's identity is its hash, not its position — she inserts a paragraph at
|
||||||
|
// the top and every sentence below keeps its suggestions.
|
||||||
|
|
||||||
|
// chunk is one sentence of the document, with the hash that identifies it.
|
||||||
|
type chunk struct {
|
||||||
|
text string
|
||||||
|
hash string
|
||||||
|
}
|
||||||
|
|
||||||
|
// asciiTerminators end a sentence only when whitespace (or the end of the text)
|
||||||
|
// follows, so "3.5" and "Ms." don't split mid-word — a wrong split costs only a
|
||||||
|
// slightly smaller chunk, but a split inside a number would churn its hash on
|
||||||
|
// every keystroke around it.
|
||||||
|
const asciiTerminators = ".!?"
|
||||||
|
|
||||||
|
// cjkTerminators end a sentence outright: Chinese runs sentences together with
|
||||||
|
// no space after 。, and she writes in both languages in one document.
|
||||||
|
const cjkTerminators = "。!?"
|
||||||
|
|
||||||
|
// closers are swallowed into the sentence they close, so the quote mark travels
|
||||||
|
// with the sentence rather than opening the next one.
|
||||||
|
const closers = `)]}"'’”」』`
|
||||||
|
|
||||||
|
// splitChunks divides text into sentences, dropping whitespace-only runs.
|
||||||
|
// Newlines always break a chunk, so a list or a line of dialogue is its own unit.
|
||||||
|
//
|
||||||
|
// `salt` distinguishes two *readings* of the same sentence. The grammar
|
||||||
|
// checkpoint's advice depends on the document's tone — the same line gets
|
||||||
|
// different notes as an academic essay than as a journal entry — so switching
|
||||||
|
// tone must re-open every sentence rather than serve back advice written for the
|
||||||
|
// old register.
|
||||||
|
func splitChunks(text, salt string) []chunk {
|
||||||
|
var out []chunk
|
||||||
|
runes := []rune(text)
|
||||||
|
start := 0
|
||||||
|
add := func(end int) {
|
||||||
|
if s := string(runes[start:end]); strings.TrimSpace(s) != "" {
|
||||||
|
out = append(out, chunk{text: s, hash: hashChunk(s, salt)})
|
||||||
|
}
|
||||||
|
start = end
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 0; i < len(runes); i++ {
|
||||||
|
r := runes[i]
|
||||||
|
if r == '\n' {
|
||||||
|
add(i + 1)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cjk := strings.ContainsRune(cjkTerminators, r)
|
||||||
|
if !cjk && !strings.ContainsRune(asciiTerminators, r) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Swallow a run of terminators ("?!", "…") and any closing punctuation.
|
||||||
|
j := i + 1
|
||||||
|
for j < len(runes) && (strings.ContainsRune(asciiTerminators+cjkTerminators+closers, runes[j])) {
|
||||||
|
j++
|
||||||
|
}
|
||||||
|
if cjk || j >= len(runes) || unicode.IsSpace(runes[j]) {
|
||||||
|
add(j)
|
||||||
|
i = j - 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if start < len(runes) {
|
||||||
|
add(len(runes))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// hashChunk identifies a sentence by its content under the same normalization
|
||||||
|
// the suppression logic uses: quote style and whitespace runs churn constantly
|
||||||
|
// (the editor rewrites quotes as she types, a paragraph reflows) and none of
|
||||||
|
// that changes what the sentence says, so none of it should cost a re-check.
|
||||||
|
func hashChunk(s, salt string) string {
|
||||||
|
sum := sha256.Sum256([]byte(salt + "\x00" + normalizeForDedup(s)))
|
||||||
|
return hex.EncodeToString(sum[:])[:16]
|
||||||
|
}
|
||||||
|
|
||||||
|
// hashSet indexes chunks by hash — "is this sentence in the document?"
|
||||||
|
func hashSet(chunks []chunk) map[string]bool {
|
||||||
|
out := make(map[string]bool, len(chunks))
|
||||||
|
for _, c := range chunks {
|
||||||
|
out[c.hash] = true
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// changedChunks returns the chunks whose hash wasn't in the last checked set,
|
||||||
|
// in document order and deduplicated — a sentence repeated verbatim is one
|
||||||
|
// question, not two.
|
||||||
|
func changedChunks(chunks []chunk, checked map[string]bool) []chunk {
|
||||||
|
seen := make(map[string]bool, len(chunks))
|
||||||
|
var out []chunk
|
||||||
|
for _, c := range chunks {
|
||||||
|
if checked[c.hash] || seen[c.hash] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[c.hash] = true
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// joinChunks renders a chunk set as the text to hand the model: one sentence per
|
||||||
|
// line, so two sentences pulled from opposite ends of the document don't read as
|
||||||
|
// one run-on.
|
||||||
|
func joinChunks(chunks []chunk) string {
|
||||||
|
parts := make([]string, 0, len(chunks))
|
||||||
|
for _, c := range chunks {
|
||||||
|
parts = append(parts, strings.TrimSpace(c.text))
|
||||||
|
}
|
||||||
|
return strings.Join(parts, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// chunkFor names the sentence a suggestion belongs to: the first chunk whose
|
||||||
|
// text contains the flagged span. Returns "" when the span straddles a sentence
|
||||||
|
// boundary or the model paraphrased what it quoted — such a row is re-examined
|
||||||
|
// on every pass rather than cached, which is the safe direction.
|
||||||
|
func chunkFor(original string, chunks []chunk) string {
|
||||||
|
o := normalizeForDedup(original)
|
||||||
|
if o == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
for _, c := range chunks {
|
||||||
|
if strings.Contains(normalizeForDedup(c.text), o) {
|
||||||
|
return c.hash
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func texts(chunks []chunk) []string {
|
||||||
|
out := make([]string, 0, len(chunks))
|
||||||
|
for _, c := range chunks {
|
||||||
|
out = append(out, strings.TrimSpace(c.text))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSplitChunks(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
in string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "plain sentences",
|
||||||
|
in: "I has two apple. She go to market yesterday! Why?",
|
||||||
|
want: []string{"I has two apple.", "She go to market yesterday!", "Why?"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// A decimal must not split, or the sentence's identity would churn
|
||||||
|
// while she types the number.
|
||||||
|
name: "decimals stay whole",
|
||||||
|
in: "It costs 3.50 today. Tomorrow, more.",
|
||||||
|
want: []string{"It costs 3.50 today.", "Tomorrow, more."},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "closing quote travels with its sentence",
|
||||||
|
in: `He said "early," and left. She stayed.`,
|
||||||
|
want: []string{`He said "early," and left.`, "She stayed."},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Chinese runs sentences together with no space after 。 — she writes
|
||||||
|
// in both languages in one document.
|
||||||
|
name: "cjk terminators split without a space",
|
||||||
|
in: "我想说这句话。但是不知道用英语怎么说。",
|
||||||
|
want: []string{"我想说这句话。", "但是不知道用英语怎么说。"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "newlines break chunks",
|
||||||
|
in: "A list item\nAnother item\n",
|
||||||
|
want: []string{"A list item", "Another item"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank runs are dropped",
|
||||||
|
in: "\n\n \nOnly this.\n\n",
|
||||||
|
want: []string{"Only this."},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "trailing fragment is its own chunk",
|
||||||
|
in: "Done. Still writing",
|
||||||
|
want: []string{"Done.", "Still writing"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got := texts(splitChunks(tc.in, ""))
|
||||||
|
if len(got) != len(tc.want) {
|
||||||
|
t.Fatalf("want %q, got %q", tc.want, got)
|
||||||
|
}
|
||||||
|
for i := range got {
|
||||||
|
if got[i] != tc.want[i] {
|
||||||
|
t.Fatalf("chunk %d: want %q, got %q", i, tc.want[i], got[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A sentence's identity survives the churn that doesn't change what it says:
|
||||||
|
// the editor rewrites quotes as she types, and a paragraph reflows.
|
||||||
|
func TestChunkIdentityIgnoresCosmeticChurn(t *testing.T) {
|
||||||
|
a := splitChunks(`She said "hello" softly.`, "")
|
||||||
|
b := splitChunks("She said “hello” softly.", "")
|
||||||
|
if len(a) != 1 || len(b) != 1 {
|
||||||
|
t.Fatalf("want one chunk each, got %d and %d", len(a), len(b))
|
||||||
|
}
|
||||||
|
if a[0].hash != b[0].hash {
|
||||||
|
t.Fatalf("quote/whitespace churn changed the sentence's identity")
|
||||||
|
}
|
||||||
|
if same := splitChunks(`She said "hello" softly.`, "academic"); same[0].hash == a[0].hash {
|
||||||
|
t.Fatalf("a different tone must be a different reading of the sentence")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChangedChunksAndLookup(t *testing.T) {
|
||||||
|
chunks := splitChunks("One thing. Another thing. One thing.", "")
|
||||||
|
if len(chunks) != 3 {
|
||||||
|
t.Fatalf("want 3 chunks, got %d", len(chunks))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A repeated sentence is one question, not two.
|
||||||
|
if got := changedChunks(chunks, nil); len(got) != 2 {
|
||||||
|
t.Fatalf("want 2 distinct changed chunks, got %d", len(got))
|
||||||
|
}
|
||||||
|
|
||||||
|
checked := hashSet(chunks[:1])
|
||||||
|
changed := changedChunks(chunks, checked)
|
||||||
|
if len(changed) != 1 || strings.TrimSpace(changed[0].text) != "Another thing." {
|
||||||
|
t.Fatalf("want only the unread sentence, got %q", texts(changed))
|
||||||
|
}
|
||||||
|
|
||||||
|
if chunkFor("Another", chunks) != chunks[1].hash {
|
||||||
|
t.Fatalf("span was attributed to the wrong sentence")
|
||||||
|
}
|
||||||
|
// A span the document doesn't contain has no sentence, so it is never cached.
|
||||||
|
if chunkFor("nowhere in here", chunks) != "" {
|
||||||
|
t.Fatalf("unanchorable span should have no chunk")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,179 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
// What language is this DOCUMENT in — as opposed to this span.
|
||||||
|
//
|
||||||
|
// `readsAsPairLang` next door answers a span-level question for the purpose of
|
||||||
|
// labelling one card, and it is written to under-claim: two marker words, or
|
||||||
|
// nothing. A whole document needs the opposite temperament. A proportion, not a
|
||||||
|
// presence — one Portuguese quotation inside an English essay must not flip the
|
||||||
|
// entire pass into Portuguese, and one English sentence at the end of a
|
||||||
|
// Portuguese journal must not keep it in English.
|
||||||
|
//
|
||||||
|
// Three properties, in the order they bite:
|
||||||
|
//
|
||||||
|
// - Decided over the WHOLE document, never a chunk. The grammar checkpoint
|
||||||
|
// sends only the sentences that changed, so a verdict computed from what it
|
||||||
|
// asked about would put an English card in a Portuguese journal the moment
|
||||||
|
// she edits its one English line. The caller passes content_text, always.
|
||||||
|
//
|
||||||
|
// - Hysteresis. A bilingual paragraph sits near whatever single threshold we
|
||||||
|
// pick, and a document crossing it every few keystrokes would alternate card
|
||||||
|
// languages between passes — the same instability the mascot needed a band
|
||||||
|
// for. Flip to the pair at 70% and back only below 40%; in between, whatever
|
||||||
|
// it was last time stands. The band is the feature, not a rounding
|
||||||
|
// tolerance.
|
||||||
|
//
|
||||||
|
// - Plain code, no model call. The house rule is that the LLM is garnish,
|
||||||
|
// never a gatekeeper: a document must not become uncheckable because the
|
||||||
|
// inference box is down.
|
||||||
|
const (
|
||||||
|
docLangEnglish = "en"
|
||||||
|
docLangPair = "pair"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The band. Deliberately wide: the cost of an unnecessary flip is every card in
|
||||||
|
// the document changing language, which is far more startling than a paragraph
|
||||||
|
// of mixed writing being read as whichever language it was a minute ago.
|
||||||
|
const (
|
||||||
|
flipToPairAt = 0.70
|
||||||
|
flipToEnglishBelow = 0.40
|
||||||
|
)
|
||||||
|
|
||||||
|
// documentLang returns the language verdict for a document, given the verdict it
|
||||||
|
// carried before. `prev` is "" for a document that has never been read.
|
||||||
|
//
|
||||||
|
// The result is one of docLangEnglish / docLangPair — not a language code. Which
|
||||||
|
// language "pair" means is the writer's `pair_lang`, and keeping the stored
|
||||||
|
// verdict relative to her pair means changing her pair doesn't strand a stale
|
||||||
|
// language name on every document she owns.
|
||||||
|
func documentLang(contentText, pairLang, prev string) string {
|
||||||
|
if prev != docLangPair {
|
||||||
|
prev = docLangEnglish
|
||||||
|
}
|
||||||
|
p := normalizePairLang(pairLang)
|
||||||
|
if !hasLangTest(p) {
|
||||||
|
// No test for this pair: say English, which is what every surface did
|
||||||
|
// before this phase. A wrong flip is louder than a missing one.
|
||||||
|
return docLangEnglish
|
||||||
|
}
|
||||||
|
|
||||||
|
var pair, english int
|
||||||
|
for _, c := range splitChunks(contentText, "") {
|
||||||
|
switch sentenceLang(c.text, p) {
|
||||||
|
case docLangPair:
|
||||||
|
pair++
|
||||||
|
case docLangEnglish:
|
||||||
|
english++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
decided := pair + english
|
||||||
|
if decided == 0 {
|
||||||
|
// Nothing to go on — an empty document, a list of numbers, a title. Hold
|
||||||
|
// the previous verdict rather than resetting a Portuguese journal to
|
||||||
|
// English because she cleared it to start again.
|
||||||
|
return prev
|
||||||
|
}
|
||||||
|
|
||||||
|
ratio := float64(pair) / float64(decided)
|
||||||
|
switch {
|
||||||
|
case ratio >= flipToPairAt && corroborated(contentText, p):
|
||||||
|
return docLangPair
|
||||||
|
case ratio < flipToEnglishBelow:
|
||||||
|
return docLangEnglish
|
||||||
|
default:
|
||||||
|
return prev
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sentenceLang classifies one sentence as pair-language, English, or neither.
|
||||||
|
//
|
||||||
|
// Neither is a real answer and carries weight: a sentence with no evidence
|
||||||
|
// either way ("Bom dia.", "OK.", a heading) is left out of the ratio entirely
|
||||||
|
// rather than counted for the language it isn't. Counting the undecided as
|
||||||
|
// English is what would keep a Portuguese document in English forever, since
|
||||||
|
// short sentences carry no markers.
|
||||||
|
func sentenceLang(s, p string) string {
|
||||||
|
if p == "zh" {
|
||||||
|
han, latin := scriptCounts(s)
|
||||||
|
switch {
|
||||||
|
case han >= 2 && han > latin:
|
||||||
|
return docLangPair
|
||||||
|
case latin >= 3 && latin > han:
|
||||||
|
return docLangEnglish
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
// A Latin pair shares its alphabet with English, so both sides are counted
|
||||||
|
// the same way and the larger pile of evidence wins. A tie — including no
|
||||||
|
// evidence at all — is no answer, which is why the English list below is
|
||||||
|
// curated as carefully against the pair languages as theirs is against
|
||||||
|
// English.
|
||||||
|
pair := distinctMarkers(s, latinMarkers[p])
|
||||||
|
eng := distinctMarkers(s, englishMarkers)
|
||||||
|
switch {
|
||||||
|
case pair > eng:
|
||||||
|
return docLangPair
|
||||||
|
case eng > pair:
|
||||||
|
return docLangEnglish
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// corroborated requires the document as a whole to carry real evidence of the
|
||||||
|
// pair language before the pass flips into it. A two-sentence document of
|
||||||
|
// "Sim." / "Não." would otherwise reach 100% on almost nothing; a flip changes
|
||||||
|
// every card in the document, so it has to be earned document-wide and not only
|
||||||
|
// in proportion.
|
||||||
|
func corroborated(contentText, p string) bool {
|
||||||
|
if p == "zh" {
|
||||||
|
han, _ := scriptCounts(contentText)
|
||||||
|
return han >= 8
|
||||||
|
}
|
||||||
|
return distinctMarkers(contentText, latinMarkers[p]) >= 3
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasLangTest reports whether readsAsPairLang / sentenceLang know how to test
|
||||||
|
// this pair at all. Kept beside the tests it describes so a new pair that adds
|
||||||
|
// markers without adding itself here fails loudly in review rather than quietly
|
||||||
|
// at runtime.
|
||||||
|
func hasLangTest(p string) bool {
|
||||||
|
if p == "zh" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return len(latinMarkers[p]) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// English function words, curated against the pair languages exactly as
|
||||||
|
// `latinMarkers` is curated against English.
|
||||||
|
//
|
||||||
|
// Every word here is one a Portuguese, French or Spanish sentence has no reason
|
||||||
|
// to contain. Deliberately absent, each a false English vote waiting to happen
|
||||||
|
// in someone's own language: "on" and "son" (French), "as", "a", "o", "e", "no",
|
||||||
|
// "os" (Portuguese), "no", "para", "sin" (Spanish), and "is"-alikes that are
|
||||||
|
// really other languages' words. The list is short on purpose — it does not need
|
||||||
|
// coverage, only a reliable vote in the sentences where the pair list is silent.
|
||||||
|
var englishMarkers = words(
|
||||||
|
"the", "and", "is", "are", "was", "were", "be", "been", "being",
|
||||||
|
"of", "to", "that", "this", "these", "those", "with", "from", "for",
|
||||||
|
"have", "has", "had", "they", "them", "their", "there", "then", "than",
|
||||||
|
"what", "which", "when", "where", "why", "how", "who",
|
||||||
|
"will", "would", "should", "could", "can", "about", "because",
|
||||||
|
"into", "some", "such", "only", "very", "much", "many", "other",
|
||||||
|
"our", "your", "its", "it", "he", "she", "we", "you", "but", "not",
|
||||||
|
"just", "like", "also", "most", "over", "after", "before", "between",
|
||||||
|
"through", "said", "says", "get", "got", "make", "made", "know", "think",
|
||||||
|
"thing", "things", "time", "people", "here", "always", "never", "something",
|
||||||
|
"want", "need", "feel", "day", "today", "good", "really", "still", "even",
|
||||||
|
)
|
||||||
|
|
||||||
|
// normalizeDocLang folds a stored verdict into the two values the rest of the
|
||||||
|
// code reasons about: the column holds "" for a document nothing has read yet,
|
||||||
|
// and that means English, which is what every surface did before this phase.
|
||||||
|
func normalizeDocLang(v string) string {
|
||||||
|
if strings.TrimSpace(v) == docLangPair {
|
||||||
|
return docLangPair
|
||||||
|
}
|
||||||
|
return docLangEnglish
|
||||||
|
}
|
||||||
@@ -0,0 +1,536 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A monolingual document in either language has to be read as that language, and
|
||||||
|
// the mixed cases in between are where the whole design lives: one quotation
|
||||||
|
// must not move a document, and one leftover English line must not hold a
|
||||||
|
// journal in English.
|
||||||
|
func TestDocumentLangReadsWholeDocuments(t *testing.T) {
|
||||||
|
const ptJournal = "Hoje foi um dia muito bom. Eu gosto de escrever aqui todas as noites. " +
|
||||||
|
"A minha irmã também quer aprender. Não sei porque isso é tão difícil para mim."
|
||||||
|
const enEssay = "The weather was very cold this morning. I think that the bus was late again. " +
|
||||||
|
"She told me about the meeting, but I could not hear what they said."
|
||||||
|
const zhJournal = "今天天气很好。我和妹妹一起去公园散步。我们看到很多花。"
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
text string
|
||||||
|
pairLang string
|
||||||
|
prev string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"portuguese journal", ptJournal, "pt-PT", "", docLangPair},
|
||||||
|
{"english essay", enEssay, "pt-PT", "", docLangEnglish},
|
||||||
|
{"chinese journal", zhJournal, "zh", "", docLangPair},
|
||||||
|
{"english essay, zh writer", enEssay, "zh", "", docLangEnglish},
|
||||||
|
|
||||||
|
// One English sentence at the end of a Portuguese journal is the case that
|
||||||
|
// motivated the whole phase: the pass must stay in Portuguese.
|
||||||
|
{
|
||||||
|
"portuguese with one english line",
|
||||||
|
ptJournal + " I will write more tomorrow.",
|
||||||
|
"pt-PT", "", docLangPair,
|
||||||
|
},
|
||||||
|
// And the mirror: an English essay quoting a line of Portuguese is still an
|
||||||
|
// English essay.
|
||||||
|
{
|
||||||
|
"english quoting portuguese",
|
||||||
|
enEssay + " She wrote: \"Eu não sei o que dizer.\"",
|
||||||
|
"pt-PT", "", docLangEnglish,
|
||||||
|
},
|
||||||
|
// A pair Petal has no test for cannot flip anything. Saying English is what
|
||||||
|
// every surface did before this phase.
|
||||||
|
{"untested pair", ptJournal, "de", "", docLangEnglish},
|
||||||
|
// Nothing to go on holds the previous answer rather than resetting a
|
||||||
|
// journal because she cleared it to start again.
|
||||||
|
{"emptied portuguese journal", "", "pt-PT", docLangPair, docLangPair},
|
||||||
|
{"emptied english essay", " \n ", "pt-PT", docLangEnglish, docLangEnglish},
|
||||||
|
// Proportion, not presence: a couple of Portuguese words are not a
|
||||||
|
// Portuguese document even though readsAsPairLang would label that span.
|
||||||
|
{
|
||||||
|
"english with a portuguese phrase",
|
||||||
|
enEssay + " The sign said pão com manteiga.",
|
||||||
|
"pt-PT", "", docLangEnglish,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := documentLang(tc.text, tc.pairLang, tc.prev); got != tc.want {
|
||||||
|
t.Fatalf("documentLang = %q, want %q", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The band, from both directions. A document sitting inside it keeps whatever it
|
||||||
|
// was, and that is the point: without it, a bilingual paragraph would alternate
|
||||||
|
// its cards' language every few keystrokes as she typed across the threshold.
|
||||||
|
func TestDocumentLangHysteresis(t *testing.T) {
|
||||||
|
// Half and half: two Portuguese sentences, two English ones. Inside the band
|
||||||
|
// from either side.
|
||||||
|
const mixed = "Eu gosto muito de escrever aqui. A minha irmã não sabe porque é difícil. " +
|
||||||
|
"The weather was very cold this morning. I think that they said the same thing."
|
||||||
|
|
||||||
|
if got := documentLang(mixed, "pt-PT", docLangEnglish); got != docLangEnglish {
|
||||||
|
t.Fatalf("mixed document from english = %q, want it to stay %q", got, docLangEnglish)
|
||||||
|
}
|
||||||
|
if got := documentLang(mixed, "pt-PT", docLangPair); got != docLangPair {
|
||||||
|
t.Fatalf("mixed document from pair = %q, want it to stay %q", got, docLangPair)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Above the upper threshold it flips regardless of where it came from; below
|
||||||
|
// the lower one it flips back regardless.
|
||||||
|
const mostlyPT = "Eu gosto muito de escrever aqui. A minha irmã não sabe porque é difícil. " +
|
||||||
|
"Hoje foi um dia bom para mim. Amanhã também quero escrever mais uma coisa. " +
|
||||||
|
"I think so too."
|
||||||
|
if got := documentLang(mostlyPT, "pt-PT", docLangEnglish); got != docLangPair {
|
||||||
|
t.Fatalf("mostly-portuguese from english = %q, want %q", got, docLangPair)
|
||||||
|
}
|
||||||
|
const mostlyEN = "The weather was very cold this morning. I think that they said the same thing. " +
|
||||||
|
"She could not hear what the other people were saying about it. Eu não sei."
|
||||||
|
if got := documentLang(mostlyEN, "pt-PT", docLangPair); got != docLangEnglish {
|
||||||
|
t.Fatalf("mostly-english from pair = %q, want %q", got, docLangEnglish)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Corroboration: a proportion computed over almost nothing is not evidence. Two
|
||||||
|
// bare words at 100% must not flip a document, because a flip rewrites every
|
||||||
|
// card in it.
|
||||||
|
func TestDocumentLangNeedsCorroboration(t *testing.T) {
|
||||||
|
if got := documentLang("Não. Eu.", "pt-PT", docLangEnglish); got != docLangEnglish {
|
||||||
|
t.Fatalf("two bare words flipped the document: %q", got)
|
||||||
|
}
|
||||||
|
if got := documentLang("我。", "zh", docLangEnglish); got != docLangEnglish {
|
||||||
|
t.Fatalf("two Han runes flipped the document: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The Portuguese half of the same rule, and the bug it was reported as: "the
|
||||||
|
// Portuguese option isn't translating the advice in English — it's just
|
||||||
|
// reprinting Portuguese."
|
||||||
|
//
|
||||||
|
// Nothing was wrong with targetFor when that was reported. It was reading a
|
||||||
|
// direction the account could not leave: `learnerPairs` held only zh, so every
|
||||||
|
// pt-PT writer was learning_en by force and this function correctly explained a
|
||||||
|
// Portuguese document in Portuguese. Pinned here rather than only in the auth
|
||||||
|
// package because this is where the consequence actually lands — the language
|
||||||
|
// the writer reads her advice in.
|
||||||
|
func TestTargetExplainsPortugueseInEnglishForALearner(t *testing.T) {
|
||||||
|
learner := targetFor("pt-PT", auth.DirectionLearningPair, docLangPair)
|
||||||
|
if learner.Correct.Code != "pt-PT" {
|
||||||
|
t.Fatalf("corrected in %s, want the document's own Portuguese", learner.Correct.Code)
|
||||||
|
}
|
||||||
|
if learner.Explain.Code != "en" {
|
||||||
|
t.Fatalf("explained in %s, want English", learner.Explain.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the native Portuguese speaker practising English is untouched: her
|
||||||
|
// Portuguese is still explained in Portuguese.
|
||||||
|
native := targetFor("pt-PT", auth.DirectionLearningEn, docLangPair)
|
||||||
|
if native.Correct.Code != "pt-PT" || native.Explain.Code != "pt-PT" {
|
||||||
|
t.Fatalf("learning_en on a Portuguese document: correct=%s explain=%s", native.Correct.Code, native.Explain.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The two language decisions are genuinely independent, and zh was the first
|
||||||
|
// pair that could prove it — the first that could be travelled in both
|
||||||
|
// directions.
|
||||||
|
//
|
||||||
|
// A Mandarin native practising English who writes Chinese wants Chinese
|
||||||
|
// corrections explained in Chinese. An English native learning Chinese who writes
|
||||||
|
// Chinese wants the same Chinese corrections explained in English. Same document,
|
||||||
|
// same Correct, different Explain.
|
||||||
|
func TestTargetSeparatesCorrectedFromExplained(t *testing.T) {
|
||||||
|
learningEn := targetFor("zh", auth.DirectionLearningEn, docLangPair)
|
||||||
|
if learningEn.Correct.Code != "zh" || learningEn.Explain.Code != "zh" {
|
||||||
|
t.Fatalf("learning_en on a Chinese document: correct=%s explain=%s", learningEn.Correct.Code, learningEn.Explain.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
learningPair := targetFor("zh", auth.DirectionLearningPair, docLangPair)
|
||||||
|
if learningPair.Correct.Code != "zh" {
|
||||||
|
t.Fatalf("learner direction changed what gets corrected: %s", learningPair.Correct.Code)
|
||||||
|
}
|
||||||
|
if learningPair.Explain.Code != "en" {
|
||||||
|
t.Fatalf("learner direction explained in %s, want English", learningPair.Explain.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An English document is the path every account is on today, in either
|
||||||
|
// direction: English corrections, English explanations, her language still on
|
||||||
|
// the Ask Petal and translate taps.
|
||||||
|
for _, dir := range []string{auth.DirectionLearningEn, auth.DirectionLearningPair} {
|
||||||
|
got := targetFor("zh", dir, docLangEnglish)
|
||||||
|
if got.Flipped() || got.Explain.Code != "en" {
|
||||||
|
t.Fatalf("english document with direction %s: %+v", dir, got)
|
||||||
|
}
|
||||||
|
if got.Pair.Code != "zh" {
|
||||||
|
t.Fatalf("english document lost the writer's pair: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// newDirectedServer seeds one writer on a given pair and direction, with a
|
||||||
|
// document of her own. Like newPairServer, but the direction is the variable.
|
||||||
|
func newDirectedServer(t *testing.T, client llm.LLMClient, pairLang, direction, text string) (http.Handler, string, *Handler) {
|
||||||
|
t.Helper()
|
||||||
|
database, err := db.Open(filepath.Join(t.TempDir(), "doclang.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open db: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { database.Close() })
|
||||||
|
|
||||||
|
const userID = "writer-directed"
|
||||||
|
if _, err := database.Exec(
|
||||||
|
`INSERT INTO users (id, email, display_name, pair_lang, direction) VALUES (?, ?, ?, ?, ?)`,
|
||||||
|
userID, "d@example.com", "Writer", pairLang, direction,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("seed user: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var docID string
|
||||||
|
if err := database.QueryRow(
|
||||||
|
`INSERT INTO documents (user_id, content_text) VALUES (?, ?) RETURNING id`,
|
||||||
|
userID, text,
|
||||||
|
).Scan(&docID); err != nil {
|
||||||
|
t.Fatalf("seed doc: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
h := New(database, client)
|
||||||
|
h.Limit = llm.NewRateLimiter(0)
|
||||||
|
h.VoiceLimit = llm.NewRateLimiter(0)
|
||||||
|
r := chi.NewRouter()
|
||||||
|
r.Route("/docs", func(dr chi.Router) { h.RegisterDocRoutes(dr) })
|
||||||
|
r.Mount("/suggestions", h.Routes())
|
||||||
|
return auth.Middleware(auth.StaticResolver(userID))(r), docID, h
|
||||||
|
}
|
||||||
|
|
||||||
|
const ptDocument = "Hoje foi um dia muito bom. Eu gosto de escrever aqui todas as noites. " +
|
||||||
|
"A minha irmã também quer aprender comigo. Não sei porque isso é tão difícil para mim."
|
||||||
|
|
||||||
|
// End to end: a Portuguese document reaches the model as a Portuguese
|
||||||
|
// checkpoint. This is the observed bug from 2026-07-28 — two pt-PT sentences
|
||||||
|
// drew no cards at all, because Petal was reading them as bad English.
|
||||||
|
func TestCheckpointFollowsTheDocumentLanguage(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[]}`}
|
||||||
|
srv, docID, _ := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)
|
||||||
|
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(client.lastPrompt, "European Portuguese") {
|
||||||
|
t.Fatalf("checkpoint didn't follow the document into Portuguese:\n%s", client.lastPrompt)
|
||||||
|
}
|
||||||
|
if strings.Contains(client.lastPrompt, "second language") {
|
||||||
|
t.Fatalf("checkpoint kept the ESL framing on a Portuguese document:\n%s", client.lastPrompt)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the voice pass, which had no language argument at all before this phase.
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/voice", ""); rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("voice: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(client.lastPrompt, "European Portuguese") {
|
||||||
|
t.Fatalf("voice pass didn't follow the document:\n%s", client.lastPrompt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The verdict is persisted, because hysteresis needs a yesterday.
|
||||||
|
func TestDocumentLangIsRemembered(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[]}`}
|
||||||
|
srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)
|
||||||
|
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
var stored string
|
||||||
|
if err := h.DB.QueryRow(`SELECT doc_lang FROM documents WHERE id = ?`, docID).Scan(&stored); err != nil {
|
||||||
|
t.Fatalf("read doc_lang: %v", err)
|
||||||
|
}
|
||||||
|
if stored != docLangPair {
|
||||||
|
t.Fatalf("doc_lang = %q, want %q", stored, docLangPair)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A document that changes language re-opens every sentence. Without the verdict
|
||||||
|
// in the chunk salt, the sentences she didn't touch would keep serving cards
|
||||||
|
// written in the language the document no longer speaks.
|
||||||
|
func TestLanguageFlipReopensCheckedSentences(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[]}`}
|
||||||
|
const enStart = "The weather was very cold this morning."
|
||||||
|
srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, enStart)
|
||||||
|
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("first check: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
first := client.calls
|
||||||
|
|
||||||
|
// She rewrites the document in Portuguese, keeping the first sentence.
|
||||||
|
setDocText(t, h, docID, enStart+" "+ptDocument)
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", ""); rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("second check: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if client.calls == first {
|
||||||
|
t.Fatal("the flipped document was never sent to the model")
|
||||||
|
}
|
||||||
|
if !strings.Contains(client.lastPrompt, "The weather was very cold") {
|
||||||
|
t.Fatalf("the already-checked sentence was not re-opened by the flip:\n%s", client.lastPrompt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The translate card, pointed the other way. She is writing her journal in
|
||||||
|
// Portuguese and drops in the one English sentence she knows; Petal renders it
|
||||||
|
// into Portuguese, and that card is a translation — not a correction to prose
|
||||||
|
// that was never wrong.
|
||||||
|
func TestEnglishSpanBecomesATranslateCardInAPortugueseDocument(t *testing.T) {
|
||||||
|
const english = "I want to say this but I don't know how to say it."
|
||||||
|
// The model volunteers "clarity", as it did for the zh case. Not consulted.
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"` + english + `","replacement":"Eu quero dizer isto mas não sei como o dizer.","explanation":"Aqui está em português.","type":"clarity"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, _ := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument+" "+english)
|
||||||
|
|
||||||
|
var out []db.Suggestion
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if len(out) != 1 {
|
||||||
|
t.Fatalf("want 1 card, got %d: %+v", len(out), out)
|
||||||
|
}
|
||||||
|
if out[0].Type != db.SuggestionTypeTranslate {
|
||||||
|
t.Fatalf("card type = %q, want %q", out[0].Type, db.SuggestionTypeTranslate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the half that keeps it honest: a genuine Portuguese correction in the same
|
||||||
|
// document stays a correction. Reading the English-document test backwards would
|
||||||
|
// have called this a translation, because every Portuguese sentence also "reads
|
||||||
|
// as English" by that test's deliberately low bar.
|
||||||
|
func TestPortugueseCorrectionKeepsItsTypeInAPortugueseDocument(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"Não sei porque isso é tão difícil para mim.","replacement":"Não sei porque isto é tão difícil para mim.","explanation":"Aqui usa-se isto.","type":"grammar"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, _ := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)
|
||||||
|
|
||||||
|
var out []db.Suggestion
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if len(out) != 1 {
|
||||||
|
t.Fatalf("want 1 card, got %d: %+v", len(out), out)
|
||||||
|
}
|
||||||
|
if out[0].Type == db.SuggestionTypeTranslate {
|
||||||
|
t.Fatal("a Portuguese correction inside a Portuguese document was labelled a translation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setDocLang writes a document's language verdict directly, so a test of the
|
||||||
|
// tap-through doesn't have to run a checkpoint through the same stub client to
|
||||||
|
// get one.
|
||||||
|
func setDocLang(t *testing.T, h *Handler, docID, lang string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := h.DB.Exec(`UPDATE documents SET doc_lang = ? WHERE id = ?`, lang, docID); err != nil {
|
||||||
|
t.Fatalf("set doc_lang: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// seedExplanation files one card carrying a given explanation and returns its
|
||||||
|
// id — the shape the translate tap-through needs, where only the explanation and
|
||||||
|
// the document it hangs off matter.
|
||||||
|
func seedExplanation(t *testing.T, h *Handler, docID, explanation string) string {
|
||||||
|
t.Helper()
|
||||||
|
var sugID string
|
||||||
|
if err := h.DB.QueryRow(
|
||||||
|
`INSERT INTO suggestions (doc_id, original, replacement, explanation, type, from_pos, to_pos)
|
||||||
|
VALUES (?, ?, ?, ?, ?, 0, 5) RETURNING id`,
|
||||||
|
docID, "isso", "isto", explanation, "grammar",
|
||||||
|
).Scan(&sugID); err != nil {
|
||||||
|
t.Fatalf("seed suggestion: %v", err)
|
||||||
|
}
|
||||||
|
return sugID
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tap-through has to read the same decision the card was written under. On a
|
||||||
|
// Portuguese document by a Portuguese writer the explanation already arrived in
|
||||||
|
// Portuguese, so the destination is the other half of the pair — the English she
|
||||||
|
// is practising — and never Portuguese into Portuguese again.
|
||||||
|
//
|
||||||
|
// This pins the report that "Petal presents the Ask Petal advice in both
|
||||||
|
// sections as Portuguese": the endpoint used to answer "" here, which left the
|
||||||
|
// card Portuguese, the bubble beneath it the same Portuguese, and no English on
|
||||||
|
// the card at all for a writer whose whole reason for the pair is English.
|
||||||
|
func TestTranslateRendersHerExplanationIntoTheEnglishSheIsLearning(t *testing.T) {
|
||||||
|
client := &stubClient{response: "Use this one here."}
|
||||||
|
srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)
|
||||||
|
setDocLang(t, h, docID, docLangPair)
|
||||||
|
sugID := seedExplanation(t, h, docID, "Aqui usa-se isto.")
|
||||||
|
|
||||||
|
rec := do(t, srv, http.MethodPost, "/suggestions/"+sugID+"/translate", "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("translate: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
var out translateResponse
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if out.Translation == "" {
|
||||||
|
t.Fatal("a Portuguese explanation left the tap with nowhere to go")
|
||||||
|
}
|
||||||
|
if !strings.Contains(client.lastPrompt, "into natural, friendly English") {
|
||||||
|
t.Fatalf("translate didn't render into English:\n%s", client.lastPrompt)
|
||||||
|
}
|
||||||
|
if strings.Contains(client.lastPrompt, "into natural, friendly European Portuguese") {
|
||||||
|
t.Fatalf("the model was asked to render Portuguese into Portuguese:\n%s", client.lastPrompt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The learner travelling the other way is the case that proves the endpoint
|
||||||
|
// derives its destination rather than skipping whenever a document is flipped: a
|
||||||
|
// native English speaker learning Chinese, writing Chinese, gets her
|
||||||
|
// explanations in English — and the tap still has somewhere to go.
|
||||||
|
func TestTranslateStillRendersForALearnersEnglishExplanation(t *testing.T) {
|
||||||
|
const zhDocument = "今天天气很好。我早上去公园散步。下午我在家里写作业。晚上我和朋友一起吃饭。"
|
||||||
|
client := &stubClient{response: "这里应该用这个。"}
|
||||||
|
srv, docID, h := newDirectedServer(t, client, "zh", auth.DirectionLearningPair, zhDocument)
|
||||||
|
setDocLang(t, h, docID, docLangPair)
|
||||||
|
sugID := seedExplanation(t, h, docID, "This measure word doesn't fit here.")
|
||||||
|
|
||||||
|
rec := do(t, srv, http.MethodPost, "/suggestions/"+sugID+"/translate", "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("translate: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
var out translateResponse
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if out.Translation == "" {
|
||||||
|
t.Fatal("a learner's English explanation was left untranslated")
|
||||||
|
}
|
||||||
|
if !strings.Contains(client.lastPrompt, "Simplified Chinese") {
|
||||||
|
t.Fatalf("translate didn't render into the pair language:\n%s", client.lastPrompt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPassAnnouncesItsVerdict pins the header the client reads. Storing the
|
||||||
|
// verdict on the document row is not enough on its own: the editor sees that row
|
||||||
|
// only when the document is opened or saved, and the pass that decides the
|
||||||
|
// verdict runs *after* a save — so the client would always be one save behind,
|
||||||
|
// and read-aloud is reached for exactly when she has stopped typing and no
|
||||||
|
// further save is coming. Caught in a browser: a Portuguese paragraph read in an
|
||||||
|
// American voice, twice, until another keystroke went in.
|
||||||
|
//
|
||||||
|
// Asserted on both endpoints that can flip it, and on the empty-document early
|
||||||
|
// return, which answers without ever reaching the model.
|
||||||
|
func TestPassAnnouncesItsVerdict(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[]}`}
|
||||||
|
srv, docID, h := newDirectedServer(t, client, "pt-PT", auth.DirectionLearningEn, ptDocument)
|
||||||
|
|
||||||
|
for _, path := range []string{"/check", "/voice"} {
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+path, "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: code=%d body=%s", path, rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get("X-Petal-Doc-Lang"); got != docLangPair {
|
||||||
|
t.Fatalf("%s: X-Petal-Doc-Lang = %q, want %q", path, got, docLangPair)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An English document says so rather than saying nothing — the client has to
|
||||||
|
// be able to hear a flip back, not just a flip away.
|
||||||
|
if _, err := h.DB.Exec(
|
||||||
|
`UPDATE documents SET content_text = ?, doc_lang = '' WHERE id = ?`,
|
||||||
|
"The weather was very cold this morning. I walked to the shop and bought some bread.", docID,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("rewrite doc: %v", err)
|
||||||
|
}
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get("X-Petal-Doc-Lang"); got != docLangEnglish {
|
||||||
|
t.Fatalf("English document: X-Petal-Doc-Lang = %q, want %q", got, docLangEnglish)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The empty-document path returns before the model call, and still answers.
|
||||||
|
if _, err := h.DB.Exec(`UPDATE documents SET content_text = '' WHERE id = ?`, docID); err != nil {
|
||||||
|
t.Fatalf("empty doc: %v", err)
|
||||||
|
}
|
||||||
|
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("empty check: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get("X-Petal-Doc-Lang"); got == "" {
|
||||||
|
t.Fatal("empty document answered with no verdict header at all")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOrdinaryProseIsEnoughEvidence is the regression for what the marker lists
|
||||||
|
// were caught doing on 2026-07-29, live, in a browser: unremarkable Portuguese
|
||||||
|
// read as English, because the list was curated against English so tightly that
|
||||||
|
// it had also been curated against ordinary writing. The document below scored
|
||||||
|
// two pair markers and zero English ones, and two is below the corroboration
|
||||||
|
// floor — so a paragraph with no evidence of English in it at all came back
|
||||||
|
// English, and was corrected and read aloud as English.
|
||||||
|
//
|
||||||
|
// Every sample here is prose a person might actually write, not prose chosen to
|
||||||
|
// contain markers. That is the whole point of the test: the failure was invisible
|
||||||
|
// to a suite whose fixtures all argued their own case.
|
||||||
|
func TestOrdinaryProseIsEnoughEvidence(t *testing.T) {
|
||||||
|
samples := []struct{ name, text string }{
|
||||||
|
{"the one seen live", "Esta manhã acordei cedo e fui correr ao longo da marginal. O ar estava fresco e havia poucas pessoas na rua. Depois comprei um jornal e li-o sentado num banco ao sol."},
|
||||||
|
{"an afternoon out", "Hoje o céu estava limpo e fomos até ao jardim junto ao rio. A minha mãe trouxe uma manta velha e sentámos-nos debaixo de uma árvore."},
|
||||||
|
{"plans", "Amanhã vamos ao cinema depois do trabalho. Ontem estava demasiado cansada para sair de casa."},
|
||||||
|
}
|
||||||
|
for _, s := range samples {
|
||||||
|
if got := documentLang(s.text, "pt-PT", ""); got != docLangPair {
|
||||||
|
t.Errorf("%s: documentLang = %q, want %q — ordinary Portuguese must not read as English\n%s",
|
||||||
|
s.name, got, docLangPair, s.text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEnglishDidNotGetEasierToMistake is the other half, and the reason the
|
||||||
|
// additions were held to "a word an English sentence has no reason to contain".
|
||||||
|
// Widening a marker list is only safe if it widens in one direction: these are
|
||||||
|
// English documents, including ones about Portugal and ones quoting Portuguese,
|
||||||
|
// and every one of them must still come back English.
|
||||||
|
func TestEnglishDidNotGetEasierToMistake(t *testing.T) {
|
||||||
|
samples := []struct{ name, text string }{
|
||||||
|
{"plain English", "This morning I woke up early and went for a run along the seafront. The air was fresh and there were few people about. Afterwards I bought a newspaper and read it on a bench."},
|
||||||
|
{"English about Portugal", "We spent a week in Lisbon last summer. The trams were crowded but the food was wonderful, and we walked up to the castle every evening."},
|
||||||
|
{"English quoting her", "My mother always says \"até amanhã\" when she leaves, never goodbye. I asked her why once and she said it sounded less final to her."},
|
||||||
|
{"an English diary", "Today was long. I had two meetings before lunch and another one after, and by the time I got home I could not think straight. Tomorrow should be quieter."},
|
||||||
|
}
|
||||||
|
for _, s := range samples {
|
||||||
|
if got := documentLang(s.text, "pt-PT", ""); got != docLangEnglish {
|
||||||
|
t.Errorf("%s: documentLang = %q, want %q — the widened list must not pull English across\n%s",
|
||||||
|
s.name, got, docLangEnglish, s.text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And the same document must not flip once it is already sitting in English:
|
||||||
|
// the hysteresis band is only a safety net if the low side holds too.
|
||||||
|
for _, s := range samples {
|
||||||
|
if got := documentLang(s.text, "pt-PT", docLangEnglish); got != docLangEnglish {
|
||||||
|
t.Errorf("%s: held verdict flipped to %q", s.name, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,230 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/vocab"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The growth journal.
|
||||||
|
//
|
||||||
|
// The suggestions table already records everything this needs — it is purely a
|
||||||
|
// read-side view, with no new capture and no model call. Two framing rules
|
||||||
|
// decide what may appear here, and they are enforced in the SQL rather than left
|
||||||
|
// to the copy:
|
||||||
|
//
|
||||||
|
// 1. It reports growth, never an error tally. Nothing counts what she got
|
||||||
|
// wrong this month; the signals are things that *stopped* happening and
|
||||||
|
// phrasing that *stuck*.
|
||||||
|
// 2. It only ever compares the writer to her own past self. There is no
|
||||||
|
// target, no average, no other user anywhere in these queries.
|
||||||
|
//
|
||||||
|
// A quiet month is quiet: every signal below is omitted rather than softened
|
||||||
|
// when the data isn't there, because an invented milestone is worse than none.
|
||||||
|
|
||||||
|
// Journal is one writer's growth over the recent windows.
|
||||||
|
type Journal struct {
|
||||||
|
// Kept / KeptBefore are edits she took on board in the last 30 days and in
|
||||||
|
// the 30 before that — her own past self, the only comparison offered.
|
||||||
|
Kept int `json:"kept"`
|
||||||
|
KeptBefore int `json:"kept_before"`
|
||||||
|
// Stuck: phrasing she was given that now turns up across her own writing.
|
||||||
|
Stuck []Chunk `json:"stuck"`
|
||||||
|
// Faded: things she used to need fixing and hasn't, recently.
|
||||||
|
Faded []Fade `json:"faded"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunk is a phrase that has stuck: it appears in Docs of her documents now.
|
||||||
|
type Chunk struct {
|
||||||
|
Phrase string `json:"phrase"`
|
||||||
|
Docs int `json:"docs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fade is a pattern that has stopped appearing. Times is how often it came up
|
||||||
|
// during the earlier window — context for "and not since", never a scoreboard.
|
||||||
|
type Fade struct {
|
||||||
|
Pattern string `json:"pattern"`
|
||||||
|
Times int `json:"times"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Journal windows, in days. `recent` is the month being reported on; `history`
|
||||||
|
// reaches back far enough that a pattern's absence means something (one quiet
|
||||||
|
// fortnight doesn't).
|
||||||
|
const (
|
||||||
|
recentDays = 30
|
||||||
|
historyDays = 120
|
||||||
|
maxSignals = 3 // per list: a journal is a couple of warm lines, not a report
|
||||||
|
)
|
||||||
|
|
||||||
|
// growth serves GET /api/suggestions/growth.
|
||||||
|
func (h *Handler) growth(w http.ResponseWriter, r *http.Request) {
|
||||||
|
userID := auth.UserID(r.Context())
|
||||||
|
j := Journal{Stuck: []Chunk{}, Faded: []Fade{}}
|
||||||
|
|
||||||
|
err := h.DB.QueryRow(
|
||||||
|
`SELECT
|
||||||
|
sum(CASE WHEN s.resolved_at >= datetime('now', '-30 days') THEN 1 ELSE 0 END),
|
||||||
|
sum(CASE WHEN s.resolved_at < datetime('now', '-30 days')
|
||||||
|
AND s.resolved_at >= datetime('now', '-60 days') THEN 1 ELSE 0 END)
|
||||||
|
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||||
|
WHERE d.user_id = ? AND s.status = 'accepted' AND s.resolved_at IS NOT NULL`,
|
||||||
|
userID,
|
||||||
|
).Scan(&nullInt{&j.Kept}, &nullInt{&j.KeptBefore})
|
||||||
|
if err != nil {
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
stuck, err := h.stuck(userID)
|
||||||
|
if err != nil {
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
j.Stuck = stuck
|
||||||
|
|
||||||
|
faded, err := h.faded(userID)
|
||||||
|
if err != nil {
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
j.Faded = faded
|
||||||
|
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, j)
|
||||||
|
}
|
||||||
|
|
||||||
|
// stuck finds accepted phrasing that now appears in more than one of her own
|
||||||
|
// documents. One document is just the edit itself, still sitting where it was
|
||||||
|
// applied; a second is her reaching for the phrase on her own, which is the
|
||||||
|
// whole claim the line makes.
|
||||||
|
func (h *Handler) stuck(userID string) ([]Chunk, error) {
|
||||||
|
rows, err := h.DB.Query(
|
||||||
|
`SELECT DISTINCT s.replacement
|
||||||
|
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||||
|
WHERE d.user_id = ? AND s.status = 'accepted'
|
||||||
|
AND s.resolved_at >= datetime('now', '-120 days')
|
||||||
|
AND trim(s.replacement) != ''
|
||||||
|
ORDER BY s.resolved_at DESC
|
||||||
|
LIMIT 40`,
|
||||||
|
userID,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
// vocab.PhraseKey is the same definition of "a learnable chunk" the garden
|
||||||
|
// plants, so the journal and the garden can never disagree about what counts.
|
||||||
|
var phrases []string
|
||||||
|
for rows.Next() {
|
||||||
|
var replacement string
|
||||||
|
if err := rows.Scan(&replacement); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if key := vocab.PhraseKey(replacement); key != "" {
|
||||||
|
phrases = append(phrases, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
out := []Chunk{}
|
||||||
|
for _, p := range phrases {
|
||||||
|
var docs int
|
||||||
|
if err := h.DB.QueryRow(
|
||||||
|
`SELECT count(*) FROM documents WHERE user_id = ? AND instr(lower(content_text), ?) > 0`,
|
||||||
|
userID, p,
|
||||||
|
).Scan(&docs); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if docs >= 2 {
|
||||||
|
out = append(out, Chunk{Phrase: p, Docs: docs})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sortDesc(out, func(c Chunk) int { return c.Docs })
|
||||||
|
return trim(out, maxSignals), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// faded finds patterns she used to be corrected on during the earlier part of
|
||||||
|
// the history window and hasn't been since.
|
||||||
|
//
|
||||||
|
// The guard that makes this honest: it says nothing at all unless she has
|
||||||
|
// actually been writing lately. Without it, a month away from Petal would be
|
||||||
|
// reported back to her as progress, which is the one way this feature could lie.
|
||||||
|
func (h *Handler) faded(userID string) ([]Fade, error) {
|
||||||
|
var wroteRecently int
|
||||||
|
if err := h.DB.QueryRow(
|
||||||
|
`SELECT count(*) FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||||
|
WHERE d.user_id = ? AND s.resolved_at >= datetime('now', '-30 days')`,
|
||||||
|
userID,
|
||||||
|
).Scan(&wroteRecently); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if wroteRecently == 0 {
|
||||||
|
return []Fade{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := h.DB.Query(
|
||||||
|
`SELECT lower(trim(s.original)) AS pattern, count(*) AS times
|
||||||
|
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||||
|
WHERE d.user_id = ? AND s.status = 'accepted'
|
||||||
|
AND s.resolved_at < datetime('now', '-30 days')
|
||||||
|
AND s.resolved_at >= datetime('now', '-120 days')
|
||||||
|
AND trim(s.original) != ''
|
||||||
|
AND pattern NOT IN (
|
||||||
|
SELECT lower(trim(s2.original))
|
||||||
|
FROM suggestions s2 JOIN documents d2 ON d2.id = s2.doc_id
|
||||||
|
WHERE d2.user_id = ? AND s2.status = 'accepted'
|
||||||
|
AND s2.resolved_at >= datetime('now', '-30 days'))
|
||||||
|
GROUP BY pattern
|
||||||
|
HAVING times >= 2
|
||||||
|
ORDER BY times DESC
|
||||||
|
LIMIT 3`,
|
||||||
|
userID, userID,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
out := []Fade{}
|
||||||
|
for rows.Next() {
|
||||||
|
var f Fade
|
||||||
|
if err := rows.Scan(&f.Pattern, &f.Times); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, f)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// nullInt scans a possibly-NULL aggregate into an int (SUM over no rows is
|
||||||
|
// NULL, which is a zero here, not an error).
|
||||||
|
type nullInt struct{ dst *int }
|
||||||
|
|
||||||
|
func (n *nullInt) Scan(v any) error {
|
||||||
|
switch t := v.(type) {
|
||||||
|
case int64:
|
||||||
|
*n.dst = int(t)
|
||||||
|
case nil:
|
||||||
|
*n.dst = 0
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortDesc[T any](s []T, key func(T) int) {
|
||||||
|
for i := 1; i < len(s); i++ {
|
||||||
|
for j := i; j > 0 && key(s[j]) > key(s[j-1]); j-- {
|
||||||
|
s[j], s[j-1] = s[j-1], s[j]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func trim[T any](s []T, n int) []T {
|
||||||
|
if len(s) > n {
|
||||||
|
return s[:n]
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
// resolved seeds one already-settled suggestion, dated `daysAgo` at the moment
|
||||||
|
// she decided it (the journal reads decisions, not proposals).
|
||||||
|
func resolved(t *testing.T, h *Handler, docID, status, original, replacement string, daysAgo int) {
|
||||||
|
t.Helper()
|
||||||
|
_, err := h.DB.Exec(
|
||||||
|
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at)
|
||||||
|
VALUES (?, 0, 0, ?, ?, '', 'collocation', ?, datetime('now', ?), datetime('now', ?))`,
|
||||||
|
docID, original, replacement, status,
|
||||||
|
"-"+strconv.Itoa(daysAgo)+" days", "-"+strconv.Itoa(daysAgo)+" days",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("seed resolved suggestion: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func seedDoc(t *testing.T, h *Handler, userID, text string) string {
|
||||||
|
t.Helper()
|
||||||
|
var id string
|
||||||
|
if err := h.DB.QueryRow(
|
||||||
|
`INSERT INTO documents (user_id, content_text) VALUES (?, ?) RETURNING id`, userID, text,
|
||||||
|
).Scan(&id); err != nil {
|
||||||
|
t.Fatalf("seed doc: %v", err)
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
func readJournal(t *testing.T, srv http.Handler) Journal {
|
||||||
|
t.Helper()
|
||||||
|
rec := do(t, srv, http.MethodGet, "/suggestions/growth", "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("growth: got %d, want 200 (body %s)", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var j Journal
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &j); err != nil {
|
||||||
|
t.Fatalf("decode journal: %v", err)
|
||||||
|
}
|
||||||
|
return j
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestJournalIsEmptyForANewWriter: nothing to report reports nothing. Empty
|
||||||
|
// lists, not nulls, so the frontend never has to guess.
|
||||||
|
func TestJournalIsEmptyForANewWriter(t *testing.T) {
|
||||||
|
srv, _, _ := newTestServer(t, &stubClient{})
|
||||||
|
j := readJournal(t, srv)
|
||||||
|
if j.Kept != 0 || j.KeptBefore != 0 || len(j.Stuck) != 0 || len(j.Faded) != 0 {
|
||||||
|
t.Fatalf("new writer got a journal: %+v", j)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestKeptComparesHerToHerOwnPastSelf.
|
||||||
|
func TestKeptCountsTwoWindows(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
resolved(t, h, docID, "accepted", "do a decision", "make a decision", 5)
|
||||||
|
}
|
||||||
|
resolved(t, h, docID, "accepted", "big rain", "heavy rain", 40)
|
||||||
|
resolved(t, h, docID, "rejected", "no thanks", "no, thank you", 5) // decisions kept only
|
||||||
|
resolved(t, h, docID, "accepted", "long ago", "long since", 200) // outside both windows
|
||||||
|
|
||||||
|
j := readJournal(t, srv)
|
||||||
|
if j.Kept != 3 {
|
||||||
|
t.Errorf("Kept = %d, want 3", j.Kept)
|
||||||
|
}
|
||||||
|
if j.KeptBefore != 1 {
|
||||||
|
t.Errorf("KeptBefore = %d, want 1", j.KeptBefore)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStuckNeedsASecondDocument: a phrase sitting in the one document it was
|
||||||
|
// applied to has not stuck — it's just the edit, where she left it. A second
|
||||||
|
// document is her reaching for it herself, which is the claim the line makes.
|
||||||
|
func TestStuckNeedsASecondDocument(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
if _, err := h.DB.Exec(`UPDATE documents SET content_text = ? WHERE id = ?`,
|
||||||
|
"I had to make a decision.", docID); err != nil {
|
||||||
|
t.Fatalf("set content: %v", err)
|
||||||
|
}
|
||||||
|
resolved(t, h, docID, "accepted", "do a decision", "make a decision", 10)
|
||||||
|
resolved(t, h, docID, "accepted", "do a photo", "take a photo", 10)
|
||||||
|
|
||||||
|
if j := readJournal(t, srv); len(j.Stuck) != 0 {
|
||||||
|
t.Fatalf("one document counted as sticking: %+v", j.Stuck)
|
||||||
|
}
|
||||||
|
|
||||||
|
// She uses it again, elsewhere, on her own.
|
||||||
|
seedDoc(t, h, db.LocalUserID, "Later I had to Make A Decision about the flat.")
|
||||||
|
j := readJournal(t, srv)
|
||||||
|
if len(j.Stuck) != 1 {
|
||||||
|
t.Fatalf("Stuck = %+v, want just the phrase she reused", j.Stuck)
|
||||||
|
}
|
||||||
|
if j.Stuck[0].Phrase != "make a decision" || j.Stuck[0].Docs != 2 {
|
||||||
|
t.Errorf("Stuck[0] = %+v, want {make a decision 2} (case-insensitive)", j.Stuck[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFadedNeedsRecentWriting is the guard that keeps this feature honest: a
|
||||||
|
// month away from Petal must never be reported back as progress.
|
||||||
|
func TestFadedNeedsRecentWriting(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 60)
|
||||||
|
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 55)
|
||||||
|
|
||||||
|
if j := readJournal(t, srv); len(j.Faded) != 0 {
|
||||||
|
t.Fatalf("silence reported as growth: %+v", j.Faded)
|
||||||
|
}
|
||||||
|
|
||||||
|
// She has been writing again this month — now the absence means something.
|
||||||
|
resolved(t, h, docID, "accepted", "big rain", "heavy rain", 3)
|
||||||
|
j := readJournal(t, srv)
|
||||||
|
if len(j.Faded) != 1 || j.Faded[0].Pattern != "在 the morning" || j.Faded[0].Times != 2 {
|
||||||
|
t.Fatalf("Faded = %+v, want the pattern she stopped needing (twice, back then)", j.Faded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFadedExcludesWhatStillHappens: a pattern corrected again this month has
|
||||||
|
// not faded, however often it came up before.
|
||||||
|
func TestFadedExcludesWhatStillHappens(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 60)
|
||||||
|
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 55)
|
||||||
|
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 2)
|
||||||
|
|
||||||
|
if j := readJournal(t, srv); len(j.Faded) != 0 {
|
||||||
|
t.Fatalf("Faded = %+v, want empty — it still happens", j.Faded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestJournalIsPerWriter: another account's learning is never anyone else's
|
||||||
|
// journal, and the only comparison Petal draws is with her own past self.
|
||||||
|
func TestJournalIsPerWriter(t *testing.T) {
|
||||||
|
srv, _, h := newTestServer(t, &stubClient{})
|
||||||
|
if _, err := h.DB.Exec(`INSERT INTO users (id, email) VALUES ('bob', 'bob@example.com')`); err != nil {
|
||||||
|
t.Fatalf("seed user: %v", err)
|
||||||
|
}
|
||||||
|
bobDoc := seedDoc(t, h, "bob", "Bob had to make a decision.")
|
||||||
|
seedDoc(t, h, "bob", "Bob will make a decision again.")
|
||||||
|
resolved(t, h, bobDoc, "accepted", "do a decision", "make a decision", 5)
|
||||||
|
resolved(t, h, bobDoc, "accepted", "big rain", "heavy rain", 60)
|
||||||
|
resolved(t, h, bobDoc, "accepted", "big rain", "heavy rain", 55)
|
||||||
|
|
||||||
|
j := readJournal(t, srv)
|
||||||
|
if j.Kept != 0 || j.KeptBefore != 0 || len(j.Stuck) != 0 || len(j.Faded) != 0 {
|
||||||
|
t.Fatalf("bob's learning leaked into the local user's journal: %+v", j)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -20,6 +21,7 @@ import (
|
|||||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/vocab"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Handler holds the dependencies for the checkpoint + suggestion routes. The
|
// Handler holds the dependencies for the checkpoint + suggestion routes. The
|
||||||
@@ -54,12 +56,17 @@ func (h *Handler) RegisterDocRoutes(r chi.Router) {
|
|||||||
r.Post("/{id}/collocation", h.collocation)
|
r.Post("/{id}/collocation", h.collocation)
|
||||||
r.Post("/{id}/rewrite", h.rewrite)
|
r.Post("/{id}/rewrite", h.rewrite)
|
||||||
r.Get("/{id}/suggestions", h.listForDoc)
|
r.Get("/{id}/suggestions", h.listForDoc)
|
||||||
|
r.Get("/{id}/settled", h.listSettled)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Routes returns the router mounted at /api/suggestions for per-suggestion
|
// Routes returns the router mounted at /api/suggestions for per-suggestion
|
||||||
// actions.
|
// actions.
|
||||||
func (h *Handler) Routes() chi.Router {
|
func (h *Handler) Routes() chi.Router {
|
||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
|
// The growth journal reads the same table these actions write, so it lives
|
||||||
|
// here rather than growing its own mount. A literal segment, so it can never
|
||||||
|
// be shadowed by an id.
|
||||||
|
r.Get("/growth", h.growth)
|
||||||
r.Post("/{id}/accept", h.accept)
|
r.Post("/{id}/accept", h.accept)
|
||||||
r.Post("/{id}/dismiss", h.dismiss)
|
r.Post("/{id}/dismiss", h.dismiss)
|
||||||
r.Post("/{id}/chat", h.chat)
|
r.Post("/{id}/chat", h.chat)
|
||||||
@@ -84,6 +91,22 @@ type mechanicsFinding struct {
|
|||||||
Original string `json:"original"`
|
Original string `json:"original"`
|
||||||
Replacement string `json:"replacement"`
|
Replacement string `json:"replacement"`
|
||||||
Explanation string `json:"explanation"`
|
Explanation string `json:"explanation"`
|
||||||
|
// Which family this offline finding belongs to. Empty (the historical shape)
|
||||||
|
// means mechanics; the miscollocation rules send 'collocation' so a chunk the
|
||||||
|
// rule pack caught is indistinguishable from one the coach caught — same
|
||||||
|
// family, same rail, and the same planting into the garden on accept.
|
||||||
|
Type string `json:"type"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// localType maps a client-supplied family onto the two an offline rule may claim.
|
||||||
|
// Anything else — including the empty string older clients send — is mechanics,
|
||||||
|
// so a stray label can never smuggle a row into an LLM family and survive that
|
||||||
|
// pass's DELETE.
|
||||||
|
func localType(t string) string {
|
||||||
|
if strings.ToLower(strings.TrimSpace(t)) == db.SuggestionTypeCollocation {
|
||||||
|
return db.SuggestionTypeCollocation
|
||||||
|
}
|
||||||
|
return db.SuggestionTypeMechanics
|
||||||
}
|
}
|
||||||
|
|
||||||
// maxMechanicsFindings caps a single submission so a runaway client can't flood
|
// maxMechanicsFindings caps a single submission so a runaway client can't flood
|
||||||
@@ -138,10 +161,21 @@ func (h *Handler) mechanics(w http.ResponseWriter, r *http.Request) {
|
|||||||
httputil.WriteJSON(w, http.StatusOK, out)
|
httputil.WriteJSON(w, http.StatusOK, out)
|
||||||
}
|
}
|
||||||
|
|
||||||
// replaceMechanics swaps the document's pending mechanics rows for the supplied
|
// replaceMechanics brings the document's pending offline rows in line with the
|
||||||
// findings in one transaction, leaving the LLM families and actioned rows
|
// supplied findings in one transaction, leaving the LLM families and actioned
|
||||||
// untouched. Findings the user already accepted or dismissed are suppressed (the
|
// rows untouched. Findings the user already accepted or dismissed are suppressed
|
||||||
// detector has no memory between runs), and malformed spans are skipped.
|
// (the detector has no memory between runs), and malformed spans are skipped.
|
||||||
|
//
|
||||||
|
// A finding the detector still reports keeps its existing row — same id, same
|
||||||
|
// created_at — and only its offsets move. This pass fires 250 ms after a
|
||||||
|
// keystroke, so deleting and re-inserting the family would hand every card a new
|
||||||
|
// identity several times a sentence: the rail would remount, a card expanded for
|
||||||
|
// Ask Petal would collapse under her, and the arrival chime would re-fire.
|
||||||
|
//
|
||||||
|
// The scope is *source*, not type: the rule pack owns both the mechanics family
|
||||||
|
// and its share of the collocation family, and every run is a full recompute of
|
||||||
|
// the document. Scoping by type instead would strand offline collocations the
|
||||||
|
// current text no longer warrants — the one row nobody would ever replace.
|
||||||
func (h *Handler) replaceMechanics(docID string, findings []mechanicsFinding) error {
|
func (h *Handler) replaceMechanics(docID string, findings []mechanicsFinding) error {
|
||||||
tx, err := h.DB.Begin()
|
tx, err := h.DB.Begin()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -149,18 +183,18 @@ func (h *Handler) replaceMechanics(docID string, findings []mechanicsFinding) er
|
|||||||
}
|
}
|
||||||
defer tx.Rollback()
|
defer tx.Rollback()
|
||||||
|
|
||||||
if _, err := tx.Exec(
|
existing, err := loadPending(tx, docID, "source = '"+db.SuggestionSourceLocal+"'")
|
||||||
`DELETE FROM suggestions WHERE doc_id = ? AND status = ? AND type = ?`,
|
if err != nil {
|
||||||
docID, db.SuggestionStatusPending, db.SuggestionTypeMechanics,
|
|
||||||
); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
index := indexByEdit(existing)
|
||||||
|
|
||||||
sup, err := buildSuppressor(tx, docID)
|
sup, err := buildSuppressor(tx, docID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
kept := make(map[string]bool, len(existing))
|
||||||
for _, f := range findings {
|
for _, f := range findings {
|
||||||
if f.From < 0 || f.To <= f.From || strings.TrimSpace(f.Original) == "" {
|
if f.From < 0 || f.To <= f.From || strings.TrimSpace(f.Original) == "" {
|
||||||
continue // malformed span — the client re-anchors by string anyway
|
continue // malformed span — the client re-anchors by string anyway
|
||||||
@@ -168,15 +202,34 @@ func (h *Handler) replaceMechanics(docID string, findings []mechanicsFinding) er
|
|||||||
if sup.suppressed(f.Original, f.Replacement) {
|
if sup.suppressed(f.Original, f.Replacement) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
typ := localType(f.Type)
|
||||||
|
if row, ok := index.take(f.Original, f.Replacement, f.From); ok {
|
||||||
|
kept[row.id] = true
|
||||||
|
if err := reposition(tx, row, f.From, f.To, ""); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
if _, err := tx.Exec(
|
if _, err := tx.Exec(
|
||||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, source)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||||
docID, f.From, f.To, f.Original, f.Replacement, f.Explanation, db.SuggestionTypeMechanics,
|
docID, f.From, f.To, f.Original, f.Replacement, f.Explanation,
|
||||||
|
typ, db.SuggestionSourceLocal,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Whatever the detector no longer reports, she has fixed.
|
||||||
|
for _, row := range existing {
|
||||||
|
if kept[row.id] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, row.id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return tx.Commit()
|
return tx.Commit()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -194,12 +247,45 @@ func (h *Handler) collocation(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// pass is the signature shared by the grammar checkpoint and the voice pass:
|
// pass is the signature shared by the grammar checkpoint and the voice pass:
|
||||||
// given the document text, the document's tone and the writer's pair language it
|
// given the document text, the document's tone and the languages this document
|
||||||
// returns the model's raw suggestions. The voice pass ignores both extras (see
|
// is to be corrected and explained in, it returns the model's raw suggestions.
|
||||||
// llm.RunVoice) and the checkpoint ignores the language — only the collocation
|
// The voice pass ignores the tone (see llm.RunVoice) and the collocation coach
|
||||||
// coach writes a word of it — but one signature keeps runPass free of special
|
// reads only the writer's pair language, but one signature keeps runPass free of
|
||||||
// cases.
|
// special cases.
|
||||||
type pass func(ctx context.Context, client llm.LLMClient, contentText, tone string, lang llm.Lang) ([]llm.RawSuggestion, error)
|
type pass func(ctx context.Context, client llm.LLMClient, contentText, tone string, t llm.Target) ([]llm.RawSuggestion, error)
|
||||||
|
|
||||||
|
// targetFor resolves the two language decisions for one pass over one document.
|
||||||
|
//
|
||||||
|
// They read different state on purpose. What gets *corrected* follows the
|
||||||
|
// document, because Portuguese prose wants Portuguese corrections. What language
|
||||||
|
// the correction is *explained* in follows the writer — the half of her pair she
|
||||||
|
// is not learning — because an explanation is teaching, and teaching lands in the
|
||||||
|
// language she reads most easily. A native Portuguese speaker practising English
|
||||||
|
// gets Portuguese explained in Portuguese; a native English speaker learning
|
||||||
|
// French gets French explained in English. Neither is trapped: the other language
|
||||||
|
// stays one tap away, in both directions.
|
||||||
|
//
|
||||||
|
// An English document keeps the pre-Phase-28 behaviour exactly — explained in
|
||||||
|
// English, with her language on the Ask Petal / translate taps — which is the
|
||||||
|
// path every account today is on.
|
||||||
|
//
|
||||||
|
// The direction lookup costs nothing today: `learnerPairs` is {"zh"}, so fr, es
|
||||||
|
// and pt-PT accounts are all learning_en and their non-learned half *is* the pair
|
||||||
|
// language. This rule therefore produces "explain in the document's language" for
|
||||||
|
// every writer who currently exists. It is written out anyway to stop the
|
||||||
|
// coincidence being baked into the prompts, the way "English is the language
|
||||||
|
// being learned" was baked into pair_lang before migration 0016.
|
||||||
|
func targetFor(pairLang, direction, docLang string) llm.Target {
|
||||||
|
pair := llm.LangFor(pairLang)
|
||||||
|
if normalizeDocLang(docLang) != docLangPair {
|
||||||
|
return llm.EnglishTarget(pair)
|
||||||
|
}
|
||||||
|
explain := pair
|
||||||
|
if direction == auth.DirectionLearningPair {
|
||||||
|
explain = llm.English
|
||||||
|
}
|
||||||
|
return llm.Target{Correct: pair, Explain: explain, Pair: pair}
|
||||||
|
}
|
||||||
|
|
||||||
// runPass is the shared body for both LLM passes. It loads the document text,
|
// runPass is the shared body for both LLM passes. It loads the document text,
|
||||||
// enforces the pass's per-document rate limit, runs the model, swaps in the
|
// enforces the pass's per-document rate limit, runs the model, swaps in the
|
||||||
@@ -209,17 +295,19 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
|||||||
docID := chi.URLParam(r, "id")
|
docID := chi.URLParam(r, "id")
|
||||||
userID := auth.UserID(r.Context())
|
userID := auth.UserID(r.Context())
|
||||||
|
|
||||||
// The writer's pair language rides along with the document rather than in a
|
// The writer's pair language and direction ride along with the document
|
||||||
// second query: it is read from the same row-scoped lookup that already
|
// rather than in a second query: they are read from the same row-scoped
|
||||||
// proves she owns this document.
|
// lookup that already proves she owns this document. `doc_lang` is the
|
||||||
var contentText, tone, pairLang string
|
// previous language verdict, which the new one needs (hysteresis).
|
||||||
|
var contentText, tone, pairLang, direction, prevLang string
|
||||||
err := h.DB.QueryRow(
|
err := h.DB.QueryRow(
|
||||||
`SELECT d.content_text, d.tone, COALESCE(u.pair_lang, '')
|
`SELECT d.content_text, d.tone, d.doc_lang,
|
||||||
|
COALESCE(u.pair_lang, ''), COALESCE(u.direction, '')
|
||||||
FROM documents d
|
FROM documents d
|
||||||
JOIN users u ON u.id = d.user_id
|
JOIN users u ON u.id = d.user_id
|
||||||
WHERE d.id = ? AND d.user_id = ?`,
|
WHERE d.id = ? AND d.user_id = ?`,
|
||||||
docID, userID,
|
docID, userID,
|
||||||
).Scan(&contentText, &tone, &pairLang)
|
).Scan(&contentText, &tone, &prevLang, &pairLang, &direction)
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
httputil.ErrorJSON(w, http.StatusNotFound, "document not found")
|
httputil.ErrorJSON(w, http.StatusNotFound, "document not found")
|
||||||
return
|
return
|
||||||
@@ -229,11 +317,103 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Nothing to analyze on an empty document — skip the LLM round-trip.
|
// What language is this document in, and so what language should its cards be
|
||||||
|
// written in? Computed from the whole content_text — never from `askText`,
|
||||||
|
// which on a chunked pass is only the sentences that changed, and would put an
|
||||||
|
// English card in a Portuguese journal the moment she edits its one English
|
||||||
|
// line.
|
||||||
|
//
|
||||||
|
// Decided before the empty-document exit so every reconcile below is told the
|
||||||
|
// same verdict. An emptied document has nothing to go on and holds whatever it
|
||||||
|
// said last (see documentLang), which is what keeps a Portuguese journal
|
||||||
|
// Portuguese while she clears it to start the entry again.
|
||||||
|
docLang := documentLang(contentText, pairLang, prevLang)
|
||||||
|
|
||||||
|
// Announce the verdict on every answer this pass gives, including the early
|
||||||
|
// ones below. This pass is the only thing that decides the value, so it is
|
||||||
|
// the only moment the client can learn it promptly — and the client needs it
|
||||||
|
// promptly for read-aloud, which is reached for exactly when she has stopped
|
||||||
|
// typing and no further save is coming. Carrying it back on the document row
|
||||||
|
// alone means the editor is always one save behind the truth, and a paragraph
|
||||||
|
// of Portuguese read in an American voice is how that sounds.
|
||||||
|
//
|
||||||
|
// A header rather than a wider body: /check and /voice answer with a bare
|
||||||
|
// array of the unified pending set, and every caller of both endpoints reads
|
||||||
|
// it as one. A verdict is metadata about the pass, not another suggestion.
|
||||||
|
w.Header().Set("X-Petal-Doc-Lang", docLang)
|
||||||
|
|
||||||
|
// Nothing to analyze on an empty document — skip the LLM round-trip. The
|
||||||
|
// family's rows go with the text they were about.
|
||||||
if strings.TrimSpace(contentText) == "" {
|
if strings.TrimSpace(contentText) == "" {
|
||||||
httputil.WriteJSON(w, http.StatusOK, []db.Suggestion{})
|
if err := h.reconcilePending(docID, contentText, pairLang, docLang, nil, scope, nil, nil, false); err != nil {
|
||||||
|
httputil.ServerError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
out, err := h.fetchPending(userID, docID)
|
||||||
|
if err != nil {
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, out)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if docLang != normalizeDocLang(prevLang) {
|
||||||
|
if _, err := h.DB.Exec(
|
||||||
|
`UPDATE documents SET doc_lang = ? WHERE id = ? AND user_id = ?`,
|
||||||
|
docLang, docID, userID,
|
||||||
|
); err != nil {
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
target := targetFor(pairLang, direction, docLang)
|
||||||
|
|
||||||
|
// Decide what to ask about before spending anything: a chunked pass asks only
|
||||||
|
// about the sentences that changed since it last read the document, and when
|
||||||
|
// none did it doesn't call the model at all — nor consume its rate-limit slot,
|
||||||
|
// so the next real edit isn't throttled by a check that had nothing to do.
|
||||||
|
//
|
||||||
|
// Only a chunked pass consults that record, so only it needs the tone folded
|
||||||
|
// into a sentence's identity — and, next to it, the language verdict. A
|
||||||
|
// document that flips language changes every sentence's identity, so its
|
||||||
|
// old-language cards are re-checked rather than left sitting there in a
|
||||||
|
// language the rest of the document no longer speaks.
|
||||||
|
salt := ""
|
||||||
|
if scope.chunked {
|
||||||
|
salt = tone + "\x00" + docLang
|
||||||
|
}
|
||||||
|
chunks := splitChunks(contentText, salt)
|
||||||
|
askText, fresh := contentText, chunks
|
||||||
|
if scope.chunked {
|
||||||
|
checked, err := h.checkedChunks(docID, scope.family)
|
||||||
|
if err != nil {
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
changed := changedChunks(chunks, checked)
|
||||||
|
if len(changed) == 0 {
|
||||||
|
// Every sentence has already been read. Drop the rows whose sentence is
|
||||||
|
// gone, keep the rest exactly as they are, and answer immediately.
|
||||||
|
if err := h.reconcilePending(docID, contentText, pairLang, docLang, nil, scope, chunks, nil, false); err != nil {
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
out, err := h.fetchPending(userID, docID)
|
||||||
|
if err != nil {
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, out)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// When every sentence is new — a first pass, a paste, a tone switch — hand
|
||||||
|
// over the document verbatim so the model reads it with its paragraphing
|
||||||
|
// intact. Otherwise send just the delta, one sentence per line.
|
||||||
|
if len(changed) < len(hashSet(chunks)) {
|
||||||
|
askText, fresh = joinChunks(changed), changed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
ok, _, slotAt := limiter.Allow(docID)
|
ok, _, slotAt := limiter.Allow(docID)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -248,17 +428,19 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
raw, err := run(r.Context(), h.Client, contentText, tone, llm.LangFor(pairLang))
|
raw, err := run(r.Context(), h.Client, askText, tone, target)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Allow ran before the model call, so a failed pass would otherwise hold
|
// Allow ran before the model call, so a failed pass would otherwise hold
|
||||||
// the per-document slot for the full interval — stranding the frontend's
|
// the per-document slot for the full interval — stranding the frontend's
|
||||||
// auto-retry on the throttle path. Release it so a retry can re-run.
|
// auto-retry on the throttle path. Release it so a retry can re-run.
|
||||||
limiter.Release(docID, slotAt)
|
limiter.Release(docID, slotAt)
|
||||||
httputil.ErrorJSON(w, http.StatusBadGateway, "llm pass failed: "+err.Error())
|
httputil.UpstreamError(w, "pass", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := h.replacePending(docID, contentText, raw, scope); err != nil {
|
// A whole-document pass re-read everything, so every one of its rows is up for
|
||||||
|
// re-proposal; a chunked pass only puts the sentences it asked about in play.
|
||||||
|
if err := h.reconcilePending(docID, contentText, pairLang, docLang, raw, scope, chunks, fresh, !scope.chunked); err != nil {
|
||||||
httputil.ServerError(w, err)
|
httputil.ServerError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -279,73 +461,49 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
|||||||
// inserts. The grammar checkpoint and voice pass each own a disjoint family, so
|
// inserts. The grammar checkpoint and voice pass each own a disjoint family, so
|
||||||
// running one never disturbs the other's pending flags.
|
// running one never disturbs the other's pending flags.
|
||||||
type pendingScope struct {
|
type pendingScope struct {
|
||||||
deleteWhere string // extra WHERE clause scoping the DELETE to this family
|
deleteWhere string // extra WHERE clause scoping this pass to its own family
|
||||||
forceType string // if set, every inserted row gets this type; else normalizeType
|
forceType string // if set, every inserted row gets this type; else normalizeType
|
||||||
|
// family keys the sentences this pass has already read (see checked_chunks).
|
||||||
|
family string
|
||||||
|
// chunked passes re-read only the sentences that changed. True for the typing-
|
||||||
|
// cadence grammar checkpoint, which fires constantly and must feel still;
|
||||||
|
// false for the explicit whole-document passes, where she pressed a button
|
||||||
|
// asking for a fresh read of everything.
|
||||||
|
chunked bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Every scope below is confined to source='llm'. The offline rule pack owns its
|
||||||
|
// own rows and recomputes them on each edit (see replaceMechanics); its findings
|
||||||
|
// must survive all three model passes — including the collocation coach, which
|
||||||
|
// now shares the collocation family with it.
|
||||||
var (
|
var (
|
||||||
// grammarScope owns the grammar/phrasing/idiom/clarity flags — everything but
|
// grammarScope owns the grammar/phrasing/idiom/clarity flags — everything but
|
||||||
// the other self-owned families (voice, collocation, mechanics), which run on
|
// the other self-owned families (voice, collocation), which run on their own
|
||||||
// their own cadence/pass and must survive a grammar checkpoint. Notably the
|
// cadence/pass and must survive a grammar checkpoint. Notably the offline pass
|
||||||
// deterministic mechanics pass writes its rows in the same /check request just
|
// writes its rows in the same /check request just before this pass reconciles,
|
||||||
// before this DELETE runs, so excluding it here is what keeps them alive.
|
// so the source clause is also what keeps them alive.
|
||||||
grammarScope = pendingScope{deleteWhere: "type NOT IN ('voice','collocation','mechanics')", forceType: ""}
|
grammarScope = pendingScope{
|
||||||
// voiceScope owns the voice flags only.
|
deleteWhere: "source = 'llm' AND type NOT IN ('voice','collocation')",
|
||||||
voiceScope = pendingScope{deleteWhere: "type = 'voice'", forceType: db.SuggestionTypeVoice}
|
family: "grammar",
|
||||||
// collocationScope owns the collocation flags only.
|
chunked: true,
|
||||||
collocationScope = pendingScope{deleteWhere: "type = 'collocation'", forceType: db.SuggestionTypeCollocation}
|
}
|
||||||
|
// voiceScope owns the model's voice flags only. Voice is a property of the
|
||||||
|
// document as a whole — a sentence isn't inconsistent with itself — so this
|
||||||
|
// pass always reads everything.
|
||||||
|
voiceScope = pendingScope{
|
||||||
|
deleteWhere: "source = 'llm' AND type = 'voice'",
|
||||||
|
forceType: db.SuggestionTypeVoice,
|
||||||
|
family: "voice",
|
||||||
|
}
|
||||||
|
// collocationScope owns the model's collocation flags only — the rule pack's
|
||||||
|
// share of the same family is left standing.
|
||||||
|
collocationScope = pendingScope{
|
||||||
|
deleteWhere: "source = 'llm' AND type = 'collocation'",
|
||||||
|
forceType: db.SuggestionTypeCollocation,
|
||||||
|
family: "collocation",
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
// replacePending swaps a document's pending suggestions within one family for a
|
|
||||||
// fresh batch in a single transaction. Accepted/rejected suggestions and the
|
|
||||||
// other family's pending rows are left untouched.
|
|
||||||
//
|
|
||||||
// Suggestions touching a sentence the user already settled are suppressed from
|
|
||||||
// the fresh batch (see suppressor): not just the identical edit re-proposed, but
|
|
||||||
// reversals and re-polishing of the model's own just-accepted output — the
|
|
||||||
// "fickle, keeps going back and forth on a few sentences" behavior. The model has
|
|
||||||
// no memory between passes, so without this it re-opens resolved sentences every
|
|
||||||
// checkpoint.
|
|
||||||
func (h *Handler) replacePending(docID, contentText string, raw []llm.RawSuggestion, scope pendingScope) error {
|
|
||||||
tx, err := h.DB.Begin()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer tx.Rollback()
|
|
||||||
|
|
||||||
if _, err := tx.Exec(
|
|
||||||
`DELETE FROM suggestions WHERE doc_id = ? AND status = ? AND `+scope.deleteWhere,
|
|
||||||
docID, db.SuggestionStatusPending,
|
|
||||||
); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
sup, err := buildSuppressor(tx, docID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, s := range raw {
|
|
||||||
if sup.suppressed(s.Original, s.Replacement) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
typ := scope.forceType
|
|
||||||
if typ == "" {
|
|
||||||
typ = normalizeType(s.Type)
|
|
||||||
}
|
|
||||||
from, to := locate(contentText, s.Original)
|
|
||||||
if _, err := tx.Exec(
|
|
||||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
||||||
docID, from, to, s.Original, s.Replacement, s.Explanation, typ,
|
|
||||||
); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return tx.Commit()
|
|
||||||
}
|
|
||||||
|
|
||||||
// dedupQuoteReplacer folds every straight/curly single- and double-quote variant
|
// dedupQuoteReplacer folds every straight/curly single- and double-quote variant
|
||||||
// (and backtick/acute accent) onto one canonical character. The editor and the
|
// (and backtick/acute accent) onto one canonical character. The editor and the
|
||||||
// model both rewrite quotes between passes — a sentence accepted with "…" comes
|
// model both rewrite quotes between passes — a sentence accepted with "…" comes
|
||||||
@@ -484,6 +642,76 @@ func (h *Handler) listForDoc(w http.ResponseWriter, r *http.Request) {
|
|||||||
httputil.WriteJSON(w, http.StatusOK, out)
|
httputil.WriteJSON(w, http.StatusOK, out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// listSettled returns the normalized originals of every edit the user has
|
||||||
|
// already accepted or dismissed on this document — the same spans buildSuppressor
|
||||||
|
// drops on the server, handed to the client so its instant rule-pack pass can
|
||||||
|
// drop them too.
|
||||||
|
//
|
||||||
|
// Without this the offline half of the loop has no memory. The rule pack detects
|
||||||
|
// from the text alone and re-runs 250 ms after a keystroke, so a dismissed "the
|
||||||
|
// the" comes straight back the moment she types anywhere in the document; the
|
||||||
|
// server's reply then removes it again. That flicker is the visible symptom, but
|
||||||
|
// the real one is worse: with the server unreachable — the case the rule pack
|
||||||
|
// exists for — the reply never comes and a card she dismissed simply stays.
|
||||||
|
//
|
||||||
|
// Only the originals are sent. Replacements are the model's words, not hers, and
|
||||||
|
// the client only needs to answer "has she settled this span?"
|
||||||
|
func (h *Handler) listSettled(w http.ResponseWriter, r *http.Request) {
|
||||||
|
out, err := h.fetchSettled(auth.UserID(r.Context()), chi.URLParam(r, "id"))
|
||||||
|
if err != nil {
|
||||||
|
httputil.ServerError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
httputil.WriteJSON(w, http.StatusOK, settledResponse{Originals: out})
|
||||||
|
}
|
||||||
|
|
||||||
|
// settledResponse wraps the list so the endpoint can grow a second field without
|
||||||
|
// breaking a client that reads a bare array.
|
||||||
|
type settledResponse struct {
|
||||||
|
Originals []string `json:"originals"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchSettled loads the distinct normalized originals of the document's actioned
|
||||||
|
// rows. Scoped through documents for the same reason fetchPending is: an original
|
||||||
|
// is a quotation of her writing.
|
||||||
|
func (h *Handler) fetchSettled(userID, docID string) ([]string, error) {
|
||||||
|
rows, err := h.DB.Query(
|
||||||
|
`SELECT DISTINCT s.original
|
||||||
|
FROM suggestions s
|
||||||
|
JOIN documents d ON d.id = s.doc_id
|
||||||
|
WHERE s.doc_id = ? AND d.user_id = ? AND s.status IN (?, ?)`,
|
||||||
|
docID, userID, db.SuggestionStatusAccepted, db.SuggestionStatusRejected,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
// DISTINCT is on the raw text; normalizing can collapse two rows into one, so
|
||||||
|
// dedupe again on this side to keep the payload honest.
|
||||||
|
seen := map[string]struct{}{}
|
||||||
|
out := []string{}
|
||||||
|
for rows.Next() {
|
||||||
|
var original string
|
||||||
|
if err := rows.Scan(&original); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
norm := normalizeForDedup(original)
|
||||||
|
if norm == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, dup := seen[norm]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[norm] = struct{}{}
|
||||||
|
out = append(out, norm)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// fetchPending loads a document's pending suggestions, joined through documents
|
// fetchPending loads a document's pending suggestions, joined through documents
|
||||||
// so the rows are only reachable by the document's owner. A suggestion quotes the
|
// so the rows are only reachable by the document's owner. A suggestion quotes the
|
||||||
// sentence it corrects, so an unscoped read here would leak document text to
|
// sentence it corrects, so an unscoped read here would leak document text to
|
||||||
@@ -491,7 +719,7 @@ func (h *Handler) listForDoc(w http.ResponseWriter, r *http.Request) {
|
|||||||
func (h *Handler) fetchPending(userID, docID string) ([]db.Suggestion, error) {
|
func (h *Handler) fetchPending(userID, docID string) ([]db.Suggestion, error) {
|
||||||
rows, err := h.DB.Query(
|
rows, err := h.DB.Query(
|
||||||
`SELECT s.id, s.doc_id, s.from_pos, s.to_pos, s.original, s.replacement,
|
`SELECT s.id, s.doc_id, s.from_pos, s.to_pos, s.original, s.replacement,
|
||||||
s.explanation, s.type, s.status, s.created_at
|
s.explanation, s.type, s.status, s.source, s.created_at
|
||||||
FROM suggestions s
|
FROM suggestions s
|
||||||
JOIN documents d ON d.id = s.doc_id
|
JOIN documents d ON d.id = s.doc_id
|
||||||
WHERE s.doc_id = ? AND d.user_id = ? AND s.status = ?
|
WHERE s.doc_id = ? AND d.user_id = ? AND s.status = ?
|
||||||
@@ -508,7 +736,7 @@ func (h *Handler) fetchPending(userID, docID string) ([]db.Suggestion, error) {
|
|||||||
var s db.Suggestion
|
var s db.Suggestion
|
||||||
if err := rows.Scan(
|
if err := rows.Scan(
|
||||||
&s.ID, &s.DocID, &s.FromPos, &s.ToPos, &s.Original, &s.Replacement,
|
&s.ID, &s.DocID, &s.FromPos, &s.ToPos, &s.Original, &s.Replacement,
|
||||||
&s.Explanation, &s.Type, &s.Status, &s.CreatedAt,
|
&s.Explanation, &s.Type, &s.Status, &s.Source, &s.CreatedAt,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -520,11 +748,14 @@ func (h *Handler) fetchPending(userID, docID string) ([]db.Suggestion, error) {
|
|||||||
return dedupeSpans(out), nil
|
return dedupeSpans(out), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// dedupeSpans resolves collisions between the deterministic mechanics family and
|
// dedupeSpans resolves collisions between the offline rule pack and the model:
|
||||||
// the LLM families: when a mechanics finding and an LLM suggestion fight over the
|
// when a local finding and an LLM suggestion fight over the same characters, the
|
||||||
// same characters, mechanics wins and the LLM card is dropped. Its span is exact
|
// local one wins and the LLM card is dropped. Its span is exact (the detector
|
||||||
// (the detector matched it), whereas the LLM positions are only advisory
|
// matched it), whereas the LLM positions are only advisory (re-anchored by string
|
||||||
// (re-anchored by string at render), so the precise fix should own the span.
|
// at render), so the precise fix should own the span. This is why the split is by
|
||||||
|
// source rather than by type — an offline miscollocation is as exact as an
|
||||||
|
// offline comma, and the coach's fuzzy version of the same chunk shouldn't
|
||||||
|
// double up next to it.
|
||||||
//
|
//
|
||||||
// This deliberately does NOT dedupe LLM-vs-LLM overlaps: voice (awareness-only,
|
// This deliberately does NOT dedupe LLM-vs-LLM overlaps: voice (awareness-only,
|
||||||
// no replacement) and collocation legitimately co-occupy the same span, and that
|
// no replacement) and collocation legitimately co-occupy the same span, and that
|
||||||
@@ -534,7 +765,7 @@ func dedupeSpans(in []db.Suggestion) []db.Suggestion {
|
|||||||
type span struct{ from, to int }
|
type span struct{ from, to int }
|
||||||
var claimed []span
|
var claimed []span
|
||||||
for _, s := range in {
|
for _, s := range in {
|
||||||
if s.Type == db.SuggestionTypeMechanics && s.FromPos >= 0 {
|
if s.Source == db.SuggestionSourceLocal && s.FromPos >= 0 {
|
||||||
claimed = append(claimed, span{s.FromPos, s.ToPos})
|
claimed = append(claimed, span{s.FromPos, s.ToPos})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -544,7 +775,7 @@ func dedupeSpans(in []db.Suggestion) []db.Suggestion {
|
|||||||
|
|
||||||
out := make([]db.Suggestion, 0, len(in))
|
out := make([]db.Suggestion, 0, len(in))
|
||||||
for _, s := range in {
|
for _, s := range in {
|
||||||
if s.Type != db.SuggestionTypeMechanics && s.FromPos >= 0 {
|
if s.Source != db.SuggestionSourceLocal && s.FromPos >= 0 {
|
||||||
overlaps := false
|
overlaps := false
|
||||||
for _, sp := range claimed {
|
for _, sp := range claimed {
|
||||||
if s.FromPos < sp.to && sp.from < s.ToPos {
|
if s.FromPos < sp.to && sp.from < s.ToPos {
|
||||||
@@ -553,7 +784,7 @@ func dedupeSpans(in []db.Suggestion) []db.Suggestion {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if overlaps {
|
if overlaps {
|
||||||
continue // an exact mechanics fix owns these characters
|
continue // an exact offline fix owns these characters
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out = append(out, s)
|
out = append(out, s)
|
||||||
@@ -577,7 +808,7 @@ func (h *Handler) dismiss(w http.ResponseWriter, r *http.Request) {
|
|||||||
// no rows and surfaces as a 404.
|
// no rows and surfaces as a 404.
|
||||||
func (h *Handler) setStatus(w http.ResponseWriter, r *http.Request, status string) {
|
func (h *Handler) setStatus(w http.ResponseWriter, r *http.Request, status string) {
|
||||||
res, err := h.DB.Exec(
|
res, err := h.DB.Exec(
|
||||||
`UPDATE suggestions SET status = ?
|
`UPDATE suggestions SET status = ?, resolved_at = datetime('now')
|
||||||
WHERE id = ? AND status = ?
|
WHERE id = ? AND status = ?
|
||||||
AND doc_id IN (SELECT id FROM documents WHERE user_id = ?)`,
|
AND doc_id IN (SELECT id FROM documents WHERE user_id = ?)`,
|
||||||
status, chi.URLParam(r, "id"), db.SuggestionStatusPending,
|
status, chi.URLParam(r, "id"), db.SuggestionStatusPending,
|
||||||
@@ -591,9 +822,75 @@ func (h *Handler) setStatus(w http.ResponseWriter, r *http.Request, status strin
|
|||||||
httputil.ErrorJSON(w, http.StatusNotFound, "pending suggestion not found")
|
httputil.ErrorJSON(w, http.StatusNotFound, "pending suggestion not found")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if status == db.SuggestionStatusAccepted {
|
||||||
|
h.plant(chi.URLParam(r, "id"), auth.UserID(r.Context()))
|
||||||
|
}
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// plant grows an accepted collocation into a vocabulary-garden phrase card. It
|
||||||
|
// runs after the status write and swallows its own errors: accepting an edit is
|
||||||
|
// the thing the writer asked for, and it must not fail — or even feel slower —
|
||||||
|
// because a flashcard couldn't be made.
|
||||||
|
//
|
||||||
|
// Only collocations are planted. The other families correct *this* sentence
|
||||||
|
// ("their" → "there", a comma, a clearer clause); a collocation is the one that
|
||||||
|
// hands over a reusable chunk, which is the only thing worth reviewing in a week.
|
||||||
|
func (h *Handler) plant(id, userID string) {
|
||||||
|
var s db.Suggestion
|
||||||
|
var contentText, docLang string
|
||||||
|
err := h.DB.QueryRow(
|
||||||
|
`SELECT s.type, s.original, s.replacement, s.explanation, s.doc_id, d.content_text, d.doc_lang
|
||||||
|
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||||
|
WHERE s.id = ? AND d.user_id = ?`,
|
||||||
|
id, userID,
|
||||||
|
).Scan(&s.Type, &s.Original, &s.Replacement, &s.Explanation, &s.DocID, &contentText, &docLang)
|
||||||
|
if err != nil {
|
||||||
|
if !errors.Is(err, sql.ErrNoRows) {
|
||||||
|
log.Printf("suggestions: could not read %s for planting: %v", id, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if s.Type != db.SuggestionTypeCollocation || strings.TrimSpace(s.Replacement) == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// The stored text is still the pre-accept draft — the client applies the
|
||||||
|
// replacement in the editor. Correct the sentence here so the flashcard
|
||||||
|
// quizzes the phrasing she is keeping, not the one she just left behind.
|
||||||
|
docID := s.DocID
|
||||||
|
if _, err := vocab.Plant(h.DB, userID, vocab.Phrase{
|
||||||
|
Text: s.Replacement,
|
||||||
|
Meaning: s.Explanation,
|
||||||
|
Example: correctedSentence(contentText, s.Original, s.Replacement),
|
||||||
|
DocID: &docID,
|
||||||
|
// The chunk is her own sentence, corrected — so it is in the document's
|
||||||
|
// language, whatever the collocation pass was framed in.
|
||||||
|
Lang: docLang,
|
||||||
|
}); err != nil {
|
||||||
|
log.Printf("suggestions: could not plant %s: %v", id, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// correctedSentence returns the sentence of contentText containing original,
|
||||||
|
// with original swapped for replacement. Returns "" when the original isn't
|
||||||
|
// found (the draft moved on) — a card with no example still reviews, just
|
||||||
|
// without the cloze, so there's nothing to fall back to and nothing to guess.
|
||||||
|
func correctedSentence(contentText, original, replacement string) string {
|
||||||
|
idx := strings.Index(contentText, original)
|
||||||
|
if original == "" || idx < 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
start := strings.LastIndexAny(contentText[:idx], ".!?\n")
|
||||||
|
end := strings.IndexAny(contentText[idx+len(original):], ".!?\n")
|
||||||
|
if end < 0 {
|
||||||
|
end = len(contentText)
|
||||||
|
} else {
|
||||||
|
end += idx + len(original) + 1 // keep the terminator
|
||||||
|
}
|
||||||
|
sentence := strings.TrimSpace(contentText[start+1 : end])
|
||||||
|
return strings.Replace(sentence, original, replacement, 1)
|
||||||
|
}
|
||||||
|
|
||||||
// locate finds the plaintext offsets of original within contentText. Returns
|
// locate finds the plaintext offsets of original within contentText. Returns
|
||||||
// (-1, -1) when not found; the frontend anchors by string regardless, so a miss
|
// (-1, -1) when not found; the frontend anchors by string regardless, so a miss
|
||||||
// here is non-fatal.
|
// here is non-fatal.
|
||||||
@@ -607,6 +904,11 @@ func locate(contentText, original string) (int, int) {
|
|||||||
|
|
||||||
// normalizeType maps the model's type string onto a valid suggestion type,
|
// normalizeType maps the model's type string onto a valid suggestion type,
|
||||||
// defaulting unknown values to grammar so a stray label never trips the CHECK.
|
// defaulting unknown values to grammar so a stray label never trips the CHECK.
|
||||||
|
//
|
||||||
|
// 'translate' is absent on purpose, and stays absent even though the type now
|
||||||
|
// exists: it is decided from the span (see language.go), never taken from the
|
||||||
|
// model. A model that volunteers the label anyway lands on grammar here and is
|
||||||
|
// then promoted — or not — on the evidence.
|
||||||
func normalizeType(t string) string {
|
func normalizeType(t string) string {
|
||||||
switch strings.ToLower(strings.TrimSpace(t)) {
|
switch strings.ToLower(strings.TrimSpace(t)) {
|
||||||
case db.SuggestionTypeGrammar, db.SuggestionTypePhrasing, db.SuggestionTypeIdiom, db.SuggestionTypeClarity, db.SuggestionTypeCollocation:
|
case db.SuggestionTypeGrammar, db.SuggestionTypePhrasing, db.SuggestionTypeIdiom, db.SuggestionTypeClarity, db.SuggestionTypeCollocation:
|
||||||
|
|||||||
@@ -20,10 +20,17 @@ import (
|
|||||||
type stubClient struct {
|
type stubClient struct {
|
||||||
response string
|
response string
|
||||||
calls int
|
calls int
|
||||||
|
// The full prompt of the most recent call, so a test can assert which
|
||||||
|
// sentences a chunked pass actually asked about.
|
||||||
|
lastPrompt string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *stubClient) Complete(_ context.Context, _ llm.CompletionRequest) (string, error) {
|
func (s *stubClient) Complete(_ context.Context, req llm.CompletionRequest) (string, error) {
|
||||||
s.calls++
|
s.calls++
|
||||||
|
s.lastPrompt = ""
|
||||||
|
for _, m := range req.Messages {
|
||||||
|
s.lastPrompt += m.Content + "\n"
|
||||||
|
}
|
||||||
return s.response, nil
|
return s.response, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -64,6 +71,17 @@ func newTestServer(t *testing.T, client llm.LLMClient) (http.Handler, string, *H
|
|||||||
return authed, docID, h
|
return authed, docID, h
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// setDocText rewrites the seeded document, standing in for the writer editing.
|
||||||
|
// The grammar checkpoint only asks the model about sentences that changed since
|
||||||
|
// it last read the document, so a test that wants a second real pass has to
|
||||||
|
// change something first — as she always has.
|
||||||
|
func setDocText(t *testing.T, h *Handler, docID, text string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := h.DB.Exec(`UPDATE documents SET content_text = ? WHERE id = ?`, text, docID); err != nil {
|
||||||
|
t.Fatalf("update doc text: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func do(t *testing.T, srv http.Handler, method, path, body string) *httptest.ResponseRecorder {
|
func do(t *testing.T, srv http.Handler, method, path, body string) *httptest.ResponseRecorder {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
var r *http.Request
|
var r *http.Request
|
||||||
@@ -183,6 +201,7 @@ func TestFickleEditsSuppressed(t *testing.T) {
|
|||||||
]}`}
|
]}`}
|
||||||
srv, docID, h := newTestServer(t, client)
|
srv, docID, h := newTestServer(t, client)
|
||||||
h.Limit = llm.NewRateLimiter(0)
|
h.Limit = llm.NewRateLimiter(0)
|
||||||
|
setDocText(t, h, docID, `He left "early," because of the rain. The cat always have a calm face.`)
|
||||||
|
|
||||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
var got []db.Suggestion
|
var got []db.Suggestion
|
||||||
@@ -194,6 +213,10 @@ func TestFickleEditsSuppressed(t *testing.T) {
|
|||||||
do(t, srv, http.MethodPost, "/suggestions/"+s.ID+"/accept", "")
|
do(t, srv, http.MethodPost, "/suggestions/"+s.ID+"/accept", "")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Both edits are now in the document, which is what re-opens those sentences
|
||||||
|
// for a second reading.
|
||||||
|
setDocText(t, h, docID, `He left "early," due to the rain. The cat always has a calm face.`)
|
||||||
|
|
||||||
// Reversal of the first accept (note the " → ' quote churn) and a re-polish of
|
// Reversal of the first accept (note the " → ' quote churn) and a re-polish of
|
||||||
// the second accept must both be dropped; only the unrelated edit survives.
|
// the second accept must both be dropped; only the unrelated edit survives.
|
||||||
client.response = `{"suggestions":[
|
client.response = `{"suggestions":[
|
||||||
@@ -314,7 +337,9 @@ func TestCollocationPassCoexists(t *testing.T) {
|
|||||||
t.Fatalf("collocation response should carry all three families, got %+v", got)
|
t.Fatalf("collocation response should carry all three families, got %+v", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
// A grammar checkpoint must NOT wipe the voice or collocation flags.
|
// A grammar checkpoint must NOT wipe the voice or collocation flags. She fixes
|
||||||
|
// the flagged sentence, so its own grammar row goes and nothing replaces it.
|
||||||
|
setDocText(t, h, docID, "I have two apples.")
|
||||||
client.response = `{"suggestions":[]}`
|
client.response = `{"suggestions":[]}`
|
||||||
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
if rec.Code != http.StatusOK {
|
if rec.Code != http.StatusOK {
|
||||||
|
|||||||
@@ -116,4 +116,14 @@ func TestSuggestionIsolation(t *testing.T) {
|
|||||||
if rec.Code != http.StatusNoContent {
|
if rec.Code != http.StatusNoContent {
|
||||||
t.Fatalf("owner accept = %d, want 204 (body: %s)", rec.Code, rec.Body)
|
t.Fatalf("owner accept = %d, want 204 (body: %s)", rec.Code, rec.Body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// That accept created a settled span, which is the other read of this table.
|
||||||
|
// It carries originals only — but an original is a verbatim quotation of her
|
||||||
|
// sentence, so it is the same leak as the pending list through a smaller hole.
|
||||||
|
if got := getSettled(t, owner, docID); len(got) != 1 {
|
||||||
|
t.Fatalf("owner should see their own settled span, got %v", got)
|
||||||
|
}
|
||||||
|
if got := getSettled(t, stranger, docID); len(got) != 0 {
|
||||||
|
t.Fatalf("stranger read %d settled span(s) (leaking %q)", len(got), got[0])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,228 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Telling her language from English, well enough to label a card.
|
||||||
|
//
|
||||||
|
// When the checkpoint quotes a span she wrote in her own language and hands back
|
||||||
|
// an English rendering, that is not a correction — nothing was wrong with what
|
||||||
|
// she wrote — and it should not be filed under 'clarity'. The label is derived
|
||||||
|
// here rather than asked of the model: a type is structural, and a model that
|
||||||
|
// re-reasons every pass would drift between labels for the same sentence.
|
||||||
|
//
|
||||||
|
// The failure mode is deliberately cheap. Getting this wrong changes a card's
|
||||||
|
// coloured pill and nothing else — the replacement, the explanation and the
|
||||||
|
// Accept button are identical either way — so a heuristic is the right tool. It
|
||||||
|
// is written to under-claim: a span it isn't sure about stays whatever the model
|
||||||
|
// called it.
|
||||||
|
//
|
||||||
|
// The two pair families need genuinely different tests, and pretending otherwise
|
||||||
|
// would be the bug:
|
||||||
|
//
|
||||||
|
// - zh is a different script. Counting Han runes is close to certain.
|
||||||
|
// - pt-PT, fr and es share the Latin alphabet with English, where no such
|
||||||
|
// signal exists. Those fall back to function words — the short, extremely
|
||||||
|
// common words a sentence in that language can hardly avoid and an English
|
||||||
|
// sentence has no reason to contain.
|
||||||
|
|
||||||
|
// isTranslation reports whether this edit is a rendering of one language into
|
||||||
|
// the other, rather than a correction. Both halves must hold: the quoted span
|
||||||
|
// reads as one language, and what Petal offers back reads as the other. The
|
||||||
|
// second half matters — a Chinese span rewritten into different Chinese is
|
||||||
|
// something else entirely, and Petal has no business calling it a translation.
|
||||||
|
//
|
||||||
|
// Which way it points follows the document (Phase 28). In an English document
|
||||||
|
// the translate card is her language rendered into English — she reached for a
|
||||||
|
// sentence she couldn't say yet, and Petal said it for her. In a document she
|
||||||
|
// wrote in her own language the useful card is the mirror image: an English
|
||||||
|
// sentence she dropped into her Portuguese, rendered into Portuguese. Asking the
|
||||||
|
// English-document question there would label nothing, and the card would file
|
||||||
|
// as a correction to prose that was never wrong.
|
||||||
|
//
|
||||||
|
// The flipped direction cannot be the same test read backwards. `readsAsEnglish`
|
||||||
|
// is a low bar on purpose — Latin letters, not swamped by another script — which
|
||||||
|
// every Portuguese sentence also clears, so using it on the *original* would
|
||||||
|
// call every genuine Portuguese correction a translation. The flipped test
|
||||||
|
// instead uses the sentence-level vote from doclang.go, where English has its
|
||||||
|
// own marker list and has to out-evidence the pair language to win.
|
||||||
|
func isTranslation(original, replacement, pairLang, docLang string) bool {
|
||||||
|
if strings.TrimSpace(original) == "" || strings.TrimSpace(replacement) == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if normalizeDocLang(docLang) == docLangPair {
|
||||||
|
p := normalizePairLang(pairLang)
|
||||||
|
return sentenceLang(original, p) == docLangEnglish &&
|
||||||
|
sentenceLang(replacement, p) == docLangPair
|
||||||
|
}
|
||||||
|
return readsAsPairLang(original, pairLang) && readsAsEnglish(replacement)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readsAsPairLang reports whether s is predominantly in the writer's language.
|
||||||
|
func readsAsPairLang(s, pairLang string) bool {
|
||||||
|
switch normalizePairLang(pairLang) {
|
||||||
|
case "zh":
|
||||||
|
han, latin := scriptCounts(s)
|
||||||
|
// Predominantly, not merely partly: one Chinese word inside an English
|
||||||
|
// sentence is a vocabulary question, and the sentence around it is still
|
||||||
|
// English prose with its own grammar to correct. Two runes is the floor
|
||||||
|
// because a single Han character is as likely to be a stray keystroke.
|
||||||
|
return han >= 2 && han > latin
|
||||||
|
case "pt-PT", "fr", "es":
|
||||||
|
return distinctMarkers(s, latinMarkers[normalizePairLang(pairLang)]) >= 2
|
||||||
|
}
|
||||||
|
// A pair Petal has no test for. Say no: an unlabelled card is a card that
|
||||||
|
// reads as it did yesterday, and a wrongly-labelled one is a new defect.
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// readsAsEnglish reports whether s is English prose rather than more of her own
|
||||||
|
// language. It is not a language identifier — it only has to separate "English"
|
||||||
|
// from "the pair language", and it is only ever asked about text Petal itself
|
||||||
|
// generated, so the bar is low on purpose: Latin letters present, and not
|
||||||
|
// swamped by another script.
|
||||||
|
func readsAsEnglish(s string) bool {
|
||||||
|
han, latin := scriptCounts(s)
|
||||||
|
return latin > 0 && latin > han
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizePairLang folds the stored `users.pair_lang` into the codes below.
|
||||||
|
// Empty (a document whose owner has no pair recorded) falls through to no test.
|
||||||
|
func normalizePairLang(pairLang string) string {
|
||||||
|
switch p := strings.ToLower(strings.TrimSpace(pairLang)); p {
|
||||||
|
case "zh", "zh-cn", "zh-hans":
|
||||||
|
return "zh"
|
||||||
|
case "pt", "pt-pt":
|
||||||
|
return "pt-PT"
|
||||||
|
case "fr", "fr-fr":
|
||||||
|
return "fr"
|
||||||
|
case "es", "es-es":
|
||||||
|
return "es"
|
||||||
|
default:
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// scriptCounts counts Han runes and ASCII letters. Everything else — digits,
|
||||||
|
// punctuation, spaces, emoji — is ignored, so trailing 。or a stray comma
|
||||||
|
// changes nothing.
|
||||||
|
func scriptCounts(s string) (han, latin int) {
|
||||||
|
for _, r := range s {
|
||||||
|
switch {
|
||||||
|
case unicode.Is(unicode.Han, r):
|
||||||
|
han++
|
||||||
|
case r < unicode.MaxASCII && unicode.IsLetter(r):
|
||||||
|
latin++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return han, latin
|
||||||
|
}
|
||||||
|
|
||||||
|
// distinctMarkers counts how many *different* marker words appear in s. Distinct
|
||||||
|
// rather than total: "que ... que" is one writer's habit, while "eu quero" is two
|
||||||
|
// independent pieces of evidence.
|
||||||
|
func distinctMarkers(s string, markers map[string]bool) int {
|
||||||
|
if len(markers) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
|
||||||
|
// Split on anything that isn't a letter, so punctuation and digits are
|
||||||
|
// separators. Apostrophes included: French elision (j'ai, n'est) should
|
||||||
|
// yield its parts.
|
||||||
|
return !unicode.IsLetter(r)
|
||||||
|
}) {
|
||||||
|
if markers[w] {
|
||||||
|
seen[w] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(seen)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function words that a sentence in each Latin pair can hardly avoid.
|
||||||
|
//
|
||||||
|
// Curated against English, not for coverage: every entry here is a word an
|
||||||
|
// English sentence has essentially no reason to contain, which is why the lists
|
||||||
|
// omit plenty of far more common words. Deliberately absent — each of them a
|
||||||
|
// false positive waiting to happen — is anything that is *also* an English word:
|
||||||
|
// the pan-Romance shorts (a, o, e, as, no, on, en, de, se, na, mi, son, era,
|
||||||
|
// plus, pour, si, ma, ce, ne), Portuguese "do", Spanish "con", "ya" and "todo".
|
||||||
|
// Dropping "con" costs the Spanish list one of its commonest words, and that is
|
||||||
|
// the right trade — a marker that fires on English corroborates the wrong
|
||||||
|
// answer, which is worse than a sentence Petal declines to label.
|
||||||
|
//
|
||||||
|
// A single marker is not enough (see readsAsPairLang), so these lists are read
|
||||||
|
// as evidence to be corroborated rather than as a decision.
|
||||||
|
//
|
||||||
|
// **Curated against English is not the same as curated thinly**, and the first
|
||||||
|
// version of these lists confused the two. Seen live 2026-07-29: "Esta manhã
|
||||||
|
// acordei cedo e fui correr ao longo da marginal. O ar estava fresco e havia
|
||||||
|
// poucas pessoas na rua." — unremarkable Portuguese, two marker hits, *zero*
|
||||||
|
// English hits, and a verdict of English, because the document-level floor wants
|
||||||
|
// three. The list was missing the ordinary machinery of the language: the
|
||||||
|
// contractions (ao, à, num), the past tenses a diary is written in (estava,
|
||||||
|
// havia, fomos), and the words that join two clauses (até, depois, então,
|
||||||
|
// onde). Every one of them clears the bar above — an English sentence has no
|
||||||
|
// reason to contain them — so their absence bought nothing and cost the verdict.
|
||||||
|
// The floor stays at three; what changed is that three is now reachable by
|
||||||
|
// prose rather than only by a paragraph that happens to argue with itself.
|
||||||
|
var latinMarkers = map[string]map[string]bool{
|
||||||
|
"fr": words(
|
||||||
|
"je", "tu", "il", "elle", "ils", "elles", "nous", "vous", "est", "sont",
|
||||||
|
"était", "étais", "une", "des", "les", "du", "dans", "avec", "que", "qui",
|
||||||
|
"mais", "très", "être", "avoir", "pas", "cette", "cet", "ces", "mon",
|
||||||
|
"mes", "notre", "votre", "leur", "aussi", "alors", "parce", "comme",
|
||||||
|
"beaucoup", "toujours", "jamais", "quand", "bien", "chose", "temps",
|
||||||
|
"moi", "toi", "lui", "peux", "veux", "sais", "faire", "dit", "aujourd",
|
||||||
|
"hui", "quelque", "chez", "tout", "tous", "rien", "déjà", "encore",
|
||||||
|
// The same gap the pt-PT list was caught with, closed by analogy rather
|
||||||
|
// than by observation — no fr account exists yet to catch it live.
|
||||||
|
"aux", "après", "où", "avait", "étaient", "depuis", "jusqu", "chaque",
|
||||||
|
"autre", "même", "hier", "demain", "matin", "soir", "nôtre", "leurs",
|
||||||
|
),
|
||||||
|
"pt-PT": words(
|
||||||
|
"eu", "você", "ele", "ela", "eles", "elas", "nós", "são", "uma", "os",
|
||||||
|
"da", "dos", "das", "com", "que", "mas", "muito", "não", "meu",
|
||||||
|
"minha", "seu", "sua", "isso", "este", "esta", "está", "estou", "quero",
|
||||||
|
"também", "quando", "porque", "coisa", "tempo", "fazer", "sempre",
|
||||||
|
"nunca", "bem", "obrigado", "obrigada", "gosto", "tenho", "tem", "foi",
|
||||||
|
"ser", "ter", "mais", "já", "ainda", "aqui", "ali", "nada", "tudo",
|
||||||
|
"todos", "para", "pela", "pelo", "sobre", "assim",
|
||||||
|
// The contractions, which no English sentence has any use for.
|
||||||
|
"ao", "aos", "à", "às", "num", "numa", "dum", "duma", "pelos", "pelas",
|
||||||
|
"neste", "nesta", "disso", "deste", "desta",
|
||||||
|
// The tenses a journal is actually written in.
|
||||||
|
"estava", "estavam", "estão", "estamos", "havia", "houve", "era", "eram",
|
||||||
|
"fui", "fomos", "foram", "vai", "vamos", "tinha", "tinham",
|
||||||
|
// The joins between two clauses.
|
||||||
|
"até", "depois", "antes", "onde", "então", "enquanto", "embora",
|
||||||
|
"sem", "quem", "entre",
|
||||||
|
// And the everyday determiners and time words a diary can hardly avoid.
|
||||||
|
"nosso", "nossa", "outro", "outra", "mesmo", "mesma", "tão",
|
||||||
|
"muitos", "muitas", "poucos", "poucas", "hoje", "ontem", "amanhã",
|
||||||
|
),
|
||||||
|
"es": words(
|
||||||
|
"yo", "él", "ella", "ellos", "ellas", "nosotros", "una", "los", "las",
|
||||||
|
"del", "que", "pero", "muy", "esto", "esta", "este", "está",
|
||||||
|
"estoy", "quiero", "también", "cuando", "porque", "cosa", "tiempo",
|
||||||
|
"hacer", "siempre", "nunca", "bien", "gracias", "tengo", "tiene", "fue",
|
||||||
|
"ser", "tener", "más", "aquí", "allí", "nada", "todos",
|
||||||
|
"para", "sobre", "así", "hola", "señor", "usted", "muchas",
|
||||||
|
// Likewise by analogy: no es account exists yet either. "sin" and "tan"
|
||||||
|
// stay out — both are English words, which is the one disqualification.
|
||||||
|
"al", "después", "antes", "donde", "entonces", "mientras", "aunque",
|
||||||
|
"estaba", "estaban", "están", "había", "hubo", "fuimos", "fueron",
|
||||||
|
"nuestro", "nuestra", "otro", "otra", "mismo", "misma", "quién", "quien",
|
||||||
|
"muchos", "pocas", "pocos", "hoy", "ayer", "mañana",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
func words(list ...string) map[string]bool {
|
||||||
|
out := make(map[string]bool, len(list))
|
||||||
|
for _, w := range list {
|
||||||
|
out[w] = true
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,265 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// The flagship case, and the ones next to it that must NOT become translations.
|
||||||
|
//
|
||||||
|
// Every case here is an ENGLISH document — the path every account was on before
|
||||||
|
// Phase 28 — so `docLang` is left at "". The mirror image lives in
|
||||||
|
// TestIsTranslationInAPairLanguageDocument below.
|
||||||
|
func TestIsTranslation(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
original string
|
||||||
|
replacement string
|
||||||
|
pairLang string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
// The sentence from the UX review, verbatim.
|
||||||
|
name: "whole Chinese sentence rendered into English",
|
||||||
|
original: "我想说这句话但是不知道用英语怎么说。",
|
||||||
|
replacement: "I want to say this but I don't know how to say it in English.",
|
||||||
|
pairLang: "zh",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ordinary English correction is not a translation",
|
||||||
|
original: "She goes to market yesterday",
|
||||||
|
replacement: "She went to the market yesterday",
|
||||||
|
pairLang: "zh",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// One Chinese word inside English prose. The sentence around it is
|
||||||
|
// still English with its own grammar to fix, and calling the card a
|
||||||
|
// translation would mislabel a grammar fix.
|
||||||
|
name: "single Chinese word inside an English sentence",
|
||||||
|
original: "I bought a 苹果 at the store",
|
||||||
|
replacement: "I bought an apple at the store",
|
||||||
|
pairLang: "zh",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "a lone stray Han rune is not a sentence",
|
||||||
|
original: "的",
|
||||||
|
replacement: "of",
|
||||||
|
pairLang: "zh",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Chinese in, Chinese out: whatever this is, Petal is not translating.
|
||||||
|
name: "Chinese rewritten as Chinese",
|
||||||
|
original: "我想说这句话",
|
||||||
|
replacement: "我要说这句话",
|
||||||
|
pairLang: "zh",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// The same Chinese span, but the writer is on the French pair. Petal
|
||||||
|
// has no business offering to translate a language she never claimed.
|
||||||
|
name: "Chinese span on a non-zh pair",
|
||||||
|
original: "我想说这句话但是不知道用英语怎么说。",
|
||||||
|
replacement: "I want to say this in English.",
|
||||||
|
pairLang: "fr",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "French sentence rendered into English",
|
||||||
|
original: "Je ne sais pas comment le dire en anglais.",
|
||||||
|
replacement: "I don't know how to say it in English.",
|
||||||
|
pairLang: "fr",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Portuguese sentence rendered into English",
|
||||||
|
original: "Eu quero dizer isso mas não sei como.",
|
||||||
|
replacement: "I want to say this but I don't know how.",
|
||||||
|
pairLang: "pt-PT",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Spanish sentence rendered into English",
|
||||||
|
original: "Yo quiero decir esto pero no sé cómo.",
|
||||||
|
replacement: "I want to say this but I don't know how.",
|
||||||
|
pairLang: "es",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// A single marker is not evidence. "Que" appears in English writing
|
||||||
|
// about other languages, in names, in quoted phrases.
|
||||||
|
name: "one Latin marker is not enough",
|
||||||
|
original: "The word que confused me",
|
||||||
|
replacement: "The word que confuses me",
|
||||||
|
pairLang: "pt-PT",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// The words most likely to sink this heuristic: English function words
|
||||||
|
// that are also Romance function words. They are kept out of the lists
|
||||||
|
// precisely so this sentence stays a grammar fix.
|
||||||
|
name: "English full of pan-Romance lookalikes",
|
||||||
|
original: "I do not know if a con man on the plus side as no era",
|
||||||
|
replacement: "I do not know whether a con man, on the plus side, is no era",
|
||||||
|
pairLang: "es",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "English with a borrowed French phrase stays English",
|
||||||
|
original: "It was a pas de deux, more or less",
|
||||||
|
replacement: "It was a pas de deux, more or less.",
|
||||||
|
pairLang: "fr",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty replacement (an awareness-only finding)",
|
||||||
|
original: "我想说这句话但是不知道用英语怎么说。",
|
||||||
|
replacement: "",
|
||||||
|
pairLang: "zh",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// A document whose owner has no pair recorded. No test, no label.
|
||||||
|
name: "no pair language",
|
||||||
|
original: "我想说这句话但是不知道用英语怎么说。",
|
||||||
|
replacement: "I want to say this in English.",
|
||||||
|
pairLang: "",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// An unshipped pair. Same rule: decline rather than guess.
|
||||||
|
name: "unknown pair language",
|
||||||
|
original: "Ich weiß nicht wie man das sagt.",
|
||||||
|
replacement: "I don't know how to say that.",
|
||||||
|
pairLang: "de",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
if got := isTranslation(c.original, c.replacement, c.pairLang, ""); got != c.want {
|
||||||
|
t.Errorf("isTranslation(%q, %q, %q, en) = %v, want %v",
|
||||||
|
c.original, c.replacement, c.pairLang, got, c.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The mirror image (Phase 28): in a document she wrote in her own language, the
|
||||||
|
// translate card is the English sentence rendered into her language — and the
|
||||||
|
// English-document question, asked here, would label nothing.
|
||||||
|
//
|
||||||
|
// The case this file exists to pin is the third one: a genuine Portuguese
|
||||||
|
// correction inside a Portuguese document. Reading the English-document test
|
||||||
|
// backwards would call it a translation, because `readsAsEnglish` is a low bar
|
||||||
|
// that Portuguese clears too. It has to stay a correction.
|
||||||
|
func TestIsTranslationInAPairLanguageDocument(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
original string
|
||||||
|
replacement string
|
||||||
|
pairLang string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "English sentence rendered into Portuguese",
|
||||||
|
original: "I want to say this but I don't know how to say it.",
|
||||||
|
replacement: "Eu quero dizer isso mas não sei como o dizer.",
|
||||||
|
pairLang: "pt-PT",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "English sentence rendered into Chinese",
|
||||||
|
original: "I don't know how to say this in Chinese.",
|
||||||
|
replacement: "我不知道这句话用中文怎么说。",
|
||||||
|
pairLang: "zh",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// The one that matters. Portuguese in, Portuguese out, inside a
|
||||||
|
// Portuguese document: a correction, and nothing else.
|
||||||
|
name: "Portuguese corrected as Portuguese",
|
||||||
|
original: "Eu quero dizer isso mas não sei como.",
|
||||||
|
replacement: "Eu quero dizer isto mas não sei como.",
|
||||||
|
pairLang: "pt-PT",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// And its Chinese twin, which the script test already caught.
|
||||||
|
name: "Chinese corrected as Chinese",
|
||||||
|
original: "我想说这句话",
|
||||||
|
replacement: "我要说这句话",
|
||||||
|
pairLang: "zh",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// The old direction, asked in the new document. She quoted English in
|
||||||
|
// her Portuguese and Petal rendered it into Portuguese — which IS a
|
||||||
|
// translation, and is the case above. This is its reverse: Portuguese
|
||||||
|
// out of an English document that isn't one. No label.
|
||||||
|
name: "Portuguese rendered into English is not this document's translation",
|
||||||
|
original: "Eu quero dizer isso mas não sei como.",
|
||||||
|
replacement: "I want to say this but I don't know how.",
|
||||||
|
pairLang: "pt-PT",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// English prose without enough evidence to vote. Silence, not a guess.
|
||||||
|
name: "too short to read as English",
|
||||||
|
original: "OK",
|
||||||
|
replacement: "Está bem, muito obrigado.",
|
||||||
|
pairLang: "pt-PT",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown pair language declines in both directions",
|
||||||
|
original: "I don't know how to say that.",
|
||||||
|
replacement: "Ich weiß nicht wie man das sagt.",
|
||||||
|
pairLang: "de",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
if got := isTranslation(c.original, c.replacement, c.pairLang, docLangPair); got != c.want {
|
||||||
|
t.Errorf("isTranslation(%q, %q, %q, pair) = %v, want %v",
|
||||||
|
c.original, c.replacement, c.pairLang, got, c.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pair_lang is stored as the pack code, but a stored value has drifted before
|
||||||
|
// (see the picker's history), so the fold is tested rather than assumed.
|
||||||
|
func TestNormalizePairLang(t *testing.T) {
|
||||||
|
for in, want := range map[string]string{
|
||||||
|
"zh": "zh", "zh-CN": "zh", "ZH": "zh",
|
||||||
|
"pt": "pt-PT", "pt-PT": "pt-PT", "pt-pt": "pt-PT",
|
||||||
|
"fr": "fr", "fr-FR": "fr",
|
||||||
|
"es": "es", "es-ES": "es",
|
||||||
|
" zh ": "zh",
|
||||||
|
"": "",
|
||||||
|
"de": "de",
|
||||||
|
} {
|
||||||
|
if got := normalizePairLang(in); got != want {
|
||||||
|
t.Errorf("normalizePairLang(%q) = %q, want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// French elision must yield its parts, or "j'ai" and "n'est" — two of the
|
||||||
|
// commonest shapes in the language — count for nothing.
|
||||||
|
func TestElisionYieldsMarkers(t *testing.T) {
|
||||||
|
if n := distinctMarkers("Je n'est pas", latinMarkers["fr"]); n < 3 {
|
||||||
|
t.Errorf("elided French: got %d markers, want >= 3 (je, est, pas)", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Distinct, not total: one word repeated is one piece of evidence.
|
||||||
|
func TestRepeatedMarkerCountsOnce(t *testing.T) {
|
||||||
|
if n := distinctMarkers("que que que", latinMarkers["pt-PT"]); n != 1 {
|
||||||
|
t.Errorf("repeated marker: got %d, want 1", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,236 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The offline rule pack and the LLM now share the collocation family, which is
|
||||||
|
// the point: the writer sees one rail and is never told which engine spoke. What
|
||||||
|
// makes that safe is `source` — each pass replaces only its own rows. These tests
|
||||||
|
// pin the two ways that could go wrong, both of which the old type-scoped DELETEs
|
||||||
|
// would have hit.
|
||||||
|
|
||||||
|
// pendingOfType counts the pending rows of one family in a response body.
|
||||||
|
func pendingOfType(got []db.Suggestion, typ string) []db.Suggestion {
|
||||||
|
var out []db.Suggestion
|
||||||
|
for _, s := range got {
|
||||||
|
if s.Type == typ {
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOfflineCollocationFilesAsCollocation proves a miscollocation the rule pack
|
||||||
|
// found is stored in the collocation family (so accepting it plants a garden
|
||||||
|
// card, exactly as the coach's would) while still being marked as locally found.
|
||||||
|
func TestOfflineCollocationFilesAsCollocation(t *testing.T) {
|
||||||
|
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||||
|
|
||||||
|
got := postMechanics(t, srv, docID, `[
|
||||||
|
{"from":0,"to":13,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"},
|
||||||
|
{"from":20,"to":27,"original":"the the","replacement":"the","explanation":"doubled word","type":"mechanics"}
|
||||||
|
]`)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("want both findings, got %+v", got)
|
||||||
|
}
|
||||||
|
coll := pendingOfType(got, db.SuggestionTypeCollocation)
|
||||||
|
if len(coll) != 1 {
|
||||||
|
t.Fatalf("want 1 collocation, got %+v", got)
|
||||||
|
}
|
||||||
|
if coll[0].Source != db.SuggestionSourceLocal {
|
||||||
|
t.Errorf("offline finding should be source=local, got %q", coll[0].Source)
|
||||||
|
}
|
||||||
|
if mech := pendingOfType(got, db.SuggestionTypeMechanics); len(mech) != 1 {
|
||||||
|
t.Fatalf("want 1 mechanics finding, got %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUnknownLocalTypeFallsBackToMechanics: a family the offline pass isn't
|
||||||
|
// allowed to claim (or an older client sending none at all) must land in
|
||||||
|
// mechanics. Otherwise a stray label would smuggle a row into an LLM family,
|
||||||
|
// where nothing would ever replace it.
|
||||||
|
func TestUnknownLocalTypeFallsBackToMechanics(t *testing.T) {
|
||||||
|
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||||
|
|
||||||
|
got := postMechanics(t, srv, docID, `[
|
||||||
|
{"from":0,"to":5,"original":"aaaaa","replacement":"bbbbb","explanation":"x","type":"voice"},
|
||||||
|
{"from":6,"to":11,"original":"ccccc","replacement":"ddddd","explanation":"y"}
|
||||||
|
]`)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("want 2 findings, got %+v", got)
|
||||||
|
}
|
||||||
|
for _, s := range got {
|
||||||
|
if s.Type != db.SuggestionTypeMechanics {
|
||||||
|
t.Errorf("offline finding claimed family %q; only mechanics/collocation are allowed", s.Type)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCoachDoesNotWipeOfflineCollocations is the collision the source column
|
||||||
|
// exists for: the LLM collocation pass replaces the collocation family, and the
|
||||||
|
// rule pack's share of that family has to survive it. Before `source`, running
|
||||||
|
// the coach silently deleted every offline chunk on the page.
|
||||||
|
func TestCoachDoesNotWipeOfflineCollocations(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"apple","replacement":"an apple","explanation":"article","type":"collocation"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, _ := newTestServer(t, client)
|
||||||
|
|
||||||
|
// The seeded doc is "I has two apple." — the coach's flag anchors on "apple"
|
||||||
|
// at [10,15], so the offline finding is given a span well clear of it. Two
|
||||||
|
// findings fighting over the same characters is a different rule (see
|
||||||
|
// TestOfflineCardWinsSpanCollision); this test is about the DELETE.
|
||||||
|
postMechanics(t, srv, docID, `[
|
||||||
|
{"from":0,"to":5,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||||
|
]`)
|
||||||
|
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/collocation", "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("collocation pass: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
var got []db.Suggestion
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var local, llm int
|
||||||
|
for _, s := range pendingOfType(got, db.SuggestionTypeCollocation) {
|
||||||
|
if s.Source == db.SuggestionSourceLocal {
|
||||||
|
local++
|
||||||
|
} else {
|
||||||
|
llm++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if local != 1 {
|
||||||
|
t.Errorf("the coach wiped the offline collocation: local=%d, got %+v", local, got)
|
||||||
|
}
|
||||||
|
if llm != 1 {
|
||||||
|
t.Errorf("want the coach's own flag alongside it: llm=%d, got %+v", llm, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOfflinePassReplacesItsOwnCollocations is the mirror: the rule pack
|
||||||
|
// recomputes the whole document every run, so a chunk the current text no longer
|
||||||
|
// warrants must go — and the coach's flags must stay. Scoping the offline DELETE
|
||||||
|
// by type instead of source would have stranded the first row forever.
|
||||||
|
func TestOfflinePassReplacesItsOwnCollocations(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"apple","replacement":"an apple","explanation":"article","type":"collocation"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, _ := newTestServer(t, client)
|
||||||
|
|
||||||
|
// A coach flag, then an offline chunk, then a rerun that no longer finds it.
|
||||||
|
do(t, srv, http.MethodPost, "/docs/"+docID+"/collocation", "")
|
||||||
|
postMechanics(t, srv, docID, `[
|
||||||
|
{"from":0,"to":13,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||||
|
]`)
|
||||||
|
got := postMechanics(t, srv, docID, `[]`)
|
||||||
|
|
||||||
|
for _, s := range got {
|
||||||
|
if s.Source == db.SuggestionSourceLocal {
|
||||||
|
t.Errorf("stale offline finding survived a recompute: %+v", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(pendingOfType(got, db.SuggestionTypeCollocation)) != 1 {
|
||||||
|
t.Fatalf("the coach's own flag should be untouched, got %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOfflineCollocationPlantsOnAccept closes the loop the family split was for:
|
||||||
|
// a chunk the rule pack found, accepted, becomes a vocabulary-garden card — with
|
||||||
|
// no model involved anywhere in the path.
|
||||||
|
func TestOfflineCollocationPlantsOnAccept(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||||
|
if _, err := h.DB.Exec(
|
||||||
|
`UPDATE documents SET content_text = ? WHERE id = ?`,
|
||||||
|
"I had to do a decision about the job.", docID,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("set content: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := postMechanics(t, srv, docID, `[
|
||||||
|
{"from":9,"to":22,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||||
|
]`)
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("want the offline chunk, got %+v", got)
|
||||||
|
}
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/suggestions/"+got[0].ID+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("accept: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
cards := gardenCards(t, h)
|
||||||
|
if len(cards) != 1 || cards[0].word != "make a decision" {
|
||||||
|
t.Fatalf("want a planted phrase card, got %+v", cards)
|
||||||
|
}
|
||||||
|
// The example is the corrected sentence — the phrasing she kept, not the one
|
||||||
|
// she just left behind.
|
||||||
|
if cards[0].example != "I had to make a decision about the job." {
|
||||||
|
t.Errorf("example should be the corrected sentence, got %q", cards[0].example)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOfflineCardWinsSpanCollision: the tiebreak is by engine, not by family. An
|
||||||
|
// offline miscollocation has an exact span; the coach's overlapping flag is only
|
||||||
|
// advisory, so it is the one that goes.
|
||||||
|
func TestOfflineCardWinsSpanCollision(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"do a decision about","replacement":"decide about","explanation":"wordy","type":"collocation"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, h := newTestServer(t, client)
|
||||||
|
if _, err := h.DB.Exec(
|
||||||
|
`UPDATE documents SET content_text = ? WHERE id = ?`,
|
||||||
|
"I had to do a decision about the job.", docID,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("set content: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
do(t, srv, http.MethodPost, "/docs/"+docID+"/collocation", "")
|
||||||
|
got := postMechanics(t, srv, docID, `[
|
||||||
|
{"from":9,"to":22,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||||
|
]`)
|
||||||
|
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("want the overlapping coach flag dropped, got %+v", got)
|
||||||
|
}
|
||||||
|
if got[0].Source != db.SuggestionSourceLocal {
|
||||||
|
t.Errorf("the exact offline card should own the span, got %+v", got[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOfflineHanziFindingStaysMechanics: a 错别字 the Chinese rule pack found —
|
||||||
|
// both halves written in hanzi — files as an ordinary mechanics row.
|
||||||
|
//
|
||||||
|
// The check is worth its own test because there is a rule one layer over that
|
||||||
|
// would plausibly claim it. `isTranslation` re-labels an edit whose original
|
||||||
|
// reads as the writer's language and whose replacement reads as English, which
|
||||||
|
// is exactly how a zh-pair writer's quoted Chinese becomes a 'translate' card.
|
||||||
|
// A wrong-character fix looks like the first half of that and nothing like the
|
||||||
|
// second: 己经 → 已经 never leaves Chinese. It must stay a tidy-up in her own
|
||||||
|
// sentence, on the same rail as a doubled word, with no rendering-into-English
|
||||||
|
// implied anywhere.
|
||||||
|
func TestOfflineHanziFindingStaysMechanics(t *testing.T) {
|
||||||
|
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||||
|
|
||||||
|
got := postMechanics(t, srv, docID, `[
|
||||||
|
{"from":1,"to":3,"original":"己经","replacement":"已经","explanation":"已经 (already) takes 已","type":"mechanics"}
|
||||||
|
]`)
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("want the one finding, got %+v", got)
|
||||||
|
}
|
||||||
|
if got[0].Type != db.SuggestionTypeMechanics {
|
||||||
|
t.Errorf("hanzi fix filed as %q, want %q", got[0].Type, db.SuggestionTypeMechanics)
|
||||||
|
}
|
||||||
|
if got[0].Source != db.SuggestionSourceLocal {
|
||||||
|
t.Errorf("source = %q, want %q", got[0].Source, db.SuggestionSourceLocal)
|
||||||
|
}
|
||||||
|
// The characters survive the round trip intact — a mangled span here would
|
||||||
|
// replace the wrong characters in her document.
|
||||||
|
if got[0].Original != "己经" || got[0].Replacement != "已经" {
|
||||||
|
t.Errorf("round-tripped as %q → %q", got[0].Original, got[0].Replacement)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,243 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
// seedSuggestion writes one pending suggestion against the seeded doc, after
|
||||||
|
// replacing the doc's text so the sentence around `original` is under the test's
|
||||||
|
// control.
|
||||||
|
func seedSuggestion(t *testing.T, h *Handler, docID, text, sType, original, replacement, explanation string) string {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := h.DB.Exec(`UPDATE documents SET content_text = ? WHERE id = ?`, text, docID); err != nil {
|
||||||
|
t.Fatalf("set content: %v", err)
|
||||||
|
}
|
||||||
|
var id string
|
||||||
|
err := h.DB.QueryRow(
|
||||||
|
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, status)
|
||||||
|
VALUES (?, 0, 0, ?, ?, ?, ?, 'pending') RETURNING id`,
|
||||||
|
docID, original, replacement, explanation, sType,
|
||||||
|
).Scan(&id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("seed suggestion: %v", err)
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
type card struct {
|
||||||
|
word, definition, example string
|
||||||
|
interval int
|
||||||
|
}
|
||||||
|
|
||||||
|
func gardenCards(t *testing.T, h *Handler) []card {
|
||||||
|
t.Helper()
|
||||||
|
rows, err := h.DB.Query(
|
||||||
|
`SELECT word, definition, example, interval_days FROM vocab_words WHERE user_id = ? ORDER BY word`,
|
||||||
|
db.LocalUserID,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read garden: %v", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []card
|
||||||
|
for rows.Next() {
|
||||||
|
var c card
|
||||||
|
if err := rows.Scan(&c.word, &c.definition, &c.example, &c.interval); err != nil {
|
||||||
|
t.Fatalf("scan: %v", err)
|
||||||
|
}
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAcceptedCollocationIsPlanted walks the whole hand-over: a collocation the
|
||||||
|
// writer accepts becomes a phrase card whose example is the *corrected*
|
||||||
|
// sentence, so the flashcard quizzes the phrasing she kept.
|
||||||
|
func TestAcceptedCollocationIsPlanted(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
id := seedSuggestion(t, h, docID,
|
||||||
|
"Yesterday was hard. I had to do a decision about the job. Then I slept.",
|
||||||
|
db.SuggestionTypeCollocation, "do a decision", "make a decision",
|
||||||
|
"English pairs “make” with “decision”.")
|
||||||
|
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("accept: got %d, want 204", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
cards := gardenCards(t, h)
|
||||||
|
if len(cards) != 1 {
|
||||||
|
t.Fatalf("garden has %d cards, want 1: %+v", len(cards), cards)
|
||||||
|
}
|
||||||
|
got := cards[0]
|
||||||
|
if got.word != "make a decision" {
|
||||||
|
t.Errorf("word = %q, want %q", got.word, "make a decision")
|
||||||
|
}
|
||||||
|
if got.example != "I had to make a decision about the job." {
|
||||||
|
t.Errorf("example = %q — want the corrected sentence, bounded to its own sentence", got.example)
|
||||||
|
}
|
||||||
|
if got.definition != "English pairs “make” with “decision”." {
|
||||||
|
t.Errorf("definition = %q, want the explanation", got.definition)
|
||||||
|
}
|
||||||
|
if got.interval != 1 {
|
||||||
|
t.Errorf("interval_days = %d, want 1 (due tomorrow, like a fresh capture)", got.interval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOnlyCollocationsArePlanted: the other families correct this sentence and
|
||||||
|
// hand over nothing reusable. A dismissed collocation is not a lesson either.
|
||||||
|
func TestOnlyCollocationsArePlanted(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
|
||||||
|
grammar := seedSuggestion(t, h, docID, "I has two apples.",
|
||||||
|
db.SuggestionTypeGrammar, "I has", "I have", "Subject–verb agreement.")
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/suggestions/"+grammar+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("accept grammar: got %d", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
dismissed := seedSuggestion(t, h, docID, "We must take a photo of it.",
|
||||||
|
db.SuggestionTypeCollocation, "do a photo", "take a photo", "Photos are taken.")
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/suggestions/"+dismissed+"/dismiss", ""); rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("dismiss: got %d", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cards := gardenCards(t, h); len(cards) != 0 {
|
||||||
|
t.Fatalf("garden grew %d card(s) from a grammar fix and a dismissal: %+v", len(cards), cards)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPlantingIsIdempotentAndNeverResets: accepting the same chunk again is
|
||||||
|
// evidence it's still being learned — the worst possible response is to wipe the
|
||||||
|
// card's first context and the schedule it has been climbing.
|
||||||
|
func TestPlantingIsIdempotentAndNeverResets(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
first := seedSuggestion(t, h, docID, "I had to do a decision.",
|
||||||
|
db.SuggestionTypeCollocation, "do a decision", "make a decision", "First explanation.")
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/suggestions/"+first+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("accept: got %d", rec.Code)
|
||||||
|
}
|
||||||
|
// The card climbs a little.
|
||||||
|
if _, err := h.DB.Exec(
|
||||||
|
`UPDATE vocab_words SET reps = 3, interval_days = 7 WHERE user_id = ? AND word = 'make a decision'`,
|
||||||
|
db.LocalUserID,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("advance card: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
second := seedSuggestion(t, h, docID, "Later I must do a decision again.",
|
||||||
|
db.SuggestionTypeCollocation, "do a decision", "make a decision", "Second explanation.")
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/suggestions/"+second+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("accept again: got %d", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
cards := gardenCards(t, h)
|
||||||
|
if len(cards) != 1 {
|
||||||
|
t.Fatalf("garden has %d cards, want 1 (one chunk, one card)", len(cards))
|
||||||
|
}
|
||||||
|
if cards[0].definition != "First explanation." {
|
||||||
|
t.Errorf("definition = %q — the existing card should win", cards[0].definition)
|
||||||
|
}
|
||||||
|
if cards[0].example != "I had to make a decision." {
|
||||||
|
t.Errorf("example = %q — the first context should survive", cards[0].example)
|
||||||
|
}
|
||||||
|
if cards[0].interval != 7 {
|
||||||
|
t.Errorf("interval_days = %d, want 7 — progress must not be reset", cards[0].interval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSentenceRewriteIsNotAPhraseCard: a "collocation" long enough to be a
|
||||||
|
// rewritten sentence makes a miserable flashcard, so it is dropped rather than
|
||||||
|
// planted — and the accept still succeeds.
|
||||||
|
func TestSentenceRewriteIsNotAPhraseCard(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
long := "I would like to take this opportunity to thank you for everything"
|
||||||
|
id := seedSuggestion(t, h, docID, "I want thank you for everything.",
|
||||||
|
db.SuggestionTypeCollocation, "I want thank you for everything", long, "More natural.")
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("accept: got %d, want 204 — a skipped card must never fail the accept", rec.Code)
|
||||||
|
}
|
||||||
|
if cards := gardenCards(t, h); len(cards) != 0 {
|
||||||
|
t.Fatalf("planted a sentence as a phrase card: %+v", cards)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCorrectedSentence(t *testing.T) {
|
||||||
|
const text = "One thing. I had to do a decision fast! Another thing."
|
||||||
|
cases := []struct {
|
||||||
|
name, original, replacement, want string
|
||||||
|
}{
|
||||||
|
{"bounded to its sentence", "do a decision", "make a decision", "I had to make a decision fast!"},
|
||||||
|
{"original no longer present", "do a choice", "make a choice", ""},
|
||||||
|
{"empty original", "", "make a decision", ""},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := correctedSentence(text, tc.original, tc.replacement); got != tc.want {
|
||||||
|
t.Errorf("correctedSentence = %q, want %q", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// A document with no terminator at all is one sentence, and still works.
|
||||||
|
if got := correctedSentence("i had to do a decision", "do a decision", "make a decision"); got != "i had to make a decision" {
|
||||||
|
t.Errorf("unterminated doc: got %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPlantedPhraseCarriesTheDocumentLanguage: a chunk planted out of a document
|
||||||
|
// written in her own language is a card in that language. The collocation pass
|
||||||
|
// itself deliberately did not flip in Phase 28 — its prompt is per-language
|
||||||
|
// knowledge, not framing — but the phrase it hands over is still lifted from her
|
||||||
|
// prose, so the card has to know what language that prose was in or the garden
|
||||||
|
// will read it aloud in the wrong voice.
|
||||||
|
func TestPlantedPhraseCarriesTheDocumentLanguage(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
if _, err := h.DB.Exec(`UPDATE documents SET doc_lang = 'pair' WHERE id = ?`, docID); err != nil {
|
||||||
|
t.Fatalf("set doc_lang: %v", err)
|
||||||
|
}
|
||||||
|
id := seedSuggestion(t, h, docID,
|
||||||
|
"Ontem foi difícil. Tive de tomar uma decisão sobre o trabalho.",
|
||||||
|
db.SuggestionTypeCollocation, "tomar uma decisão", "tomar uma decisão",
|
||||||
|
"Em português diz-se “tomar” uma decisão.")
|
||||||
|
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("accept: got %d, want 204", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
var lang string
|
||||||
|
if err := h.DB.QueryRow(
|
||||||
|
`SELECT lang FROM vocab_words WHERE user_id = ? AND word = ?`,
|
||||||
|
db.LocalUserID, "tomar uma decisão",
|
||||||
|
).Scan(&lang); err != nil {
|
||||||
|
t.Fatalf("read planted card: %v", err)
|
||||||
|
}
|
||||||
|
if lang != "pair" {
|
||||||
|
t.Fatalf("planted card lang = %q, want pair", lang)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPlantedPhraseOnAnEnglishDocumentIsUnchanged is the other half, and the one
|
||||||
|
// every account is on today: an English document plants an English card, and the
|
||||||
|
// empty backfill and 'en' both mean that.
|
||||||
|
func TestPlantedPhraseOnAnEnglishDocumentIsUnchanged(t *testing.T) {
|
||||||
|
srv, docID, h := newTestServer(t, &stubClient{})
|
||||||
|
id := seedSuggestion(t, h, docID, "I had to do a decision about the job.",
|
||||||
|
db.SuggestionTypeCollocation, "do a decision", "make a decision",
|
||||||
|
"English pairs “make” with “decision”.")
|
||||||
|
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("accept: got %d, want 204", rec.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
var lang string
|
||||||
|
if err := h.DB.QueryRow(
|
||||||
|
`SELECT lang FROM vocab_words WHERE user_id = ? AND word = ?`,
|
||||||
|
db.LocalUserID, "make a decision",
|
||||||
|
).Scan(&lang); err != nil {
|
||||||
|
t.Fatalf("read planted card: %v", err)
|
||||||
|
}
|
||||||
|
if lang == "pair" {
|
||||||
|
t.Fatalf("planted card lang = %q on an English document, want en or empty", lang)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,305 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Reconciliation replaces the old "delete the family, insert the new batch"
|
||||||
|
// shape of every pass. A suggestion the pass proposes again is the *same*
|
||||||
|
// suggestion: it keeps its row, and therefore its id, its created_at and — most
|
||||||
|
// visibly — the explanation it was first given. The model re-words its reasoning
|
||||||
|
// every time it is asked, so re-inserting meant one unchanged mistake carried
|
||||||
|
// three different explanations in a single sitting.
|
||||||
|
//
|
||||||
|
// The id is what the frontend keys its cards on, so a stable id is also what
|
||||||
|
// keeps the rail from emptying and refilling, a card from collapsing mid-read,
|
||||||
|
// and the arrival chime from re-firing for advice she has already seen.
|
||||||
|
|
||||||
|
// pendingRow is the part of an existing pending suggestion reconciliation cares
|
||||||
|
// about.
|
||||||
|
type pendingRow struct {
|
||||||
|
id string
|
||||||
|
original string
|
||||||
|
replacement string
|
||||||
|
chunkHash string
|
||||||
|
from int
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadPending reads the pending rows a pass owns. `where` is the pass's own
|
||||||
|
// scoping clause (by source, and for the model passes by family) — the same
|
||||||
|
// fragment that used to scope its DELETE.
|
||||||
|
func loadPending(tx *sql.Tx, docID, where string) ([]pendingRow, error) {
|
||||||
|
rows, err := tx.Query(
|
||||||
|
`SELECT id, original, replacement, chunk_hash, from_pos FROM suggestions
|
||||||
|
WHERE doc_id = ? AND status = ? AND `+where,
|
||||||
|
docID, db.SuggestionStatusPending,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var out []pendingRow
|
||||||
|
for rows.Next() {
|
||||||
|
var r pendingRow
|
||||||
|
if err := rows.Scan(&r.id, &r.original, &r.replacement, &r.chunkHash, &r.from); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// editKey identifies an edit by what it proposes, not where: "this exact change
|
||||||
|
// to this exact text". Normalized like the suppression comparisons, so the
|
||||||
|
// editor's quote rewriting and a reflowed paragraph don't read as a new edit.
|
||||||
|
func editKey(original, replacement string) string {
|
||||||
|
return normalizeForDedup(original) + "\x00" + normalizeForDedup(replacement)
|
||||||
|
}
|
||||||
|
|
||||||
|
// editIndex matches freshly proposed edits against the rows already standing.
|
||||||
|
type editIndex struct {
|
||||||
|
rows []pendingRow
|
||||||
|
used []bool
|
||||||
|
byKey map[string][]int
|
||||||
|
}
|
||||||
|
|
||||||
|
func indexByEdit(rows []pendingRow) *editIndex {
|
||||||
|
idx := &editIndex{rows: rows, used: make([]bool, len(rows)), byKey: map[string][]int{}}
|
||||||
|
for i, r := range rows {
|
||||||
|
k := editKey(r.original, r.replacement)
|
||||||
|
idx.byKey[k] = append(idx.byKey[k], i)
|
||||||
|
}
|
||||||
|
return idx
|
||||||
|
}
|
||||||
|
|
||||||
|
// take claims the standing row for this edit, if there is one. When a document
|
||||||
|
// repeats the same mistake, `near` (the fresh span's start) picks the closest
|
||||||
|
// standing row, so two identical cards keep their own identities instead of
|
||||||
|
// trading them whenever the text between them grows.
|
||||||
|
func (i *editIndex) take(original, replacement string, near int) (pendingRow, bool) {
|
||||||
|
best, bestDist := -1, 0
|
||||||
|
for _, n := range i.byKey[editKey(original, replacement)] {
|
||||||
|
if i.used[n] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
d := i.rows[n].from - near
|
||||||
|
if d < 0 {
|
||||||
|
d = -d
|
||||||
|
}
|
||||||
|
if best < 0 || d < bestDist {
|
||||||
|
best, bestDist = n, d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if best < 0 {
|
||||||
|
return pendingRow{}, false
|
||||||
|
}
|
||||||
|
i.used[best] = true
|
||||||
|
return i.rows[best], true
|
||||||
|
}
|
||||||
|
|
||||||
|
// reposition updates the advisory offsets (and the sentence a row belongs to)
|
||||||
|
// without touching anything the writer can see. The frontend re-anchors by
|
||||||
|
// string at render time, so these only matter for the local-vs-model span
|
||||||
|
// arbitration in dedupeSpans.
|
||||||
|
func reposition(tx *sql.Tx, row pendingRow, from, to int, chunkHash string) error {
|
||||||
|
if row.from == from && row.chunkHash == chunkHash {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err := tx.Exec(
|
||||||
|
`UPDATE suggestions SET from_pos = ?, to_pos = ?, chunk_hash = ? WHERE id = ?`,
|
||||||
|
from, to, chunkHash, row.id,
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconcilePending brings a model pass's family in line with what it just
|
||||||
|
// proposed, sentence by sentence:
|
||||||
|
//
|
||||||
|
// - A row on a sentence this pass didn't ask about is kept untouched — that
|
||||||
|
// is the whole point of chunking. Only its offsets are refreshed.
|
||||||
|
// - A row on a sentence that no longer exists in the document is dropped: she
|
||||||
|
// rewrote or deleted it.
|
||||||
|
// - A row on a sentence the pass *did* ask about survives only if the model
|
||||||
|
// proposed the same edit again, in which case it keeps its identity.
|
||||||
|
//
|
||||||
|
// `fresh` names the sentences the model was asked about (nil when it wasn't
|
||||||
|
// called at all). inPlayAll marks the whole-document passes — voice and the
|
||||||
|
// collocation coach — where every row is up for re-proposal because the model
|
||||||
|
// just re-read everything.
|
||||||
|
//
|
||||||
|
// `pairLang` is the writer's own language and `docLang` this document's language
|
||||||
|
// verdict; between them they type a finding that turns out to be one language
|
||||||
|
// rendered into the other, in whichever direction this document makes useful
|
||||||
|
// (see language.go).
|
||||||
|
func (h *Handler) reconcilePending(
|
||||||
|
docID, contentText, pairLang, docLang string,
|
||||||
|
raw []llm.RawSuggestion,
|
||||||
|
scope pendingScope,
|
||||||
|
chunks, fresh []chunk,
|
||||||
|
inPlayAll bool,
|
||||||
|
) error {
|
||||||
|
tx, err := h.DB.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
|
||||||
|
existing, err := loadPending(tx, docID, scope.deleteWhere)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
present := hashSet(chunks)
|
||||||
|
asked := hashSet(fresh)
|
||||||
|
modelRan := inPlayAll || fresh != nil
|
||||||
|
|
||||||
|
// Sentences to hand back to the model next time, because a row we were
|
||||||
|
// caching on them turned out to be unanchorable (see below).
|
||||||
|
reopen := map[string]bool{}
|
||||||
|
|
||||||
|
var inPlay []pendingRow
|
||||||
|
for _, r := range existing {
|
||||||
|
switch {
|
||||||
|
// A row whose sentence we can't name is never cached — it is re-examined
|
||||||
|
// whenever the model speaks, and left alone when it doesn't.
|
||||||
|
case inPlayAll, r.chunkHash == "" && modelRan, asked[r.chunkHash]:
|
||||||
|
inPlay = append(inPlay, r)
|
||||||
|
case r.chunkHash != "" && !present[r.chunkHash]:
|
||||||
|
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, r.id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
// Untouched sentence: keep the card exactly as she last saw it.
|
||||||
|
from, to := locate(contentText, r.original)
|
||||||
|
if from < 0 {
|
||||||
|
// The sentence is unchanged in substance but the quoted span no
|
||||||
|
// longer matches byte for byte — a quote mark the editor rewrote
|
||||||
|
// inside it, say. The frontend anchors by that string, so this card
|
||||||
|
// can't be shown; drop it and let the sentence be read again rather
|
||||||
|
// than cache advice nobody can see.
|
||||||
|
reopen[r.chunkHash] = true
|
||||||
|
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, r.id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := reposition(tx, r, from, to, r.chunkHash); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for h := range reopen {
|
||||||
|
delete(present, h)
|
||||||
|
}
|
||||||
|
|
||||||
|
sup, err := buildSuppressor(tx, docID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
index := indexByEdit(inPlay)
|
||||||
|
kept := make(map[string]bool, len(inPlay))
|
||||||
|
for _, s := range raw {
|
||||||
|
if sup.suppressed(s.Original, s.Replacement) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
from, to := locate(contentText, s.Original)
|
||||||
|
// Attribute the finding to a sentence the model was actually shown before
|
||||||
|
// falling back to the whole document: a short span ("the the") can occur in
|
||||||
|
// two sentences, and crediting it to the cached one would drop it as advice
|
||||||
|
// we already have.
|
||||||
|
hash := chunkFor(s.Original, fresh)
|
||||||
|
if hash == "" {
|
||||||
|
hash = chunkFor(s.Original, chunks)
|
||||||
|
}
|
||||||
|
// A sentence we didn't ask about already has whatever advice it deserves.
|
||||||
|
// The model can't normally quote one — it was only shown the delta — but if
|
||||||
|
// it wanders there anyway, the cached card stands rather than gaining a
|
||||||
|
// twin.
|
||||||
|
if !inPlayAll && hash != "" && present[hash] && !asked[hash] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if row, ok := index.take(s.Original, s.Replacement, from); ok {
|
||||||
|
kept[row.id] = true
|
||||||
|
if err := reposition(tx, row, from, to, hash); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// A pass with a forced type owns its family outright and is never asked
|
||||||
|
// about translation: voice reads whole paragraphs for tone, and the
|
||||||
|
// collocation coach is about English word pairings. Only the open-typed
|
||||||
|
// grammar checkpoint can turn out to have been handed her own language.
|
||||||
|
typ := scope.forceType
|
||||||
|
if typ == "" {
|
||||||
|
typ = normalizeType(s.Type)
|
||||||
|
if isTranslation(s.Original, s.Replacement, pairLang, docLang) {
|
||||||
|
typ = db.SuggestionTypeTranslate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(
|
||||||
|
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, source, chunk_hash)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||||
|
docID, from, to, s.Original, s.Replacement, s.Explanation, typ, db.SuggestionSourceLLM, hash,
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Asked about and not proposed again: the model has changed its mind, or she
|
||||||
|
// has fixed it.
|
||||||
|
for _, r := range inPlay {
|
||||||
|
if kept[r.id] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, r.id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Record the sentences this family has now read. Every sentence still in the
|
||||||
|
// document has been read by *some* pass: the ones just asked about now, the
|
||||||
|
// rest in an earlier round.
|
||||||
|
if scope.chunked {
|
||||||
|
if _, err := tx.Exec(
|
||||||
|
`DELETE FROM checked_chunks WHERE doc_id = ? AND family = ?`, docID, scope.family,
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for h := range present {
|
||||||
|
if _, err := tx.Exec(
|
||||||
|
`INSERT INTO checked_chunks (doc_id, family, hash) VALUES (?, ?, ?)`,
|
||||||
|
docID, scope.family, h,
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkedChunks loads the sentences a family read on its last pass.
|
||||||
|
func (h *Handler) checkedChunks(docID, family string) (map[string]bool, error) {
|
||||||
|
rows, err := h.DB.Query(
|
||||||
|
`SELECT hash FROM checked_chunks WHERE doc_id = ? AND family = ?`, docID, family,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
out := map[string]bool{}
|
||||||
|
for rows.Next() {
|
||||||
|
var hash string
|
||||||
|
if err := rows.Scan(&hash); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out[hash] = true
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
@@ -69,7 +69,7 @@ func (h *Handler) rewrite(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
out, err := llm.RunRewrite(r.Context(), h.Client, text, body.Style)
|
out, err := llm.RunRewrite(r.Context(), h.Client, text, body.Style)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
httputil.ErrorJSON(w, http.StatusBadGateway, "rewrite failed: "+err.Error())
|
httputil.UpstreamError(w, "rewrite", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The settled endpoint exists for the offline half of the loop. The rule pack
|
||||||
|
// detects from the text alone, 250 ms after a keystroke, and has no memory
|
||||||
|
// between runs — so without the document's record of what she has already
|
||||||
|
// answered, a dismissed finding is re-detected and re-rendered on the next
|
||||||
|
// keystroke, and stays there for as long as the server can't be reached.
|
||||||
|
|
||||||
|
func getSettled(t *testing.T, srv http.Handler, docID string) []string {
|
||||||
|
t.Helper()
|
||||||
|
rec := do(t, srv, http.MethodGet, "/docs/"+docID+"/settled", "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("settled: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
var out struct {
|
||||||
|
Originals []string `json:"originals"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
return out.Originals
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(list []string, want string) bool {
|
||||||
|
for _, s := range list {
|
||||||
|
if s == want {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSettledListsActionedSpans proves the endpoint reports exactly the spans the
|
||||||
|
// suppressor would drop: accepted and dismissed, never pending. A pending row
|
||||||
|
// leaking in would be the damaging direction — the client would hide a card she
|
||||||
|
// has never been shown an answer to.
|
||||||
|
func TestSettledListsActionedSpans(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"},
|
||||||
|
{"original":"two apple","replacement":"two apples","explanation":"plural","type":"grammar"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, _ := newTestServer(t, client)
|
||||||
|
|
||||||
|
if got := getSettled(t, srv, docID); len(got) != 0 {
|
||||||
|
t.Fatalf("nothing actioned yet, got %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
var got []db.Suggestion
|
||||||
|
_ = json.Unmarshal(rec.Body.Bytes(), &got)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("first pass: want 2, got %d", len(got))
|
||||||
|
}
|
||||||
|
|
||||||
|
// One accepted, one still pending: only the accepted span is settled.
|
||||||
|
do(t, srv, http.MethodPost, "/suggestions/"+got[0].ID+"/accept", "")
|
||||||
|
settled := getSettled(t, srv, docID)
|
||||||
|
if len(settled) != 1 || settled[0] != got[0].Original {
|
||||||
|
t.Fatalf("want just %q settled, got %v", got[0].Original, settled)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A dismissal settles a span just as an accept does — the whole point of the
|
||||||
|
// item: "you already decided about this one" doesn't mean "you agreed".
|
||||||
|
do(t, srv, http.MethodPost, "/suggestions/"+got[1].ID+"/dismiss", "")
|
||||||
|
settled = getSettled(t, srv, docID)
|
||||||
|
if len(settled) != 2 || !contains(settled, got[1].Original) {
|
||||||
|
t.Fatalf("dismissed span missing from %v", settled)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSettledNormalizesAndDedupes proves the payload is normalized server-side
|
||||||
|
// and collapsed. The client compares its freshly-detected findings against these
|
||||||
|
// strings, so the two sides have to agree on what "the same span" is — the
|
||||||
|
// editor's quote churn is the case that breaks a byte-exact match, and it is why
|
||||||
|
// normalizeForDedup exists at all.
|
||||||
|
func TestSettledNormalizesAndDedupes(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[]}`}
|
||||||
|
srv, docID, h := newTestServer(t, client)
|
||||||
|
|
||||||
|
// The same span twice, differing only in quote style and line breaks — one
|
||||||
|
// accepted, one dismissed. Distinct rows; one settled span.
|
||||||
|
a := seedSuggestion(t, h, docID, "text", db.SuggestionTypeGrammar,
|
||||||
|
"She said \"hello\"\n to me", "She said 'hello' to me", "quotes")
|
||||||
|
b := seedSuggestion(t, h, docID, "text", db.SuggestionTypeGrammar,
|
||||||
|
"She said “hello” to me", "She said 'hello' to me", "quotes")
|
||||||
|
do(t, srv, http.MethodPost, "/suggestions/"+a+"/accept", "")
|
||||||
|
do(t, srv, http.MethodPost, "/suggestions/"+b+"/dismiss", "")
|
||||||
|
|
||||||
|
settled := getSettled(t, srv, docID)
|
||||||
|
if len(settled) != 1 {
|
||||||
|
t.Fatalf("two spellings of one span should collapse to one, got %v", settled)
|
||||||
|
}
|
||||||
|
if want := "She said 'hello' to me"; settled[0] != want {
|
||||||
|
t.Fatalf("settled[0] = %q, want normalized %q", settled[0], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNormalizeMatchesTheClient is the Go half of a pair. Every case here also
|
||||||
|
// appears in web/src/lib/settled.test.ts, asserted against the TypeScript
|
||||||
|
// reimplementation of this function. The two are compared across a network
|
||||||
|
// boundary — the server normalizes what it sends, the client normalizes what it
|
||||||
|
// checks against it — so they have to fold the same characters the same way, and
|
||||||
|
// nothing but a shared list of cases can say so. Add to both or neither.
|
||||||
|
func TestNormalizeMatchesTheClient(t *testing.T) {
|
||||||
|
cases := []struct{ in, want string }{
|
||||||
|
{"She said “hello”", "She said 'hello'"},
|
||||||
|
{"She said \"hello\"", "She said 'hello'"},
|
||||||
|
{"it‘s", "it's"},
|
||||||
|
{"it’s", "it's"},
|
||||||
|
{"`code´", "'code'"},
|
||||||
|
{" a apple\n here ", "a apple here"},
|
||||||
|
{"a\tapple", "a apple"},
|
||||||
|
{" \n ", ""},
|
||||||
|
{"我想说这句话", "我想说这句话"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := normalizeForDedup(c.in); got != c.want {
|
||||||
|
t.Errorf("normalizeForDedup(%q) = %q, want %q", c.in, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestSettledEmptyIsAList guards the shape rather than the content: the client
|
||||||
|
// spreads this array into its settled set, and a null would throw there. Go
|
||||||
|
// marshals a nil slice as null, so this is one `[]string{}` away from breaking.
|
||||||
|
func TestSettledEmptyIsAList(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[]}`}
|
||||||
|
srv, docID, _ := newTestServer(t, client)
|
||||||
|
|
||||||
|
rec := do(t, srv, http.MethodGet, "/docs/"+docID+"/settled", "")
|
||||||
|
if body := rec.Body.String(); body != "{\"originals\":[]}\n" && body != "{\"originals\":[]}" {
|
||||||
|
t.Fatalf("empty settled body = %q, want an empty list", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,191 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// byOriginal indexes a pending set by the text each card flags.
|
||||||
|
func byOriginal(in []db.Suggestion) map[string]db.Suggestion {
|
||||||
|
out := map[string]db.Suggestion{}
|
||||||
|
for _, s := range in {
|
||||||
|
out[s.Original] = s
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUntouchedSentencesKeepTheirCards is the heart of the stability work: she
|
||||||
|
// edits one sentence, and the cards on every other sentence stay exactly as they
|
||||||
|
// were — same id (so the rail keeps the card instead of remounting it), same
|
||||||
|
// explanation (the model re-words its reasoning every time it is asked, and one
|
||||||
|
// unchanged mistake used to carry three different explanations in a sitting).
|
||||||
|
// The model is only asked about the sentence that changed.
|
||||||
|
func TestUntouchedSentencesKeepTheirCards(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"I has two apple","replacement":"I have two apples","explanation":"first wording","type":"grammar"},
|
||||||
|
{"original":"She go to market","replacement":"She goes to market","explanation":"agreement","type":"grammar"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, h := newTestServer(t, client)
|
||||||
|
h.Limit = llm.NewRateLimiter(0)
|
||||||
|
setDocText(t, h, docID, "I has two apple. She go to market yesterday.")
|
||||||
|
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
var first []db.Suggestion
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &first); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if len(first) != 2 {
|
||||||
|
t.Fatalf("first pass: want 2, got %d: %+v", len(first), first)
|
||||||
|
}
|
||||||
|
kept := byOriginal(first)["I has two apple"]
|
||||||
|
|
||||||
|
// She fixes only the second sentence. The model, asked again, re-words its
|
||||||
|
// reasoning about the first — which it must never get the chance to do.
|
||||||
|
setDocText(t, h, docID, "I has two apple. She goes to market yesterday.")
|
||||||
|
client.response = `{"suggestions":[
|
||||||
|
{"original":"I has two apple","replacement":"I have two apples","explanation":"REWORDED","type":"grammar"}
|
||||||
|
]}`
|
||||||
|
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
var second []db.Suggestion
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &second); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(client.lastPrompt, "I has two apple") {
|
||||||
|
t.Fatalf("untouched sentence was sent to the model:\n%s", client.lastPrompt)
|
||||||
|
}
|
||||||
|
if !strings.Contains(client.lastPrompt, "She goes to market") {
|
||||||
|
t.Fatalf("edited sentence was not sent to the model:\n%s", client.lastPrompt)
|
||||||
|
}
|
||||||
|
|
||||||
|
now := byOriginal(second)["I has two apple"]
|
||||||
|
if now.ID != kept.ID {
|
||||||
|
t.Fatalf("card was remounted: id %q became %q", kept.ID, now.ID)
|
||||||
|
}
|
||||||
|
if now.Explanation != "first wording" {
|
||||||
|
t.Fatalf("explanation drifted: %q", now.Explanation)
|
||||||
|
}
|
||||||
|
// The fixed sentence's card is gone, and the model's stray re-proposal for the
|
||||||
|
// cached sentence did not become a second card.
|
||||||
|
if len(second) != 1 {
|
||||||
|
t.Fatalf("want exactly one card left, got %d: %+v", len(second), second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUnchangedDocumentSkipsTheModel proves a check with nothing new to read
|
||||||
|
// costs nothing: no model call, and every card left standing untouched. This is
|
||||||
|
// the doc-open and tone-less re-check path.
|
||||||
|
func TestUnchangedDocumentSkipsTheModel(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, h := newTestServer(t, client)
|
||||||
|
h.Limit = llm.NewRateLimiter(0)
|
||||||
|
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
var first []db.Suggestion
|
||||||
|
_ = json.Unmarshal(rec.Body.Bytes(), &first)
|
||||||
|
if len(first) != 1 || client.calls != 1 {
|
||||||
|
t.Fatalf("first pass: %d cards, %d calls", len(first), client.calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
var second []db.Suggestion
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &second); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if client.calls != 1 {
|
||||||
|
t.Fatalf("re-checking an unedited document called the model %d times", client.calls)
|
||||||
|
}
|
||||||
|
if len(second) != 1 || second[0].ID != first[0].ID {
|
||||||
|
t.Fatalf("card did not survive an idle re-check: %+v", second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDeletedSentenceDropsItsCard covers the other half of the skip path: she
|
||||||
|
// removes a flagged sentence outright, so nothing changed that the model could
|
||||||
|
// be asked about — but its card must still go.
|
||||||
|
func TestDeletedSentenceDropsItsCard(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, h := newTestServer(t, client)
|
||||||
|
h.Limit = llm.NewRateLimiter(0)
|
||||||
|
|
||||||
|
do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
setDocText(t, h, docID, "")
|
||||||
|
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
var got []db.Suggestion
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Fatalf("card outlived its sentence: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestToneChangeReopensEverySentence: the checkpoint's advice is written for the
|
||||||
|
// document's tone, so switching from a journal to an academic essay has to
|
||||||
|
// re-read sentences that haven't changed a character.
|
||||||
|
func TestToneChangeReopensEverySentence(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, h := newTestServer(t, client)
|
||||||
|
h.Limit = llm.NewRateLimiter(0)
|
||||||
|
|
||||||
|
do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
if _, err := h.DB.Exec(`UPDATE documents SET tone = 'academic' WHERE id = ?`, docID); err != nil {
|
||||||
|
t.Fatalf("set tone: %v", err)
|
||||||
|
}
|
||||||
|
do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
|
||||||
|
if client.calls != 2 {
|
||||||
|
t.Fatalf("tone change did not re-read the document: %d model calls", client.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMechanicsFindingsKeepTheirRows: the rule pack re-runs 250 ms after every
|
||||||
|
// keystroke. A finding it still reports must keep its row, or the rail would
|
||||||
|
// remount several times a sentence — collapsing a card she has open, and
|
||||||
|
// re-firing the arrival chime for advice she is already reading.
|
||||||
|
func TestMechanicsFindingsKeepTheirRows(t *testing.T) {
|
||||||
|
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||||
|
body := `{"findings":[
|
||||||
|
{"from":0,"to":5,"original":"I has","replacement":"I have","explanation":"agreement"},
|
||||||
|
{"from":6,"to":15,"original":"two apple","replacement":"two apples","explanation":"plural"}
|
||||||
|
]}`
|
||||||
|
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/mechanics", body)
|
||||||
|
var first []db.Suggestion
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &first); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if len(first) != 2 {
|
||||||
|
t.Fatalf("want 2 rows, got %d", len(first))
|
||||||
|
}
|
||||||
|
|
||||||
|
// She types elsewhere: same findings, shifted spans, one of them now fixed.
|
||||||
|
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/mechanics", `{"findings":[
|
||||||
|
{"from":20,"to":25,"original":"I has","replacement":"I have","explanation":"agreement"}
|
||||||
|
]}`)
|
||||||
|
var second []db.Suggestion
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &second); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if len(second) != 1 {
|
||||||
|
t.Fatalf("want 1 row, got %d: %+v", len(second), second)
|
||||||
|
}
|
||||||
|
if second[0].ID != byOriginal(first)["I has"].ID {
|
||||||
|
t.Fatalf("surviving finding was given a new identity: %+v", second[0])
|
||||||
|
}
|
||||||
|
if second[0].FromPos != 20 {
|
||||||
|
t.Fatalf("span did not follow the text: %+v", second[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,23 +17,44 @@ type translateResponse struct {
|
|||||||
Translation string `json:"translation"`
|
Translation string `json:"translation"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// translate renders a suggestion's English explanation into Simplified Chinese
|
// translate renders a suggestion's explanation into the other half of the
|
||||||
// for the Ask Petal bubble, so the ESL reader sees the "why" in her first
|
// writer's pair for the Ask Petal bubble, so she sees the "why" in the language
|
||||||
// language instead of a second copy of the same English text. The explanation is
|
// she reads most easily instead of a second copy of the same text. The
|
||||||
// loaded server-side from the suggestion id (scoped to the local user) and never
|
// explanation is loaded server-side from the suggestion id (scoped to the
|
||||||
// trusted from the client, mirroring chat (spec Note #10).
|
// caller) and never trusted from the client, mirroring chat (spec Note #10).
|
||||||
|
//
|
||||||
|
// Which language it renders into cannot be assumed (Phase 28). Before that phase
|
||||||
|
// every explanation was English and every rendering went into her language, so
|
||||||
|
// "the pair language" was a safe constant. Now the explanation's language is a
|
||||||
|
// decision — `targetFor`, from the document's verdict and her direction — and
|
||||||
|
// this endpoint has to read the same decision back, or it round-trips Portuguese
|
||||||
|
// into Portuguese and calls it a translation.
|
||||||
|
//
|
||||||
|
// So: render into whichever half the explanation is NOT already in — and that
|
||||||
|
// is the whole rule, in both directions. When it first shipped this endpoint
|
||||||
|
// answered "" for a Portuguese explanation on the reasoning that an English
|
||||||
|
// rendering she hadn't asked for was noise. It was reported as the opposite: a
|
||||||
|
// writer whose pair is Portuguese and English, learning English, met a
|
||||||
|
// Portuguese card with a Portuguese bubble under it and no English anywhere on
|
||||||
|
// the card. The tap is the one place the other language was promised, and the
|
||||||
|
// half she is *practising* is exactly the half worth a tap.
|
||||||
|
//
|
||||||
|
// The bubble sits directly beneath the explanation inside the card, so between
|
||||||
|
// the two the writer always has both languages, whichever way round the document
|
||||||
|
// put them.
|
||||||
func (h *Handler) translate(w http.ResponseWriter, r *http.Request) {
|
func (h *Handler) translate(w http.ResponseWriter, r *http.Request) {
|
||||||
sugID := chi.URLParam(r, "id")
|
sugID := chi.URLParam(r, "id")
|
||||||
|
|
||||||
var explanation, pairLang string
|
var explanation, pairLang, direction, docLang string
|
||||||
err := h.DB.QueryRow(
|
err := h.DB.QueryRow(
|
||||||
`SELECT s.explanation, COALESCE(u.pair_lang, '')
|
`SELECT s.explanation, COALESCE(u.pair_lang, ''),
|
||||||
|
COALESCE(u.direction, ''), d.doc_lang
|
||||||
FROM suggestions s
|
FROM suggestions s
|
||||||
JOIN documents d ON d.id = s.doc_id
|
JOIN documents d ON d.id = s.doc_id
|
||||||
JOIN users u ON u.id = d.user_id
|
JOIN users u ON u.id = d.user_id
|
||||||
WHERE s.id = ? AND d.user_id = ?`,
|
WHERE s.id = ? AND d.user_id = ?`,
|
||||||
sugID, auth.UserID(r.Context()),
|
sugID, auth.UserID(r.Context()),
|
||||||
).Scan(&explanation, &pairLang)
|
).Scan(&explanation, &pairLang, &direction, &docLang)
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
httputil.ErrorJSON(w, http.StatusNotFound, "suggestion not found")
|
httputil.ErrorJSON(w, http.StatusNotFound, "suggestion not found")
|
||||||
return
|
return
|
||||||
@@ -49,9 +70,22 @@ func (h *Handler) translate(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
out, err := llm.RunTranslate(r.Context(), h.Client, explanation, llm.LangFor(pairLang))
|
// The explanation's own language, recovered from the same rule that chose it
|
||||||
|
// when the card was written. A card written before this phase — or on a
|
||||||
|
// document whose verdict has since flipped — is read as whatever the rule says
|
||||||
|
// today; the alternative is a language column on every suggestion row, and the
|
||||||
|
// cost of being wrong is one bubble seeded in the language it was already in.
|
||||||
|
target := targetFor(pairLang, direction, docLang)
|
||||||
|
from, to := target.Explain, target.Pair
|
||||||
|
if from.Code == to.Code {
|
||||||
|
// The explanation is already in her language, so the half this tap has to
|
||||||
|
// reach is the other one: the English she is practising.
|
||||||
|
to = llm.English
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := llm.RunTranslate(r.Context(), h.Client, explanation, from, to)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
httputil.ErrorJSON(w, http.StatusBadGateway, "translate failed: "+err.Error())
|
httputil.UpstreamError(w, "translate", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
package suggestions
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The pair model's flagship moment, end to end: she reaches for a sentence in
|
||||||
|
// her own language mid-document, and the card that comes back is labelled as a
|
||||||
|
// translation rather than as a tidy-up of her Chinese.
|
||||||
|
//
|
||||||
|
// The label is asserted through the real /check path rather than against
|
||||||
|
// isTranslation directly, because the point of the item was never the detector —
|
||||||
|
// Petal already found these spans and already rendered them into English. What
|
||||||
|
// was wrong was the type that reached the rail.
|
||||||
|
func TestChineseSpanBecomesATranslateCard(t *testing.T) {
|
||||||
|
// Note the model calls it "clarity", as the live build did. The type it
|
||||||
|
// volunteers is not consulted.
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"我想说这句话但是不知道用英语怎么说。","replacement":"I want to say this but I don't know how to say it in English.","explanation":"这是英文说法 · Here is how to say it in English","type":"clarity"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, database := newPairServer(t, client, "zh")
|
||||||
|
setDocTextDB(t, database, docID, "My weekend was good. 我想说这句话但是不知道用英语怎么说。")
|
||||||
|
|
||||||
|
var out []db.Suggestion
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if len(out) != 1 {
|
||||||
|
t.Fatalf("want 1 card, got %d: %+v", len(out), out)
|
||||||
|
}
|
||||||
|
if out[0].Type != db.SuggestionTypeTranslate {
|
||||||
|
t.Fatalf("card type = %q, want %q", out[0].Type, db.SuggestionTypeTranslate)
|
||||||
|
}
|
||||||
|
// The rendering and the reasoning are the model's, untouched — only the label
|
||||||
|
// is Petal's.
|
||||||
|
if out[0].Replacement != "I want to say this but I don't know how to say it in English." {
|
||||||
|
t.Fatalf("replacement was rewritten: %q", out[0].Replacement)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The other half of the same claim: an ordinary English correction on the same
|
||||||
|
// writer's document keeps the type the model gave it. A relabel that fired on
|
||||||
|
// everything would be no better than the label it replaced.
|
||||||
|
func TestEnglishCorrectionKeepsItsType(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"My weekend was very good","replacement":"My weekend was wonderful","explanation":"stronger wording","type":"phrasing"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, database := newPairServer(t, client, "zh")
|
||||||
|
setDocTextDB(t, database, docID, "My weekend was very good.")
|
||||||
|
|
||||||
|
var out []db.Suggestion
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
if len(out) != 1 {
|
||||||
|
t.Fatalf("want 1 card, got %d: %+v", len(out), out)
|
||||||
|
}
|
||||||
|
if out[0].Type != db.SuggestionTypePhrasing {
|
||||||
|
t.Fatalf("card type = %q, want %q", out[0].Type, db.SuggestionTypePhrasing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The voice pass reads whole paragraphs for tone and stamps its own family. A
|
||||||
|
// Chinese paragraph must not be able to smuggle a translate row into it — voice
|
||||||
|
// rows carry no replacement to accept, so a "translation" there would be a card
|
||||||
|
// offering nothing.
|
||||||
|
func TestVoicePassCannotProduceATranslateCard(t *testing.T) {
|
||||||
|
client := &stubClient{response: `{"suggestions":[
|
||||||
|
{"original":"我想说这句话但是不知道用英语怎么说。","replacement":"I want to say this in English.","explanation":"tone","type":"clarity"}
|
||||||
|
]}`}
|
||||||
|
srv, docID, database := newPairServer(t, client, "zh")
|
||||||
|
setDocTextDB(t, database, docID, "A first paragraph.\n\n我想说这句话但是不知道用英语怎么说。")
|
||||||
|
|
||||||
|
var out []db.Suggestion
|
||||||
|
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/voice", "")
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("voice: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||||
|
t.Fatalf("decode: %v", err)
|
||||||
|
}
|
||||||
|
for _, s := range out {
|
||||||
|
if s.Type == db.SuggestionTypeTranslate {
|
||||||
|
t.Fatalf("voice pass produced a translate card: %+v", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setDocTextDB is setDocText for the pair harness, which hands back the DB
|
||||||
|
// rather than the Handler.
|
||||||
|
func setDocTextDB(t *testing.T, database *db.DB, docID, text string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := database.Exec(
|
||||||
|
`UPDATE documents SET content_text = ? WHERE id = ?`, text, docID,
|
||||||
|
); err != nil {
|
||||||
|
t.Fatalf("update doc text: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+122
-17
@@ -19,7 +19,9 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
@@ -36,6 +38,12 @@ const maxTextBytes = 4000
|
|||||||
// with the words, mirroring the old utterance.rate = 0.95. Higher = slower.
|
// with the words, mirroring the old utterance.rate = 0.95. Higher = slower.
|
||||||
const lengthScale = 1.1
|
const lengthScale = 1.1
|
||||||
|
|
||||||
|
// slowLengthScale is the "say it slower" replay (SUGGESTIONS §5e): roughly 0.75×
|
||||||
|
// the normal pace, which is the speed listening drills have used for decades.
|
||||||
|
// Piper stretches durations rather than resampling, so the voice keeps its pitch
|
||||||
|
// instead of turning into a slowed tape.
|
||||||
|
const slowLengthScale = lengthScale / 0.75
|
||||||
|
|
||||||
// audioFormat describes one output encoding: the cache-file extension, the
|
// audioFormat describes one output encoding: the cache-file extension, the
|
||||||
// response Content-Type, and the ffmpeg args that turn Piper's WAV (on stdin)
|
// response Content-Type, and the ffmpeg args that turn Piper's WAV (on stdin)
|
||||||
// into this format (on stdout). A nil ffmpegArgs means "serve the WAV as-is".
|
// into this format (on stdout). A nil ffmpegArgs means "serve the WAV as-is".
|
||||||
@@ -90,6 +98,7 @@ type Handler struct {
|
|||||||
cacheDir string
|
cacheDir string
|
||||||
format audioFormat
|
format audioFormat
|
||||||
client *http.Client
|
client *http.Client
|
||||||
|
writes atomic.Uint64 // cache writes since boot; drives the prune throttle
|
||||||
}
|
}
|
||||||
|
|
||||||
// New builds a Handler from config. It returns (nil, false) when TTS_ENDPOINT is
|
// New builds a Handler from config. It returns (nil, false) when TTS_ENDPOINT is
|
||||||
@@ -105,16 +114,14 @@ func New(cfg *config.Config) (*Handler, bool) {
|
|||||||
format = formats["mp3"]
|
format = formats["mp3"]
|
||||||
}
|
}
|
||||||
|
|
||||||
// Map by base language so en-US, en-GB, etc. all resolve to the English
|
// Keyed by base language so en-US, en-GB — and pt-PT, pt-BR, bare pt —
|
||||||
// instance (the client sends BCP-47 tags like the old Web Speech path did).
|
// resolve to the one instance that has that language's model loaded (the
|
||||||
// A language is only routable when both its endpoint and voice are set;
|
// client sends BCP-47 tags, as the old Web Speech path did). Config has
|
||||||
// otherwise the client falls back to Web Speech for that language.
|
// already dropped any language configured by halves, so an unroutable
|
||||||
|
// language reaches the client as a 404 and falls back to Web Speech.
|
||||||
routes := map[string]route{}
|
routes := map[string]route{}
|
||||||
if cfg.TTSVoiceEN != "" {
|
for lang, v := range cfg.TTSVoices {
|
||||||
routes["en"] = route{strings.TrimRight(cfg.TTSEndpoint, "/"), cfg.TTSVoiceEN}
|
routes[lang] = route{endpoint: v.Endpoint, voice: v.Voice}
|
||||||
}
|
|
||||||
if cfg.TTSEndpointZH != "" && cfg.TTSVoiceZH != "" {
|
|
||||||
routes["zh"] = route{strings.TrimRight(cfg.TTSEndpointZH, "/"), cfg.TTSVoiceZH}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := os.MkdirAll(cfg.TTSCacheDir, 0o755); err != nil {
|
if err := os.MkdirAll(cfg.TTSCacheDir, 0o755); err != nil {
|
||||||
@@ -132,6 +139,19 @@ func New(cfg *config.Config) (*Handler, bool) {
|
|||||||
}, true
|
}, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Languages lists the base language tags this handler can synthesize, sorted,
|
||||||
|
// each with the voice serving it — for the startup line, so a deployment says
|
||||||
|
// which sidecars it actually reached rather than which ones it was configured
|
||||||
|
// to want.
|
||||||
|
func (h *Handler) Languages() []string {
|
||||||
|
out := make([]string, 0, len(h.routes))
|
||||||
|
for lang, rt := range h.routes {
|
||||||
|
out = append(out, lang+"="+rt.voice)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// Routes mounts the synthesis endpoint. Mount under "/tts" so the full path is
|
// Routes mounts the synthesis endpoint. Mount under "/tts" so the full path is
|
||||||
// POST /api/tts.
|
// POST /api/tts.
|
||||||
func (h *Handler) Routes() chi.Router {
|
func (h *Handler) Routes() chi.Router {
|
||||||
@@ -140,11 +160,13 @@ func (h *Handler) Routes() chi.Router {
|
|||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
// synthRequest is the body the editor posts: a passage and the BCP-47 language
|
// synthRequest is the body the editor posts: a passage, the BCP-47 language tag
|
||||||
// tag it's written in (e.g. "en-US", "zh-CN").
|
// it's written in (e.g. "en-US", "zh-CN", "pt-PT"), and whether to say it slowly
|
||||||
|
// — the replay a learner reaches for when the sentence went past too fast.
|
||||||
type synthRequest struct {
|
type synthRequest struct {
|
||||||
Text string `json:"text"`
|
Text string `json:"text"`
|
||||||
Lang string `json:"lang"`
|
Lang string `json:"lang"`
|
||||||
|
Slow bool `json:"slow"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// synth resolves a voice for the requested language, returns cached audio when
|
// synth resolves a voice for the requested language, returns cached audio when
|
||||||
@@ -180,9 +202,17 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Content-addressed: identical (voice, text) → identical clip. The format
|
scale := lengthScale
|
||||||
// extension keeps encodings from colliding in the same dir.
|
if req.Slow {
|
||||||
sum := sha256.Sum256([]byte(rt.voice + "\n" + text))
|
scale = slowLengthScale
|
||||||
|
}
|
||||||
|
|
||||||
|
// Content-addressed: identical (voice, pace, text) → identical clip. The pace
|
||||||
|
// belongs in the key — without it the slow replay of a word already heard at
|
||||||
|
// normal speed would be served from cache at normal speed, which is the one
|
||||||
|
// request where the difference is the whole point. The format extension keeps
|
||||||
|
// encodings from colliding in the same dir.
|
||||||
|
sum := sha256.Sum256([]byte(fmt.Sprintf("%s\n%.3f\n%s", rt.voice, scale, text)))
|
||||||
name := hex.EncodeToString(sum[:])[:32] + h.format.ext
|
name := hex.EncodeToString(sum[:])[:32] + h.format.ext
|
||||||
path := filepath.Join(h.cacheDir, name)
|
path := filepath.Join(h.cacheDir, name)
|
||||||
|
|
||||||
@@ -191,7 +221,7 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
audio, err := h.synthesize(r.Context(), rt, text)
|
audio, err := h.synthesize(r.Context(), rt, text, scale)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "synthesis failed", http.StatusBadGateway)
|
http.Error(w, "synthesis failed", http.StatusBadGateway)
|
||||||
fmt.Fprintf(os.Stderr, "tts: synthesize: %v\n", err)
|
fmt.Fprintf(os.Stderr, "tts: synthesize: %v\n", err)
|
||||||
@@ -205,6 +235,7 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
|||||||
if err := os.WriteFile(tmp, audio, 0o644); err == nil {
|
if err := os.WriteFile(tmp, audio, 0o644); err == nil {
|
||||||
_ = os.Rename(tmp, path)
|
_ = os.Rename(tmp, path)
|
||||||
}
|
}
|
||||||
|
h.pruneCache()
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Set("Content-Type", h.format.contentType)
|
w.Header().Set("Content-Type", h.format.contentType)
|
||||||
@@ -212,6 +243,80 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
|||||||
_, _ = w.Write(audio)
|
_, _ = w.Write(audio)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// maxCacheBytes bounds the whole clip cache at 512 MiB.
|
||||||
|
//
|
||||||
|
// Each clip is small, so nothing about ordinary reading approaches this — a
|
||||||
|
// year of tapping words is tens of megabytes. What it bounds is the shape of
|
||||||
|
// the endpoint: the cache key is the *text*, so a client asking for four
|
||||||
|
// thousand distinct characters at a time writes a new file every request, for
|
||||||
|
// as long as it cares to. That is an authenticated writer filling the same
|
||||||
|
// encrypted volume the database lives on, and a full disk is SQLite failing to
|
||||||
|
// write, not merely read-aloud getting slower.
|
||||||
|
const maxCacheBytes = 512 << 20
|
||||||
|
|
||||||
|
// pruneEvery throttles the sweep: checking the directory on every synthesis
|
||||||
|
// would stat the whole cache for each new word. Synthesis is already the slow
|
||||||
|
// path and misses are rare once a writer settles, so one sweep per this many
|
||||||
|
// cache writes keeps the cost invisible while still converging long before the
|
||||||
|
// limit means anything.
|
||||||
|
const pruneEvery = 64
|
||||||
|
|
||||||
|
// pruneCache trims the cache back under maxCacheBytes, oldest-first, and is a
|
||||||
|
// no-op the great majority of the time it is called.
|
||||||
|
//
|
||||||
|
// Oldest by modification time is a fair approximation of least-recently-useful
|
||||||
|
// here: a clip is written once and only ever read afterwards, so its age is how
|
||||||
|
// long ago someone wanted it. Evicting one costs a re-synthesis, never data —
|
||||||
|
// which is why this can be as approximate as it likes, and why every error
|
||||||
|
// along the way is simply given up on.
|
||||||
|
func (h *Handler) pruneCache() {
|
||||||
|
if n := h.writes.Add(1); n%pruneEvery != 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(h.cacheDir)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
type clip struct {
|
||||||
|
path string
|
||||||
|
size int64
|
||||||
|
mod time.Time
|
||||||
|
}
|
||||||
|
var clips []clip
|
||||||
|
var total int64
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
info, err := e.Info()
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
clips = append(clips, clip{filepath.Join(h.cacheDir, e.Name()), info.Size(), info.ModTime()})
|
||||||
|
total += info.Size()
|
||||||
|
}
|
||||||
|
if total <= maxCacheBytes {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(clips, func(i, j int) bool { return clips[i].mod.Before(clips[j].mod) })
|
||||||
|
// Drop to 80% rather than exactly to the line, so the next few hundred
|
||||||
|
// clips don't each trigger another sweep.
|
||||||
|
target := int64(maxCacheBytes / 100 * 80)
|
||||||
|
removed := 0
|
||||||
|
for _, c := range clips {
|
||||||
|
if total <= target {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if os.Remove(c.path) == nil {
|
||||||
|
total -= c.size
|
||||||
|
removed++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintf(os.Stderr, "tts: cache over %d bytes — evicted %d oldest clip(s)\n", int64(maxCacheBytes), removed)
|
||||||
|
}
|
||||||
|
|
||||||
// serve streams a cached clip with a long-lived immutable cache header (the URL
|
// serve streams a cached clip with a long-lived immutable cache header (the URL
|
||||||
// is content-addressed, so the bytes never change for a given request).
|
// is content-addressed, so the bytes never change for a given request).
|
||||||
func (h *Handler) serve(w http.ResponseWriter, r *http.Request, path string) {
|
func (h *Handler) serve(w http.ResponseWriter, r *http.Request, path string) {
|
||||||
@@ -222,11 +327,11 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, path string) {
|
|||||||
|
|
||||||
// synthesize POSTs to the route's Piper instance, then transcodes the returned
|
// synthesize POSTs to the route's Piper instance, then transcodes the returned
|
||||||
// WAV when the configured format calls for it.
|
// WAV when the configured format calls for it.
|
||||||
func (h *Handler) synthesize(ctx context.Context, rt route, text string) ([]byte, error) {
|
func (h *Handler) synthesize(ctx context.Context, rt route, text string, scale float64) ([]byte, error) {
|
||||||
body, _ := json.Marshal(map[string]any{
|
body, _ := json.Marshal(map[string]any{
|
||||||
"text": text,
|
"text": text,
|
||||||
"voice": rt.voice,
|
"voice": rt.voice,
|
||||||
"length_scale": lengthScale,
|
"length_scale": scale,
|
||||||
})
|
})
|
||||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, rt.endpoint+h.synthURI, bytes.NewReader(body))
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, rt.endpoint+h.synthURI, bytes.NewReader(body))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ func newStubPiper(t *testing.T, body []byte) (*httptest.Server, *int32, *synthEc
|
|||||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||||
last.voice, _ = req["voice"].(string)
|
last.voice, _ = req["voice"].(string)
|
||||||
last.text, _ = req["text"].(string)
|
last.text, _ = req["text"].(string)
|
||||||
|
last.scale, _ = req["length_scale"].(float64)
|
||||||
w.Header().Set("Content-Type", "audio/wav")
|
w.Header().Set("Content-Type", "audio/wav")
|
||||||
_, _ = w.Write(body)
|
_, _ = w.Write(body)
|
||||||
}))
|
}))
|
||||||
@@ -29,7 +30,10 @@ func newStubPiper(t *testing.T, body []byte) (*httptest.Server, *int32, *synthEc
|
|||||||
return srv, &calls, last
|
return srv, &calls, last
|
||||||
}
|
}
|
||||||
|
|
||||||
type synthEcho struct{ voice, text string }
|
type synthEcho struct {
|
||||||
|
voice, text string
|
||||||
|
scale float64
|
||||||
|
}
|
||||||
|
|
||||||
// newHandler builds a wav-format handler (no ffmpeg) pointed at a stub server.
|
// newHandler builds a wav-format handler (no ffmpeg) pointed at a stub server.
|
||||||
func newHandler(t *testing.T, endpoint string) *Handler {
|
func newHandler(t *testing.T, endpoint string) *Handler {
|
||||||
@@ -88,7 +92,12 @@ func TestSynthPathNormalisation(t *testing.T) {
|
|||||||
|
|
||||||
func post(t *testing.T, h *Handler, text, lang string) *httptest.ResponseRecorder {
|
func post(t *testing.T, h *Handler, text, lang string) *httptest.ResponseRecorder {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
b, _ := json.Marshal(synthRequest{Text: text, Lang: lang})
|
return postReq(t, h, synthRequest{Text: text, Lang: lang})
|
||||||
|
}
|
||||||
|
|
||||||
|
func postReq(t *testing.T, h *Handler, body synthRequest) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
b, _ := json.Marshal(body)
|
||||||
req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(b))
|
req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(b))
|
||||||
rr := httptest.NewRecorder()
|
rr := httptest.NewRecorder()
|
||||||
h.synth(rr, req)
|
h.synth(rr, req)
|
||||||
@@ -192,8 +201,87 @@ func TestTextIsCapped(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The slow replay is the whole of SUGGESTIONS §5e: same text, same voice, more
|
||||||
|
// time per phoneme.
|
||||||
|
func TestSlowRequestStretchesTheVoice(t *testing.T) {
|
||||||
|
srv, _, last := newStubPiper(t, []byte("RIFF....fake-wav"))
|
||||||
|
h := newHandler(t, srv.URL)
|
||||||
|
|
||||||
|
if rr := postReq(t, h, synthRequest{Text: "reception", Lang: "en-US"}); rr.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want 200", rr.Code)
|
||||||
|
}
|
||||||
|
if last.scale != lengthScale {
|
||||||
|
t.Fatalf("normal length_scale = %v, want %v", last.scale, lengthScale)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rr := postReq(t, h, synthRequest{Text: "reception", Lang: "en-US", Slow: true}); rr.Code != http.StatusOK {
|
||||||
|
t.Fatalf("slow status = %d, want 200", rr.Code)
|
||||||
|
}
|
||||||
|
if last.scale != slowLengthScale {
|
||||||
|
t.Fatalf("slow length_scale = %v, want %v", last.scale, slowLengthScale)
|
||||||
|
}
|
||||||
|
if slowLengthScale <= lengthScale {
|
||||||
|
t.Fatalf("slowLengthScale %v is not slower than %v", slowLengthScale, lengthScale)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The pace has to be part of the cache key. Without it, asking for the slow
|
||||||
|
// replay of a word already heard at normal speed serves the normal clip — the
|
||||||
|
// one request where hearing the difference is the entire point.
|
||||||
|
func TestSlowClipIsNotServedFromTheNormalCache(t *testing.T) {
|
||||||
|
srv, calls, last := newStubPiper(t, []byte("RIFF....fake-wav"))
|
||||||
|
h := newHandler(t, srv.URL)
|
||||||
|
|
||||||
|
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US"})
|
||||||
|
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US", Slow: true})
|
||||||
|
if *calls != 2 {
|
||||||
|
t.Fatalf("piper calls = %d, want 2 (the slow clip is a different clip)", *calls)
|
||||||
|
}
|
||||||
|
if last.scale != slowLengthScale {
|
||||||
|
t.Fatalf("second call length_scale = %v, want the slow one", last.scale)
|
||||||
|
}
|
||||||
|
|
||||||
|
// …and each pace still caches on its own.
|
||||||
|
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US", Slow: true})
|
||||||
|
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US"})
|
||||||
|
if *calls != 2 {
|
||||||
|
t.Fatalf("piper calls = %d, want 2 (both paces now cached)", *calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A Portuguese request must reach the Portuguese instance on the base tag alone:
|
||||||
|
// env var names cannot hold the hyphen in pt-PT, so config keys the map on "pt"
|
||||||
|
// and the handler has to meet it there. pt-BR resolves to the same instance
|
||||||
|
// because there is only one Portuguese voice loaded — and it is the European one.
|
||||||
|
func TestPortugueseRoutesOnTheBaseTag(t *testing.T) {
|
||||||
|
enSrv, enCalls, _ := newStubPiper(t, []byte("EN-wav"))
|
||||||
|
ptSrv, ptCalls, ptLast := newStubPiper(t, []byte("PT-wav"))
|
||||||
|
h := &Handler{
|
||||||
|
routes: map[string]route{
|
||||||
|
"en": {strings.TrimRight(enSrv.URL, "/"), "en_US-amy-medium"},
|
||||||
|
"pt": {strings.TrimRight(ptSrv.URL, "/"), "pt_PT-tugão-medium"},
|
||||||
|
},
|
||||||
|
cacheDir: t.TempDir(),
|
||||||
|
format: formats["wav"],
|
||||||
|
client: http.DefaultClient,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Distinct text per tag, so a cache hit can't stand in for a route.
|
||||||
|
for i, tag := range []string{"pt-PT", "pt", "pt-BR"} {
|
||||||
|
if rr := post(t, h, strings.Repeat("receção ", i+1), tag); rr.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s status = %d, want 200", tag, rr.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if *ptCalls != 3 || *enCalls != 0 {
|
||||||
|
t.Fatalf("calls en=%d pt=%d, want en=0 pt=3", *enCalls, *ptCalls)
|
||||||
|
}
|
||||||
|
if ptLast.voice != "pt_PT-tugão-medium" {
|
||||||
|
t.Fatalf("pt voice = %q, want the European Portuguese voice", ptLast.voice)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBaseLang(t *testing.T) {
|
func TestBaseLang(t *testing.T) {
|
||||||
cases := map[string]string{"en-US": "en", "EN_gb": "en", "zh-CN": "zh", "en": "en", "": ""}
|
cases := map[string]string{"en-US": "en", "EN_gb": "en", "zh-CN": "zh", "pt-PT": "pt", "en": "en", "": ""}
|
||||||
for in, want := range cases {
|
for in, want := range cases {
|
||||||
if got := baseLang(in); got != want {
|
if got := baseLang(in); got != want {
|
||||||
t.Errorf("baseLang(%q) = %q, want %q", in, got, want)
|
t.Errorf("baseLang(%q) = %q, want %q", in, got, want)
|
||||||
|
|||||||
@@ -26,6 +26,11 @@ type Word struct {
|
|||||||
Phonetic string `json:"phonetic"`
|
Phonetic string `json:"phonetic"`
|
||||||
Example string `json:"example"`
|
Example string `json:"example"`
|
||||||
DocID *string `json:"doc_id"`
|
DocID *string `json:"doc_id"`
|
||||||
|
// Lang is '' | 'en' | 'pair' — the language of the document the word was met
|
||||||
|
// in (see migration 0018). '' reads as English, like everywhere else this
|
||||||
|
// vocabulary appears. The client needs it to pick a read-aloud voice: "comum"
|
||||||
|
// is unguessable from its letters, so the card has to carry the answer.
|
||||||
|
Lang string `json:"lang"`
|
||||||
DueAt time.Time `json:"due_at"`
|
DueAt time.Time `json:"due_at"`
|
||||||
IntervalDays int `json:"interval_days"`
|
IntervalDays int `json:"interval_days"`
|
||||||
Ease float64 `json:"ease"`
|
Ease float64 `json:"ease"`
|
||||||
@@ -54,7 +59,7 @@ func (h *Handler) Routes() chi.Router {
|
|||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
const vocabColumns = `id, word, gloss, definition, phonetic, example, doc_id,
|
const vocabColumns = `id, word, gloss, definition, phonetic, example, doc_id, lang,
|
||||||
due_at, interval_days, ease, reps, lapses, last_reviewed, created_at`
|
due_at, interval_days, ease, reps, lapses, last_reviewed, created_at`
|
||||||
|
|
||||||
func scanWord(s interface {
|
func scanWord(s interface {
|
||||||
@@ -62,7 +67,7 @@ func scanWord(s interface {
|
|||||||
}) (Word, error) {
|
}) (Word, error) {
|
||||||
var w Word
|
var w Word
|
||||||
err := s.Scan(
|
err := s.Scan(
|
||||||
&w.ID, &w.Word, &w.Gloss, &w.Definition, &w.Phonetic, &w.Example, &w.DocID,
|
&w.ID, &w.Word, &w.Gloss, &w.Definition, &w.Phonetic, &w.Example, &w.DocID, &w.Lang,
|
||||||
&w.DueAt, &w.IntervalDays, &w.Ease, &w.Reps, &w.Lapses, &w.LastReviewed, &w.CreatedAt,
|
&w.DueAt, &w.IntervalDays, &w.Ease, &w.Reps, &w.Lapses, &w.LastReviewed, &w.CreatedAt,
|
||||||
)
|
)
|
||||||
return w, err
|
return w, err
|
||||||
@@ -165,15 +170,22 @@ func (h *Handler) capture(w http.ResponseWriter, r *http.Request) {
|
|||||||
// would hit the foreign key and leak a raw "FOREIGN KEY constraint" 500
|
// would hit the foreign key and leak a raw "FOREIGN KEY constraint" 500
|
||||||
// instead of a clean 400 (and, once auth lands, would let a word be attached
|
// instead of a clean 400 (and, once auth lands, would let a word be attached
|
||||||
// to another user's document).
|
// to another user's document).
|
||||||
|
//
|
||||||
|
// The same row-scoped lookup answers what language the card is in: a word is
|
||||||
|
// met inside a document, so the document's verdict is the word's language.
|
||||||
|
// Asking the document rather than trusting a `lang` in the request body is
|
||||||
|
// the same choice `runPass` makes — the client never gets to name a language
|
||||||
|
// the server can already read. A word with no document is '', which reads as
|
||||||
|
// English.
|
||||||
|
lang := ""
|
||||||
if req.DocID != nil {
|
if req.DocID != nil {
|
||||||
if strings.TrimSpace(*req.DocID) == "" {
|
if strings.TrimSpace(*req.DocID) == "" {
|
||||||
req.DocID = nil
|
req.DocID = nil
|
||||||
} else {
|
} else {
|
||||||
var ok int
|
|
||||||
err := h.DB.QueryRow(
|
err := h.DB.QueryRow(
|
||||||
`SELECT 1 FROM documents WHERE id = ? AND user_id = ?`,
|
`SELECT doc_lang FROM documents WHERE id = ? AND user_id = ?`,
|
||||||
*req.DocID, userID,
|
*req.DocID, userID,
|
||||||
).Scan(&ok)
|
).Scan(&lang)
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
httputil.ErrorJSON(w, http.StatusBadRequest, "unknown doc_id")
|
httputil.ErrorJSON(w, http.StatusBadRequest, "unknown doc_id")
|
||||||
return
|
return
|
||||||
@@ -189,15 +201,19 @@ func (h *Handler) capture(w http.ResponseWriter, r *http.Request) {
|
|||||||
// schedule (due_at/reps/interval/ease) alone so re-looking-up a word never
|
// schedule (due_at/reps/interval/ease) alone so re-looking-up a word never
|
||||||
// resets its progress.
|
// resets its progress.
|
||||||
_, err := h.DB.Exec(
|
_, err := h.DB.Exec(
|
||||||
`INSERT INTO vocab_words (user_id, word, gloss, definition, phonetic, example, doc_id, due_at, interval_days)
|
`INSERT INTO vocab_words (user_id, word, gloss, definition, phonetic, example, doc_id, lang, due_at, interval_days)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now', '+1 day'), 1)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now', '+1 day'), 1)
|
||||||
ON CONFLICT(user_id, word) DO UPDATE SET
|
ON CONFLICT(user_id, word) DO UPDATE SET
|
||||||
gloss = excluded.gloss,
|
gloss = excluded.gloss,
|
||||||
definition = excluded.definition,
|
definition = excluded.definition,
|
||||||
phonetic = excluded.phonetic,
|
phonetic = excluded.phonetic,
|
||||||
example = CASE WHEN excluded.example != '' THEN excluded.example ELSE vocab_words.example END,
|
example = CASE WHEN excluded.example != '' THEN excluded.example ELSE vocab_words.example END,
|
||||||
doc_id = COALESCE(excluded.doc_id, vocab_words.doc_id)`,
|
doc_id = COALESCE(excluded.doc_id, vocab_words.doc_id),
|
||||||
userID, word, req.Gloss, req.Definition, req.Phonetic, req.Example, req.DocID,
|
-- lang travels with doc_id, and for the same reason: it is the new
|
||||||
|
-- context or it is nothing. A lookup made outside any document must
|
||||||
|
-- not relabel a card that was captured inside one.
|
||||||
|
lang = CASE WHEN excluded.doc_id IS NOT NULL THEN excluded.lang ELSE vocab_words.lang END`,
|
||||||
|
userID, word, req.Gloss, req.Definition, req.Phonetic, req.Example, req.DocID, lang,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
httputil.ServerError(w, err)
|
httputil.ServerError(w, err)
|
||||||
|
|||||||
@@ -253,3 +253,89 @@ func TestDocLinkSurvivesDocDelete(t *testing.T) {
|
|||||||
t.Fatalf("doc_id should be nulled after doc delete, got %v", *all[0].DocID)
|
t.Fatalf("doc_id should be nulled after doc delete, got %v", *all[0].DocID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCaptureTakesLanguageFromItsDocument is the garden's half of Phase 28: a
|
||||||
|
// word met inside a document written in her own language is a card in that
|
||||||
|
// language, and the server reads that off the document rather than being told.
|
||||||
|
//
|
||||||
|
// The three cases are the three the client can actually produce: a lookup inside
|
||||||
|
// a flipped document, a lookup inside an English one, and a lookup with no
|
||||||
|
// document at all (the search box) — the last of which is '', which reads as
|
||||||
|
// English everywhere this value is used.
|
||||||
|
func TestCaptureTakesLanguageFromItsDocument(t *testing.T) {
|
||||||
|
srv, database := newTestServer(t)
|
||||||
|
|
||||||
|
seed := func(lang string) string {
|
||||||
|
t.Helper()
|
||||||
|
var id string
|
||||||
|
if err := database.QueryRow(
|
||||||
|
`INSERT INTO documents (user_id, content_text, doc_lang) VALUES (?, 'hi', ?) RETURNING id`,
|
||||||
|
db.LocalUserID, lang,
|
||||||
|
).Scan(&id); err != nil {
|
||||||
|
t.Fatalf("seed doc: %v", err)
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
pairDoc, enDoc := seed("pair"), seed("en")
|
||||||
|
|
||||||
|
capture := func(word, body string) Word {
|
||||||
|
t.Helper()
|
||||||
|
rec := do(t, srv, http.MethodPost, "/vocab", body)
|
||||||
|
if rec.Code != http.StatusCreated {
|
||||||
|
t.Fatalf("capture %s: code=%d body=%s", word, rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
var w Word
|
||||||
|
if err := json.Unmarshal(rec.Body.Bytes(), &w); err != nil {
|
||||||
|
t.Fatalf("decode %s: %v", word, err)
|
||||||
|
}
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := capture("comum", `{"word":"comum","doc_id":"`+pairDoc+`"}`); got.Lang != "pair" {
|
||||||
|
t.Fatalf("word from a flipped document: lang=%q, want pair", got.Lang)
|
||||||
|
}
|
||||||
|
if got := capture("reception", `{"word":"reception","doc_id":"`+enDoc+`"}`); got.Lang != "en" {
|
||||||
|
t.Fatalf("word from an English document: lang=%q, want en", got.Lang)
|
||||||
|
}
|
||||||
|
if got := capture("orphan", `{"word":"orphan"}`); got.Lang != "" {
|
||||||
|
t.Fatalf("word with no document: lang=%q, want empty", got.Lang)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A client that names a language is ignored: the document is the authority,
|
||||||
|
// the same way runPass never lets the request pick its own target.
|
||||||
|
if got := capture("comum", `{"word":"comum","lang":"en","doc_id":"`+pairDoc+`"}`); got.Lang != "pair" {
|
||||||
|
t.Fatalf("client-supplied lang should not win: lang=%q, want pair", got.Lang)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRecaptureOutsideADocumentKeepsItsLanguage pins the one asymmetry in the
|
||||||
|
// upsert. Re-looking-up a word refreshes its context, but a lookup made with no
|
||||||
|
// document carries no verdict — and relabelling a Portuguese card English
|
||||||
|
// because she checked the word again from the search box would silently move it
|
||||||
|
// to the wrong voice. lang travels with doc_id, or it doesn't travel.
|
||||||
|
func TestRecaptureOutsideADocumentKeepsItsLanguage(t *testing.T) {
|
||||||
|
srv, database := newTestServer(t)
|
||||||
|
var docID string
|
||||||
|
if err := database.QueryRow(
|
||||||
|
`INSERT INTO documents (user_id, content_text, doc_lang) VALUES (?, 'olá', 'pair') RETURNING id`,
|
||||||
|
db.LocalUserID,
|
||||||
|
).Scan(&docID); err != nil {
|
||||||
|
t.Fatalf("seed doc: %v", err)
|
||||||
|
}
|
||||||
|
if rec := do(t, srv, http.MethodPost, "/vocab",
|
||||||
|
`{"word":"saudade","gloss":"","doc_id":"`+docID+`"}`); rec.Code != http.StatusCreated {
|
||||||
|
t.Fatalf("capture: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
rec := do(t, srv, http.MethodPost, "/vocab", `{"word":"saudade","gloss":"longing"}`)
|
||||||
|
if rec.Code != http.StatusCreated {
|
||||||
|
t.Fatalf("recapture: code=%d body=%s", rec.Code, rec.Body)
|
||||||
|
}
|
||||||
|
var w Word
|
||||||
|
_ = json.Unmarshal(rec.Body.Bytes(), &w)
|
||||||
|
if w.Lang != "pair" {
|
||||||
|
t.Fatalf("recapture outside a document: lang=%q, want pair held", w.Lang)
|
||||||
|
}
|
||||||
|
if w.Gloss != "longing" {
|
||||||
|
t.Fatalf("recapture should still refresh the gloss, got %q", w.Gloss)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,105 @@
|
|||||||
|
package vocab
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Planting: the garden's second source.
|
||||||
|
//
|
||||||
|
// Capture (handlers.go) records words the writer *sought out*. Planting records
|
||||||
|
// phrasing she was gently *given* — an accepted collocation like "make a
|
||||||
|
// decision" is a learnable chunk exactly like a looked-up word, and the SM-2-lite
|
||||||
|
// scheduler doesn't care that it's three words rather than one. Together the two
|
||||||
|
// halves make the garden a record of both sides of learning.
|
||||||
|
//
|
||||||
|
// Everything here is best-effort by design: planting hangs off accepting a
|
||||||
|
// suggestion, and that accept must succeed whether or not a card comes of it.
|
||||||
|
|
||||||
|
// Execer is the slice of *sql.DB (or *sql.Tx) that planting needs.
|
||||||
|
type Execer interface {
|
||||||
|
Exec(query string, args ...any) (sql.Result, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phrase is one chunk to plant.
|
||||||
|
type Phrase struct {
|
||||||
|
Text string // the corrected phrasing, e.g. "make a decision"
|
||||||
|
Meaning string // why it's better — the suggestion's explanation
|
||||||
|
Example string // the sentence she met it in, already corrected
|
||||||
|
DocID *string // where, so "where did I see this?" stays one tap
|
||||||
|
// Lang is the document's verdict ('' | 'en' | 'pair'), because the chunk is
|
||||||
|
// lifted out of her own prose and is therefore in whatever language that
|
||||||
|
// prose is. See migration 0018.
|
||||||
|
Lang string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phrase-card caps. A collocation is a short chunk; anything longer is a
|
||||||
|
// rewritten sentence wearing a collocation's label, and a sentence makes a
|
||||||
|
// miserable flashcard. Both bounds are deliberately tight — the cost of
|
||||||
|
// skipping a real chunk is one missing card, the cost of planting a sentence is
|
||||||
|
// a garden the writer stops trusting.
|
||||||
|
const (
|
||||||
|
maxPhraseRunes = 60
|
||||||
|
maxPhraseWords = 6
|
||||||
|
minPhraseWords = 2
|
||||||
|
)
|
||||||
|
|
||||||
|
// PhraseKey normalizes a replacement into a garden key, or returns "" when the
|
||||||
|
// text isn't a plantable chunk.
|
||||||
|
//
|
||||||
|
// Lowercasing matches capture's normalization, so a phrase and a looked-up word
|
||||||
|
// share one UNIQUE(user_id, word) namespace rather than colliding sideways.
|
||||||
|
// Single words are rejected on purpose: a one-word fix is word choice, and word
|
||||||
|
// choice already reaches the garden through lookup — planting it here would give
|
||||||
|
// it a card with no gloss and no phonetic, which reviews badly.
|
||||||
|
func PhraseKey(text string) string {
|
||||||
|
// Collapse all whitespace (a replacement can carry a newline from the
|
||||||
|
// editor) so the key is stable and the word count is honest.
|
||||||
|
s := strings.Join(strings.Fields(strings.ToLower(text)), " ")
|
||||||
|
// Trim the punctuation a phrase picks up from the sentence around it, but
|
||||||
|
// leave inner marks alone: "can't afford" and "in one's own time" are chunks.
|
||||||
|
s = strings.Trim(s, `.,;:!?…"'“”‘’()[]`)
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" || len([]rune(s)) > maxPhraseRunes {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
n := len(strings.Fields(s))
|
||||||
|
if n < minPhraseWords || n > maxPhraseWords {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
// A chunk of pure digits or symbols ("12 000", "-- --") isn't vocabulary.
|
||||||
|
if !strings.ContainsFunc(s, unicode.IsLetter) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plant adds a phrase card to the garden, due tomorrow like any fresh capture.
|
||||||
|
// It reports whether a new card was created.
|
||||||
|
//
|
||||||
|
// ON CONFLICT DO NOTHING, unlike capture's refresh-the-context upsert: accepting
|
||||||
|
// the same collocation again months later is evidence the chunk is still being
|
||||||
|
// learned, and the last thing that should do is overwrite the card's first
|
||||||
|
// context or disturb a schedule it has been climbing. An existing card wins.
|
||||||
|
func Plant(ex Execer, userID string, p Phrase) (bool, error) {
|
||||||
|
key := PhraseKey(p.Text)
|
||||||
|
if key == "" {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
res, err := ex.Exec(
|
||||||
|
`INSERT INTO vocab_words (user_id, word, gloss, definition, phonetic, example, doc_id, lang, due_at, interval_days)
|
||||||
|
VALUES (?, ?, '', ?, '', ?, ?, ?, datetime('now', '+1 day'), 1)
|
||||||
|
ON CONFLICT(user_id, word) DO NOTHING`,
|
||||||
|
userID, key,
|
||||||
|
clamp(strings.TrimSpace(p.Meaning), maxDefinitionLen),
|
||||||
|
clamp(strings.TrimSpace(p.Example), maxExampleLen),
|
||||||
|
p.DocID,
|
||||||
|
p.Lang,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
n, err := res.RowsAffected()
|
||||||
|
return n > 0, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
package vocab
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPhraseKey(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
in, want string
|
||||||
|
}{
|
||||||
|
{"make a decision", "make a decision"},
|
||||||
|
{"Make A Decision", "make a decision"}, // shares one namespace with lookups
|
||||||
|
{" make a\ndecision ", "make a decision"}, // the editor's whitespace
|
||||||
|
{"“make a decision.”", "make a decision"}, // punctuation from the sentence around it
|
||||||
|
{"can’t afford it", "can’t afford it"}, // inner marks are part of the chunk
|
||||||
|
{"decision", ""}, // word choice, not a chunk — lookup's job
|
||||||
|
{"", ""}, //
|
||||||
|
{"...", ""}, //
|
||||||
|
{"12 000", ""}, // digits aren't vocabulary
|
||||||
|
{"a b c d e f g", ""}, // a clause wearing a chunk's label
|
||||||
|
{"in one’s own good time again", "in one’s own good time again"}, // six words is still a chunk
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
if got := PhraseKey(tc.in); got != tc.want {
|
||||||
|
t.Errorf("PhraseKey(%q) = %q, want %q", tc.in, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The length cap counts runes, not bytes — otherwise a Portuguese chunk well
|
||||||
|
// inside the limit would be dropped for being accented.
|
||||||
|
accented := "ãããããã ãããããã ãããããã ãããããã ãããããã" // 34 runes, 64 bytes
|
||||||
|
if got := PhraseKey(accented); got != accented {
|
||||||
|
t.Errorf("PhraseKey(%d runes / %d bytes) = %q, want it kept", len([]rune(accented)), len(accented), got)
|
||||||
|
}
|
||||||
|
long := "ãããããããããããã ãããããããããããã ãããããããããããã ãããããããããããã ãããããããããããã ãããããããããããã"
|
||||||
|
if got := PhraseKey(long); got != "" {
|
||||||
|
t.Errorf("PhraseKey(%d runes) = %q, want \"\"", len([]rune(long)), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPlantCreatesOnceAndReportsIt(t *testing.T) {
|
||||||
|
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open db: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { database.Close() })
|
||||||
|
|
||||||
|
p := Phrase{Text: "make a decision", Meaning: "why", Example: "I had to make a decision."}
|
||||||
|
created, err := Plant(database, db.LocalUserID, p)
|
||||||
|
if err != nil || !created {
|
||||||
|
t.Fatalf("first plant: created=%v err=%v", created, err)
|
||||||
|
}
|
||||||
|
created, err = Plant(database, db.LocalUserID, p)
|
||||||
|
if err != nil || created {
|
||||||
|
t.Fatalf("second plant: created=%v err=%v, want false", created, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A card that isn't plantable is a silent no-op, not an error: planting hangs
|
||||||
|
// off accepting an edit, and that accept must never fail for a flashcard.
|
||||||
|
created, err = Plant(database, db.LocalUserID, Phrase{Text: "decision"})
|
||||||
|
if err != nil || created {
|
||||||
|
t.Fatalf("unplantable: created=%v err=%v", created, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var n int
|
||||||
|
if err := database.QueryRow(`SELECT count(*) FROM vocab_words WHERE user_id = ?`, db.LocalUserID).Scan(&n); err != nil {
|
||||||
|
t.Fatalf("count: %v", err)
|
||||||
|
}
|
||||||
|
if n != 1 {
|
||||||
|
t.Fatalf("garden has %d cards, want 1", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,322 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Build the two Chinese assets the learner direction of the zh pair needs.
|
||||||
|
|
||||||
|
Why two, and why they are split the way they are
|
||||||
|
------------------------------------------------
|
||||||
|
Every other pair Petal ships needs one asset: a word list the browser loads so
|
||||||
|
it can underline. Chinese needs two, because the browser and the server want
|
||||||
|
different halves of the same dictionary and for different reasons.
|
||||||
|
|
||||||
|
* **The browser needs a word list, and it needs it offline.** Chinese is
|
||||||
|
written without spaces, so there is no such thing as "the word under the
|
||||||
|
cursor" until something segments the sentence. Every ESL surface Petal
|
||||||
|
already has — the hover gloss, the right-click lookup, Ctrl/Cmd+D, the
|
||||||
|
vocabulary garden capture — is built on `wordAt`, and `wordAt` is a regex
|
||||||
|
over Latin letters. Segmentation is what replaces that regex, it runs on
|
||||||
|
every hover, and a round-trip per hover is not a hover. So the word list
|
||||||
|
ships to the browser: `web/public/dictionaries/zh/words.txt`.
|
||||||
|
|
||||||
|
* **The server holds the whole dictionary.** Pinyin and English senses are
|
||||||
|
only ever wanted one word at a time, in answer to a hover or a click, which
|
||||||
|
is exactly what `/api/gloss/{word}` already does for the other direction. So
|
||||||
|
the readings stay in the binary — `internal/lexicon/data/hanzi.json.gz` —
|
||||||
|
where their size costs a browser nothing.
|
||||||
|
|
||||||
|
That split is what makes the coverage decisions below come out *opposite* to
|
||||||
|
each other, and both are deliberate.
|
||||||
|
|
||||||
|
Two sources, because neither one has both halves
|
||||||
|
------------------------------------------------
|
||||||
|
* **CC-CEDICT** (CC BY-SA 4.0, https://www.mdbg.net/) has the headwords,
|
||||||
|
pinyin and English senses, and no frequency information at all.
|
||||||
|
* **jieba's `dict.txt`** (MIT, https://github.com/fxsjy/jieba) has ~349k
|
||||||
|
headwords with corpus frequencies, and no definitions.
|
||||||
|
|
||||||
|
Segmentation needs the frequencies: the standard algorithm is a shortest-path
|
||||||
|
walk over log-probabilities, not longest-match, and without frequencies the
|
||||||
|
classic ambiguities go the wrong way. The client list therefore carries
|
||||||
|
`word freq` per line; the gloss map carries readings.
|
||||||
|
|
||||||
|
The size decision is the client list, and it is a size decision only
|
||||||
|
--------------------------------------------------------------------
|
||||||
|
Measured on ordinary learner prose, the segmentation produced by the full jieba
|
||||||
|
dictionary (381,886 hanzi headwords once CC-CEDICT is unioned in) and by a
|
||||||
|
frequency-gated one is **identical**, including on the textbook ambiguities
|
||||||
|
(研究生命的起源, 乒乓球拍卖完了, 南京市长江大桥). What the long tail contains is
|
||||||
|
rare proper nouns, and the max-probability walk almost never chooses one: a
|
||||||
|
freq-3 name loses to two common words every time. The cases where a missing word
|
||||||
|
does change the answer degrade *gracefully* — the sentence splits into smaller
|
||||||
|
real words, which is a slightly clumsier gloss, not a wrong underline.
|
||||||
|
|
||||||
|
So the gate is set where the size is, at **freq >= 5**: 188,522 words, ~0.97 MB
|
||||||
|
gzipped over the wire, in line with fr (1.19 MB) and es (1.74 MB) rather than in
|
||||||
|
excess of them. Every CC-CEDICT headword is unioned back in regardless of
|
||||||
|
frequency, so the segmenter can always see a word the server can explain.
|
||||||
|
|
||||||
|
The gloss map is gated by nothing, for the opposite reason
|
||||||
|
-----------------------------------------------------------
|
||||||
|
The es phase settled that a *spelling* dictionary should hold the union of every
|
||||||
|
variety, because its only power is to underline and it must not underline
|
||||||
|
correct writing. This asset's only power is to **explain**, and the word a
|
||||||
|
learner stops on is precisely the one they do not know — which is to say, the
|
||||||
|
rare one. Trimming this by frequency would remove exactly the entries it exists
|
||||||
|
for. All 113,637 glossable headwords ship, ~3.1 MB gzipped, which is less than
|
||||||
|
half of what `synonyms.json.gz` has embedded since Phase 9.
|
||||||
|
|
||||||
|
Simplified only, and said out loud
|
||||||
|
-----------------------------------
|
||||||
|
The zh langpack is written in simplified characters and jieba's frequencies are
|
||||||
|
counted over simplified text, so the traditional headword in each CC-CEDICT line
|
||||||
|
is dropped and simplified is what both assets are keyed by. Glossing traditional
|
||||||
|
would be nearly free *here* and useless in the app: nothing would segment it, so
|
||||||
|
nothing would ever ask. Traditional support is a real feature and it starts with
|
||||||
|
a traditional word list, not with this file.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
curl -sL https://www.mdbg.net/chinese/export/cedict/cedict_1_0_ts_utf-8_mdbg.txt.gz | gunzip > cedict.txt
|
||||||
|
curl -sL https://raw.githubusercontent.com/fxsjy/jieba/master/jieba/dict.txt -o jieba.txt
|
||||||
|
python3 scripts/build_cedict.py cedict.txt jieba.txt \
|
||||||
|
web/public/dictionaries/zh/words.txt.gz \
|
||||||
|
internal/lexicon/data/hanzi.json.gz
|
||||||
|
"""
|
||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# Frequency gate for the *client* list only (see the module docstring). Words
|
||||||
|
# below it survive if CC-CEDICT knows them, so "segmentable" is always a superset
|
||||||
|
# of "glossable" and a hover can never land on a word the server cannot explain.
|
||||||
|
MIN_FREQ = 5
|
||||||
|
|
||||||
|
# A CC-CEDICT headword we keep must be nothing but han characters. This drops the
|
||||||
|
# entries that are really English or numerals with a Chinese gloss attached
|
||||||
|
# ("AA制", "PM2.5", "11区"): the segmenter walks runs of hanzi, so a mixed
|
||||||
|
# headword can never be matched anyway, and a Latin one would collide with the
|
||||||
|
# English tokenizer that is still running on the same paragraph.
|
||||||
|
HANZI_ONLY = re.compile(r'^[一-鿿]+$')
|
||||||
|
|
||||||
|
CEDICT_LINE = re.compile(r'^(\S+) (\S+) \[(.*?)\] /(.*)/$')
|
||||||
|
|
||||||
|
# At most this many readings per word, and this many senses per reading. Two
|
||||||
|
# readings is not an arbitrary cap: it is what the particles need. 得 is dé "to
|
||||||
|
# obtain" *and* de, the complement marker — and a learner who hovers 得 in
|
||||||
|
# 说得很好 and is told only "to obtain" has been actively misinformed. Beyond two
|
||||||
|
# the tail is dialect and surnames, which crowd out the sense actually wanted.
|
||||||
|
MAX_READINGS = 2
|
||||||
|
MAX_SENSES = 3
|
||||||
|
MAX_SENSE_CHARS = 110
|
||||||
|
|
||||||
|
# Senses that describe the *dictionary* rather than the word. A learner hovering
|
||||||
|
# a word wants to know what it means, not that it is an orthographic variant of
|
||||||
|
# another headword they also do not know.
|
||||||
|
SKIP_SENSE_PREFIXES = ('variant of', 'old variant', 'see ', 'used in', 'abbr. for')
|
||||||
|
|
||||||
|
# ── pinyin: numbered syllables to tone marks ────────────────────────────────
|
||||||
|
# CC-CEDICT stores "gong1 yuan2". A learner reading their own writing back wants
|
||||||
|
# gōngyuán: the tone mark is the part that is hard to remember and the part that
|
||||||
|
# changes the word. The placement rule is the standard one — a/o/e take the mark
|
||||||
|
# if present, otherwise the last vowel of the final — and it is small enough to
|
||||||
|
# do here rather than to take a dependency for.
|
||||||
|
TONE_VOWELS = {
|
||||||
|
'a': 'āáǎà',
|
||||||
|
'e': 'ēéěè',
|
||||||
|
'i': 'īíǐì',
|
||||||
|
'o': 'ōóǒò',
|
||||||
|
'u': 'ūúǔù',
|
||||||
|
'ü': 'ǖǘǚǜ',
|
||||||
|
}
|
||||||
|
|
||||||
|
SYLLABLE = re.compile(r'^([a-zA-Zü:]+)([1-5])$')
|
||||||
|
|
||||||
|
|
||||||
|
def tone_mark(syllable: str) -> str:
|
||||||
|
"""One numbered pinyin syllable to its tone-marked form."""
|
||||||
|
m = SYLLABLE.match(syllable)
|
||||||
|
if not m:
|
||||||
|
# Punctuation, a bare letter (CC-CEDICT writes "X" for unknown), or an
|
||||||
|
# already-marked syllable: pass it through rather than mangling it.
|
||||||
|
return syllable
|
||||||
|
body, tone = m.group(1), int(m.group(2))
|
||||||
|
# CC-CEDICT writes ü as "u:" and, in a few entries, as "v".
|
||||||
|
body = body.replace('u:', 'ü').replace('U:', 'Ü').replace('v', 'ü').replace('V', 'Ü')
|
||||||
|
if tone == 5: # neutral tone carries no mark
|
||||||
|
return body
|
||||||
|
low = body.lower()
|
||||||
|
idx = -1
|
||||||
|
for vowel in ('a', 'o', 'e'):
|
||||||
|
idx = low.find(vowel)
|
||||||
|
if idx >= 0:
|
||||||
|
break
|
||||||
|
if idx < 0:
|
||||||
|
# No a/o/e: the mark goes on the last of i/u/ü (liú, guǐ, nǚ).
|
||||||
|
idx = max(low.rfind('i'), low.rfind('u'), low.rfind('ü'))
|
||||||
|
if idx < 0:
|
||||||
|
return body
|
||||||
|
marked = TONE_VOWELS[low[idx]][tone - 1]
|
||||||
|
if body[idx].isupper():
|
||||||
|
marked = marked.upper()
|
||||||
|
return body[:idx] + marked + body[idx + 1:]
|
||||||
|
|
||||||
|
|
||||||
|
def pinyin(numbered: str) -> str:
|
||||||
|
"""A whole CC-CEDICT pinyin field to tone marks, syllables joined up.
|
||||||
|
|
||||||
|
Joined rather than spaced because that is how a word is written when it is
|
||||||
|
being read as a word (gōngyuán, not gōng yuán); the spaces in the source are
|
||||||
|
a storage convention, not orthography.
|
||||||
|
"""
|
||||||
|
return ''.join(tone_mark(s) for s in numbered.split())
|
||||||
|
|
||||||
|
|
||||||
|
def clean_senses(raw: list[str]) -> list[str]:
|
||||||
|
"""Strip the apparatus CC-CEDICT carries for lexicographers, not learners."""
|
||||||
|
out = []
|
||||||
|
for sense in raw:
|
||||||
|
# "CL:座[zuo4]" is the measure-word field, useful and not a definition.
|
||||||
|
sense = re.sub(r'\s*CL:.*$', '', sense).strip()
|
||||||
|
# Bracketed pinyin cross-references ("abbr. for 的士[di1 shi4]").
|
||||||
|
sense = re.sub(r'\[[a-zA-Z0-9: ]+\]', '', sense).strip()
|
||||||
|
# Both edits cut inside parentheses — "cat (CL:只)" loses its closing
|
||||||
|
# bracket and leaves "cat (" on the card. Drop a dangling opener rather
|
||||||
|
# than trying to rebalance: what it introduced is gone.
|
||||||
|
if sense.count('(') > sense.count(')'):
|
||||||
|
sense = re.sub(r'\s*\([^()]*$', '', sense).strip()
|
||||||
|
if not sense or sense.startswith(SKIP_SENSE_PREFIXES):
|
||||||
|
continue
|
||||||
|
out.append(sense)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def read_cedict(path: str) -> dict[str, list[tuple[str, list[str]]]]:
|
||||||
|
entries: dict[str, list[tuple[str, list[str]]]] = {}
|
||||||
|
for line in open(path, encoding='utf-8'):
|
||||||
|
if line.startswith('#'):
|
||||||
|
continue
|
||||||
|
m = CEDICT_LINE.match(line.strip())
|
||||||
|
if not m:
|
||||||
|
continue
|
||||||
|
_traditional, simplified, py, defs = m.groups()
|
||||||
|
if not HANZI_ONLY.match(simplified):
|
||||||
|
continue
|
||||||
|
entries.setdefault(simplified, []).append((py, defs.split('/')))
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def read_jieba(path: str) -> dict[str, int]:
|
||||||
|
freqs: dict[str, int] = {}
|
||||||
|
for line in open(path, encoding='utf-8'):
|
||||||
|
parts = line.split()
|
||||||
|
if len(parts) >= 2 and HANZI_ONLY.match(parts[0]):
|
||||||
|
freqs[parts[0]] = int(parts[1])
|
||||||
|
return freqs
|
||||||
|
|
||||||
|
|
||||||
|
# ── the assertions ──────────────────────────────────────────────────────────
|
||||||
|
# The es phase's lesson, in the place it applies here: a check that every
|
||||||
|
# plausible input would pass is not a check. The Spanish MUST_ACCEPT list
|
||||||
|
# asserted vocabulary that all twenty-four builds carried, so it could not tell
|
||||||
|
# them apart. These assert the things that actually go wrong in *this* build —
|
||||||
|
# a mis-parsed pinyin field, a missing particle reading, a word list gated so
|
||||||
|
# hard the segmenter can no longer see a word the server can explain.
|
||||||
|
|
||||||
|
# Tone marking, including the three cases the placement rule exists for.
|
||||||
|
MUST_MARK = {
|
||||||
|
'gong1 yuan2': 'gōngyuán', # a/o/e rule, first syllable
|
||||||
|
'pao3 bu4': 'pǎobù',
|
||||||
|
'liu2': 'liú', # no a/o/e: mark the *last* of i/u
|
||||||
|
'gui3': 'guǐ',
|
||||||
|
'nu:3': 'nǚ', # u: is ü
|
||||||
|
'lu:e4': 'lüè', # ü and an e in the same syllable: e wins
|
||||||
|
'de5': 'de', # neutral tone takes no mark at all
|
||||||
|
'Zhong1 wen2': 'Zhōngwén', # capitalised headword keeps its capital
|
||||||
|
}
|
||||||
|
|
||||||
|
# The particles the 错别字 rules are about must each carry the *grammatical*
|
||||||
|
# reading, not only the lexical one. 的/地/得 are the single most confused triple
|
||||||
|
# in written Chinese and all three are neutral-tone "de" in the use that matters;
|
||||||
|
# an entry that only knows 得 as dé is worse than no entry.
|
||||||
|
MUST_READ_DE = ('的', '地', '得')
|
||||||
|
|
||||||
|
# Words the segmenter must be able to see. 图书馆 and 乒乓球 are ordinary
|
||||||
|
# vocabulary; 我 and 的 are the two commonest words in the language and a gate
|
||||||
|
# that dropped either would be visibly broken; 的士 is a CC-CEDICT headword rare
|
||||||
|
# enough to fall below the frequency gate, and is here to prove the union.
|
||||||
|
MUST_SEGMENT = ('我', '的', '图书馆', '乒乓球', '公园', '的士')
|
||||||
|
|
||||||
|
|
||||||
|
def check(words: dict[str, int], gloss: dict[str, list[list[str]]]) -> None:
|
||||||
|
for numbered, want in MUST_MARK.items():
|
||||||
|
got = pinyin(numbered)
|
||||||
|
assert got == want, f'pinyin({numbered!r}) = {got!r}, want {want!r}'
|
||||||
|
|
||||||
|
for particle in MUST_READ_DE:
|
||||||
|
readings = gloss.get(particle)
|
||||||
|
assert readings, f'{particle} has no gloss entry at all'
|
||||||
|
assert any(r[0] == 'de' for r in readings), \
|
||||||
|
f'{particle} never reads as neutral "de": {readings}'
|
||||||
|
|
||||||
|
for word in MUST_SEGMENT:
|
||||||
|
assert word in words, f'{word} missing from the segmentation list'
|
||||||
|
|
||||||
|
# The invariant the two gates exist to keep: everything the server can
|
||||||
|
# explain, the browser can find.
|
||||||
|
missing = [w for w in gloss if w not in words]
|
||||||
|
assert not missing, f'{len(missing)} glossable words are unsegmentable, e.g. {missing[:5]}'
|
||||||
|
|
||||||
|
# Nothing Latin leaked into either asset (see HANZI_ONLY).
|
||||||
|
for name, keys in (('words', words), ('gloss', gloss)):
|
||||||
|
bad = [k for k in keys if not HANZI_ONLY.match(k)]
|
||||||
|
assert not bad, f'non-hanzi headwords in {name}: {bad[:5]}'
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
if len(sys.argv) != 5:
|
||||||
|
sys.exit(__doc__.strip().rsplit('Usage:', 1)[-1].strip())
|
||||||
|
cedict_path, jieba_path, words_out, gloss_out = sys.argv[1:]
|
||||||
|
|
||||||
|
entries = read_cedict(cedict_path)
|
||||||
|
freqs = read_jieba(jieba_path)
|
||||||
|
|
||||||
|
# The client list: frequency-gated, then unioned with every glossable word.
|
||||||
|
# A CC-CEDICT word jieba has never seen gets frequency 1 — real, and rare
|
||||||
|
# enough that the max-probability walk will only choose it when nothing else
|
||||||
|
# fits, which is exactly the standing it should have.
|
||||||
|
words = {w: f for w, f in freqs.items() if f >= MIN_FREQ}
|
||||||
|
for w in entries:
|
||||||
|
words.setdefault(w, 1)
|
||||||
|
|
||||||
|
gloss: dict[str, list[list[str]]] = {}
|
||||||
|
for word, rows in entries.items():
|
||||||
|
readings: list[list[str]] = []
|
||||||
|
for numbered, defs in rows:
|
||||||
|
senses = clean_senses(defs)
|
||||||
|
if not senses:
|
||||||
|
continue
|
||||||
|
readings.append([pinyin(numbered), '; '.join(senses[:MAX_SENSES])[:MAX_SENSE_CHARS]])
|
||||||
|
if len(readings) == MAX_READINGS:
|
||||||
|
break
|
||||||
|
if readings:
|
||||||
|
gloss[word] = readings
|
||||||
|
|
||||||
|
check(words, gloss)
|
||||||
|
|
||||||
|
# Gzipped on disk, like the pt-PT/fr/es word lists: the browser inflates it
|
||||||
|
# with DecompressionStream (see useSpellChecker.fetchText), which costs no
|
||||||
|
# bundle bytes, and 0.97 MB over the wire rather than 2.23 MB is the whole
|
||||||
|
# difference between this and the biggest asset Petal ships.
|
||||||
|
body = ('\n'.join(f'{w} {words[w]}' for w in sorted(words)) + '\n').encode('utf-8')
|
||||||
|
with gzip.open(words_out, 'wb', compresslevel=9) as fh:
|
||||||
|
fh.write(body)
|
||||||
|
|
||||||
|
payload = json.dumps(gloss, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
|
||||||
|
with gzip.open(gloss_out, 'wb', compresslevel=9) as fh:
|
||||||
|
fh.write(payload)
|
||||||
|
|
||||||
|
print(f'{words_out}: {len(words)} words, {len(body) / 1e6:.2f} MB raw, '
|
||||||
|
f'{len(gzip.compress(body, 9)) / 1e6:.2f} MB gzipped')
|
||||||
|
print(f'{gloss_out}: {len(gloss)} entries, {len(gzip.compress(payload, 9)) / 1e6:.2f} MB gzipped')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,543 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Build one of Petal's browser spelling dictionaries from a Hunspell one.
|
||||||
|
|
||||||
|
Why this script exists at all
|
||||||
|
----------------------------
|
||||||
|
English is vendored the obvious way: `dictionary-en`'s `en.aff` + `en.dic` go
|
||||||
|
into web/public/dictionaries/en and nspell reads them in the browser. The plan
|
||||||
|
for Phase 21 said "Hunspell pt-PT vendored like en-US", and that turns out not to
|
||||||
|
work, for a measured reason.
|
||||||
|
|
||||||
|
nspell expands affixes **eagerly at load time** — it materialises every surface
|
||||||
|
form into a hash the moment you construct it. English gets away with this: ~50k
|
||||||
|
stems and a small rule set. European Portuguese does not. `pt_PT.aff` carries
|
||||||
|
1,340 affix rules (the full verb paradigm: six persons x a dozen tenses, plus
|
||||||
|
diminutives, plus productive prefixes) over 44,257 stems. Measured on this
|
||||||
|
machine, nspell needed ~340 MB of heap for the first 12,000 entries alone and had
|
||||||
|
not returned after three minutes on the whole file; extrapolated, it wants well
|
||||||
|
over a gigabyte. That is not something to hand a browser, still less a tablet.
|
||||||
|
French is larger again: 5,600 affix rules over 84,140 stems.
|
||||||
|
|
||||||
|
So the expansion happens **here**, once, at build time, and the browser gets a
|
||||||
|
flat word list it can load with no affix machinery at all. The runtime code path
|
||||||
|
is then *identical* to English — same nspell, same interface — which is the real
|
||||||
|
prize. The aff shipped alongside keeps only the suggestion-shaping directives
|
||||||
|
(TRY/KEY/REP/MAP), so corrections still know that "cao" wants "ção" and that a
|
||||||
|
missing acute accent is a near miss.
|
||||||
|
|
||||||
|
What Phase 24 had to add
|
||||||
|
------------------------
|
||||||
|
The build plan recorded that the pt-PT version of this script "generalizes" to
|
||||||
|
French. It did not. It handled single-character flags and plain PFX/SFX and
|
||||||
|
stopped on anything else — the right call, because `fr.aff` uses four of the
|
||||||
|
things it stopped on, and getting any of them wrong changes which words are
|
||||||
|
accepted:
|
||||||
|
|
||||||
|
* **`FLAG long`** — French flags are *two characters* (`S.`, `L'`, `Um`). The
|
||||||
|
pt-PT reader took `set(flagstr)`, one flag per character, which on a French
|
||||||
|
entry yields a bag of unrelated single letters: every entry would have been
|
||||||
|
expanded through the wrong paradigm. This is the one that fails silently.
|
||||||
|
* **Continuation flags** — pt-PT's affixes append plain text, so that script
|
||||||
|
dropped anything after a `/` and asserted the drop was safe. French really
|
||||||
|
does affix an affixed form: `PFX Um 0 0/S.` says the prefixed form then takes
|
||||||
|
the plural suffix, and the elision prefixes arrive the same way from the
|
||||||
|
other side (`SFX ... ait/n'q'l'm't's'`).
|
||||||
|
* **`NEEDAFFIX`** — French marks thousands of stems "not a word on its own"
|
||||||
|
(`Allemagne/S.()`), the bare form arriving instead through a zero-append
|
||||||
|
rule. Ignoring the flag accepts stems the real dictionary rejects.
|
||||||
|
* **`FULLSTRIP`** — a rule may strip the whole stem.
|
||||||
|
|
||||||
|
`CIRCUMFIX` and `FORBIDDENWORD` are *declared* in `fr.aff` and used by nothing,
|
||||||
|
which this script asserts rather than assumes: an upstream release that started
|
||||||
|
using either would otherwise change what is accepted without changing this file.
|
||||||
|
`KEEPCASE` and `NOSUGGEST` are honoured by being ignored on purpose — they shape
|
||||||
|
casing and suggestions, not membership, and a NOSUGGEST word is still a word.
|
||||||
|
|
||||||
|
Elision is handled at lookup, not here — and that is the size decision
|
||||||
|
----------------------------------------------------------------------
|
||||||
|
Most of French's affix machinery by volume is elision: `l'`, `d'`, `qu'`, `j'`,
|
||||||
|
`n'`, `s'`, `jusqu'`, `puisqu'`. Hunspell treats `l'arbre` as one word, so a
|
||||||
|
faithful expansion carries much of the language thirty-four times over — and
|
||||||
|
Petal's tokenizer keeps internal apostrophes, so `l'arbre` really does arrive at
|
||||||
|
the dictionary as one token and really would be underlined if it were absent.
|
||||||
|
|
||||||
|
Both halves were built and measured. Keeping the elided forms: **3,159,832 forms,
|
||||||
|
8.25 MB gzipped**, ~45 MB of text for nspell to hash on a tablet. Dropping them:
|
||||||
|
**473,326 forms, 1.19 MB gzipped**. The elided seven-eighths are not new words —
|
||||||
|
they are thirteen little words glued to words already in the list — so the third
|
||||||
|
option is the one taken: rules whose append carries an apostrophe are skipped
|
||||||
|
here (the count is printed), and `withElision` in `useSpellChecker.ts` splits a
|
||||||
|
token at a *known clitic* and checks the remainder. `l'arbre` costs one extra
|
||||||
|
lookup instead of seven megabytes, and `zzz'arbre` is still flagged because
|
||||||
|
`zzz` is not one of the thirteen.
|
||||||
|
|
||||||
|
Stems that carry an apostrophe of their own — `aujourd'hui`, `quelqu'un`,
|
||||||
|
`presqu'île`, `prud'homme` — are dictionary entries rather than affixed forms,
|
||||||
|
so they are kept verbatim and matched directly. `entr'aide` and `grand'mère` are
|
||||||
|
absent for the same reason they are absent from Dicollecte: modern French spells
|
||||||
|
them `entraide` and `grand-mère`.
|
||||||
|
|
||||||
|
Choosing the source
|
||||||
|
-------------------
|
||||||
|
Both languages have a trap here, and they are different traps.
|
||||||
|
|
||||||
|
**pt-PT: the wrong country.** npm's `dictionary-pt` is not European Portuguese.
|
||||||
|
Both it and `dictionary-pt-br` package VERO ("Verificador Ortográfico Livre",
|
||||||
|
Brasil), so vendoring the obvious npm name would have shipped Brazilian spellings
|
||||||
|
under a pt-PT label — the pt-BR drift SUGGESTIONS.md §3 warns about, arriving
|
||||||
|
through the packaging rather than through the model. The authentic dictionary is
|
||||||
|
the Projecto Natura one (Universidade do Minho) that LibreOffice ships and Debian
|
||||||
|
packages as `hunspell-pt-pt`; its aff declares `LANG pt_PT`.
|
||||||
|
|
||||||
|
**es: the wrong country again, hidden one layer further down.** Spanish looked
|
||||||
|
like it would repeat the pt trap — `hunspell-es` installs twenty country codes,
|
||||||
|
`es_AR` through `es_VE` — and then looked like it did not, because every one of
|
||||||
|
them is a symlink to a single `es_ES.aff`/`es_ES.dic`. Both readings were wrong.
|
||||||
|
Debian collapses the twenty because it ships **one** of upstream's builds, and
|
||||||
|
the one it ships is the **peninsular** `es_ES`. RLA (Santiago Bosio's project,
|
||||||
|
`sbosio/rla-es`) publishes twenty-four dictionaries per release: one per country,
|
||||||
|
plus a **generic `es`** that is the union of all of them. Debian packages neither
|
||||||
|
the generic one nor a choice — it packages Spain, under a name that reads like
|
||||||
|
"Spanish".
|
||||||
|
|
||||||
|
Measured against the v2.9 release: Debian's file is 659,085 expanded forms and
|
||||||
|
upstream `es_ES` is 659,018; the generic `es` is **717,640**. The 58,622-form
|
||||||
|
difference is almost entirely **voseo** — `vení`, `tenés`, `querés`, `sabés`,
|
||||||
|
`andá` — the present tense of most of Latin America, which Debian's package
|
||||||
|
rejects as misspellings. Petal ships the **generic** build.
|
||||||
|
|
||||||
|
**Vocabulary cannot detect this and morphology can.** The first version of the es
|
||||||
|
profile asserted the pan-Hispanic lexicon — *computadora* and *ordenador*, *papa*
|
||||||
|
and *patata* — and passed happily on the peninsular file, because **every** RLA
|
||||||
|
variant carries the full pan-Hispanic vocabulary; only the verb paradigms are
|
||||||
|
localised. The `REP` table is no help either: its `ll`↔`y` and `ás`↔`az` entries
|
||||||
|
look like evidence of yeísmo and seseo, but they are shared by all twenty-four
|
||||||
|
builds. What separates them is exactly two things, and the profile now demands
|
||||||
|
both at once: **voseo** (absent from `es_ES`) and **vosotros** (largely absent
|
||||||
|
from `es_MX`). Only the generic build has both, so only the generic build passes.
|
||||||
|
|
||||||
|
This is the same decision fr made between `-classical` and `-revised`, arriving
|
||||||
|
by a different road. The only thing this dictionary can do is underline
|
||||||
|
something, and *tienes* and *tenés* are both correct Spanish taught in different
|
||||||
|
countries — so Petal takes the build that accepts every variety rather than one
|
||||||
|
that makes a writer wrong for where she is from. Nothing is generated to get
|
||||||
|
there: the forms come from a real upstream package, which is what lets the
|
||||||
|
MUST_ACCEPT list prove which package it was.
|
||||||
|
|
||||||
|
Licensing note: RLA is tri-licensed GPL-3+ / LGPL-3+ / MPL-1.1+; Petal
|
||||||
|
redistributes under the MPL. The upstream README and LICENSE are vendored beside
|
||||||
|
the output.
|
||||||
|
|
||||||
|
**fr: the wrong side of an argument the French have not settled.** The regional
|
||||||
|
question turns out to be a non-question — Debian's `fr_FR`, `fr_CA`, `fr_BE`,
|
||||||
|
`fr_CH`, `fr_LU` and `fr_MC` are all symlinks to one `fr.dic`, so unlike pt there
|
||||||
|
is no country here to get wrong. What there is instead is the 1990 spelling
|
||||||
|
reform, packaged three ways: `hunspell-fr-classical` (traditional), `-revised`
|
||||||
|
(reform only) and `-comprehensive` (both). Petal ships **comprehensive**, because
|
||||||
|
Petal never corrects her French — the only thing this dictionary can do is
|
||||||
|
underline something. *coût* and *cout* are both correct French, taught in
|
||||||
|
different decades to different people, and a writing companion has no business
|
||||||
|
underlining one of them to take a side. The `fr` MUST_ACCEPT list is written to
|
||||||
|
*prove* which package was used: classical rejects `cout`, revised rejects `coût`,
|
||||||
|
and only comprehensive accepts both.
|
||||||
|
|
||||||
|
Licensing: pt-PT is GPL-2 or LGPL-2.1 or MPL-1.1, (c) José João de Almeida, Rui
|
||||||
|
Vilela, Alberto Simões. fr is MPL-2.0, (c) 2007-2018 the Dicollecte contributors
|
||||||
|
(grammalecte.net). The upstream copyright file is vendored beside each output.
|
||||||
|
|
||||||
|
Usage
|
||||||
|
-----
|
||||||
|
apt-get download hunspell-fr-comprehensive # or hunspell-pt-pt
|
||||||
|
dpkg-deb -x hunspell-fr-comprehensive_*.deb src
|
||||||
|
python3 scripts/build_hunspell_dictionary.py fr \\
|
||||||
|
src/usr/share/hunspell/fr.aff \\
|
||||||
|
src/usr/share/hunspell/fr.dic \\
|
||||||
|
web/public/dictionaries/fr
|
||||||
|
|
||||||
|
Spanish does not come from Debian — see below; `hunspell-es` is the peninsular
|
||||||
|
build. Take the generic dictionary from an upstream release instead:
|
||||||
|
|
||||||
|
curl -LO https://github.com/sbosio/rla-es/releases/download/v2.9/es.oxt
|
||||||
|
unzip -d src es.oxt # an .oxt is a zip
|
||||||
|
python3 scripts/build_hunspell_dictionary.py es \\
|
||||||
|
src/es.aff src/es.dic web/public/dictionaries/es
|
||||||
|
"""
|
||||||
|
import gzip
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
# Directives worth keeping in the shipped aff. These shape *suggestions*, not
|
||||||
|
# membership: TRY orders the alphabet the corrector tries, KEY knows which keys
|
||||||
|
# are adjacent, REP holds the language's own confusions (cao/ção, ss/ç), and MAP
|
||||||
|
# says an accented vowel and its bare form are the same letter for scoring —
|
||||||
|
# which is most of what an ESL writer gets wrong in either language.
|
||||||
|
KEEP_DIRECTIVES = ("SET", "TRY", "KEY", "REP", "MAP", "WORDCHARS")
|
||||||
|
|
||||||
|
# Directives that would change which words are *accepted* and that this expander
|
||||||
|
# does not implement. If a future upstream release starts using one, the output
|
||||||
|
# would silently disagree with the real dictionary, so the build stops instead.
|
||||||
|
UNSUPPORTED = (
|
||||||
|
"COMPOUNDFLAG", "COMPOUNDMIN", "COMPOUNDRULE", "COMPOUNDBEGIN",
|
||||||
|
"ONLYINCOMPOUND", "PSEUDOROOT", "AF", "AM",
|
||||||
|
)
|
||||||
|
|
||||||
|
APOSTROPHES = "'’"
|
||||||
|
|
||||||
|
|
||||||
|
class Aff:
|
||||||
|
"""The parts of an .aff file that decide which words exist."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.pfx = {} # flag -> [(strip, append, condition, continuation)]
|
||||||
|
self.sfx = {}
|
||||||
|
# Cross-product, per flag — and kept per table, because PFX and SFX are
|
||||||
|
# separate flag namespaces in hunspell: the same flag may name a prefix
|
||||||
|
# table and a suffix table, with different cross-product settings. One
|
||||||
|
# shared dict let the second block silently overwrite the first.
|
||||||
|
self.cross_pfx = {} # flag -> bool
|
||||||
|
self.cross_sfx = {}
|
||||||
|
self.flag_kind = "char"
|
||||||
|
self.needaffix = None
|
||||||
|
self.circumfix = None
|
||||||
|
self.forbidden = None
|
||||||
|
self.dropped_apostrophe_rules = 0
|
||||||
|
|
||||||
|
|
||||||
|
def parse_flags(raw, kind):
|
||||||
|
"""Split a flag string into flags, per the aff's FLAG declaration."""
|
||||||
|
raw = raw.strip()
|
||||||
|
if not raw:
|
||||||
|
return set()
|
||||||
|
if kind == "long":
|
||||||
|
# Two characters per flag, exactly. An odd length is a malformed flag
|
||||||
|
# string, and silently dropping the trailing character would quietly
|
||||||
|
# expand an entry through the wrong paradigm — the failure mode this
|
||||||
|
# whole FLAG-aware rewrite exists to avoid.
|
||||||
|
if len(raw) % 2:
|
||||||
|
raise SystemExit(f"odd-length long flag string {raw!r}")
|
||||||
|
return {raw[i:i + 2] for i in range(0, len(raw), 2)}
|
||||||
|
if kind == "num":
|
||||||
|
return {f for f in raw.split(",") if f}
|
||||||
|
return set(raw)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_aff(path):
|
||||||
|
with open(path, encoding="utf-8") as fh:
|
||||||
|
lines = fh.read().splitlines()
|
||||||
|
|
||||||
|
aff = Aff()
|
||||||
|
|
||||||
|
# FLAG has to be known before anything containing a flag is read, and it can
|
||||||
|
# sit anywhere in the file. So: header pass first, rules second.
|
||||||
|
for line in lines:
|
||||||
|
parts = line.split()
|
||||||
|
if not parts:
|
||||||
|
continue
|
||||||
|
head = parts[0]
|
||||||
|
if head in UNSUPPORTED:
|
||||||
|
raise SystemExit(
|
||||||
|
f"{path}: unsupported directive {head!r} — this expander handles "
|
||||||
|
"PFX/SFX affixation with continuation flags, and honouring "
|
||||||
|
f"{head} would change which words are accepted. Extend the "
|
||||||
|
"script before shipping."
|
||||||
|
)
|
||||||
|
if len(parts) < 2:
|
||||||
|
continue
|
||||||
|
if head == "FLAG":
|
||||||
|
aff.flag_kind = parts[1]
|
||||||
|
if aff.flag_kind not in ("long", "num", "UTF-8"):
|
||||||
|
raise SystemExit(f"{path}: unknown FLAG type {aff.flag_kind!r}")
|
||||||
|
elif head == "NEEDAFFIX":
|
||||||
|
aff.needaffix = parts[1]
|
||||||
|
elif head == "CIRCUMFIX":
|
||||||
|
aff.circumfix = parts[1]
|
||||||
|
elif head == "FORBIDDENWORD":
|
||||||
|
aff.forbidden = parts[1]
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
while i < len(lines):
|
||||||
|
parts = lines[i].split()
|
||||||
|
if parts and parts[0] in ("PFX", "SFX"):
|
||||||
|
kind, flag, cross_flag, count = parts[0], parts[1], parts[2], int(parts[3])
|
||||||
|
table = aff.pfx if kind == "PFX" else aff.sfx
|
||||||
|
cross = aff.cross_pfx if kind == "PFX" else aff.cross_sfx
|
||||||
|
cross[flag] = cross_flag == "Y"
|
||||||
|
rules = table.setdefault(flag, [])
|
||||||
|
for j in range(1, count + 1):
|
||||||
|
p = lines[i + j].split()
|
||||||
|
strip = "" if p[2] == "0" else p[2]
|
||||||
|
append, _, cont_raw = p[3].partition("/")
|
||||||
|
if append == "0":
|
||||||
|
append = ""
|
||||||
|
# Elision. See the header: these forms are `l'` and its twelve
|
||||||
|
# siblings glued to words already in the list, they multiply the
|
||||||
|
# download by seven, and `withElision` reconstructs them at
|
||||||
|
# lookup for the cost of one extra hash probe.
|
||||||
|
if any(a in append for a in APOSTROPHES):
|
||||||
|
aff.dropped_apostrophe_rules += 1
|
||||||
|
continue
|
||||||
|
cont = parse_flags(cont_raw, aff.flag_kind)
|
||||||
|
if aff.circumfix and aff.circumfix in cont:
|
||||||
|
raise SystemExit(
|
||||||
|
f"{path}: CIRCUMFIX is used by a {kind} {flag} rule. It "
|
||||||
|
"was declared-but-unused when this expander was written "
|
||||||
|
"and is not implemented; honouring it would change which "
|
||||||
|
"words are accepted."
|
||||||
|
)
|
||||||
|
cond = p[4] if len(p) > 4 else "."
|
||||||
|
anchored = ("^" + cond) if kind == "PFX" else (cond + "$")
|
||||||
|
rules.append((strip, append, re.compile(anchored), cont))
|
||||||
|
i += count + 1
|
||||||
|
continue
|
||||||
|
i += 1
|
||||||
|
return aff
|
||||||
|
|
||||||
|
|
||||||
|
def apply_suffix(word, rules):
|
||||||
|
"""Every (form, continuation flags) a suffix table yields for `word`."""
|
||||||
|
out = []
|
||||||
|
for strip, append, cond, cont in rules:
|
||||||
|
if strip and not word.endswith(strip):
|
||||||
|
continue
|
||||||
|
if not cond.search(word):
|
||||||
|
continue
|
||||||
|
stem = word[: len(word) - len(strip)] if strip else word
|
||||||
|
out.append((stem + append, cont))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def apply_prefix(word, rules):
|
||||||
|
out = []
|
||||||
|
for strip, append, cond, cont in rules:
|
||||||
|
if strip and not word.startswith(strip):
|
||||||
|
continue
|
||||||
|
if not cond.search(word):
|
||||||
|
continue
|
||||||
|
stem = word[len(strip):] if strip else word
|
||||||
|
out.append((append + stem, cont))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def expand_entry(word, flags, aff, out):
|
||||||
|
"""Add every surface form of one dictionary entry to `out`.
|
||||||
|
|
||||||
|
Hunspell's model without compounding: a form is the stem plus at most one
|
||||||
|
prefix and at most one suffix. A flag reaches an affix either from the stem's
|
||||||
|
own flags or from the continuation flags of the affix applied on the other
|
||||||
|
side; when both sides apply, both rules must be declared cross-product.
|
||||||
|
|
||||||
|
NEEDAFFIX is why the bare form is not simply added: the flag says *this* form
|
||||||
|
is not a word, only whatever can be built from it — and it arrives both on
|
||||||
|
stems and on continuations.
|
||||||
|
"""
|
||||||
|
def is_word(carried):
|
||||||
|
return not (aff.needaffix and aff.needaffix in carried)
|
||||||
|
|
||||||
|
if is_word(flags):
|
||||||
|
out.add(word)
|
||||||
|
|
||||||
|
suffixed = [] # (form, flag, continuation flags)
|
||||||
|
for f in flags:
|
||||||
|
if f in aff.sfx:
|
||||||
|
for form, cont in apply_suffix(word, aff.sfx[f]):
|
||||||
|
suffixed.append((form, f, cont))
|
||||||
|
if is_word(cont):
|
||||||
|
out.add(form)
|
||||||
|
|
||||||
|
prefixed = []
|
||||||
|
for f in flags:
|
||||||
|
if f in aff.pfx:
|
||||||
|
for form, cont in apply_prefix(word, aff.pfx[f]):
|
||||||
|
prefixed.append((form, f, cont))
|
||||||
|
if is_word(cont):
|
||||||
|
out.add(form)
|
||||||
|
|
||||||
|
# Prefix then suffix. The suffix flag may come from the stem or from the
|
||||||
|
# prefix's own continuation (`PFX Um 0 0/S.`), and the suffix condition is
|
||||||
|
# matched against the whole prefixed word, which is what hunspell does.
|
||||||
|
# NEEDAFFIX is checked here too, exactly as on the single-affix paths above:
|
||||||
|
# a doubly-affixed form whose last continuation still carries the flag is
|
||||||
|
# "not a word on its own", and without compounding there is no third affix
|
||||||
|
# left to make it one.
|
||||||
|
for form, pf, pcont in prefixed:
|
||||||
|
if not aff.cross_pfx.get(pf):
|
||||||
|
continue
|
||||||
|
for f in flags | pcont:
|
||||||
|
if f in aff.sfx and aff.cross_sfx.get(f):
|
||||||
|
for full, fcont in apply_suffix(form, aff.sfx[f]):
|
||||||
|
if is_word(fcont):
|
||||||
|
out.add(full)
|
||||||
|
|
||||||
|
# Suffix then prefix — the same pair reached from the other side, which is
|
||||||
|
# how the elision prefixes arrive in French. Only the flags the suffix hands
|
||||||
|
# forward are new here; the stem's own were covered above.
|
||||||
|
for form, sf, scont in suffixed:
|
||||||
|
if not aff.cross_sfx.get(sf):
|
||||||
|
continue
|
||||||
|
for f in scont:
|
||||||
|
if f in aff.pfx and aff.cross_pfx.get(f):
|
||||||
|
for full, fcont in apply_prefix(form, aff.pfx[f]):
|
||||||
|
if is_word(fcont):
|
||||||
|
out.add(full)
|
||||||
|
|
||||||
|
|
||||||
|
def expand(aff_path, dic_path):
|
||||||
|
aff = parse_aff(aff_path)
|
||||||
|
forms = set()
|
||||||
|
needaffix_stems = 0
|
||||||
|
with open(dic_path, encoding="utf-8") as fh:
|
||||||
|
fh.readline() # leading entry count, not a word
|
||||||
|
for raw in fh:
|
||||||
|
# Morphological fields (po:nom is:fem) follow the entry, separated by
|
||||||
|
# a tab in pt-PT and by a space in fr.
|
||||||
|
entry = raw.strip().split("\t")[0].split(" ")[0]
|
||||||
|
if not entry:
|
||||||
|
continue
|
||||||
|
word, _, flagstr = entry.partition("/")
|
||||||
|
word = word.strip()
|
||||||
|
if not word:
|
||||||
|
continue
|
||||||
|
flags = parse_flags(flagstr, aff.flag_kind)
|
||||||
|
if aff.forbidden and aff.forbidden in flags:
|
||||||
|
raise SystemExit(
|
||||||
|
f"{dic_path}: FORBIDDENWORD is in use ({word!r}). It was "
|
||||||
|
"declared-but-unused when this expander was written; the "
|
||||||
|
"forms it removes would be wrongly accepted."
|
||||||
|
)
|
||||||
|
if aff.needaffix and aff.needaffix in flags:
|
||||||
|
needaffix_stems += 1
|
||||||
|
expand_entry(word, flags, aff, forms)
|
||||||
|
|
||||||
|
# NFC, because the aff's own ICONV table normalises decomposed accents on the
|
||||||
|
# way in and the browser hands nspell whatever the keyboard produced.
|
||||||
|
forms = {unicodedata.normalize("NFC", f) for f in forms}
|
||||||
|
return forms, aff, needaffix_stems
|
||||||
|
|
||||||
|
|
||||||
|
def shipped_aff(aff_path):
|
||||||
|
keep = []
|
||||||
|
for line in open(aff_path, encoding="utf-8").read().splitlines():
|
||||||
|
head = line.split()[0] if line.split() else ""
|
||||||
|
if head in KEEP_DIRECTIVES:
|
||||||
|
keep.append(line)
|
||||||
|
return "\n".join(keep) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
# Words the built list must accept, and must reject, before it is written. Each
|
||||||
|
# set is chosen to fail loudly on the *specific* wrong source that language has a
|
||||||
|
# packaged, plausible way of reaching — not to spot-check spelling in general.
|
||||||
|
PROFILES = {
|
||||||
|
# The pt-PT/pt-BR fault lines: post-Acordo spellings, the European lexicon,
|
||||||
|
# and the first-person-plural preterite accent that only pt-PT writes.
|
||||||
|
"pt-PT": {
|
||||||
|
"accept": ("receção", "húmido", "telemóvel", "autocarro", "comboio",
|
||||||
|
"ótimo", "pensámos", "escrevêssemos", "jardim"),
|
||||||
|
"reject": ("recepção", "úmido", "ônibus", "óptimo"),
|
||||||
|
"wrong": "this does not look like European Portuguese",
|
||||||
|
},
|
||||||
|
# Which of the three 1990-reform packagings this is. `coût`/`cout` and
|
||||||
|
# `paraître`/`paraitre` are each accepted by exactly one of classical and
|
||||||
|
# revised, so a build accepting all four is comprehensive and one that drops
|
||||||
|
# any of them is not. `Allemagne` is a NEEDAFFIX stem reachable only through
|
||||||
|
# a zero-append rule and `km` only through a prefix continuation, so between
|
||||||
|
# them they also check that this expander honoured the two features the
|
||||||
|
# pt-PT one refused.
|
||||||
|
"fr": {
|
||||||
|
"accept": ("coût", "cout", "paraître", "paraitre", "nénuphar", "nénufar",
|
||||||
|
"oignon", "ognon", "événement", "évènement", "jardin",
|
||||||
|
"Allemagne", "écrivissions", "km"),
|
||||||
|
"reject": ("jardinn", "écrivaitz", "xyzzyque"),
|
||||||
|
"wrong": "this does not look like the comprehensive French dictionary",
|
||||||
|
},
|
||||||
|
# The generic RLA build, and the accept list is written to reject the four
|
||||||
|
# neighbouring builds rather than to describe this one.
|
||||||
|
#
|
||||||
|
# The first version of this profile demanded *computadora* and *ordenador*,
|
||||||
|
# *papa* and *patata*, and passed — on the peninsular file, because **every**
|
||||||
|
# RLA variant carries the whole pan-Hispanic vocabulary. Vocabulary does not
|
||||||
|
# discriminate here at all; only morphology does, and it discriminates
|
||||||
|
# completely:
|
||||||
|
#
|
||||||
|
# * **voseo** (`vení`, `tenés`, `querés`) is in `es` and `es_AR` and not in
|
||||||
|
# `es_ES` or Debian's package. Demanding it rejects the peninsular build.
|
||||||
|
# * **vosotros** (`tenéis`, `escribid`) is in `es`, `es_AR` and `es_ES`, and
|
||||||
|
# largely absent from `es_MX`. Demanding it rejects the Mexican build.
|
||||||
|
#
|
||||||
|
# Requiring both at once leaves exactly one package standing: the generic
|
||||||
|
# `es`, which is the only one that accepts every variety of Spanish. That is
|
||||||
|
# the same reason fr ships `-comprehensive` — the only thing this dictionary
|
||||||
|
# can do is underline something, and *tienes* and *tenés* are both correct
|
||||||
|
# Spanish taught in different countries.
|
||||||
|
#
|
||||||
|
# The rest are shape checks: `escribiésemos` is the -se imperfect subjunctive,
|
||||||
|
# `dámelo` proves the enclitic pronoun rules ran, and `jardín`/`niño` prove
|
||||||
|
# FLAG UTF-8 was read as characters rather than bytes.
|
||||||
|
"es": {
|
||||||
|
"accept": (
|
||||||
|
# Rejects es_ES and Debian's hunspell-es.
|
||||||
|
"vení", "tenés", "querés", "sabés", "andá",
|
||||||
|
# Rejects es_MX.
|
||||||
|
"tenéis", "escribid",
|
||||||
|
# Rejects es_AR, which has both voseo and vosotros and would
|
||||||
|
# otherwise pass. Caribbean and Andean everyday words: the generic
|
||||||
|
# build is the union of all twenty-four, so it is the only one that
|
||||||
|
# holds another region's vocabulary as well as its own.
|
||||||
|
"arepa", "chévere", "bacán",
|
||||||
|
# Pan-Hispanic vocabulary. These pass on every RLA build, so they
|
||||||
|
# prove nothing on their own — kept because a source that stopped
|
||||||
|
# being RLA at all would fail them.
|
||||||
|
"computadora", "ordenador", "papa", "patata", "jugo", "zumo",
|
||||||
|
# Morphology and encoding.
|
||||||
|
"escribiéramos", "escribiésemos", "escríbeme", "dámelo",
|
||||||
|
"jardín", "niño", "corazón",
|
||||||
|
),
|
||||||
|
"reject": ("jardinn", "escribiz", "xyzzyque", "haiga"),
|
||||||
|
"wrong": "this is not the generic RLA build (a per-country one accepts "
|
||||||
|
"only some of these)",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main(lang, aff_path, dic_path, out_dir):
|
||||||
|
profile = PROFILES.get(lang)
|
||||||
|
if profile is None:
|
||||||
|
raise SystemExit(f"no profile for {lang!r}; known: {', '.join(PROFILES)}")
|
||||||
|
|
||||||
|
forms, aff, needaffix_stems = expand(aff_path, dic_path)
|
||||||
|
|
||||||
|
missing = [w for w in profile["accept"] if w not in forms]
|
||||||
|
present = [w for w in profile["reject"] if w in forms]
|
||||||
|
if missing or present:
|
||||||
|
raise SystemExit(
|
||||||
|
f"{profile['wrong']}: missing {missing}, unexpectedly present {present}"
|
||||||
|
)
|
||||||
|
|
||||||
|
os.makedirs(out_dir, exist_ok=True)
|
||||||
|
ordered = sorted(forms)
|
||||||
|
body = f"{len(ordered)}\n" + "\n".join(ordered) + "\n"
|
||||||
|
|
||||||
|
dic_out = os.path.join(out_dir, f"{lang}.dic.gz")
|
||||||
|
# mtime=0 so rebuilding identical input produces an identical file — a
|
||||||
|
# vendored asset that changes on every build is noise in the diff.
|
||||||
|
with gzip.GzipFile(dic_out, "wb", compresslevel=9, mtime=0) as fh:
|
||||||
|
fh.write(body.encode("utf-8"))
|
||||||
|
|
||||||
|
aff_out = os.path.join(out_dir, f"{lang}.aff")
|
||||||
|
with open(aff_out, "w", encoding="utf-8") as fh:
|
||||||
|
fh.write(shipped_aff(aff_path))
|
||||||
|
|
||||||
|
print(f"{len(ordered)} forms -> {dic_out} "
|
||||||
|
f"({os.path.getsize(dic_out) / 1e6:.2f} MB gzipped); "
|
||||||
|
f"{needaffix_stems} NEEDAFFIX stems, "
|
||||||
|
f"{aff.dropped_apostrophe_rules} elision rules skipped", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if len(sys.argv) != 5:
|
||||||
|
raise SystemExit(
|
||||||
|
"usage: build_hunspell_dictionary.py <lang> <aff> <dic> <out-dir>\n"
|
||||||
|
f" lang is one of: {', '.join(PROFILES)}"
|
||||||
|
)
|
||||||
|
main(*sys.argv[1:5])
|
||||||
Generated
+28
-10
@@ -30,6 +30,7 @@
|
|||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@tailwindcss/vite": "^4.0.0",
|
"@tailwindcss/vite": "^4.0.0",
|
||||||
|
"@types/node": "^26.1.2",
|
||||||
"@types/react": "^19.1.0",
|
"@types/react": "^19.1.0",
|
||||||
"@types/react-dom": "^19.1.0",
|
"@types/react-dom": "^19.1.0",
|
||||||
"@vitejs/plugin-react": "^4.3.4",
|
"@vitejs/plugin-react": "^4.3.4",
|
||||||
@@ -2107,6 +2108,16 @@
|
|||||||
"integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==",
|
"integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/node": {
|
||||||
|
"version": "26.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.2.tgz",
|
||||||
|
"integrity": "sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"undici-types": "~8.3.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@types/react": {
|
"node_modules/@types/react": {
|
||||||
"version": "19.2.17",
|
"version": "19.2.17",
|
||||||
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz",
|
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz",
|
||||||
@@ -2928,9 +2939,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/linkify-it": {
|
"node_modules/linkify-it": {
|
||||||
"version": "5.0.1",
|
"version": "5.0.2",
|
||||||
"resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.2.tgz",
|
||||||
"integrity": "sha512-wVoTjP4Q6R0NW5hiZkVJaFZPWgtXfoGF+6LucL3/FtiNjmcHhYjEr5f1Kqjirc1nBW07J/ZuRFumqr2oqccEWg==",
|
"integrity": "sha512-ONTm2jCMAVZjgQa/Fy1kScXsuOoF5NPTsoFBdE1KVIZ2vAh/r9+Bqo+0jINCBYnavTPQZz38QzFTme79ENoN3Q==",
|
||||||
"funding": [
|
"funding": [
|
||||||
{
|
{
|
||||||
"type": "github",
|
"type": "github",
|
||||||
@@ -3019,9 +3030,9 @@
|
|||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/nanoid": {
|
"node_modules/nanoid": {
|
||||||
"version": "3.3.15",
|
"version": "3.3.16",
|
||||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz",
|
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz",
|
||||||
"integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==",
|
"integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"funding": [
|
"funding": [
|
||||||
{
|
{
|
||||||
@@ -3104,9 +3115,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/postcss": {
|
"node_modules/postcss": {
|
||||||
"version": "8.5.15",
|
"version": "8.5.23",
|
||||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz",
|
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz",
|
||||||
"integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==",
|
"integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"funding": [
|
"funding": [
|
||||||
{
|
{
|
||||||
@@ -3124,7 +3135,7 @@
|
|||||||
],
|
],
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"nanoid": "^3.3.12",
|
"nanoid": "^3.3.16",
|
||||||
"picocolors": "^1.1.1",
|
"picocolors": "^1.1.1",
|
||||||
"source-map-js": "^1.2.1"
|
"source-map-js": "^1.2.1"
|
||||||
},
|
},
|
||||||
@@ -3559,6 +3570,13 @@
|
|||||||
"integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
|
"integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/undici-types": {
|
||||||
|
"version": "8.3.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
|
||||||
|
"integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT"
|
||||||
|
},
|
||||||
"node_modules/update-browserslist-db": {
|
"node_modules/update-browserslist-db": {
|
||||||
"version": "1.2.3",
|
"version": "1.2.3",
|
||||||
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz",
|
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz",
|
||||||
|
|||||||
@@ -33,6 +33,7 @@
|
|||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@tailwindcss/vite": "^4.0.0",
|
"@tailwindcss/vite": "^4.0.0",
|
||||||
|
"@types/node": "^26.1.2",
|
||||||
"@types/react": "^19.1.0",
|
"@types/react": "^19.1.0",
|
||||||
"@types/react-dom": "^19.1.0",
|
"@types/react-dom": "^19.1.0",
|
||||||
"@vitejs/plugin-react": "^4.3.4",
|
"@vitejs/plugin-react": "^4.3.4",
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
Spanish spelling dictionary
|
||||||
|
===========================
|
||||||
|
|
||||||
|
The word list in `es.dic.gz` and the suggestion directives in `es.aff` are
|
||||||
|
derived from the **generic** Spanish Hunspell dictionary published by the RLA-ES
|
||||||
|
project ("Recursos Lingüísticos Abiertos del Español"), release v2.9.
|
||||||
|
|
||||||
|
Copyright (C) Santiago Bosio and the RLA-ES contributors
|
||||||
|
|
||||||
|
License: GPL-3+ or LGPL-3+ or MPL-1.1+
|
||||||
|
Tri-licensed; you may choose freely among the three. Petal
|
||||||
|
redistributes under the MPL. Full texts:
|
||||||
|
https://www.gnu.org/licenses/gpl-3.0.en.html
|
||||||
|
https://www.gnu.org/licenses/lgpl-3.0.en.html
|
||||||
|
https://www.mozilla.org/en-US/MPL/1.1/
|
||||||
|
|
||||||
|
Upstream: https://github.com/sbosio/rla-es
|
||||||
|
Source: https://github.com/sbosio/rla-es/releases/download/v2.9/es.oxt
|
||||||
|
(an .oxt is a zip; es.aff and es.dic are at its root)
|
||||||
|
|
||||||
|
Not the Debian package, and that is the point
|
||||||
|
---------------------------------------------
|
||||||
|
`hunspell-es` looks like the obvious source and is the wrong one. It installs
|
||||||
|
twenty country codes — `es_AR` through `es_VE` — all symlinked to a single file,
|
||||||
|
which reads like "one pan-Hispanic dictionary". It is not. RLA publishes
|
||||||
|
twenty-four dictionaries per release: one per country, plus a **generic `es`**
|
||||||
|
that is the union of all of them, and Debian ships the **peninsular `es_ES`**
|
||||||
|
build under the collapsed name.
|
||||||
|
|
||||||
|
Measured against v2.9, expanded to surface forms:
|
||||||
|
|
||||||
|
Debian hunspell-es 659,085 forms voseo: no vosotros: yes
|
||||||
|
upstream es_ES 659,018 forms voseo: no vosotros: yes
|
||||||
|
upstream es_MX 554,923 forms voseo: no vosotros: no
|
||||||
|
upstream es_AR 669,605 forms voseo: yes vosotros: yes
|
||||||
|
upstream es (generic) 717,640 forms voseo: yes vosotros: yes <-- this
|
||||||
|
|
||||||
|
The 58,622-form gap between Debian's file and the generic one is essentially the
|
||||||
|
**voseo** paradigm — `vení`, `tenés`, `querés`, `sabés`, `andá` — the ordinary
|
||||||
|
present tense of Argentina, Uruguay, Paraguay and much of Central America. Under
|
||||||
|
the Debian package, a writer using it would have had her own verbs underlined as
|
||||||
|
misspellings.
|
||||||
|
|
||||||
|
Why the generic build rather than one country
|
||||||
|
---------------------------------------------
|
||||||
|
The only thing this dictionary can do is underline something. *Tienes* and
|
||||||
|
*tenés* are both correct Spanish, taught in different countries, and a writing
|
||||||
|
companion has no business marking one of them wrong — the same reasoning that
|
||||||
|
makes the French dictionary here the `-comprehensive` packaging rather than
|
||||||
|
`-classical` or `-revised`. The generic build accepts every variety, so Petal
|
||||||
|
underlines only what no Spanish speaker anywhere would write.
|
||||||
|
|
||||||
|
How the build proves it got this file
|
||||||
|
-------------------------------------
|
||||||
|
Vocabulary cannot tell these builds apart: *every* RLA variant carries the full
|
||||||
|
pan-Hispanic lexicon, so *computadora* alongside *ordenador* passes on the
|
||||||
|
peninsular file too. (The `REP` table is likewise no evidence — its `ll`/`y` and
|
||||||
|
`ás`/`az` entries look like yeísmo and seseo but are shared by all builds.) Only
|
||||||
|
the verb paradigms are localised, so the `es` profile in
|
||||||
|
`scripts/build_hunspell_dictionary.py` demands, all at once:
|
||||||
|
|
||||||
|
* **voseo** (`vení`, `tenés`, `querés`) — rejects `es_ES` and Debian's package;
|
||||||
|
* **vosotros** (`tenéis`, `escribid`) — rejects `es_MX`;
|
||||||
|
* **another region's everyday words** (`arepa`, `chévere`, `bacán`) — rejects
|
||||||
|
`es_AR`, which has both paradigms and would otherwise pass.
|
||||||
|
|
||||||
|
Only the generic build satisfies all three. Each of the four neighbouring builds
|
||||||
|
was run through the profile and confirmed to fail.
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
SET UTF-8
|
||||||
|
TRY aeroinsctldumpbgfvhzóíjáqéñxyúükwAEROINSCTLDUMPBGFVHZÓÍJÁQÉÑXYÚÜKW
|
||||||
|
REP 19
|
||||||
|
REP ás az
|
||||||
|
REP az ás
|
||||||
|
REP cc x
|
||||||
|
REP és ez
|
||||||
|
REP ez és
|
||||||
|
REP güe hue
|
||||||
|
REP güi hui
|
||||||
|
REP hue güe
|
||||||
|
REP hui güi
|
||||||
|
REP ís iz
|
||||||
|
REP ío ido
|
||||||
|
REP ke que
|
||||||
|
REP ki qui
|
||||||
|
REP ll y
|
||||||
|
REP mb nv
|
||||||
|
REP nv mb
|
||||||
|
REP seci cesi
|
||||||
|
REP x cc
|
||||||
|
REP y ll
|
||||||
|
MAP 6
|
||||||
|
MAP aáAÁ
|
||||||
|
MAP eéEÉ
|
||||||
|
MAP iíIÍ
|
||||||
|
MAP oóOÓ
|
||||||
|
MAP uúüUÚÜ
|
||||||
|
MAP nñNÑ
|
||||||
Binary file not shown.
@@ -0,0 +1,49 @@
|
|||||||
|
French spelling dictionary
|
||||||
|
==========================
|
||||||
|
|
||||||
|
The word list in `fr.dic.gz` and the suggestion directives in `fr.aff` are
|
||||||
|
derived from the Dicollecte / Grammalecte Hunspell dictionary for French
|
||||||
|
(`fr.aff` / `fr.dic`), as packaged by Debian/Ubuntu in
|
||||||
|
`hunspell-fr-comprehensive`.
|
||||||
|
|
||||||
|
Copyright (C) 2007-2018 the Dicollecte contributors
|
||||||
|
(full list at
|
||||||
|
https://grammalecte.net/members.php?prj=fr)
|
||||||
|
Dictionary author: Olivier R.
|
||||||
|
|
||||||
|
License: MPL-2.0
|
||||||
|
This Source Code Form is subject to the terms of the Mozilla
|
||||||
|
Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||||
|
with this file, You can obtain one at
|
||||||
|
http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
Upstream: https://grammalecte.net/home.php?prj=fr
|
||||||
|
|
||||||
|
Which of the three
|
||||||
|
------------------
|
||||||
|
|
||||||
|
Debian packages this dictionary three ways, by how it treats the 1990 spelling
|
||||||
|
reform: `hunspell-fr-classical` (traditional spellings), `hunspell-fr-revised`
|
||||||
|
(reform spellings) and `hunspell-fr-comprehensive` (both). Petal ships the
|
||||||
|
**comprehensive** one, because Petal never corrects her French — the only thing
|
||||||
|
this dictionary can do is underline something, and *coût* and *cout* are both
|
||||||
|
correct French. The regional packages (`fr_FR`, `fr_CA`, `fr_BE`, `fr_CH`,
|
||||||
|
`fr_LU`, `fr_MC`) are all symlinks to the same word list, so there is no
|
||||||
|
regional choice being made here.
|
||||||
|
|
||||||
|
What Petal changed
|
||||||
|
------------------
|
||||||
|
|
||||||
|
`scripts/build_hunspell_dictionary.py` applies the upstream affix rules ahead of
|
||||||
|
time — Hunspell's PFX/SFX expansion run once at build time instead of once per
|
||||||
|
browser — and writes the resulting 473,326 surface forms as a flat word list.
|
||||||
|
The shipped `.aff` keeps only upstream's TRY/KEY/REP/MAP/WORDCHARS lines, which
|
||||||
|
shape *corrections* rather than membership.
|
||||||
|
|
||||||
|
One thing about membership did change, and it is reversible at lookup rather
|
||||||
|
than lost: the elided forms (`l'arbre`, `qu'elle`, `jusqu'ici`) are **not** in
|
||||||
|
the word list. Expanding them costs 8.25 MB gzipped against 1.19 MB, and they
|
||||||
|
are not new words — they are thirteen clitics glued to words already present —
|
||||||
|
so `withElision` in `web/src/hooks/useSpellChecker.ts` splits the token and
|
||||||
|
checks the remainder instead. Stems that carry an apostrophe of their own
|
||||||
|
(`aujourd'hui`, `quelqu'un`, `presqu'île`, `prud'homme`) are kept verbatim.
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
SET UTF-8
|
||||||
|
WORDCHARS -’'1234567890.
|
||||||
|
TRY esntiarulodcpmévqfgbhàxèjyêMILzACçôîPâùJFSûBVœRDGNETHXkïOwKWYUëQÉZŒüãÎáöóÈíæÅñäśńÿ
|
||||||
|
MAP 25
|
||||||
|
MAP aàâäAÀÂÄ
|
||||||
|
MAP eéèêëEÉÈÊË
|
||||||
|
MAP iîïyIÎÏY
|
||||||
|
MAP oôöOÔÖ
|
||||||
|
MAP uùûüUÙÛÜ
|
||||||
|
MAP cçCÇ
|
||||||
|
MAP bB
|
||||||
|
MAP dD
|
||||||
|
MAP fF
|
||||||
|
MAP gG
|
||||||
|
MAP hH
|
||||||
|
MAP jJ
|
||||||
|
MAP kK
|
||||||
|
MAP lL
|
||||||
|
MAP mM
|
||||||
|
MAP nN
|
||||||
|
MAP pP
|
||||||
|
MAP qQ
|
||||||
|
MAP rR
|
||||||
|
MAP sS
|
||||||
|
MAP tT
|
||||||
|
MAP vV
|
||||||
|
MAP wW
|
||||||
|
MAP xX
|
||||||
|
MAP zZ
|
||||||
|
REP 110
|
||||||
|
REP a â
|
||||||
|
REP â a
|
||||||
|
REP e é
|
||||||
|
REP é e
|
||||||
|
REP e ê
|
||||||
|
REP ê e
|
||||||
|
REP e è
|
||||||
|
REP è e
|
||||||
|
REP i î
|
||||||
|
REP î i
|
||||||
|
REP o ô
|
||||||
|
REP ô o
|
||||||
|
REP u û
|
||||||
|
REP û u
|
||||||
|
REP A Â
|
||||||
|
REP Â A
|
||||||
|
REP E É
|
||||||
|
REP É E
|
||||||
|
REP E Ê
|
||||||
|
REP Ê E
|
||||||
|
REP E È
|
||||||
|
REP È E
|
||||||
|
REP I Î
|
||||||
|
REP Î I
|
||||||
|
REP O Ô
|
||||||
|
REP Ô O
|
||||||
|
REP U Û
|
||||||
|
REP Û U
|
||||||
|
REP ^Ca$ Ça
|
||||||
|
REP ^l l'
|
||||||
|
REP ^d d'
|
||||||
|
REP ^n n'
|
||||||
|
REP ^s s'
|
||||||
|
REP ^j j'
|
||||||
|
REP ^m m'
|
||||||
|
REP ^t t'
|
||||||
|
REP ^c c'
|
||||||
|
REP f ph
|
||||||
|
REP ph f
|
||||||
|
REP c qu
|
||||||
|
REP qu c
|
||||||
|
REP k qu
|
||||||
|
REP qu k
|
||||||
|
REP x ct
|
||||||
|
REP ct x
|
||||||
|
REP bb b
|
||||||
|
REP b bb
|
||||||
|
REP cc c
|
||||||
|
REP c cc
|
||||||
|
REP ff f
|
||||||
|
REP f ff
|
||||||
|
REP ll l
|
||||||
|
REP l ll
|
||||||
|
REP mm m
|
||||||
|
REP m mm
|
||||||
|
REP nn n
|
||||||
|
REP n nn
|
||||||
|
REP pp p
|
||||||
|
REP p pp
|
||||||
|
REP rr r
|
||||||
|
REP r rr
|
||||||
|
REP ss s
|
||||||
|
REP s ss
|
||||||
|
REP ss c
|
||||||
|
REP c ss
|
||||||
|
REP ss ç
|
||||||
|
REP ç ss
|
||||||
|
REP tt t
|
||||||
|
REP t tt
|
||||||
|
REP œ oe
|
||||||
|
REP oe œ
|
||||||
|
REP æ ae
|
||||||
|
REP ae æ
|
||||||
|
REP ai é
|
||||||
|
REP é ai
|
||||||
|
REP ai è
|
||||||
|
REP è ai
|
||||||
|
REP ai ê
|
||||||
|
REP ê ai
|
||||||
|
REP ei é
|
||||||
|
REP é ei
|
||||||
|
REP ei è
|
||||||
|
REP è ei
|
||||||
|
REP ei ê
|
||||||
|
REP ê ei
|
||||||
|
REP o au
|
||||||
|
REP au o
|
||||||
|
REP o eau
|
||||||
|
REP eau o
|
||||||
|
REP ett èt
|
||||||
|
REP èt ett
|
||||||
|
REP ell èl
|
||||||
|
REP èl ell
|
||||||
|
REP t th
|
||||||
|
REP th t
|
||||||
|
REP ième$ e
|
||||||
|
REP ème$ e
|
||||||
|
REP è$ e
|
||||||
|
REP mn$ min
|
||||||
|
REP ogue$ ogiste
|
||||||
|
REP ogiste$ ogue
|
||||||
|
REP disez$ dites
|
||||||
|
REP fesez$ faites
|
||||||
|
REP faisez$ faites
|
||||||
|
REP puit puits
|
||||||
|
REP sanctionnable punissable
|
||||||
|
REP questionnable discutable
|
||||||
|
REP antitartre détartrant
|
||||||
|
REP email courriel
|
||||||
|
REP construirent construisirent
|
||||||
|
KEY azertyuiop|qsdfghjklmù|wxcvbn|aéz|yèu|iço|oàp|aqz|zse|edr|rft|tgy|yhu|uji|iko|olpm|qws|sxd|dcf|fvg|gbh|hnj
|
||||||
Binary file not shown.
@@ -0,0 +1,32 @@
|
|||||||
|
European Portuguese spelling dictionary
|
||||||
|
=======================================
|
||||||
|
|
||||||
|
The word list in `pt-PT.dic.gz` and the suggestion directives in `pt-PT.aff` are
|
||||||
|
derived from the LibreOffice/Projecto Natura Hunspell dictionary for European
|
||||||
|
Portuguese (`pt_PT.aff` / `pt_PT.dic`), as packaged by Debian/Ubuntu in
|
||||||
|
`hunspell-pt-pt`.
|
||||||
|
|
||||||
|
Copyright (C) 2006-2012 José João de Almeida <jj@di.uminho.pt>
|
||||||
|
Rui Vilela <ruivilela@di.uminho.pt>
|
||||||
|
Alberto Simões <ambs@di.uminho.pt>
|
||||||
|
Universidade do Minho — Projecto Natura
|
||||||
|
|
||||||
|
License: GPL-2 or LGPL-2.1 or MPL-1.1
|
||||||
|
(Petal redistributes it under the MPL-1.1 option.)
|
||||||
|
|
||||||
|
Upstream: https://natura.di.uminho.pt/ — via
|
||||||
|
https://git.libreoffice.org/dictionaries/+/refs/heads/master/pt_PT
|
||||||
|
|
||||||
|
What Petal changed
|
||||||
|
------------------
|
||||||
|
|
||||||
|
Nothing about which words are correct. `scripts/build_hunspell_dictionary.py`
|
||||||
|
applies the upstream affix rules ahead of time — Hunspell's PFX/SFX expansion
|
||||||
|
run once at build time instead of once per browser — and writes the resulting
|
||||||
|
1,039,058 surface forms as a flat word list. The shipped `.aff` keeps only
|
||||||
|
upstream's TRY/KEY/REP/MAP/WORDCHARS lines, which shape *corrections* rather
|
||||||
|
than membership. See that script's header for why the dictionary could not be
|
||||||
|
vendored in its original form.
|
||||||
|
|
||||||
|
Note that npm's `dictionary-pt` is *not* this dictionary: both it and
|
||||||
|
`dictionary-pt-br` package the Brazilian VERO word list.
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
SET UTF-8
|
||||||
|
TRY aerisontcdmlupvgbfzáhçqjíxãóéêâúõACMPSBTELGRIFVDkHJONôywUKXZWQÁYÍÉàÓèÂÚ
|
||||||
|
KEY qwertyuiop|asdfghjkl|zxcvbnm
|
||||||
|
WORDCHARS -
|
||||||
|
REP 25
|
||||||
|
REP por pro
|
||||||
|
REP pre per
|
||||||
|
REP damente mente
|
||||||
|
REP mente damente
|
||||||
|
REP iz íz
|
||||||
|
REP cao ção
|
||||||
|
REP ç ss
|
||||||
|
REP ss ç
|
||||||
|
REP c ss
|
||||||
|
REP ss c
|
||||||
|
REP ch x
|
||||||
|
REP x ch
|
||||||
|
REP cç x
|
||||||
|
REP x cç
|
||||||
|
REP k qu
|
||||||
|
REP íti ití
|
||||||
|
REP ití íti
|
||||||
|
REP issí íssi
|
||||||
|
REP ilí íli
|
||||||
|
REP íli ilí
|
||||||
|
REP ífi ifí
|
||||||
|
REP ifí ífi
|
||||||
|
REP nume mune
|
||||||
|
REP coen quen
|
||||||
|
REP concerteza com_certeza
|
||||||
|
MAP 11
|
||||||
|
MAP aá
|
||||||
|
MAP aã
|
||||||
|
MAP aâ
|
||||||
|
MAP eé
|
||||||
|
MAP eê
|
||||||
|
MAP ií
|
||||||
|
MAP cç
|
||||||
|
MAP oó
|
||||||
|
MAP oô
|
||||||
|
MAP oõ
|
||||||
|
MAP uú
|
||||||
Binary file not shown.
@@ -0,0 +1,59 @@
|
|||||||
|
Chinese word list (segmentation)
|
||||||
|
================================
|
||||||
|
|
||||||
|
`words.txt.gz` is not a spelling dictionary — Chinese has no spelling to check
|
||||||
|
in the Hunspell sense. It is the word list Petal's segmenter walks, so that a
|
||||||
|
sentence written without spaces has words in it to hover, look up and capture.
|
||||||
|
Each line is `word frequency`. See scripts/build_cedict.py for how it is built
|
||||||
|
and why it is gated where it is.
|
||||||
|
|
||||||
|
It is derived from two upstream sources, both redistributable, both credited
|
||||||
|
here because the file itself has no room for a header.
|
||||||
|
|
||||||
|
|
||||||
|
CC-CEDICT — the headwords
|
||||||
|
-------------------------
|
||||||
|
Community maintained free Chinese-English dictionary, published by MDBG.
|
||||||
|
https://www.mdbg.net/chinese/dictionary?page=cedict
|
||||||
|
|
||||||
|
Licensed under the Creative Commons Attribution-ShareAlike 4.0 International
|
||||||
|
License — https://creativecommons.org/licenses/by-sa/4.0/
|
||||||
|
|
||||||
|
Referenced works:
|
||||||
|
CEDICT — Copyright (C) 1997, 1998 Paul Andrew Denisowski
|
||||||
|
|
||||||
|
CC-CEDICT is also the source of `internal/lexicon/data/hanzi.json.gz`, the
|
||||||
|
pinyin and English senses embedded in the Petal binary. The same attribution and
|
||||||
|
the same ShareAlike terms apply to that file; it is named here because it has
|
||||||
|
nowhere of its own to say so.
|
||||||
|
|
||||||
|
|
||||||
|
jieba — the frequencies
|
||||||
|
-----------------------
|
||||||
|
"结巴" Chinese word segmentation, by Sun Junyi.
|
||||||
|
https://github.com/fxsjy/jieba
|
||||||
|
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2013 Sun Junyi
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
|
||||||
|
Only the word/frequency columns are used; jieba's part-of-speech tags and its
|
||||||
|
algorithm are not (Petal's segmenter is its own, in web/src/lib/segment.ts).
|
||||||
Binary file not shown.
+158
-27
@@ -1,12 +1,23 @@
|
|||||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||||
import { api, type DocSummary, type DocUpdate, type Document, type Suggestion, type Tag, type TagColor } from './api/client'
|
import {
|
||||||
|
api,
|
||||||
|
onDocLang,
|
||||||
|
type DocSummary,
|
||||||
|
type DocUpdate,
|
||||||
|
type Document,
|
||||||
|
type Suggestion,
|
||||||
|
type Tag,
|
||||||
|
type TagColor,
|
||||||
|
} from './api/client'
|
||||||
import { useAutoSave } from './hooks/useAutoSave'
|
import { useAutoSave } from './hooks/useAutoSave'
|
||||||
import { useCheckpoint } from './hooks/useCheckpoint'
|
import { findingKey, useCheckpoint } from './hooks/useCheckpoint'
|
||||||
import { useSpellChecker } from './hooks/useSpellChecker'
|
import { useSpellChecker } from './hooks/useSpellChecker'
|
||||||
|
import { useSegmenter } from './hooks/useSegmenter'
|
||||||
import { useTags } from './hooks/useTags'
|
import { useTags } from './hooks/useTags'
|
||||||
import { DocList } from './components/DocList/DocList'
|
import { DocList } from './components/DocList/DocList'
|
||||||
import { EditorCore, type EditorChange } from './components/Editor/EditorCore'
|
import { EditorCore, type EditorChange } from './components/Editor/EditorCore'
|
||||||
import { ToneSelect } from './components/Editor/ToneSelect'
|
import { ToneSelect } from './components/Editor/ToneSelect'
|
||||||
|
import { ChromeStrip } from './components/Editor/ChromeStrip'
|
||||||
import { ExportMenu } from './components/Export/ExportMenu'
|
import { ExportMenu } from './components/Export/ExportMenu'
|
||||||
import { HistoryPanel } from './components/History/HistoryPanel'
|
import { HistoryPanel } from './components/History/HistoryPanel'
|
||||||
import { GardenPanel } from './components/Garden/GardenPanel'
|
import { GardenPanel } from './components/Garden/GardenPanel'
|
||||||
@@ -21,6 +32,7 @@ import { PetalFall } from './effects/PetalFall'
|
|||||||
import { usePack } from './i18n'
|
import { usePack } from './i18n'
|
||||||
import { useNightMode } from './hooks/useNightMode'
|
import { useNightMode } from './hooks/useNightMode'
|
||||||
import { playSuggestionSound } from './audio/sounds'
|
import { playSuggestionSound } from './audio/sounds'
|
||||||
|
import { fromIME } from './lib/ime'
|
||||||
|
|
||||||
export default function App() {
|
export default function App() {
|
||||||
const updateAvailable = useVersionWatch()
|
const updateAvailable = useVersionWatch()
|
||||||
@@ -30,7 +42,7 @@ export default function App() {
|
|||||||
const night = useNightMode()
|
const night = useNightMode()
|
||||||
// Who's writing, and whether the server still recognises them. `signedOut`
|
// Who's writing, and whether the server still recognises them. `signedOut`
|
||||||
// flips the moment any call comes back 401.
|
// flips the moment any call comes back 401.
|
||||||
const { me, signedOut } = useSession()
|
const { me, signedOut, setDirection, setPair } = useSession()
|
||||||
const t = usePack()
|
const t = usePack()
|
||||||
// A real account to sign out of, as opposed to the hardcoded local user a
|
// A real account to sign out of, as opposed to the hardcoded local user a
|
||||||
// build without auth configured runs as.
|
// build without auth configured runs as.
|
||||||
@@ -78,7 +90,42 @@ export default function App() {
|
|||||||
return wordCountRef.current === 0 && (t === '' || t === 'Untitled')
|
return wordCountRef.current === 0 && (t === '' || t === 'Untitled')
|
||||||
}, [])
|
}, [])
|
||||||
|
|
||||||
const { status, schedule, saveNow } = useAutoSave(currentDoc?.id ?? null)
|
// The pass announces its verdict the moment it decides one, which is the only
|
||||||
|
// moment that is prompt enough: read-aloud is reached for when she has stopped
|
||||||
|
// typing, so waiting for the next save means waiting for a save that isn't
|
||||||
|
// coming. Registered once, and it updates the same one field the save path
|
||||||
|
// does — whichever arrives first wins, and they agree.
|
||||||
|
useEffect(() => {
|
||||||
|
onDocLang((docId, lang) =>
|
||||||
|
setCurrentDoc((prev) => (prev && prev.id === docId && prev.doc_lang !== lang ? { ...prev, doc_lang: lang } : prev)),
|
||||||
|
)
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
// Only `doc_lang` is lifted out of the save response, and only when it moved.
|
||||||
|
// It is the one field the server decides on its own — the checkpoint pass reads
|
||||||
|
// the whole document and writes back whether it is English or hers — so it is
|
||||||
|
// the one field that would otherwise go stale under her while she writes. Read
|
||||||
|
// -aloud is what notices: a Portuguese paragraph read in an American voice.
|
||||||
|
// Everything else in the response is what the client just sent, and copying it
|
||||||
|
// back mid-keystroke would be a way to lose a character, not to gain one.
|
||||||
|
const { status, schedule, saveNow } = useAutoSave(currentDoc?.id ?? null, (saved) =>
|
||||||
|
setCurrentDoc((prev) =>
|
||||||
|
prev && prev.id === saved.id && prev.doc_lang !== saved.doc_lang ? { ...prev, doc_lang: saved.doc_lang } : prev,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
// The Chinese word list, for a writer going the other way through the zh pair.
|
||||||
|
// Gated on the account's own setting rather than on anything in the text: a
|
||||||
|
// Mandarin native drafting English quotes Chinese constantly, and none of that
|
||||||
|
// is what segmentation is for. Declared above the checkpoint because the
|
||||||
|
// offline 错别字 pass reads it.
|
||||||
|
//
|
||||||
|
// Both halves of the gate matter now that Chinese is not the only pair with a
|
||||||
|
// learner direction. `learning_pair` alone used to imply zh; a writer learning
|
||||||
|
// Portuguese is also learning_pair and has no use for a megabyte of Chinese
|
||||||
|
// word list — nor for the hanzi hover it turns on, which would ask /api/hanzi
|
||||||
|
// about Portuguese words.
|
||||||
|
const segmenter = useSegmenter(me?.direction === 'learning_pair' && me?.pair_lang === 'zh')
|
||||||
|
|
||||||
const {
|
const {
|
||||||
suggestions,
|
suggestions,
|
||||||
checking,
|
checking,
|
||||||
@@ -89,7 +136,8 @@ export default function App() {
|
|||||||
runVoice,
|
runVoice,
|
||||||
runCollocation,
|
runCollocation,
|
||||||
removeSuggestion,
|
removeSuggestion,
|
||||||
} = useCheckpoint(currentDoc?.id ?? null)
|
resolveServerId,
|
||||||
|
} = useCheckpoint(currentDoc?.id ?? null, segmenter)
|
||||||
// Browser-side spell checker — loads the en-US dictionary once per session.
|
// Browser-side spell checker — loads the en-US dictionary once per session.
|
||||||
const { checker: spellChecker, addWord } = useSpellChecker()
|
const { checker: spellChecker, addWord } = useSpellChecker()
|
||||||
// The tag roster (with counts). Assignments live on the doc summaries below.
|
// The tag roster (with counts). Assignments live on the doc summaries below.
|
||||||
@@ -307,16 +355,39 @@ export default function App() {
|
|||||||
[currentDoc, patchSummary, schedule],
|
[currentDoc, patchSummary, schedule],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// She took the kitten up on its daily invitation. The prompt becomes the
|
||||||
|
// blank page's title, so the question she agreed to answer stays in front of
|
||||||
|
// her while she answers it — rather than being said once and then gone the
|
||||||
|
// moment the bubble fades.
|
||||||
|
const handleAcceptInvitation = useCallback(
|
||||||
|
(prompt: string) => {
|
||||||
|
if (!currentDoc) return
|
||||||
|
handleTitleChange(prompt)
|
||||||
|
},
|
||||||
|
[currentDoc, handleTitleChange],
|
||||||
|
)
|
||||||
|
|
||||||
const handleEditorChange = useCallback(
|
const handleEditorChange = useCallback(
|
||||||
(change: EditorChange) => {
|
(change: EditorChange) => {
|
||||||
|
const { composing, ...patch } = change
|
||||||
setWordCount(change.word_count)
|
setWordCount(change.word_count)
|
||||||
setDocText(change.content_text)
|
|
||||||
setEditTick((n) => n + 1)
|
|
||||||
if (currentDoc) {
|
if (currentDoc) {
|
||||||
patchSummary(currentDoc.id, { word_count: change.word_count })
|
patchSummary(currentDoc.id, { word_count: change.word_count })
|
||||||
schedule(change)
|
// The save is never held: see EditorChange.composing. The flag itself
|
||||||
scheduleCheckpoint(change.content_text)
|
// stays out of the patch — it describes the keyboard, not the document,
|
||||||
|
// and the stashed draft a signed-out save leaves behind should be the
|
||||||
|
// document alone.
|
||||||
|
schedule(patch)
|
||||||
}
|
}
|
||||||
|
// Everything below reads the text as prose. While an IME composition is
|
||||||
|
// in flight it is not prose yet — it is the pinyin she is converting — so
|
||||||
|
// the checkpoint, the rule pack and the companion all wait for the word
|
||||||
|
// to commit. EditorCore emits one more change the moment it does, so
|
||||||
|
// nothing is skipped, only deferred by the length of a word.
|
||||||
|
if (composing) return
|
||||||
|
setDocText(change.content_text)
|
||||||
|
setEditTick((n) => n + 1)
|
||||||
|
if (currentDoc) scheduleCheckpoint(change.content_text)
|
||||||
},
|
},
|
||||||
[currentDoc, patchSummary, schedule, scheduleCheckpoint],
|
[currentDoc, patchSummary, schedule, scheduleCheckpoint],
|
||||||
)
|
)
|
||||||
@@ -337,17 +408,44 @@ export default function App() {
|
|||||||
|
|
||||||
// Accept applies the replacement in the editor (handled in EditorCore) and
|
// Accept applies the replacement in the editor (handled in EditorCore) and
|
||||||
// marks the suggestion accepted; dismiss just rejects it. Both drop it locally.
|
// marks the suggestion accepted; dismiss just rejects it. Both drop it locally.
|
||||||
|
// A rule-pack card can be accepted before its row exists — the edit has already
|
||||||
|
// landed either way, so a missing id just means there's nothing to file.
|
||||||
const handleAccept = useCallback(
|
const handleAccept = useCallback(
|
||||||
async (s: Suggestion) => {
|
async (s: Suggestion) => {
|
||||||
removeSuggestion(s.id)
|
removeSuggestion(s.id)
|
||||||
setAcceptTick((n) => n + 1)
|
setAcceptTick((n) => n + 1)
|
||||||
try {
|
try {
|
||||||
await api.acceptSuggestion(s.id)
|
const id = await resolveServerId(s)
|
||||||
|
if (id) await api.acceptSuggestion(id)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('accept failed', err)
|
console.error('accept failed', err)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[removeSuggestion],
|
[removeSuggestion, resolveServerId],
|
||||||
|
)
|
||||||
|
|
||||||
|
// Accept-all: EditorCore has already applied the whole category in one editor
|
||||||
|
// transaction, so this is only the bookkeeping. Each row is filed individually
|
||||||
|
// (there's no batch endpoint, and each accept plants its own word in the
|
||||||
|
// garden), but the kitten cheers once — five cheers for one click would read as
|
||||||
|
// five separate congratulations for a decision she made once.
|
||||||
|
const handleAcceptMany = useCallback(
|
||||||
|
async (list: Suggestion[]) => {
|
||||||
|
if (list.length === 0) return
|
||||||
|
for (const s of list) removeSuggestion(s.id)
|
||||||
|
setAcceptTick((n) => n + 1)
|
||||||
|
await Promise.all(
|
||||||
|
list.map(async (s) => {
|
||||||
|
try {
|
||||||
|
const id = await resolveServerId(s)
|
||||||
|
if (id) await api.acceptSuggestion(id)
|
||||||
|
} catch (err) {
|
||||||
|
console.error('accept failed', err)
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
},
|
||||||
|
[removeSuggestion, resolveServerId],
|
||||||
)
|
)
|
||||||
|
|
||||||
// After restoring a version, swap the restored doc into the editor. Bumping
|
// After restoring a version, swap the restored doc into the editor. Bumping
|
||||||
@@ -365,11 +463,14 @@ export default function App() {
|
|||||||
[patchSummary],
|
[patchSummary],
|
||||||
)
|
)
|
||||||
|
|
||||||
// Escape always restores the sidebar while in distraction-free mode.
|
// Escape always restores the sidebar while in distraction-free mode — unless
|
||||||
|
// it belongs to an IME, where it cancels a candidate and never reaches Petal
|
||||||
|
// at all. This is the writer typing Chinese in the very mode built for
|
||||||
|
// uninterrupted writing, so it is the one worth getting right.
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!focusMode) return
|
if (!focusMode) return
|
||||||
const onKey = (e: KeyboardEvent) => {
|
const onKey = (e: KeyboardEvent) => {
|
||||||
if (e.key === 'Escape') setFocusMode(false)
|
if (e.key === 'Escape' && !fromIME(e)) setFocusMode(false)
|
||||||
}
|
}
|
||||||
window.addEventListener('keydown', onKey)
|
window.addEventListener('keydown', onKey)
|
||||||
return () => window.removeEventListener('keydown', onKey)
|
return () => window.removeEventListener('keydown', onKey)
|
||||||
@@ -385,12 +486,13 @@ export default function App() {
|
|||||||
async (s: Suggestion) => {
|
async (s: Suggestion) => {
|
||||||
removeSuggestion(s.id)
|
removeSuggestion(s.id)
|
||||||
try {
|
try {
|
||||||
await api.dismissSuggestion(s.id)
|
const id = await resolveServerId(s)
|
||||||
|
if (id) await api.dismissSuggestion(id)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('dismiss failed', err)
|
console.error('dismiss failed', err)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[removeSuggestion],
|
[removeSuggestion, resolveServerId],
|
||||||
)
|
)
|
||||||
|
|
||||||
// Play a soft sound when freshly-checked suggestions arrive — one per distinct
|
// Play a soft sound when freshly-checked suggestions arrive — one per distinct
|
||||||
@@ -398,10 +500,14 @@ export default function App() {
|
|||||||
// a pile-up. We track which ids we've already chimed for, and only chime for
|
// a pile-up. We track which ids we've already chimed for, and only chime for
|
||||||
// recently-created suggestions so opening a doc with old pending advice stays
|
// recently-created suggestions so opening a doc with old pending advice stays
|
||||||
// silent (the existing set was created in a past session).
|
// silent (the existing set was created in a past session).
|
||||||
|
// Rule-pack findings are chimed by their wording, not their id: the same fix
|
||||||
|
// appears first as a provisional card and then as its persisted row, and the
|
||||||
|
// writer should hear it once.
|
||||||
const chimedRef = useRef<Set<string>>(new Set())
|
const chimedRef = useRef<Set<string>>(new Set())
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const fresh = suggestions.filter((s) => !chimedRef.current.has(s.id))
|
const key = (s: Suggestion) => (s.source === 'local' ? `local:${findingKey(s)}` : s.id)
|
||||||
fresh.forEach((s) => chimedRef.current.add(s.id))
|
const fresh = suggestions.filter((s) => !chimedRef.current.has(key(s)))
|
||||||
|
fresh.forEach((s) => chimedRef.current.add(key(s)))
|
||||||
const justMade = fresh.filter(
|
const justMade = fresh.filter(
|
||||||
(s) => Date.now() - new Date(s.created_at).getTime() < 12_000,
|
(s) => Date.now() - new Date(s.created_at).getTime() < 12_000,
|
||||||
)
|
)
|
||||||
@@ -423,7 +529,7 @@ export default function App() {
|
|||||||
<PetalFall night={night} />
|
<PetalFall night={night} />
|
||||||
<header
|
<header
|
||||||
onMouseDown={handleChromeDown}
|
onMouseDown={handleChromeDown}
|
||||||
className="petal-no-print flex h-12 shrink-0 items-center gap-2 px-5"
|
className="petal-no-print flex h-12 shrink-0 items-center gap-2 px-3 md:px-5"
|
||||||
style={{ borderBottom: '1px solid var(--color-border)' }}
|
style={{ borderBottom: '1px solid var(--color-border)' }}
|
||||||
>
|
>
|
||||||
<button
|
<button
|
||||||
@@ -439,7 +545,15 @@ export default function App() {
|
|||||||
☰
|
☰
|
||||||
</button>
|
</button>
|
||||||
<span className="text-xl">🌸</span>
|
<span className="text-xl">🌸</span>
|
||||||
<span className="text-lg font-extrabold text-plum">Petal</span>
|
{/* The wordmark is the first thing to go on a small phone. The header
|
||||||
|
holds a hamburger, a name and the garden button, and the garden
|
||||||
|
button's label is a langpack string: 词汇花园 · Garden is 130px
|
||||||
|
where Jardim de palavras · Garden is nearly 200, which is the
|
||||||
|
difference between fitting a 320px screen and scrolling the whole
|
||||||
|
app sideways. Of everything in this row, the one that can be spared
|
||||||
|
is the app's own name — she is already inside the app, and the
|
||||||
|
blossom stays. */}
|
||||||
|
<span className="petal-wordmark text-lg font-extrabold text-plum">Petal</span>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => setGardenOpen(true)}
|
onClick={() => setGardenOpen(true)}
|
||||||
@@ -474,6 +588,9 @@ export default function App() {
|
|||||||
onToggleTag={handleToggleTag}
|
onToggleTag={handleToggleTag}
|
||||||
onCreateTag={handleCreateTag}
|
onCreateTag={handleCreateTag}
|
||||||
account={account}
|
account={account}
|
||||||
|
direction={me?.direction}
|
||||||
|
onDirection={setDirection}
|
||||||
|
onPair={setPair}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -488,27 +605,36 @@ export default function App() {
|
|||||||
<>
|
<>
|
||||||
<div
|
<div
|
||||||
onMouseDown={handleChromeDown}
|
onMouseDown={handleChromeDown}
|
||||||
className="flex flex-1 flex-col overflow-y-auto px-6 py-8"
|
// `petal-scrollport` marks this as the editor's scrolling
|
||||||
|
// ancestor; EditorCore measures it to pin the text column while
|
||||||
|
// the suggestion rail's overhang is scrolled.
|
||||||
|
className="petal-scrollport flex flex-1 flex-col overflow-y-auto px-6 py-8"
|
||||||
>
|
>
|
||||||
<div ref={canvasRef} className="mx-auto flex w-full max-w-[720px] flex-1 flex-col">
|
<div ref={canvasRef} className="mx-auto flex w-full max-w-[720px] flex-1 flex-col">
|
||||||
<div className="mb-5 flex items-center gap-3">
|
{/* Title, then the three chrome pills. Their labels are
|
||||||
|
bilingual and don't shrink, so how much width this row
|
||||||
|
wants is a property of the langpack — 历史 is two glyphs
|
||||||
|
where Historique is ten — and on a phone no language's
|
||||||
|
version of it fits. The pills therefore live in a strip
|
||||||
|
that scrolls itself; the title takes its own line below
|
||||||
|
the drawer breakpoint so it keeps its full width. */}
|
||||||
|
<div className="mb-5 flex flex-wrap items-center gap-2 md:gap-3">
|
||||||
<input
|
<input
|
||||||
value={title}
|
value={title}
|
||||||
onChange={(e) => handleTitleChange(e.target.value)}
|
onChange={(e) => handleTitleChange(e.target.value)}
|
||||||
placeholder="Untitled"
|
placeholder="Untitled"
|
||||||
aria-label="Document title"
|
aria-label="Document title"
|
||||||
className="min-w-0 flex-1 bg-transparent text-3xl font-extrabold text-plum focus:outline-none"
|
className="min-w-0 flex-1 basis-full bg-transparent text-3xl font-extrabold text-plum focus:outline-none md:basis-0"
|
||||||
style={{ fontFamily: 'var(--font-ui)' }}
|
style={{ fontFamily: 'var(--font-ui)' }}
|
||||||
/>
|
/>
|
||||||
<div className="petal-no-print shrink-0">
|
<ChromeStrip className="petal-no-print flex items-center gap-2 py-0.5 md:gap-3">
|
||||||
<ToneSelect value={tone} onChange={handleToneChange} />
|
<ToneSelect value={tone} onChange={handleToneChange} />
|
||||||
</div>
|
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => setHistoryOpen(true)}
|
onClick={() => setHistoryOpen(true)}
|
||||||
aria-label="Version history"
|
aria-label="Version history"
|
||||||
title="Browse and restore earlier versions"
|
title="Browse and restore earlier versions"
|
||||||
className="petal-no-print inline-flex h-9 shrink-0 items-center gap-1.5 whitespace-nowrap px-3 text-sm font-bold"
|
className="inline-flex h-9 shrink-0 items-center gap-1.5 whitespace-nowrap px-3 text-sm font-bold"
|
||||||
style={{
|
style={{
|
||||||
borderRadius: 'var(--radius-pill)',
|
borderRadius: 'var(--radius-pill)',
|
||||||
background: 'var(--color-surface)',
|
background: 'var(--color-surface)',
|
||||||
@@ -520,17 +646,19 @@ export default function App() {
|
|||||||
<span>{t.app.history}</span>
|
<span>{t.app.history}</span>
|
||||||
<span style={{ color: 'var(--color-muted)' }}>· History</span>
|
<span style={{ color: 'var(--color-muted)' }}>· History</span>
|
||||||
</button>
|
</button>
|
||||||
<div className="petal-no-print">
|
|
||||||
<ExportMenu docId={currentDoc.id} />
|
<ExportMenu docId={currentDoc.id} />
|
||||||
</div>
|
</ChromeStrip>
|
||||||
</div>
|
</div>
|
||||||
<EditorCore
|
<EditorCore
|
||||||
key={`${currentDoc.id}:${editorEpoch}`}
|
key={`${currentDoc.id}:${editorEpoch}`}
|
||||||
docId={currentDoc.id}
|
docId={currentDoc.id}
|
||||||
|
docLang={currentDoc.doc_lang}
|
||||||
initialContent={currentDoc.content}
|
initialContent={currentDoc.content}
|
||||||
onChange={handleEditorChange}
|
onChange={handleEditorChange}
|
||||||
|
segmenter={segmenter}
|
||||||
suggestions={suggestions}
|
suggestions={suggestions}
|
||||||
onAccept={handleAccept}
|
onAccept={handleAccept}
|
||||||
|
onAcceptMany={handleAcceptMany}
|
||||||
onDismiss={handleDismiss}
|
onDismiss={handleDismiss}
|
||||||
onVoiceCheck={runVoice}
|
onVoiceCheck={runVoice}
|
||||||
voicing={voicing}
|
voicing={voicing}
|
||||||
@@ -551,6 +679,7 @@ export default function App() {
|
|||||||
voicing={voicing}
|
voicing={voicing}
|
||||||
collocating={collocating}
|
collocating={collocating}
|
||||||
llmDown={llmDown}
|
llmDown={llmDown}
|
||||||
|
suggestionCount={suggestions.length}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
@@ -597,6 +726,8 @@ export default function App() {
|
|||||||
editTick={editTick}
|
editTick={editTick}
|
||||||
acceptTick={acceptTick}
|
acceptTick={acceptTick}
|
||||||
text={docText}
|
text={docText}
|
||||||
|
blankPage={wordCount === 0 && docText.trim() === ''}
|
||||||
|
onAcceptInvitation={handleAcceptInvitation}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
+149
-8
@@ -44,8 +44,18 @@ export interface Document {
|
|||||||
// When true, this document's automatic snapshots are never pruned, so its
|
// When true, this document's automatic snapshots are never pruned, so its
|
||||||
// full writing trail survives as authorship evidence (see the passport).
|
// full writing trail survives as authorship evidence (see the passport).
|
||||||
preserve_history: boolean
|
preserve_history: boolean
|
||||||
|
// Which language this document is written in, as decided server-side by the
|
||||||
|
// checkpoint pass: '' | 'en' | 'pair' ('' reads as English). Read-only — the
|
||||||
|
// editor never sends it. It is here so read-aloud can use the right voice on a
|
||||||
|
// document written in her own language.
|
||||||
|
doc_lang: DocLang
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The document-language verdict, shared by documents and garden cards. 'pair'
|
||||||
|
// names the writer's own language rather than a language code, so changing her
|
||||||
|
// pair re-reads her documents instead of stranding a stale name on them.
|
||||||
|
export type DocLang = '' | 'en' | 'pair'
|
||||||
|
|
||||||
// Fields the editor sends on auto-save. All optional so a rename can send title
|
// Fields the editor sends on auto-save. All optional so a rename can send title
|
||||||
// alone; the editor sends the full set.
|
// alone; the editor sends the full set.
|
||||||
export interface DocUpdate {
|
export interface DocUpdate {
|
||||||
@@ -68,19 +78,51 @@ export interface WordMeaning {
|
|||||||
// a list of synonyms. Any of these may be empty when the word isn't a headword.
|
// a list of synonyms. Any of these may be empty when the word isn't a headword.
|
||||||
export interface WordInfo {
|
export interface WordInfo {
|
||||||
word: string
|
word: string
|
||||||
gloss: string // Chinese translation; '' when the word isn't in the gloss set
|
gloss: string // translation into the writer's language; '' when absent
|
||||||
phonetic: string // IPA for the English word; '' when absent
|
phonetic: string // IPA for the English word; '' when absent
|
||||||
definitions: WordMeaning[]
|
definitions: WordMeaning[]
|
||||||
synonyms: string[]
|
synonyms: string[]
|
||||||
|
// From DreamDict only; the embedded datasets leave them unknown. `frequency`
|
||||||
|
// is 0 and `difficulty` is -1 when the dictionary has no score — see
|
||||||
|
// wordBand, which turns the pair into a band or into nothing at all.
|
||||||
|
frequency: number
|
||||||
|
difficulty: number
|
||||||
|
etymology: string // free-form, already trimmed to a line by the server; '' when absent
|
||||||
|
// The same token read as a word of the writer's own language, when it is one.
|
||||||
|
// Absent for a zh-pair writer and for almost every word in a Latin pair — see
|
||||||
|
// lexicon.Reverse for why Petal asks both directions instead of guessing.
|
||||||
|
reverse?: WordReverse
|
||||||
|
}
|
||||||
|
|
||||||
|
// A word looked up in the other direction: the writer's language -> English.
|
||||||
|
export interface WordReverse {
|
||||||
|
lang: string
|
||||||
|
gloss: string
|
||||||
|
definitions?: WordMeaning[]
|
||||||
|
phonetic?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
// The lightweight Chinese-only gloss behind the inline hover/select tooltip.
|
// The lightweight Chinese-only gloss behind the inline hover/select tooltip.
|
||||||
export interface Gloss {
|
export interface Gloss {
|
||||||
word: string
|
word: string
|
||||||
gloss: string
|
gloss: string
|
||||||
|
// The English meaning of the token read as a word of her own language.
|
||||||
|
// Present only on a collision (Portuguese *sale*, French *chat*).
|
||||||
|
reverse?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export type SuggestionType = 'grammar' | 'phrasing' | 'idiom' | 'clarity' | 'voice' | 'collocation' | 'mechanics'
|
// 'translate' is a span she wrote in her own language, rendered into English —
|
||||||
|
// not a correction. The server decides the label from the span itself, never from
|
||||||
|
// the model, so the client can trust it (see suggestions/language.go).
|
||||||
|
export type SuggestionType =
|
||||||
|
| 'grammar'
|
||||||
|
| 'phrasing'
|
||||||
|
| 'idiom'
|
||||||
|
| 'clarity'
|
||||||
|
| 'translate'
|
||||||
|
| 'voice'
|
||||||
|
| 'collocation'
|
||||||
|
| 'mechanics'
|
||||||
|
|
||||||
// One word in the vocabulary garden: a looked-up word with its gloss/phonetic,
|
// One word in the vocabulary garden: a looked-up word with its gloss/phonetic,
|
||||||
// the sentence it was met in, and its spaced-repetition state. `reps` drives how
|
// the sentence it was met in, and its spaced-repetition state. `reps` drives how
|
||||||
@@ -93,6 +135,9 @@ export interface VocabWord {
|
|||||||
phonetic: string
|
phonetic: string
|
||||||
example: string
|
example: string
|
||||||
doc_id: string | null
|
doc_id: string | null
|
||||||
|
// The language of the document this word was met in — the card's own language
|
||||||
|
// (migration 0018). Read-aloud needs it: "comum" is unguessable from letters.
|
||||||
|
lang: DocLang
|
||||||
due_at: string
|
due_at: string
|
||||||
interval_days: number
|
interval_days: number
|
||||||
ease: number
|
ease: number
|
||||||
@@ -136,18 +181,39 @@ export interface Suggestion {
|
|||||||
explanation: string
|
explanation: string
|
||||||
type: SuggestionType
|
type: SuggestionType
|
||||||
status: 'pending' | 'accepted' | 'rejected'
|
status: 'pending' | 'accepted' | 'rejected'
|
||||||
|
// Which engine proposed it — the offline rule pack or the model. The rail
|
||||||
|
// deliberately renders both identically; this is here because the wire format
|
||||||
|
// carries it, not because the writer is ever shown it.
|
||||||
|
source?: 'llm' | 'local'
|
||||||
created_at: string
|
created_at: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The growth journal (GET /api/suggestions/growth). `kept`/`kept_before` are
|
||||||
|
// the last thirty days and the thirty before them — the only comparison Petal
|
||||||
|
// draws is with her own past self. `stuck` is phrasing she was given that now
|
||||||
|
// turns up across her own documents; `faded` is what she used to be corrected
|
||||||
|
// on and hasn't been lately. Both lists are empty when the data isn't there:
|
||||||
|
// nothing here is padded to fill a page.
|
||||||
|
export interface GrowthJournal {
|
||||||
|
kept: number
|
||||||
|
kept_before: number
|
||||||
|
stuck: { phrase: string; docs: number }[]
|
||||||
|
faded: { pattern: string; times: number }[]
|
||||||
|
}
|
||||||
|
|
||||||
// A deterministic, rule-based fix detected client-side (see Companion/prose.ts).
|
// A deterministic, rule-based fix detected client-side (see Companion/prose.ts).
|
||||||
// The frontend owns mechanics detection; the backend only persists these as the
|
// The frontend owns offline detection; the backend only persists these. Spans
|
||||||
// 'mechanics' suggestion family. Spans are exact plaintext offsets.
|
// are exact plaintext offsets. `type` names the family the finding belongs to:
|
||||||
|
// 'mechanics' for a fix to this sentence, 'collocation' for the miscollocation
|
||||||
|
// rules, whose findings are chunks worth keeping and are filed — and planted in
|
||||||
|
// the garden on accept — exactly like the LLM coach's.
|
||||||
export interface MechanicsFinding {
|
export interface MechanicsFinding {
|
||||||
from: number
|
from: number
|
||||||
to: number
|
to: number
|
||||||
original: string
|
original: string
|
||||||
replacement: string
|
replacement: string
|
||||||
explanation: string
|
explanation: string
|
||||||
|
type: 'mechanics' | 'collocation'
|
||||||
}
|
}
|
||||||
|
|
||||||
// One dictionary's worth of personal words — the ones she's excused from
|
// One dictionary's worth of personal words — the ones she's excused from
|
||||||
@@ -157,6 +223,21 @@ export interface PersonalWords {
|
|||||||
words: string[]
|
words: string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One pronunciation of a Chinese word, and what it means in that pronunciation.
|
||||||
|
// A list, because 得 is dé "to obtain" and also the particle in 说得很好.
|
||||||
|
export interface HanziReading {
|
||||||
|
pinyin: string
|
||||||
|
senses: string
|
||||||
|
}
|
||||||
|
|
||||||
|
// A Chinese word lookup. `readings` is empty for a word with no headword, in
|
||||||
|
// which case `chars` may carry the character-by-character reading.
|
||||||
|
export interface HanziInfo {
|
||||||
|
word: string
|
||||||
|
readings: HanziReading[]
|
||||||
|
chars: { char: string; pinyin: string; senses: string }[]
|
||||||
|
}
|
||||||
|
|
||||||
// Who's writing. Mirrors the backend db.User.
|
// Who's writing. Mirrors the backend db.User.
|
||||||
export interface Me {
|
export interface Me {
|
||||||
id: string
|
id: string
|
||||||
@@ -164,6 +245,11 @@ export interface Me {
|
|||||||
display_name: string
|
display_name: string
|
||||||
created_at: string
|
created_at: string
|
||||||
pair_lang: string
|
pair_lang: string
|
||||||
|
// Which half of the pair is being learned: 'learning_en' (the writer is
|
||||||
|
// native in pair_lang and practising English) or 'learning_pair' (the other
|
||||||
|
// way round). Mirrors users.direction; the server refuses 'learning_pair' for
|
||||||
|
// a pair it has no word list for.
|
||||||
|
direction: string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Thrown when the server says the session is gone. Callers can tell it apart
|
// Thrown when the server says the session is gone. Callers can tell it apart
|
||||||
@@ -190,11 +276,34 @@ function signedOut(): UnauthorizedError {
|
|||||||
return new UnauthorizedError()
|
return new UnauthorizedError()
|
||||||
}
|
}
|
||||||
|
|
||||||
async function req<T>(path: string, init?: RequestInit): Promise<T> {
|
// The document-language verdict is decided by the checkpoint pass, so the pass's
|
||||||
|
// own response is the first moment the client can know it. It rides on a header
|
||||||
|
// (the pass answers with a bare array of suggestions, and every caller reads it
|
||||||
|
// as one), and reaches the app through a handler registered here — the same
|
||||||
|
// shape onUnauthorized already uses, for the same reason: it is one fact from
|
||||||
|
// deep inside a request that a component several layers up needs.
|
||||||
|
//
|
||||||
|
// Without it the editor learns the verdict only from a document save, which is
|
||||||
|
// always one save behind the pass — and read-aloud is reached for precisely when
|
||||||
|
// she has stopped typing and no further save is coming.
|
||||||
|
let docLangHandler: ((docId: string, lang: DocLang) => void) | null = null
|
||||||
|
|
||||||
|
export function onDocLang(handler: (docId: string, lang: DocLang) => void) {
|
||||||
|
docLangHandler = handler
|
||||||
|
}
|
||||||
|
|
||||||
|
// `verdictFor` names the document whose language this response may announce.
|
||||||
|
// Only the three pass endpoints pass it; everything else has no verdict to carry
|
||||||
|
// and never touches the handler.
|
||||||
|
async function req<T>(path: string, init?: RequestInit, verdictFor?: string): Promise<T> {
|
||||||
const res = await fetch(`/api${path}`, {
|
const res = await fetch(`/api${path}`, {
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
...init,
|
...init,
|
||||||
})
|
})
|
||||||
|
if (verdictFor && res.ok) {
|
||||||
|
const lang = res.headers.get('X-Petal-Doc-Lang')
|
||||||
|
if (lang === '' || lang === 'en' || lang === 'pair') docLangHandler?.(verdictFor, lang)
|
||||||
|
}
|
||||||
if (res.status === 401) throw signedOut()
|
if (res.status === 401) throw signedOut()
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
const detail = await res.text().catch(() => '')
|
const detail = await res.text().catch(() => '')
|
||||||
@@ -209,6 +318,22 @@ export const api = {
|
|||||||
// the hardcoded local user, so the frontend needs no separate mode for it.
|
// the hardcoded local user, so the frontend needs no separate mode for it.
|
||||||
me: () => req<Me>('/me'),
|
me: () => req<Me>('/me'),
|
||||||
|
|
||||||
|
// Move to another (English + X) pair. Answers with the whole updated user, so
|
||||||
|
// the caller re-reads the pair from the server rather than assuming its own
|
||||||
|
// request took — a code the server won't ship comes back 400 and the app is
|
||||||
|
// still on a language it can render.
|
||||||
|
setPairLang: (lang: string) =>
|
||||||
|
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ pair_lang: lang }) }),
|
||||||
|
|
||||||
|
// Turn the pair around. Same endpoint, same contract, and deliberately a
|
||||||
|
// separate call: the two fields are validated together server-side, so a
|
||||||
|
// client that wants to change both says both in one request rather than
|
||||||
|
// sending two that each pass on their own.
|
||||||
|
setDirection: (direction: string) =>
|
||||||
|
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ direction }) }),
|
||||||
|
setPair: (lang: string, direction: string) =>
|
||||||
|
req<Me>('/me', { method: 'PATCH', body: JSON.stringify({ pair_lang: lang, direction }) }),
|
||||||
|
|
||||||
listDocs: () => req<DocSummary[]>('/docs'),
|
listDocs: () => req<DocSummary[]>('/docs'),
|
||||||
createDoc: () => req<Document>('/docs', { method: 'POST' }),
|
createDoc: () => req<Document>('/docs', { method: 'POST' }),
|
||||||
getDoc: (id: string) => req<Document>(`/docs/${id}`),
|
getDoc: (id: string) => req<Document>(`/docs/${id}`),
|
||||||
@@ -220,14 +345,14 @@ export const api = {
|
|||||||
// Rate-limited per document server-side (returns the existing set if too soon).
|
// Rate-limited per document server-side (returns the existing set if too soon).
|
||||||
// Both passes return the UNIFIED pending set (grammar + voice), so the client
|
// Both passes return the UNIFIED pending set (grammar + voice), so the client
|
||||||
// never drops one family's highlights when the other refreshes.
|
// never drops one family's highlights when the other refreshes.
|
||||||
checkDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/check`, { method: 'POST' }),
|
checkDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/check`, { method: 'POST' }, id),
|
||||||
// Voice-consistency pass: whole-document, explicit-action, slower. Returns the
|
// Voice-consistency pass: whole-document, explicit-action, slower. Returns the
|
||||||
// unified pending set too. Rate-limited per document server-side.
|
// unified pending set too. Rate-limited per document server-side.
|
||||||
voiceDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/voice`, { method: 'POST' }),
|
voiceDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/voice`, { method: 'POST' }, id),
|
||||||
// Collocation coach: whole-document, explicit-action pass flagging non-native
|
// Collocation coach: whole-document, explicit-action pass flagging non-native
|
||||||
// word pairings ("do a decision" → "make a decision"). Returns the unified
|
// word pairings ("do a decision" → "make a decision"). Returns the unified
|
||||||
// pending set too. Rate-limited per document server-side.
|
// pending set too. Rate-limited per document server-side.
|
||||||
collocationDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/collocation`, { method: 'POST' }),
|
collocationDoc: (id: string) => req<Suggestion[]>(`/docs/${id}/collocation`, { method: 'POST' }, id),
|
||||||
// Mechanics pass: persist the client-detected deterministic fixes as the
|
// Mechanics pass: persist the client-detected deterministic fixes as the
|
||||||
// 'mechanics' family and return the unified pending set. Not rate-limited (it's
|
// 'mechanics' family and return the unified pending set. Not rate-limited (it's
|
||||||
// free, local detection); runs alongside the grammar checkpoint.
|
// free, local detection); runs alongside the grammar checkpoint.
|
||||||
@@ -238,6 +363,12 @@ export const api = {
|
|||||||
}),
|
}),
|
||||||
// Pending suggestions for a doc, loaded when the editor opens it.
|
// Pending suggestions for a doc, loaded when the editor opens it.
|
||||||
listSuggestions: (id: string) => req<Suggestion[]>(`/docs/${id}/suggestions`),
|
listSuggestions: (id: string) => req<Suggestion[]>(`/docs/${id}/suggestions`),
|
||||||
|
// The spans she has already accepted or dismissed on this doc, normalized. The
|
||||||
|
// server suppresses these itself; the client needs them so the instant rule-pack
|
||||||
|
// pass doesn't hand back a dismissed card before the server can say otherwise —
|
||||||
|
// or, with the server unreachable, at all. See lib/settled.ts.
|
||||||
|
listSettled: (id: string) =>
|
||||||
|
req<{ originals: string[] }>(`/docs/${id}/settled`),
|
||||||
acceptSuggestion: (id: string) =>
|
acceptSuggestion: (id: string) =>
|
||||||
req<void>(`/suggestions/${id}/accept`, { method: 'POST' }),
|
req<void>(`/suggestions/${id}/accept`, { method: 'POST' }),
|
||||||
dismissSuggestion: (id: string) =>
|
dismissSuggestion: (id: string) =>
|
||||||
@@ -246,6 +377,10 @@ export const api = {
|
|||||||
// opening bubble (the explanation itself stays English in the card body).
|
// opening bubble (the explanation itself stays English in the card body).
|
||||||
translateSuggestion: (id: string) =>
|
translateSuggestion: (id: string) =>
|
||||||
req<{ translation: string }>(`/suggestions/${id}/translate`, { method: 'POST' }),
|
req<{ translation: string }>(`/suggestions/${id}/translate`, { method: 'POST' }),
|
||||||
|
// The growth journal: her own accepted edits read back as patterns. Purely a
|
||||||
|
// read-side view of a table Petal already keeps, computed locally with no
|
||||||
|
// model call, so it costs nothing and leaves nothing.
|
||||||
|
growth: () => req<GrowthJournal>('/suggestions/growth'),
|
||||||
|
|
||||||
// Version history. listVersions returns metadata only (no bodies); getVersion
|
// Version history. listVersions returns metadata only (no bodies); getVersion
|
||||||
// loads one full snapshot for preview; snapshotDoc takes an explicit restore
|
// loads one full snapshot for preview; snapshotDoc takes an explicit restore
|
||||||
@@ -278,6 +413,12 @@ export const api = {
|
|||||||
// Lightweight Chinese-only gloss for the inline hover/select tooltip — instant
|
// Lightweight Chinese-only gloss for the inline hover/select tooltip — instant
|
||||||
// and offline, so it fires on hover without spinning up the heavier lookup.
|
// and offline, so it fires on hover without spinning up the heavier lookup.
|
||||||
glossWord: (word: string) => req<Gloss>(`/gloss/${encodeURIComponent(word)}`),
|
glossWord: (word: string) => req<Gloss>(`/gloss/${encodeURIComponent(word)}`),
|
||||||
|
// The same lookup pointing the other way: a Chinese word to its pinyin and
|
||||||
|
// English senses, for an account learning the pair language rather than
|
||||||
|
// English. A word the dictionary has no headword for comes back with empty
|
||||||
|
// readings and — when its characters are known — a per-character reading
|
||||||
|
// instead, which is a real second answer for a compound.
|
||||||
|
hanziWord: (word: string) => req<HanziInfo>(`/hanzi/${encodeURIComponent(word)}`),
|
||||||
// Tone-rewrite: rewrites a selected passage in the given style ('natural',
|
// Tone-rewrite: rewrites a selected passage in the given style ('natural',
|
||||||
// 'academic', …) and returns the rewritten text for an in-editor preview. Not
|
// 'academic', …) and returns the rewritten text for an in-editor preview. Not
|
||||||
// persisted — the editor applies it directly on accept.
|
// persisted — the editor applies it directly on accept.
|
||||||
|
|||||||
@@ -0,0 +1,117 @@
|
|||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||||
|
import { docLang, nativeLang, speak, stopSpeech } from './speech'
|
||||||
|
import { resetPackForTests, setPackLang } from '../i18n'
|
||||||
|
|
||||||
|
// Read-aloud has two jobs beyond "make a sound": ask for the right pace, and ask
|
||||||
|
// in the right language. Both are decided at the call site and travel in the
|
||||||
|
// request body, so this checks the body — the part a component author can get
|
||||||
|
// wrong without anything failing loudly.
|
||||||
|
|
||||||
|
let bodies: Array<Record<string, unknown>>
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
bodies = []
|
||||||
|
vi.stubGlobal(
|
||||||
|
'fetch',
|
||||||
|
vi.fn((_url: string, init: RequestInit) => {
|
||||||
|
bodies.push(JSON.parse(String(init.body)))
|
||||||
|
// Never resolves to audio: the fallback path needs no window.Audio here,
|
||||||
|
// and rejecting would run the Web Speech branch instead of the server one.
|
||||||
|
return new Promise(() => {})
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
stopSpeech()
|
||||||
|
vi.unstubAllGlobals()
|
||||||
|
resetPackForTests()
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('speak', () => {
|
||||||
|
it('asks for the normal pace by default', () => {
|
||||||
|
speak('reception')
|
||||||
|
expect(bodies).toHaveLength(1)
|
||||||
|
expect(bodies[0]).toMatchObject({ text: 'reception', lang: 'en-US', slow: false })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('asks for the slow replay when the slow control is used', () => {
|
||||||
|
speak('reception', undefined, true)
|
||||||
|
expect(bodies[0]).toMatchObject({ text: 'reception', slow: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('still detects Chinese by script, so a zh selection is never read in English', () => {
|
||||||
|
speak('你好世界')
|
||||||
|
expect(bodies[0]).toMatchObject({ lang: 'zh-CN' })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('sends nothing for empty text', () => {
|
||||||
|
speak(' ')
|
||||||
|
expect(bodies).toHaveLength(0)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('nativeLang', () => {
|
||||||
|
// The voice for her own language comes from the pack, not from the letters.
|
||||||
|
// "comum" is spelled the same in both halves of the pt pair, so a detector
|
||||||
|
// would have to guess; the component that knows it is rendering her language
|
||||||
|
// says so instead.
|
||||||
|
it('follows the pair language', () => {
|
||||||
|
setPackLang('zh')
|
||||||
|
expect(nativeLang()).toBe('zh-CN')
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(nativeLang()).toBe('pt-PT')
|
||||||
|
setPackLang('fr')
|
||||||
|
expect(nativeLang()).toBe('fr-FR')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('names a European Portuguese voice, never a Brazilian one', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(nativeLang()).not.toBe('pt-BR')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('is what a Latin-pair lookup speaks the other reading in', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
speak('comum', nativeLang())
|
||||||
|
expect(bodies[0]).toMatchObject({ text: 'comum', lang: 'pt-PT' })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('speaks the French reading of a collision in French', () => {
|
||||||
|
// "chat" is the sharpest case in the fr pair: an English word, a French
|
||||||
|
// word, and the companion's own animal. Nothing about the letters says
|
||||||
|
// which — only the pack does.
|
||||||
|
setPackLang('fr')
|
||||||
|
speak('chat', nativeLang())
|
||||||
|
expect(bodies.at(-1)).toMatchObject({ text: 'chat', lang: 'fr-FR' })
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('docLang', () => {
|
||||||
|
// The document's verdict is the answer for a passage lifted out of it, because
|
||||||
|
// for a Latin pair there is no other answer available: an English sentence and
|
||||||
|
// a Portuguese one are the same letters.
|
||||||
|
it('reads a flipped document in her own language', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(docLang('Ontem foi difícil.', 'pair')).toBe('pt-PT')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reads an English document in English, and treats the backfill as English', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(docLang('Yesterday was hard.', 'en')).toBe('en-US')
|
||||||
|
expect(docLang('Yesterday was hard.', '')).toBe('en-US')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('lets the script win over the verdict, so quoted Chinese is never spelled out', () => {
|
||||||
|
// An English document quoting Chinese is 'en' by verdict, and the English
|
||||||
|
// voice reads Han characters one "Chinese letter" at a time — the one
|
||||||
|
// failure worse than silence.
|
||||||
|
setPackLang('zh')
|
||||||
|
expect(docLang('你好世界', 'en')).toBe('zh-CN')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('is what the editor selection and the garden card ask for', () => {
|
||||||
|
setPackLang('fr')
|
||||||
|
speak('Le chat dort.', docLang('Le chat dort.', 'pair'))
|
||||||
|
expect(bodies.at(-1)).toMatchObject({ text: 'Le chat dort.', lang: 'fr-FR' })
|
||||||
|
})
|
||||||
|
})
|
||||||
+44
-9
@@ -6,6 +6,8 @@
|
|||||||
// (TTS disabled) or unreachable, we fall back to the browser's Web Speech API so
|
// (TTS disabled) or unreachable, we fall back to the browser's Web Speech API so
|
||||||
// the buttons still do something. No model or network is strictly required.
|
// the buttons still do something. No model or network is strictly required.
|
||||||
|
|
||||||
|
import { pack } from '../i18n'
|
||||||
|
|
||||||
// speechSupported reports whether read-aloud can do anything at all. Audio
|
// speechSupported reports whether read-aloud can do anything at all. Audio
|
||||||
// playback is universal, so as long as we can construct an Audio element OR the
|
// playback is universal, so as long as we can construct an Audio element OR the
|
||||||
// Web Speech API exists, the buttons should show. The server path is tried at
|
// Web Speech API exists, the buttons should show. The server path is tried at
|
||||||
@@ -51,8 +53,10 @@ function pickVoice(lang: string): SpeechSynthesisVoice | undefined {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// speakWebSpeech is the fallback: the browser's built-in synthesizer. A touch
|
// speakWebSpeech is the fallback: the browser's built-in synthesizer. A touch
|
||||||
// slower than default so learners can follow along.
|
// slower than default so learners can follow along, and slower still when the
|
||||||
function speakWebSpeech(text: string, lang: string): void {
|
// slow replay was asked for — the fallback should degrade in voice quality, not
|
||||||
|
// in what the button does.
|
||||||
|
function speakWebSpeech(text: string, lang: string, slow: boolean): void {
|
||||||
if (!webSpeechSupported()) return
|
if (!webSpeechSupported()) return
|
||||||
const synth = window.speechSynthesis
|
const synth = window.speechSynthesis
|
||||||
synth.cancel()
|
synth.cancel()
|
||||||
@@ -60,7 +64,7 @@ function speakWebSpeech(text: string, lang: string): void {
|
|||||||
utterance.lang = lang
|
utterance.lang = lang
|
||||||
const voice = pickVoice(lang)
|
const voice = pickVoice(lang)
|
||||||
if (voice) utterance.voice = voice
|
if (voice) utterance.voice = voice
|
||||||
utterance.rate = 0.95
|
utterance.rate = slow ? 0.7 : 0.95
|
||||||
synth.speak(utterance)
|
synth.speak(utterance)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,13 +78,44 @@ export function detectLang(text: string): string {
|
|||||||
return CJK.test(text) ? 'zh-CN' : 'en-US'
|
return CJK.test(text) ? 'zh-CN' : 'en-US'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// nativeLang is the locale of the writer's own language — the voice for the
|
||||||
|
// *other* reading of a word that exists in both halves of a Latin pair.
|
||||||
|
//
|
||||||
|
// It is asked for explicitly rather than detected, and that is the point. A
|
||||||
|
// script boundary can be detected (the CJK test above); "comum" cannot. So the
|
||||||
|
// component that knows it is rendering her language says so, and everything
|
||||||
|
// rendering English lets the default stand. No guess, therefore no wrong guess
|
||||||
|
// about her writing — the same rule the both-directions gloss follows.
|
||||||
|
export function nativeLang(): string {
|
||||||
|
return pack().locale
|
||||||
|
}
|
||||||
|
|
||||||
|
// docLang turns a document-language verdict ('' | 'en' | 'pair', decided
|
||||||
|
// server-side — see internal/suggestions/doclang.go) into a locale for a passage
|
||||||
|
// taken out of that document. It is what the editor's read-aloud and the garden's
|
||||||
|
// review card ask instead of guessing.
|
||||||
|
//
|
||||||
|
// The script test still wins, and that is not redundant with the verdict. A
|
||||||
|
// Chinese sentence quoted inside an English document is 'en' by verdict and
|
||||||
|
// still has to be read by the Chinese voice: the English voice spells Han
|
||||||
|
// characters out one "Chinese letter" at a time, which is the one failure loud
|
||||||
|
// enough to be worse than no audio. In the other direction there is nothing to
|
||||||
|
// test — an English sentence inside Portuguese prose looks exactly like the
|
||||||
|
// Portuguese around it — so the document's verdict is the only answer available,
|
||||||
|
// and it is the answer this phase decided on.
|
||||||
|
export function docLang(text: string, verdict: string): string {
|
||||||
|
if (CJK.test(text)) return 'zh-CN'
|
||||||
|
return verdict === 'pair' ? pack().locale : 'en-US'
|
||||||
|
}
|
||||||
|
|
||||||
// speak reads `text` aloud, cancelling anything already in flight so rapid taps
|
// speak reads `text` aloud, cancelling anything already in flight so rapid taps
|
||||||
// don't queue up. `lang` defaults to a guess from the text (Chinese vs English)
|
// don't queue up. `lang` defaults to a guess from the text (Chinese vs English)
|
||||||
// so callers can just pass the selection; pass an explicit locale to override.
|
// so callers can just pass the selection; pass an explicit locale to override.
|
||||||
// It tries the server's neural voice first and silently falls back to the browser
|
// `slow` asks for the stretched replay (SUGGESTIONS §5e) — the second tap on a
|
||||||
// voice if that's unavailable (route off, network error, or a 404 for a language
|
// sentence that went by too fast. It tries the server's neural voice first and
|
||||||
// with no configured voice).
|
// silently falls back to the browser voice if that's unavailable (route off,
|
||||||
export function speak(text: string, lang = detectLang(text)): void {
|
// network error, or a 404 for a language with no configured voice).
|
||||||
|
export function speak(text: string, lang = detectLang(text), slow = false): void {
|
||||||
if (!text.trim()) return
|
if (!text.trim()) return
|
||||||
stopSpeech()
|
stopSpeech()
|
||||||
const seq = ++requestSeq
|
const seq = ++requestSeq
|
||||||
@@ -88,7 +123,7 @@ export function speak(text: string, lang = detectLang(text)): void {
|
|||||||
fetch('/api/tts', {
|
fetch('/api/tts', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({ text, lang }),
|
body: JSON.stringify({ text, lang, slow }),
|
||||||
})
|
})
|
||||||
.then((res) => {
|
.then((res) => {
|
||||||
if (!res.ok) throw new Error(`tts ${res.status}`)
|
if (!res.ok) throw new Error(`tts ${res.status}`)
|
||||||
@@ -115,6 +150,6 @@ export function speak(text: string, lang = detectLang(text)): void {
|
|||||||
// Server TTS unavailable for this request — use the browser voice instead,
|
// Server TTS unavailable for this request — use the browser voice instead,
|
||||||
// unless a newer tap has already superseded this one.
|
// unless a newer tap has already superseded this one.
|
||||||
if (seq !== requestSeq) return
|
if (seq !== requestSeq) return
|
||||||
speakWebSpeech(text, lang)
|
speakWebSpeech(text, lang, slow)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import type { SaveStatus } from '../../hooks/useAutoSave'
|
|||||||
import { useCompanion, type Mood } from './useCompanion'
|
import { useCompanion, type Mood } from './useCompanion'
|
||||||
import { LottiePlayer } from './LottiePlayer'
|
import { LottiePlayer } from './LottiePlayer'
|
||||||
import { COMPANIONS, DEFAULT_COMPANION } from './companions'
|
import { COMPANIONS, DEFAULT_COMPANION } from './companions'
|
||||||
|
import { useCardOverlap } from './useCardOverlap'
|
||||||
import { onPrefsScopeChange, readPref, writePref } from '../../lib/prefs'
|
import { onPrefsScopeChange, readPref, writePref } from '../../lib/prefs'
|
||||||
import { usePack } from '../../i18n'
|
import { usePack } from '../../i18n'
|
||||||
|
|
||||||
@@ -13,6 +14,11 @@ interface Props {
|
|||||||
editTick: number
|
editTick: number
|
||||||
acceptTick: number
|
acceptTick: number
|
||||||
text: string
|
text: string
|
||||||
|
// The open document is still empty — the one state the daily writing
|
||||||
|
// invitation is offered in.
|
||||||
|
blankPage: boolean
|
||||||
|
// Called when she takes the invitation up, with the English prompt.
|
||||||
|
onAcceptInvitation: (prompt: string) => void
|
||||||
}
|
}
|
||||||
|
|
||||||
// Emoji placeholder per mood, used for any mood a companion has no Lottie for.
|
// Emoji placeholder per mood, used for any mood a companion has no Lottie for.
|
||||||
@@ -30,17 +36,38 @@ const STORAGE_KEY = 'petal.companion'
|
|||||||
// useCompanion and shows a Mandarin-first speech bubble for cheers, tips, and
|
// useCompanion and shows a Mandarin-first speech bubble for cheers, tips, and
|
||||||
// break reminders. Clicking the mascot opens a picker to switch companions
|
// break reminders. Clicking the mascot opens a picker to switch companions
|
||||||
// (the choice persists in localStorage).
|
// (the choice persists in localStorage).
|
||||||
export function PetalCompanion({ wordCount, saveStatus, llmDown, editTick, acceptTick, text }: Props) {
|
export function PetalCompanion({
|
||||||
const t = usePack()
|
|
||||||
const { mood, bubble, dismiss, holdBubble, releaseBubble } = useCompanion({
|
|
||||||
wordCount,
|
wordCount,
|
||||||
saveStatus,
|
saveStatus,
|
||||||
llmDown,
|
llmDown,
|
||||||
editTick,
|
editTick,
|
||||||
acceptTick,
|
acceptTick,
|
||||||
text,
|
text,
|
||||||
|
blankPage,
|
||||||
|
onAcceptInvitation,
|
||||||
|
}: Props) {
|
||||||
|
const t = usePack()
|
||||||
|
const {
|
||||||
|
mood,
|
||||||
|
bubble,
|
||||||
|
dismiss,
|
||||||
|
holdBubble,
|
||||||
|
releaseBubble,
|
||||||
|
acceptInvite,
|
||||||
|
declineInvite,
|
||||||
|
setInviteHandler,
|
||||||
|
} = useCompanion({
|
||||||
|
wordCount,
|
||||||
|
saveStatus,
|
||||||
|
llmDown,
|
||||||
|
editTick,
|
||||||
|
acceptTick,
|
||||||
|
text,
|
||||||
|
blankPage,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
useEffect(() => setInviteHandler(onAcceptInvitation), [setInviteHandler, onAcceptInvitation])
|
||||||
|
|
||||||
const [companionId, setCompanionId] = useState<string>(
|
const [companionId, setCompanionId] = useState<string>(
|
||||||
() => readPref(STORAGE_KEY) || DEFAULT_COMPANION,
|
() => readPref(STORAGE_KEY) || DEFAULT_COMPANION,
|
||||||
)
|
)
|
||||||
@@ -60,6 +87,20 @@ export function PetalCompanion({ wordCount, saveStatus, llmDown, editTick, accep
|
|||||||
const companion = COMPANIONS.find((c) => c.id === companionId) ?? COMPANIONS[0]
|
const companion = COMPANIONS.find((c) => c.id === companionId) ?? COMPANIONS[0]
|
||||||
const [pickerOpen, setPickerOpen] = useState(false)
|
const [pickerOpen, setPickerOpen] = useState(false)
|
||||||
const rootRef = useRef<HTMLDivElement>(null)
|
const rootRef = useRef<HTMLDivElement>(null)
|
||||||
|
const badgeRef = useRef<HTMLButtonElement>(null)
|
||||||
|
// Stand-in for the badge's corner, used only for measuring — see
|
||||||
|
// useCardOverlap. It sits exactly where the badge sits but never bobs, shrinks
|
||||||
|
// or hovers, so what the mascot yields to can't depend on whether it is
|
||||||
|
// currently yielding.
|
||||||
|
const probeRef = useRef<HTMLSpanElement>(null)
|
||||||
|
|
||||||
|
// When suggestion cards stack down into the corner, the kitten fades to
|
||||||
|
// translucent and shrinks a step so the card stays readable and clickable.
|
||||||
|
// It wakes back up whenever it has something to say (bubble) or is being
|
||||||
|
// interacted with (picker open) — except under an open History or Garden
|
||||||
|
// panel, where even a cheer would cover the controls she just reached for.
|
||||||
|
const crowded = useCardOverlap(probeRef)
|
||||||
|
const faded = crowded.modal || (crowded.cards && !pickerOpen && !bubble)
|
||||||
|
|
||||||
// Awake companions (no sleeping clip) don't visibly nap — when the engine
|
// Awake companions (no sleeping clip) don't visibly nap — when the engine
|
||||||
// dozes them, keep their normal idle pose instead of a sleepy face. Only a
|
// dozes them, keep their normal idle pose instead of a sleepy face. Only a
|
||||||
@@ -99,7 +140,7 @@ export function PetalCompanion({ wordCount, saveStatus, llmDown, editTick, accep
|
|||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
ref={rootRef}
|
ref={rootRef}
|
||||||
className="pointer-events-none fixed bottom-4 right-4 z-40 flex flex-col items-end gap-2"
|
className="petal-corner pointer-events-none fixed bottom-4 right-4 z-40 flex flex-col items-end gap-2"
|
||||||
>
|
>
|
||||||
{pickerOpen && (
|
{pickerOpen && (
|
||||||
<div
|
<div
|
||||||
@@ -150,7 +191,10 @@ export function PetalCompanion({ wordCount, saveStatus, llmDown, editTick, accep
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{bubble && !pickerOpen && (
|
{/* The bubble is its own layer, so fading the badge doesn't hide it —
|
||||||
|
hold it back explicitly while a panel is open. useCompanion keeps the
|
||||||
|
bubble in state, so it reappears when she closes the panel. */}
|
||||||
|
{bubble && !pickerOpen && !crowded.modal && (
|
||||||
<div
|
<div
|
||||||
role="status"
|
role="status"
|
||||||
onClick={dismiss}
|
onClick={dismiss}
|
||||||
@@ -179,15 +223,58 @@ export function PetalCompanion({ wordCount, saveStatus, llmDown, editTick, accep
|
|||||||
>
|
>
|
||||||
{bubble.en}
|
{bubble.en}
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
|
{/* The daily invitation's two answers. "Not today" is a real button
|
||||||
|
sitting level with the other one, not a small grey escape — a no
|
||||||
|
that has to be hunted for isn't much of a no. */}
|
||||||
|
{bubble.invite && (
|
||||||
|
<div className="mt-3 flex flex-wrap items-center gap-2">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation()
|
||||||
|
acceptInvite(bubble.invite!.prompt)
|
||||||
|
}}
|
||||||
|
className="rounded-full px-3.5 py-1.5 text-sm font-bold"
|
||||||
|
style={{ background: 'var(--color-accent)', color: 'var(--color-plum)' }}
|
||||||
|
>
|
||||||
|
{t.companion.inviteAccept}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation()
|
||||||
|
declineInvite()
|
||||||
|
}}
|
||||||
|
className="rounded-full px-3.5 py-1.5 text-sm font-semibold"
|
||||||
|
style={{ background: 'var(--color-surface-alt)', color: 'var(--color-plum)' }}
|
||||||
|
>
|
||||||
|
{t.companion.inviteDecline}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
<span
|
||||||
|
ref={probeRef}
|
||||||
|
aria-hidden
|
||||||
|
className="pointer-events-none absolute bottom-0 right-0"
|
||||||
|
style={{
|
||||||
|
width: 'var(--petal-companion-size, 9rem)',
|
||||||
|
height: 'var(--petal-companion-size, 9rem)',
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
|
||||||
<button
|
<button
|
||||||
|
ref={badgeRef}
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => setPickerOpen((o) => !o)}
|
onClick={() => setPickerOpen((o) => !o)}
|
||||||
title="Choose a companion"
|
title="Choose a companion"
|
||||||
aria-label="Choose a companion"
|
aria-label="Choose a companion"
|
||||||
className={`petal-companion pointer-events-auto select-none ${napping ? 'petal-companion-sleep' : ''}`}
|
className={`petal-companion select-none ${faded ? 'petal-companion-faded' : 'pointer-events-auto'}${
|
||||||
|
napping ? ' petal-companion-sleep' : ''
|
||||||
|
}`}
|
||||||
style={{
|
style={{
|
||||||
// Size scales with the viewport — see --petal-companion-size in index.css.
|
// Size scales with the viewport — see --petal-companion-size in index.css.
|
||||||
width: 'var(--petal-companion-size)',
|
width: 'var(--petal-companion-size)',
|
||||||
|
|||||||
@@ -0,0 +1,261 @@
|
|||||||
|
import { afterEach, describe, expect, it } from 'vitest'
|
||||||
|
import { analyzeProse, mechanicsFindings } from './prose'
|
||||||
|
import { resetPackForTests, setPackLang } from '../../i18n'
|
||||||
|
|
||||||
|
// Phase 22's offline half: the grammar-lite rule pack, the embedded
|
||||||
|
// miscollocation list, the false-friend heads-up, and the per-pair L1 rules.
|
||||||
|
//
|
||||||
|
// The bar these tests enforce is the one the pack promises: **precision over
|
||||||
|
// recall**. Every rule is pinned in two directions — the mistake it must catch,
|
||||||
|
// and the correct English next to it that it must leave alone. A rule that
|
||||||
|
// cannot be guarded that way was left out of the pack rather than tested
|
||||||
|
// loosely here.
|
||||||
|
|
||||||
|
afterEach(() => resetPackForTests())
|
||||||
|
|
||||||
|
const rules = (text: string) => analyzeProse(text).map((h) => h.rule)
|
||||||
|
const findings = (text: string) => mechanicsFindings(text)
|
||||||
|
|
||||||
|
// The one-click fix a given rule produced, if any.
|
||||||
|
function fix(text: string, original: string) {
|
||||||
|
return findings(text).find((f) => f.original.toLowerCase() === original.toLowerCase())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every finding must be able to anchor: its span has to be exactly the text it
|
||||||
|
// claims, or the editor applies the edit to the wrong characters.
|
||||||
|
function expectExactSpans(text: string) {
|
||||||
|
for (const f of findings(text)) {
|
||||||
|
expect(text.slice(f.from, f.to), `span mismatch for "${f.original}"`).toBe(f.original)
|
||||||
|
}
|
||||||
|
return findings(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('prepositions', () => {
|
||||||
|
it('depend of → depend on, keeping the writer\'s own verb form', () => {
|
||||||
|
expect(fix('It all depends of the weather on the day we travel.', 'depends of')?.replacement).toBe(
|
||||||
|
'depends on',
|
||||||
|
)
|
||||||
|
expect(fix('Depending of the weather we will go to the beach today.', 'Depending of')?.replacement).toBe(
|
||||||
|
'Depending on',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('discuss about → discuss (the preposition simply goes)', () => {
|
||||||
|
expect(fix('We discussed about the plan for a long time yesterday.', 'discussed about')?.replacement).toBe(
|
||||||
|
'discussed',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('explain me → explain to me', () => {
|
||||||
|
expect(fix('Can you explain me the rules of this game again please.', 'explain me')?.replacement).toBe(
|
||||||
|
'explain to me',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('listen the radio → listen to the radio', () => {
|
||||||
|
expect(fix('I listen the radio every morning while I make my coffee.', 'listen the')?.replacement).toBe(
|
||||||
|
'listen to the',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
// The pairings deliberately NOT in the list, because they are only usually
|
||||||
|
// wrong. Each of these is correct English and must stay silent.
|
||||||
|
it('leaves the correct prepositions alone', () => {
|
||||||
|
expect(rules('It all depends on the weather on the day we travel.')).not.toContain('preposition')
|
||||||
|
expect(rules('We discussed the plan for a long time yesterday afternoon.')).not.toContain('preposition')
|
||||||
|
expect(rules('I listen to the radio every morning while I make coffee.')).not.toContain('preposition')
|
||||||
|
// Left out of the pack on purpose — "married with children" is a phrase,
|
||||||
|
// "arrive to" wants at or in, "different than" is ordinary American usage.
|
||||||
|
expect(rules('She is married with children and lives near the old harbour.')).not.toContain('preposition')
|
||||||
|
expect(rules('This result is different than the one we saw last week.')).not.toContain('preposition')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('doubled comparatives', () => {
|
||||||
|
it('more better → better', () => {
|
||||||
|
expect(fix('This one is more better than the other one we tried.', 'more better')?.replacement).toBe('better')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('most easiest → easiest', () => {
|
||||||
|
expect(fix('That was the most easiest question on the whole exam paper.', 'most easiest')?.replacement).toBe(
|
||||||
|
'easiest',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
// The guard the generic /\w+er/ pattern would have failed: these are correct.
|
||||||
|
it('leaves ordinary "more/most + adjective" alone', () => {
|
||||||
|
expect(rules('She is more clever than anyone else in the whole class.')).not.toContain('doublecomp')
|
||||||
|
expect(rules('He was the most eager student in the room that morning.')).not.toContain('doublecomp')
|
||||||
|
expect(rules('This is the most beautiful garden I have ever seen here.')).not.toContain('doublecomp')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('people is', () => {
|
||||||
|
it('people is → people are, and people has → people have', () => {
|
||||||
|
expect(fix('Many people is waiting outside the hall in the rain.', 'people is')?.replacement).toBe('people are')
|
||||||
|
expect(fix('Some people has never seen the sea in their whole life.', 'people has')?.replacement).toBe(
|
||||||
|
'people have',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('leaves the plural alone', () => {
|
||||||
|
expect(rules('Many people are waiting outside the hall in the rain.')).not.toContain('peopleare')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('miscollocations', () => {
|
||||||
|
// The whole point of the family: these file as 'collocation', not
|
||||||
|
// 'mechanics', so an accepted chunk plants in the vocabulary garden exactly
|
||||||
|
// as one the LLM coach proposed would.
|
||||||
|
it('files as the collocation family, not as mechanics', () => {
|
||||||
|
const f = fix('I had to do a decision about the job offer quickly.', 'do a decision')
|
||||||
|
expect(f?.replacement).toBe('make a decision')
|
||||||
|
expect(f?.type).toBe('collocation')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('mechanics fixes keep their own family', () => {
|
||||||
|
expect(fix('I saw the the cat in the garden this morning.', 'the the')?.type).toBe('mechanics')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('agrees with the tense the writer was already using', () => {
|
||||||
|
expect(fix('She did a mistake on the form and had to start again.', 'did a mistake')?.replacement).toBe(
|
||||||
|
'made a mistake',
|
||||||
|
)
|
||||||
|
expect(fix('He is making his homework at the kitchen table right now.', 'making his homework')?.replacement).toBe(
|
||||||
|
'doing his homework',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('say me → tell me', () => {
|
||||||
|
expect(fix('Please say me what happened at the meeting this afternoon.', 'say me')?.replacement).toBe('tell me')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('make a photo → take a photo', () => {
|
||||||
|
expect(fix('We made a photo together in front of the old church.', 'made a photo')?.replacement).toBe(
|
||||||
|
'took a photo',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('strong rain → heavy rain', () => {
|
||||||
|
expect(fix('There was strong rain all afternoon and we stayed inside.', 'strong rain')?.replacement).toBe(
|
||||||
|
'heavy rain',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
// "strong wind" is the correct pairing, and the rule that fixes "big wind"
|
||||||
|
// must not propose it as a change to itself.
|
||||||
|
it('never proposes a phrase identical to what she wrote', () => {
|
||||||
|
expect(rules('There was a strong wind blowing across the open field today.')).not.toContain('collocation')
|
||||||
|
expect(fix('There was a big wind blowing across the open field today.', 'big wind')?.replacement).toBe(
|
||||||
|
'strong wind',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('leaves the correct pairings alone', () => {
|
||||||
|
expect(rules('I had to make a decision about the job offer quickly.')).not.toContain('collocation')
|
||||||
|
expect(rules('She does her homework at the kitchen table every evening.')).not.toContain('collocation')
|
||||||
|
expect(rules('We took a photo together in front of the old church.')).not.toContain('collocation')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('false friends', () => {
|
||||||
|
it('flags a Portuguese false friend for the pt-PT pair, awareness-only', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
const hints = analyzeProse('I will eventually finish the report before the end of the week.')
|
||||||
|
const ff = hints.find((h) => h.rule === 'falsefriend')
|
||||||
|
expect(ff).toBeDefined()
|
||||||
|
// Never a card: the word may well be the one she meant, and a one-click
|
||||||
|
// "fix" would be Petal deciding that for her.
|
||||||
|
expect(ff?.fix).toBeUndefined()
|
||||||
|
expect(findings('I will eventually finish the report before the end of the week.')).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('raises at most one per pass — a heads-up, not a sweep', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
const hints = analyzeProse(
|
||||||
|
'Actually I did not pretend to assist the lecture at the library this week.',
|
||||||
|
)
|
||||||
|
expect(hints.filter((h) => h.rule === 'falsefriend')).toHaveLength(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('says nothing for the zh pair, which has no false friends at all', () => {
|
||||||
|
setPackLang('zh')
|
||||||
|
expect(rules('I will eventually finish the report before the end of the week.')).not.toContain('falsefriend')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('per-pair L1 interference', () => {
|
||||||
|
it('pt-PT: "have 30 years" → "am 30 years old"', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(fix('My sister I have 30 years and she is older than me.', 'I have 30 years')?.replacement).toBe(
|
||||||
|
'I am 30 years old',
|
||||||
|
)
|
||||||
|
// The subject and tense she wrote in are carried into the correction.
|
||||||
|
expect(fix('When we met she had twenty years and I was still at school.', 'she had twenty years')?.replacement).toBe(
|
||||||
|
'she was twenty years old',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('pt-PT: "I am agree" → "I agree"', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(fix('I am agree with everything that was said at the meeting.', 'I am agree')?.replacement).toBe('I agree')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('pt-PT: "since three years" → "for three years"', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(fix('I have lived in this city since three years and I love it.', 'since three years')?.replacement).toBe(
|
||||||
|
'for three years',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('pt-PT: leaves "since" with a starting point alone', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(rules('I have lived in this city since 2020 and I still love it.')).not.toContain('since')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('zh: "very like" → "really like", and "open the light" → "turn on the light"', () => {
|
||||||
|
setPackLang('zh')
|
||||||
|
expect(fix('I very like the small garden behind my grandmother house.', 'very like')?.replacement).toBe(
|
||||||
|
'really like',
|
||||||
|
)
|
||||||
|
expect(fix('Please open the light before you come into the dark room.', 'open the light')?.replacement).toBe(
|
||||||
|
'turn on the light',
|
||||||
|
)
|
||||||
|
expect(fix('She closed the television and went straight to bed last night.', 'closed the television')?.replacement).toBe(
|
||||||
|
'turned off the television',
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('zh: although…but is awareness-only — the "but" is too common to anchor a card to', () => {
|
||||||
|
setPackLang('zh')
|
||||||
|
const text = 'Although it was raining hard, but we still went to the park.'
|
||||||
|
expect(rules(text)).toContain('althoughbut')
|
||||||
|
expect(findings(text).some((f) => f.original.includes('but'))).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('zh: leaves "very + adjective" alone', () => {
|
||||||
|
setPackLang('zh')
|
||||||
|
expect(rules('I am very happy about the small garden behind the house.')).not.toContain('veryverb')
|
||||||
|
})
|
||||||
|
|
||||||
|
// The gating is the reason these rules can be confident. A rule that is a
|
||||||
|
// near-certainty for one L1 is only a guess for another, and a guess does not
|
||||||
|
// belong in a rule pack that runs on every keystroke.
|
||||||
|
it('does not run one pair\'s interference rules for the other pair', () => {
|
||||||
|
setPackLang('zh')
|
||||||
|
expect(rules('I have 30 years and I still live near the old harbour.')).not.toContain('haveyears')
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(rules('I very like the small garden behind my grandmother house.')).not.toContain('veryverb')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('spans stay exact across every new rule', () => {
|
||||||
|
it('anchors each finding to the text it claims', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expectExactSpans(
|
||||||
|
'I have 30 years and I am agree that we depends of the weather, and she did a mistake since three years.',
|
||||||
|
)
|
||||||
|
setPackLang('zh')
|
||||||
|
expectExactSpans('I very like to open the light, and many people is more better at it than me.')
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
import { readFileSync } from 'node:fs'
|
||||||
|
import { gunzipSync } from 'node:zlib'
|
||||||
|
import { describe, expect, it } from 'vitest'
|
||||||
|
|
||||||
|
import { CONFUSION_PAIRS, hanziFindings } from './hanzi'
|
||||||
|
import { buildSegmenter } from '../../lib/segment'
|
||||||
|
|
||||||
|
// The 错别字 pack, held to the bar Phase 22 set for the English rule pack: every
|
||||||
|
// rule pinned in *two* directions — the mistake it must catch, and the correct
|
||||||
|
// writing next to it that it must leave alone.
|
||||||
|
//
|
||||||
|
// Here the second direction is the one that matters, and it is unusually easy to
|
||||||
|
// get wrong. Chinese has no spaces, so every one of these rules is a substring
|
||||||
|
// match on running text, and for most of them there exists an ordinary correct
|
||||||
|
// sentence that contains the substring across a word boundary. Those sentences
|
||||||
|
// are the real test.
|
||||||
|
|
||||||
|
const raw = gunzipSync(readFileSync(new URL('../../../public/dictionaries/zh/words.txt.gz', import.meta.url)))
|
||||||
|
const seg = buildSegmenter(raw.toString('utf8'))
|
||||||
|
|
||||||
|
const flagged = (text: string) => hanziFindings(text, seg).map((f) => `${f.original}→${f.replacement}`)
|
||||||
|
|
||||||
|
describe('the gate that admits a rule', () => {
|
||||||
|
// The pack's own claim about itself, checked against the shipped dictionary
|
||||||
|
// rather than asserted in a comment. A pair whose wrong form is a real word
|
||||||
|
// cannot be decided mechanically and does not belong here.
|
||||||
|
it('every wrong form is not a word, and every right form is', () => {
|
||||||
|
for (const { wrong, right } of CONFUSION_PAIRS) {
|
||||||
|
expect(seg.has(wrong), `${wrong} is a dictionary word and must not be flagged`).toBe(false)
|
||||||
|
expect(seg.has(right), `${right} is not a dictionary word`).toBe(true)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// The errors this pack deliberately refuses, and why — each is a genuine
|
||||||
|
// mistake by a modern standard whose wrong form is itself a headword. If a
|
||||||
|
// dictionary rebuild ever drops one of these, this test fails and the pair
|
||||||
|
// becomes admissible; that is the intended way to find out.
|
||||||
|
it('refuses the well-known errors it cannot decide', () => {
|
||||||
|
for (const undecidable of ['自已', '好象', '倒底', '帐号', '部份']) {
|
||||||
|
expect(seg.has(undecidable), `${undecidable} is no longer a word — reconsider the rule`).toBe(true)
|
||||||
|
expect(flagged(`这是${undecidable}的例子`)).toEqual([])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('the mistakes it catches', () => {
|
||||||
|
it('已 / 己 / 以', () => {
|
||||||
|
expect(flagged('我己经写完了作业')).toEqual(['己经→已经'])
|
||||||
|
expect(flagged('我以经吃过饭了')).toEqual(['以经→已经'])
|
||||||
|
expect(flagged('下课已后我们去公园')).toEqual(['已后→以后'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('在 / 再', () => {
|
||||||
|
expect(flagged('明天在见')).toEqual(['在见→再见'])
|
||||||
|
expect(flagged('他正再看书')).toEqual(['正再→正在'])
|
||||||
|
expect(flagged('现再几点了')).toEqual(['现再→现在'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('做 / 作', () => {
|
||||||
|
expect(flagged('我的工做很忙')).toEqual(['工做→工作'])
|
||||||
|
expect(flagged('老师给我们很多做业')).toEqual(['做业→作业'])
|
||||||
|
expect(flagged('这本书的做者是谁')).toEqual(['做者→作者'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the rest', () => {
|
||||||
|
expect(flagged('我觉的这个很好')).toEqual(['觉的→觉得'])
|
||||||
|
expect(flagged('你因该早点睡')).toEqual(['因该→应该'])
|
||||||
|
expect(flagged('即然你来了就坐下吧')).toEqual(['即然→既然'])
|
||||||
|
expect(flagged('你知到吗')).toEqual(['知到→知道'])
|
||||||
|
expect(flagged('请输入你的蜜码')).toEqual(['蜜码→密码'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reports an exact span, so the card replaces the right characters', () => {
|
||||||
|
const text = '我己经到了'
|
||||||
|
const [f] = hanziFindings(text, seg)
|
||||||
|
expect(text.slice(f.from, f.to)).toBe('己经')
|
||||||
|
expect(text.slice(0, f.from) + f.replacement + text.slice(f.to)).toBe('我已经到了')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('finds every occurrence, in document order', () => {
|
||||||
|
expect(flagged('我己经吃了,他也己经吃了')).toEqual(['己经→已经', '己经→已经'])
|
||||||
|
expect(flagged('我的工做很忙,所以我觉的很累')).toEqual(['工做→工作', '觉的→觉得'])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
// ── the direction that matters ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
describe('the correct writing it must not touch', () => {
|
||||||
|
// Each of these is an ordinary sentence containing a flagged substring across
|
||||||
|
// a word boundary. Without the boundary gate, every one would be corrupted —
|
||||||
|
// and corrupted silently, into text that is still made of real characters.
|
||||||
|
it('leaves two real words alone where they happen to abut', () => {
|
||||||
|
// 自己 + 经常. The substring is 己经.
|
||||||
|
expect(flagged('他自己经常做饭')).toEqual([])
|
||||||
|
// 睡觉 + 的. The substring is 觉的.
|
||||||
|
expect(flagged('睡觉的时候不要看手机')).toEqual([])
|
||||||
|
// 感觉 + 的.
|
||||||
|
expect(flagged('这是我感觉的方向')).toEqual([])
|
||||||
|
// 不知 + 到底.
|
||||||
|
expect(flagged('我不知到底该怎么办')).toEqual([])
|
||||||
|
// 因 + 位置.
|
||||||
|
expect(flagged('因位置不好我们换了座位')).toEqual([])
|
||||||
|
// 已 + 后悔.
|
||||||
|
expect(flagged('他已后悔了')).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('leaves ordinary correct prose entirely alone', () => {
|
||||||
|
for (const good of [
|
||||||
|
'我今天早上去公园跑步了',
|
||||||
|
'他的中文说得很好',
|
||||||
|
'我已经完成了我的作业',
|
||||||
|
'现在几点了,我们再见面吧',
|
||||||
|
'我觉得这个工作很有意思',
|
||||||
|
'既然你已经知道了,就按照计划做',
|
||||||
|
]) {
|
||||||
|
expect(flagged(good), good).toEqual([])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Where the gate costs the pack a real catch, and the trade it is making.
|
||||||
|
// 不知 is itself a word, so 我不知到他在哪里 — which really is 知到 for 知道 —
|
||||||
|
// reads to the segmenter as 不知 + 到 and is left alone. That is the gate
|
||||||
|
// preferring a missed error to a corrupted sentence, which is the whole
|
||||||
|
// premise: 我不知到底该怎么办 is the same three characters and is correct.
|
||||||
|
it('declines a real error rather than risk the sentence beside it', () => {
|
||||||
|
expect(flagged('我不知到他在哪里')).toEqual([])
|
||||||
|
expect(flagged('你知到吗')).toEqual(['知到→知道'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('says nothing about English, or about nothing', () => {
|
||||||
|
expect(flagged('I already finished my homework')).toEqual([])
|
||||||
|
expect(flagged('')).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
// The direction gate. The word list is loaded only for an account learning
|
||||||
|
// Chinese, so without one this pack is silent — a writer practising English
|
||||||
|
// must never be told her own quoted Chinese is wrong.
|
||||||
|
it('is silent without a segmenter, which is how the direction gate works', () => {
|
||||||
|
expect(hanziFindings('我己经写完了', null)).toEqual([])
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
import type { MechanicsFinding } from '../../api/client'
|
||||||
|
import type { Segmenter } from '../../lib/segment'
|
||||||
|
|
||||||
|
// 错别字 — wrong-character detection, the Chinese counterpart of the spell
|
||||||
|
// checker, and a different problem from the one Hunspell solves.
|
||||||
|
//
|
||||||
|
// Chinese has no misspellings in the English sense: every character a writer can
|
||||||
|
// type is a real character, correctly formed, and an IME will not offer one that
|
||||||
|
// is not. What it *will* offer is the wrong one. Typing pinyin `yijing` and
|
||||||
|
// taking the first candidate gives 已经 or 己经 depending on the moment, and both
|
||||||
|
// are made of real characters. So the unit of error is not a malformed word but
|
||||||
|
// a **substituted character inside a correct-looking one** — which is why this
|
||||||
|
// is a rule pack over confusable pairs rather than a dictionary membership test.
|
||||||
|
//
|
||||||
|
// The discipline is Phase 22's, and the bar is the same: **precision over
|
||||||
|
// recall**. A wrong nudge costs more trust than a missed one earns, and it costs
|
||||||
|
// double here, because a learner has no way to know the tool is wrong. Two
|
||||||
|
// mechanical gates enforce it, and both are checked in the tests rather than
|
||||||
|
// asserted in prose.
|
||||||
|
|
||||||
|
// A confusable pair: `wrong` is never a word, `right` is what was meant.
|
||||||
|
//
|
||||||
|
// **Gate one — the pair must be decidable by the dictionary.** Each entry is
|
||||||
|
// admitted only if `wrong` is absent from the 188k-word list *and* `right` is
|
||||||
|
// present. That is what makes the correction a fact rather than a preference,
|
||||||
|
// and it is checked against the shipped asset in hanzi.test.ts.
|
||||||
|
//
|
||||||
|
// It is also the gate that keeps out errors everyone knows are errors. 自已 for
|
||||||
|
// 自己 is among the commonest slips in written Chinese, and 自已 is itself a
|
||||||
|
// dictionary headword — so this pack does not flag it, exactly as Phase 22's
|
||||||
|
// English pack left out `married with`. The same fate for 好象 (an older form of
|
||||||
|
// 好像, still in the dictionary), 倒底, 帐号 and 部份: all real errors by a modern
|
||||||
|
// standard, none of them decidable here.
|
||||||
|
interface Confusion {
|
||||||
|
wrong: string
|
||||||
|
right: string
|
||||||
|
// The note on the card. English, because this pack only ever runs for a writer
|
||||||
|
// whose English is the language they think in — see the direction gate below.
|
||||||
|
why: string
|
||||||
|
}
|
||||||
|
|
||||||
|
const CONFUSIONS: Confusion[] = [
|
||||||
|
// 已 / 己 / 以 — three characters that differ by one stroke and share a
|
||||||
|
// syllable. The most productive source of 错别字 there is.
|
||||||
|
{ wrong: '己经', right: '已经', why: '已经 (already) — 己 is the "self" character; the one you want is 已.' },
|
||||||
|
{ wrong: '以经', right: '已经', why: '已经 (already) — 以 is a different word; 已 is the one that means "already".' },
|
||||||
|
{ wrong: '已后', right: '以后', why: '以后 (afterwards) takes 以, not 已.' },
|
||||||
|
|
||||||
|
// 在 / 再 — same pinyin (zài), completely different jobs: one is location and
|
||||||
|
// ongoing action, the other is repetition.
|
||||||
|
{ wrong: '在见', right: '再见', why: '再见 (goodbye) — 再 is "again", which is what "see you again" needs.' },
|
||||||
|
{ wrong: '正再', right: '正在', why: '正在 (in the middle of doing) takes 在, the one about being somewhere.' },
|
||||||
|
{ wrong: '现再', right: '现在', why: '现在 (now) takes 在.' },
|
||||||
|
|
||||||
|
// 做 / 作 — both zuò, both "to do", and which one a compound takes is simply
|
||||||
|
// fixed by convention. A learner cannot reason it out, which is what makes a
|
||||||
|
// reminder worth having.
|
||||||
|
{ wrong: '工做', right: '工作', why: '工作 (work) is written with 作.' },
|
||||||
|
{ wrong: '做业', right: '作业', why: '作业 (homework) is written with 作.' },
|
||||||
|
{ wrong: '做者', right: '作者', why: '作者 (author) is written with 作.' },
|
||||||
|
{ wrong: '做文', right: '作文', why: '作文 (an essay) is written with 作.' },
|
||||||
|
{ wrong: '做用', right: '作用', why: '作用 (effect, function) is written with 作.' },
|
||||||
|
|
||||||
|
// 得 / 的 — the pair everyone knows about. Only the fixed compound is flagged:
|
||||||
|
// deciding 的 against 地 against 得 in the general case needs to know whether
|
||||||
|
// the next word is a verb or a noun, which nothing here can tell.
|
||||||
|
{ wrong: '觉的', right: '觉得', why: '觉得 (to feel, to think) ends in 得.' },
|
||||||
|
|
||||||
|
// 即 / 既 — one stroke apart, opposite meanings ("namely" against "since").
|
||||||
|
{ wrong: '即然', right: '既然', why: '既然 (since, given that) takes 既.' },
|
||||||
|
{ wrong: '既使', right: '即使', why: '即使 (even if) takes 即.' },
|
||||||
|
|
||||||
|
// The rest: ordinary IME slips where the wrong character is a homophone.
|
||||||
|
{ wrong: '因该', right: '应该', why: '应该 (should) — 因 means "because"; the word you want starts with 应.' },
|
||||||
|
{ wrong: '因位', right: '因为', why: '因为 (because) ends in 为.' },
|
||||||
|
{ wrong: '知到', right: '知道', why: '知道 (to know) ends in 道.' },
|
||||||
|
{ wrong: '安照', right: '按照', why: '按照 (according to) takes 按.' },
|
||||||
|
{ wrong: '蜜码', right: '密码', why: '密码 (password) takes 密 — 蜜 is honey.' },
|
||||||
|
{ wrong: '犹其', right: '尤其', why: '尤其 (especially) takes 尤.' },
|
||||||
|
{ wrong: '甘净', right: '干净', why: '干净 (clean) takes 干.' },
|
||||||
|
{ wrong: '什末', right: '什么', why: '什么 (what) ends in 么.' },
|
||||||
|
{ wrong: '一像', right: '一样', why: '一样 (the same) ends in 样 — 像 is "to resemble".' },
|
||||||
|
{ wrong: '必须品', right: '必需品', why: '必需品 (a necessity) takes 需. 必须 is "must", which is a different word.' },
|
||||||
|
]
|
||||||
|
|
||||||
|
// **Gate two — the characters must not already belong to two different words.**
|
||||||
|
//
|
||||||
|
// This is the gate that stops the pack from destroying correct writing, and
|
||||||
|
// without it every rule above is dangerous. 自己经常 ("oneself, often") contains
|
||||||
|
// the string 己经. 睡觉的时候 ("when sleeping") contains 觉的. 不知到底 contains 知到.
|
||||||
|
// A substring match would corrupt all three.
|
||||||
|
//
|
||||||
|
// The segmenter already knows the difference, so the test is: split the text,
|
||||||
|
// and if the two characters land in different tokens *and* either token is a
|
||||||
|
// real multi-character word, this is a word boundary and not an error. Two
|
||||||
|
// adjacent single-character tokens is what the walk produces when it has nothing
|
||||||
|
// better to offer — which is exactly what a mistyped compound looks like.
|
||||||
|
function isWordBoundary(tokens: { word: string; from: number; to: number }[], at: number): boolean {
|
||||||
|
const left = tokens.find((t) => at >= t.from && at < t.to)
|
||||||
|
const right = tokens.find((t) => at + 1 >= t.from && at + 1 < t.to)
|
||||||
|
if (!left || !right || left === right) return false
|
||||||
|
return left.word.length > 1 || right.word.length > 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// hanziFindings returns the 错别字 in a piece of text, as ordinary mechanics
|
||||||
|
// findings — the same shape, the same rail, the same cards, the same accept.
|
||||||
|
//
|
||||||
|
// It needs the segmenter and does nothing without one, which is also the
|
||||||
|
// direction gate: the word list is loaded only for an account learning Chinese
|
||||||
|
// (useSegmenter), so a writer practising English can never be told her quoted
|
||||||
|
// Chinese is wrong. That is not a nicety. Petal deliberately never corrects the
|
||||||
|
// pair language — the fr and es dictionaries are chosen to hold every variety
|
||||||
|
// precisely so they cannot underline correct writing — and a Mandarin native
|
||||||
|
// does not need her own language checked by a rule pack of two dozen entries.
|
||||||
|
export function hanziFindings(text: string, segmenter: Segmenter | null): MechanicsFinding[] {
|
||||||
|
if (!segmenter || !text) return []
|
||||||
|
// One segmentation for the whole text, shared by every rule. The walk is
|
||||||
|
// linear, but running it two dozen times over a long document would not be.
|
||||||
|
const tokens = segmenter.segment(text)
|
||||||
|
const found: MechanicsFinding[] = []
|
||||||
|
|
||||||
|
for (const c of CONFUSIONS) {
|
||||||
|
let from = text.indexOf(c.wrong)
|
||||||
|
while (from !== -1) {
|
||||||
|
// The boundary test is asked at the seam the substitution sits on: the
|
||||||
|
// gap between the first two characters, which is where a mistyped
|
||||||
|
// compound and two adjacent words look different from each other.
|
||||||
|
if (!isWordBoundary(tokens, from)) {
|
||||||
|
found.push({
|
||||||
|
from,
|
||||||
|
to: from + c.wrong.length,
|
||||||
|
original: c.wrong,
|
||||||
|
replacement: c.right,
|
||||||
|
explanation: c.why,
|
||||||
|
type: 'mechanics',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
from = text.indexOf(c.wrong, from + 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Document order, so the rail reads down the page rather than down this file.
|
||||||
|
return found.sort((a, b) => a.from - b.from)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exported for the tests, which check every pair against the shipped word list.
|
||||||
|
// A pack whose own gate is only described in a comment is a pack whose gate can
|
||||||
|
// rot; this is how the description is made to stay true.
|
||||||
|
export const CONFUSION_PAIRS = CONFUSIONS.map((c) => ({ wrong: c.wrong, right: c.right }))
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
|
|
||||||
|
import { markInvited, mayInvite, todayKey } from './invitation'
|
||||||
|
import { resetPackForTests, setPackLang } from '../../i18n'
|
||||||
|
import { zh } from '../../i18n/packs/zh'
|
||||||
|
import { ptPT } from '../../i18n/packs/pt-PT'
|
||||||
|
import { declined, invitations } from './tips'
|
||||||
|
|
||||||
|
// The suite runs without a DOM, so localStorage is stubbed the same way
|
||||||
|
// prefs.test.ts does it.
|
||||||
|
function fakeStorage(): Storage {
|
||||||
|
const map = new Map<string, string>()
|
||||||
|
return {
|
||||||
|
get length() {
|
||||||
|
return map.size
|
||||||
|
},
|
||||||
|
key: (i: number) => [...map.keys()][i] ?? null,
|
||||||
|
getItem: (k: string) => map.get(k) ?? null,
|
||||||
|
setItem: (k: string, v: string) => void map.set(k, v),
|
||||||
|
removeItem: (k: string) => void map.delete(k),
|
||||||
|
clear: () => map.clear(),
|
||||||
|
} as Storage
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(() => vi.stubGlobal('localStorage', fakeStorage()))
|
||||||
|
afterEach(() => {
|
||||||
|
vi.unstubAllGlobals()
|
||||||
|
resetPackForTests()
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('once a day', () => {
|
||||||
|
it('offers, then does not offer again the same day', () => {
|
||||||
|
expect(mayInvite()).toBe(true)
|
||||||
|
markInvited()
|
||||||
|
expect(mayInvite()).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('offers again the next day', () => {
|
||||||
|
const monday = new Date(2026, 6, 27, 10, 0)
|
||||||
|
const tuesday = new Date(2026, 6, 28, 9, 0)
|
||||||
|
markInvited(monday)
|
||||||
|
expect(mayInvite(monday)).toBe(false)
|
||||||
|
expect(mayInvite(tuesday)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
// The whole promise of §5c: nothing is counting. A month away has to look
|
||||||
|
// exactly like a day away, because the alternative is a streak, and a streak
|
||||||
|
// punishes exactly the person this feature is for.
|
||||||
|
it('treats a month away the same as a day away', () => {
|
||||||
|
const june = new Date(2026, 5, 1, 10, 0)
|
||||||
|
const july = new Date(2026, 6, 27, 10, 0)
|
||||||
|
markInvited(june)
|
||||||
|
expect(mayInvite(july)).toBe(true)
|
||||||
|
// And after being asked again, still only ever one stored value: a date.
|
||||||
|
markInvited(july)
|
||||||
|
expect(localStorage.getItem('petal.invited')).toBe(todayKey(july))
|
||||||
|
// One stored value, and it is a date. Nothing accumulates.
|
||||||
|
expect(localStorage.length).toBe(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('uses the writer\'s local day, so a late night and the small hours differ', () => {
|
||||||
|
const lateNight = new Date(2026, 6, 27, 23, 30)
|
||||||
|
const smallHours = new Date(2026, 6, 28, 1, 15)
|
||||||
|
markInvited(lateNight)
|
||||||
|
expect(mayInvite(smallHours)).toBe(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('the invitation copy', () => {
|
||||||
|
const PACKS = [
|
||||||
|
{ name: 'zh', pack: zh },
|
||||||
|
{ name: 'pt-PT', pack: ptPT },
|
||||||
|
]
|
||||||
|
|
||||||
|
it('every pack offers something to write about, and a way to say no', () => {
|
||||||
|
for (const { name, pack } of PACKS) {
|
||||||
|
expect(pack.companion.invitations.length, name).toBeGreaterThan(3)
|
||||||
|
expect(pack.companion.inviteAccept.trim(), name).not.toBe('')
|
||||||
|
expect(pack.companion.inviteDecline.trim(), name).not.toBe('')
|
||||||
|
expect(pack.companion.declined.native.trim(), name).not.toBe('')
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// The copy is bound by the same rule as the timing: no streaks, no guilt, no
|
||||||
|
// counting of days. This is the part a well-meaning future edit would undo —
|
||||||
|
// "day 4 in a row!" is a natural thing to write and the wrong thing to say.
|
||||||
|
it('never invokes a streak, a target, or a missed day', () => {
|
||||||
|
const forbidden =
|
||||||
|
/streak|in a row|every day|don't break|dont break|missed|behind|连续|打卡|坚持|seguidos|todos os dias|falhaste/i
|
||||||
|
for (const { name, pack } of PACKS) {
|
||||||
|
const copy = [
|
||||||
|
...pack.companion.invitations.flatMap((l) => [l.native, l.en]),
|
||||||
|
pack.companion.inviteAccept,
|
||||||
|
pack.companion.inviteDecline,
|
||||||
|
pack.companion.declined.native,
|
||||||
|
pack.companion.declined.en,
|
||||||
|
]
|
||||||
|
for (const line of copy) {
|
||||||
|
expect(line, `${name}: ${line}`).not.toMatch(forbidden)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reads from the pair in force, like every other companion line', () => {
|
||||||
|
setPackLang('pt-PT')
|
||||||
|
expect(invitations()).toBe(ptPT.companion.invitations)
|
||||||
|
expect(declined()).toBe(ptPT.companion.declined)
|
||||||
|
setPackLang('zh')
|
||||||
|
expect(invitations()).toBe(zh.companion.invitations)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
// When the companion may offer its daily invitation to write.
|
||||||
|
//
|
||||||
|
// A handful of lines, pulled out of the timing engine on purpose: this is the
|
||||||
|
// part of §5c with the ethics in it, and it should be readable and testable on
|
||||||
|
// its own rather than buried in a heartbeat.
|
||||||
|
//
|
||||||
|
// The rule is a *date*, not a count and not a run of days. Petal remembers the
|
||||||
|
// last day it asked and nothing else — so there is no streak to break, no tally
|
||||||
|
// of days missed, and nothing that gets worse for being away a week. Coming
|
||||||
|
// back after a month looks exactly like coming back tomorrow, which is the only
|
||||||
|
// version of this feature worth shipping to someone learning a language.
|
||||||
|
//
|
||||||
|
// Either answer spends the day's invitation. Being asked again after saying no
|
||||||
|
// would turn "not today" into a negotiation.
|
||||||
|
|
||||||
|
import { readPref, writePref } from '../../lib/prefs'
|
||||||
|
|
||||||
|
const INVITED_KEY = 'petal.invited'
|
||||||
|
|
||||||
|
// The local calendar day. Local rather than UTC because "today" is the writer's
|
||||||
|
// day: a nudge at 11pm and another at 1am would otherwise be two different days.
|
||||||
|
export function todayKey(now = new Date()): string {
|
||||||
|
const pad = (n: number) => String(n).padStart(2, '0')
|
||||||
|
return `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())}`
|
||||||
|
}
|
||||||
|
|
||||||
|
// mayInvite reports whether today's invitation is still unspent.
|
||||||
|
export function mayInvite(now = new Date()): boolean {
|
||||||
|
return readPref(INVITED_KEY) !== todayKey(now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// markInvited spends it — called when the invitation is *offered*, not when it
|
||||||
|
// is accepted, because declining has to count too.
|
||||||
|
export function markInvited(now = new Date()): void {
|
||||||
|
writePref(INVITED_KEY, todayKey(now))
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user