Compare commits
71
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8f2ad34a10 | ||
|
|
5659312358 | ||
|
|
a216614c81 | ||
|
|
c348a9b8ae | ||
|
|
77f284f65c | ||
|
|
9224c44fff | ||
|
|
39d4e4770a | ||
|
|
bd92cdc9b6 | ||
|
|
1acc23244e | ||
|
|
40de65b3d1 | ||
|
|
b2d50e9136 | ||
|
|
1aa3a14030 | ||
|
|
978cb80642 | ||
|
|
f082a930cb | ||
|
|
ce5de68b9b | ||
|
|
b23c5a9a13 | ||
|
|
25e415daa2 | ||
|
|
3bcc967f51 | ||
|
|
963fc1754d | ||
|
|
de251ceae2 | ||
|
|
10e8aef86c | ||
|
|
c33de1175b | ||
|
|
ba06d904f0 | ||
|
|
ea14eb5e88 | ||
|
|
aac15b5ac5 | ||
|
|
be9aa13287 | ||
|
|
ec9fba9252 | ||
|
|
9c40a8ad3f | ||
|
|
ac1c6cddb0 | ||
|
|
3e714b6f00 | ||
|
|
69bf3ffde1 | ||
|
|
9a0edd6679 | ||
|
|
be1ab5cef7 | ||
|
|
071ea7b835 | ||
|
|
9a2e909b85 | ||
|
|
1f4ca4775a | ||
|
|
1bbc8fc8d3 | ||
|
|
e9b8595456 | ||
|
|
7b845644be | ||
|
|
3b714e297a | ||
|
|
24c3533e18 | ||
|
|
ccb43e5a4d | ||
|
|
4de83d0da5 | ||
|
|
74bf600593 | ||
|
|
86175f1559 | ||
|
|
3640ce9324 | ||
|
|
97e9c269ec | ||
|
|
336cae93e0 | ||
|
|
30d5e691c9 | ||
|
|
ddc4164228 | ||
|
|
84ee6bfb9c | ||
|
|
151df4565b | ||
|
|
1b4a5f26df | ||
|
|
e2f967c92b | ||
|
|
6d71276513 | ||
|
|
1cf207d73f | ||
|
|
42d857a878 | ||
|
|
33e49ddb62 | ||
|
|
1d76ab1c82 | ||
|
|
623bd02b9c | ||
|
|
d01a0f1f0a | ||
|
|
5b221cc7a3 | ||
|
|
2363ef2d37 | ||
|
|
df6bc4989c | ||
|
|
8410b6315b | ||
|
|
dae1213c68 | ||
|
|
316b6b305d | ||
|
|
023882a722 | ||
|
|
6901cdbbe4 | ||
|
|
61b3c6cd62 | ||
|
|
78ed1dd281 |
@@ -0,0 +1,33 @@
|
||||
# Keep the build context small and the image reproducible. Anything the build
|
||||
# needs but that is gitignored (web/dist) is produced inside the image instead.
|
||||
|
||||
.git
|
||||
.gitignore
|
||||
.env
|
||||
|
||||
# Built by stage 1 — never copy a stale local build into the image.
|
||||
web/dist
|
||||
web/node_modules
|
||||
|
||||
# Local runtime state: the live database, images and TTS cache must never end
|
||||
# up baked into an image layer.
|
||||
data/
|
||||
*.db
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
backups/
|
||||
|
||||
# Local build outputs
|
||||
/petal
|
||||
*.test
|
||||
*.out
|
||||
*.log
|
||||
|
||||
# Docs and tooling that don't affect the binary
|
||||
*.md
|
||||
!web/**/*.md
|
||||
deploy/
|
||||
scripts/
|
||||
.vscode/
|
||||
.idea/
|
||||
.DS_Store
|
||||
+36
-5
@@ -10,6 +10,12 @@ DATABASE_PATH=./data/petal.db
|
||||
# On-disk store for images pasted/dropped/inserted in the editor
|
||||
IMAGE_DIR=./data/images
|
||||
|
||||
# DreamDict's built dictionary (French, European Portuguese, Spanish, Mandarin),
|
||||
# opened read-only beside petal.db. Optional: with no file here, word lookups use
|
||||
# the embedded English/Chinese datasets, which is how a laptop checkout runs.
|
||||
# Build one with `go run ./cmd/dictimport` in the dreamdict repo.
|
||||
DICT_PATH=./data/dict.db
|
||||
|
||||
# LLM
|
||||
LLM_BACKEND=vllm # vllm | ollama
|
||||
LLM_ENDPOINT=http://localhost:8000 # vLLM :8000, Ollama :11434
|
||||
@@ -27,17 +33,42 @@ TTS_ENDPOINT= # e.g. http://127.0.0.1:5005 — empty disable
|
||||
TTS_ENDPOINT_ZH= # e.g. http://127.0.0.1:5006 — Chinese Piper instance
|
||||
TTS_VOICE_EN=en_US-amy-medium # Piper voice id for English
|
||||
TTS_VOICE_ZH=zh_CN-huayan-medium # Piper voice id for Chinese
|
||||
TTS_PATH=/ # path Piper serves synthesis on: "/" up to piper-tts 1.5, "/synthesize" from 1.6.0
|
||||
TTS_CACHE_DIR=./data/tts # on-disk store for synthesized clips (content-addressed)
|
||||
TTS_TIMEOUT=15s
|
||||
TTS_AUDIO_FORMAT=mp3 # mp3 | opus | wav — mp3/opus transcode Piper's WAV via ffmpeg
|
||||
|
||||
# --- Deferred (not wired in the local-dev build) ---
|
||||
|
||||
# Auth (Authentik OIDC) — deferred; single hardcoded local user for now
|
||||
# SESSION_SECRET=change-me-to-random-64-char-string
|
||||
# AUTHENTIK_URL=https://auth.parodia.dev
|
||||
# --- Auth (Authentik OIDC) ---
|
||||
#
|
||||
# Login turns on only when the issuer, client id and secret are all set. Leave
|
||||
# them commented out for local development and Petal runs as the single
|
||||
# hardcoded `local` user, exactly as it did before auth landed.
|
||||
#
|
||||
# That fallback is scoped to development on purpose. With a BASE_URL naming
|
||||
# anything but localhost, Petal refuses to start rather than run open — see
|
||||
# PETAL_REQUIRE_AUTH — because the fallback on a reachable host means every
|
||||
# anonymous visitor is the `local` user, with full read and write over every
|
||||
# document in the database.
|
||||
#
|
||||
# AUTHENTIK_URL is the issuer of the Petal provider in Authentik (the value of
|
||||
# its "OpenID Configuration Issuer" field). The redirect URI to register there
|
||||
# is BASE_URL + /auth/callback.
|
||||
# AUTHENTIK_URL=https://auth.parodia.dev/application/o/petal/
|
||||
# AUTHENTIK_CLIENT_ID=petal
|
||||
# AUTHENTIK_CLIENT_SECRET=
|
||||
#
|
||||
# Who may sign in: comma-separated OIDC subject ids and/or email addresses.
|
||||
# Empty = anyone Authentik authenticates, which is right for a single-household
|
||||
# instance and wrong the moment the IdP serves a wider audience than Petal. An
|
||||
# empty list is warned about at every boot rather than assumed either way.
|
||||
# PETAL_ALLOWED_SUBS=her@example.com,me@example.com
|
||||
#
|
||||
# Whether a missing OIDC configuration is fatal. Defaults to false for a
|
||||
# loopback BASE_URL and true for anything else, so neither a laptop nor a
|
||||
# deployment normally has to name it.
|
||||
# PETAL_REQUIRE_AUTH=true
|
||||
|
||||
# --- Deferred (not wired in the local-dev build) ---
|
||||
|
||||
# Copyleaks (Tier-2 plagiarism) — deferred; needs a public webhook
|
||||
# COPYLEAKS_ENABLED=false
|
||||
|
||||
@@ -10,6 +10,10 @@ web/dist/*
|
||||
!web/dist/.gitkeep
|
||||
*.log
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Local env & data
|
||||
.env
|
||||
*.db
|
||||
|
||||
+313
-3
File diff suppressed because one or more lines are too long
+70
@@ -0,0 +1,70 @@
|
||||
# Petal — multi-stage build producing the single self-contained binary.
|
||||
#
|
||||
# Stage 1 builds the frontend; stage 2 compiles the Go server with web/dist
|
||||
# embedded (go:embed), so the runtime image carries one executable and no
|
||||
# assets. modernc's SQLite is pure Go, so CGO stays off and the binary is
|
||||
# static — the runtime layer exists only for ffmpeg (read-aloud transcodes
|
||||
# Piper's WAV to mp3) and CA certificates.
|
||||
|
||||
# ---------- stage 1: frontend ----------
|
||||
FROM node:22-alpine AS web
|
||||
|
||||
WORKDIR /src/web
|
||||
|
||||
# Install deps against the lockfile alone so this layer caches across source
|
||||
# edits. The Hunspell dictionaries come from a devDependency, so a plain
|
||||
# `npm ci` (not --omit=dev) is required for the spell checker to ship.
|
||||
COPY web/package.json web/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY web/ ./
|
||||
RUN npm run build
|
||||
|
||||
# ---------- stage 2: server ----------
|
||||
FROM golang:1.25-alpine AS build
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
# The build context's web/dist is gitignored and excluded by .dockerignore;
|
||||
# take the freshly built one from stage 1 so go:embed picks it up.
|
||||
COPY --from=web /src/web/dist ./web/dist
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/petal ./cmd/server
|
||||
|
||||
# ---------- stage 3: runtime ----------
|
||||
FROM alpine:3.21
|
||||
|
||||
# ffmpeg: read-aloud pipes Piper's WAV through it to mp3/opus. tzdata: the
|
||||
# companion's bedtime nag and night mode read the local clock, so the container
|
||||
# needs a real timezone rather than bare UTC.
|
||||
RUN apk add --no-cache ca-certificates ffmpeg tzdata \
|
||||
&& adduser -D -u 10001 petal
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/petal /app/petal
|
||||
|
||||
# Mount point for petal.db (+ -wal/-shm), the image store, the TTS cache and
|
||||
# DreamDict's read-only dict.db. dict.db is deployed alongside rather than baked
|
||||
# in: it is ~450 MB, changes a few times a year, and is shared with other
|
||||
# services on the host — putting it in the image would multiply it by every tag.
|
||||
RUN mkdir -p /data && chown -R petal:petal /data
|
||||
VOLUME ["/data"]
|
||||
|
||||
USER petal
|
||||
EXPOSE 8080
|
||||
|
||||
ENV PORT=8080 \
|
||||
DATABASE_PATH=/data/petal.db \
|
||||
IMAGE_DIR=/data/images \
|
||||
TTS_CACHE_DIR=/data/tts \
|
||||
DICT_PATH=/data/dict.db
|
||||
|
||||
# Same endpoint Traefik and the uptime probe use; needs no session by design.
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD wget -qO- http://127.0.0.1:8080/api/health || exit 1
|
||||
|
||||
ENTRYPOINT ["/app/petal"]
|
||||
@@ -0,0 +1,390 @@
|
||||
# Petal multi-user plan
|
||||
|
||||
**Status:** Phase 0 (identity plumbing) landed 2026-07-26 in `6901cdb`.
|
||||
**Phase A (authentication) + Phase C's image store built 2026-07-27** — in-app
|
||||
OIDC, server-side sessions, allowlist, provisioning, the frontend 401 path and
|
||||
per-owner images. Not yet configured against the live Authentik; see
|
||||
`BUILD_PLAN.md` Phase 16 and `deploy/README.md` §4. Phase B (migrating the
|
||||
`local` user) still waits on her first real login.
|
||||
**All OPEN decisions ratified by the user 2026-07-26** (recommendations
|
||||
accepted as written) — see each OPEN for its settled answer. Execution phases
|
||||
live in `BUILD_PLAN.md` (Phase 15 onward); product rationale for the language
|
||||
work is in `SUGGESTIONS.md`.
|
||||
|
||||
Settled context that postdates the original draft: Petal will be hosted on the
|
||||
**parodia.dev VPS**, reaching vLLM on millenia over **headscale VPN**; Piper
|
||||
TTS is already installed on parodia (VPS-local, no VPN hop). The LLM is the
|
||||
only cross-VPN dependency, and per the LLM-minimalism principle
|
||||
(SUGGESTIONS.md §6) it must never gate essential functionality.
|
||||
|
||||
---
|
||||
|
||||
## 1. Where Petal is today
|
||||
|
||||
Single user, by construction. `db.Open` seeds one row (`users.id = 'local'`) and,
|
||||
until this week, every query named that constant directly.
|
||||
|
||||
What was already right: **the schema has been multi-user-shaped from day one.**
|
||||
`documents`, `tags`, and `vocab_words` all carry `user_id`; `document_versions`
|
||||
and `suggestions` scope through their parent document. No migration is needed to
|
||||
support a second user — only a way to know which user is asking.
|
||||
|
||||
### What Phase 0 changed
|
||||
|
||||
- New `internal/auth`: `Middleware(Resolver)` resolves the caller once per API
|
||||
request and stores the id in the request context. Handlers read
|
||||
`auth.UserID(r.Context())`.
|
||||
- `Resolver` is a one-method interface — `Resolve(*http.Request) (string, error)`
|
||||
— and is the only thing a real identity provider has to implement.
|
||||
- `StaticResolver(db.LocalUserID)` supplies today's single user, so behavior is
|
||||
unchanged.
|
||||
- `/api` is split into a public group (`/health`, `/version`) and an
|
||||
authenticated group (everything else).
|
||||
- Two pre-existing access-control gaps fixed: `setStatus` (accept/dismiss) had
|
||||
**no ownership check at all**, and `fetchPending` read suggestions by `doc_id`
|
||||
alone — which leaked the quoted source sentences.
|
||||
- Two-user isolation test suites (`docs`, `suggestions`) mount the same routers
|
||||
twice behind two resolvers over one database.
|
||||
|
||||
**The remaining work is not "make Petal multi-user."** It is "authenticate
|
||||
someone, provision them, and clean up the three places where data is still
|
||||
global."
|
||||
|
||||
---
|
||||
|
||||
## 2. Goals and non-goals
|
||||
|
||||
**Goals**
|
||||
- Two or more people use one Petal instance without seeing each other's writing.
|
||||
- The existing local user's data survives, attached to a real account.
|
||||
- Adding a user is an operator action, not a code change.
|
||||
|
||||
**Non-goals (explicitly out, unless a reviewer argues otherwise)**
|
||||
- Sharing, collaboration, or multi-author documents. Petal is a private writing
|
||||
space; every feature to date assumes one reader. Sharing would change the
|
||||
passport's meaning (authorship evidence) and is a product decision, not an
|
||||
auth one.
|
||||
- Roles, permissions, or an admin UI.
|
||||
- Public signup. Accounts are provisioned deliberately.
|
||||
|
||||
---
|
||||
|
||||
## 3. Phase A — authentication
|
||||
|
||||
### OPEN #1: forward-auth vs. in-app OIDC — **SETTLED: Option B (in-app OIDC)**
|
||||
|
||||
Ratified 2026-07-26. The deciding fact arrived with the deployment plan: Petal
|
||||
will live on the public parodia.dev VPS, which is exactly the environment where
|
||||
Option A's "must never be reachable except through Traefik" invariant is a
|
||||
footgun. Original analysis kept below for the record.
|
||||
|
||||
**Option A — Traefik forward-auth to an Authentik outpost.**
|
||||
Traefik is already in the deferred deploy bucket. Authentik's outpost terminates
|
||||
the login, and Petal receives a trusted header (`X-authentik-uid`, plus email and
|
||||
name). The `Resolver` becomes ~20 lines: read the header, map to a user id.
|
||||
|
||||
- *For:* no OIDC library, no session store, no cookie handling, no redirect
|
||||
plumbing, no token refresh, no logout endpoint. Petal keeps zero auth code.
|
||||
Login/MFA/password reset are entirely Authentik's problem.
|
||||
- *Against:* Petal is only secure if it is **never reachable except through
|
||||
Traefik**. Anyone who can hit the container directly can forge the header and
|
||||
become any user. That's a deployment invariant enforced by network config, not
|
||||
by code — and it is exactly the kind of invariant that quietly breaks. It also
|
||||
makes local development awkward (no proxy → no identity), though
|
||||
`StaticResolver` covers that.
|
||||
|
||||
**Option B — Petal is an OIDC client itself.**
|
||||
`github.com/coreos/go-oidc` + `golang.org/x/oauth2`, a `/auth/callback` route,
|
||||
and a signed session cookie. The config fields already exist
|
||||
(`AUTHENTIK_URL`, `AUTHENTIK_CLIENT_ID`, `AUTHENTIK_CLIENT_SECRET`,
|
||||
`SESSION_SECRET`).
|
||||
|
||||
- *For:* self-contained and safe to expose directly. No trust-the-proxy
|
||||
invariant. Works the same in dev and prod.
|
||||
- *Against:* meaningfully more code — a session table or signed-cookie scheme,
|
||||
CSRF on the callback, token expiry, logout. Two new dependencies in a project
|
||||
that currently has exactly two.
|
||||
|
||||
**My recommendation: Option B**, but not confidently. The deciding factor for me
|
||||
is that "must never be reachable directly" is a footgun that survives long after
|
||||
whoever set it up has forgotten, and Petal already holds someone's private
|
||||
journals. But if the deployment is definitively a single Traefik-fronted box on a
|
||||
LAN and will stay that way, Option A is *much* less code and I'd not object.
|
||||
|
||||
A reviewer should weigh: how likely is this instance ever exposed beyond the LAN?
|
||||
Does the operator want Petal to be independently deployable?
|
||||
|
||||
### Session handling (assumes Option B)
|
||||
|
||||
- Server-side sessions in a `sessions` table (id, user_id, expires_at,
|
||||
created_at, user_agent), cookie holds an opaque random id.
|
||||
Preferred over signed stateless cookies because it makes logout and
|
||||
revocation actually work — worth the one table.
|
||||
- Cookie: `HttpOnly`, `SameSite=Lax`, `Secure` when `BASE_URL` is https.
|
||||
- **OPEN #2 — SETTLED: 30-day sliding expiry** (ratified 2026-07-26). An
|
||||
editor that logs you out mid-draft is hostile, and auto-save makes a
|
||||
surprise 401 genuinely costly. Sliding: each authenticated request extends
|
||||
the session.
|
||||
|
||||
### The 401 problem
|
||||
|
||||
Every frontend fetch currently assumes success. Once a session can expire,
|
||||
**any** call can return 401 mid-session — including the 1.5s auto-save, which is
|
||||
the one that must not fail silently.
|
||||
|
||||
Proposal: a single interceptor in `web/src/api/client.ts` that, on 401, halts
|
||||
auto-save, surfaces a warm bilingual "请重新登录 / Please sign in again" state
|
||||
rather than a raw error, and preserves unsaved editor content across the
|
||||
re-login (localStorage draft keyed by doc id). This is small but easy to forget,
|
||||
and getting it wrong means lost writing.
|
||||
|
||||
---
|
||||
|
||||
## 4. Phase B — user provisioning and migration
|
||||
|
||||
### Provisioning
|
||||
|
||||
On first successful login, upsert a `users` row from the OIDC claims
|
||||
(`sub` → `users.id`, plus email and name). No signup flow; whoever Authentik lets
|
||||
in gets an account.
|
||||
|
||||
**OPEN #3 — SETTLED: yes, allowlist** (ratified 2026-07-26). Authentik may
|
||||
host other applications with a broader user set than Petal should have. Gate
|
||||
via a `PETAL_ALLOWED_SUBS` env var (or an Authentik group claim check —
|
||||
implementer's choice, env var is simpler); a valid login not on the list gets
|
||||
a warm bilingual "this Petal isn't yours to write in" page, not a 500.
|
||||
|
||||
### Migrating the existing `local` user
|
||||
|
||||
The live database on millenia holds real writing under `user_id = 'local'`. That
|
||||
data must end up owned by the wife's real account.
|
||||
|
||||
Recommended: a migration that **renames** rather than copies — update the
|
||||
`users.id` and let `ON UPDATE CASCADE`… except SQLite FKs here are declared
|
||||
without `ON UPDATE`, so this needs either a deliberate multi-table update inside
|
||||
one transaction (`documents`, `tags`, `vocab_words` — versions and suggestions
|
||||
follow their parents) with `PRAGMA foreign_keys=OFF` around it, or an explicit
|
||||
one-off admin command.
|
||||
|
||||
I'd rather do this as a **documented one-off script run with the app stopped and
|
||||
a copy of the DB taken first** than as an automatic startup migration, because it
|
||||
depends on knowing the new OIDC subject id, which isn't available until that
|
||||
person logs in once. Sequence: deploy auth → she logs in → new empty account is
|
||||
created → stop app, back up, run script to move `local`'s rows onto her real id,
|
||||
delete the empty row → restart.
|
||||
|
||||
**OPEN #4 — SETTLED: documented one-off script** (ratified 2026-07-26), run
|
||||
with the app stopped and a DB backup taken first, per the existing `scripts/`
|
||||
convention. No admin endpoint.
|
||||
|
||||
---
|
||||
|
||||
## 5. Phase C — the data that is still global
|
||||
|
||||
Found during the Phase 0 audit. None of these break with two users; all of them
|
||||
leak or bleed.
|
||||
|
||||
### Image store — the real one
|
||||
|
||||
`internal/images` is a flat content-addressed directory. There is no per-user
|
||||
association and no database row at all. Any authenticated user who knows a
|
||||
sha256 can fetch any other user's image.
|
||||
|
||||
That is capability-URL security. Hashes aren't guessable, so this is not an
|
||||
emergency — but "unguessable filename" is not access control, and images pasted
|
||||
into a private journal are exactly the content that shouldn't rely on it.
|
||||
|
||||
Proposal: an `images` table (hash, user_id, content_type, created_at, size) with
|
||||
the fetch handler joining on the caller. Content addressing is kept — the same
|
||||
image uploaded by two users is stored once on disk and simply has two rows, so
|
||||
deduplication survives. Deleting the last row referencing a hash removes the
|
||||
file.
|
||||
|
||||
**OPEN #5 — SETTLED: fix it in the same phase as auth** (ratified
|
||||
2026-07-26). The moment a second account exists the exposure is real, and the
|
||||
fix requires a migration either way.
|
||||
|
||||
### Frontend `localStorage`
|
||||
|
||||
`petal.spell.personal` (personal dictionary), `petal.companion` (chosen mascot),
|
||||
plus sound and petal-effect preferences are all per-browser. Two users on one
|
||||
device share them — and the personal dictionary is the one that matters, since
|
||||
it's built from someone's own writing.
|
||||
|
||||
Cheapest fix: namespace every key by user id once the client knows who it is.
|
||||
The honest fix for the dictionary is to move it server-side into a table, which
|
||||
also means it follows a user between devices — arguably a feature.
|
||||
|
||||
### Not affected
|
||||
|
||||
`export-all` is correctly scoped. The TTS cache is content-addressed audio of
|
||||
text the requester supplied, no cross-user inference. The lexicon is a static
|
||||
dataset identical for everyone.
|
||||
|
||||
---
|
||||
|
||||
## 6. Phase D — per-user language (and DreamDict)
|
||||
|
||||
Tracked here because it lands on the same `users` row and shouldn't be designed
|
||||
twice.
|
||||
|
||||
**English is always the target language.** What varies is the user's *native*
|
||||
language — the one glosses and explanations are written in. Mandarin ships today;
|
||||
European Portuguese (pt-PT, explicitly not pt-BR) is wanted; French is possible.
|
||||
|
||||
### OPEN #6 is answered: DreamDict
|
||||
|
||||
The original worry here was data sourcing — Petal's gloss comes from ECDICT
|
||||
(English↔Chinese), and a pt-PT equivalent of comparable quality and license
|
||||
looked like the blocker.
|
||||
|
||||
`~/git/dreamdict` already solves it, and more completely than expected. It
|
||||
supports **en, fr, pt-PT, and zh** (~136k/56k/136k/121k words), and its shape maps
|
||||
almost 1:1 onto `lexicon.Result`:
|
||||
|
||||
| Petal field | DreamDict |
|
||||
|---|---|
|
||||
| `Gloss` | `Translate(word, "en", L1)` |
|
||||
| `Phonetic` | pronunciation (CMU + IPA for en, Wiktionary IPA elsewhere) |
|
||||
| `Definitions` | `Define(word, lang)` — curated sources ranked above Wiktionary |
|
||||
| `Synonyms` | `Synonyms(word, lang)` |
|
||||
|
||||
It also carries data Petal has no equivalent for and could use: `Antonyms`,
|
||||
`Frequency`, `Difficulty`, and `Etymology`.
|
||||
|
||||
> **Correction (2026-07-27, Phase 20).** The `Gloss` row of that table is wrong.
|
||||
> `Translate(word, "en", L1)` reads Wiktionary's translation sections, which are
|
||||
> thin in the en→X direction: measured on the real `dict.db`, it answers for
|
||||
> **17%** of the 2,000 commonest English words into pt-PT and 16% into fr.
|
||||
> Meaning has to come through shared WordNet synset ids instead (**61%**), which
|
||||
> is what DreamDict's new `Equivalents(word, from, to)` does — falling back to
|
||||
> the translations table, for 62% combined. The 1:1 mapping was assumed from the
|
||||
> API surface and never checked against the data; it did not survive contact
|
||||
> with it. Same measurement on zh reads 53% against ECDICT's near-total coverage
|
||||
> of those words, which is why the zh pair did **not** converge.
|
||||
|
||||
So Phase D stops being gated on data and becomes an integration decision.
|
||||
|
||||
### OPEN #6a (new): how to integrate — **SETTLED: Option 3** (ratified 2026-07-26)
|
||||
|
||||
Import the package, open `dict.db` read-only. Prerequisite stands: DreamDict's
|
||||
module path must be renamed (or `replace`-directed) first — **that change
|
||||
lives in the dreamdict repo, not this one.** The migration caution below also
|
||||
stands: pt-PT/fr wire to DreamDict first; zh stays on ECDICT until compared on
|
||||
real lookups. Options kept below for the record.
|
||||
|
||||
**Option 1 — HTTP client.** Petal calls DreamDict on localhost:7777, exactly the
|
||||
pattern already used for Piper TTS (including graceful degradation when it's
|
||||
down).
|
||||
- *For:* zero coupling, DreamDict updates independently, all endpoints available.
|
||||
- *Against:* a second service Petal now depends on at runtime, and the gloss is a
|
||||
350ms hover tooltip where "the dictionary service is down" is a visible
|
||||
regression from today's always-there embedded data.
|
||||
|
||||
**Option 2 — build-time extraction.** A script (sibling to the existing
|
||||
`scripts/build_gloss.py`) generates Petal's embedded `.json.gz` datasets per
|
||||
language from DreamDict's `dict.db`.
|
||||
- *For:* preserves the embedded/offline property exactly; no runtime dependency;
|
||||
no architectural change at all.
|
||||
- *Against:* every language multiplies the binary (the four current gz files are
|
||||
already ~11.6 MB); updating the dictionary means rebuilding and redeploying
|
||||
Petal; the richer fields are lost unless separately extracted.
|
||||
|
||||
**Option 3 — import the package, open `dict.db` read-only.** DreamDict's
|
||||
`internal/dictionary` is a plain library with `NewReadOnly(dbPath)`, and its only
|
||||
dependency is `modernc.org/sqlite` — the same CGO-free driver Petal already uses.
|
||||
Petal opens `dict.db` as a second read-only handle beside `petal.db`.
|
||||
- *For:* no service, no HTTP, no new dependency, lookups stay local-file fast,
|
||||
all four languages at once, and it deletes ~11.6 MB of embedded gz plus the
|
||||
ECDICT build scripts. One dictionary, maintained once, shared with GogoBee.
|
||||
- *Against:* Petal stops being a self-contained binary in the "just run it" sense
|
||||
— `dict.db` has to be deployed alongside. In practice Petal already ships a
|
||||
data directory (`petal.db`, images, TTS cache), so this is a smaller loss than
|
||||
it first sounds.
|
||||
|
||||
**My recommendation: Option 3.** It is the only one that gets all four languages,
|
||||
keeps lookups offline and instant, and *removes* code rather than adding a
|
||||
subsystem. Option 1's runtime dependency buys flexibility Petal doesn't need for
|
||||
a dictionary that changes a few times a year.
|
||||
|
||||
**Prerequisite:** DreamDict's module path is currently `module dreamdict`, which
|
||||
isn't fetchable. Importing it needs the module renamed to something like
|
||||
`gitea.parodia.dev/drwily/dreamdict` (or a local `replace` directive for
|
||||
development). Small, but it must happen first.
|
||||
|
||||
### Migration caution
|
||||
|
||||
Whichever option wins, the zh path is **currently working and in daily use**. The
|
||||
gloss quality difference between ECDICT and CC-CEDICT is unknown and matters more
|
||||
than the architecture.
|
||||
|
||||
Proposal: introduce DreamDict behind Petal's existing lexicon interface as a
|
||||
*provider*, wire pt-PT and fr to it first (nothing to regress — they don't exist
|
||||
yet), and keep zh on ECDICT until the two have been compared on real lookups from
|
||||
her actual documents. Converge only if quality holds. This also de-risks the whole
|
||||
change: if DreamDict turns out to be a poor fit, only the unshipped languages are
|
||||
affected.
|
||||
|
||||
### Still per-user regardless
|
||||
|
||||
Native language becomes a `users` column, and these become per-user lookups:
|
||||
LLM prompt copy (`internal/llm/prompts.go`, currently Mandarin-first), companion
|
||||
tips (`tips.ts`), the L1 Piper voice (Piper has pt-PT voices), and the CJK font
|
||||
stacks (not needed for Latin-script L1). English-side machinery — nspell en-US,
|
||||
the phonetic dataset, the EN voice — is unaffected and stays shared.
|
||||
|
||||
## 7. Suggested sequence
|
||||
|
||||
1. ~~Settle OPEN #1~~ **Settled: in-app OIDC.**
|
||||
2. Deploy plumbing: Dockerfile, Traefik, real hostname on parodia.dev, HTTPS,
|
||||
headscale route to vLLM on millenia (bound to the headscale interface
|
||||
only), VPS-local Piper, off-VPS DB backup. Auth needs a stable `BASE_URL`
|
||||
and a redirect URI.
|
||||
3. Auth itself: OIDC `Resolver`, sessions table (30-day sliding), allowlist,
|
||||
frontend 401 handling with draft preservation.
|
||||
4. Image store table + migration (same phase, per OPEN #5).
|
||||
5. Provision the second real account; migrate `local`'s data (script, app
|
||||
stopped, backup first).
|
||||
6. `localStorage` namespacing (key by user **and** language — see
|
||||
SUGGESTIONS.md §8).
|
||||
7. Per-user language pair. **No longer gated on data** — DreamDict covers all
|
||||
four languages. Sequence within it: rename DreamDict's module path → wire
|
||||
it in as a lexicon provider → pt-PT/fr first → compare zh quality →
|
||||
converge if it holds. The pair model and langpack shape are specified in
|
||||
SUGGESTIONS.md §1–§3.
|
||||
|
||||
These are expanded into checkboxed execution phases in `BUILD_PLAN.md`
|
||||
(Phase 15 onward) — that file remains the source of truth for progress.
|
||||
|
||||
---
|
||||
|
||||
## 8. Risks
|
||||
|
||||
- **Silent unscoping.** Phase 0 hit this exactly once: `docs.fetch` took a
|
||||
`userID` parameter and kept binding `db.LocalUserID` in the query. Unused
|
||||
parameters are legal Go — it compiled, `vet` was silent, and every existing
|
||||
test passed while the lookup stayed unscoped. Only the two-user isolation test
|
||||
caught it. **Every new user-scoped endpoint should get an isolation case in the
|
||||
same commit**; the existing suites are the template.
|
||||
- **Migrating live data.** The wife's real writing is the thing being moved.
|
||||
Back up first, run with the app stopped, verify counts before deleting
|
||||
anything.
|
||||
- **A 401 mid-draft losing work.** See Phase A.
|
||||
- **Scope creep into sharing.** Multi-user and collaboration are different
|
||||
products. Adding accounts should not quietly become adding sharing.
|
||||
|
||||
---
|
||||
|
||||
## 9. Questions for the reviewer — all answered 2026-07-26
|
||||
|
||||
1. ~~Forward-auth or in-app OIDC?~~ **In-app OIDC** (OPEN #1).
|
||||
2. ~~Session lifetime?~~ **30-day sliding** (OPEN #2).
|
||||
3. ~~Allowlist?~~ **Yes**, `PETAL_ALLOWED_SUBS` or group claim (OPEN #3).
|
||||
4. ~~Script vs. admin endpoint?~~ **Script**, app stopped, backup first (OPEN #4).
|
||||
5. ~~Image store timing?~~ **Same phase as auth** (OPEN #5).
|
||||
6. ~~pt-PT dictionary data?~~ **DreamDict**, integrated per **Option 3**
|
||||
(import package, read-only `dict.db`; module rename is the prerequisite)
|
||||
(OPEN #6a).
|
||||
7. ~~zh gloss regression risk?~~ **zh stays on ECDICT** until compared against
|
||||
DreamDict on real lookups from her actual documents; converge only if
|
||||
quality holds.
|
||||
@@ -32,6 +32,16 @@ cd .. && go build -o petal ./cmd/server
|
||||
|
||||
Configuration is via environment variables — copy `.env.example` to `.env`.
|
||||
|
||||
## Deployment
|
||||
|
||||
```bash
|
||||
docker compose up -d --build # petal + the two Piper read-aloud sidecars
|
||||
```
|
||||
|
||||
Behind Traefik on the parodia.dev VPS; vLLM stays on millenia over headscale.
|
||||
Full runbook — first deploy, the LLM link, backups and restore — in
|
||||
[`deploy/README.md`](./deploy/README.md).
|
||||
|
||||
## Status
|
||||
Early build, multi-session. Auth (Authentik), Copyleaks plagiarism, and Docker/Traefik
|
||||
deployment are deferred — see `BUILD_PLAN.md`.
|
||||
Early build, multi-session. Auth (Authentik OIDC) is next; Copyleaks plagiarism is
|
||||
still parked — see `BUILD_PLAN.md`.
|
||||
|
||||
+364
@@ -0,0 +1,364 @@
|
||||
# Petal — product suggestions: becoming essential for language learners
|
||||
|
||||
**Status:** written 2026-07-26 against `feat/writing-passport`; **ratified by
|
||||
the user 2026-07-26** (recommendations accepted — reviewer Q1–Q3 settled
|
||||
below; Q4–Q6 remain genuinely open and don't block execution). This document
|
||||
is the *why*; the checkboxed execution phases live in `BUILD_PLAN.md`
|
||||
(Phase 15 onward).
|
||||
|
||||
**The brief:** make Petal essential for two audiences — ESL writers (native
|
||||
Mandarin / pt-PT / French → English), and English natives learning Mandarin,
|
||||
European Portuguese, French, maybe Spanish. Preserve privacy and warmth.
|
||||
|
||||
**The language model (settled by the user, 2026-07-26):** every user has
|
||||
exactly one language **pair, with English always one half** — (en + X),
|
||||
X ∈ {zh, pt-PT, fr, maybe es}. This is a deliberate scope decision: never an
|
||||
X↔Y pair without English, never more than one pair per user. The UI is
|
||||
bilingual in the pair everywhere (tips, pet responses, cards), and the user
|
||||
may **type in either language of the pair**; Petal infers direction from the
|
||||
text rather than asking.
|
||||
|
||||
---
|
||||
|
||||
## 1. What the pair model implies
|
||||
|
||||
The wife's zh setup is already exactly this — she writes Mandarin and English
|
||||
mixed, the UI is zh+en bilingual, and Petal adapts per span (CJK is never
|
||||
spellchecked, English words gloss to Chinese). So the pair model isn't a new
|
||||
design; it's a *promotion of today's behavior to the spec*. Three consequences:
|
||||
|
||||
- **Schema:** one column, `users.pair_lang` (the X half; default `'zh'`).
|
||||
No per-document language, no target/native split. Add it in whatever
|
||||
migration Phase B's provisioning touches — one column now vs. a real
|
||||
migration later, the same logic that put `user_id` in the schema on day one.
|
||||
- **Direction is inferred, not declared.** The zh pair gets inference for free
|
||||
(script boundaries separate the languages). Latin pairs don't — see §3a,
|
||||
which is the one genuinely new problem the pair model creates.
|
||||
- **Every bilingual surface stays two-language**, just parameterized: the
|
||||
`中文 · English` pattern becomes `X · English`. Nothing about the UI's shape
|
||||
changes, which is why the copy extraction in §2 is safe to do early.
|
||||
|
||||
## 2. Languages as data, not code ("langpacks")
|
||||
|
||||
Adding pt-PT today means editing code in many places. A quick census: **29
|
||||
frontend files** carry hardcoded zh-first bilingual strings (`tips.ts`,
|
||||
`GardenPanel`, `StatusBar`, `WordCard`, every popover…), plus the
|
||||
Mandarin-first prompt copy in `internal/llm/prompts.go`. Adding each new
|
||||
language by hunting through those files doesn't scale to four pairs and would
|
||||
slowly erode the bilingual-copy quality that makes Petal feel cared-for.
|
||||
|
||||
Because English is always one half, a **langpack is keyed by X alone** — one
|
||||
pack per pair, holding everything that varies:
|
||||
|
||||
- UI copy pairs — extract the existing `中文 · English` strings into a copy
|
||||
module; the current strings become the `zh` pack verbatim, so nothing
|
||||
visible changes. This is the biggest single chore in the whole effort;
|
||||
better done once than per-language.
|
||||
- Companion tip/cheer/bedtime lines (`tips.ts` is already data-shaped —
|
||||
closest to done).
|
||||
- LLM prompt copy: bilingual explanation phrasing, "natives usually say…"
|
||||
example pairs, and the both-directions framing (the text may be English, X,
|
||||
or mixed — respond appropriately).
|
||||
- Hunspell dictionary for X where one exists (`pt-PT`, `fr`, `es` upstream;
|
||||
zh has none — see §4). The en-US dictionary is shared by every pair.
|
||||
- DreamDict wiring: gloss both directions (`en→X` and `X→en`), phonetics.
|
||||
- Piper voices for X (both for reading X text aloud and as the L1 voice);
|
||||
font stack (CJK stacks only for zh).
|
||||
|
||||
Shared across all pairs, untouched: nspell en-US, the English IPA dataset, the
|
||||
EN Piper voice, and all of the editor machinery.
|
||||
|
||||
This is refactoring, not product, so it's tempting to skip. Don't: it's the
|
||||
difference between "Spanish is a data drop" and "Spanish is a month."
|
||||
|
||||
## 3. Sequencing: Latin-script targets first, and in this order
|
||||
|
||||
**pt-PT → fr → es.** Everything needed for these exists already: Hunspell
|
||||
dictionaries, Piper voices, DreamDict data (en/fr/pt-PT/zh), and — critically —
|
||||
the entire decoration/anchoring machinery (`wordAt`, spell tokenizer, suggestion
|
||||
re-anchoring) already works, because these languages are space-delimited and
|
||||
Latin-script like English.
|
||||
|
||||
Caveats worth writing down now:
|
||||
|
||||
- **DreamDict has no Spanish.** "Maybe Spanish" is gated on adding es to
|
||||
DreamDict first, or a separate dataset. Cheap to note, expensive to discover
|
||||
later.
|
||||
- **pt-BR drift is the main quality risk.** Qwen will default to Brazilian
|
||||
Portuguese in both explanations and "natives say…" examples. Prompts must pin
|
||||
European Portuguese explicitly, and the pt-PT pack should be reviewed by a
|
||||
pt-PT speaker before it's trusted — same standard the zh copy got by being
|
||||
written for a real reader. The multi-user plan's ECDICT-vs-DreamDict
|
||||
compare-on-real-lookups discipline applies here too.
|
||||
### 3a. The Latin+Latin wrinkle: inferring direction without a script boundary
|
||||
|
||||
The zh pair gets "which language is this word?" for free — the script answers
|
||||
it, and all of today's behavior (CJK never spellchecked, English words gloss
|
||||
to Chinese) hangs off that. In an en+fr or en+pt pair, both halves are Latin
|
||||
script, so the two per-word decisions need a real answer:
|
||||
|
||||
- **Spellcheck:** load both Hunspell dictionaries and pass a token if *either*
|
||||
accepts it; flag only words wrong in both. This never falsely squiggles
|
||||
correct writing in either language — the failure mode is missing a French
|
||||
word that happens to be a valid English word, which is the gentle direction
|
||||
to fail in. Correction pills can offer both dictionaries' suggestions.
|
||||
- **Gloss/WordCard:** look the word up in both directions via DreamDict; if it
|
||||
exists in only one language, done. For collisions (*chat*, *pain*, *sale*
|
||||
are all real words in both English and French), show both compactly — a
|
||||
two-line card ("🇫🇷 chat → cat · 🇬🇧 chat → bavarder") is honest, needs no
|
||||
detector, and is arguably *delightful* for a learner. A sentence-level
|
||||
language guess can order the lines, but shouldn't hide either.
|
||||
|
||||
No trained language detector, no heuristics that can be wrong about someone's
|
||||
writing — both-dictionaries membership plus show-both-on-collision covers it.
|
||||
The LLM passes need nothing: the prompt already sees the mixed text whole.
|
||||
|
||||
- The Hunspell tokenizer's current rule "CJK is never tokenized" stays correct
|
||||
for the zh pair unchanged.
|
||||
|
||||
## 4. The zh pair's *other* direction is a separate epic — say so explicitly
|
||||
|
||||
The en+zh pair already exists, but only one direction of it is built: today
|
||||
Petal deliberately ignores typed hanzi (never tokenized, never flagged, never
|
||||
glossed) — exactly right for a zh-native writer practicing English, and
|
||||
exactly insufficient for an English native *learning* Chinese, for whom the
|
||||
hanzi side is the whole point. Supporting that direction breaks assumptions
|
||||
that are load-bearing everywhere:
|
||||
|
||||
- No spaces → `wordAt`, the spell tokenizer, and word-boundary lookups need
|
||||
real word segmentation (a jieba-style segmenter, client- or server-side).
|
||||
- Hunspell has no concept of Chinese; "spellcheck" becomes wrong-character
|
||||
(错别字) detection — a different problem, probably LLM-assisted.
|
||||
- Smart-typography input rules and the IME interact; input rules are currently
|
||||
ASCII-gated, which is correct, but selection/caret behavior mid-IME
|
||||
composition needs testing.
|
||||
- The learning aids that matter are different: pinyin annotation (useful here,
|
||||
unlike for the current user who reads hanzi), tone-mark help, HSK-level word
|
||||
difficulty, hanzi stroke/handwriting practice.
|
||||
|
||||
None of this is unbuildable, but it is its **own phase with its own spec**, not
|
||||
part of the langpack drop. Recommendation: ship the pt-PT/fr pairs first to
|
||||
prove the pair model, and treat learner-facing Chinese writing as Petal's next
|
||||
big product bet after that — it's also the most differentiated one (very few
|
||||
warm, private tools exist for writing practice in Chinese).
|
||||
|
||||
## 5. Deepening the learning loop (all local, all gentle)
|
||||
|
||||
Petal's suggestion pipeline currently *corrects and forgets*. The vocabulary
|
||||
garden proved that capturing what the user already does (lookups) creates a
|
||||
learning surface for free. The same move is available twice more:
|
||||
|
||||
### 5a. Growth journal (patterns from accepted suggestions)
|
||||
|
||||
Accepted grammar/collocation suggestions are a record of what the writer is
|
||||
learning. Aggregate them **locally** into gentle patterns: "this month you've
|
||||
mostly stopped mixing 在/at" / "make a decision has stuck — you've used it
|
||||
right 4 times since." Two framing rules that keep it warm: it reports *growth*,
|
||||
never an error tally, and it only ever compares the writer to her own past
|
||||
self. Feeds the companion's cheer pool with genuinely personal material
|
||||
("上次你还问过这个词,这次自己用对了! 🌱"). Data is already in the
|
||||
`suggestions` table (status + type + original/replacement); this is a read-side
|
||||
feature, no new capture needed.
|
||||
|
||||
### 5b. Plant accepted collocations in the garden
|
||||
|
||||
An accepted collocation ("do a decision" → "make a decision") is a learnable
|
||||
chunk, exactly like a looked-up word. Auto-capture it into the vocabulary
|
||||
garden as a phrase card (the SM-2-lite scheduler doesn't care that it's two
|
||||
words). The garden then reflects *both* halves of learning: words she sought
|
||||
out, and phrasing she was gently given.
|
||||
|
||||
### 5c. Companion as tutor-lite: a daily invitation to write
|
||||
|
||||
The companion nudges about breaks and bedtime but never *invites writing*. A
|
||||
once-a-day bilingual prompt ("写 50 个字:今天让你微笑的一件小事 · Write 50
|
||||
words: one small thing that made you smile today"), offered when a session
|
||||
starts with no doc open. Explicitly **no streaks, no guilt** — the existing
|
||||
no-streak-shaming ethos in the SR scheduler is the right precedent; a declined
|
||||
prompt just gets a sleepy "好吧,我继续睡 😴". Prompt lists live in the
|
||||
native-language pack.
|
||||
|
||||
### 5d. Use DreamDict's richer fields
|
||||
|
||||
The multi-user plan notes DreamDict carries `Frequency`, `Difficulty`,
|
||||
`Antonyms`, `Etymology` with "no equivalent" in Petal. Three cheap, high-value
|
||||
surfaces:
|
||||
- A **frequency/difficulty chip** in the WordCard ("common word" / "advanced")
|
||||
— helps a learner decide whether a word is worth gardening.
|
||||
- **Etymology for the en-native audience**: Romance-language learners live on
|
||||
cognates; a one-line "from Latin *decidere*, like English *decide*" is the
|
||||
single best memory hook for pt/fr/es vocabulary.
|
||||
- **False friends**: a small curated list per pair (en↔pt: *embarrassed* ≠
|
||||
*embaraçada*-adjacent traps, *actually*/*atualmente*; en↔fr likewise),
|
||||
surfaced as a warm heads-up in the WordCard and as a collocation-style
|
||||
gentle flag when one is used suspiciously. Tiny data, disproportionate
|
||||
trust-building — this is the mistake that makes learners feel foolish, and
|
||||
catching it kindly is very Petal.
|
||||
|
||||
### 5e. Read-aloud, slower
|
||||
|
||||
Piper voices exist per target language; wire the target-language voice into the
|
||||
existing 🔊 surfaces, and add a **slow toggle** (Piper's `length_scale`) —
|
||||
learners replaying a sentence at 0.75× is one of the oldest, most-loved
|
||||
listening aids, and it's a query parameter away.
|
||||
|
||||
## 6. LLM-minimalism: essential help in plain code, the model as garnish
|
||||
|
||||
**Stated by the user (2026-07-26):** with the LLM on the far side of a VPN,
|
||||
preserve as much essential functionality as possible in ordinary code inside
|
||||
Petal, and rely on the LLM as little as possible. This deserves to be a
|
||||
standing design principle, not just a deployment reaction — it's also what
|
||||
keeps Petal instant (no 38-second checkpoints for things a lookup can answer)
|
||||
and private by construction.
|
||||
|
||||
Where Petal stands today, by dependency:
|
||||
|
||||
| Already pure code (survives VPN-down) | LLM-only today |
|
||||
|---|---|
|
||||
| Spellcheck (Hunspell), gloss/definitions/synonyms/phonetics (embedded lexicon → DreamDict), thesaurus, vocabulary garden + SR review, search, tags, versions + writing passport, export, find/replace, typography, TTS (Piper, VPS-local) | Grammar checkpoint, collocation coach, voice pass, Ask Petal, tone rewrite |
|
||||
|
||||
Everything in §5 lands in the left column by design (growth journal, garden
|
||||
planting, daily prompts, DreamDict fields, false friends — all lookups and
|
||||
local aggregation). The right column splits into two groups:
|
||||
|
||||
**Worth a code-first layer (the essential two):**
|
||||
|
||||
- **Grammar lite** — a rule-pack of high-precision, data-driven checks for
|
||||
the classic ESL patterns: a/an before vowel sounds, uncountables
|
||||
("informations", "advices", "furnitures"), subject–verb agreement in simple
|
||||
clauses, doubled comparatives, common preposition pairs ("depend of" →
|
||||
"depend on"), per-pair L1-interference rules (zh: dropped articles, he/she
|
||||
slips; pt/fr: "have X years" for age). These run instantly on every edit —
|
||||
no debounce, no 30s rate limit — as a fourth suggestion family through the
|
||||
existing rail. The bar is **precision over recall**: an offline rule must be
|
||||
near-certain before it flags, because a wrong correction is colder than a
|
||||
missed one. LanguageTool's open rule corpus is a mineable source for
|
||||
vetted patterns (extract data, not the Java).
|
||||
- **Collocation data** — the same curated-list move as false friends: the
|
||||
do/make, say/tell, strong-tea/heavy-rain families that fill every ESL
|
||||
collocation workbook are a few hundred entries of data, not a model. A
|
||||
small embedded miscollocation list catches the top offenders offline; the
|
||||
LLM pass, when reachable, adds the long tail. Same family, same rail, same
|
||||
warm phrasing — the writer never needs to know which engine spoke.
|
||||
|
||||
**Inherently LLM (degrade warmly, don't imitate):** Ask Petal, tone rewrite,
|
||||
and the voice pass are open-ended language generation — a code fake would be
|
||||
worse than the existing honest "小助手在休息" state. Leave them as the
|
||||
garnish they are.
|
||||
|
||||
The framing that falls out: **the LLM never holds essential functionality
|
||||
hostage.** Every daily-writing need — spell, define, gloss, pronounce, catch
|
||||
the common mistakes, review vocabulary, prove authorship — works on a
|
||||
disconnected VPS. The model adds depth and conversation when the tunnel is up.
|
||||
|
||||
## 7. The writing passport is an ESL flagship — treat it as one
|
||||
|
||||
The passport exists because AI detectors misfire on non-native English (the
|
||||
commit message cites the Stanford TOEFL finding). That's not a side feature —
|
||||
for the ESL audience it may be *the* reason to adopt Petal over any other
|
||||
editor: **the tool that protects you from being wrongly accused, instead of
|
||||
scoring you.** No product change needed beyond making sure it works identically
|
||||
for any target language (it should — it's language-agnostic snapshot history).
|
||||
Worth a prominent place in the README/landing copy when Petal gets one.
|
||||
|
||||
## 8. Ties into MULTIUSER_PLAN.md
|
||||
|
||||
For the open questions there, this document's brief implies:
|
||||
|
||||
- **OPEN #1 (auth):** Option B (in-app OIDC), and the planned deployment
|
||||
settles it. The user's stated topology (2026-07-26) is: **Petal hosted on
|
||||
the parodia.dev VPS, reaching vLLM on millenia over headscale VPN.** A
|
||||
public-internet app is exactly the case where "must never be reachable
|
||||
except through Traefik" is a footgun — one proxy misconfiguration on a VPS
|
||||
and forged identity headers reach the app. In-app OIDC is safe to expose
|
||||
directly.
|
||||
- **Deployment topology consequences** worth writing into the plan's Phase 2
|
||||
(deploy plumbing):
|
||||
- The LLM becomes the only cross-VPN runtime dependency (Piper is already
|
||||
installed on parodia.dev, so TTS stays VPS-local). The warm
|
||||
"小助手在休息" degradation path was built for a flaky co-tenant Ollama; a
|
||||
VPN link-down hits the same path, so the architecture already fails
|
||||
gently — but checkpoint latency now includes a WAN+VPN round trip, worth
|
||||
a look at the 60s LLM timeout.
|
||||
- Everything offline-by-design (DreamDict lookups, spellcheck, gloss,
|
||||
garden, search, the whole editor) keeps working when the VPN is down —
|
||||
another argument for MULTIUSER_PLAN Option 3 over an HTTP dictionary
|
||||
service, which would otherwise add a second cross-machine dependency.
|
||||
- vLLM and Piper on millenia should bind to the headscale interface only,
|
||||
never 0.0.0.0 on the LAN-facing side.
|
||||
- The writing moves onto rented VPS disk. "The writing never leaves the
|
||||
box" (§8) becomes "the box is a VPS" — at-rest encryption and an
|
||||
off-VPS backup of `petal.db` (e.g. nightly to millenia over the same
|
||||
VPN) deserve a line in the deploy phase.
|
||||
- **OPEN #6a (DreamDict):** Option 3 (import, read-only `dict.db`), agreed —
|
||||
it's the only option where four languages stay offline and instant, which
|
||||
§6 and §9 treat as non-negotiable.
|
||||
- **Phase B provisioning** should set `users.pair_lang` from the operator's
|
||||
provisioning step or a first-run picker — add the column in the same
|
||||
migration. (The plan's Phase D "native language becomes a `users` column"
|
||||
becomes this: one column, the X half of the pair.)
|
||||
- **localStorage namespacing** matters slightly more than the plan says once a
|
||||
household mixes pairs: the personal spell dictionary is per-*language* as
|
||||
well as per-user (a user's en words and pt-PT words must not merge into one
|
||||
Hunspell overlay). Key by `user + lang`.
|
||||
|
||||
## 9. Privacy & warmth guardrails (the checklist for every item above)
|
||||
|
||||
Everything suggested here passes these; future ideas should too.
|
||||
|
||||
1. **Offline-first, always — and code-first (§6).** Every essential surface
|
||||
must work with the LLM unreachable and the network unplugged
|
||||
(DreamDict-as-local-file preserves this; an HTTP dictionary service would
|
||||
not). The LLM only ever adds depth to something that already works.
|
||||
Cloud APIs are off the table even when they'd be easier.
|
||||
2. **The writing never leaves the box.** No telemetry, no "anonymous usage
|
||||
stats," ever. The growth journal (§5a) is computed locally from local rows.
|
||||
3. **No scores, no percentages, no red.** Petal already refuses AI-detection
|
||||
scores and classic red squiggles; the growth journal and false-friend flags
|
||||
must hold the same line — evidence and gentle phrasing, never grades.
|
||||
4. **No streaks, no guilt.** The SR scheduler set the precedent (gentle
|
||||
"again", no wipe). Daily prompts (§5c) are invitations, not obligations.
|
||||
5. **Both languages of the pair, always visible** — every explanation, tip,
|
||||
and pet response renders bilingual in (en + X). That's the warmth: being
|
||||
helped in the language you think in, next to the one you're learning.
|
||||
6. **The kitten stays asleep.** Every new companion behavior routes through
|
||||
the existing mood/cooldown engine; 瞌睡猫 keeps mumbling helpful things
|
||||
without waking up. (A reactive-animation puppy is on the wishlist — low
|
||||
priority per the user; the `companions.ts` roster + mood engine is already
|
||||
the drop-in point, richer per-mood Lottie segments are the only new work.)
|
||||
|
||||
## 10. Suggested sequence (interleaved with the multi-user plan's)
|
||||
|
||||
1. Add the `users.pair_lang` column (with the Phase B migration or sooner).
|
||||
2. Extract the bilingual UI copy into the langpack (zh pack = today's strings
|
||||
verbatim; pure refactor, no visible change).
|
||||
3. DreamDict integration per MULTIUSER_PLAN Option 3 (module rename → lexicon
|
||||
provider → pt-PT/fr wired first, zh compared before converging).
|
||||
4. pt-PT as the first full second pair: Hunspell pt-PT, Piper pt-PT voices,
|
||||
pinned-pt-PT prompts, native-speaker copy review, and the both-dictionaries
|
||||
spellcheck + show-both-gloss behavior from §3a. French follows the same
|
||||
groove; Spanish gated on DreamDict es data.
|
||||
5. Learning-loop features (§5) — each is small and independent; growth journal
|
||||
and garden-planting of collocations first, since they're read-side over
|
||||
existing data.
|
||||
6. Code-first layers (§6): the embedded miscollocation list first (same shape
|
||||
as false friends, drops into the existing collocation family), then
|
||||
grammar lite as its own suggestion family. Both are per-pair data, so
|
||||
they slot naturally into the langpacks from step 2.
|
||||
7. Learner-facing Chinese writing (the zh pair's second direction): spec it as
|
||||
its own phase (§4) only after the pair model is proven on pt-PT/fr.
|
||||
|
||||
## 11. Questions for the reviewer (1–3 settled 2026-07-26)
|
||||
|
||||
1. ~~§3a's no-detector stance~~ **Settled: yes** — pass if either dictionary
|
||||
accepts it, show both glosses on collision, no language detector.
|
||||
2. ~~UI-copy extraction first?~~ **Settled: yes** — the extraction (§2) is a
|
||||
prerequisite chore, done before pt-PT is wired.
|
||||
3. ~~Growth journal framing~~ **Settled: build it** with the two framing rules
|
||||
as hard constraints (growth only, self-comparison only).
|
||||
4. When (not whether) to build the learner-facing hanzi direction of the zh
|
||||
pair — after pt-PT/fr, or is it wanted sooner?
|
||||
5. Spanish: worth asking DreamDict to grow an es dataset now, or park it?
|
||||
6. Grammar lite (§6): hand-curate the rule pack from ESL teaching materials
|
||||
(small, fully understood), or mine LanguageTool's open rule corpus for
|
||||
vetted patterns (bigger head start, needs licensing + quality triage)?
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The baseline policy has to actually permit the frontend Vite builds. These
|
||||
// are the allowances dist/index.html needs; if a future build starts emitting
|
||||
// an inline script or pulling from a new host, that shows up here rather than
|
||||
// as a blank page in production.
|
||||
func TestBaselineCSPCoversTheBuiltFrontend(t *testing.T) {
|
||||
required := []string{
|
||||
"script-src 'self'", // Vite emits no inline script
|
||||
"'unsafe-inline' https://fonts.googleapis.com", // React style={{…}} + the font link
|
||||
"https://fonts.gstatic.com", // the font files themselves
|
||||
"blob:", // read-aloud plays an object URL
|
||||
"object-src 'none'",
|
||||
"base-uri 'self'",
|
||||
"frame-ancestors 'self'",
|
||||
}
|
||||
for _, want := range required {
|
||||
if !strings.Contains(contentSecurityPolicy, want) {
|
||||
t.Errorf("baseline CSP is missing %q:\n%s", want, contentSecurityPolicy)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The middleware is a floor, not a ceiling: it runs *before* the handler
|
||||
// precisely so a route serving untrusted bytes can overwrite the policy with a
|
||||
// stricter one. This is the contract the image store depends on, and the reason
|
||||
// the policy no longer lives in the Traefik labels — customresponseheaders
|
||||
// would overwrite it in the other direction.
|
||||
func TestRouteMayTightenTheBaselineCSP(t *testing.T) {
|
||||
strict := "default-src 'none'; sandbox"
|
||||
handler := securityHeaders(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Security-Policy", strict)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/images/x.svg", nil))
|
||||
|
||||
if got := rec.Header().Get("Content-Security-Policy"); got != strict {
|
||||
t.Fatalf("handler's policy was not honoured: got %q, want %q", got, strict)
|
||||
}
|
||||
// The headers it didn't touch still stand.
|
||||
if rec.Header().Get("X-Content-Type-Options") != "nosniff" {
|
||||
t.Error("baseline nosniff was lost")
|
||||
}
|
||||
}
|
||||
|
||||
// Every ordinary response carries the baseline.
|
||||
func TestBaselineHeadersOnAPlainResponse(t *testing.T) {
|
||||
handler := securityHeaders(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
||||
|
||||
for header, want := range map[string]string{
|
||||
"Content-Security-Policy": contentSecurityPolicy,
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Referrer-Policy": "same-origin",
|
||||
} {
|
||||
if got := rec.Header().Get(header); got != want {
|
||||
t.Errorf("%s = %q, want %q", header, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
+196
-15
@@ -1,9 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"flag"
|
||||
"io/fs"
|
||||
"log"
|
||||
"net/http"
|
||||
@@ -12,12 +14,14 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/config"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/docs"
|
||||
"gitea.parodia.dev/drwily/petal/internal/images"
|
||||
"gitea.parodia.dev/drwily/petal/internal/lexicon"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
"gitea.parodia.dev/drwily/petal/internal/spell"
|
||||
"gitea.parodia.dev/drwily/petal/internal/suggestions"
|
||||
"gitea.parodia.dev/drwily/petal/internal/tts"
|
||||
"gitea.parodia.dev/drwily/petal/internal/vocab"
|
||||
@@ -25,8 +29,25 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
backupTo := flag.String("backup", "",
|
||||
"write a consistent copy of the database to this path and exit (no server)")
|
||||
flag.Parse()
|
||||
|
||||
cfg := config.Load()
|
||||
|
||||
// Backup mode short-circuits before anything else starts: no migrations, no
|
||||
// seed, no listener. It runs against the live database safely (VACUUM INTO
|
||||
// takes only a read transaction), so the nightly job is
|
||||
// docker compose exec petal /app/petal -backup /data/backups/<name>.db
|
||||
// against the running container rather than a copy of three WAL files.
|
||||
if *backupTo != "" {
|
||||
if err := db.Backup(cfg.DatabasePath, *backupTo); err != nil {
|
||||
log.Fatalf("backup: %v", err)
|
||||
}
|
||||
log.Printf("backup written to %s", *backupTo)
|
||||
return
|
||||
}
|
||||
|
||||
database, err := db.Open(cfg.DatabasePath)
|
||||
if err != nil {
|
||||
log.Fatalf("database: %v", err)
|
||||
@@ -34,11 +55,82 @@ func main() {
|
||||
defer database.Close()
|
||||
log.Printf("database ready at %s", cfg.DatabasePath)
|
||||
|
||||
// Identity. With Authentik configured, Petal is an OIDC client in its own
|
||||
// right: /auth/login starts a real login and the session cookie it issues is
|
||||
// what every API request is resolved from. Without it — local development,
|
||||
// and every deployment before auth landed — StaticResolver hands out the
|
||||
// single hardcoded local user, so nothing about running Petal on a laptop
|
||||
// changes.
|
||||
sessions := auth.NewSessionStore(database.DB)
|
||||
users := auth.NewUserStore(database.DB)
|
||||
|
||||
// …and the fallback is exactly what must not happen quietly on a public
|
||||
// host. Refuse to start rather than serve someone's journals to the open
|
||||
// internet because one environment variable was misspelled. See
|
||||
// config.RequireAuth for why this defaults on for any non-loopback BASE_URL.
|
||||
if !cfg.AuthEnabled() && cfg.RequireAuth {
|
||||
log.Fatalf("auth: refusing to start unauthenticated at %s.\n"+
|
||||
" Petal would resolve every anonymous request to the single %q user, with full\n"+
|
||||
" read and write over every document in the database.\n"+
|
||||
" Set AUTHENTIK_URL, AUTHENTIK_CLIENT_ID and AUTHENTIK_CLIENT_SECRET, or set\n"+
|
||||
" PETAL_REQUIRE_AUTH=false if this really is a trusted private network.",
|
||||
cfg.BaseURL, db.LocalUserID)
|
||||
}
|
||||
|
||||
var resolver auth.Resolver = auth.StaticResolver(db.LocalUserID)
|
||||
var oidcClient *auth.OIDC
|
||||
if cfg.AuthEnabled() {
|
||||
allowed := auth.ParseAllowlist(cfg.AllowedSubs)
|
||||
oidcClient = auth.NewOIDC(context.Background(), auth.Options{
|
||||
IssuerURL: cfg.AuthentikURL,
|
||||
ClientID: cfg.AuthentikClientID,
|
||||
ClientSecret: cfg.AuthentikClientSecret,
|
||||
BaseURL: cfg.BaseURL,
|
||||
Allowed: allowed,
|
||||
}, sessions, users)
|
||||
resolver = sessions
|
||||
if n, err := sessions.Prune(); err == nil && n > 0 {
|
||||
log.Printf("auth: pruned %d expired session(s)", n)
|
||||
}
|
||||
log.Printf("auth: OIDC enabled (issuer=%s, redirect=%s)", cfg.AuthentikURL, oidcClient.RedirectURI())
|
||||
// An empty allowlist is a legitimate choice for a single-household
|
||||
// instance and a wide-open door in front of an IdP that fronts anything
|
||||
// else. Petal cannot tell which it is, so it says so every boot rather
|
||||
// than assuming.
|
||||
if len(allowed) == 0 {
|
||||
log.Printf("auth: WARNING — PETAL_ALLOWED_SUBS is empty, so EVERY account %s "+
|
||||
"authenticates may sign in and start writing here. Set it to the "+
|
||||
"comma-separated emails (or subject ids) that belong in this Petal.",
|
||||
cfg.AuthentikURL)
|
||||
}
|
||||
} else {
|
||||
log.Printf("auth: OIDC not configured — running as the single %q user", db.LocalUserID)
|
||||
}
|
||||
|
||||
// The dictionary behind word lookups. dict.db is DreamDict's built database
|
||||
// — French, European Portuguese, Spanish and Mandarin in one read-only file
|
||||
// beside petal.db. It is optional on purpose: a laptop checkout has never
|
||||
// had one, and the Chinese pair doesn't need one, so its absence downgrades
|
||||
// lookups rather than stopping Petal. A file that is present but broken is
|
||||
// a different matter and gets said out loud.
|
||||
dict, err := lexicon.OpenDreamDict(cfg.DictPath)
|
||||
if err != nil {
|
||||
log.Printf("dictionary: %s unusable (%v) — falling back to the embedded datasets", cfg.DictPath, err)
|
||||
}
|
||||
defer dict.Close()
|
||||
lexSet := lexicon.NewSet(dict)
|
||||
if lexSet.HasDreamDict() {
|
||||
log.Printf("dictionary: DreamDict open at %s (%s)", cfg.DictPath, lexSet.Contents())
|
||||
} else {
|
||||
log.Printf("dictionary: no dict.db at %s — English/Chinese only", cfg.DictPath)
|
||||
}
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(middleware.RequestID)
|
||||
r.Use(middleware.RealIP)
|
||||
r.Use(middleware.Logger)
|
||||
r.Use(middleware.Recoverer)
|
||||
r.Use(securityHeaders)
|
||||
|
||||
// Build version: a hash of the embedded SPA shell. Vite rewrites index.html
|
||||
// with content-hashed asset names on every build, so this string changes
|
||||
@@ -65,6 +157,29 @@ func main() {
|
||||
_, _ = w.Write([]byte(`{"version":"` + version + `"}`))
|
||||
})
|
||||
|
||||
// Everything below serves or mutates a particular user's data, so it sits
|
||||
// behind the auth middleware. /health and /version deliberately stay
|
||||
// outside it: they carry no user data, and a monitoring probe (or the
|
||||
// client's update poll) must not need a session to reach them.
|
||||
//
|
||||
// The middleware resolves the caller once and hands handlers the answer via
|
||||
// auth.UserID(r.Context()). Which resolver it runs is the only thing that
|
||||
// changed when auth landed: the session store in a deployment with
|
||||
// Authentik configured, the static local user otherwise. No handler or
|
||||
// query moved for either.
|
||||
api.Group(func(pr chi.Router) {
|
||||
pr.Use(auth.Middleware(resolver))
|
||||
|
||||
// Who am I? The frontend namespaces its per-account browser state by
|
||||
// this id and shows the signed-in writer.
|
||||
pr.Get("/me", users.MeHandler())
|
||||
|
||||
// …and the one thing about herself she can change: which language
|
||||
// Petal is her pair in. It lives here rather than under a /settings
|
||||
// tree because there is exactly one setting and it is a property of
|
||||
// the user row — the same row /me reads back.
|
||||
pr.Patch("/me", users.UpdateMeHandler())
|
||||
|
||||
llmClient := llm.NewLLMClient(cfg)
|
||||
sug := suggestions.New(database, llmClient)
|
||||
|
||||
@@ -73,41 +188,66 @@ func main() {
|
||||
docsHandler := docs.New(database)
|
||||
docsRouter := docsHandler.Routes()
|
||||
sug.RegisterDocRoutes(docsRouter)
|
||||
api.Mount("/docs", docsRouter)
|
||||
pr.Mount("/docs", docsRouter)
|
||||
|
||||
// Tag management (the roster) and cross-document full-text search.
|
||||
api.Mount("/tags", docsHandler.TagRoutes())
|
||||
api.Mount("/search", docsHandler.SearchRoutes())
|
||||
pr.Mount("/tags", docsHandler.TagRoutes())
|
||||
pr.Mount("/search", docsHandler.SearchRoutes())
|
||||
|
||||
// Per-suggestion actions (accept/dismiss) under /api/suggestions.
|
||||
api.Mount("/suggestions", sug.Routes())
|
||||
pr.Mount("/suggestions", sug.Routes())
|
||||
|
||||
// Offline lexicon: full word lookups (gloss + definition + synonyms) for
|
||||
// the right-click popover, and the lightweight Chinese-only gloss for the
|
||||
// inline hover/select tooltip. One handler so the datasets load once.
|
||||
lex := lexicon.NewHandler()
|
||||
api.Mount("/word", lex.Routes())
|
||||
api.Mount("/gloss", lex.GlossRoutes())
|
||||
// the right-click popover, and the lightweight gloss-only lookup for the
|
||||
// inline hover/select tooltip. One handler over one provider Set, so the
|
||||
// embedded datasets and dict.db are each opened once. Which of them
|
||||
// answers depends on the caller's language pair — so unlike before, the
|
||||
// response is no longer identical for everyone, and it stays behind auth
|
||||
// for that reason as much as for the API surface.
|
||||
lex := lexicon.NewHandler(database.DB, lexSet)
|
||||
pr.Mount("/word", lex.Routes())
|
||||
pr.Mount("/gloss", lex.GlossRoutes())
|
||||
// The same lookup pointing the other way: a Chinese word to its pinyin
|
||||
// and English senses, for an account whose direction is learning_pair.
|
||||
pr.Mount("/hanzi", lex.HanziRoutes())
|
||||
|
||||
// Vocabulary garden: words the writer looks up are captured here and
|
||||
// surfaced for gentle spaced-repetition review.
|
||||
api.Mount("/vocab", vocab.New(database).Routes())
|
||||
pr.Mount("/vocab", vocab.New(database).Routes())
|
||||
|
||||
// Editor image uploads, stored on disk and served back by content hash.
|
||||
imgHandler, err := images.New(cfg.ImageDir)
|
||||
// The personal spelling dictionary — the words she's told Petal to stop
|
||||
// flagging. Kept server-side (rather than in the browser) so it belongs
|
||||
// to her account and follows her between devices.
|
||||
pr.Mount("/spell", spell.New(database).Routes())
|
||||
|
||||
// Editor image uploads, stored on disk and served back by content hash
|
||||
// to whoever owns them. Files already on disk from before ownership
|
||||
// existed are claimed for the local user at startup.
|
||||
imgHandler, err := images.New(cfg.ImageDir, database.DB, db.LocalUserID)
|
||||
if err != nil {
|
||||
log.Fatalf("image store: %v", err)
|
||||
}
|
||||
api.Mount("/images", imgHandler.Routes())
|
||||
pr.Mount("/images", imgHandler.Routes())
|
||||
|
||||
// Read-aloud: proxy short passages to a local Piper TTS server. Only
|
||||
// mounted when TTS_ENDPOINT is configured; otherwise the frontend falls
|
||||
// back to the browser's Web Speech API on its own.
|
||||
if ttsHandler, ok := tts.New(cfg); ok {
|
||||
api.Mount("/tts", ttsHandler.Routes())
|
||||
log.Printf("read-aloud enabled (TTS endpoint=%s)", cfg.TTSEndpoint)
|
||||
pr.Mount("/tts", ttsHandler.Routes())
|
||||
// Name the languages, not just the English endpoint: which
|
||||
// voices a deployment actually reached is the thing worth
|
||||
// seeing at boot, and a missing sidecar is silent otherwise
|
||||
// (a 404 the client answers by quietly using Web Speech).
|
||||
log.Printf("read-aloud enabled (voices: %s)", strings.Join(ttsHandler.Languages(), ", "))
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
// Login lives outside /api: these are browser navigations, and they must be
|
||||
// reachable without a session — that is their entire job.
|
||||
if oidcClient != nil {
|
||||
r.Mount("/auth", oidcClient.Routes())
|
||||
}
|
||||
|
||||
// Everything else: serve the embedded SPA (with index.html fallback for client routing).
|
||||
r.NotFound(spaHandler())
|
||||
@@ -119,6 +259,47 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// contentSecurityPolicy is the default policy for everything Petal serves.
|
||||
//
|
||||
// It lives here rather than in the Traefik labels, and that move is the point:
|
||||
// Traefik's customResponseHeaders *sets* a header, overwriting whatever the
|
||||
// application chose, so a policy declared at the edge silently replaces the
|
||||
// stricter one an individual route needs. Stored images need exactly that (an
|
||||
// uploaded SVG is a document that can carry script — see internal/images), and
|
||||
// a rule the edge can quietly undo is not a rule.
|
||||
//
|
||||
// The allowances are what the built frontend actually uses, no more: script
|
||||
// only from Petal itself (Vite emits no inline script — this policy is checked
|
||||
// against dist/index.html), inline *styles* because React's style={{…}} props
|
||||
// compile to style attributes, and Google's font hosts because index.html links
|
||||
// them. object-src and base-uri close the two attribute-injection routes that
|
||||
// survive HTML escaping.
|
||||
const contentSecurityPolicy = "default-src 'self'; " +
|
||||
"script-src 'self'; " +
|
||||
"style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " +
|
||||
"font-src 'self' data: https://fonts.gstatic.com; " +
|
||||
"img-src 'self' data: blob:; " +
|
||||
"media-src 'self' data: blob:; " +
|
||||
"connect-src 'self'; " +
|
||||
"object-src 'none'; " +
|
||||
"base-uri 'self'; " +
|
||||
"form-action 'self'; " +
|
||||
"frame-ancestors 'self'"
|
||||
|
||||
// securityHeaders lays down the baseline response headers before the handler
|
||||
// runs, so a route that needs something stricter — the image store — simply
|
||||
// overwrites its own copy on the way past. Ordering is the mechanism: this is a
|
||||
// floor, not a ceiling.
|
||||
func securityHeaders(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
h := w.Header()
|
||||
h.Set("Content-Security-Policy", contentSecurityPolicy)
|
||||
h.Set("X-Content-Type-Options", "nosniff")
|
||||
h.Set("Referrer-Policy", "same-origin")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// maxAPIBodyBytes caps a JSON API request body at 2 MiB. That's far above any
|
||||
// real document save (the body is text plus lightweight marks; images upload
|
||||
// separately by reference) while still bounding abuse. Exceeding it makes the
|
||||
|
||||
+683
-48
@@ -1,57 +1,608 @@
|
||||
# Deploying read-aloud (Piper TTS) to millenia
|
||||
# Deploying Petal
|
||||
|
||||
Petal's read-aloud generates audio with a local **Piper** neural-TTS server and
|
||||
transcodes it to mp3 with **ffmpeg**. Both run on millenia (192.168.1.212);
|
||||
nothing leaves the box. If Piper is down or `TTS_ENDPOINT` is unset, the frontend
|
||||
falls back to the browser's Web Speech API automatically.
|
||||
Two deployments exist right now:
|
||||
|
||||
## 1. Piper as a systemd service (one-time)
|
||||
| | host | shape | status |
|
||||
|---|---|---|---|
|
||||
| **parodia** | `petal.parodia.dev` / `100.64.0.1` | docker compose behind the host's Traefik | **canonical** since 2026-07-27 — she signs in here |
|
||||
| **millenia** | `192.168.1.212` / `100.64.0.2` | `petal.service`, bare binary on `:8088`, Piper as user systemd units | frozen fallback: a copy of her writing as it stood at the move, still owned by the pre-auth `local` user |
|
||||
|
||||
Her writing moved to the VPS when sign-in landed (Phase 16/17): it is the
|
||||
instance that actually authenticates, its data directory is LUKS-encrypted, and
|
||||
it is reachable from anywhere. millenia was left running and untouched as a
|
||||
fallback — but the two diverge the moment anything is written on either, so it
|
||||
should be retired rather than kept in step. Everything below is the VPS side;
|
||||
the millenia Piper notes are kept in the appendix because that instance still
|
||||
runs them.
|
||||
|
||||
---
|
||||
|
||||
## 1. The VPS stack
|
||||
|
||||
`docker-compose.yml` at the repo root brings up three containers:
|
||||
|
||||
- **petal** — the single Go binary with the frontend embedded. Publishes no host
|
||||
port; Traefik is the only way in.
|
||||
- **piper-en** / **piper-zh** / **piper-pt** / **piper-fr** — read-aloud. Each
|
||||
Piper HTTP server loads exactly one voice, so every language is its own
|
||||
container off one image, with the models cached in a shared volume. They sit
|
||||
on an internal network with no published ports, so only Petal can reach them.
|
||||
Adding pt-PT in Phase 21 was a third service and fr in Phase 24 a fourth —
|
||||
never a new image, and since Phase 21 never any Go either (the languages are
|
||||
discovered from `TTS_ENDPOINT_<LANG>`/`TTS_VOICE_<LANG>`).
|
||||
|
||||
They run as containers rather than the host systemd units millenia uses because
|
||||
Piper was never actually installed on the VPS, and the `reala` account has no
|
||||
lingering session to keep user units alive across logout.
|
||||
|
||||
### Prerequisites on the host
|
||||
|
||||
- Docker with the compose plugin, and the existing external `traefik` network
|
||||
- A DNS A record for the hostname pointing at the VPS (`petal.parodia.dev` is
|
||||
already in place)
|
||||
|
||||
### First deploy
|
||||
|
||||
```bash
|
||||
ssh reala@100.64.0.1
|
||||
git clone https://gitea.parodia.dev/drwily/petal.git ~/petal
|
||||
cd ~/petal
|
||||
cp deploy/petal.env.example .env
|
||||
```
|
||||
|
||||
Then edit `.env`:
|
||||
|
||||
- `PETAL_UID` / `PETAL_GID` — `id -u` / `id -g` for this account. `./data` is a
|
||||
bind mount, so the image's own `petal` user has no claim on it; a mismatch
|
||||
shows up as `unable to open database file (14)` and a restart loop.
|
||||
- `AUTHENTIK_URL` / `AUTHENTIK_CLIENT_ID` / `AUTHENTIK_CLIENT_SECRET` /
|
||||
`PETAL_ALLOWED_SUBS` — sign-in, see §4. Without them Petal runs as the single
|
||||
`local` user and must not be exposed.
|
||||
- `LLM_MODEL` / `LLM_CHAT_MODEL` — see §3.
|
||||
|
||||
```bash
|
||||
mkdir -p data/backups
|
||||
docker compose up -d --build
|
||||
docker compose ps # all three healthy
|
||||
```
|
||||
|
||||
### Updating
|
||||
|
||||
```bash
|
||||
cd ~/petal && git pull && docker compose up -d --build
|
||||
```
|
||||
|
||||
The frontend is embedded in the binary, so a rebuild is the whole deploy. The
|
||||
client polls `/api/version` (a hash of the built `index.html`) and offers a
|
||||
refresh when it changes.
|
||||
|
||||
---
|
||||
|
||||
## 2. What Traefik does
|
||||
|
||||
Labels follow the convention the other services on this box use: the external
|
||||
`traefik` network, the `web-secure` entrypoint, the `default` cert resolver and
|
||||
`compression@file`. Petal adds its own response-header middleware
|
||||
(`frame-ancestors 'self'`, HSTS, nosniff, `Referrer-Policy: same-origin`).
|
||||
|
||||
There is no auth middleware at the edge: Petal does its own (§4). `/api/health`
|
||||
and `/api/version` sit outside Petal's own auth for the same reason they always
|
||||
did — a monitoring probe must not need a session, and neither carries user
|
||||
data.
|
||||
|
||||
---
|
||||
|
||||
## 3. The LLM link over headscale
|
||||
|
||||
The vLLM backend stays on millenia and is reached over headscale
|
||||
(`100.64.0.2`). **This is the only cross-VPN dependency**, and by the
|
||||
LLM-minimalism principle it never gates essential functionality — spell check,
|
||||
gloss, vocabulary garden, search, export and read-aloud all keep working with
|
||||
the link down, and the status bar shows the warm
|
||||
`🌙 小助手在休息 · Petal's helper is resting · 文字已保存`.
|
||||
|
||||
`LLM_TIMEOUT` is raised from the local-network default of 30s to **90s**: the
|
||||
voice and collocation passes send a whole document, the timeout is a hard
|
||||
deadline on the completion call, and a WAN+VPN round trip eats the margin.
|
||||
|
||||
### How the link is exposed — a forwarder, not a rebind
|
||||
|
||||
vLLM stays bound to `127.0.0.1:8000`. `vllm-headscale-proxy.service` (a socat
|
||||
unit, in this directory) adds a second listener on `100.64.0.2:8000` that
|
||||
forwards to it.
|
||||
|
||||
The plan originally said to rebind vLLM itself. That turned out to be the
|
||||
expensive option: `vllm-chat.service` is **shared** — Petal, Gogobee and Open
|
||||
WebUI all point at `127.0.0.1:8000`, and Open WebUI stores its endpoint in its
|
||||
own database rather than in env — so moving the bind address would mean editing
|
||||
three consumers and reloading a 35B AWQ model, minutes of downtime for all of
|
||||
them. The forwarder adds a door instead of moving one: local callers are
|
||||
untouched, and the only new exposure is on the VPN interface.
|
||||
|
||||
It binds `100.64.0.2` specifically, **never** `0.0.0.0`: the far end of this
|
||||
link is a public host, and the LAN has no business seeing an unauthenticated
|
||||
inference endpoint.
|
||||
|
||||
```bash
|
||||
sudo install -m 0644 deploy/vllm-headscale-proxy.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload && sudo systemctl enable --now vllm-headscale-proxy
|
||||
ss -lntp | grep 8000 # expect BOTH 127.0.0.1:8000 and 100.64.0.2:8000
|
||||
```
|
||||
|
||||
The model id (`qwen3.6-35b`) goes into `LLM_MODEL` / `LLM_CHAT_MODEL` on the
|
||||
VPS. Verified end to end: a grammar checkpoint from `petal.parodia.dev` returns
|
||||
real suggestions in ~3s over the VPN.
|
||||
|
||||
---
|
||||
|
||||
## 4. Sign-in (Authentik OIDC)
|
||||
|
||||
Petal is an OIDC client in its own right: it runs the login itself rather than
|
||||
trusting a header from the proxy. Nothing about the container has to be
|
||||
unreachable for that to be safe.
|
||||
|
||||
Login turns on only when `AUTHENTIK_URL`, `AUTHENTIK_CLIENT_ID` and
|
||||
`AUTHENTIK_CLIENT_SECRET` are all set. With any of them missing Petal falls back
|
||||
to the single hardcoded `local` user — which is what local development wants,
|
||||
and what every deployment did before this landed. A host serving the public
|
||||
must have them set.
|
||||
|
||||
### Register Petal in Authentik
|
||||
|
||||
In the Authentik admin UI (**Applications → Providers → Create → OAuth2/OpenID
|
||||
Provider**):
|
||||
|
||||
| Field | Value |
|
||||
| --- | --- |
|
||||
| Client type | Confidential |
|
||||
| Redirect URI | `https://petal.parodia.dev/auth/callback` (strict) |
|
||||
| Scopes | `openid`, `profile`, `email` |
|
||||
| Signing key | any (Petal fetches the JWKS from discovery) |
|
||||
|
||||
Then create an **Application** bound to that provider, and copy the client id,
|
||||
the client secret, and the provider's **OpenID Configuration Issuer** (it looks
|
||||
like `https://auth.parodia.dev/application/o/petal/` — the issuer, not the
|
||||
`.well-known` URL; Petal appends that itself).
|
||||
|
||||
Put them in `.env`:
|
||||
|
||||
```
|
||||
AUTHENTIK_URL=https://auth.parodia.dev/application/o/petal/
|
||||
AUTHENTIK_CLIENT_ID=…
|
||||
AUTHENTIK_CLIENT_SECRET=…
|
||||
PETAL_ALLOWED_SUBS=her@example.com,me@example.com
|
||||
```
|
||||
|
||||
`PETAL_ALLOWED_SUBS` is the guest list: comma-separated OIDC subject ids and/or
|
||||
email addresses. Authentik fronts several applications on this host, and being a
|
||||
valid user there does not mean being a user here. Leaving it empty lets in
|
||||
everyone Authentik authenticates. Emails are accepted alongside subject ids
|
||||
precisely so the list can be written *before* anyone has logged in — a subject
|
||||
is an opaque uuid that doesn't exist until first sign-in.
|
||||
|
||||
A valid login that isn't on the list gets a warm bilingual "this Petal isn't
|
||||
yours to write in" page, and no account is provisioned.
|
||||
|
||||
### Checking it
|
||||
|
||||
```bash
|
||||
curl -si https://petal.parodia.dev/api/docs | head -1 # 401 without a session
|
||||
curl -si https://petal.parodia.dev/auth/login | grep -i location # → Authentik
|
||||
docker compose logs petal | grep '^.*auth:' # issuer + redirect at boot
|
||||
```
|
||||
|
||||
The startup log prints the redirect URI it will use; if Authentik rejects the
|
||||
login with a redirect-uri mismatch, compare that line against what's registered.
|
||||
|
||||
Two things bit this deployment, both worth checking first if a login dies early:
|
||||
|
||||
- **The issuer's trailing slash is significant.** Authentik's is
|
||||
`…/application/o/petal/`, OIDC requires the discovered issuer to match the
|
||||
configured one byte-for-byte, and normalising the slash away makes discovery
|
||||
fail with `did not match the issuer URL returned by provider`.
|
||||
- **A provider created through the API or `ak shell` has an empty
|
||||
`grant_types`**, which authentik reads as "no grant type is permitted here"
|
||||
and answers with `invalid_request` / *The request is otherwise malformed*
|
||||
before the login page ever appears. The admin UI fills the list in for you;
|
||||
scripted creation must set it (`authorization_code`, `refresh_token`).
|
||||
|
||||
Discovery is lazy and retried, so an Authentik outage blocks *new* logins but
|
||||
leaves existing sessions working — those only need Petal's own database.
|
||||
|
||||
### Sessions
|
||||
|
||||
Opaque token in a `petal_session` cookie (`HttpOnly`, `SameSite=Lax`, `Secure`
|
||||
on https); the `sessions` table stores only its SHA-256, so a database copy
|
||||
yields nothing usable. Thirty-day sliding expiry — every request pushes it out,
|
||||
throttled to one write an hour. `/auth/logout` deletes the row, not just the
|
||||
cookie. Expired rows are pruned at startup.
|
||||
|
||||
To sign someone out everywhere immediately:
|
||||
|
||||
```bash
|
||||
docker compose exec petal sh -c \
|
||||
"sqlite3 /data/petal.db \"DELETE FROM sessions WHERE user_id = '<sub>'\""
|
||||
```
|
||||
|
||||
### The edge gate is gone
|
||||
|
||||
Until Phase 16 there was a Traefik basic-auth middleware in front of everything,
|
||||
because Petal authenticated nobody and a public hostname was a public API. It
|
||||
was removed when OIDC went live on 2026-07-27, together with the separate
|
||||
unauthenticated `/api/health` router that existed only to escape it: every `/api`
|
||||
route now answers 401 without a session, and the only thing an anonymous visitor
|
||||
gets is the app shell and a redirect to sign in.
|
||||
|
||||
Removing that gate meant a missing `AUTHENTIK_*` variable stopped being a
|
||||
nuisance and became an exposure: Petal would fall back to the single `local`
|
||||
user and hand every anonymous visitor full read and write over the database,
|
||||
saying so only in a log line. So **it now refuses to start instead**:
|
||||
|
||||
```
|
||||
auth: refusing to start unauthenticated at https://petal.parodia.dev.
|
||||
Petal would resolve every anonymous request to the single "local" user, ...
|
||||
```
|
||||
|
||||
The guard defaults on for any `BASE_URL` that isn't loopback, so a laptop
|
||||
checkout still runs open and a deployment cannot. If you genuinely want an
|
||||
unauthenticated instance on a trusted private network, say so out loud with
|
||||
`PETAL_REQUIRE_AUTH=false` — and if it is reachable from anywhere else, put a
|
||||
gate back in front of it first:
|
||||
|
||||
```yaml
|
||||
traefik.http.routers.petal.middlewares: compression@file,petal-headers,petal-auth
|
||||
traefik.http.middlewares.petal-auth.basicauth.users: ${PETAL_BASIC_AUTH:?}
|
||||
```
|
||||
|
||||
with `htpasswd -nbB petal 'your-password'` in `.env` as `PETAL_BASIC_AUTH`.
|
||||
|
||||
### Who gets in
|
||||
|
||||
`PETAL_ALLOWED_SUBS` is a comma-separated list of emails and/or OIDC subject
|
||||
ids. **Set it.** Empty means everyone authentik authenticates, and authentik on
|
||||
this host fronts several applications — a valid account there is not the same as
|
||||
belonging in someone's private journal. An empty list is legal (a single-
|
||||
household instance may want it) and warns at every boot:
|
||||
|
||||
```
|
||||
auth: WARNING — PETAL_ALLOWED_SUBS is empty, so EVERY account ... may sign in
|
||||
```
|
||||
|
||||
### Response headers
|
||||
|
||||
`Content-Security-Policy`, `X-Content-Type-Options` and `Referrer-Policy` are set
|
||||
by the binary, not by the Traefik labels. Traefik's `customresponseheaders`
|
||||
*overwrites*, which would silently replace the stricter policy an individual
|
||||
route picks for itself — the image store serves stored uploads under
|
||||
`default-src 'none'; sandbox` so that an SVG someone pasted into a document
|
||||
cannot run as a page on Petal's own origin. Only HSTS stays at the edge, where
|
||||
TLS is actually terminated.
|
||||
|
||||
---
|
||||
|
||||
## 4a. Moving an account (`scripts/migrate_local_user.py`)
|
||||
|
||||
Petal ran as one hardcoded user (`users.id = 'local'`) before sign-in existed.
|
||||
Moving that writing onto a real account is a deliberate, one-off operation:
|
||||
|
||||
```bash
|
||||
docker compose stop petal
|
||||
python3 scripts/migrate_local_user.py data/petal.db --to <oidc-sub> # dry run
|
||||
python3 scripts/migrate_local_user.py data/petal.db --to <oidc-sub> \
|
||||
--email her@example.com --name "Her Name" --apply
|
||||
docker compose up -d petal
|
||||
```
|
||||
|
||||
The subject id is **knowable before she has ever logged in**. With authentik's
|
||||
default `hashed_user_id` sub mode it is the user's `uid`:
|
||||
|
||||
```bash
|
||||
docker exec authentik-server-1 ak shell -c \
|
||||
"from authentik.core.models import User; print(User.objects.get(username='claire').uid)"
|
||||
```
|
||||
|
||||
so the data can move first and she signs in to find it already there.
|
||||
|
||||
**Start Petal once on the incoming database before migrating.** A database
|
||||
carried over from another instance may be a schema behind, and the app applies
|
||||
migrations at startup; the script moves rows and does not touch the schema.
|
||||
|
||||
The script is dry-run by default, takes its own `VACUUM INTO` backup, runs as
|
||||
one transaction with foreign keys off, and verifies the row counts before it
|
||||
commits. It refuses to run while anything else has the database open, and
|
||||
refuses to merge into an account that already owns writing.
|
||||
|
||||
**`local` comes back, and that's expected.** `db.Open` seeds that row on every
|
||||
startup, so it reappears the moment Petal restarts after a migration. It owns
|
||||
nothing — the writing is on the real account — and it is only ever resolved to
|
||||
by `StaticResolver`, which a deployment with `AUTHENTIK_*` set never uses. Check
|
||||
`SELECT COUNT(*) FROM documents WHERE user_id = 'local'` if you want to be sure
|
||||
a migration took; the presence of the row itself says nothing.
|
||||
|
||||
---
|
||||
|
||||
## 4b. The dictionary (`dict.db`)
|
||||
|
||||
Word lookups for the French, European Portuguese and Spanish pairs come from
|
||||
[DreamDict](https://github.com/prosolis/dreamdict)'s built database, which Petal
|
||||
opens **read-only** beside `petal.db`. Petal imports DreamDict's `dictionary`
|
||||
package directly — there is no DreamDict service to run and nothing to reach
|
||||
over the VPN, which matters because a hover gloss must answer in milliseconds.
|
||||
|
||||
`dict.db` is **optional**. With no file at `DICT_PATH` Petal logs
|
||||
|
||||
```
|
||||
dictionary: no dict.db at /data/dict.db — English/Chinese only
|
||||
```
|
||||
|
||||
and serves lookups from the datasets compiled into the binary. The Chinese pair
|
||||
is unaffected either way — it stays on ECDICT (see below) — and a non-Chinese
|
||||
writer still gets English definitions, synonyms and pronunciation, losing only
|
||||
the translation. **A dictionary that failed to deploy costs the gloss, not the
|
||||
popover.** A file that is present but was never imported is a different matter
|
||||
and is logged as an error.
|
||||
|
||||
### Installing it
|
||||
|
||||
The database is built by DreamDict's own import CLI from ~6 GB of source data;
|
||||
it is not built on the VPS. Copy the built file into the data volume:
|
||||
|
||||
```bash
|
||||
# on the machine holding a built dict.db (millenia: ~/dreamdict/data/dict.db)
|
||||
scp ~/dreamdict/data/dict.db reala@100.64.0.1:/home/reala/petal/data/dict.db
|
||||
# on parodia
|
||||
chown "$(id -u):$(id -g)" /home/reala/petal/data/dict.db
|
||||
docker compose restart petal # the handle is opened once, at startup
|
||||
```
|
||||
|
||||
Expect ~450 MB. It sits inside the LUKS volume with everything else (§6). The
|
||||
backups name `petal.db` explicitly rather than sweeping the data directory
|
||||
(§5), so `dict.db` stays out of them — which is the right outcome and worth
|
||||
keeping: it is rebuildable from public data and would otherwise dominate every
|
||||
nightly snapshot. Petal never writes to it.
|
||||
|
||||
### Why Chinese doesn't use it
|
||||
|
||||
The zh pair stays on the embedded ECDICT gloss, deliberately. Measured on the
|
||||
deployed database, DreamDict reaches a Chinese gloss for 53% of the 2,000
|
||||
commonest English words; ECDICT covers essentially all of them and is in daily
|
||||
use by a real writer. `lexicon.Set.For` is where that decision lives — one
|
||||
`switch`, changed the day a comparison on her actual lookups says otherwise.
|
||||
|
||||
For pt-PT and French the same measurement reads 62% and 63%, which is why they
|
||||
use DreamDict: there is no alternative source for them at all.
|
||||
|
||||
### Rebuilding it
|
||||
|
||||
Rebuilt 2026-07-27 to add Spanish (the previous file predated DreamDict's
|
||||
Spanish support). The recipe, since it will be needed again:
|
||||
|
||||
```bash
|
||||
# on millenia, from a clean checkout of dreamdict main
|
||||
./scripts/download-dict-data.sh ~/dreamdict/data # idempotent; skips what's there
|
||||
go run ./cmd/dictimport --data ~/dreamdict/data --db ./dict.db --clean
|
||||
```
|
||||
|
||||
~6 minutes on 32 cores; the data directory is ~7 GB and mostly already
|
||||
downloaded. **Build to a new path, never over a file in use** — then verify by
|
||||
hash on both ends before swapping.
|
||||
|
||||
Two things worth knowing before trusting a rebuild:
|
||||
|
||||
- The SUBTLEX-US download fails (the source moved behind a manual export). It
|
||||
does not matter: the loader falls back to `SUBTLEX-US.txt`, which is present,
|
||||
and English "frequency" is mostly SCOWL's commonness bucket anyway —
|
||||
1000/800/600/…/50, refined by SUBTLEX for only ~1,600 words. That is why the
|
||||
word-difficulty chip reads `difficulty`, not `frequency`.
|
||||
- Check the *other* languages' counts are unchanged before shipping. The 2026-07
|
||||
rebuild came out byte-identical for en/fr/pt-PT/zh, which is what says it
|
||||
added a language rather than quietly shifting the rest.
|
||||
|
||||
Gloss coverage of the 2,000 commonest English words, after the rebuild:
|
||||
**es 68.6%**, fr 63.1%, pt-PT 62.1%, zh 53.2%. The startup line reports actual
|
||||
per-language row counts, so a database missing a language says so.
|
||||
|
||||
---
|
||||
|
||||
## 5. Backups
|
||||
|
||||
### On the VPS — folded into `parodia-backup`
|
||||
|
||||
Petal rides the host's existing offsite job (`/usr/local/bin/parodia-backup`,
|
||||
`parodia-backup.timer`, nightly ~03:40): age-encrypted to S3, 14-day retention,
|
||||
dead-man snitch. The host holds only the age *public* recipient, so it writes
|
||||
backups it cannot itself decrypt.
|
||||
|
||||
```
|
||||
push petal.db.age sqlite_file_dump /home/reala/petal/data/petal.db
|
||||
```
|
||||
|
||||
`/home/reala/petal/.env` is in the same job's secrets tarball — it carries the
|
||||
interim basic-auth hash and, from Phase 16, the OIDC client secret.
|
||||
|
||||
**Why `sqlite_file_dump` and not the script's existing `sqlite_dump`:** that
|
||||
helper uses Python's `iterdump`, which **does not reproduce an FTS5 virtual
|
||||
table**. It emits `documents_fts` as a raw `sqlite_master` row plus its shadow
|
||||
tables, and replaying the result dies with `no such table: documents_fts` —
|
||||
verified by round-tripping a real dump on 2026-07-27. Petal's cross-document
|
||||
search would have been silently missing after any restore. `sqlite_file_dump`
|
||||
runs `VACUUM INTO` instead: a genuine database file, virtual tables intact, WAL
|
||||
folded in, no write lock. Restore is a copy rather than a replay.
|
||||
|
||||
> If `apply.db` ever gains a virtual table, it needs the same treatment.
|
||||
|
||||
### Restore (VPS)
|
||||
|
||||
```bash
|
||||
age -d -i <offline-identity> petal.db.age > /tmp/petal.db # from S3
|
||||
cd ~/petal
|
||||
docker compose stop petal # stop writers first
|
||||
mv data/petal.db data/petal.db.before-restore # keep the current state
|
||||
rm -f data/petal.db-wal data/petal.db-shm # a stale WAL against a new file
|
||||
cp /tmp/petal.db data/petal.db
|
||||
docker compose start petal
|
||||
docker compose logs petal --tail 5 # expect "database ready"
|
||||
```
|
||||
|
||||
To sanity-check an archive before committing to it, have Petal open it in a
|
||||
scratch directory — a clean exit means it reads end to end:
|
||||
|
||||
```bash
|
||||
mkdir -p /tmp/restore-check && cp /tmp/petal.db /tmp/restore-check/petal.db
|
||||
docker run --rm -v /tmp/restore-check:/data --user "$(id -u):$(id -g)" \
|
||||
--entrypoint sh petal:local -c '/app/petal -backup /data/verify.db'
|
||||
```
|
||||
|
||||
### On millenia — `petal-backup.timer`
|
||||
|
||||
Until 2026-07-27 her actual writing had **no scheduled backup at all**; the
|
||||
newest snapshot was a month old. It now runs nightly at 03:20
|
||||
(`Persistent=true`, because the box isn't on 24/7 and a missed window would
|
||||
otherwise be skipped silently):
|
||||
|
||||
```bash
|
||||
sudo install -m 0644 deploy/petal-backup.service deploy/petal-backup.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload && sudo systemctl enable --now petal-backup.timer
|
||||
sudo systemctl start petal-backup.service # prove it before trusting it
|
||||
```
|
||||
|
||||
`deploy/backup-petal.sh` snapshots via `petal -backup`, gzips, **age-encrypts
|
||||
with the parodia public recipient**, pushes to the VPS over headscale with a
|
||||
post-transfer size check, and prunes both ends. The private identity is offline,
|
||||
so neither millenia nor the VPS can decrypt what it is holding — verified.
|
||||
|
||||
A manual snapshot any time, no tooling required:
|
||||
|
||||
```bash
|
||||
cd ~/petal && ./petal -backup ~/petal/backups/manual-$(date -u +%Y%m%dT%H%M%SZ).db
|
||||
```
|
||||
|
||||
Restore is a copy — stop Petal, drop the file in as `data/petal.db`, remove any
|
||||
stale `-wal`/`-shm`, start.
|
||||
|
||||
---
|
||||
|
||||
## 6. Encryption at rest
|
||||
|
||||
### VPS — `/home/reala/petal/data` is a LUKS volume
|
||||
|
||||
`deploy/setup-encrypted-data.sh` puts the data directory on LUKS2 over a sparse
|
||||
file at `/var/lib/petal-crypt.img`. That covers `petal.db`, uploaded `images/`,
|
||||
**and the TTS cache** — which is synthesized audio of her sentences and is easy
|
||||
to forget.
|
||||
|
||||
LUKS-on-a-file rather than gocryptfs because Petal is SQLite in WAL mode: WAL
|
||||
needs a shared-memory index (`-shm`) mapped consistently across processes, and
|
||||
FUSE has a long history of subtle mmap/locking differences. A block device with
|
||||
ext4 behaves exactly like a disk to SQLite, which is the only guarantee worth
|
||||
having under a database.
|
||||
|
||||
**What it protects, honestly.** The key lives at `/etc/petal/dataset.key` on the
|
||||
same host so the volume auto-unlocks at boot. That is a deliberate availability
|
||||
tradeoff:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| protects against | a decommissioned or resold disk; reading the raw block device; casual browsing of a filesystem snapshot that excludes `/etc` |
|
||||
| does **not** protect against | anyone holding the whole VM image — they get the keyfile with the ciphertext; or anything at all while the host is running and mounted |
|
||||
|
||||
Real protection from a provider-side snapshot needs the key off-box (fetched
|
||||
over the VPN at boot). Considered, not chosen.
|
||||
|
||||
Two things this setup got wrong the first time, both caught by rehearsing a
|
||||
reboot rather than trusting a clean run — worth knowing if you rebuild it:
|
||||
|
||||
- **Mounting over a directory hides its contents, it does not remove them.** The
|
||||
first pass left the original plaintext `petal.db` and WAL sitting on the
|
||||
unencrypted root filesystem, invisible under the mount. The script now shreds
|
||||
the originals before mounting and refuses to continue if the mountpoint will
|
||||
not come up empty.
|
||||
- **`systemd-cryptsetup` was not installed**, so `/etc/crypttab` was ignored
|
||||
entirely and the volume would never have unlocked at boot. The script now
|
||||
refuses to run without the generator present.
|
||||
|
||||
Check it any time:
|
||||
|
||||
```bash
|
||||
sudo ./deploy/setup-encrypted-data.sh --status
|
||||
```
|
||||
|
||||
### The mount-liveness guard
|
||||
|
||||
The mountpoint directory exists whether or not the volume is mounted, so a boot
|
||||
where the unlock failed would start Petal against an empty unencrypted
|
||||
directory and quietly serve a blank database — the failure that looks like data
|
||||
loss. `data/.volume-ok` lives on the encrypted filesystem and is bind-mounted
|
||||
with `create_host_path: false`, turning that into a loud container start
|
||||
failure:
|
||||
|
||||
```
|
||||
Error response from daemon: invalid mount config for type "bind":
|
||||
bind source path does not exist: /home/reala/petal/data/.volume-ok
|
||||
```
|
||||
|
||||
Verified by unmounting and attempting a start.
|
||||
|
||||
**A true reboot has not been tested** — the VPS also runs matrix, lemmy, akkoma,
|
||||
gitea and authentik, so rebooting it is your call. The boot path was rehearsed
|
||||
through `local-fs.target`, which pulls the mount, which pulls the unlock.
|
||||
|
||||
### millenia is not encrypted at rest
|
||||
|
||||
LVM, no LUKS. Her canonical writing sits in plaintext on the home box. Backups
|
||||
leaving it are age-encrypted; the disk itself is not.
|
||||
|
||||
---
|
||||
|
||||
## 7. Supervision and monitoring
|
||||
|
||||
Petal on millenia ran for months as a bare `./petal` with PPID 1 — no unit, no
|
||||
screen session — so a crash or reboot left it down until someone noticed. It is
|
||||
now `petal.service`:
|
||||
|
||||
```bash
|
||||
sudo install -m 0644 deploy/petal.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload && sudo systemctl enable --now petal.service
|
||||
```
|
||||
|
||||
Verified by `kill -9`-ing it and watching systemd bring it back.
|
||||
|
||||
**Piper's silent-failure mode is fixed.** Both units now set
|
||||
`StartLimitIntervalSec=300` / `StartLimitBurst=5`. With `RestartSec=3` and
|
||||
systemd's default 10-second window, only ~3 restarts ever landed inside it, so
|
||||
the burst limit was never reached and a dead service looped **26,800+ times over
|
||||
a day without ever entering `failed`**. A genuinely broken Piper now shows up in
|
||||
`systemctl --user --failed`.
|
||||
|
||||
### Still to do — an external probe
|
||||
|
||||
Nothing yet watches millenia from outside. uptime-kuma already runs on the VPS
|
||||
and can reach millenia over headscale, so the missing piece is two monitors
|
||||
(they need the uptime-kuma UI, hence not scripted here):
|
||||
|
||||
- `http://100.64.0.2:8088/api/health` — Petal itself
|
||||
- a POST to `http://100.64.0.2:8088/api/tts` — catches a dead Piper, which
|
||||
`/api/health` will not, because read-aloud degrades silently to browser
|
||||
speech
|
||||
|
||||
---
|
||||
|
||||
## Appendix — Piper on millenia (user systemd units)
|
||||
|
||||
millenia still runs Piper as user services; these are the original notes.
|
||||
|
||||
```bash
|
||||
# from this repo, on your workstation:
|
||||
scp deploy/piper.service deploy/setup-piper.sh 192.168.1.212:/tmp/
|
||||
ssh 192.168.1.212 'cd /tmp && sudo ./setup-piper.sh'
|
||||
```
|
||||
|
||||
`setup-piper.sh` creates `~/piper/venv`, installs `piper-tts[http]`, downloads the
|
||||
`en_US-amy-medium` voice into `~/piper/voices`, installs+enables `piper.service`
|
||||
(loopback :5005), and smoke-tests it. Idempotent.
|
||||
`setup-piper.sh` creates `~/piper/venv`, installs `piper-tts[http]`, downloads
|
||||
`en_US-amy-medium` into `~/piper/voices`, installs and enables `piper.service`
|
||||
(loopback `:5005`), and smoke-tests it. Idempotent. Check it with
|
||||
`systemctl status piper` / `journalctl -u piper -f`.
|
||||
|
||||
Check it any time: `systemctl status piper`, `journalctl -u piper -f`.
|
||||
|
||||
## 2. Point petal at Piper + redeploy the binary
|
||||
|
||||
Add to petal's environment (its `.env` or launch env):
|
||||
|
||||
```
|
||||
TTS_ENDPOINT=http://127.0.0.1:5005
|
||||
TTS_VOICE_EN=en_US-amy-medium
|
||||
TTS_AUDIO_FORMAT=mp3
|
||||
```
|
||||
|
||||
Then ship the rebuilt binary (`go build -o petal ./cmd/server` already done) and
|
||||
restart the petal `:8088` session. Confirm the log line:
|
||||
`read-aloud enabled (TTS endpoint=http://127.0.0.1:5005)`.
|
||||
|
||||
## 3. Verify
|
||||
|
||||
```bash
|
||||
# on millenia — end-to-end through petal, including ffmpeg transcode:
|
||||
curl -sf -X POST localhost:8088/api/tts \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"text":"hello there","lang":"en-US"}' -o /tmp/petal-tts.mp3 \
|
||||
&& file /tmp/petal-tts.mp3 # expect: Audio file ... MPEG ... layer III
|
||||
```
|
||||
|
||||
- Second identical call is served from the cache (`~/petal/.../data/tts/*.mp3`).
|
||||
- A language with no configured instance returns 404 → client uses Web Speech.
|
||||
- Browser check via the uitest harness (`~/petal/uitest`): tap a word in the
|
||||
WordCard / select a sentence and hit speak — expect the natural Piper voice.
|
||||
|
||||
## Chinese voice (live)
|
||||
|
||||
Each Piper HTTP server loads ONE model, so Chinese runs as a **second instance**:
|
||||
`piper-zh.service` on :5006 with `zh_CN-huayan-medium`. Deployed via:
|
||||
Chinese runs as a second instance (`piper-zh.service`, `:5006`,
|
||||
`zh_CN-huayan-medium`):
|
||||
|
||||
```bash
|
||||
scp deploy/piper-zh.service 192.168.1.212:~/.config/systemd/user/
|
||||
@@ -60,6 +611,90 @@ ssh 192.168.1.212 'export XDG_RUNTIME_DIR=/run/user/$(id -u)
|
||||
systemctl --user daemon-reload && systemctl --user enable --now piper-zh.service'
|
||||
```
|
||||
|
||||
Then in petal's `start.sh`: `TTS_ENDPOINT_ZH=http://127.0.0.1:5006` and
|
||||
`TTS_VOICE_ZH=zh_CN-huayan-medium`. The handler maps language → instance from config,
|
||||
so adding more languages is just another instance + env pair (no code change).
|
||||
Petal's env then carries `TTS_ENDPOINT=http://127.0.0.1:5005`,
|
||||
`TTS_ENDPOINT_ZH=http://127.0.0.1:5006` and the matching voice ids. The handler
|
||||
maps language → instance from config, so another language is another instance
|
||||
plus an env pair, no code change.
|
||||
|
||||
**Adding a language (Phase 21 made this literal).** Petal discovers its Piper
|
||||
instances from the environment: English is the unsuffixed
|
||||
`TTS_ENDPOINT`/`TTS_VOICE_EN`, and every other language is a
|
||||
`TTS_ENDPOINT_<LANG>`/`TTS_VOICE_<LANG>` pair. `<LANG>` is the *base* tag —
|
||||
`PT`, not `PT_PT`, because an environment variable name cannot hold a hyphen and
|
||||
only one Portuguese model is loaded regardless. Both halves must be set: an
|
||||
endpoint with no voice is dropped, so a half-finished language reads to the
|
||||
browser as "no voice here, use Web Speech" instead of erroring on every tap. The
|
||||
startup line names what it actually resolved:
|
||||
|
||||
```
|
||||
read-aloud enabled (voices: en=en_US-amy-medium, pt=pt_PT-tugão-medium, zh=zh_CN-huayan-medium)
|
||||
```
|
||||
|
||||
**Portuguese: `pt_PT-tugão-medium` is the only European voice Piper ships.** The
|
||||
other five `pt_*` models in the catalogue are all Brazilian, so the voice has to
|
||||
be named explicitly for the same reason the Hunspell dictionary did (Phase 21):
|
||||
the obvious default is the wrong country. Check what exists before assuming:
|
||||
|
||||
```bash
|
||||
docker exec petal-piper-en python -c "import urllib.request,json; \
|
||||
d=json.load(urllib.request.urlopen('https://huggingface.co/rhasspy/piper-voices/resolve/main/voices.json')); \
|
||||
print([k for k in d if k.startswith('pt')])"
|
||||
```
|
||||
|
||||
**French: the opposite situation, and worth knowing it is.** Every `fr_*` voice
|
||||
in the catalogue is `fr_FR`, so there is no wrong country to land on by default
|
||||
and no Québec voice to choose instead; `fr_FR-siwis-medium` is picked to match
|
||||
the register of the other three rather than to avoid anything. The name is also
|
||||
plain ASCII, so the entrypoint's percent-encoded download fallback — which
|
||||
exists only because `tugão` broke `piper.download_voices` — never fires here.
|
||||
|
||||
**Slow replay.** `POST /api/tts` takes `slow: true`, which raises Piper's
|
||||
`length_scale` to about 4/3 (≈0.75× pace). It is a separate cache entry, not a
|
||||
playback-rate trick, so the slow clip is synthesized once and then instant.
|
||||
|
||||
**Piper version note:** piper-tts moved synthesis from `POST /` to
|
||||
`POST /synthesize` in 1.6.0, with an identical request body. `TTS_PATH` selects
|
||||
which — it defaults to `/`, and both the VPS compose and millenia's `start.sh`
|
||||
now set `/synthesize`. If read-aloud starts returning 502 after a Piper upgrade,
|
||||
that flag is the fix.
|
||||
|
||||
**The venv is fragile across Python upgrades.** On 2026-07-27 millenia's Piper
|
||||
was found dead with **26,800+ failed restarts**, silently since the Jul 26
|
||||
reboot — read-aloud had been falling back to browser Web Speech the whole time.
|
||||
Root cause: an OS upgrade moved `/usr/bin/python3` from 3.13 to 3.14, and
|
||||
`venv/bin/python3` is a *symlink to the system interpreter*, so the venv's
|
||||
`lib/python3.13/site-packages` became invisible — `sys.path` contained no
|
||||
site-packages at all. The failure surfaced as the misleading
|
||||
`No module named piper.http_server` even though `http_server.py` was sitting
|
||||
right there on disk.
|
||||
|
||||
Fix (what was done — recreating the venv, not repairing it):
|
||||
|
||||
```bash
|
||||
systemctl --user stop piper.service piper-zh.service
|
||||
mv ~/piper/venv ~/piper/venv.broken-py313
|
||||
python3 -m venv ~/piper/venv
|
||||
~/piper/venv/bin/pip install "piper-tts[http]"
|
||||
~/piper/venv/bin/python -c 'import piper.http_server' # must not raise
|
||||
systemctl --user start piper.service piper-zh.service
|
||||
```
|
||||
|
||||
That reinstall lands 1.6.0, so it must be paired with `TTS_PATH=/synthesize` in
|
||||
`start.sh` and a binary new enough to read that variable. Voices in
|
||||
`~/piper/voices` survive and do not need re-downloading.
|
||||
|
||||
Worth knowing: `Restart=on-failure` will retry forever without ever alerting.
|
||||
Neither service reports its health anywhere, which is why this went unnoticed
|
||||
for a day. A `/api/tts` probe in uptime-kuma would have caught it.
|
||||
|
||||
Verify end to end (through Petal, including the ffmpeg transcode):
|
||||
|
||||
```bash
|
||||
curl -sf -X POST localhost:8088/api/tts \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"text":"hello there","lang":"en-US"}' -o /tmp/petal-tts.mp3 \
|
||||
&& file /tmp/petal-tts.mp3 # expect: MPEG ADTS, layer III
|
||||
```
|
||||
|
||||
A second identical call is served from the cache; a language with no configured
|
||||
instance returns 404 so the client falls back to Web Speech.
|
||||
|
||||
Executable
+113
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
# Nightly off-box backup of Petal's database.
|
||||
#
|
||||
# ./backup-petal.sh # snapshot, compress, encrypt, push, prune
|
||||
# ./backup-petal.sh --local-only # snapshot + prune, skip the remote push
|
||||
#
|
||||
# Used on millenia, driven by petal-backup.timer (see deploy/README.md). The
|
||||
# VPS does not use this script -- Petal rides parodia-backup there.
|
||||
#
|
||||
# The snapshot goes through `petal -backup`, which uses SQLite's VACUUM INTO:
|
||||
# one coherent file including anything still in the WAL, taken without a write
|
||||
# lock, so it is safe against the live running app. That is why this script
|
||||
# never touches petal.db / -wal / -shm directly — copying those three
|
||||
# separately can capture a torn mid-checkpoint state.
|
||||
#
|
||||
# Everything below is overridable from the environment.
|
||||
set -euo pipefail
|
||||
|
||||
# Stack directory (holds docker-compose.yml and ./data).
|
||||
STACK_DIR="${STACK_DIR:-$HOME/petal}"
|
||||
# Where snapshots land on the VPS before being pushed off-box. Inside ./data so
|
||||
# the container can write it through the existing bind mount.
|
||||
LOCAL_DIR="${LOCAL_DIR:-$STACK_DIR/data/backups}"
|
||||
# Off-VPS destination: millenia over headscale. Empty disables the push.
|
||||
REMOTE_HOST="${REMOTE_HOST:-100.64.0.2}"
|
||||
REMOTE_USER="${REMOTE_USER:-}"
|
||||
REMOTE_DIR="${REMOTE_DIR:-petal-backups}"
|
||||
# Retention, in days, on each side.
|
||||
KEEP_LOCAL_DAYS="${KEEP_LOCAL_DAYS:-7}"
|
||||
KEEP_REMOTE_DAYS="${KEEP_REMOTE_DAYS:-30}"
|
||||
# age public recipient. Set it and every archive is encrypted before it leaves
|
||||
# (and at rest locally too); leave it empty and the script says so loudly.
|
||||
AGE_RECIPIENT="${AGE_RECIPIENT:-}"
|
||||
|
||||
local_only=0
|
||||
[ "${1:-}" = "--local-only" ] && local_only=1
|
||||
|
||||
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
name="petal-${stamp}.db"
|
||||
|
||||
cd "$STACK_DIR"
|
||||
|
||||
mkdir -p "$LOCAL_DIR"
|
||||
snapshot="${LOCAL_DIR}/${name}"
|
||||
|
||||
# Two deployment shapes: the VPS runs the compose stack, millenia runs a bare
|
||||
# binary. Either way the snapshot goes through `petal -backup` (VACUUM INTO),
|
||||
# which is safe against the live process, so neither has to stop writing.
|
||||
if [ -f "$STACK_DIR/docker-compose.yml" ] && docker compose ps --status running 2>/dev/null | grep -q petal; then
|
||||
echo ">> snapshotting via the running container -> data/backups/${name}"
|
||||
# ./data/backups on the host is the container's /data/backups.
|
||||
docker compose exec -T petal /app/petal -backup "/data/backups/${name}"
|
||||
elif [ -x "$STACK_DIR/petal" ]; then
|
||||
echo ">> snapshotting via the local binary -> ${snapshot}"
|
||||
# DATABASE_PATH must match the running instance; start.sh is the source of
|
||||
# truth for it, so read it from there rather than guessing.
|
||||
DB_PATH="$(sed -n 's/^export DATABASE_PATH=//p' "$STACK_DIR/start.sh" 2>/dev/null | tail -1)"
|
||||
DATABASE_PATH="${DB_PATH:-$STACK_DIR/data/petal.db}" "$STACK_DIR/petal" -backup "$snapshot"
|
||||
else
|
||||
echo "no way to snapshot: neither a running petal container nor $STACK_DIR/petal" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[ -s "$snapshot" ] || { echo "snapshot missing or empty: $snapshot" >&2; exit 1; }
|
||||
|
||||
echo ">> compressing"
|
||||
gzip -9 "$snapshot"
|
||||
archive="${snapshot}.gz"
|
||||
|
||||
# Encrypt with age when a recipient is configured. The recipient is a PUBLIC
|
||||
# key -- this host can write backups it cannot itself decrypt, and the private
|
||||
# identity stays offline. Same custody model as parodia-backup. Without this,
|
||||
# an off-box copy is just her writing sitting in plaintext on another machine.
|
||||
if [ -n "$AGE_RECIPIENT" ]; then
|
||||
age -r "$AGE_RECIPIENT" -o "${archive}.age" "$archive"
|
||||
shred -uz "$archive" 2>/dev/null || rm -f "$archive"
|
||||
archive="${archive}.age"
|
||||
else
|
||||
echo " (AGE_RECIPIENT unset: this backup is NOT encrypted)" >&2
|
||||
fi
|
||||
echo " $(du -h "$archive" | cut -f1) ${archive}"
|
||||
|
||||
if [ "$local_only" -eq 0 ] && [ -n "$REMOTE_HOST" ]; then
|
||||
target="${REMOTE_HOST}"
|
||||
[ -n "$REMOTE_USER" ] && target="${REMOTE_USER}@${REMOTE_HOST}"
|
||||
|
||||
echo ">> pushing to ${target}:${REMOTE_DIR}/"
|
||||
ssh -o BatchMode=yes "$target" "mkdir -p '${REMOTE_DIR}'"
|
||||
scp -q -o BatchMode=yes "$archive" "${target}:${REMOTE_DIR}/"
|
||||
|
||||
# Verify by size rather than trusting scp's exit code alone — a truncated
|
||||
# transfer that still exits 0 would leave a backup that only looks fine.
|
||||
local_size="$(stat -c%s "$archive")"
|
||||
remote_size="$(ssh -o BatchMode=yes "$target" "stat -c%s '${REMOTE_DIR}/$(basename "$archive")'")"
|
||||
if [ "$local_size" != "$remote_size" ]; then
|
||||
echo "size mismatch after transfer: local ${local_size}, remote ${remote_size}" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo " verified ${remote_size} bytes"
|
||||
|
||||
echo ">> pruning remote copies older than ${KEEP_REMOTE_DAYS} days"
|
||||
ssh -o BatchMode=yes "$target" \
|
||||
"find '${REMOTE_DIR}' \\( -name 'petal-*.db.gz' -o -name 'petal-*.db.gz.age' \\) -type f -mtime +${KEEP_REMOTE_DAYS} -delete"
|
||||
elif [ "$local_only" -eq 1 ]; then
|
||||
echo ">> --local-only: skipping the remote push"
|
||||
else
|
||||
echo ">> REMOTE_HOST is empty: skipping the remote push" >&2
|
||||
fi
|
||||
|
||||
echo ">> pruning local copies older than ${KEEP_LOCAL_DAYS} days"
|
||||
find "$LOCAL_DIR" \( -name 'petal-*.db.gz' -o -name 'petal-*.db.gz.age' \) -type f -mtime "+${KEEP_LOCAL_DAYS}" -delete
|
||||
|
||||
echo ">> done"
|
||||
@@ -0,0 +1,23 @@
|
||||
[Unit]
|
||||
Description=Nightly backup of Petal's database (millenia)
|
||||
Documentation=file:///home/reala/petal/deploy/README.md
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=reala
|
||||
WorkingDirectory=/home/reala/petal
|
||||
# Encrypted with the parodia age recipient before it leaves the box, then
|
||||
# pushed to the VPS over headscale. The recipient is a public key and the
|
||||
# private identity is offline, so neither millenia nor the VPS can decrypt what
|
||||
# they are holding. Replaces nothing -- before this there was no scheduled
|
||||
# backup of her writing at all; the newest snapshot on 2026-07-27 was a month
|
||||
# old.
|
||||
Environment=AGE_RECIPIENT=age19n4k55m9d50xew5vj2ehmcsf3wuj7fhgmfpckadpvcya4032q9dqrt4yjw
|
||||
Environment=REMOTE_USER=reala
|
||||
Environment=REMOTE_HOST=100.64.0.1
|
||||
Environment=REMOTE_DIR=petal-backups-millenia
|
||||
ExecStart=/home/reala/petal/deploy/backup-petal.sh
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Nightly Petal database backup (millenia)
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 03:20:00
|
||||
# The box is not on 24/7; without this a missed window would just be skipped
|
||||
# and the backup would silently never run.
|
||||
Persistent=true
|
||||
RandomizedDelaySec=300
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,102 @@
|
||||
# Petal — production environment for the parodia.dev VPS.
|
||||
# Copy to the stack directory as `.env` (docker-compose.yml reads it via
|
||||
# env_file) and fill in the model names. Values the image already fixes
|
||||
# (PORT, DATABASE_PATH, IMAGE_DIR, TTS_CACHE_DIR, TTS endpoints) are set in
|
||||
# docker-compose.yml, not here.
|
||||
|
||||
# --- Routing -----------------------------------------------------------------
|
||||
# Must match the DNS A record and the Traefik Host() rule.
|
||||
PETAL_HOST=petal.parodia.dev
|
||||
# Absolute origin the app knows itself by. Phase 16's OIDC redirect URI is
|
||||
# built from this, so it has to be the real public HTTPS origin.
|
||||
BASE_URL=https://petal.parodia.dev
|
||||
|
||||
# The companion's bedtime nag and the night theme read the container clock.
|
||||
TZ=Europe/Lisbon
|
||||
|
||||
# The container runs as this uid/gid so it can write the ./data bind mount.
|
||||
# Set both to the output of `id -u` / `id -g` for the account owning the stack
|
||||
# directory. Wrong values show up as "unable to open database file (14)".
|
||||
PETAL_UID=1001
|
||||
PETAL_GID=1001
|
||||
|
||||
# (The interim PETAL_BASIC_AUTH edge gate is gone: Petal authenticates for
|
||||
# itself now, and the auth block at the bottom of this file is what holds the
|
||||
# door. A second password in front of a real login is one more thing to lose.)
|
||||
|
||||
# --- LLM (millenia, over headscale) ------------------------------------------
|
||||
# The only cross-VPN dependency. Petal degrades warmly when it's unreachable:
|
||||
# spell check, gloss, garden, search, export and read-aloud all keep working and
|
||||
# the status bar shows 小助手在休息 · Petal's helper is resting.
|
||||
#
|
||||
# 100.64.0.2 is millenia on the headscale network. vLLM must be bound to that
|
||||
# interface (NOT 0.0.0.0 — this host is public); see deploy/README.md.
|
||||
LLM_BACKEND=vllm
|
||||
LLM_ENDPOINT=http://100.64.0.2:8000
|
||||
LLM_MODEL=
|
||||
LLM_CHAT_MODEL=
|
||||
# 30s is the local-network default. Over WAN + VPN, with the voice and
|
||||
# collocation passes sending a whole document, that truncates real work — the
|
||||
# request is a hard deadline on Complete, and a timeout surfaces as the same
|
||||
# warm 502 as an unreachable model. 90s leaves headroom without letting a
|
||||
# genuinely wedged backend hang the pass forever.
|
||||
LLM_TIMEOUT=90s
|
||||
|
||||
# --- Read-aloud (Piper sidecars) ---------------------------------------------
|
||||
# Endpoints are wired in docker-compose.yml; these pick the voice each sidecar
|
||||
# loads. Changing one means recreating that container so it downloads the model.
|
||||
#
|
||||
# A language is routable only when both halves are set — a TTS_ENDPOINT_XX with
|
||||
# no TTS_VOICE_XX reads as "no voice for this language" and the browser's own
|
||||
# synthesizer takes over, rather than as an instance that errors on every
|
||||
# request. Adding es is a compose service plus a pair of lines here.
|
||||
#
|
||||
# pt_PT-tugão-medium is the only European Portuguese voice Piper ships; every
|
||||
# other pt model in the catalogue is Brazilian. French has the opposite
|
||||
# property — every fr voice in the catalogue is fr_FR — so there is no wrong
|
||||
# country to land on and no non-ASCII name to trip the downloader.
|
||||
TTS_VOICE_EN=en_US-amy-medium
|
||||
TTS_VOICE_ZH=zh_CN-huayan-medium
|
||||
TTS_VOICE_PT=pt_PT-tugão-medium
|
||||
TTS_VOICE_FR=fr_FR-siwis-medium
|
||||
# Mexican, not peninsular — the es pack is written in neutral Latin American
|
||||
# Spanish, and es_ES-davefx-medium would read it in the accent it avoids.
|
||||
TTS_VOICE_ES=es_MX-ald-medium
|
||||
TTS_AUDIO_FORMAT=mp3
|
||||
TTS_TIMEOUT=15s
|
||||
|
||||
# --- Auth (Authentik OIDC) ---------------------------------------------------
|
||||
# NOT OPTIONAL HERE. Authentik already runs on this host; set all three and
|
||||
# Petal authenticates for itself.
|
||||
#
|
||||
# Leave any of them unset and Petal REFUSES TO START, because the alternative is
|
||||
# worse: it would otherwise fall back to resolving every anonymous request to
|
||||
# the single `local` user, handing the open internet full read and write over
|
||||
# every document in the database. That fallback is right on a laptop and a
|
||||
# catastrophe on this host, so the guard is on for any non-loopback BASE_URL.
|
||||
# See PETAL_REQUIRE_AUTH below.
|
||||
#
|
||||
# AUTHENTIK_URL is the provider's issuer, and the redirect URI to register in
|
||||
# Authentik is https://petal.parodia.dev/auth/callback.
|
||||
AUTHENTIK_URL=https://auth.parodia.dev/application/o/petal/
|
||||
AUTHENTIK_CLIENT_ID=petal
|
||||
AUTHENTIK_CLIENT_SECRET=
|
||||
|
||||
# Who may sign in: comma-separated subject ids and/or emails.
|
||||
#
|
||||
# SET THIS. Empty means everyone Authentik authenticates, and Authentik on this
|
||||
# host fronts half a dozen applications — being a valid user there is not the
|
||||
# same as belonging in someone's private journal. An empty value is legal (a
|
||||
# single-household instance may genuinely want it) and says so loudly in the
|
||||
# startup log every boot.
|
||||
#
|
||||
# An email is knowable in advance; a subject id is an opaque uuid nobody can
|
||||
# know before that person's first login. Use emails to invite, subject ids to
|
||||
# pin.
|
||||
PETAL_ALLOWED_SUBS=her@example.com,me@example.com
|
||||
|
||||
# The guard itself. Defaulted from BASE_URL — loopback origins run open, real
|
||||
# ones demand a login — so it does not normally need setting. Set it to false
|
||||
# only for a deployment genuinely reachable from nowhere but a trusted network,
|
||||
# and understand that it means anyone who reaches Petal is the `local` user.
|
||||
# PETAL_REQUIRE_AUTH=true
|
||||
@@ -0,0 +1,24 @@
|
||||
[Unit]
|
||||
Description=Petal writing editor (millenia)
|
||||
# Petal ran unsupervised for a long time -- a bare ./petal with PPID 1, no unit
|
||||
# and no screen session -- so a crash or a reboot left it silently down until
|
||||
# somebody noticed. It also wants vLLM up first, though it degrades warmly if
|
||||
# the model is unreachable, so this is Wants and not Requires.
|
||||
After=network-online.target vllm-chat.service
|
||||
Wants=network-online.target vllm-chat.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=reala
|
||||
WorkingDirectory=/home/reala/petal
|
||||
# start.sh carries the environment (ports, LLM endpoint, Piper endpoints and
|
||||
# TTS_PATH) and execs the binary, so the service supervises Petal itself rather
|
||||
# than a shell wrapper.
|
||||
ExecStart=/home/reala/petal/start.sh
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
StandardOutput=append:/home/reala/petal/petal.log
|
||||
StandardError=append:/home/reala/petal/petal.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -2,6 +2,14 @@
|
||||
Description=Piper TTS HTTP server — Chinese voice (read-aloud backend for petal)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Give up loudly instead of retrying forever. This service once failed 26,800+
|
||||
# times over a day without anyone noticing: RestartSec=3 means only ~3 restarts
|
||||
# land inside systemd's default 10s StartLimitIntervalSec, so the default burst
|
||||
# of 5 was never reached and the unit never entered `failed`. Widening the
|
||||
# window to 5 minutes makes a genuinely broken Piper show up in
|
||||
# `systemctl --user --failed` while still riding out transient blips.
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
|
||||
@@ -2,6 +2,14 @@
|
||||
Description=Piper TTS HTTP server (read-aloud backend for petal)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Give up loudly instead of retrying forever. This service once failed 26,800+
|
||||
# times over a day without anyone noticing: RestartSec=3 means only ~3 restarts
|
||||
# land inside systemd's default 10s StartLimitIntervalSec, so the default burst
|
||||
# of 5 was never reached and the unit never entered `failed`. Widening the
|
||||
# window to 5 minutes makes a genuinely broken Piper show up in
|
||||
# `systemctl --user --failed` while still riding out transient blips.
|
||||
StartLimitIntervalSec=300
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# Piper neural-TTS HTTP server — the read-aloud backend Petal proxies to.
|
||||
#
|
||||
# One image, any voice: the model is named by PIPER_VOICE at runtime and
|
||||
# downloaded into the shared /voices volume on first start. Each Piper server
|
||||
# loads exactly one voice, so a new language is a new service in
|
||||
# docker-compose.yml, not a new image (English and Chinese today; pt-PT lands
|
||||
# with the Portuguese pair).
|
||||
#
|
||||
# python:3.12 rather than 3.13 — piper-tts pulls onnxruntime, whose wheel
|
||||
# coverage for 3.13 still lags.
|
||||
FROM python:3.12-slim
|
||||
|
||||
RUN pip install --no-cache-dir "piper-tts[http]" \
|
||||
&& useradd -m -u 10002 piper
|
||||
|
||||
ENV PIPER_VOICE=en_US-amy-medium \
|
||||
PIPER_DATA_DIR=/voices \
|
||||
PIPER_PORT=5000
|
||||
|
||||
RUN mkdir -p /voices && chown piper:piper /voices
|
||||
VOLUME ["/voices"]
|
||||
|
||||
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||
|
||||
USER piper
|
||||
EXPOSE 5000
|
||||
|
||||
# The server has no dedicated health route, so synthesizing a single word is
|
||||
# the honest check: it proves the model loaded, not just that a port is open.
|
||||
HEALTHCHECK --interval=60s --timeout=20s --start-period=180s --retries=3 \
|
||||
CMD python -c "import os,urllib.request,json; \
|
||||
urllib.request.urlopen(urllib.request.Request('http://127.0.0.1:'+os.environ['PIPER_PORT']+'/synthesize', \
|
||||
data=json.dumps({'text':'ok','voice':os.environ['PIPER_VOICE']}).encode(), \
|
||||
headers={'Content-Type':'application/json'}), timeout=15).read(1)"
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fetch the configured voice if the shared volume doesn't have it yet, then
|
||||
# serve it. The download is the only step that needs the internet, and it runs
|
||||
# once per voice for the life of the volume — Petal itself stays offline-first.
|
||||
set -euo pipefail
|
||||
|
||||
voice="${PIPER_VOICE:?PIPER_VOICE must be set}"
|
||||
data_dir="${PIPER_DATA_DIR:-/voices}"
|
||||
port="${PIPER_PORT:-5000}"
|
||||
|
||||
if [ ! -f "${data_dir}/${voice}.onnx" ]; then
|
||||
echo ">> downloading voice ${voice} into ${data_dir}"
|
||||
# piper.download_voices cannot fetch a voice whose name isn't ASCII, and the
|
||||
# only European Portuguese voice in the catalogue is pt_PT-tugão-medium:
|
||||
# the downloader pastes the name straight into the request line, and
|
||||
# http.client encodes that as ASCII, so it dies with UnicodeEncodeError on
|
||||
# the ã before a byte leaves the container. Every pt_BR voice downloads
|
||||
# fine — the failure lands precisely on the voice the pt-PT pair needs.
|
||||
#
|
||||
# So: try the supported path, and fall back to fetching the two files
|
||||
# ourselves with the URL percent-encoded, which is all the downloader was
|
||||
# missing. Same host, same files, same destination names.
|
||||
python -m piper.download_voices "${voice}" --data-dir "${data_dir}" || {
|
||||
echo ">> download_voices failed for ${voice}; fetching directly (non-ASCII voice name)"
|
||||
python - "${voice}" "${data_dir}" <<'PY'
|
||||
import json, sys, urllib.parse, urllib.request
|
||||
|
||||
voice, data_dir = sys.argv[1], sys.argv[2]
|
||||
BASE = "https://huggingface.co/rhasspy/piper-voices/resolve/main/"
|
||||
|
||||
catalogue = json.load(urllib.request.urlopen(BASE + "voices.json", timeout=120))
|
||||
entry = catalogue.get(voice)
|
||||
if entry is None:
|
||||
sys.exit(f"no voice named {voice!r} in the catalogue")
|
||||
|
||||
# The catalogue keys the files by repo path; only the model and its config are
|
||||
# needed to serve (MODEL_CARD is licence text).
|
||||
for path in entry["files"]:
|
||||
if not path.endswith((".onnx", ".onnx.json")):
|
||||
continue
|
||||
url = BASE + urllib.parse.quote(path)
|
||||
dest = f"{data_dir}/{path.rsplit('/', 1)[-1]}"
|
||||
print(f">> {url} -> {dest}", flush=True)
|
||||
with urllib.request.urlopen(url, timeout=600) as r, open(dest, "wb") as out:
|
||||
while chunk := r.read(1 << 20):
|
||||
out.write(chunk)
|
||||
PY
|
||||
}
|
||||
fi
|
||||
|
||||
echo ">> serving ${voice} on :${port}"
|
||||
# 0.0.0.0 is safe here: the container sits on Petal's internal compose network
|
||||
# with no published ports, so only Petal can reach it.
|
||||
exec python -m piper.http_server \
|
||||
-m "${voice}" \
|
||||
--data-dir "${data_dir}" \
|
||||
--host 0.0.0.0 --port "${port}"
|
||||
Executable
+149
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env bash
|
||||
# One-time setup: put Petal's data directory on an encrypted volume.
|
||||
#
|
||||
# sudo ./setup-encrypted-data.sh # create + migrate + persist
|
||||
# sudo ./setup-encrypted-data.sh --status # report, change nothing
|
||||
#
|
||||
# WHAT THIS DOES AND DOES NOT PROTECT
|
||||
# -----------------------------------
|
||||
# The volume auto-unlocks from a keyfile stored on the same host. That is a
|
||||
# deliberate choice (availability over paranoia), and it means:
|
||||
#
|
||||
# protects against : a decommissioned or resold disk, someone reading the
|
||||
# raw block device, casual browsing of a filesystem-level
|
||||
# snapshot that does not include /etc
|
||||
# does NOT protect : anyone who takes the whole VM image -- they get
|
||||
# against /etc/petal/dataset.key along with the ciphertext; and
|
||||
# anything at all once the host is running and mounted
|
||||
#
|
||||
# For real protection against a provider-side snapshot the key has to live off
|
||||
# the box (fetched over the VPN at boot). That was considered and not chosen.
|
||||
#
|
||||
# WHY LUKS-ON-A-FILE RATHER THAN gocryptfs
|
||||
# ----------------------------------------
|
||||
# Petal is SQLite in WAL mode. WAL needs a shared-memory index (-shm) mapped
|
||||
# consistently across processes, and FUSE filesystems have a long history of
|
||||
# subtle mmap/locking differences. A LUKS block device with ext4 on top behaves
|
||||
# exactly like a normal disk to SQLite, which is the only guarantee worth having
|
||||
# under a database.
|
||||
set -euo pipefail
|
||||
|
||||
IMG="${IMG:-/var/lib/petal-crypt.img}"
|
||||
SIZE="${SIZE:-8G}"
|
||||
MAPPER_NAME="${MAPPER_NAME:-petal-data}"
|
||||
KEYFILE="${KEYFILE:-/etc/petal/dataset.key}"
|
||||
MOUNTPOINT="${MOUNTPOINT:-/home/reala/petal/data}"
|
||||
STACK_DIR="${STACK_DIR:-/home/reala/petal}"
|
||||
OWNER_UID="${OWNER_UID:-1001}"
|
||||
OWNER_GID="${OWNER_GID:-1001}"
|
||||
|
||||
[ "$(id -u)" -eq 0 ] || { echo "must run as root" >&2; exit 1; }
|
||||
|
||||
# systemd-cryptsetup ships the generator that turns /etc/crypttab into units.
|
||||
# On a minimal Debian it is NOT installed, and without it crypttab is silently
|
||||
# ignored -- the volume simply never unlocks at boot. Found the hard way.
|
||||
if [ ! -x /usr/lib/systemd/system-generators/systemd-cryptsetup-generator ]; then
|
||||
echo "!! systemd-cryptsetup-generator is missing: /etc/crypttab would be ignored at boot."
|
||||
echo " install it first: apt-get install systemd-cryptsetup"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
status() {
|
||||
echo "image : $IMG $( [ -f "$IMG" ] && echo "($(du -h --apparent-size "$IMG" | cut -f1) apparent, $(du -h "$IMG" | cut -f1) on disk)" || echo "(absent)")"
|
||||
echo "mapper : /dev/mapper/$MAPPER_NAME $( [ -e "/dev/mapper/$MAPPER_NAME" ] && echo "(open)" || echo "(closed)")"
|
||||
echo "keyfile : $KEYFILE $( [ -f "$KEYFILE" ] && echo "(present, mode $(stat -c%a "$KEYFILE"))" || echo "(absent)")"
|
||||
echo "mountpoint : $MOUNTPOINT $(mountpoint -q "$MOUNTPOINT" && echo "(mounted)" || echo "(NOT mounted)")"
|
||||
grep -q "^$MAPPER_NAME " /etc/crypttab 2>/dev/null && echo "crypttab : present" || echo "crypttab : MISSING"
|
||||
grep -q " $MOUNTPOINT " /etc/fstab 2>/dev/null && echo "fstab : present" || echo "fstab : MISSING"
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--status" ]; then status; exit 0; fi
|
||||
|
||||
if [ -f "$IMG" ]; then
|
||||
echo "$IMG already exists — refusing to re-create. Use --status." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ">> stopping the stack so nothing is writing to $MOUNTPOINT"
|
||||
if [ -f "$STACK_DIR/docker-compose.yml" ]; then
|
||||
( cd "$STACK_DIR" && docker compose down )
|
||||
fi
|
||||
|
||||
echo ">> generating keyfile $KEYFILE (root-only)"
|
||||
install -d -m 0700 "$(dirname "$KEYFILE")"
|
||||
if [ ! -f "$KEYFILE" ]; then
|
||||
dd if=/dev/urandom of="$KEYFILE" bs=512 count=1 status=none
|
||||
chmod 0400 "$KEYFILE"
|
||||
fi
|
||||
|
||||
echo ">> creating $SIZE sparse image at $IMG"
|
||||
truncate -s "$SIZE" "$IMG"
|
||||
chmod 0600 "$IMG"
|
||||
|
||||
echo ">> LUKS format + open"
|
||||
cryptsetup luksFormat --type luks2 --batch-mode --key-file "$KEYFILE" "$IMG"
|
||||
cryptsetup luksOpen --key-file "$KEYFILE" "$IMG" "$MAPPER_NAME"
|
||||
|
||||
echo ">> mkfs + mount"
|
||||
mkfs.ext4 -q -L petal-data "/dev/mapper/$MAPPER_NAME"
|
||||
|
||||
# Preserve whatever is already in the plaintext directory, then swap it in.
|
||||
STAGING=""
|
||||
if [ -d "$MOUNTPOINT" ] && [ -n "$(ls -A "$MOUNTPOINT" 2>/dev/null)" ]; then
|
||||
STAGING="$(mktemp -d)"
|
||||
echo ">> preserving existing plaintext data -> $STAGING"
|
||||
cp -a "$MOUNTPOINT/." "$STAGING/"
|
||||
|
||||
# Critical, and easy to miss: mounting over a directory HIDES its contents,
|
||||
# it does not remove them. Skip this and the original plaintext petal.db sits
|
||||
# on the unencrypted root filesystem forever, invisible under the mount,
|
||||
# defeating the entire exercise. Clear the mountpoint before mounting.
|
||||
echo ">> shredding the plaintext originals under the mountpoint"
|
||||
find "$MOUNTPOINT" -mindepth 1 -type f -exec shred -uz {} + 2>/dev/null || true
|
||||
find "$MOUNTPOINT" -mindepth 1 -depth -type d -exec rmdir {} + 2>/dev/null || true
|
||||
[ -z "$(ls -A "$MOUNTPOINT" 2>/dev/null)" ] || {
|
||||
echo "!! $MOUNTPOINT is not empty after cleanup; refusing to mount over live data" >&2
|
||||
echo " (data is preserved at $STAGING)" >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
mkdir -p "$MOUNTPOINT"
|
||||
mount "/dev/mapper/$MAPPER_NAME" "$MOUNTPOINT"
|
||||
|
||||
if [ -n "$STAGING" ]; then
|
||||
echo ">> restoring data onto the encrypted volume"
|
||||
cp -a "$STAGING/." "$MOUNTPOINT/"
|
||||
find "$STAGING" -type f -exec shred -uz {} + 2>/dev/null || true
|
||||
rm -rf "$STAGING"
|
||||
fi
|
||||
|
||||
# Mount-liveness sentinel: docker-compose bind-mounts this file with
|
||||
# create_host_path:false, so an unmounted volume becomes a loud container start
|
||||
# failure rather than Petal quietly serving an empty database.
|
||||
touch "$MOUNTPOINT/.volume-ok"
|
||||
|
||||
chown -R "$OWNER_UID:$OWNER_GID" "$MOUNTPOINT"
|
||||
|
||||
echo ">> persisting across reboots"
|
||||
# systemd-cryptsetup loop-mounts a regular file source on its own.
|
||||
if ! grep -q "^$MAPPER_NAME " /etc/crypttab 2>/dev/null; then
|
||||
echo "$MAPPER_NAME $IMG $KEYFILE luks,nofail" >> /etc/crypttab
|
||||
fi
|
||||
# nofail: a problem here must never wedge the boot of a host running half a
|
||||
# dozen other services.
|
||||
# x-systemd.before=docker.service is the important one: without it Docker can
|
||||
# start first, find $MOUNTPOINT empty, and bring Petal up against a blank
|
||||
# unencrypted directory that the real volume then hides.
|
||||
if ! grep -q " $MOUNTPOINT " /etc/fstab 2>/dev/null; then
|
||||
echo "/dev/mapper/$MAPPER_NAME $MOUNTPOINT ext4 defaults,nofail,x-systemd.requires=/dev/mapper/$MAPPER_NAME,x-systemd.before=docker.service 0 2" >> /etc/fstab
|
||||
fi
|
||||
systemctl daemon-reload
|
||||
|
||||
echo ">> restarting the stack"
|
||||
if [ -f "$STACK_DIR/docker-compose.yml" ]; then
|
||||
( cd "$STACK_DIR" && docker compose up -d )
|
||||
fi
|
||||
|
||||
echo
|
||||
status
|
||||
@@ -0,0 +1,40 @@
|
||||
[Unit]
|
||||
Description=Expose millenia's vLLM chat server on the headscale interface only
|
||||
# Why a forwarder instead of just rebinding vLLM: vllm-chat.service is shared.
|
||||
# Petal, Gogobee and Open WebUI all talk to 127.0.0.1:8000, and Open WebUI keeps
|
||||
# its endpoint in its own database rather than in env, so moving vLLM's bind
|
||||
# address would mean editing three consumers and reloading a 35B AWQ model
|
||||
# (minutes of downtime for all of them). This adds a second door instead: local
|
||||
# callers keep loopback untouched, and only the headscale address gains a
|
||||
# listener. Nothing about vllm-chat changes.
|
||||
#
|
||||
# Deliberately NOT 0.0.0.0 — this reaches a public VPS over the VPN, and the
|
||||
# LAN has no business seeing an unauthenticated inference endpoint.
|
||||
After=network-online.target tailscaled.service vllm-chat.service
|
||||
Wants=network-online.target
|
||||
BindsTo=vllm-chat.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
# fork: one child per connection, so a single client can't block the others.
|
||||
# reuseaddr: survive a restart while sockets are still in TIME_WAIT.
|
||||
# The bind address is millenia's headscale IP; if tailscaled hasn't brought the
|
||||
# interface up yet the bind fails and Restart retries until it has.
|
||||
ExecStart=/usr/bin/socat -d TCP-LISTEN:8000,bind=100.64.0.2,fork,reuseaddr TCP:127.0.0.1:8000
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
# Long generations hold a connection open; don't let systemd reap a healthy one.
|
||||
TimeoutStopSec=10
|
||||
|
||||
# The process only shuttles bytes between two sockets — give it nothing else.
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectControlGroups=true
|
||||
RestrictAddressFamilies=AF_INET AF_INET6
|
||||
DynamicUser=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,211 @@
|
||||
# Petal on the parodia.dev VPS.
|
||||
#
|
||||
# docker compose up -d --build
|
||||
#
|
||||
# Fronted by the host's existing Traefik (external `traefik` network, the
|
||||
# `web-secure` entrypoint and the `default` cert resolver — same convention the
|
||||
# other services on this box use). Petal itself never binds a host port; the
|
||||
# only way in is through Traefik over HTTPS.
|
||||
#
|
||||
# Read-aloud runs as two sibling containers rather than host systemd services:
|
||||
# each Piper HTTP server loads exactly one voice, the host has no lingering
|
||||
# user session to keep systemd units alive, and keeping them on the internal
|
||||
# network means the TTS ports are unreachable from anywhere but Petal.
|
||||
#
|
||||
# Copy deploy/petal.env.example to .env before the first `up`.
|
||||
|
||||
name: petal
|
||||
|
||||
services:
|
||||
petal:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: petal:local
|
||||
container_name: petal
|
||||
restart: unless-stopped
|
||||
# ./data is a bind mount, so the image's own `petal` user (uid 10001) has no
|
||||
# claim on it — the host's ownership wins and the container can't open
|
||||
# petal.db. Run as whoever owns the stack directory instead. Keeping it the
|
||||
# host user (rather than chowning ./data to 10001) is deliberate: the backup
|
||||
# script gzips snapshots in place from the host, so the host account needs
|
||||
# write access to the same directory. Still never root.
|
||||
user: "${PETAL_UID:-1001}:${PETAL_GID:-1001}"
|
||||
env_file: .env
|
||||
environment:
|
||||
# Fixed by the image layout; kept here so they're visible at a glance.
|
||||
PORT: "8080"
|
||||
DATABASE_PATH: /data/petal.db
|
||||
IMAGE_DIR: /data/images
|
||||
TTS_CACHE_DIR: /data/tts
|
||||
# DreamDict's built dictionary, read-only, deployed into the data volume
|
||||
# (see deploy/README.md). Absent it, word lookups fall back to the
|
||||
# embedded English/Chinese datasets rather than failing.
|
||||
DICT_PATH: /data/dict.db
|
||||
# Piper sidecars. Each server loads one voice, so English and Chinese are
|
||||
# separate containers; the handler maps language → instance from config.
|
||||
TTS_ENDPOINT: http://piper-en:5000
|
||||
TTS_ENDPOINT_ZH: http://piper-zh:5000
|
||||
# A language is discovered from the TTS_ENDPOINT_<LANG>/TTS_VOICE_<LANG>
|
||||
# pair, so fr and es cost a service and two lines rather than a code
|
||||
# change. <LANG> is the base tag — an env var name can't hold pt-PT's
|
||||
# hyphen, and there is one Portuguese voice loaded either way.
|
||||
TTS_ENDPOINT_PT: http://piper-pt:5000
|
||||
TTS_ENDPOINT_FR: http://piper-fr:5000
|
||||
TTS_ENDPOINT_ES: http://piper-es:5000
|
||||
# The sidecars run piper-tts 1.6.0, which serves synthesis on
|
||||
# /synthesize; millenia's older server keeps the default "/".
|
||||
TTS_PATH: /synthesize
|
||||
# The companion's bedtime nag and night mode read the local clock.
|
||||
TZ: ${TZ:-Europe/Lisbon}
|
||||
volumes:
|
||||
# A bind mount, not a named volume: petal.db must be trivially reachable
|
||||
# from the host for the nightly backup and for a restore.
|
||||
- ./data:/data
|
||||
# Mount-liveness guard. On the VPS ./data is an encrypted LUKS volume, and
|
||||
# the mountpoint directory still exists when that volume is NOT mounted —
|
||||
# so without this, a boot where the unlock failed would start Petal
|
||||
# against an empty unencrypted directory and quietly serve a blank
|
||||
# database. .volume-ok lives on the encrypted filesystem, and
|
||||
# create_host_path: false turns its absence into a container start
|
||||
# failure instead. Harmless elsewhere: create the file once and it is a
|
||||
# no-op. See deploy/README.md §6.
|
||||
- type: bind
|
||||
source: ./data/.volume-ok
|
||||
target: /data/.volume-ok
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: false
|
||||
networks:
|
||||
- traefik
|
||||
- internal
|
||||
depends_on:
|
||||
- piper-en
|
||||
- piper-zh
|
||||
- piper-pt
|
||||
labels:
|
||||
traefik.enable: "true"
|
||||
traefik.docker.network: traefik
|
||||
traefik.http.routers.petal.rule: Host(`${PETAL_HOST:-petal.parodia.dev}`)
|
||||
traefik.http.routers.petal.entrypoints: web-secure
|
||||
traefik.http.routers.petal.tls: "true"
|
||||
traefik.http.routers.petal.tls.certResolver: default
|
||||
traefik.http.routers.petal.service: petal
|
||||
# No edge gate: Petal authenticates for itself now (Authentik OIDC), so
|
||||
# every /api route answers 401 without a session and the only thing served
|
||||
# to an anonymous visitor is the app shell and its sign-in redirect. The
|
||||
# basic-auth middleware that stood here until Phase 16 — plus the separate
|
||||
# unauthenticated router /api/health needed to escape it — is gone; a
|
||||
# second password in front of a real login is just one more thing to lose.
|
||||
traefik.http.routers.petal.middlewares: compression@file,petal-headers
|
||||
traefik.http.services.petal.loadbalancer.server.port: "8080"
|
||||
# HSTS is the edge's business — it is a statement about the TLS
|
||||
# termination, which happens here and not in the container.
|
||||
#
|
||||
# The Content-Security-Policy that used to sit alongside it has moved into
|
||||
# the app (see securityHeaders in cmd/server/main.go). customresponseheaders
|
||||
# *overwrites*, so a policy set here would silently replace the stricter
|
||||
# one an individual route chooses for itself — which is exactly what the
|
||||
# image store does to keep an uploaded SVG from running as a page. A rule
|
||||
# the edge can quietly undo is not a rule. X-Content-Type-Options and
|
||||
# Referrer-Policy moved with it for the same reason: one place to read,
|
||||
# and no dependence on this file being deployed alongside the binary.
|
||||
traefik.http.middlewares.petal-headers.headers.customresponseheaders.Strict-Transport-Security: max-age=31536000; includeSubDomains
|
||||
|
||||
piper-en:
|
||||
build:
|
||||
context: deploy/piper
|
||||
image: petal-piper:local
|
||||
container_name: petal-piper-en
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PIPER_VOICE: ${TTS_VOICE_EN:-en_US-amy-medium}
|
||||
volumes:
|
||||
- piper-voices:/voices
|
||||
networks:
|
||||
- internal
|
||||
|
||||
piper-zh:
|
||||
build:
|
||||
context: deploy/piper
|
||||
image: petal-piper:local
|
||||
container_name: petal-piper-zh
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PIPER_VOICE: ${TTS_VOICE_ZH:-zh_CN-huayan-medium}
|
||||
volumes:
|
||||
- piper-voices:/voices
|
||||
networks:
|
||||
- internal
|
||||
|
||||
# European Portuguese, for the pt-PT pair. pt_PT-tugão-medium is the *only*
|
||||
# European voice in Piper's catalogue — the other five Portuguese models are
|
||||
# all pt_BR — so the default anyone reaches for is the Brazilian one, exactly
|
||||
# as it was with the Hunspell dictionary in Phase 21. Named here rather than
|
||||
# left to the image default for that reason.
|
||||
piper-pt:
|
||||
build:
|
||||
context: deploy/piper
|
||||
image: petal-piper:local
|
||||
container_name: petal-piper-pt
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PIPER_VOICE: ${TTS_VOICE_PT:-pt_PT-tugão-medium}
|
||||
volumes:
|
||||
- piper-voices:/voices
|
||||
networks:
|
||||
- internal
|
||||
|
||||
# French, for the fr pair. The opposite situation to Portuguese: every French
|
||||
# voice Piper ships is fr_FR, so there is no wrong country to land on by
|
||||
# default, and the name is plain ASCII so the entrypoint's percent-encoded
|
||||
# fallback (added for tugão) never has to fire. siwis-medium to match the
|
||||
# register of the other three.
|
||||
piper-fr:
|
||||
build:
|
||||
context: deploy/piper
|
||||
image: petal-piper:local
|
||||
container_name: petal-piper-fr
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PIPER_VOICE: ${TTS_VOICE_FR:-fr_FR-siwis-medium}
|
||||
volumes:
|
||||
- piper-voices:/voices
|
||||
networks:
|
||||
- internal
|
||||
|
||||
# Spanish, for the es pair — and the Portuguese trap rather than the French
|
||||
# one. Piper's catalogue has nine Spanish voices, six of them es_ES, and the
|
||||
# obvious pick (es_ES-davefx-medium, which the build plan itself named) is
|
||||
# peninsular. The es pack is written in neutral Latin American Spanish, so a
|
||||
# Castilian voice would read it aloud in the accent the copy was written to
|
||||
# avoid — the same wrong-country default that pt-PT hit through packaging,
|
||||
# arriving here through the voice list. Only two Latin American voices exist,
|
||||
# es_AR-daniela-high and es_MX; Mexican is the neutral broadcast standard and
|
||||
# ald-medium matches the register of the other four. ASCII, so the
|
||||
# percent-encoded download fallback added for tugão never has to fire.
|
||||
piper-es:
|
||||
build:
|
||||
context: deploy/piper
|
||||
image: petal-piper:local
|
||||
container_name: petal-piper-es
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PIPER_VOICE: ${TTS_VOICE_ES:-es_MX-ald-medium}
|
||||
volumes:
|
||||
- piper-voices:/voices
|
||||
networks:
|
||||
- internal
|
||||
|
||||
networks:
|
||||
# Created and owned by the host's Traefik stack.
|
||||
traefik:
|
||||
external: true
|
||||
# Petal ↔ Piper only. Not reachable from the internet or the other stacks.
|
||||
internal:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
# Downloaded voice models, shared read-mostly by both Piper instances so the
|
||||
# same model is never fetched twice.
|
||||
piper-voices:
|
||||
@@ -3,7 +3,11 @@ module gitea.parodia.dev/drwily/petal
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/coreos/go-oidc/v3 v3.20.0
|
||||
github.com/go-chi/chi/v5 v5.3.0
|
||||
github.com/go-jose/go-jose/v4 v4.1.4
|
||||
github.com/prosolis/dreamdict v0.0.0-20260727163219-302a39d5c768
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
modernc.org/sqlite v1.53.0
|
||||
)
|
||||
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
github.com/coreos/go-oidc/v3 v3.20.0 h1:EtE0WIBHk03N+DqGkY4+UONzzZHk7amKt6IyNd7OsZE=
|
||||
github.com/coreos/go-oidc/v3 v3.20.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
|
||||
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@@ -12,10 +16,14 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/prosolis/dreamdict v0.0.0-20260727163219-302a39d5c768 h1:+7NP78QlAVcXMviA23cMBtxwvYwDhADnABa3JBhbApI=
|
||||
github.com/prosolis/dreamdict v0.0.0-20260727163219-302a39d5c768/go.mod h1:s4D+Q++6Qjq8X7/ZXEMStGXA6tScm4GadFo0+HF4BHY=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
||||
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
// Package auth answers one question for every API request: who is asking?
|
||||
//
|
||||
// Until now Petal ran as a single hardcoded user and every query passed
|
||||
// db.LocalUserID directly. That made the identity of the caller a compile-time
|
||||
// constant scattered across ~35 call sites — nothing a real login could ever
|
||||
// replace without touching all of them. This package moves that identity into
|
||||
// the request context, resolved once by [Middleware], so handlers read the
|
||||
// current user instead of naming one.
|
||||
//
|
||||
// The identity itself still comes from [StaticResolver] today, which returns
|
||||
// the same local user for everyone. Swapping in Authentik later means writing
|
||||
// one Resolver (validate the session cookie → user id) and changing the single
|
||||
// line in main.go that constructs it. No handler changes.
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
)
|
||||
|
||||
// ctxKey is unexported so no other package can plant a user id in the context
|
||||
// without going through [WithUser].
|
||||
type ctxKey struct{}
|
||||
|
||||
// WithUser returns a copy of ctx carrying userID as the authenticated caller.
|
||||
// Handlers never call this; [Middleware] does, and tests use it to build a
|
||||
// request that looks authenticated.
|
||||
func WithUser(ctx context.Context, userID string) context.Context {
|
||||
return context.WithValue(ctx, ctxKey{}, userID)
|
||||
}
|
||||
|
||||
// UserID returns the authenticated user id carried by ctx, or "" if the request
|
||||
// never passed through [Middleware].
|
||||
//
|
||||
// Returning "" rather than panicking keeps an unauthenticated request failing
|
||||
// *closed*: every query in Petal is scoped `WHERE user_id = ?`, so an empty id
|
||||
// matches no rows — a missing middleware leaks nothing, it just returns empty
|
||||
// results. Handlers may therefore use the value directly without checking it.
|
||||
func UserID(ctx context.Context) string {
|
||||
id, _ := ctx.Value(ctxKey{}).(string)
|
||||
return id
|
||||
}
|
||||
|
||||
// Resolver maps an inbound request to the id of the user making it. Returning
|
||||
// an error, or an empty id, rejects the request with a 401.
|
||||
//
|
||||
// This is the seam a real identity provider drops into: an Authentik resolver
|
||||
// validates the session cookie and returns the user id it maps to.
|
||||
type Resolver interface {
|
||||
Resolve(r *http.Request) (string, error)
|
||||
}
|
||||
|
||||
// StaticResolver resolves every request to the same user id, ignoring the
|
||||
// request entirely. It is how Petal runs today — a single-user app whose one
|
||||
// user now arrives through the same path a logged-in user eventually will.
|
||||
type StaticResolver string
|
||||
|
||||
// Resolve implements [Resolver].
|
||||
func (s StaticResolver) Resolve(*http.Request) (string, error) { return string(s), nil }
|
||||
|
||||
// Middleware resolves the caller with res and stores the result in the request
|
||||
// context for [UserID]. Requests the resolver rejects — or resolves to an empty
|
||||
// id — never reach the handler; they get a 401 instead.
|
||||
func Middleware(res Resolver) func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
userID, err := res.Resolve(r)
|
||||
if err != nil || userID == "" {
|
||||
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r.WithContext(WithUser(r.Context(), userID)))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// errResolver rejects every request, standing in for a real resolver that finds
|
||||
// no valid session.
|
||||
type errResolver struct{ err error }
|
||||
|
||||
func (e errResolver) Resolve(*http.Request) (string, error) { return "", e.err }
|
||||
|
||||
func TestUserIDRoundTrip(t *testing.T) {
|
||||
ctx := WithUser(context.Background(), "alice")
|
||||
if got := UserID(ctx); got != "alice" {
|
||||
t.Fatalf("UserID = %q, want alice", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A request that never passed through the middleware must report no user rather
|
||||
// than panicking — every query is `WHERE user_id = ?`, so an empty id fails
|
||||
// closed (matches nothing) instead of falling back to some default account.
|
||||
func TestUserIDAbsentIsEmpty(t *testing.T) {
|
||||
if got := UserID(context.Background()); got != "" {
|
||||
t.Fatalf("UserID on bare context = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMiddlewareInjectsResolvedUser(t *testing.T) {
|
||||
var seen string
|
||||
h := Middleware(StaticResolver("local"))(http.HandlerFunc(
|
||||
func(_ http.ResponseWriter, r *http.Request) { seen = UserID(r.Context()) },
|
||||
))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
||||
|
||||
if seen != "local" {
|
||||
t.Fatalf("handler saw user %q, want local", seen)
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Both rejection paths — an explicit error and a silent empty id — must 401
|
||||
// without ever entering the handler. The empty case matters most: a resolver
|
||||
// that returns ("", nil) by mistake would otherwise hand handlers an empty user
|
||||
// id, and while that fails closed at the SQL layer, it should never get there.
|
||||
func TestMiddlewareRejectsUnresolved(t *testing.T) {
|
||||
for name, res := range map[string]Resolver{
|
||||
"resolver error": errResolver{err: http.ErrNoCookie},
|
||||
"empty user id": StaticResolver(""),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
called := false
|
||||
h := Middleware(res)(http.HandlerFunc(
|
||||
func(http.ResponseWriter, *http.Request) { called = true },
|
||||
))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
||||
|
||||
if called {
|
||||
t.Fatal("handler ran for an unauthenticated request")
|
||||
}
|
||||
if rec.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want 401", rec.Code)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/coreos/go-oidc/v3/oidc"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
// Temporary cookies that carry one login attempt from /auth/login to
|
||||
// /auth/callback. They live for ten minutes and are cleared the moment the
|
||||
// callback runs.
|
||||
const (
|
||||
stateCookie = "petal_oidc_state"
|
||||
nonceCookie = "petal_oidc_nonce"
|
||||
pkceCookie = "petal_oidc_pkce"
|
||||
|
||||
loginAttemptTTL = 600 // seconds
|
||||
)
|
||||
|
||||
// Options configures the OIDC client.
|
||||
type Options struct {
|
||||
IssuerURL string // Authentik's issuer, e.g. https://auth.example.com/application/o/petal/
|
||||
ClientID string
|
||||
ClientSecret string
|
||||
BaseURL string // Petal's public base URL; the redirect URI is derived from it
|
||||
Allowed Allowlist
|
||||
}
|
||||
|
||||
// OIDC implements Petal's half of an authorization-code login against
|
||||
// Authentik: /auth/login starts it, /auth/callback finishes it by provisioning
|
||||
// the account and issuing a session, /auth/logout ends it.
|
||||
//
|
||||
// Petal is the OIDC client itself rather than trusting a proxy-injected header.
|
||||
// The header approach is far less code, but it is only safe while the container
|
||||
// is unreachable except through that proxy — an invariant enforced by network
|
||||
// configuration, not by anything in the repository, on a public host that also
|
||||
// runs half a dozen other services. Petal holds someone's private journals; it
|
||||
// should be safe to expose directly.
|
||||
type OIDC struct {
|
||||
opts Options
|
||||
sessions *SessionStore
|
||||
users *UserStore
|
||||
secure bool
|
||||
|
||||
// The provider is discovered over the network, which means it can fail at
|
||||
// startup for reasons that have nothing to do with Petal. Discovery is
|
||||
// therefore lazy and retried: an Authentik outage blocks new logins but
|
||||
// leaves every existing session working, since those only need the database.
|
||||
mu sync.Mutex
|
||||
provider *oidc.Provider
|
||||
oauth *oauth2.Config
|
||||
verifier *oidc.IDTokenVerifier
|
||||
}
|
||||
|
||||
// NewOIDC builds the login flow. It attempts discovery once so a misconfigured
|
||||
// issuer shows up in the startup log rather than on the writer's first login,
|
||||
// but a failure here is not fatal.
|
||||
func NewOIDC(ctx context.Context, opts Options, sessions *SessionStore, users *UserStore) *OIDC {
|
||||
o := &OIDC{
|
||||
opts: opts,
|
||||
sessions: sessions,
|
||||
users: users,
|
||||
secure: strings.HasPrefix(strings.ToLower(opts.BaseURL), "https://"),
|
||||
}
|
||||
if err := o.discover(ctx); err != nil {
|
||||
log.Printf("auth: OIDC discovery failed (%v) — login will retry on demand", err)
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// RedirectURI is the callback Authentik must have registered for this client.
|
||||
func (o *OIDC) RedirectURI() string {
|
||||
return strings.TrimSuffix(o.opts.BaseURL, "/") + "/auth/callback"
|
||||
}
|
||||
|
||||
// discover resolves the provider metadata and builds the oauth2 config.
|
||||
func (o *OIDC) discover(ctx context.Context) error {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
if o.provider != nil {
|
||||
return nil
|
||||
}
|
||||
// The issuer is passed through exactly as configured, trailing slash and
|
||||
// all: OIDC requires the discovered issuer to match the requested one
|
||||
// byte-for-byte, and Authentik's ends in a slash. (go-oidc trims it itself
|
||||
// when building the .well-known URL, so a slash here costs nothing.)
|
||||
provider, err := oidc.NewProvider(ctx, o.opts.IssuerURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.provider = provider
|
||||
o.verifier = provider.Verifier(&oidc.Config{ClientID: o.opts.ClientID})
|
||||
o.oauth = &oauth2.Config{
|
||||
ClientID: o.opts.ClientID,
|
||||
ClientSecret: o.opts.ClientSecret,
|
||||
Endpoint: provider.Endpoint(),
|
||||
RedirectURL: o.RedirectURI(),
|
||||
Scopes: []string{oidc.ScopeOpenID, "profile", "email"},
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ready returns the discovered client, discovering it first if an earlier
|
||||
// attempt failed.
|
||||
func (o *OIDC) ready(ctx context.Context) (*oauth2.Config, *oidc.IDTokenVerifier, error) {
|
||||
if err := o.discover(ctx); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
return o.oauth, o.verifier, nil
|
||||
}
|
||||
|
||||
// Routes mounts the login endpoints. Mount at "/auth", outside /api: these are
|
||||
// browser navigations, not API calls, and they must be reachable without a
|
||||
// session — that is their whole purpose.
|
||||
func (o *OIDC) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/login", o.login)
|
||||
r.Get("/callback", o.callback)
|
||||
// POST only. Signing out is a state change, and SameSite=Lax deliberately
|
||||
// *does* send the session cookie on a top-level cross-site GET — so a GET
|
||||
// route here means any page on the internet can sign her out mid-draft by
|
||||
// linking to it, or embedding it as an image. Small harm, free to remove.
|
||||
r.Post("/logout", o.logout)
|
||||
return r
|
||||
}
|
||||
|
||||
// login starts an authorization-code flow with PKCE.
|
||||
func (o *OIDC) login(w http.ResponseWriter, r *http.Request) {
|
||||
// Already signed in? Don't bounce a valid session through the IdP.
|
||||
if _, err := o.sessions.Resolve(r); err == nil {
|
||||
http.Redirect(w, r, "/", http.StatusFound)
|
||||
return
|
||||
}
|
||||
|
||||
conf, _, err := o.ready(r.Context())
|
||||
if err != nil {
|
||||
o.page(w, http.StatusServiceUnavailable,
|
||||
"登录暂时不可用", "Sign-in is unavailable right now",
|
||||
"Petal 联系不上登录服务。请稍后再试。",
|
||||
"Petal can't reach the sign-in service. Please try again in a moment.")
|
||||
return
|
||||
}
|
||||
|
||||
state, err := randomToken()
|
||||
if err != nil {
|
||||
o.page(w, http.StatusInternalServerError, "出了点问题", "Something went wrong", "请再试一次。", "Please try again.")
|
||||
return
|
||||
}
|
||||
nonce, err := randomToken()
|
||||
if err != nil {
|
||||
o.page(w, http.StatusInternalServerError, "出了点问题", "Something went wrong", "请再试一次。", "Please try again.")
|
||||
return
|
||||
}
|
||||
pkce := oauth2.GenerateVerifier()
|
||||
|
||||
// state defends the callback against CSRF (a forged callback can't know the
|
||||
// cookie); nonce ties the returned ID token to this attempt; PKCE binds the
|
||||
// code to this client even if it leaks in transit.
|
||||
o.setTemp(w, stateCookie, state)
|
||||
o.setTemp(w, nonceCookie, nonce)
|
||||
o.setTemp(w, pkceCookie, pkce)
|
||||
|
||||
http.Redirect(w, r, conf.AuthCodeURL(state,
|
||||
oidc.Nonce(nonce),
|
||||
oauth2.S256ChallengeOption(pkce),
|
||||
), http.StatusFound)
|
||||
}
|
||||
|
||||
// callback completes the flow: verify, allowlist, provision, issue a session.
|
||||
func (o *OIDC) callback(w http.ResponseWriter, r *http.Request) {
|
||||
// Expire the one-shot login cookies up front, not on the way out: every exit
|
||||
// from here writes a response, and a Set-Cookie added after the header is
|
||||
// written is silently dropped. They're read from the request below, so
|
||||
// clearing them on the response now costs nothing.
|
||||
o.clearTemp(w)
|
||||
|
||||
if errParam := r.URL.Query().Get("error"); errParam != "" {
|
||||
o.page(w, http.StatusForbidden,
|
||||
"登录未完成", "Sign-in didn't finish",
|
||||
"登录服务拒绝了这次请求。你可以再试一次。",
|
||||
"The sign-in service turned that request down. You can try again.")
|
||||
return
|
||||
}
|
||||
|
||||
state, err := r.Cookie(stateCookie)
|
||||
if err != nil || state.Value == "" ||
|
||||
subtle.ConstantTimeCompare([]byte(state.Value), []byte(r.URL.Query().Get("state"))) != 1 {
|
||||
o.page(w, http.StatusBadRequest,
|
||||
"这个登录链接过期了", "That sign-in link expired",
|
||||
"请回到 Petal 重新登录。",
|
||||
"Head back to Petal and sign in again.")
|
||||
return
|
||||
}
|
||||
|
||||
conf, verifier, err := o.ready(r.Context())
|
||||
if err != nil {
|
||||
o.page(w, http.StatusServiceUnavailable,
|
||||
"登录暂时不可用", "Sign-in is unavailable right now",
|
||||
"Petal 联系不上登录服务。请稍后再试。",
|
||||
"Petal can't reach the sign-in service. Please try again in a moment.")
|
||||
return
|
||||
}
|
||||
|
||||
pkce, err := r.Cookie(pkceCookie)
|
||||
if err != nil {
|
||||
o.page(w, http.StatusBadRequest, "这个登录链接过期了", "That sign-in link expired",
|
||||
"请回到 Petal 重新登录。", "Head back to Petal and sign in again.")
|
||||
return
|
||||
}
|
||||
|
||||
token, err := conf.Exchange(r.Context(), r.URL.Query().Get("code"), oauth2.VerifierOption(pkce.Value))
|
||||
if err != nil {
|
||||
log.Printf("auth: code exchange failed: %v", err)
|
||||
o.page(w, http.StatusBadGateway, "登录没有成功", "Sign-in didn't go through",
|
||||
"请再试一次。", "Please try again.")
|
||||
return
|
||||
}
|
||||
|
||||
claims, err := o.claims(r.Context(), verifier, token)
|
||||
if err != nil {
|
||||
log.Printf("auth: id token rejected: %v", err)
|
||||
o.page(w, http.StatusBadGateway, "登录没有成功", "Sign-in didn't go through",
|
||||
"请再试一次。", "Please try again.")
|
||||
return
|
||||
}
|
||||
|
||||
// Nonce check: this ID token must belong to the attempt that started here.
|
||||
nonce, err := r.Cookie(nonceCookie)
|
||||
if err != nil || subtle.ConstantTimeCompare([]byte(nonce.Value), []byte(claims.nonce)) != 1 {
|
||||
o.page(w, http.StatusBadRequest, "这个登录链接过期了", "That sign-in link expired",
|
||||
"请回到 Petal 重新登录。", "Head back to Petal and sign in again.")
|
||||
return
|
||||
}
|
||||
|
||||
if !o.opts.Allowed.Permits(claims.Subject, claims.Email) {
|
||||
log.Printf("auth: rejected sign-in for sub=%s email=%s (not on the allowlist)", claims.Subject, claims.Email)
|
||||
o.page(w, http.StatusForbidden,
|
||||
"这个 Petal 不是给你写的", "This Petal isn't yours to write in",
|
||||
"你的账号是有效的,但还没有被邀请到这个 Petal。如果这是个误会,找管理员说一声就好。",
|
||||
"Your account is valid, but it hasn't been invited to this Petal. If that's a mistake, a word with whoever runs it will sort it out.")
|
||||
return
|
||||
}
|
||||
|
||||
if err := o.users.Upsert(claims.Subject, claims.Email, claims.displayName()); err != nil {
|
||||
log.Printf("auth: provisioning failed: %v", err)
|
||||
o.page(w, http.StatusInternalServerError, "出了点问题", "Something went wrong",
|
||||
"请再试一次。", "Please try again.")
|
||||
return
|
||||
}
|
||||
|
||||
session, err := o.sessions.Create(claims.Subject, r.UserAgent())
|
||||
if err != nil {
|
||||
log.Printf("auth: session creation failed: %v", err)
|
||||
o.page(w, http.StatusInternalServerError, "出了点问题", "Something went wrong",
|
||||
"请再试一次。", "Please try again.")
|
||||
return
|
||||
}
|
||||
SetSessionCookie(w, session, o.secure)
|
||||
log.Printf("auth: signed in %s (%s)", claims.Email, claims.Subject)
|
||||
|
||||
http.Redirect(w, r, "/", http.StatusFound)
|
||||
}
|
||||
|
||||
// logout revokes the session server-side and clears the cookie. Doing both
|
||||
// matters: clearing only the cookie leaves a token that still works if it was
|
||||
// ever captured.
|
||||
func (o *OIDC) logout(w http.ResponseWriter, r *http.Request) {
|
||||
if token := SessionToken(r); token != "" {
|
||||
if err := o.sessions.Revoke(token); err != nil {
|
||||
log.Printf("auth: revoke failed: %v", err)
|
||||
}
|
||||
}
|
||||
ClearSessionCookie(w, o.secure)
|
||||
http.Redirect(w, r, "/", http.StatusFound)
|
||||
}
|
||||
|
||||
// idClaims is the subset of the ID token Petal cares about.
|
||||
type idClaims struct {
|
||||
Subject string `json:"sub"`
|
||||
Email string `json:"email"`
|
||||
Name string `json:"name"`
|
||||
PreferredUsername string `json:"preferred_username"`
|
||||
|
||||
nonce string
|
||||
}
|
||||
|
||||
func (c idClaims) displayName() string {
|
||||
if c.Name != "" {
|
||||
return c.Name
|
||||
}
|
||||
if c.PreferredUsername != "" {
|
||||
return c.PreferredUsername
|
||||
}
|
||||
return c.Email
|
||||
}
|
||||
|
||||
// claims verifies the ID token in a token response and extracts its claims.
|
||||
func (o *OIDC) claims(ctx context.Context, verifier *oidc.IDTokenVerifier, token *oauth2.Token) (idClaims, error) {
|
||||
raw, ok := token.Extra("id_token").(string)
|
||||
if !ok || raw == "" {
|
||||
return idClaims{}, errors.New("no id_token in the token response")
|
||||
}
|
||||
idToken, err := verifier.Verify(ctx, raw)
|
||||
if err != nil {
|
||||
return idClaims{}, err
|
||||
}
|
||||
var claims idClaims
|
||||
if err := idToken.Claims(&claims); err != nil {
|
||||
return idClaims{}, err
|
||||
}
|
||||
if claims.Subject == "" {
|
||||
claims.Subject = idToken.Subject
|
||||
}
|
||||
claims.nonce = idToken.Nonce
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
func (o *OIDC) setTemp(w http.ResponseWriter, name, value string) {
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: name,
|
||||
Value: value,
|
||||
Path: "/auth",
|
||||
HttpOnly: true,
|
||||
Secure: o.secure,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
MaxAge: loginAttemptTTL,
|
||||
})
|
||||
}
|
||||
|
||||
func (o *OIDC) clearTemp(w http.ResponseWriter) {
|
||||
for _, name := range []string{stateCookie, nonceCookie, pkceCookie} {
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: name, Value: "", Path: "/auth",
|
||||
HttpOnly: true, Secure: o.secure, SameSite: http.SameSiteLaxMode, MaxAge: -1,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func randomToken() (string, error) {
|
||||
b := make([]byte, 24)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
// page renders one of the flow's dead ends. Every one of them is a full stop in
|
||||
// front of someone who was just trying to write, so they read as warm bilingual
|
||||
// sentences rather than as a status code — the same standard as the rest of the
|
||||
// app, and the reason these aren't plain http.Error calls.
|
||||
func (o *OIDC) page(w http.ResponseWriter, status int, titleZH, titleEN, bodyZH, bodyEN string) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
fmt.Fprintf(w, `<!doctype html>
|
||||
<html lang="zh"><head><meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>%s · Petal</title>
|
||||
<style>
|
||||
:root { color-scheme: light dark; }
|
||||
body { margin:0; min-height:100vh; display:flex; align-items:center; justify-content:center;
|
||||
background:#fdf8f5; color:#5b4b52;
|
||||
font-family:'Nunito','PingFang SC','Microsoft YaHei','Noto Sans CJK SC',system-ui,sans-serif; }
|
||||
main { max-width:30rem; padding:2.5rem; text-align:center; }
|
||||
.mark { font-size:2.5rem; }
|
||||
h1 { font-size:1.5rem; margin:.75rem 0 .25rem; font-weight:700; }
|
||||
h2 { font-size:1rem; margin:0 0 1.25rem; font-weight:600; opacity:.65; }
|
||||
p { line-height:1.7; margin:.4rem 0; }
|
||||
p.en { opacity:.7; font-size:.95rem; }
|
||||
a { display:inline-block; margin-top:1.75rem; padding:.6rem 1.4rem; border-radius:999px;
|
||||
background:#f3c7d3; color:#5b4b52; text-decoration:none; font-weight:700; }
|
||||
@media (prefers-color-scheme: dark) { body { background:#231b28; color:#e9dfe6; } a { background:#7c5f78; color:#fdf8f5; } }
|
||||
</style></head>
|
||||
<body><main>
|
||||
<div class="mark">🌸</div>
|
||||
<h1>%s</h1><h2>%s</h2>
|
||||
<p>%s</p><p class="en">%s</p>
|
||||
<a href="/">回到 Petal · Back to Petal</a>
|
||||
</main></body></html>
|
||||
`, titleEN, titleZH, titleEN, bodyZH, bodyEN)
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
jose "github.com/go-jose/go-jose/v4"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// These tests run the whole login round-trip against a stub identity provider:
|
||||
// discovery, the redirect out, the callback back, and the session that comes out
|
||||
// the other end. The flow is the one place in Petal where getting a detail wrong
|
||||
// (an unchecked state, a nonce nobody compares) is both easy and invisible —
|
||||
// everything still "works" from the browser's point of view.
|
||||
|
||||
// stubIdP is a minimal OpenID provider: discovery, a JWKS, and a token endpoint
|
||||
// that mints a signed ID token for whoever the test says just logged in.
|
||||
type stubIdP struct {
|
||||
*httptest.Server
|
||||
key *rsa.PrivateKey
|
||||
clientID string
|
||||
// issuer as advertised by discovery and asserted in tokens. Defaults to the
|
||||
// server's URL; a test can give it a trailing slash, which is what Authentik
|
||||
// does and which OIDC requires to match byte-for-byte.
|
||||
issuer string
|
||||
|
||||
// Claims the next token exchange will assert.
|
||||
sub, email, name string
|
||||
// nonce echoed into the token; set from the login attempt's cookie.
|
||||
nonce string
|
||||
// lastForm records what Petal sent to /token, so the test can assert PKCE.
|
||||
lastForm url.Values
|
||||
}
|
||||
|
||||
func newStubIdP(t *testing.T, clientID string) *stubIdP {
|
||||
t.Helper()
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
idp := &stubIdP{key: key, clientID: clientID}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
idp.Server = httptest.NewServer(mux)
|
||||
idp.issuer = idp.URL
|
||||
t.Cleanup(idp.Close)
|
||||
|
||||
mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"issuer": idp.issuer,
|
||||
"authorization_endpoint": idp.URL + "/authorize",
|
||||
"token_endpoint": idp.URL + "/token",
|
||||
"jwks_uri": idp.URL + "/jwks",
|
||||
"id_token_signing_alg_values_supported": []string{"RS256"},
|
||||
})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/jwks", func(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(jose.JSONWebKeySet{
|
||||
Keys: []jose.JSONWebKey{{Key: key.Public(), Algorithm: "RS256", Use: "sig"}},
|
||||
})
|
||||
})
|
||||
|
||||
mux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = r.ParseForm()
|
||||
idp.lastForm = r.PostForm
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"access_token": "stub-access-token",
|
||||
"token_type": "Bearer",
|
||||
"id_token": idp.idToken(t),
|
||||
})
|
||||
})
|
||||
|
||||
return idp
|
||||
}
|
||||
|
||||
// idToken mints a signed ID token asserting the currently configured claims.
|
||||
func (idp *stubIdP) idToken(t *testing.T) string {
|
||||
t.Helper()
|
||||
signer, err := jose.NewSigner(
|
||||
jose.SigningKey{Algorithm: jose.RS256, Key: idp.key},
|
||||
(&jose.SignerOptions{}).WithType("JWT"),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]any{
|
||||
"iss": idp.issuer,
|
||||
"aud": idp.clientID,
|
||||
"sub": idp.sub,
|
||||
"email": idp.email,
|
||||
"name": idp.name,
|
||||
"nonce": idp.nonce,
|
||||
"exp": time.Now().Add(time.Hour).Unix(),
|
||||
"iat": time.Now().Unix(),
|
||||
})
|
||||
signed, err := signer.Sign(payload)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := signed.CompactSerialize()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// newFlow wires Petal's login routes to a stub provider.
|
||||
func newFlow(t *testing.T, allowed Allowlist) (*stubIdP, http.Handler, *SessionStore, *UserStore) {
|
||||
t.Helper()
|
||||
sessions, users, _ := newStores(t)
|
||||
idp := newStubIdP(t, "petal")
|
||||
|
||||
o := NewOIDC(context.Background(), Options{
|
||||
IssuerURL: idp.URL,
|
||||
ClientID: "petal",
|
||||
ClientSecret: "shh",
|
||||
BaseURL: "http://petal.test",
|
||||
Allowed: allowed,
|
||||
}, sessions, users)
|
||||
return idp, o.Routes(), sessions, users
|
||||
}
|
||||
|
||||
// cookieJar collects Set-Cookie headers across the redirect chain, standing in
|
||||
// for the browser that would normally carry them.
|
||||
type cookieJar map[string]string
|
||||
|
||||
func (j cookieJar) absorb(rec *httptest.ResponseRecorder) {
|
||||
for _, c := range rec.Result().Cookies() {
|
||||
if c.MaxAge < 0 || c.Value == "" {
|
||||
delete(j, c.Name)
|
||||
continue
|
||||
}
|
||||
j[c.Name] = c.Value
|
||||
}
|
||||
}
|
||||
|
||||
func (j cookieJar) attach(r *http.Request) *http.Request {
|
||||
for name, value := range j {
|
||||
r.AddCookie(&http.Cookie{Name: name, Value: value})
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// start runs /auth/login and returns the redirect target plus the cookies it set.
|
||||
func start(t *testing.T, flow http.Handler) (*url.URL, cookieJar) {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
flow.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/login", nil))
|
||||
if rec.Code != http.StatusFound {
|
||||
t.Fatalf("login status=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
target, err := url.Parse(rec.Header().Get("Location"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jar := cookieJar{}
|
||||
jar.absorb(rec)
|
||||
return target, jar
|
||||
}
|
||||
|
||||
func TestLoginRoundTrip(t *testing.T) {
|
||||
idp, flow, sessions, users := newFlow(t, nil)
|
||||
idp.sub, idp.email, idp.name = "sub-her", "her@example.com", "Her Name"
|
||||
|
||||
target, jar := start(t, flow)
|
||||
|
||||
// The redirect must carry everything the flow depends on later.
|
||||
q := target.Query()
|
||||
if q.Get("state") == "" || q.Get("nonce") == "" {
|
||||
t.Fatalf("login redirect missing state/nonce: %s", target)
|
||||
}
|
||||
if q.Get("code_challenge") == "" || q.Get("code_challenge_method") != "S256" {
|
||||
t.Fatalf("login redirect missing PKCE challenge: %s", target)
|
||||
}
|
||||
if q.Get("redirect_uri") != "http://petal.test/auth/callback" {
|
||||
t.Fatalf("redirect_uri = %q", q.Get("redirect_uri"))
|
||||
}
|
||||
if jar[stateCookie] != q.Get("state") {
|
||||
t.Fatal("the state cookie does not match the state sent to the provider")
|
||||
}
|
||||
idp.nonce = jar[nonceCookie]
|
||||
|
||||
// Come back as the provider would.
|
||||
rec := httptest.NewRecorder()
|
||||
flow.ServeHTTP(rec, jar.attach(
|
||||
httptest.NewRequest(http.MethodGet, "/callback?code=abc&state="+url.QueryEscape(q.Get("state")), nil)))
|
||||
if rec.Code != http.StatusFound || rec.Header().Get("Location") != "/" {
|
||||
t.Fatalf("callback status=%d location=%q body=%s", rec.Code, rec.Header().Get("Location"), rec.Body)
|
||||
}
|
||||
|
||||
// PKCE: the code verifier must reach the token endpoint.
|
||||
if v := idp.lastForm.Get("code_verifier"); v == "" {
|
||||
t.Fatal("token exchange sent no code_verifier")
|
||||
}
|
||||
|
||||
// The account was provisioned from the token's claims...
|
||||
user, err := users.Get("sub-her")
|
||||
if err != nil {
|
||||
t.Fatalf("user was not provisioned: %v", err)
|
||||
}
|
||||
if user.Email != "her@example.com" || user.DisplayName != "Her Name" {
|
||||
t.Fatalf("unexpected provisioned user %+v", user)
|
||||
}
|
||||
|
||||
// ...and the response carries a session that resolves to them.
|
||||
jar.absorb(rec)
|
||||
token := jar[SessionCookie]
|
||||
if token == "" {
|
||||
t.Fatal("callback issued no session cookie")
|
||||
}
|
||||
got, err := sessions.Resolve(withCookie(token))
|
||||
if err != nil || got != "sub-her" {
|
||||
t.Fatalf("session resolved to %q (err=%v), want sub-her", got, err)
|
||||
}
|
||||
|
||||
// The one-shot login cookies must not linger.
|
||||
for _, name := range []string{stateCookie, nonceCookie, pkceCookie} {
|
||||
if jar[name] != "" {
|
||||
t.Fatalf("%s survived the callback", name)
|
||||
}
|
||||
}
|
||||
|
||||
// Signing out revokes server-side, not just in the browser.
|
||||
out := httptest.NewRecorder()
|
||||
flow.ServeHTTP(out, jar.attach(httptest.NewRequest(http.MethodPost, "/logout", nil)))
|
||||
if out.Code != http.StatusFound {
|
||||
t.Fatalf("logout status=%d", out.Code)
|
||||
}
|
||||
if _, err := sessions.Resolve(withCookie(token)); err == nil {
|
||||
t.Fatal("the session survived signing out")
|
||||
}
|
||||
}
|
||||
|
||||
// Signing out is a state change, so it must not be reachable by GET: with
|
||||
// SameSite=Lax the session cookie *is* sent on a top-level cross-site
|
||||
// navigation, which would let any page on the internet sign her out mid-draft.
|
||||
func TestLogoutRejectsGET(t *testing.T) {
|
||||
_, flow, _, _ := newFlow(t, nil)
|
||||
|
||||
out := httptest.NewRecorder()
|
||||
flow.ServeHTTP(out, httptest.NewRequest(http.MethodGet, "/logout", nil))
|
||||
if out.Code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("GET /logout status=%d, want 405", out.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// A callback whose state doesn't match the cookie is a forged one.
|
||||
func TestCallbackRejectsBadState(t *testing.T) {
|
||||
idp, flow, sessions, _ := newFlow(t, nil)
|
||||
idp.sub, idp.email = "sub-her", "her@example.com"
|
||||
|
||||
_, jar := start(t, flow)
|
||||
idp.nonce = jar[nonceCookie]
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
flow.ServeHTTP(rec, jar.attach(
|
||||
httptest.NewRequest(http.MethodGet, "/callback?code=abc&state=some-other-state", nil)))
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status=%d, want 400", rec.Code)
|
||||
}
|
||||
assertNoSession(t, sessions, rec)
|
||||
|
||||
// And so is one with no state cookie at all.
|
||||
bare := httptest.NewRecorder()
|
||||
flow.ServeHTTP(bare, httptest.NewRequest(http.MethodGet, "/callback?code=abc&state=x", nil))
|
||||
if bare.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status=%d for a cookieless callback, want 400", bare.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// An ID token minted for a different login attempt must not be accepted, even
|
||||
// though it is perfectly valid and correctly signed.
|
||||
func TestCallbackRejectsReplayedNonce(t *testing.T) {
|
||||
idp, flow, sessions, _ := newFlow(t, nil)
|
||||
idp.sub, idp.email = "sub-her", "her@example.com"
|
||||
|
||||
_, jarA := start(t, flow)
|
||||
_, jarB := start(t, flow)
|
||||
idp.nonce = jarB[nonceCookie] // a token belonging to the *other* attempt
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
flow.ServeHTTP(rec, jarA.attach(
|
||||
httptest.NewRequest(http.MethodGet, "/callback?code=abc&state="+url.QueryEscape(jarA[stateCookie]), nil)))
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status=%d, want 400", rec.Code)
|
||||
}
|
||||
assertNoSession(t, sessions, rec)
|
||||
}
|
||||
|
||||
// Being a valid user at the identity provider is not the same as being a user
|
||||
// here, and the refusal has to read like Petal rather than like a stack trace.
|
||||
func TestCallbackHonoursAllowlist(t *testing.T) {
|
||||
idp, flow, sessions, users := newFlow(t, ParseAllowlist("her@example.com"))
|
||||
idp.sub, idp.email, idp.name = "sub-stranger", "stranger@example.com", "A Stranger"
|
||||
|
||||
_, jar := start(t, flow)
|
||||
idp.nonce = jar[nonceCookie]
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
flow.ServeHTTP(rec, jar.attach(
|
||||
httptest.NewRequest(http.MethodGet, "/callback?code=abc&state="+url.QueryEscape(jar[stateCookie]), nil)))
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status=%d, want 403", rec.Code)
|
||||
}
|
||||
if body := rec.Body.String(); !strings.Contains(body, "这个 Petal 不是给你写的") ||
|
||||
!strings.Contains(body, "isn't yours to write in") {
|
||||
t.Fatalf("refusal page is not the warm bilingual one: %s", body)
|
||||
}
|
||||
assertNoSession(t, sessions, rec)
|
||||
if _, err := users.Get("sub-stranger"); err == nil {
|
||||
t.Fatal("a rejected login still provisioned an account")
|
||||
}
|
||||
|
||||
// The person on the list gets in through the same door.
|
||||
idp.sub, idp.email, idp.name = "sub-her", "her@example.com", "Her Name"
|
||||
_, jar2 := start(t, flow)
|
||||
idp.nonce = jar2[nonceCookie]
|
||||
ok := httptest.NewRecorder()
|
||||
flow.ServeHTTP(ok, jar2.attach(
|
||||
httptest.NewRequest(http.MethodGet, "/callback?code=abc&state="+url.QueryEscape(jar2[stateCookie]), nil)))
|
||||
if ok.Code != http.StatusFound {
|
||||
t.Fatalf("an allowed writer was turned away: status=%d body=%s", ok.Code, ok.Body)
|
||||
}
|
||||
}
|
||||
|
||||
// The provider refusing the login (a cancelled consent, a locked account) is a
|
||||
// dead end, not a session.
|
||||
func TestCallbackHandlesProviderError(t *testing.T) {
|
||||
_, flow, sessions, _ := newFlow(t, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
flow.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/callback?error=access_denied", nil))
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status=%d, want 403", rec.Code)
|
||||
}
|
||||
assertNoSession(t, sessions, rec)
|
||||
}
|
||||
|
||||
// Authentik's issuer ends in a slash, and OIDC requires the discovered issuer to
|
||||
// match the configured one byte-for-byte. Normalising it away made discovery
|
||||
// fail against the real provider while every stub test still passed.
|
||||
func TestDiscoveryKeepsTrailingSlashIssuer(t *testing.T) {
|
||||
sessions, users, _ := newStores(t)
|
||||
idp := newStubIdP(t, "petal")
|
||||
idp.issuer = idp.URL + "/"
|
||||
idp.sub, idp.email = "sub-her", "her@example.com"
|
||||
|
||||
o := NewOIDC(context.Background(), Options{
|
||||
IssuerURL: idp.issuer,
|
||||
ClientID: "petal",
|
||||
ClientSecret: "shh",
|
||||
BaseURL: "http://petal.test",
|
||||
}, sessions, users)
|
||||
flow := o.Routes()
|
||||
|
||||
// A failed discovery renders the 503 "sign-in is unavailable" page instead
|
||||
// of redirecting, so reaching the provider at all is the assertion.
|
||||
target, jar := start(t, flow)
|
||||
if !strings.HasPrefix(target.String(), idp.URL+"/authorize") {
|
||||
t.Fatalf("login went to %q, want the provider's authorize endpoint", target)
|
||||
}
|
||||
|
||||
// And the ID token it issues, whose `iss` carries the same slash, verifies.
|
||||
idp.nonce = jar[nonceCookie]
|
||||
rec := httptest.NewRecorder()
|
||||
flow.ServeHTTP(rec, jar.attach(
|
||||
httptest.NewRequest(http.MethodGet, "/callback?code=abc&state="+url.QueryEscape(jar[stateCookie]), nil)))
|
||||
if rec.Code != http.StatusFound {
|
||||
t.Fatalf("callback status=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
}
|
||||
|
||||
// Signing in when already signed in shouldn't bounce a good session through the
|
||||
// identity provider.
|
||||
func TestLoginSkipsWhenAlreadySignedIn(t *testing.T) {
|
||||
_, flow, sessions, _ := newFlow(t, nil)
|
||||
token, err := sessions.Create(db.LocalUserID, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
flow.ServeHTTP(rec, withCookie(token))
|
||||
// withCookie builds a GET "/" request; point it at the login route.
|
||||
rec = httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/login", nil)
|
||||
req.AddCookie(&http.Cookie{Name: SessionCookie, Value: token})
|
||||
flow.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusFound || rec.Header().Get("Location") != "/" {
|
||||
t.Fatalf("status=%d location=%q, want a redirect home", rec.Code, rec.Header().Get("Location"))
|
||||
}
|
||||
}
|
||||
|
||||
// assertNoSession fails if a response handed out a usable session cookie.
|
||||
func assertNoSession(t *testing.T, sessions *SessionStore, rec *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
for _, c := range rec.Result().Cookies() {
|
||||
if c.Name == SessionCookie && c.Value != "" {
|
||||
if _, err := sessions.Resolve(withCookie(c.Value)); err == nil {
|
||||
t.Fatal("a rejected login was given a working session")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// patchMe drives UpdateMeHandler as the given user would reach it: behind the
|
||||
// middleware, which is the only thing that puts an id in the context.
|
||||
func patchMe(t *testing.T, users *UserStore, id, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
r := httptest.NewRequest(http.MethodPatch, "/me", strings.NewReader(body))
|
||||
r = r.WithContext(WithUser(r.Context(), id))
|
||||
w := httptest.NewRecorder()
|
||||
users.UpdateMeHandler()(w, r)
|
||||
return w
|
||||
}
|
||||
|
||||
func TestSetPairLang(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPair("bob", "pt-PT", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set pt-PT: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "pt-PT" {
|
||||
t.Fatalf("pair_lang = %q, want pt-PT", u.PairLang)
|
||||
}
|
||||
|
||||
// Every pair with a langpack, not just the first one: this list and the
|
||||
// frontend's PACKS are two copies of the same fact, and the day they
|
||||
// disagree is the day she can pick a pair the app cannot render.
|
||||
if err := users.SetPair("bob", "fr", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set fr: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "fr" {
|
||||
t.Fatalf("pair_lang = %q, want fr", u.PairLang)
|
||||
}
|
||||
|
||||
if err := users.SetPair("bob", "es", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set es: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "es" {
|
||||
t.Fatalf("pair_lang = %q, want es", u.PairLang)
|
||||
}
|
||||
|
||||
// And back — a writer who tries a pair and doesn't like it must be able to
|
||||
// return, which is the whole reason the picker exists.
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set zh: %v", err)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "zh" {
|
||||
t.Fatalf("pair_lang = %q, want zh", u.PairLang)
|
||||
}
|
||||
}
|
||||
|
||||
// A pair the frontend has no langpack for must not be storable. Accepting it
|
||||
// would leave her looking at Chinese copy with no way back except a lucky guess.
|
||||
func TestSetPairLangRejectsUnshippedPairs(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
// The near-misses are the ones that matter, and there are two of them now.
|
||||
// "pt-BR" must not be quietly served European copy and a European voice;
|
||||
// "es-ES" is the same mistake pointing the other way, because the es pack is
|
||||
// deliberately Latin American and reads itself aloud in a Mexican voice. A
|
||||
// regional code Petal has not decided about is refused rather than rounded
|
||||
// to the nearest pack it happens to have.
|
||||
for _, lang := range []string{"es-ES", "pt-BR", "fr-CA", "de", "klingon", "", " "} {
|
||||
if err := users.SetPair("bob", lang, DirectionLearningEn); err == nil {
|
||||
t.Fatalf("stored unshipped pair %q", lang)
|
||||
}
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "zh" {
|
||||
t.Fatalf("a refused write still moved pair_lang to %q", u.PairLang)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetPairLangUnknownUser(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
if err := users.SetPair("nobody", "pt-PT", DirectionLearningEn); err == nil {
|
||||
t.Fatal("set a pair language on an account that does not exist")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMeHandler(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
w := patchMe(t, users, "bob", `{"pair_lang":"pt-PT"}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
// The whole user comes back, so the client can re-read the pair from the
|
||||
// server instead of assuming its request took.
|
||||
var got db.User
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.ID != "bob" || got.PairLang != "pt-PT" {
|
||||
t.Fatalf("response = %+v, want bob on pt-PT", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMeHandlerRejects(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
for name, body := range map[string]string{
|
||||
"unshipped pair": `{"pair_lang":"es-ES"}`,
|
||||
"unknown direction": `{"direction":"learning_klingon"}`,
|
||||
"not json": `pt-PT`,
|
||||
} {
|
||||
if w := patchMe(t, users, "bob", body); w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: status = %d, want 400", name, w.Code)
|
||||
}
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "zh" {
|
||||
t.Fatalf("a rejected request still moved pair_lang to %q", u.PairLang)
|
||||
}
|
||||
|
||||
// A caller the middleware never resolved (or whose row is gone) is a lapsed
|
||||
// session, not a bad request — the client turns 401 into the sign-in overlay.
|
||||
if w := patchMe(t, users, "nobody", `{"pair_lang":"pt-PT"}`); w.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("unknown user: status = %d, want 401", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// An empty body used to be a 400, back when pair_lang was the only field and a
|
||||
// request that named none of it could only be a client bug. With two optional
|
||||
// fields it is an ordinary PATCH that changes nothing, and it has to be: the
|
||||
// picker sends one field without knowing the other, and "omitted" has to mean
|
||||
// "leave it alone" for that to be safe.
|
||||
func TestUpdateMeHandlerEmptyBodyChangesNothing(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningPair); err != nil {
|
||||
t.Fatalf("set up: %v", err)
|
||||
}
|
||||
w := patchMe(t, users, "bob", `{}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
u, _ := users.Get("bob")
|
||||
if u.PairLang != "zh" || u.Direction != DirectionLearningPair {
|
||||
t.Fatalf("empty PATCH moved the account to %q/%q", u.PairLang, u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The direction axis: an account can be turned around and turned back, and the
|
||||
// default every existing row already carries is the one it had before the column
|
||||
// existed.
|
||||
func TestDirectionRoundTrip(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("a fresh account starts at %q, want %q", u.Direction, DirectionLearningEn)
|
||||
}
|
||||
|
||||
w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("turn around: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
var got db.User
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
// The response carries the direction, not just the pair — the client reads
|
||||
// its whole state back from here rather than assuming the write took.
|
||||
if got.Direction != DirectionLearningPair || got.PairLang != "zh" {
|
||||
t.Fatalf("response = %+v, want bob learning zh", got)
|
||||
}
|
||||
|
||||
if w := patchMe(t, users, "bob", `{"direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||
t.Fatalf("turn back: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("direction = %q after turning back", u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The refusal this axis exists to make: a pair with no word list cannot be
|
||||
// learned toward, however good its langpack is. fr, es and pt-PT all have copy,
|
||||
// voices and spelling dictionaries — and nothing that could segment a sentence
|
||||
// or read from that language into English, which is what a learner needs.
|
||||
func TestLearnerDirectionRefusedForPairsWithoutData(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
for _, lang := range []string{"pt-PT", "fr", "es"} {
|
||||
if err := users.SetPair("bob", lang, DirectionLearningEn); err != nil {
|
||||
t.Fatalf("set %s: %v", lang, err)
|
||||
}
|
||||
w := patchMe(t, users, "bob", `{"direction":"learning_pair"}`)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: status = %d, want 400", lang, w.Code)
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("%s: a refused write still moved direction to %q", lang, u.Direction)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The two-field combination the handler validates as one decision. An account
|
||||
// already learning Chinese that asks only to change pair is asking for a state
|
||||
// neither field names on its own — French with segmentation — and it must not
|
||||
// arrive by leaving one field out.
|
||||
func TestPairChangeCannotStrandTheLearnerDirection(t *testing.T) {
|
||||
_, users, _ := newStores(t)
|
||||
|
||||
if err := users.SetPair("bob", "zh", DirectionLearningPair); err != nil {
|
||||
t.Fatalf("set up: %v", err)
|
||||
}
|
||||
if w := patchMe(t, users, "bob", `{"pair_lang":"fr"}`); w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", w.Code)
|
||||
}
|
||||
u, _ := users.Get("bob")
|
||||
if u.PairLang != "zh" || u.Direction != DirectionLearningPair {
|
||||
t.Fatalf("refused write left the account at %q/%q", u.PairLang, u.Direction)
|
||||
}
|
||||
|
||||
// Naming both at once is how that move is actually made, and it works.
|
||||
if w := patchMe(t, users, "bob", `{"pair_lang":"fr","direction":"learning_en"}`); w.Code != http.StatusOK {
|
||||
t.Fatalf("both fields: status = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.PairLang != "fr" || u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("account = %q/%q, want fr/learning_en", u.PairLang, u.Direction)
|
||||
}
|
||||
}
|
||||
|
||||
// The CHECK constraint is the last line, below the handler and below SetPair:
|
||||
// a direction that reaches the column by any other route is still refused.
|
||||
func TestDirectionCheckConstraint(t *testing.T) {
|
||||
_, users, database := newStores(t)
|
||||
if _, err := database.Exec(`UPDATE users SET direction = 'sideways' WHERE id = 'bob'`); err == nil {
|
||||
t.Fatal("the users.direction CHECK accepted 'sideways'")
|
||||
}
|
||||
if u, _ := users.Get("bob"); u.Direction != DirectionLearningEn {
|
||||
t.Fatalf("direction = %q after a refused UPDATE", u.Direction)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The cookie carrying the opaque session token, in its two spellings.
|
||||
//
|
||||
// Over https the name takes the __Host- prefix, which is not decoration: the
|
||||
// browser will only accept such a cookie if it is Secure, Path=/, and carries
|
||||
// no Domain attribute — and, crucially, refuses to let any other host set it.
|
||||
// Without the prefix, anything that can write cookies for a sibling name under
|
||||
// parodia.dev (another service on the box, a subdomain takeover) can plant a
|
||||
// session cookie in her browser that Petal will then read as hers.
|
||||
//
|
||||
// The prefix is impossible over plain http, because it requires Secure and a
|
||||
// browser drops a Secure cookie on an insecure origin. So local development
|
||||
// keeps the bare name, and the name in use follows the same `secure` flag the
|
||||
// rest of the cookie does.
|
||||
const (
|
||||
SessionCookie = "petal_session"
|
||||
HostSessionCookie = "__Host-petal_session"
|
||||
)
|
||||
|
||||
// sessionCookieName is the name to *write* under this scheme.
|
||||
func sessionCookieName(secure bool) string {
|
||||
if secure {
|
||||
return HostSessionCookie
|
||||
}
|
||||
return SessionCookie
|
||||
}
|
||||
|
||||
const (
|
||||
// sessionTTL is how long a session lives without use. Thirty days, sliding:
|
||||
// every authenticated request pushes the expiry back out. An editor that
|
||||
// logs you out mid-draft is hostile, and Petal auto-saves every 1.5s, so a
|
||||
// surprise 401 costs real writing.
|
||||
sessionTTL = 30 * 24 * time.Hour
|
||||
|
||||
// sessionTTLModifier is the same span as a SQLite datetime() modifier. All
|
||||
// expiry math happens inside SQLite so stored values stay canonical UTC and
|
||||
// never depend on the server's local clock or on Go/SQLite parsing agreeing.
|
||||
sessionTTLModifier = "+30 days"
|
||||
|
||||
// sessionRenewAfter throttles the sliding extension: a session is only
|
||||
// pushed forward once its expiry has drifted this far from the maximum. It
|
||||
// turns "a write on every request" into "a write at most once an hour per
|
||||
// session" while leaving the sliding window indistinguishable to the user.
|
||||
sessionRenewAfter = "-1 hour"
|
||||
)
|
||||
|
||||
// ErrNoSession means the request carried no session cookie, or one that is
|
||||
// unknown or expired. It is not an internal failure: the caller is simply not
|
||||
// signed in.
|
||||
var ErrNoSession = errors.New("no valid session")
|
||||
|
||||
// SessionStore issues, validates and revokes login sessions, and is itself the
|
||||
// [Resolver] the API middleware runs on.
|
||||
//
|
||||
// The cookie holds a random token; the table stores only its SHA-256. A dump of
|
||||
// the database therefore hands an attacker no usable session — the same reason
|
||||
// passwords are never stored as given. Server-side rows (rather than a signed
|
||||
// stateless cookie) are what make logout and revocation actually revoke.
|
||||
type SessionStore struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// NewSessionStore returns a store backed by the given database.
|
||||
func NewSessionStore(db *sql.DB) *SessionStore { return &SessionStore{db: db} }
|
||||
|
||||
// Create issues a new session for userID and returns the token to put in the
|
||||
// cookie. The token is never stored; only its hash is.
|
||||
func (s *SessionStore) Create(userID, userAgent string) (string, error) {
|
||||
raw := make([]byte, 32)
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
return "", err
|
||||
}
|
||||
token := base64.RawURLEncoding.EncodeToString(raw)
|
||||
|
||||
if len(userAgent) > 256 {
|
||||
userAgent = userAgent[:256]
|
||||
}
|
||||
_, err := s.db.Exec(
|
||||
`INSERT INTO sessions (id, user_id, expires_at, user_agent)
|
||||
VALUES (?, ?, datetime('now', ?), ?)`,
|
||||
hashToken(token), userID, sessionTTLModifier, userAgent,
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
// Resolve implements [Resolver]: it reads the session cookie, validates it, and
|
||||
// returns the user it belongs to — extending the session's life while it does.
|
||||
func (s *SessionStore) Resolve(r *http.Request) (string, error) {
|
||||
token := SessionToken(r)
|
||||
if token == "" {
|
||||
return "", ErrNoSession
|
||||
}
|
||||
return s.userFor(token)
|
||||
}
|
||||
|
||||
// SessionToken pulls the raw session token out of a request, preferring the
|
||||
// __Host- spelling.
|
||||
//
|
||||
// Both are read because a deployment that was signing people in before the
|
||||
// prefix existed has browsers holding the old name; those sessions stay valid
|
||||
// and quietly re-issue under the new name at the next sign-in. The prefixed one
|
||||
// wins where both are present, since it is the one another host could not have
|
||||
// planted.
|
||||
func SessionToken(r *http.Request) string {
|
||||
for _, name := range []string{HostSessionCookie, SessionCookie} {
|
||||
if c, err := r.Cookie(name); err == nil && c.Value != "" {
|
||||
return c.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// userFor validates a raw token and slides its expiry forward.
|
||||
func (s *SessionStore) userFor(token string) (string, error) {
|
||||
id := hashToken(token)
|
||||
|
||||
var userID string
|
||||
err := s.db.QueryRow(
|
||||
`SELECT user_id FROM sessions WHERE id = ? AND expires_at > datetime('now')`, id,
|
||||
).Scan(&userID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", ErrNoSession
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Slide the window. Throttled, and deliberately not fatal: a failed
|
||||
// extension shortens one session's life, which is no reason to reject a
|
||||
// request that is otherwise perfectly authenticated.
|
||||
_, _ = s.db.Exec(
|
||||
`UPDATE sessions SET expires_at = datetime('now', ?)
|
||||
WHERE id = ? AND expires_at < datetime('now', ?, ?)`,
|
||||
sessionTTLModifier, id, sessionTTLModifier, sessionRenewAfter,
|
||||
)
|
||||
return userID, nil
|
||||
}
|
||||
|
||||
// Revoke deletes the session behind a token. Unknown tokens are not an error —
|
||||
// signing out of a session that is already gone is a success, not a failure.
|
||||
func (s *SessionStore) Revoke(token string) error {
|
||||
_, err := s.db.Exec(`DELETE FROM sessions WHERE id = ?`, hashToken(token))
|
||||
return err
|
||||
}
|
||||
|
||||
// RevokeAll deletes every session for a user, signing them out everywhere.
|
||||
func (s *SessionStore) RevokeAll(userID string) error {
|
||||
_, err := s.db.Exec(`DELETE FROM sessions WHERE user_id = ?`, userID)
|
||||
return err
|
||||
}
|
||||
|
||||
// Prune removes expired rows and returns how many it deleted. Nothing depends
|
||||
// on it for correctness — expired sessions are already rejected on lookup — it
|
||||
// just keeps the table from accumulating dead rows forever.
|
||||
func (s *SessionStore) Prune() (int64, error) {
|
||||
res, err := s.db.Exec(`DELETE FROM sessions WHERE expires_at <= datetime('now')`)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
|
||||
// hashToken maps a raw session token to the id stored in the table.
|
||||
func hashToken(token string) string {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// SetSessionCookie writes the session cookie. Secure is set only when Petal is
|
||||
// served over https — flagging it on a plain-http dev server would make the
|
||||
// browser drop the cookie and silently break local login.
|
||||
func SetSessionCookie(w http.ResponseWriter, token string, secure bool) {
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: sessionCookieName(secure),
|
||||
Value: token,
|
||||
Path: "/",
|
||||
HttpOnly: true,
|
||||
Secure: secure,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
MaxAge: int(sessionTTL / time.Second),
|
||||
})
|
||||
}
|
||||
|
||||
// ClearSessionCookie expires the session cookie in the browser. The matching
|
||||
// server-side row must be revoked separately — that's the half that counts.
|
||||
//
|
||||
// Both spellings are expired, not just the one currently written: a browser
|
||||
// carrying a pre-prefix cookie must not be left holding it after signing out,
|
||||
// which is precisely the case where "clear the cookie" is the part the user can
|
||||
// see working.
|
||||
func ClearSessionCookie(w http.ResponseWriter, secure bool) {
|
||||
for _, name := range []string{HostSessionCookie, SessionCookie} {
|
||||
if name == HostSessionCookie && !secure {
|
||||
continue // the browser would reject a non-Secure __Host- cookie
|
||||
}
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: name,
|
||||
Value: "",
|
||||
Path: "/",
|
||||
HttpOnly: true,
|
||||
Secure: secure,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
MaxAge: -1,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,375 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// newStores opens a database holding two users and returns the session and user
|
||||
// stores over it.
|
||||
func newStores(t *testing.T) (*SessionStore, *UserStore, *db.DB) {
|
||||
t.Helper()
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
|
||||
if _, err := database.Exec(
|
||||
`INSERT INTO users (id, email, display_name) VALUES (?, ?, ?)`,
|
||||
"bob", "bob@petal.local", "Bob",
|
||||
); err != nil {
|
||||
t.Fatalf("seed second user: %v", err)
|
||||
}
|
||||
return NewSessionStore(database.DB), NewUserStore(database.DB), database
|
||||
}
|
||||
|
||||
// withCookie builds a request carrying a session token.
|
||||
func withCookie(token string) *http.Request {
|
||||
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
r.AddCookie(&http.Cookie{Name: SessionCookie, Value: token})
|
||||
return r
|
||||
}
|
||||
|
||||
func TestSessionLifecycle(t *testing.T) {
|
||||
sessions, _, _ := newStores(t)
|
||||
|
||||
token, err := sessions.Create(db.LocalUserID, "test-agent")
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
got, err := sessions.Resolve(withCookie(token))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
if got != db.LocalUserID {
|
||||
t.Fatalf("resolved %q, want %q", got, db.LocalUserID)
|
||||
}
|
||||
|
||||
// Signing out must invalidate the token server-side, not just in the browser:
|
||||
// clearing only the cookie leaves a token that still works if it ever leaked.
|
||||
if err := sessions.Revoke(token); err != nil {
|
||||
t.Fatalf("revoke: %v", err)
|
||||
}
|
||||
if _, err := sessions.Resolve(withCookie(token)); !errors.Is(err, ErrNoSession) {
|
||||
t.Fatalf("revoked token still resolves (err=%v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A request with no cookie, or a token nobody issued, is simply not signed in.
|
||||
func TestSessionRejectsUnknown(t *testing.T) {
|
||||
sessions, _, _ := newStores(t)
|
||||
|
||||
if _, err := sessions.Resolve(httptest.NewRequest(http.MethodGet, "/", nil)); !errors.Is(err, ErrNoSession) {
|
||||
t.Fatalf("bare request err=%v, want ErrNoSession", err)
|
||||
}
|
||||
if _, err := sessions.Resolve(withCookie("not-a-real-token")); !errors.Is(err, ErrNoSession) {
|
||||
t.Fatalf("forged token err=%v, want ErrNoSession", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The table stores a hash, so a database dump yields no usable session.
|
||||
func TestSessionTokenIsNotStored(t *testing.T) {
|
||||
sessions, _, database := newStores(t)
|
||||
|
||||
token, err := sessions.Create(db.LocalUserID, "")
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
var stored string
|
||||
if err := database.QueryRow(`SELECT id FROM sessions`).Scan(&stored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stored == token || strings.Contains(stored, token) {
|
||||
t.Fatal("the raw session token is stored in the database")
|
||||
}
|
||||
if stored != hashToken(token) {
|
||||
t.Fatal("stored id is not the token's hash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionExpiry(t *testing.T) {
|
||||
sessions, _, database := newStores(t)
|
||||
|
||||
token, err := sessions.Create(db.LocalUserID, "")
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
if _, err := database.Exec(
|
||||
`UPDATE sessions SET expires_at = datetime('now','-1 minute')`,
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := sessions.Resolve(withCookie(token)); !errors.Is(err, ErrNoSession) {
|
||||
t.Fatalf("expired session still resolves (err=%v)", err)
|
||||
}
|
||||
|
||||
n, err := sessions.Prune()
|
||||
if err != nil {
|
||||
t.Fatalf("prune: %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("pruned %d rows, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The window slides: using a session pushes its expiry back out, so someone who
|
||||
// writes in Petal every few days is never signed out mid-draft.
|
||||
func TestSessionSlidesForward(t *testing.T) {
|
||||
sessions, _, database := newStores(t)
|
||||
|
||||
token, err := sessions.Create(db.LocalUserID, "")
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
// Pretend the session has been idle for a fortnight.
|
||||
if _, err := database.Exec(
|
||||
`UPDATE sessions SET expires_at = datetime('now','+16 days')`,
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := sessions.Resolve(withCookie(token)); err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
|
||||
var extended bool
|
||||
if err := database.QueryRow(
|
||||
`SELECT expires_at > datetime('now','+29 days') FROM sessions`,
|
||||
).Scan(&extended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !extended {
|
||||
t.Fatal("using a session did not extend it")
|
||||
}
|
||||
}
|
||||
|
||||
// Two live sessions must each resolve to their own writer — the whole point.
|
||||
func TestSessionsAreNotInterchangeable(t *testing.T) {
|
||||
sessions, _, _ := newStores(t)
|
||||
|
||||
aliceToken, err := sessions.Create(db.LocalUserID, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bobToken, err := sessions.Create("bob", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for token, want := range map[string]string{aliceToken: db.LocalUserID, bobToken: "bob"} {
|
||||
got, err := sessions.Resolve(withCookie(token))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("token resolved to %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Revoking one session leaves the other alone.
|
||||
if err := sessions.Revoke(aliceToken); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := sessions.Resolve(withCookie(bobToken)); err != nil {
|
||||
t.Fatalf("bob was signed out by alice's logout: %v", err)
|
||||
}
|
||||
|
||||
// RevokeAll signs one writer out everywhere and nobody else.
|
||||
second, _ := sessions.Create("bob", "phone")
|
||||
if err := sessions.RevokeAll("bob"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, token := range []string{bobToken, second} {
|
||||
if _, err := sessions.Resolve(withCookie(token)); !errors.Is(err, ErrNoSession) {
|
||||
t.Fatalf("RevokeAll left a session alive (err=%v)", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The session store is itself the Resolver the API middleware runs on, so a
|
||||
// valid cookie must carry all the way through to the handler.
|
||||
func TestSessionStoreDrivesMiddleware(t *testing.T) {
|
||||
sessions, _, _ := newStores(t)
|
||||
token, err := sessions.Create("bob", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var seen string
|
||||
h := Middleware(sessions)(http.HandlerFunc(
|
||||
func(_ http.ResponseWriter, r *http.Request) { seen = UserID(r.Context()) },
|
||||
))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, withCookie(token))
|
||||
if rec.Code != http.StatusOK || seen != "bob" {
|
||||
t.Fatalf("status=%d user=%q, want 200/bob", rec.Code, seen)
|
||||
}
|
||||
|
||||
rec2 := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec2, httptest.NewRequest(http.MethodGet, "/", nil))
|
||||
if rec2.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status=%d for a cookieless request, want 401", rec2.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserUpsert(t *testing.T) {
|
||||
_, users, database := newStores(t)
|
||||
|
||||
if err := users.Upsert("sub-123", "her@example.com", "Her Name"); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
user, err := users.Get("sub-123")
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if user.Email != "her@example.com" || user.DisplayName != "Her Name" {
|
||||
t.Fatalf("unexpected user %+v", user)
|
||||
}
|
||||
if user.PairLang != "zh" {
|
||||
t.Fatalf("pair_lang = %q, want the zh default", user.PairLang)
|
||||
}
|
||||
|
||||
// A rename upstream is reflected here; Petal's own settings are not touched.
|
||||
if _, err := database.Exec(`UPDATE users SET pair_lang = 'pt-PT' WHERE id = 'sub-123'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := users.Upsert("sub-123", "new@example.com", "New Name"); err != nil {
|
||||
t.Fatalf("second upsert: %v", err)
|
||||
}
|
||||
user, _ = users.Get("sub-123")
|
||||
if user.Email != "new@example.com" || user.DisplayName != "New Name" {
|
||||
t.Fatalf("login did not refresh the profile: %+v", user)
|
||||
}
|
||||
if user.PairLang != "pt-PT" {
|
||||
t.Fatalf("login reset pair_lang to %q", user.PairLang)
|
||||
}
|
||||
|
||||
// Falling back to the email keeps the sidebar from showing an empty name.
|
||||
if err := users.Upsert("sub-456", "them@example.com", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if u, _ := users.Get("sub-456"); u.DisplayName != "them@example.com" {
|
||||
t.Fatalf("display name = %q, want the email fallback", u.DisplayName)
|
||||
}
|
||||
|
||||
if err := users.Upsert("", "nobody@example.com", "Nobody"); err == nil {
|
||||
t.Fatal("a login with no subject was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// Sessions belong to their account: deleting a user takes their logins with it.
|
||||
func TestSessionsCascadeWithUser(t *testing.T) {
|
||||
sessions, _, database := newStores(t)
|
||||
token, err := sessions.Create("bob", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := database.Exec(`DELETE FROM users WHERE id = 'bob'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := sessions.Resolve(withCookie(token)); !errors.Is(err, ErrNoSession) {
|
||||
t.Fatalf("session outlived its account (err=%v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowlist(t *testing.T) {
|
||||
// No list configured = anyone the IdP authenticates, which is the right
|
||||
// default for a household instance.
|
||||
if !ParseAllowlist("").Permits("anyone", "anyone@example.com") {
|
||||
t.Fatal("an empty allowlist turned someone away")
|
||||
}
|
||||
if !ParseAllowlist(" ").Permits("anyone", "anyone@example.com") {
|
||||
t.Fatal("a whitespace-only allowlist turned someone away")
|
||||
}
|
||||
|
||||
list := ParseAllowlist(" sub-123 , Her@Example.com ,, ")
|
||||
cases := []struct {
|
||||
sub, email string
|
||||
want bool
|
||||
}{
|
||||
{"sub-123", "someone@example.com", true}, // by subject
|
||||
{"sub-999", "her@example.com", true}, // by email
|
||||
{"sub-999", "HER@EXAMPLE.COM", true}, // case-insensitively
|
||||
{"sub-999", "stranger@example.com", false}, // neither
|
||||
{"", "", false}, // no claims at all
|
||||
{"sub-1", "", false}, // a near-miss subject
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := list.Permits(c.sub, c.email); got != c.want {
|
||||
t.Fatalf("Permits(%q, %q) = %v, want %v", c.sub, c.email, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Over https the cookie takes the __Host- prefix, which the browser will only
|
||||
// accept from the exact host that set it — closing the door on a sibling
|
||||
// service under the same registrable domain planting a session in her browser.
|
||||
// Over plain http it cannot: the prefix requires Secure, and a browser drops a
|
||||
// Secure cookie on an insecure origin, so local development would silently stop
|
||||
// logging in.
|
||||
func TestSessionCookieNamePerScheme(t *testing.T) {
|
||||
secure := httptest.NewRecorder()
|
||||
SetSessionCookie(secure, "tok", true)
|
||||
c := secure.Result().Cookies()[0]
|
||||
if c.Name != HostSessionCookie {
|
||||
t.Fatalf("https cookie name=%q, want %q", c.Name, HostSessionCookie)
|
||||
}
|
||||
// The prefix is a promise about these three attributes; a browser rejects
|
||||
// the cookie outright if any is wrong.
|
||||
if !c.Secure || c.Path != "/" || c.Domain != "" {
|
||||
t.Fatalf("__Host- cookie violates its own contract: %+v", c)
|
||||
}
|
||||
|
||||
insecure := httptest.NewRecorder()
|
||||
SetSessionCookie(insecure, "tok", false)
|
||||
if name := insecure.Result().Cookies()[0].Name; name != SessionCookie {
|
||||
t.Fatalf("http cookie name=%q, want %q", name, SessionCookie)
|
||||
}
|
||||
}
|
||||
|
||||
// A browser holding a cookie issued before the prefix existed must stay signed
|
||||
// in — and start using the new name at its next sign-in, not be logged out to
|
||||
// get there.
|
||||
func TestResolveAcceptsEitherCookieName(t *testing.T) {
|
||||
store, _, _ := newStores(t)
|
||||
token, err := store.Create("bob", "test-agent")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, name := range []string{SessionCookie, HostSessionCookie} {
|
||||
r := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
r.AddCookie(&http.Cookie{Name: name, Value: token})
|
||||
got, err := store.Resolve(r)
|
||||
if err != nil || got != "bob" {
|
||||
t.Fatalf("%s: resolved to %q (err=%v)", name, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Signing out must not leave the browser holding either spelling.
|
||||
func TestClearSessionCookieExpiresBothNames(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
ClearSessionCookie(rec, true)
|
||||
|
||||
cleared := map[string]bool{}
|
||||
for _, c := range rec.Result().Cookies() {
|
||||
if c.MaxAge < 0 {
|
||||
cleared[c.Name] = true
|
||||
}
|
||||
}
|
||||
for _, name := range []string{SessionCookie, HostSessionCookie} {
|
||||
if !cleared[name] {
|
||||
t.Errorf("%s was left in the browser after signing out", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
)
|
||||
|
||||
// UserStore provisions and reads accounts. Petal has no signup flow: a row
|
||||
// appears the first time someone Authentik vouches for signs in, and that is
|
||||
// the only way one is ever created.
|
||||
type UserStore struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// NewUserStore returns a store backed by the given database.
|
||||
func NewUserStore(sqlDB *sql.DB) *UserStore { return &UserStore{db: sqlDB} }
|
||||
|
||||
// Upsert records the account behind an OIDC login, keyed by the issuer's
|
||||
// subject id.
|
||||
//
|
||||
// The subject is the id — not the email, which people change and which
|
||||
// Authentik does not promise is stable. Email and display name are refreshed on
|
||||
// every login so a rename upstream shows up here; pair_lang is deliberately not
|
||||
// touched, because it is Petal's own setting rather than the IdP's.
|
||||
func (u *UserStore) Upsert(sub, email, displayName string) error {
|
||||
if sub == "" {
|
||||
return errors.New("oidc: empty subject")
|
||||
}
|
||||
if displayName == "" {
|
||||
displayName = email
|
||||
}
|
||||
_, err := u.db.Exec(
|
||||
`INSERT INTO users (id, email, display_name) VALUES (?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
email = excluded.email,
|
||||
display_name = excluded.display_name`,
|
||||
sub, email, displayName,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// Get loads one account.
|
||||
func (u *UserStore) Get(id string) (db.User, error) {
|
||||
var user db.User
|
||||
err := u.db.QueryRow(
|
||||
`SELECT id, email, COALESCE(display_name, ''), created_at, pair_lang, direction
|
||||
FROM users WHERE id = ?`, id,
|
||||
).Scan(&user.ID, &user.Email, &user.DisplayName, &user.CreatedAt, &user.PairLang, &user.Direction)
|
||||
return user, err
|
||||
}
|
||||
|
||||
// MeHandler reports who the caller is. The frontend uses it to namespace
|
||||
// per-account browser state and to show the signed-in writer; it sits behind
|
||||
// the auth middleware, so reaching it at all already proves a valid session.
|
||||
func (u *UserStore) MeHandler() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
user, err := u.Get(UserID(r.Context()))
|
||||
if err != nil {
|
||||
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, user)
|
||||
}
|
||||
}
|
||||
|
||||
// The pairs a writer may actually choose, in the order the picker offers them.
|
||||
//
|
||||
// This is deliberately *not* internal/llm's list of languages. That one names
|
||||
// every pair the prompts know how to talk about, which is a cheap thing to add;
|
||||
// this one names the pairs Petal can render itself in, which requires a langpack
|
||||
// on the frontend. Accepting a code with no pack would leave her looking at
|
||||
// Chinese with no way back except another guess, so the server refuses it. es
|
||||
// joined on the day its pack landed, not before.
|
||||
//
|
||||
// These four are now every pair PairLang names on the frontend, which makes the
|
||||
// two lists look redundant. They are not: the next pair will exist in the type
|
||||
// and in the prompts long before it has copy, and this list is the one that
|
||||
// says a writer may actually be sent there.
|
||||
var shippedPairs = []string{"zh", "pt-PT", "fr", "es"}
|
||||
|
||||
func pairIsShipped(lang string) bool {
|
||||
for _, p := range shippedPairs {
|
||||
if p == lang {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The two directions a pair can be travelled in. `DirectionLearningEn` is the
|
||||
// original assumption made explicit: the writer is native in X and practising
|
||||
// English. `DirectionLearningPair` is the other way round.
|
||||
const (
|
||||
DirectionLearningEn = "learning_en"
|
||||
DirectionLearningPair = "learning_pair"
|
||||
)
|
||||
|
||||
// The pairs whose *learner* direction Petal can actually serve, which is a
|
||||
// narrower thing than a shipped pair and narrower again than a langpack.
|
||||
//
|
||||
// Turning a pair around needs data no langpack carries: a word list to segment
|
||||
// with, and a dictionary that reads from the pair language into English. Chinese
|
||||
// has both as of Phase 26 (CC-CEDICT + jieba); French, Spanish and Portuguese
|
||||
// have neither yet, and — unlike a missing pack, which leaves a writer looking
|
||||
// at copy she cannot read — a missing word list would leave her looking at an
|
||||
// editor that silently does nothing when she hovers. Both are bad; only one is
|
||||
// legible as a bug. So the server refuses, for the same reason and by the same
|
||||
// mechanism as `shippedPairs`.
|
||||
//
|
||||
// This list is expected to grow one pair at a time and never to be inferred:
|
||||
// segmentation is a property of a writing system, and there is no rule that
|
||||
// derives "has a word list" from a language code.
|
||||
var learnerPairs = []string{"zh"}
|
||||
|
||||
// SupportsLearnerDirection reports whether a pair can be turned around.
|
||||
func SupportsLearnerDirection(lang string) bool {
|
||||
for _, p := range learnerPairs {
|
||||
if p == lang {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func directionIsKnown(d string) bool {
|
||||
return d == DirectionLearningEn || d == DirectionLearningPair
|
||||
}
|
||||
|
||||
// SetPair moves an account to another (English + X) pair, in a given direction.
|
||||
//
|
||||
// The two are written together because they constrain each other: a direction is
|
||||
// only meaningful for a pair that can be travelled in it, and validating them a
|
||||
// field at a time would let a two-step change pass through a state that neither
|
||||
// step is allowed to leave behind.
|
||||
func (u *UserStore) SetPair(id, lang, direction string) error {
|
||||
if !pairIsShipped(lang) {
|
||||
return errors.New("auth: unshipped pair language " + lang)
|
||||
}
|
||||
if !directionIsKnown(direction) {
|
||||
return errors.New("auth: unknown direction " + direction)
|
||||
}
|
||||
if direction == DirectionLearningPair && !SupportsLearnerDirection(lang) {
|
||||
return errors.New("auth: no learner direction for " + lang)
|
||||
}
|
||||
res, err := u.db.Exec(
|
||||
`UPDATE users SET pair_lang = ?, direction = ? WHERE id = ?`, lang, direction, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n, err := res.RowsAffected(); err == nil && n == 0 {
|
||||
return sql.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateMeHandler changes the caller's own settings: which language Petal
|
||||
// speaks alongside her English, and which of the two she is learning.
|
||||
//
|
||||
// It answers with the whole updated user rather than an empty 204 so the client
|
||||
// has one shape to trust: /api/me and this return the same thing, and the app
|
||||
// re-reads the pair from the response instead of assuming its request took.
|
||||
//
|
||||
// The pair language reaches further than the UI copy — it picks her Hunspell
|
||||
// dictionary, her read-aloud voice, which word-lookup provider answers, and the
|
||||
// language the prompts ask the model to explain in. All of those read
|
||||
// `users.pair_lang` at use time, so all of them follow from this one write.
|
||||
//
|
||||
// Both fields are optional and each defaults to what the account already has, so
|
||||
// the picker can send one without knowing the other. That matters for the
|
||||
// combination this endpoint exists to prevent: a client that sent only
|
||||
// `pair_lang: "fr"` while the account sat on `learning_pair` would otherwise ask
|
||||
// for French-with-segmentation, which does not exist. Here it is one decision
|
||||
// with one validation, and the answer carries whatever actually landed.
|
||||
func (u *UserStore) UpdateMeHandler() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
PairLang *string `json:"pair_lang"`
|
||||
Direction *string `json:"direction"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
httputil.BadRequest(w, "invalid request body")
|
||||
return
|
||||
}
|
||||
id := UserID(r.Context())
|
||||
current, err := u.Get(id)
|
||||
if err != nil {
|
||||
// Only a missing row means "not signed in". A dictionary-file or
|
||||
// SQLite fault answered as 401 would trip the client's session
|
||||
// interceptor and throw a writer out of an app she is still signed
|
||||
// in to — the same distinction SetPair's error branch makes below.
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||
return
|
||||
}
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
lang, direction := current.PairLang, current.Direction
|
||||
if body.PairLang != nil {
|
||||
lang = strings.TrimSpace(*body.PairLang)
|
||||
}
|
||||
if body.Direction != nil {
|
||||
direction = strings.TrimSpace(*body.Direction)
|
||||
}
|
||||
|
||||
if !pairIsShipped(lang) {
|
||||
// Name the ones that work. A writer who lands here has picked from a
|
||||
// stale client, and "not a language" tells her nothing.
|
||||
httputil.BadRequest(w, "unsupported language pair — Petal speaks "+strings.Join(shippedPairs, ", "))
|
||||
return
|
||||
}
|
||||
if !directionIsKnown(direction) {
|
||||
httputil.BadRequest(w, "unknown direction — expected "+DirectionLearningEn+" or "+DirectionLearningPair)
|
||||
return
|
||||
}
|
||||
if direction == DirectionLearningPair && !SupportsLearnerDirection(lang) {
|
||||
// Refused rather than quietly downgraded to learning_en. A silent
|
||||
// downgrade would leave the writer looking at an editor that behaves
|
||||
// like the one she just tried to leave, with nothing to read as an
|
||||
// explanation — and the caller cannot tell the two outcomes apart
|
||||
// without diffing the response it was given.
|
||||
httputil.BadRequest(w, "Petal can only be learned toward "+strings.Join(learnerPairs, ", ")+" so far")
|
||||
return
|
||||
}
|
||||
|
||||
if err := u.SetPair(id, lang, direction); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusUnauthorized, "not signed in")
|
||||
return
|
||||
}
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
user, err := u.Get(id)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, user)
|
||||
}
|
||||
}
|
||||
|
||||
// Allowlist decides which of Authentik's users may write in this Petal.
|
||||
// Authentik fronts several applications; being a valid user there does not mean
|
||||
// being a user here.
|
||||
//
|
||||
// An entry matches a subject id or an email address, case-insensitively. Both
|
||||
// are accepted on purpose: a subject is an opaque uuid nobody can know before
|
||||
// that person's first login, so a subject-only list means the operator must let
|
||||
// someone in, read a log line, and edit config — whereas an email is knowable in
|
||||
// advance. An empty list allows everyone the IdP authenticates, which is the
|
||||
// right default for a single-household instance.
|
||||
type Allowlist map[string]bool
|
||||
|
||||
// ParseAllowlist builds an Allowlist from a comma-separated env value.
|
||||
func ParseAllowlist(raw string) Allowlist {
|
||||
list := Allowlist{}
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
if p := strings.ToLower(strings.TrimSpace(part)); p != "" {
|
||||
list[p] = true
|
||||
}
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// Permits reports whether this login may proceed.
|
||||
func (a Allowlist) Permits(sub, email string) bool {
|
||||
if len(a) == 0 {
|
||||
return true
|
||||
}
|
||||
return a[strings.ToLower(sub)] || (email != "" && a[strings.ToLower(email)])
|
||||
}
|
||||
+155
-12
@@ -1,7 +1,10 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -12,6 +15,12 @@ type Config struct {
|
||||
BaseURL string
|
||||
DatabasePath string
|
||||
ImageDir string // on-disk store for editor image uploads
|
||||
// DictPath is DreamDict's built dict.db, read-only, sitting beside
|
||||
// petal.db. It is what gives Petal French, European Portuguese and Spanish
|
||||
// word lookups; without it only the embedded English/Chinese datasets
|
||||
// exist, which is exactly how a laptop checkout runs. A missing file is
|
||||
// therefore not an error — see lexicon.OpenDreamDict.
|
||||
DictPath string
|
||||
|
||||
// LLM
|
||||
LLMBackend string // "vllm" | "ollama"
|
||||
@@ -23,28 +32,76 @@ type Config struct {
|
||||
// TTS (read-aloud). Off unless TTSEndpoint is set — when empty, the /api/tts
|
||||
// route isn't mounted and the frontend falls back to the browser's Web Speech
|
||||
// API. Endpoint points at a local Piper HTTP server.
|
||||
TTSEndpoint string // Piper instance serving the English voice
|
||||
TTSEndpointZH string // Piper instance serving the Chinese voice; empty = zh falls back to Web Speech
|
||||
TTSVoiceEN string // Piper voice id for English (e.g. en_US-amy-medium)
|
||||
TTSVoiceZH string // Piper voice id for Chinese (e.g. zh_CN-huayan-medium)
|
||||
TTSEndpoint string // Piper instance serving the English voice; also the on/off switch
|
||||
// TTSVoices is every language Petal can read aloud, keyed by base language
|
||||
// tag ("en", "zh", "pt", …). Each Piper server loads exactly one model, so
|
||||
// a language *is* an instance — and the instances are discovered from the
|
||||
// environment rather than named in this struct: one
|
||||
// TTS_ENDPOINT_<LANG>/TTS_VOICE_<LANG> pair per language, so the fr and es
|
||||
// pairs cost a compose service and two lines of .env rather than a code
|
||||
// change. English keeps the unsuffixed TTS_ENDPOINT/TTS_VOICE_EN it has
|
||||
// always had.
|
||||
TTSVoices map[string]TTSVoice
|
||||
// TTSPath is the path Piper serves synthesis on. Piper moved it from "/" to
|
||||
// "/synthesize" in 1.6.0 with an unchanged request body, so this is a
|
||||
// version knob, not a feature: millenia's older server keeps the default,
|
||||
// the containerised sidecars set "/synthesize".
|
||||
TTSPath string
|
||||
TTSCacheDir string // on-disk store for synthesized clips (content-addressed)
|
||||
TTSTimeout time.Duration
|
||||
TTSFormat string // mp3 | opus | wav — mp3/opus transcode Piper's WAV via ffmpeg
|
||||
|
||||
// Auth (deferred — not wired in the local-dev build, kept for later)
|
||||
AuthentikURL string
|
||||
// Auth. OIDC against Authentik. Login is enabled only when the issuer, the
|
||||
// client id and the secret are all present; with any of them missing Petal
|
||||
// falls back to the single hardcoded local user, which is what local
|
||||
// development wants and what every deployment did before Phase 16.
|
||||
AuthentikURL string // issuer URL of the Petal provider in Authentik
|
||||
AuthentikClientID string
|
||||
AuthentikClientSecret string
|
||||
SessionSecret string
|
||||
// AllowedSubs gates who may sign in, as a comma-separated list of OIDC
|
||||
// subject ids and/or email addresses. Empty means everyone Authentik
|
||||
// authenticates — right for a single-household instance, wrong the moment
|
||||
// the IdP serves an audience wider than Petal's.
|
||||
AllowedSubs string
|
||||
// RequireAuth refuses to start when OIDC isn't configured, instead of
|
||||
// falling back to the single local user.
|
||||
//
|
||||
// The fallback is the right behaviour on a laptop and a catastrophe on a
|
||||
// public host: a typo in AUTHENTIK_CLIENT_SECRET turns every anonymous
|
||||
// visitor into the `local` user, with full read and write over someone's
|
||||
// private journals, and says so only in a log line nobody is reading. The
|
||||
// Traefik basic-auth gate that used to stand behind that mistake was
|
||||
// removed when Petal learned to authenticate for itself, so nothing catches
|
||||
// it now.
|
||||
//
|
||||
// Defaulted from BASE_URL rather than declared: a Petal that knows itself by
|
||||
// a real public origin has no business running open, and one on localhost
|
||||
// has no business demanding an IdP. Set PETAL_REQUIRE_AUTH explicitly to
|
||||
// override in either direction.
|
||||
RequireAuth bool
|
||||
}
|
||||
|
||||
// TTSVoice is one Piper instance and the single voice it has loaded.
|
||||
type TTSVoice struct {
|
||||
Endpoint string
|
||||
Voice string
|
||||
}
|
||||
|
||||
// AuthEnabled reports whether real logins are configured. When false, Petal
|
||||
// resolves every request to the local user.
|
||||
func (c *Config) AuthEnabled() bool {
|
||||
return c.AuthentikURL != "" && c.AuthentikClientID != "" && c.AuthentikClientSecret != ""
|
||||
}
|
||||
|
||||
// Load reads configuration from the environment, applying sane local-dev defaults.
|
||||
func Load() *Config {
|
||||
baseURL := env("BASE_URL", "http://localhost:8080")
|
||||
return &Config{
|
||||
Port: env("PORT", "8080"),
|
||||
BaseURL: env("BASE_URL", "http://localhost:8080"),
|
||||
BaseURL: baseURL,
|
||||
DatabasePath: env("DATABASE_PATH", "./data/petal.db"),
|
||||
ImageDir: env("IMAGE_DIR", "./data/images"),
|
||||
DictPath: env("DICT_PATH", "./data/dict.db"),
|
||||
|
||||
LLMBackend: env("LLM_BACKEND", "vllm"),
|
||||
LLMEndpoint: env("LLM_ENDPOINT", "http://localhost:8000"),
|
||||
@@ -53,9 +110,8 @@ func Load() *Config {
|
||||
LLMTimeout: envDuration("LLM_TIMEOUT", 30*time.Second),
|
||||
|
||||
TTSEndpoint: env("TTS_ENDPOINT", ""),
|
||||
TTSEndpointZH: env("TTS_ENDPOINT_ZH", ""),
|
||||
TTSVoiceEN: env("TTS_VOICE_EN", "en_US-amy-medium"),
|
||||
TTSVoiceZH: env("TTS_VOICE_ZH", "zh_CN-huayan-medium"),
|
||||
TTSVoices: ttsVoices(os.Environ()),
|
||||
TTSPath: env("TTS_PATH", "/"),
|
||||
TTSCacheDir: env("TTS_CACHE_DIR", "./data/tts"),
|
||||
TTSTimeout: envDuration("TTS_TIMEOUT", 15*time.Second),
|
||||
TTSFormat: env("TTS_AUDIO_FORMAT", "mp3"),
|
||||
@@ -63,10 +119,85 @@ func Load() *Config {
|
||||
AuthentikURL: env("AUTHENTIK_URL", ""),
|
||||
AuthentikClientID: env("AUTHENTIK_CLIENT_ID", ""),
|
||||
AuthentikClientSecret: env("AUTHENTIK_CLIENT_SECRET", ""),
|
||||
SessionSecret: env("SESSION_SECRET", "dev-insecure-secret-change-me"),
|
||||
AllowedSubs: env("PETAL_ALLOWED_SUBS", ""),
|
||||
RequireAuth: envBool("PETAL_REQUIRE_AUTH", !isLoopbackOrigin(baseURL)),
|
||||
}
|
||||
}
|
||||
|
||||
// isLoopbackOrigin reports whether a base URL names this machine — the shape a
|
||||
// development checkout has, and the only shape where running without a login is
|
||||
// a reasonable default. Anything else (a hostname, a public origin) is a
|
||||
// deployment, however small.
|
||||
func isLoopbackOrigin(baseURL string) bool {
|
||||
u, err := url.Parse(strings.TrimSpace(baseURL))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(u.Hostname()) {
|
||||
case "localhost", "127.0.0.1", "::1", "":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ttsVoices reads the Piper instances out of an environment slice (as returned
|
||||
// by os.Environ) into a map keyed by base language tag.
|
||||
//
|
||||
// English is the unsuffixed pair, TTS_ENDPOINT + TTS_VOICE_EN, because that is
|
||||
// what every deployment already sets and read-aloud has always been English
|
||||
// first. Every other language is a TTS_ENDPOINT_<LANG>/TTS_VOICE_<LANG> pair,
|
||||
// discovered rather than enumerated — TTS_ENDPOINT_ZH is what millenia and the
|
||||
// VPS already use, and TTS_ENDPOINT_PT is all the Portuguese pair needs.
|
||||
//
|
||||
// <LANG> is the *base* tag: an environment variable name cannot hold the hyphen
|
||||
// in "pt-PT", and the handler routes on the base tag anyway (a request for
|
||||
// pt-PT, pt-BR or bare pt reaches the same instance, because there is only one
|
||||
// Portuguese voice loaded). A pair is ignored unless both halves are set: half
|
||||
// a configuration should read as "no voice for this language" and fall back to
|
||||
// the browser, not as an instance that answers every request with an error.
|
||||
func ttsVoices(environ []string) map[string]TTSVoice {
|
||||
vals := make(map[string]string, len(environ))
|
||||
for _, kv := range environ {
|
||||
if k, v, ok := strings.Cut(kv, "="); ok {
|
||||
vals[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
voices := map[string]TTSVoice{}
|
||||
add := func(lang, endpoint, voice string) {
|
||||
endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/")
|
||||
voice = strings.TrimSpace(voice)
|
||||
if endpoint == "" || voice == "" {
|
||||
return
|
||||
}
|
||||
voices[lang] = TTSVoice{Endpoint: endpoint, Voice: voice}
|
||||
}
|
||||
|
||||
// The two languages that shipped before this was a map keep their voice
|
||||
// defaults, so an existing deployment that names only the endpoints (as
|
||||
// millenia's unit does) sounds exactly as it did.
|
||||
voiceOr := func(key, fallback string) string {
|
||||
if v := strings.TrimSpace(vals[key]); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
add("en", vals["TTS_ENDPOINT"], voiceOr("TTS_VOICE_EN", "en_US-amy-medium"))
|
||||
for k, endpoint := range vals {
|
||||
suffix, ok := strings.CutPrefix(k, "TTS_ENDPOINT_")
|
||||
if !ok || suffix == "" {
|
||||
continue
|
||||
}
|
||||
voice := vals["TTS_VOICE_"+suffix]
|
||||
if suffix == "ZH" {
|
||||
voice = voiceOr("TTS_VOICE_ZH", "zh_CN-huayan-medium")
|
||||
}
|
||||
add(strings.ToLower(suffix), endpoint, voice)
|
||||
}
|
||||
return voices
|
||||
}
|
||||
|
||||
func env(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
@@ -74,6 +205,18 @@ func env(key, fallback string) string {
|
||||
return fallback
|
||||
}
|
||||
|
||||
// envBool reads a boolean knob. Anything unparseable keeps the default rather
|
||||
// than silently reading as false — a mistyped PETAL_REQUIRE_AUTH must not be the
|
||||
// thing that turns the guard off.
|
||||
func envBool(key string, fallback bool) bool {
|
||||
if v := strings.TrimSpace(os.Getenv(key)); v != "" {
|
||||
if b, err := strconv.ParseBool(v); err == nil {
|
||||
return b
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envDuration(key string, fallback time.Duration) time.Duration {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package config
|
||||
|
||||
import "testing"
|
||||
|
||||
// The Piper instances are discovered from the environment rather than named in
|
||||
// code, so that a new pair costs a compose service and two .env lines. These
|
||||
// assert the discovery rule, including the two shapes that already exist in the
|
||||
// wild (millenia's systemd unit and the VPS compose file).
|
||||
func TestTTSVoicesDiscovery(t *testing.T) {
|
||||
voices := ttsVoices([]string{
|
||||
"TTS_ENDPOINT=http://piper-en:5000",
|
||||
"TTS_VOICE_EN=en_US-amy-medium",
|
||||
"TTS_ENDPOINT_ZH=http://piper-zh:5000/",
|
||||
"TTS_VOICE_ZH=zh_CN-huayan-medium",
|
||||
"TTS_ENDPOINT_PT=http://piper-pt:5000",
|
||||
"TTS_VOICE_PT=pt_PT-tugão-medium",
|
||||
"TTS_ENDPOINT_FR=http://piper-fr:5000",
|
||||
"TTS_VOICE_FR=fr_FR-siwis-medium",
|
||||
"TTS_ENDPOINT_ES=http://piper-es:5000",
|
||||
"TTS_VOICE_ES=es_MX-ald-medium",
|
||||
// Noise that must not become a language.
|
||||
"TTS_PATH=/synthesize",
|
||||
"PATH=/usr/bin",
|
||||
})
|
||||
|
||||
want := map[string]TTSVoice{
|
||||
"en": {"http://piper-en:5000", "en_US-amy-medium"},
|
||||
// The trailing slash is trimmed here so the synthesis path concatenates
|
||||
// cleanly rather than producing a double slash at every call site.
|
||||
"zh": {"http://piper-zh:5000", "zh_CN-huayan-medium"},
|
||||
"pt": {"http://piper-pt:5000", "pt_PT-tugão-medium"},
|
||||
// Phase 24's whole TTS change: a fourth language costs two lines here
|
||||
// and a compose service, and no Go at all.
|
||||
"fr": {"http://piper-fr:5000", "fr_FR-siwis-medium"},
|
||||
// And a fifth cost exactly the same, which is the claim actually being
|
||||
// tested. The voice is Mexican on purpose: the es pack is Latin
|
||||
// American, and es_ES-davefx-medium would read it in the wrong accent.
|
||||
"es": {"http://piper-es:5000", "es_MX-ald-medium"},
|
||||
}
|
||||
if len(voices) != len(want) {
|
||||
t.Fatalf("discovered %v, want %v", voices, want)
|
||||
}
|
||||
for lang, w := range want {
|
||||
if voices[lang] != w {
|
||||
t.Errorf("%s = %+v, want %+v", lang, voices[lang], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Half a configuration is not a language. An endpoint with no voice (or the
|
||||
// reverse) must read as "no voice for this language" — a 404 the client answers
|
||||
// by falling back to Web Speech — rather than as an instance that exists and
|
||||
// errors on every request.
|
||||
func TestTTSVoicesIgnoresHalfConfiguredLanguages(t *testing.T) {
|
||||
voices := ttsVoices([]string{
|
||||
"TTS_ENDPOINT=http://piper-en:5000",
|
||||
"TTS_VOICE_EN=en_US-amy-medium",
|
||||
"TTS_ENDPOINT_FR=http://piper-fr:5000", // no TTS_VOICE_FR
|
||||
"TTS_VOICE_ES=es_ES-davefx-medium", // no TTS_ENDPOINT_ES
|
||||
})
|
||||
if _, ok := voices["fr"]; ok {
|
||||
t.Errorf("fr routed with no voice configured")
|
||||
}
|
||||
if _, ok := voices["es"]; ok {
|
||||
t.Errorf("es routed with no endpoint configured")
|
||||
}
|
||||
if len(voices) != 1 {
|
||||
t.Errorf("discovered %v, want English only", voices)
|
||||
}
|
||||
}
|
||||
|
||||
// A deployment that predates the map names only the endpoints and relies on the
|
||||
// voice defaults; it must sound exactly as it did.
|
||||
func TestTTSVoicesKeepsTheOriginalDefaults(t *testing.T) {
|
||||
voices := ttsVoices([]string{
|
||||
"TTS_ENDPOINT=http://127.0.0.1:5005",
|
||||
"TTS_ENDPOINT_ZH=http://127.0.0.1:5006",
|
||||
})
|
||||
if got := voices["en"].Voice; got != "en_US-amy-medium" {
|
||||
t.Errorf("en voice = %q, want the default", got)
|
||||
}
|
||||
if got := voices["zh"].Voice; got != "zh_CN-huayan-medium" {
|
||||
t.Errorf("zh voice = %q, want the default", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Read-aloud is off when no English instance is configured; nothing else may
|
||||
// switch it on. (tts.New gates on TTSEndpoint, so a stray TTS_ENDPOINT_PT with
|
||||
// no English sibling must not produce a routable map that outlives that gate.)
|
||||
func TestTTSVoicesEmptyWithoutEndpoints(t *testing.T) {
|
||||
if voices := ttsVoices([]string{"TTS_VOICE_EN=en_US-amy-medium"}); len(voices) != 0 {
|
||||
t.Errorf("discovered %v, want none", voices)
|
||||
}
|
||||
}
|
||||
|
||||
// The fallback to the single local user is right on a laptop and a catastrophe
|
||||
// on a public host, so it is defaulted from the origin Petal knows itself by
|
||||
// rather than left to be remembered.
|
||||
func TestRequireAuthDefaultsFromBaseURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
baseURL string
|
||||
want bool
|
||||
}{
|
||||
{"http://localhost:8080", false},
|
||||
{"http://127.0.0.1:8080", false},
|
||||
{"http://[::1]:8080", false},
|
||||
{"", false}, // no BASE_URL set at all: the local-dev default
|
||||
{"https://petal.parodia.dev", true},
|
||||
{"http://petal.parodia.dev", true},
|
||||
{"https://petal.example.com/", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Setenv("BASE_URL", c.baseURL)
|
||||
t.Setenv("PETAL_REQUIRE_AUTH", "")
|
||||
if got := Load().RequireAuth; got != c.want {
|
||||
t.Errorf("BASE_URL=%q: RequireAuth=%v, want %v", c.baseURL, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The default is a default, not a rule: a trusted private network is a real
|
||||
// deployment shape, and so is wanting the guard on locally.
|
||||
func TestRequireAuthExplicitOverride(t *testing.T) {
|
||||
t.Setenv("BASE_URL", "https://petal.parodia.dev")
|
||||
t.Setenv("PETAL_REQUIRE_AUTH", "false")
|
||||
if Load().RequireAuth {
|
||||
t.Error("an explicit false must be honoured on a public origin")
|
||||
}
|
||||
|
||||
t.Setenv("BASE_URL", "http://localhost:8080")
|
||||
t.Setenv("PETAL_REQUIRE_AUTH", "true")
|
||||
if !Load().RequireAuth {
|
||||
t.Error("an explicit true must be honoured on localhost")
|
||||
}
|
||||
|
||||
// A typo must not be the thing that disables the guard.
|
||||
t.Setenv("BASE_URL", "https://petal.parodia.dev")
|
||||
t.Setenv("PETAL_REQUIRE_AUTH", "nope")
|
||||
if !Load().RequireAuth {
|
||||
t.Error("an unparseable value must keep the default, not read as false")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// Backup writes a consistent copy of the database at srcPath to destPath using
|
||||
// SQLite's `VACUUM INTO`.
|
||||
//
|
||||
// Why not copy the file: Petal runs in WAL mode, so at any instant the newest
|
||||
// committed pages may live in petal.db-wal rather than petal.db. Copying the
|
||||
// three files separately can capture them mid-checkpoint and produce a backup
|
||||
// that is subtly torn. `VACUUM INTO` runs inside a read transaction, so it sees
|
||||
// one coherent snapshot including the WAL, and emits a single defragmented file
|
||||
// with no -wal/-shm companions — exactly what you want to ship off-box.
|
||||
//
|
||||
// It takes no write lock, so this is safe to run against the live database
|
||||
// while someone is writing.
|
||||
//
|
||||
// destPath must not already exist: SQLite refuses to overwrite, which keeps a
|
||||
// failed run from destroying the previous good backup.
|
||||
func Backup(srcPath, destPath string) error {
|
||||
if _, err := os.Stat(srcPath); err != nil {
|
||||
return fmt.Errorf("source database: %w", err)
|
||||
}
|
||||
if _, err := os.Stat(destPath); err == nil {
|
||||
return fmt.Errorf("destination %s already exists", destPath)
|
||||
}
|
||||
if dir := filepath.Dir(destPath); dir != "" && dir != "." {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("create backup dir: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Opened directly rather than through Open: a backup must never migrate or
|
||||
// seed the database it is copying.
|
||||
conn, err := sql.Open("sqlite", dsn(srcPath))
|
||||
if err != nil {
|
||||
return fmt.Errorf("open source: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
conn.SetMaxOpenConns(1)
|
||||
|
||||
if err := conn.Ping(); err != nil {
|
||||
return fmt.Errorf("ping source: %w", err)
|
||||
}
|
||||
|
||||
// The path is interpolated because VACUUM INTO takes a literal, not a bound
|
||||
// parameter. Quotes are doubled so a path containing one can't break out.
|
||||
quoted := "'" + escapeSQLiteString(destPath) + "'"
|
||||
if _, err := conn.Exec("VACUUM INTO " + quoted); err != nil {
|
||||
return fmt.Errorf("vacuum into %s: %w", destPath, err)
|
||||
}
|
||||
|
||||
// A zero-byte result would mean the vacuum silently produced nothing; catch
|
||||
// it here rather than discovering it during a restore.
|
||||
info, err := os.Stat(destPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat backup: %w", err)
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
return fmt.Errorf("backup %s is empty", destPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func escapeSQLiteString(s string) string {
|
||||
out := make([]byte, 0, len(s))
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '\'' {
|
||||
out = append(out, '\'')
|
||||
}
|
||||
out = append(out, s[i])
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The point of VACUUM INTO over a file copy is that it captures rows still
|
||||
// sitting in the WAL. This writes with the source connection open (so the WAL
|
||||
// is hot and unlikely to have been checkpointed) and asserts the backup has
|
||||
// them.
|
||||
func TestBackupCapturesLiveWrites(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := filepath.Join(dir, "petal.db")
|
||||
dest := filepath.Join(dir, "backups", "petal-backup.db")
|
||||
|
||||
d, err := Open(src)
|
||||
if err != nil {
|
||||
t.Fatalf("open source: %v", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
if _, err := d.Exec(
|
||||
`INSERT INTO documents (id, user_id, title, content_text) VALUES ('d1', ?, '春天', 'hello 春天')`,
|
||||
LocalUserID,
|
||||
); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
|
||||
if err := Backup(src, dest); err != nil {
|
||||
t.Fatalf("backup: %v", err)
|
||||
}
|
||||
|
||||
// VACUUM INTO emits a single self-contained file — no -wal/-shm to ship
|
||||
// alongside it.
|
||||
for _, suffix := range []string{"-wal", "-shm"} {
|
||||
if _, err := os.Stat(dest + suffix); err == nil {
|
||||
t.Errorf("backup left a %s companion file behind", suffix)
|
||||
}
|
||||
}
|
||||
|
||||
copyConn, err := sql.Open("sqlite", dsn(dest))
|
||||
if err != nil {
|
||||
t.Fatalf("open backup: %v", err)
|
||||
}
|
||||
defer copyConn.Close()
|
||||
|
||||
var title string
|
||||
if err := copyConn.QueryRow(`SELECT title FROM documents WHERE id = 'd1'`).Scan(&title); err != nil {
|
||||
t.Fatalf("row missing from backup: %v", err)
|
||||
}
|
||||
if title != "春天" {
|
||||
t.Errorf("title = %q, want 春天", title)
|
||||
}
|
||||
|
||||
// The seeded user has to come across too, or a restore would orphan every
|
||||
// document's foreign key.
|
||||
var users int
|
||||
if err := copyConn.QueryRow(`SELECT COUNT(*) FROM users WHERE id = ?`, LocalUserID).Scan(&users); err != nil {
|
||||
t.Fatalf("count users: %v", err)
|
||||
}
|
||||
if users != 1 {
|
||||
t.Errorf("users in backup = %d, want 1", users)
|
||||
}
|
||||
}
|
||||
|
||||
// A second run to the same path must fail loudly rather than clobber or
|
||||
// half-write the previous good backup.
|
||||
func TestBackupRefusesExistingDestination(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := filepath.Join(dir, "petal.db")
|
||||
dest := filepath.Join(dir, "petal-backup.db")
|
||||
|
||||
d, err := Open(src)
|
||||
if err != nil {
|
||||
t.Fatalf("open source: %v", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
if err := Backup(src, dest); err != nil {
|
||||
t.Fatalf("first backup: %v", err)
|
||||
}
|
||||
before, err := os.ReadFile(dest)
|
||||
if err != nil {
|
||||
t.Fatalf("read backup: %v", err)
|
||||
}
|
||||
|
||||
if err := Backup(src, dest); err == nil {
|
||||
t.Fatal("second backup to the same path succeeded; want an error")
|
||||
}
|
||||
|
||||
after, err := os.ReadFile(dest)
|
||||
if err != nil {
|
||||
t.Fatalf("re-read backup: %v", err)
|
||||
}
|
||||
if len(before) != len(after) {
|
||||
t.Errorf("existing backup was modified: %d bytes → %d", len(before), len(after))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupMissingSource(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := Backup(filepath.Join(dir, "nope.db"), filepath.Join(dir, "out.db")); err == nil {
|
||||
t.Fatal("backup of a nonexistent database succeeded; want an error")
|
||||
}
|
||||
}
|
||||
@@ -365,6 +365,230 @@ SELECT id, doc_id, from_pos, to_pos, original, replacement, explanation, type, s
|
||||
DROP TABLE suggestions;
|
||||
ALTER TABLE suggestions_new RENAME TO suggestions;
|
||||
CREATE INDEX idx_suggestions_doc_id ON suggestions(doc_id);
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Writing passport: evidence that a document was written, not pasted.
|
||||
//
|
||||
// `preserve_history` opts a document out of auto-snapshot pruning. The
|
||||
// 40-snapshot cap is right for recovery (you want recent states) but
|
||||
// wrong for provenance (you want the *whole* span, oldest included), so
|
||||
// a writer who may need to defend authorship flags the doc and keeps
|
||||
// every snapshot.
|
||||
//
|
||||
// `content_hash`/`prev_hash` chain each snapshot to the one before it:
|
||||
// hash = sha256(prev_hash | doc_id | created_at | word_count | text).
|
||||
// This proves the local history is internally consistent — no snapshot
|
||||
// was edited, reordered, or removed after the fact without breaking
|
||||
// every link downstream. It is NOT third-party attestation: anyone with
|
||||
// the DB and the algorithm could forge a fresh chain. It raises the cost
|
||||
// of a doctored history from "edit one row" to "rebuild all of them".
|
||||
// Pre-existing snapshots keep empty hashes and are reported as
|
||||
// unverifiable rather than as failures.
|
||||
name: "0009_writing_passport",
|
||||
stmt: `
|
||||
ALTER TABLE documents ADD COLUMN preserve_history INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE document_versions ADD COLUMN content_hash TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE document_versions ADD COLUMN prev_hash TEXT NOT NULL DEFAULT '';
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Real accounts. Three separate things land together because they are
|
||||
// one change: Petal can now tell users apart.
|
||||
//
|
||||
// `sessions` backs server-side login state. The cookie carries an opaque
|
||||
// random token and this table stores only its SHA-256 — a leaked database
|
||||
// copy therefore yields no usable session, the same reason passwords are
|
||||
// hashed. Server-side rows (rather than a signed stateless cookie) are
|
||||
// what make logout and revocation actually revoke.
|
||||
//
|
||||
// `images` gives the content-addressed image store an owner. Until now it
|
||||
// was a flat directory with no database row at all: any caller holding a
|
||||
// hash could fetch anyone's image, which is capability-URL security, not
|
||||
// access control. The primary key is (name, user_id), so the same picture
|
||||
// uploaded by two people is still stored once on disk and simply has two
|
||||
// rows — deduplication survives; the file is deleted only with its last
|
||||
// row. Rows for images already on disk are backfilled at startup by the
|
||||
// images package, which is the only code that knows the storage path.
|
||||
//
|
||||
// `users.pair_lang` is the writer's language pair (English + X). It is
|
||||
// unused until the langpack work, but it belongs to provisioning and
|
||||
// costs nothing to add while the users table is already being touched.
|
||||
name: "0010_sessions_images_and_pair_lang",
|
||||
stmt: `
|
||||
CREATE TABLE sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at DATETIME NOT NULL,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
user_agent TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
CREATE INDEX idx_sessions_user_id ON sessions(user_id);
|
||||
|
||||
CREATE TABLE images (
|
||||
name TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
content_type TEXT NOT NULL DEFAULT '',
|
||||
size INTEGER NOT NULL DEFAULT 0,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (name, user_id)
|
||||
);
|
||||
CREATE INDEX idx_images_user_id ON images(user_id);
|
||||
|
||||
ALTER TABLE users ADD COLUMN pair_lang TEXT NOT NULL DEFAULT 'zh';
|
||||
`,
|
||||
},
|
||||
{
|
||||
// The personal spelling dictionary moves off the browser. It used to be a
|
||||
// single `petal.spell.personal` key in localStorage, which meant two
|
||||
// people sharing a device shared a word list built from one person's
|
||||
// private writing — and one person writing on two devices had two
|
||||
// unrelated lists.
|
||||
//
|
||||
// `lang` is the *dictionary's* language, not the writer's: a word is only
|
||||
// ever added while a particular Hunspell dictionary flagged it, and an
|
||||
// en-US personal word must not silence a pt-PT flag (or vice versa) once
|
||||
// the second pair ships. `word` is stored as typed; matching is exact,
|
||||
// because case carries meaning to a speller ("polish" vs "Polish").
|
||||
name: "0011_personal_dictionary",
|
||||
stmt: `
|
||||
CREATE TABLE personal_words (
|
||||
user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
lang TEXT NOT NULL,
|
||||
word TEXT NOT NULL,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (user_id, lang, word)
|
||||
);
|
||||
`,
|
||||
},
|
||||
{
|
||||
// The growth journal reads the suggestions table as a record of what the
|
||||
// writer has been learning, and that reading only works if a row is dated
|
||||
// by *her decision* rather than by the model's proposal. `created_at` is
|
||||
// when a checkpoint offered the edit; a suggestion offered in April and
|
||||
// accepted in June is June's growth, not April's.
|
||||
//
|
||||
// Existing rows are backfilled to created_at — which is exactly the
|
||||
// approximation the journal would have had to make anyway, and is very
|
||||
// nearly right in practice since edits are settled minutes after a
|
||||
// checkpoint. Only pending rows keep a NULL: nothing has been decided.
|
||||
name: "0012_suggestion_resolved_at",
|
||||
stmt: `
|
||||
ALTER TABLE suggestions ADD COLUMN resolved_at DATETIME;
|
||||
UPDATE suggestions SET resolved_at = created_at WHERE status != 'pending';
|
||||
CREATE INDEX idx_suggestions_resolved ON suggestions(status, resolved_at);
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Which engine proposed a row. Until now `type` doubled as that answer —
|
||||
// 'mechanics' meant "the offline rule pack found this" and everything else
|
||||
// meant "the model did". That breaks the moment an offline rule proposes a
|
||||
// *collocation*: the miscollocation list (SUGGESTIONS §6) is the same
|
||||
// family, the same rail and the same warm phrasing as the LLM coach, and it
|
||||
// must stay type='collocation' so an accepted chunk still plants in the
|
||||
// garden and still counts in the journal. With type no longer naming the
|
||||
// engine, the two passes could not scope their own DELETEs — the coach
|
||||
// would wipe the offline flags, and the offline pass would leave the
|
||||
// coach's behind to accumulate.
|
||||
//
|
||||
// Existing mechanics rows are local by definition; everything else came
|
||||
// from a model.
|
||||
name: "0013_suggestion_source",
|
||||
stmt: `
|
||||
ALTER TABLE suggestions ADD COLUMN source TEXT NOT NULL DEFAULT 'llm';
|
||||
UPDATE suggestions SET source = 'local' WHERE type = 'mechanics';
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Sentence-level identity, so a re-check stops regenerating the world.
|
||||
// Every pass used to delete its whole family and re-insert it, which
|
||||
// meant accepting one edit gave every other card a new id and a newly
|
||||
// worded explanation — the rail visibly emptied and refilled, and the
|
||||
// model was asked again about sentences nobody had touched.
|
||||
//
|
||||
// `chunk_hash` records which sentence a suggestion belongs to, and
|
||||
// checked_chunks records which sentences a family has already read. A
|
||||
// re-check then asks only about the difference and keeps the rest of
|
||||
// the rows exactly as they are, id and wording included.
|
||||
//
|
||||
// Existing rows get '' — "sentence unknown", which reads as in-play, so
|
||||
// they are simply reconciled on the next pass like any fresh finding.
|
||||
name: "0014_suggestion_chunk_hash",
|
||||
stmt: `
|
||||
ALTER TABLE suggestions ADD COLUMN chunk_hash TEXT NOT NULL DEFAULT '';
|
||||
|
||||
CREATE TABLE checked_chunks (
|
||||
doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
family TEXT NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
PRIMARY KEY (doc_id, family, hash)
|
||||
);
|
||||
`,
|
||||
},
|
||||
{
|
||||
// A sentence she wrote in her own language gets its own type. Petal already
|
||||
// detected such spans and already rendered them into English — it just
|
||||
// filed the result under 'clarity', so the pair model's flagship moment
|
||||
// read as tidying up her Chinese. As with 0005 and 0008, the `type` CHECK
|
||||
// can't be ALTERed in place, so rebuild the table with the extended
|
||||
// constraint, copy every row across, and recreate both indexes.
|
||||
//
|
||||
// Existing rows are left on whatever type they have. A card she is already
|
||||
// reading keeps the label she has already read (the same rule reconcile.go
|
||||
// follows for a re-proposed edit); new findings get the new label.
|
||||
name: "0015_translate_suggestion_type",
|
||||
stmt: `
|
||||
CREATE TABLE suggestions_new (
|
||||
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
||||
doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
from_pos INTEGER NOT NULL,
|
||||
to_pos INTEGER NOT NULL,
|
||||
original TEXT NOT NULL,
|
||||
replacement TEXT NOT NULL,
|
||||
explanation TEXT NOT NULL,
|
||||
type TEXT NOT NULL CHECK(type IN ('grammar','phrasing','idiom','clarity','translate','voice','collocation','mechanics')),
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','accepted','rejected')),
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
resolved_at DATETIME,
|
||||
source TEXT NOT NULL DEFAULT 'llm',
|
||||
chunk_hash TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
INSERT INTO suggestions_new (id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash)
|
||||
SELECT id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash FROM suggestions;
|
||||
|
||||
DROP TABLE suggestions;
|
||||
ALTER TABLE suggestions_new RENAME TO suggestions;
|
||||
CREATE INDEX idx_suggestions_doc_id ON suggestions(doc_id);
|
||||
CREATE INDEX idx_suggestions_resolved ON suggestions(status, resolved_at);
|
||||
`,
|
||||
},
|
||||
{
|
||||
// Which half of the pair is being learned.
|
||||
//
|
||||
// `pair_lang` (0010) has always answered "which two languages", and every
|
||||
// surface built on it assumed the answer to a second question nobody had
|
||||
// asked: that English is the language being *learned*. That assumption is
|
||||
// load-bearing in a dozen places — CJK is deliberately never tokenized,
|
||||
// never spell-checked, never glossed; the prompts explain English in her
|
||||
// language; the vocabulary garden captures English words. All correct for
|
||||
// a Mandarin native practising English, and all backwards for an English
|
||||
// native practising Mandarin.
|
||||
//
|
||||
// A second pair code ('zh-learner') was the cheaper option and is the
|
||||
// wrong shape: it would make the two directions of one pair look like two
|
||||
// unrelated languages to every query, and it would have to be repeated for
|
||||
// fr, es and pt-PT before any of them could turn around. A column keeps
|
||||
// the two questions separate, which is what they are.
|
||||
//
|
||||
// 'learning_en' is the default and is what every existing row means — the
|
||||
// backfill is the DEFAULT itself, and it is right rather than merely
|
||||
// convenient: all three accounts today are Mandarin natives writing
|
||||
// English.
|
||||
name: "0016_user_direction",
|
||||
stmt: `
|
||||
ALTER TABLE users ADD COLUMN direction TEXT NOT NULL DEFAULT 'learning_en'
|
||||
CHECK(direction IN ('learning_en','learning_pair'));
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package db
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestOpenMigratesAndSeeds(t *testing.T) {
|
||||
@@ -83,3 +84,283 @@ func TestOpenMigratesAndSeeds(t *testing.T) {
|
||||
t.Errorf("expected exactly 1 local user after reopen, got %d", users)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResolvedAtBackfill runs migration 0012 against a database that predates
|
||||
// it, which is the only shape that matters: on the live box the suggestions
|
||||
// table is years of settled edits with no resolved_at to their name. Backfilling
|
||||
// to created_at is exactly the approximation the growth journal would otherwise
|
||||
// have had to make, and a pending row must stay NULL — nothing has been decided.
|
||||
func TestResolvedAtBackfill(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "old.db")
|
||||
d, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
|
||||
// Rewind to the state before 0012: drop the column and forget the migration.
|
||||
if _, err := d.Exec(`DROP INDEX idx_suggestions_resolved`); err != nil {
|
||||
t.Fatalf("rewind index: %v", err)
|
||||
}
|
||||
if _, err := d.Exec(`ALTER TABLE suggestions DROP COLUMN resolved_at`); err != nil {
|
||||
t.Fatalf("rewind schema: %v", err)
|
||||
}
|
||||
if _, err := d.Exec(`DELETE FROM schema_migrations WHERE name = '0012_suggestion_resolved_at'`); err != nil {
|
||||
t.Fatalf("rewind migration record: %v", err)
|
||||
}
|
||||
if _, err := d.Exec(`INSERT INTO documents (id, user_id) VALUES ('d1', ?)`, LocalUserID); err != nil {
|
||||
t.Fatalf("insert document: %v", err)
|
||||
}
|
||||
for _, s := range []struct{ id, status string }{
|
||||
{"s-old", "accepted"},
|
||||
{"s-open", "pending"},
|
||||
} {
|
||||
if _, err := d.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at)
|
||||
VALUES (?, 'd1', 0, 3, 'teh', 'the', 'x', 'grammar', ?, '2026-01-02 03:04:05')`,
|
||||
s.id, s.status,
|
||||
); err != nil {
|
||||
t.Fatalf("seed %s: %v", s.id, err)
|
||||
}
|
||||
}
|
||||
d.Close()
|
||||
|
||||
d2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen (migrate): %v", err)
|
||||
}
|
||||
defer d2.Close()
|
||||
|
||||
// Compared against created_at read back the same way: the driver renders a
|
||||
// DATETIME column itself, so the assertion is "the same instant", not a
|
||||
// particular text format.
|
||||
var settled, created *string
|
||||
if err := d2.QueryRow(
|
||||
`SELECT resolved_at, created_at FROM suggestions WHERE id = 's-old'`,
|
||||
).Scan(&settled, &created); err != nil {
|
||||
t.Fatalf("read settled row: %v", err)
|
||||
}
|
||||
if settled == nil || created == nil || *settled != *created {
|
||||
t.Errorf("resolved_at = %v, want it backfilled from created_at (%v)", settled, created)
|
||||
}
|
||||
|
||||
var pending *string
|
||||
if err := d2.QueryRow(`SELECT resolved_at FROM suggestions WHERE id = 's-open'`).Scan(&pending); err != nil {
|
||||
t.Fatalf("read pending row: %v", err)
|
||||
}
|
||||
if pending != nil {
|
||||
t.Errorf("pending row got resolved_at = %v, want NULL — nothing was decided", *pending)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSuggestionSourceBackfill runs migration 0013 against a database that
|
||||
// predates it — the shape the live box is actually in. `source` is the column
|
||||
// that lets the offline rule pack and the model share the collocation family
|
||||
// without deleting each other's rows, and it can only do that if the existing
|
||||
// rows are labelled correctly on the way in: everything the old deterministic
|
||||
// pass wrote is local, and everything else came from a model.
|
||||
func TestSuggestionSourceBackfill(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "old.db")
|
||||
d, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
|
||||
// Rewind to the state before 0013.
|
||||
if _, err := d.Exec(`ALTER TABLE suggestions DROP COLUMN source`); err != nil {
|
||||
t.Fatalf("rewind schema: %v", err)
|
||||
}
|
||||
if _, err := d.Exec(`DELETE FROM schema_migrations WHERE name = '0013_suggestion_source'`); err != nil {
|
||||
t.Fatalf("rewind migration record: %v", err)
|
||||
}
|
||||
if _, err := d.Exec(`INSERT INTO documents (id, user_id) VALUES ('d1', ?)`, LocalUserID); err != nil {
|
||||
t.Fatalf("insert document: %v", err)
|
||||
}
|
||||
for _, s := range []struct{ id, typ string }{
|
||||
{"s-mech", SuggestionTypeMechanics},
|
||||
{"s-gram", SuggestionTypeGrammar},
|
||||
{"s-coll", SuggestionTypeCollocation},
|
||||
} {
|
||||
if _, err := d.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES (?, 'd1', 0, 3, 'teh', 'the', 'x', ?)`,
|
||||
s.id, s.typ,
|
||||
); err != nil {
|
||||
t.Fatalf("seed %s: %v", s.id, err)
|
||||
}
|
||||
}
|
||||
d.Close()
|
||||
|
||||
d2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen (migrate): %v", err)
|
||||
}
|
||||
defer d2.Close()
|
||||
|
||||
// A pre-0013 collocation row can only have come from the coach — the offline
|
||||
// miscollocation list did not exist yet — so it must NOT be claimed as local.
|
||||
for id, want := range map[string]string{
|
||||
"s-mech": SuggestionSourceLocal,
|
||||
"s-gram": SuggestionSourceLLM,
|
||||
"s-coll": SuggestionSourceLLM,
|
||||
} {
|
||||
var got string
|
||||
if err := d2.QueryRow(`SELECT source FROM suggestions WHERE id = ?`, id).Scan(&got); err != nil {
|
||||
t.Fatalf("read %s: %v", id, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("%s: source = %q, want %q", id, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// And a row written after the migration defaults to the model, so a code path
|
||||
// that forgets to name a source can never silently claim to be offline.
|
||||
if _, err := d2.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES ('s-new', 'd1', 0, 3, 'teh', 'the', 'x', 'grammar')`,
|
||||
); err != nil {
|
||||
t.Fatalf("insert new row: %v", err)
|
||||
}
|
||||
var fresh string
|
||||
if err := d2.QueryRow(`SELECT source FROM suggestions WHERE id = 's-new'`).Scan(&fresh); err != nil {
|
||||
t.Fatalf("read new row: %v", err)
|
||||
}
|
||||
if fresh != SuggestionSourceLLM {
|
||||
t.Errorf("default source = %q, want %q", fresh, SuggestionSourceLLM)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTranslateTypeMigrationPreservesRows runs migration 0015 against a database
|
||||
// that predates it. Unlike the two backfills above, 0015 *rebuilds the table* —
|
||||
// SQLite can't ALTER a CHECK constraint — so it copies every row across by hand,
|
||||
// and a column left out of that copy list silently loses her data. Every test
|
||||
// elsewhere starts from a fresh database and would never notice; the live box has
|
||||
// years of rows in it.
|
||||
func TestTranslateTypeMigrationPreservesRows(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "old.db")
|
||||
d, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
|
||||
// Rewind to the pre-0015 table: the same shape, minus 'translate' in the CHECK.
|
||||
if _, err := d.Exec(`
|
||||
CREATE TABLE suggestions_old (
|
||||
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
||||
doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
||||
from_pos INTEGER NOT NULL,
|
||||
to_pos INTEGER NOT NULL,
|
||||
original TEXT NOT NULL,
|
||||
replacement TEXT NOT NULL,
|
||||
explanation TEXT NOT NULL,
|
||||
type TEXT NOT NULL CHECK(type IN ('grammar','phrasing','idiom','clarity','voice','collocation','mechanics')),
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','accepted','rejected')),
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
resolved_at DATETIME,
|
||||
source TEXT NOT NULL DEFAULT 'llm',
|
||||
chunk_hash TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
DROP TABLE suggestions;
|
||||
ALTER TABLE suggestions_old RENAME TO suggestions;
|
||||
CREATE INDEX idx_suggestions_doc_id ON suggestions(doc_id);
|
||||
CREATE INDEX idx_suggestions_resolved ON suggestions(status, resolved_at);
|
||||
DELETE FROM schema_migrations WHERE name = '0015_translate_suggestion_type';
|
||||
`); err != nil {
|
||||
t.Fatalf("rewind schema: %v", err)
|
||||
}
|
||||
|
||||
if _, err := d.Exec(`INSERT INTO documents (id, user_id) VALUES ('d1', ?)`, LocalUserID); err != nil {
|
||||
t.Fatalf("insert document: %v", err)
|
||||
}
|
||||
// One row with every column carrying a distinguishable value, so a dropped
|
||||
// column shows up as a changed value rather than as a passing test.
|
||||
if _, err := d.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash)
|
||||
VALUES ('s-1', 'd1', 7, 11, 'by foots', 'on foot', 'idiom advice she has read', 'idiom', 'accepted', '2026-01-02 03:04:05', '2026-01-02 03:05:00', 'local', 'abc123')`,
|
||||
); err != nil {
|
||||
t.Fatalf("seed row: %v", err)
|
||||
}
|
||||
d.Close()
|
||||
|
||||
d2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen (migrate): %v", err)
|
||||
}
|
||||
defer d2.Close()
|
||||
|
||||
var (
|
||||
docID, original, replacement, explanation string
|
||||
typ, status, source, chunkHash string
|
||||
from, to int
|
||||
// Scanned as instants, not strings: the driver renders a DATETIME column in
|
||||
// its own format, so the claim is "the same moment", not the same text.
|
||||
createdAt, resolvedAt time.Time
|
||||
)
|
||||
if err := d2.QueryRow(
|
||||
`SELECT doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at, source, chunk_hash
|
||||
FROM suggestions WHERE id = 's-1'`,
|
||||
).Scan(&docID, &from, &to, &original, &replacement, &explanation,
|
||||
&typ, &status, &createdAt, &resolvedAt, &source, &chunkHash); err != nil {
|
||||
t.Fatalf("read migrated row: %v", err)
|
||||
}
|
||||
for _, c := range []struct{ name, got, want string }{
|
||||
{"doc_id", docID, "d1"},
|
||||
{"original", original, "by foots"},
|
||||
{"replacement", replacement, "on foot"},
|
||||
{"explanation", explanation, "idiom advice she has read"},
|
||||
{"type", typ, SuggestionTypeIdiom},
|
||||
{"status", status, SuggestionStatusAccepted},
|
||||
{"source", source, SuggestionSourceLocal},
|
||||
{"chunk_hash", chunkHash, "abc123"},
|
||||
} {
|
||||
if c.got != c.want {
|
||||
t.Errorf("%s = %q, want %q", c.name, c.got, c.want)
|
||||
}
|
||||
}
|
||||
if from != 7 || to != 11 {
|
||||
t.Errorf("offsets = (%d, %d), want (7, 11)", from, to)
|
||||
}
|
||||
// created_at and resolved_at must survive: the rail's arrival chime keys on
|
||||
// created_at, and the growth journal counts by resolved_at. A rebuild that
|
||||
// reset either would re-chime her whole document and rewrite her history.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
got time.Time
|
||||
want string
|
||||
}{
|
||||
{"created_at", createdAt, "2026-01-02 03:04:05"},
|
||||
{"resolved_at", resolvedAt, "2026-01-02 03:05:00"},
|
||||
} {
|
||||
want, err := time.Parse("2006-01-02 15:04:05", c.want)
|
||||
if err != nil {
|
||||
t.Fatalf("parse want: %v", err)
|
||||
}
|
||||
if !c.got.Equal(want) {
|
||||
t.Errorf("%s = %v, want the original instant %v", c.name, c.got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The point of the rebuild: the new type is now insertable, and a bogus one
|
||||
// still isn't.
|
||||
if _, err := d2.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES ('s-2', 'd1', 0, 3, '苹果', 'apple', 'x', ?)`, SuggestionTypeTranslate,
|
||||
); err != nil {
|
||||
t.Fatalf("insert translate row: %v", err)
|
||||
}
|
||||
if _, err := d2.Exec(
|
||||
`INSERT INTO suggestions (id, doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES ('s-3', 'd1', 0, 3, 'x', 'y', 'x', 'nonsense')`,
|
||||
); err == nil {
|
||||
t.Error("CHECK constraint accepted an unknown type after the rebuild")
|
||||
}
|
||||
|
||||
// Both indexes must come back, or every document load starts table-scanning.
|
||||
for _, idx := range []string{"idx_suggestions_doc_id", "idx_suggestions_resolved"} {
|
||||
var name string
|
||||
if err := d2.QueryRow(
|
||||
`SELECT name FROM sqlite_master WHERE type = 'index' AND name = ?`, idx,
|
||||
).Scan(&name); err != nil {
|
||||
t.Errorf("index %s missing after rebuild: %v", idx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+50
-4
@@ -2,14 +2,32 @@ package db
|
||||
|
||||
import "time"
|
||||
|
||||
// User is an account. With auth deferred, the app runs as a single hardcoded
|
||||
// `local` user (see LocalUserID); the user_id columns and this type exist so
|
||||
// real auth can drop in later without a schema migration.
|
||||
// User is an account. Its ID is the OIDC subject for anyone who signed in, or
|
||||
// LocalUserID for the pre-auth single user (and for local development, where
|
||||
// StaticResolver still hands out that id).
|
||||
type User struct {
|
||||
ID string `json:"id"`
|
||||
Email string `json:"email"`
|
||||
DisplayName string `json:"display_name"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
|
||||
// PairLang is the X in this writer's (English + X) language pair — "zh"
|
||||
// today, "pt-PT"/"fr"/"es" once the langpacks land. It selects the UI copy
|
||||
// and dictionary set, not the language they may type in.
|
||||
PairLang string `json:"pair_lang"`
|
||||
|
||||
// Direction says which half of the pair is being *learned*. Every pair until
|
||||
// now assumed one answer: the writer is native in X and practising English,
|
||||
// so hanzi is never tokenized and English is what gets underlined. Turn it
|
||||
// around — a native English speaker learning Chinese — and the same pair
|
||||
// wants the opposite of nearly every default.
|
||||
//
|
||||
// It is a separate column from PairLang rather than a second pair code
|
||||
// ("zh-learner") because it is a genuinely separate question: the pair says
|
||||
// *which two languages*, this says *which way round*. Keeping them apart is
|
||||
// what lets fr, es and pt-PT inherit the learner direction later without a
|
||||
// second langpack each.
|
||||
Direction string `json:"direction"`
|
||||
}
|
||||
|
||||
// Document is a single piece of writing. `Content` is the Tiptap JSON document
|
||||
@@ -25,6 +43,10 @@ type Document struct {
|
||||
WordCount int `json:"word_count"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
|
||||
// PreserveHistory opts this document out of auto-snapshot pruning so its
|
||||
// full writing trail survives as authorship evidence (see the passport).
|
||||
PreserveHistory bool `json:"preserve_history"`
|
||||
}
|
||||
|
||||
// DocumentVersion is a point-in-time snapshot of a document's body, captured so
|
||||
@@ -42,6 +64,13 @@ type DocumentVersion struct {
|
||||
WordCount int `json:"word_count"`
|
||||
Kind string `json:"kind"` // auto | manual | pre_restore
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
|
||||
// ContentHash chains this snapshot to the previous one (PrevHash), so a
|
||||
// history that was edited or thinned after the fact fails verification.
|
||||
// Both are empty for snapshots taken before the chain existed. Omitted from
|
||||
// list responses; the passport loads them explicitly.
|
||||
ContentHash string `json:"content_hash,omitempty"`
|
||||
PrevHash string `json:"prev_hash,omitempty"`
|
||||
}
|
||||
|
||||
// Document version kinds, mirrored from the schema CHECK constraint.
|
||||
@@ -88,8 +117,12 @@ type Suggestion struct {
|
||||
Original string `json:"original"`
|
||||
Replacement string `json:"replacement"`
|
||||
Explanation string `json:"explanation"`
|
||||
Type string `json:"type"` // grammar | phrasing | idiom | clarity | voice | collocation
|
||||
Type string `json:"type"` // grammar | phrasing | idiom | clarity | translate | voice | collocation
|
||||
Status string `json:"status"` // pending | accepted | rejected
|
||||
// Source names the engine that proposed the edit, not its family: an offline
|
||||
// rule and the model can both propose a collocation, and the writer is never
|
||||
// told which one spoke. It exists so each pass can replace its own rows.
|
||||
Source string `json:"source"` // llm | local
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
@@ -99,10 +132,23 @@ const (
|
||||
SuggestionTypePhrasing = "phrasing"
|
||||
SuggestionTypeIdiom = "idiom"
|
||||
SuggestionTypeClarity = "clarity"
|
||||
// A span she wrote in her own language, rendered into English. Not a
|
||||
// correction — nothing was wrong with it — which is why it is its own type
|
||||
// rather than a clarity fix: the card is the pair model's flagship moment
|
||||
// (SUGGESTIONS §1), and labelling it "Clarity" reads as a tidy-up of her
|
||||
// first language. The model isn't asked for this label; it is derived from the
|
||||
// span itself (see suggestions/language.go), so it can't drift.
|
||||
SuggestionTypeTranslate = "translate"
|
||||
SuggestionTypeVoice = "voice"
|
||||
SuggestionTypeCollocation = "collocation"
|
||||
SuggestionTypeMechanics = "mechanics" // deterministic rule-based pass (no LLM)
|
||||
|
||||
// Who proposed it. The offline rule pack ('local') runs on every edit inside
|
||||
// the browser and survives a VPN-down box; the model ('llm') adds the long
|
||||
// tail when it is reachable.
|
||||
SuggestionSourceLLM = "llm"
|
||||
SuggestionSourceLocal = "local"
|
||||
|
||||
SuggestionStatusPending = "pending"
|
||||
SuggestionStatusAccepted = "accepted"
|
||||
SuggestionStatusRejected = "rejected"
|
||||
|
||||
+74
-6
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
)
|
||||
@@ -46,7 +47,7 @@ func (h *Handler) exportAll(w http.ResponseWriter, r *http.Request) {
|
||||
FROM documents
|
||||
WHERE user_id = ?
|
||||
ORDER BY updated_at DESC`,
|
||||
db.LocalUserID,
|
||||
auth.UserID(r.Context()),
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -143,7 +144,7 @@ func (h *Handler) export(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := h.fetch(chi.URLParam(r, "id"))
|
||||
doc, err := h.fetch(auth.UserID(r.Context()), chi.URLParam(r, "id"))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
notFound(w)
|
||||
return
|
||||
@@ -286,7 +287,14 @@ func mdBlock(n pmNode, depth int) string {
|
||||
return "---"
|
||||
case "image":
|
||||
alt := n.attrStr("alt")
|
||||
return fmt.Sprintf("", alt, n.attrStr("src"))
|
||||
src := safeURL(n.attrStr("src"))
|
||||
if src == "" {
|
||||
// Nowhere safe to point. Keep the alt text as plain prose — it is
|
||||
// the part that carries meaning — rather than emitting an image
|
||||
// whose destination was rejected. See safeURL.
|
||||
return alt
|
||||
}
|
||||
return fmt.Sprintf("", alt, src)
|
||||
case "table":
|
||||
return mdTable(n)
|
||||
case "bulletList", "orderedList":
|
||||
@@ -395,7 +403,9 @@ func applyMdMarks(n pmNode) string {
|
||||
if n.hasMark("underline") {
|
||||
t = "<u>" + t + "</u>"
|
||||
}
|
||||
if href := n.markAttr("link", "href"); href != "" {
|
||||
// Same rule as the HTML export: plenty of Markdown renderers pass a
|
||||
// `javascript:` destination straight through into an <a href>. See safeURL.
|
||||
if href := safeURL(n.markAttr("link", "href")); href != "" {
|
||||
t = "[" + t + "](" + href + ")"
|
||||
}
|
||||
return t
|
||||
@@ -517,7 +527,16 @@ func htmlBlock(n pmNode) string {
|
||||
return "<hr>\n"
|
||||
case "image":
|
||||
alt := htmlEscape(n.attrStr("alt"))
|
||||
return fmt.Sprintf("<p><img src=\"%s\" alt=\"%s\"></p>\n", htmlEscape(n.attrStr("src")), alt)
|
||||
src := safeURL(n.attrStr("src"))
|
||||
if src == "" {
|
||||
// Nowhere safe to point: keep the alt text, which is the part that
|
||||
// carries meaning, rather than emitting a broken image.
|
||||
if alt == "" {
|
||||
return ""
|
||||
}
|
||||
return "<p>" + alt + "</p>\n"
|
||||
}
|
||||
return fmt.Sprintf("<p><img src=\"%s\" alt=\"%s\"></p>\n", htmlEscape(src), alt)
|
||||
case "table":
|
||||
return htmlTable(n)
|
||||
case "bulletList", "orderedList":
|
||||
@@ -616,7 +635,9 @@ func applyHTMLMarks(n pmNode) string {
|
||||
if n.hasMark("highlight") {
|
||||
t = "<mark>" + t + "</mark>"
|
||||
}
|
||||
if href := n.markAttr("link", "href"); href != "" {
|
||||
// An unsafe href is dropped, not the link: the words stay, they just stop
|
||||
// being clickable. See safeURL.
|
||||
if href := safeURL(n.markAttr("link", "href")); href != "" {
|
||||
t = fmt.Sprintf("<a href=\"%s\">%s</a>", htmlEscape(href), t)
|
||||
}
|
||||
return t
|
||||
@@ -857,6 +878,53 @@ func htmlEscape(s string) string {
|
||||
return r.Replace(s)
|
||||
}
|
||||
|
||||
// safeURLSchemes are the schemes an exported document may point at. Escaping
|
||||
// makes a URL safe to sit inside an attribute; it says nothing about what
|
||||
// happens when the attribute is followed, and `javascript:` survives it
|
||||
// untouched.
|
||||
//
|
||||
// The toolbar can't produce one — it prefixes anything it doesn't recognise
|
||||
// with https:// — but the toolbar is not the only way in: PUT /api/docs/{id}
|
||||
// stores whatever Tiptap JSON it is given. And an export is the one artifact
|
||||
// here that is *meant* to leave: the passport and the .html backup are files a
|
||||
// writer hands to a teacher or an editor, opened on a machine that has no
|
||||
// reason to trust them. A link that runs code when clicked is not something to
|
||||
// ship inside one.
|
||||
//
|
||||
// Relative and fragment links pass through: they're how a document refers to
|
||||
// its own headings, and they can't reach anything.
|
||||
var safeURLSchemes = map[string]bool{
|
||||
"http": true, "https": true, "mailto": true, "tel": true, "ftp": true,
|
||||
}
|
||||
|
||||
// safeURL returns u if it is safe to follow from an exported file, and "" if it
|
||||
// isn't. A dropped href leaves the link text in place — the reader loses a
|
||||
// destination, never the writing.
|
||||
func safeURL(u string) string {
|
||||
trimmed := strings.TrimSpace(u)
|
||||
if trimmed == "" {
|
||||
return ""
|
||||
}
|
||||
// A scheme is everything before the first ':', but only when no '/', '?' or
|
||||
// '#' comes first — otherwise "notes/a:b" would read as the "notes/a" scheme.
|
||||
// Nothing before a colon means a relative or fragment link, which is fine.
|
||||
if i := strings.IndexAny(trimmed, ":/?#"); i >= 0 && trimmed[i] == ':' {
|
||||
// Control characters and whitespace are stripped by browsers *before*
|
||||
// the scheme is read, so "java\nscript:" is javascript:. Fold them out
|
||||
// before deciding rather than after.
|
||||
scheme := strings.Map(func(r rune) rune {
|
||||
if r <= ' ' || r == 0x7f {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, trimmed[:i])
|
||||
if !safeURLSchemes[strings.ToLower(scheme)] {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
func xmlEscape(s string) string {
|
||||
r := strings.NewReplacer("&", "&", "<", "<", ">", ">", `"`, """)
|
||||
return r.Replace(s)
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// richDocJSON is a Tiptap document exercising headings, marks, and a list —
|
||||
@@ -243,3 +245,73 @@ func TestExportUnsupportedFormat(t *testing.T) {
|
||||
t.Fatalf("expected 400 for unsupported format, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Escaping makes a URL safe to sit inside an attribute; it says nothing about
|
||||
// what happens when the attribute is followed. An export is the one artifact
|
||||
// here meant to leave — the file handed to a teacher, opened on a machine with
|
||||
// no reason to trust it — so a destination that runs code is dropped.
|
||||
func TestExportDropsUnsafeLinkSchemes(t *testing.T) {
|
||||
unsafe := []string{
|
||||
"javascript:alert(1)",
|
||||
"JaVaScRiPt:alert(1)",
|
||||
"java\nscript:alert(1)", // browsers strip control characters first
|
||||
" javascript:alert(1)",
|
||||
"data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==",
|
||||
"vbscript:msgbox(1)",
|
||||
}
|
||||
for _, href := range unsafe {
|
||||
if got := safeURL(href); got != "" {
|
||||
t.Errorf("safeURL(%q) = %q, want it dropped", href, got)
|
||||
}
|
||||
}
|
||||
|
||||
safe := []string{
|
||||
"https://example.com/a?b=1#c",
|
||||
"http://example.com",
|
||||
"mailto:her@example.com",
|
||||
"/api/images/abc.png",
|
||||
"#a-heading",
|
||||
"notes/chapter:one.md", // a colon that isn't a scheme
|
||||
}
|
||||
for _, href := range safe {
|
||||
if got := safeURL(href); got != href {
|
||||
t.Errorf("safeURL(%q) = %q, want it kept", href, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// End to end through the renderers: an unsafe href loses its destination, never
|
||||
// its words.
|
||||
func TestRenderedExportsCarryNoScriptURLs(t *testing.T) {
|
||||
doc := db.Document{
|
||||
Title: "Notes",
|
||||
Content: `{"type":"doc","content":[{"type":"paragraph","content":[
|
||||
{"type":"text","text":"click me","marks":[{"type":"link","attrs":{"href":"javascript:alert(1)"}}]}]},
|
||||
{"type":"image","attrs":{"src":"javascript:alert(2)","alt":"a drawing"}}]}`,
|
||||
}
|
||||
|
||||
html, err := renderHTMLFile(doc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(string(html)), "javascript:") {
|
||||
t.Fatalf("html export carried a javascript: URL:\n%s", html)
|
||||
}
|
||||
if !strings.Contains(string(html), "click me") {
|
||||
t.Fatal("html export dropped the link text along with the href")
|
||||
}
|
||||
if !strings.Contains(string(html), "a drawing") {
|
||||
t.Fatal("html export dropped the alt text of the rejected image")
|
||||
}
|
||||
|
||||
md, err := renderMarkdown(doc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(string(md)), "javascript:") {
|
||||
t.Fatalf("markdown export carried a javascript: URL:\n%s", md)
|
||||
}
|
||||
if !strings.Contains(string(md), "click me") {
|
||||
t.Fatal("markdown export dropped the link text along with the href")
|
||||
}
|
||||
}
|
||||
|
||||
+28
-14
@@ -1,6 +1,7 @@
|
||||
// Package docs implements the document CRUD HTTP handlers — the create / list /
|
||||
// read / update / delete surface that backs the editor and its 1.5s auto-save.
|
||||
// All access is scoped to the single hardcoded local user while auth is deferred.
|
||||
// Every query is scoped to the caller resolved by the auth middleware, so a
|
||||
// document is only ever reachable by the user who owns it.
|
||||
package docs
|
||||
|
||||
import (
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
)
|
||||
@@ -52,15 +54,16 @@ type docSummary struct {
|
||||
Tags []db.Tag `json:"tags"`
|
||||
}
|
||||
|
||||
// list returns the local user's documents, most-recently-updated first, each
|
||||
// list returns the caller's documents, most-recently-updated first, each
|
||||
// decorated with its tags.
|
||||
func (h *Handler) list(w http.ResponseWriter, r *http.Request) {
|
||||
userID := auth.UserID(r.Context())
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT id, title, word_count, updated_at
|
||||
FROM documents
|
||||
WHERE user_id = ?
|
||||
ORDER BY updated_at DESC`,
|
||||
db.LocalUserID,
|
||||
userID,
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -84,7 +87,7 @@ func (h *Handler) list(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
byDoc, err := h.tagsByDoc(ids)
|
||||
byDoc, err := h.tagsByDoc(userID, ids)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
@@ -104,7 +107,7 @@ func (h *Handler) create(w http.ResponseWriter, r *http.Request) {
|
||||
err := h.DB.QueryRow(
|
||||
`INSERT INTO documents (user_id) VALUES (?)
|
||||
RETURNING id, user_id, title, content, content_text, tone, word_count, created_at, updated_at`,
|
||||
db.LocalUserID,
|
||||
auth.UserID(r.Context()),
|
||||
).Scan(
|
||||
&doc.ID, &doc.UserID, &doc.Title, &doc.Content, &doc.ContentText,
|
||||
&doc.Tone, &doc.WordCount, &doc.CreatedAt, &doc.UpdatedAt,
|
||||
@@ -118,7 +121,7 @@ func (h *Handler) create(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// get returns a single full document by id.
|
||||
func (h *Handler) get(w http.ResponseWriter, r *http.Request) {
|
||||
doc, err := h.fetch(chi.URLParam(r, "id"))
|
||||
doc, err := h.fetch(auth.UserID(r.Context()), chi.URLParam(r, "id"))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
notFound(w)
|
||||
return
|
||||
@@ -139,12 +142,17 @@ type updateRequest struct {
|
||||
ContentText *string `json:"content_text"`
|
||||
Tone *string `json:"tone"`
|
||||
WordCount *int `json:"word_count"`
|
||||
|
||||
// PreserveHistory toggles the passport's keep-everything mode. Sent alone
|
||||
// by the History panel's toggle, never by the auto-save path.
|
||||
PreserveHistory *bool `json:"preserve_history"`
|
||||
}
|
||||
|
||||
// update applies the provided fields to a document and returns the saved row.
|
||||
// content and content_text are kept in sync by the client and written together.
|
||||
func (h *Handler) update(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
userID := auth.UserID(r.Context())
|
||||
|
||||
var req updateRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
@@ -159,9 +167,11 @@ func (h *Handler) update(w http.ResponseWriter, r *http.Request) {
|
||||
content_text = COALESCE(?, content_text),
|
||||
tone = COALESCE(?, tone),
|
||||
word_count = COALESCE(?, word_count),
|
||||
preserve_history = COALESCE(?, preserve_history),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ? AND user_id = ?`,
|
||||
req.Title, req.Content, req.ContentText, req.Tone, req.WordCount, id, db.LocalUserID,
|
||||
req.Title, req.Content, req.ContentText, req.Tone, req.WordCount,
|
||||
req.PreserveHistory, id, userID,
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -172,7 +182,7 @@ func (h *Handler) update(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := h.fetch(id)
|
||||
doc, err := h.fetch(userID, id)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
@@ -194,7 +204,7 @@ func (h *Handler) update(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *Handler) delete(w http.ResponseWriter, r *http.Request) {
|
||||
res, err := h.DB.Exec(
|
||||
`DELETE FROM documents WHERE id = ? AND user_id = ?`,
|
||||
chi.URLParam(r, "id"), db.LocalUserID,
|
||||
chi.URLParam(r, "id"), auth.UserID(r.Context()),
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -207,17 +217,21 @@ func (h *Handler) delete(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// fetch loads one full document scoped to the local user.
|
||||
func (h *Handler) fetch(id string) (db.Document, error) {
|
||||
// fetch loads one full document, scoped to its owner. Callers pass the id from
|
||||
// [auth.UserID]; a document belonging to anyone else comes back as
|
||||
// sql.ErrNoRows, which handlers surface as a 404 rather than a 403 (a stranger's
|
||||
// document should be indistinguishable from one that doesn't exist).
|
||||
func (h *Handler) fetch(userID, id string) (db.Document, error) {
|
||||
var doc db.Document
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT id, user_id, title, content, content_text, tone, word_count, created_at, updated_at
|
||||
`SELECT id, user_id, title, content, content_text, tone, word_count,
|
||||
created_at, updated_at, preserve_history
|
||||
FROM documents
|
||||
WHERE id = ? AND user_id = ?`,
|
||||
id, db.LocalUserID,
|
||||
id, userID,
|
||||
).Scan(
|
||||
&doc.ID, &doc.UserID, &doc.Title, &doc.Content, &doc.ContentText,
|
||||
&doc.Tone, &doc.WordCount, &doc.CreatedAt, &doc.UpdatedAt,
|
||||
&doc.Tone, &doc.WordCount, &doc.CreatedAt, &doc.UpdatedAt, &doc.PreserveHistory,
|
||||
)
|
||||
return doc, err
|
||||
}
|
||||
|
||||
@@ -8,10 +8,14 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// newTestServer spins up an isolated on-disk database and the docs router.
|
||||
// newTestServer spins up an isolated on-disk database and the docs router,
|
||||
// behind the same auth middleware main.go installs. Tests must go through it:
|
||||
// handlers read the caller from the request context, so a router mounted bare
|
||||
// would see an empty user id and match no rows.
|
||||
func newTestServer(t *testing.T) http.Handler {
|
||||
t.Helper()
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
@@ -19,7 +23,13 @@ func newTestServer(t *testing.T) http.Handler {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
return New(database).Routes()
|
||||
return withAuth(New(database).Routes())
|
||||
}
|
||||
|
||||
// withAuth wraps a router so every test request arrives authenticated as the
|
||||
// seeded local user — the stand-in for a real session until Authentik lands.
|
||||
func withAuth(h http.Handler) http.Handler {
|
||||
return auth.Middleware(auth.StaticResolver(db.LocalUserID))(h)
|
||||
}
|
||||
|
||||
func do(t *testing.T, srv http.Handler, method, path, body string) *httptest.ResponseRecorder {
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
package docs
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// This file is the point of the auth plumbing: it proves that swapping the
|
||||
// hardcoded user for a request-scoped one actually isolates accounts. Every
|
||||
// handler resolves its user from the request, so mounting the same routers twice
|
||||
// behind two different resolvers gives us two "logged-in" users over one
|
||||
// database — which is exactly the situation a real login will create.
|
||||
|
||||
// newTwoUserServer opens one database holding two users and returns a router for
|
||||
// each, identical but for who the auth middleware says is calling.
|
||||
func newTwoUserServer(t *testing.T) (alice, bob http.Handler) {
|
||||
t.Helper()
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
|
||||
// db.Open seeds the local user; add a second so both sides have a valid FK.
|
||||
if _, err := database.Exec(
|
||||
`INSERT INTO users (id, email, display_name) VALUES (?, ?, ?)`,
|
||||
"bob", "bob@petal.local", "Bob",
|
||||
); err != nil {
|
||||
t.Fatalf("seed second user: %v", err)
|
||||
}
|
||||
|
||||
mount := func(userID string) http.Handler {
|
||||
h := New(database)
|
||||
r := chi.NewRouter()
|
||||
r.Mount("/docs", h.Routes())
|
||||
r.Mount("/tags", h.TagRoutes())
|
||||
r.Mount("/search", h.SearchRoutes())
|
||||
return auth.Middleware(auth.StaticResolver(userID))(r)
|
||||
}
|
||||
return mount(db.LocalUserID), mount("bob")
|
||||
}
|
||||
|
||||
// TestDocumentIsolation walks every read and write path that takes a document id
|
||||
// and asserts Bob cannot reach Alice's document through any of them. A stranger's
|
||||
// document must be indistinguishable from a nonexistent one — 404, never 403.
|
||||
func TestDocumentIsolation(t *testing.T) {
|
||||
alice, bob := newTwoUserServer(t)
|
||||
|
||||
docID := createDoc(t, alice, "Alice's diary", "a private sentence about my day")
|
||||
|
||||
t.Run("not in list", func(t *testing.T) {
|
||||
rec := do(t, bob, http.MethodGet, "/docs", "")
|
||||
var out []docSummary
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode list: %v", err)
|
||||
}
|
||||
if len(out) != 0 {
|
||||
t.Fatalf("bob sees %d of alice's documents, want 0", len(out))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("not in search", func(t *testing.T) {
|
||||
rec := do(t, bob, http.MethodGet, "/search?q=private", "")
|
||||
var out []searchResult
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode search: %v", err)
|
||||
}
|
||||
if len(out) != 0 {
|
||||
t.Fatalf("search leaked %d of alice's documents", len(out))
|
||||
}
|
||||
})
|
||||
|
||||
// The FTS index is a separate table joined back to documents; a missing
|
||||
// user_id filter there would leak content even though the list query is
|
||||
// scoped, so assert the owner still finds her own document.
|
||||
t.Run("owner still finds it", func(t *testing.T) {
|
||||
rec := do(t, alice, http.MethodGet, "/search?q=private", "")
|
||||
var out []searchResult
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode search: %v", err)
|
||||
}
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("alice found %d results for her own document, want 1", len(out))
|
||||
}
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
name, method, path, body string
|
||||
}{
|
||||
{"get", http.MethodGet, "/docs/" + docID, ""},
|
||||
{"update", http.MethodPut, "/docs/" + docID, `{"title":"defaced"}`},
|
||||
{"delete", http.MethodDelete, "/docs/" + docID, ""},
|
||||
{"export", http.MethodGet, "/docs/" + docID + "/export?format=md", ""},
|
||||
{"passport", http.MethodGet, "/docs/" + docID + "/passport", ""},
|
||||
{"snapshot", http.MethodPost, "/docs/" + docID + "/versions", ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rec := do(t, bob, tc.method, tc.path, tc.body)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("%s %s as bob = %d, want 404 (body: %s)",
|
||||
tc.method, tc.path, rec.Code, rec.Body)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The document must have survived every attempt above unchanged.
|
||||
rec := do(t, alice, http.MethodGet, "/docs/"+docID, "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("alice lost access to her own document: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
var doc db.Document
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &doc); err != nil {
|
||||
t.Fatalf("decode doc: %v", err)
|
||||
}
|
||||
if doc.Title != "Alice's diary" {
|
||||
t.Fatalf("title = %q, want %q — bob's update went through", doc.Title, "Alice's diary")
|
||||
}
|
||||
}
|
||||
|
||||
// TestVersionIsolation covers the history endpoints, which scope through a join
|
||||
// to documents rather than a direct user_id column — an easy place to forget the
|
||||
// filter, and one where the leak would be the full text of every draft.
|
||||
func TestVersionIsolation(t *testing.T) {
|
||||
alice, bob := newTwoUserServer(t)
|
||||
|
||||
docID := createDoc(t, alice, "Draft", "the first version of my essay")
|
||||
rec := do(t, alice, http.MethodPost, "/docs/"+docID+"/versions", "")
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("snapshot: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
var v db.DocumentVersion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &v); err != nil {
|
||||
t.Fatalf("decode version: %v", err)
|
||||
}
|
||||
|
||||
t.Run("list is empty for stranger", func(t *testing.T) {
|
||||
rec := do(t, bob, http.MethodGet, "/docs/"+docID+"/versions", "")
|
||||
var out []db.DocumentVersion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(out) != 0 {
|
||||
t.Fatalf("bob sees %d of alice's snapshots, want 0", len(out))
|
||||
}
|
||||
})
|
||||
|
||||
for _, tc := range []struct{ name, method, path string }{
|
||||
{"preview", http.MethodGet, "/docs/" + docID + "/versions/" + v.ID},
|
||||
{"restore", http.MethodPost, "/docs/" + docID + "/versions/" + v.ID + "/restore"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rec := do(t, bob, tc.method, tc.path, "")
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("%s as bob = %d, want 404 (body: %s)", tc.name, rec.Code, rec.Body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestTagIsolation checks the tag roster and, more importantly, that a document
|
||||
// and a tag can't be cross-linked across accounts — the assignment endpoint takes
|
||||
// two ids from different tables and must own-check both.
|
||||
func TestTagIsolation(t *testing.T) {
|
||||
alice, bob := newTwoUserServer(t)
|
||||
|
||||
docID := createDoc(t, alice, "Essay", "some words")
|
||||
|
||||
rec := do(t, alice, http.MethodPost, "/tags", `{"name":"school","color":"mint"}`)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("create tag: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
var aliceTag db.Tag
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &aliceTag); err != nil {
|
||||
t.Fatalf("decode tag: %v", err)
|
||||
}
|
||||
|
||||
rec = do(t, bob, http.MethodPost, "/tags", `{"name":"bobs","color":"sky"}`)
|
||||
var bobTag db.Tag
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &bobTag); err != nil {
|
||||
t.Fatalf("decode bob tag: %v", err)
|
||||
}
|
||||
|
||||
t.Run("roster is per user", func(t *testing.T) {
|
||||
rec := do(t, bob, http.MethodGet, "/tags", "")
|
||||
var out []db.Tag
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(out) != 1 || out[0].Name != "bobs" {
|
||||
t.Fatalf("bob's roster = %+v, want just his own tag", out)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cannot tag a stranger's document", func(t *testing.T) {
|
||||
body, _ := json.Marshal(map[string]string{"tag_id": bobTag.ID})
|
||||
rec := do(t, bob, http.MethodPost, "/docs/"+docID+"/tags", string(body))
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("bob tagging alice's doc = %d, want 404", rec.Code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cannot rename a stranger's tag", func(t *testing.T) {
|
||||
rec := do(t, bob, http.MethodPatch, "/tags/"+aliceTag.ID, `{"name":"stolen"}`)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("bob renaming alice's tag = %d, want 404", rec.Code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cannot delete a stranger's tag", func(t *testing.T) {
|
||||
rec := do(t, bob, http.MethodDelete, "/tags/"+aliceTag.ID, "")
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("bob deleting alice's tag = %d, want 404", rec.Code)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
package docs
|
||||
|
||||
// Writing passport: a standalone, printable report showing *how* a document was
|
||||
// written — when each snapshot landed, how the word count grew, how the work
|
||||
// broke into sessions. It exists because automated "AI detector" verdicts are
|
||||
// unreliable and skew against non-native English writers, so the useful thing to
|
||||
// hand someone who doubts your authorship is not a score but a record.
|
||||
//
|
||||
// The report is deliberately modest about what it proves (see passportLimits):
|
||||
// it evidences a plausible writing process, it does not certify one.
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
)
|
||||
|
||||
// Passport tuning.
|
||||
const (
|
||||
// sessionGap is the idle time that separates one writing session from the
|
||||
// next. Auto-snapshots fire at most every 3 minutes while typing, so any
|
||||
// gap far above that means the writer stepped away. 45 minutes keeps a
|
||||
// coffee break inside one session but splits morning from evening work.
|
||||
sessionGap = 45 * time.Minute
|
||||
|
||||
// jumpNoteThreshold is the share of the final word count a single
|
||||
// snapshot-to-snapshot increase must exceed before the report calls it out.
|
||||
// A large jump is the first thing a skeptical reader will ask about, so the
|
||||
// report raises it rather than leaving it to be discovered.
|
||||
jumpNoteThreshold = 0.25
|
||||
)
|
||||
|
||||
// chainHash links a snapshot to its predecessor. Covering prev_hash makes each
|
||||
// hash depend on the entire history before it, so altering any earlier snapshot
|
||||
// invalidates every later one; covering created_at means a row cannot be
|
||||
// silently backdated.
|
||||
//
|
||||
// This detects tampering with the local database. It is not third-party
|
||||
// attestation — someone with the database and this function could regenerate a
|
||||
// consistent chain from scratch.
|
||||
func chainHash(prevHash, docID string, createdAt time.Time, wordCount int, text string) string {
|
||||
h := sha256.New()
|
||||
fmt.Fprintf(h, "%s\x00%s\x00%d\x00%d\x00%s",
|
||||
prevHash, docID, createdAt.UTC().UnixNano(), wordCount, text)
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
// --- report model -----------------------------------------------------------
|
||||
|
||||
// passportSession is one continuous stretch of work — snapshots with no
|
||||
// sessionGap-sized pause between them.
|
||||
type passportSession struct {
|
||||
Start, End time.Time
|
||||
Snapshots int
|
||||
WordsAdded int // net change across the session; negative when trimming
|
||||
}
|
||||
|
||||
// Duration is the observed length of the session: first snapshot to last. A
|
||||
// single-snapshot session reports zero, which is why total active time is
|
||||
// described as a lower bound.
|
||||
func (s passportSession) Duration() time.Duration { return s.End.Sub(s.Start) }
|
||||
|
||||
// chain verification outcomes, in the order the report prefers to report them.
|
||||
const (
|
||||
chainVerified = "verified" // every hash recomputes and every link holds
|
||||
chainGaps = "gaps" // hashes valid, links broken — consistent with pruning
|
||||
chainPartial = "partial" // some snapshots predate the hash chain
|
||||
chainUnverifiable = "unverifiable" // no snapshot carries a hash
|
||||
chainBroken = "broken" // a hash does not match its own contents
|
||||
)
|
||||
|
||||
// passportData is everything the template renders.
|
||||
type passportData struct {
|
||||
Doc db.Document
|
||||
Versions []db.DocumentVersion // ascending by time
|
||||
|
||||
Sessions []passportSession
|
||||
FirstAt time.Time
|
||||
LastAt time.Time
|
||||
Span time.Duration // wall-clock first snapshot → last
|
||||
ActiveTime time.Duration // summed session durations; a lower bound
|
||||
|
||||
LargestJump int // biggest single snapshot-to-snapshot word increase
|
||||
LargestJumpAt time.Time
|
||||
LargestJumpIdx int // index into Versions, so the chart can mark it
|
||||
NoteJump bool // jump is large enough to be worth pre-empting
|
||||
|
||||
ChainStatus string
|
||||
UnhashedCount int
|
||||
GeneratedAt time.Time
|
||||
}
|
||||
|
||||
// buildPassport derives the report from a document and its snapshots, which must
|
||||
// be ordered oldest-first. It assumes nothing about snapshot spacing.
|
||||
func buildPassport(doc db.Document, versions []db.DocumentVersion) passportData {
|
||||
d := passportData{
|
||||
Doc: doc,
|
||||
Versions: versions,
|
||||
GeneratedAt: time.Now(),
|
||||
}
|
||||
if len(versions) == 0 {
|
||||
d.ChainStatus = chainUnverifiable
|
||||
return d
|
||||
}
|
||||
|
||||
d.FirstAt = versions[0].CreatedAt
|
||||
d.LastAt = versions[len(versions)-1].CreatedAt
|
||||
d.Span = d.LastAt.Sub(d.FirstAt)
|
||||
|
||||
cur := passportSession{Start: versions[0].CreatedAt, End: versions[0].CreatedAt, Snapshots: 1}
|
||||
|
||||
// Baseline for the running session's net-words figure. Later sessions
|
||||
// measure from the *previous* session's final count, not from their own
|
||||
// first snapshot, because that first snapshot already contains the few
|
||||
// minutes of typing that preceded it — measuring from it would drop that
|
||||
// work. The first session is the exception: it measures from its own first
|
||||
// snapshot rather than from zero, so a history whose early snapshots were
|
||||
// pruned understates session one instead of reporting the words it never
|
||||
// saw as a sudden addition.
|
||||
startWords := versions[0].WordCount
|
||||
|
||||
for i := 1; i < len(versions); i++ {
|
||||
v, prev := versions[i], versions[i-1]
|
||||
|
||||
if delta := v.WordCount - prev.WordCount; delta > d.LargestJump {
|
||||
d.LargestJump, d.LargestJumpAt, d.LargestJumpIdx = delta, v.CreatedAt, i
|
||||
}
|
||||
|
||||
if v.CreatedAt.Sub(prev.CreatedAt) > sessionGap {
|
||||
cur.WordsAdded = prev.WordCount - startWords
|
||||
d.Sessions = append(d.Sessions, cur)
|
||||
cur = passportSession{Start: v.CreatedAt, End: v.CreatedAt, Snapshots: 1}
|
||||
startWords = prev.WordCount
|
||||
continue
|
||||
}
|
||||
cur.End = v.CreatedAt
|
||||
cur.Snapshots++
|
||||
}
|
||||
cur.WordsAdded = versions[len(versions)-1].WordCount - startWords
|
||||
d.Sessions = append(d.Sessions, cur)
|
||||
|
||||
for _, s := range d.Sessions {
|
||||
d.ActiveTime += s.Duration()
|
||||
}
|
||||
|
||||
final := versions[len(versions)-1].WordCount
|
||||
d.NoteJump = final > 0 && float64(d.LargestJump)/float64(final) > jumpNoteThreshold
|
||||
|
||||
d.ChainStatus, d.UnhashedCount = verifyChain(doc, versions)
|
||||
return d
|
||||
}
|
||||
|
||||
// verifyChain recomputes every snapshot's hash and checks that each links to the
|
||||
// one before it. Returns the outcome and how many snapshots predate the chain.
|
||||
//
|
||||
// Broken *links* are not evidence of tampering on their own: auto-snapshot
|
||||
// pruning legitimately removes rows from the middle of the history, which severs
|
||||
// the links across the hole. So a link break is reported as a gap unless the
|
||||
// document is in preserve-history mode, where nothing should ever be removed. A
|
||||
// hash that fails to match its *own* contents is unambiguous, and always broken.
|
||||
func verifyChain(doc db.Document, versions []db.DocumentVersion) (status string, unhashed int) {
|
||||
var (
|
||||
hashed int
|
||||
linkBreak bool
|
||||
prevHash string
|
||||
havePrev bool
|
||||
)
|
||||
|
||||
for _, v := range versions {
|
||||
if v.ContentHash == "" {
|
||||
unhashed++
|
||||
havePrev = false // can't vouch for what follows an unhashed row
|
||||
continue
|
||||
}
|
||||
hashed++
|
||||
|
||||
want := chainHash(v.PrevHash, v.DocID, v.CreatedAt, v.WordCount, v.ContentText)
|
||||
if want != v.ContentHash {
|
||||
return chainBroken, unhashed
|
||||
}
|
||||
if havePrev && v.PrevHash != prevHash {
|
||||
linkBreak = true
|
||||
}
|
||||
prevHash, havePrev = v.ContentHash, true
|
||||
}
|
||||
|
||||
switch {
|
||||
case hashed == 0:
|
||||
return chainUnverifiable, unhashed
|
||||
case linkBreak && doc.PreserveHistory:
|
||||
// Nothing should have been removed from a preserved history.
|
||||
return chainBroken, unhashed
|
||||
case linkBreak:
|
||||
return chainGaps, unhashed
|
||||
case unhashed > 0:
|
||||
return chainPartial, unhashed
|
||||
default:
|
||||
return chainVerified, unhashed
|
||||
}
|
||||
}
|
||||
|
||||
// --- HTTP -------------------------------------------------------------------
|
||||
|
||||
// passport renders the report for one document as a standalone HTML download.
|
||||
// HTML rather than PDF for the same reason as the other exports: a CJK-safe PDF
|
||||
// needs an embedded Unicode font or a headless browser. The page is styled for
|
||||
// printing, so "Save as PDF" in the browser produces the handoff artifact.
|
||||
func (h *Handler) passport(w http.ResponseWriter, r *http.Request) {
|
||||
docID := chi.URLParam(r, "id")
|
||||
userID := auth.UserID(r.Context())
|
||||
|
||||
doc, err := h.fetch(userID, docID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
notFound(w)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
versions, err := h.passportVersions(userID, docID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
body := renderPassport(buildPassport(doc, versions))
|
||||
|
||||
filename := sanitizeFilename(doc.Title)
|
||||
if filename == "" {
|
||||
filename = "untitled"
|
||||
}
|
||||
filename += " - writing passport.html"
|
||||
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.Header().Set("Content-Disposition",
|
||||
fmt.Sprintf("attachment; filename*=UTF-8''%s", urlEscapeFilename(filename)))
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body)))
|
||||
_, _ = w.Write(body)
|
||||
}
|
||||
|
||||
// passportVersions loads every snapshot oldest-first with the fields the report
|
||||
// and the chain check need — including content_text, which the list endpoint
|
||||
// omits as too heavy but verification cannot do without.
|
||||
func (h *Handler) passportVersions(userID, docID string) ([]db.DocumentVersion, error) {
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT v.id, v.doc_id, v.title, v.content_text, v.word_count, v.kind,
|
||||
v.created_at, v.content_hash, v.prev_hash
|
||||
FROM document_versions v
|
||||
JOIN documents d ON d.id = v.doc_id
|
||||
WHERE v.doc_id = ? AND d.user_id = ?
|
||||
ORDER BY v.created_at ASC, v.rowid ASC`,
|
||||
docID, userID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []db.DocumentVersion
|
||||
for rows.Next() {
|
||||
var v db.DocumentVersion
|
||||
if err := rows.Scan(
|
||||
&v.ID, &v.DocID, &v.Title, &v.ContentText, &v.WordCount, &v.Kind,
|
||||
&v.CreatedAt, &v.ContentHash, &v.PrevHash,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
package docs
|
||||
|
||||
// HTML rendering for the writing passport. Self-contained (no external assets)
|
||||
// and styled for print, so the browser's "Save as PDF" turns it into the file a
|
||||
// writer actually hands over.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Chart geometry. The plot is wide and short on purpose: the report's question
|
||||
// is "what shape did this document grow in", and a wide aspect makes a steady
|
||||
// climb read as steady rather than dramatic.
|
||||
const (
|
||||
chartW, chartH = 760, 260
|
||||
padL, padR, padT, padB = 52, 20, 18, 34
|
||||
plotW, plotH = chartW - padL - padR, chartH - padT - padB
|
||||
minBandW = 2.0 // so a single-snapshot session still shows
|
||||
|
||||
// gutterSlots is the space between sessions, in snapshot-slot widths. Wide
|
||||
// enough to read as a break and to seat its duration label.
|
||||
gutterSlots = 2.5
|
||||
)
|
||||
|
||||
// Palette — the export stylesheet's tokens, reused so a passport looks like it
|
||||
// came from the same application as the document it describes.
|
||||
const (
|
||||
rose = "#b04a6a"
|
||||
roseLight = "#f6d6e0"
|
||||
roseWash = "#fdeef3"
|
||||
surface = "#fffafb"
|
||||
)
|
||||
|
||||
func renderPassport(d passportData) []byte {
|
||||
var b strings.Builder
|
||||
|
||||
fmt.Fprintf(&b, passportHead, htmlEscape(d.Doc.Title))
|
||||
|
||||
fmt.Fprintf(&b, `<header>
|
||||
<p class="eyebrow">Writing passport</p>
|
||||
<h1>%s</h1>
|
||||
<p class="sub">Generated %s</p>
|
||||
</header>
|
||||
`, htmlEscape(d.Doc.Title), htmlEscape(formatWhen(d.GeneratedAt)))
|
||||
|
||||
if len(d.Versions) == 0 {
|
||||
b.WriteString(`<p class="empty">This document has no saved history yet, so there is
|
||||
nothing to report. History builds up automatically as you write.</p>
|
||||
</body></html>`)
|
||||
return []byte(b.String())
|
||||
}
|
||||
|
||||
b.WriteString(renderStats(d))
|
||||
b.WriteString(renderChart(d))
|
||||
b.WriteString(renderSessions(d))
|
||||
b.WriteString(renderIntegrity(d))
|
||||
b.WriteString(passportLimits)
|
||||
b.WriteString("</body></html>\n")
|
||||
|
||||
return []byte(b.String())
|
||||
}
|
||||
|
||||
// renderStats is the headline row — the numbers a reader wants before deciding
|
||||
// whether to study the chart.
|
||||
func renderStats(d passportData) string {
|
||||
final := d.Versions[len(d.Versions)-1].WordCount
|
||||
|
||||
tiles := []struct{ value, label string }{
|
||||
{fmt.Sprintf("%d", len(d.Versions)), "snapshots saved"},
|
||||
{humanDuration(d.Span), "from first to last edit"},
|
||||
{fmt.Sprintf("%d", len(d.Sessions)), pluralize(len(d.Sessions), "writing session", "writing sessions")},
|
||||
{humanDuration(d.ActiveTime), "spent actively editing"},
|
||||
{fmt.Sprintf("%d", final), "words in the final draft"},
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString(`<section class="stats">`)
|
||||
for _, t := range tiles {
|
||||
fmt.Fprintf(&b, `<div class="tile"><span class="v">%s</span><span class="l">%s</span></div>`,
|
||||
htmlEscape(t.value), htmlEscape(t.label))
|
||||
}
|
||||
b.WriteString("</section>\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// renderChart draws word count as a step line — the count is only known at
|
||||
// snapshot moments, and a step says that honestly where a smooth curve would
|
||||
// invent values in between. Shaded bands mark writing sessions.
|
||||
//
|
||||
// The x axis is snapshot order, not wall-clock time, and this is deliberate. On
|
||||
// a linear time axis an essay written in three half-hour sittings across three
|
||||
// days renders as three vertical cliffs separated by empty space: all the actual
|
||||
// writing is crushed into one percent of the width, and the result looks exactly
|
||||
// like text pasted in three chunks — the opposite of what happened. Because
|
||||
// auto-snapshots are throttled to roughly one per few minutes of *active*
|
||||
// editing, snapshot order is already close to proportional to time spent
|
||||
// writing. So the plot gives its width to the writing and compresses the breaks,
|
||||
// which are drawn as explicit labelled gaps rather than silently removed.
|
||||
//
|
||||
// One series, so no legend: the heading names it.
|
||||
func renderChart(d passportData) string {
|
||||
maxW := 0
|
||||
for _, v := range d.Versions {
|
||||
if v.WordCount > maxW {
|
||||
maxW = v.WordCount
|
||||
}
|
||||
}
|
||||
yTop := niceCeil(maxW)
|
||||
|
||||
y := func(words int) float64 {
|
||||
if yTop <= 0 {
|
||||
return padT + plotH
|
||||
}
|
||||
return padT + plotH - float64(words)/float64(yTop)*plotH
|
||||
}
|
||||
|
||||
// Lay snapshots out in slots: one per snapshot, plus a gutter between
|
||||
// sessions for the break marker.
|
||||
slots := float64(len(d.Versions)) + gutterSlots*float64(len(d.Sessions)-1)
|
||||
sw := plotW / slots
|
||||
|
||||
xs := make([]float64, len(d.Versions))
|
||||
bandStart := make([]float64, len(d.Sessions))
|
||||
bandEnd := make([]float64, len(d.Sessions))
|
||||
|
||||
cursor, vi := 0.0, 0
|
||||
for si, s := range d.Sessions {
|
||||
if si > 0 {
|
||||
cursor += gutterSlots
|
||||
}
|
||||
bandStart[si] = padL + cursor*sw
|
||||
for k := 0; k < s.Snapshots; k++ {
|
||||
xs[vi] = padL + (cursor+0.5)*sw
|
||||
cursor++
|
||||
vi++
|
||||
}
|
||||
bandEnd[si] = padL + cursor*sw
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, `<section class="chart">
|
||||
<h2>How the draft grew</h2>
|
||||
<svg viewBox="0 0 %d %d" role="img" aria-label="Word count at each saved snapshot, grouped into writing sessions">
|
||||
`, chartW, chartH)
|
||||
|
||||
// Session bands sit behind everything; each carries a native tooltip.
|
||||
for i, s := range d.Sessions {
|
||||
w := bandEnd[i] - bandStart[i]
|
||||
if w < minBandW {
|
||||
w = minBandW
|
||||
}
|
||||
fmt.Fprintf(&b, `<rect x="%.1f" y="%d" width="%.1f" height="%d" fill="%s" rx="3"><title>Session %d: %s, %s, %d snapshots</title></rect>
|
||||
`, bandStart[i], padT, w, plotH, roseWash, i+1,
|
||||
htmlEscape(formatWhen(s.Start)), htmlEscape(humanDuration(s.Duration())), s.Snapshots)
|
||||
}
|
||||
|
||||
// Recessive gridlines with y labels at 0 / half / top.
|
||||
for _, gv := range []int{0, yTop / 2, yTop} {
|
||||
gy := y(gv)
|
||||
fmt.Fprintf(&b, `<line x1="%d" y1="%.1f" x2="%d" y2="%.1f" stroke="%s" stroke-width="1"/>
|
||||
<text x="%d" y="%.1f" class="axis" text-anchor="end">%d</text>
|
||||
`, padL, gy, chartW-padR, gy, roseLight, padL-8, gy+4, gv)
|
||||
}
|
||||
|
||||
// Break markers in the gutters, so compressed time is stated, not hidden.
|
||||
for i := 1; i < len(d.Sessions); i++ {
|
||||
mid := (bandEnd[i-1] + bandStart[i]) / 2
|
||||
gap := d.Sessions[i].Start.Sub(d.Sessions[i-1].End)
|
||||
fmt.Fprintf(&b, `<line x1="%.1f" y1="%d" x2="%.1f" y2="%d" stroke="%s" stroke-width="1" stroke-dasharray="3 3"/>
|
||||
<text x="%.1f" y="%d" class="gap" text-anchor="middle">%s</text>
|
||||
`, mid, padT, mid, padT+plotH, roseLight, mid, padT+plotH+13, htmlEscape(humanDuration(gap)+" away"))
|
||||
}
|
||||
|
||||
// Step path: hold the previous value until the next snapshot lands.
|
||||
var path strings.Builder
|
||||
fmt.Fprintf(&path, "M %.1f %.1f", xs[0], y(d.Versions[0].WordCount))
|
||||
for i := 1; i < len(d.Versions); i++ {
|
||||
fmt.Fprintf(&path, " L %.1f %.1f L %.1f %.1f",
|
||||
xs[i], y(d.Versions[i-1].WordCount), xs[i], y(d.Versions[i].WordCount))
|
||||
}
|
||||
fmt.Fprintf(&b, `<path d="%s" fill="none" stroke="%s" stroke-width="2" stroke-linejoin="round"/>
|
||||
`, path.String(), rose)
|
||||
|
||||
// Pre-empt the obvious question: label the largest single jump when it is a
|
||||
// big share of the finished draft, rather than letting a reader find it.
|
||||
if d.NoteJump && d.LargestJumpIdx < len(xs) {
|
||||
jx, jy := xs[d.LargestJumpIdx], y(d.Versions[d.LargestJumpIdx].WordCount)
|
||||
|
||||
// Flip the label inboard near the right edge so it can't overflow, and
|
||||
// push it below the point when the point sits near the top.
|
||||
anchor, dx := "start", 9.0
|
||||
if jx > float64(chartW)*0.6 {
|
||||
anchor, dx = "end", -9.0
|
||||
}
|
||||
ly := jy - 10
|
||||
if ly < padT+12 {
|
||||
ly = jy + 18
|
||||
}
|
||||
|
||||
fmt.Fprintf(&b, `<circle cx="%.1f" cy="%.1f" r="4" fill="%s" stroke="%s" stroke-width="2"/>
|
||||
<text x="%.1f" y="%.1f" class="note" text-anchor="%s">largest single addition: +%d words</text>
|
||||
`, jx, jy, rose, surface, jx+dx, ly, anchor, d.LargestJump)
|
||||
}
|
||||
|
||||
fmt.Fprintf(&b, `</svg>
|
||||
<p class="caption">Each shaded band is one writing session, %s to %s. Width follows
|
||||
snapshots saved, so time spent writing gets the space and breaks are compressed to
|
||||
the labelled gaps.</p>
|
||||
</section>
|
||||
`, htmlEscape(formatDay(d.FirstAt)), htmlEscape(formatDay(d.LastAt)))
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func renderSessions(d passportData) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(`<section>
|
||||
<h2>Writing sessions</h2>
|
||||
<table>
|
||||
<thead><tr><th>#</th><th>Started</th><th>Length</th><th>Snapshots</th><th>Net words</th></tr></thead>
|
||||
<tbody>
|
||||
`)
|
||||
for i, s := range d.Sessions {
|
||||
fmt.Fprintf(&b, `<tr><td>%d</td><td>%s</td><td>%s</td><td>%d</td><td>%+d</td></tr>
|
||||
`, i+1, htmlEscape(formatWhen(s.Start)), htmlEscape(humanDuration(s.Duration())),
|
||||
s.Snapshots, s.WordsAdded)
|
||||
}
|
||||
b.WriteString("</tbody></table>\n</section>\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// renderIntegrity explains the hash chain in plain language, including when it
|
||||
// cannot vouch for something.
|
||||
func renderIntegrity(d passportData) string {
|
||||
var headline, detail string
|
||||
|
||||
switch d.ChainStatus {
|
||||
case chainVerified:
|
||||
headline = "History intact"
|
||||
detail = "Every snapshot matches its own contents and links correctly to the one before it. Nothing in this history has been altered or removed since it was recorded."
|
||||
case chainGaps:
|
||||
headline = "History intact, with gaps"
|
||||
detail = "Every snapshot matches its own contents, but some older automatic snapshots have been cleared to save space, so the record is not continuous. Turn on “keep full history” for this document to stop that happening."
|
||||
case chainPartial:
|
||||
headline = "Partly verifiable"
|
||||
detail = fmt.Sprintf("%d snapshot(s) were recorded before this document started tracking integrity, so they cannot be checked. Everything recorded since then matches.", d.UnhashedCount)
|
||||
case chainUnverifiable:
|
||||
headline = "Not verifiable"
|
||||
detail = "These snapshots were recorded before integrity tracking existed. The timeline above is still the record that was saved as you wrote; it simply cannot be checked for later alteration."
|
||||
case chainBroken:
|
||||
headline = "Integrity check failed"
|
||||
detail = "At least one snapshot does not match what was recorded for it. This can mean the history was edited after the fact, or that the database was restored from a backup or copied between machines."
|
||||
}
|
||||
|
||||
return fmt.Sprintf(`<section class="integrity %s">
|
||||
<h2>%s</h2>
|
||||
<p>%s</p>
|
||||
</section>
|
||||
`, htmlEscape(d.ChainStatus), htmlEscape(headline), htmlEscape(detail))
|
||||
}
|
||||
|
||||
// passportLimits states plainly what the report does and does not establish.
|
||||
// Overclaiming would be worse than useless: a reader who catches the report
|
||||
// overstating its case discounts the whole thing.
|
||||
const passportLimits = `<section class="limits">
|
||||
<h2>How to read this</h2>
|
||||
<p>A document written over time leaves a trail: many snapshots, uneven growth,
|
||||
words added and cut and added again across separate sittings. A document that was
|
||||
pasted in from elsewhere tends to arrive nearly whole, in one or two snapshots,
|
||||
with little revision after.</p>
|
||||
<p>What this report shows is the record Petal saved automatically while the
|
||||
document was open, roughly every few minutes of active editing.</p>
|
||||
<p><strong>What it does not show.</strong> It cannot prove who was at the
|
||||
keyboard, and it cannot tell whether text typed into the editor was composed
|
||||
there or copied from another window. It is evidence of a writing process, not a
|
||||
certificate of authorship. It is most useful read alongside the drafts
|
||||
themselves.</p>
|
||||
</section>
|
||||
`
|
||||
|
||||
// --- formatting helpers -----------------------------------------------------
|
||||
|
||||
func formatWhen(t time.Time) string { return t.Local().Format("2 Jan 2006, 3:04 PM") }
|
||||
func formatDay(t time.Time) string { return t.Local().Format("2 Jan 2006") }
|
||||
|
||||
// humanDuration renders a span at the coarsest useful precision — a reader cares
|
||||
// that a session ran "2h 40m", never that it ran 2h40m12s.
|
||||
func humanDuration(d time.Duration) string {
|
||||
if d < time.Minute {
|
||||
return "under a minute"
|
||||
}
|
||||
days := int(d.Hours()) / 24
|
||||
hours := int(d.Hours()) % 24
|
||||
mins := int(d.Minutes()) % 60
|
||||
|
||||
switch {
|
||||
case days > 0 && hours > 0:
|
||||
return fmt.Sprintf("%dd %dh", days, hours)
|
||||
case days > 0:
|
||||
return fmt.Sprintf("%dd", days)
|
||||
case hours > 0 && mins > 0:
|
||||
return fmt.Sprintf("%dh %dm", hours, mins)
|
||||
case hours > 0:
|
||||
return fmt.Sprintf("%dh", hours)
|
||||
default:
|
||||
return fmt.Sprintf("%dm", mins)
|
||||
}
|
||||
}
|
||||
|
||||
func pluralize(n int, one, many string) string {
|
||||
if n == 1 {
|
||||
return one
|
||||
}
|
||||
return many
|
||||
}
|
||||
|
||||
// niceCeil rounds a maximum up to a round number so gridlines land on values a
|
||||
// reader can hold in their head.
|
||||
func niceCeil(n int) int {
|
||||
if n <= 0 {
|
||||
return 0
|
||||
}
|
||||
mag := math.Pow(10, math.Floor(math.Log10(float64(n))))
|
||||
return int(math.Ceil(float64(n)/(mag/2)) * (mag / 2))
|
||||
}
|
||||
|
||||
// passportHead is the page shell: one %s for the title. Print rules keep the
|
||||
// chart and the caveats on the page rather than letting them break across sheets.
|
||||
const passportHead = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Writing passport — %s</title>
|
||||
<style>
|
||||
:root { color-scheme: light; }
|
||||
body {
|
||||
font-family: "Georgia", "Songti SC", "Noto Serif CJK SC", "Source Han Serif SC", serif;
|
||||
line-height: 1.7; color: #463a3f; background: #fffafb;
|
||||
max-width: 48rem; margin: 3rem auto; padding: 0 1.5rem;
|
||||
}
|
||||
header { border-bottom: 2px solid #f6d6e0; padding-bottom: 1rem; margin-bottom: 2rem; }
|
||||
.eyebrow { text-transform: uppercase; letter-spacing: .12em; font-size: .72rem;
|
||||
color: #b04a6a; margin: 0 0 .3rem; }
|
||||
h1 { font-size: 1.8rem; color: #b04a6a; margin: 0; line-height: 1.3; }
|
||||
h2 { font-size: 1.05rem; color: #b04a6a; margin: 0 0 .75rem; }
|
||||
.sub, .axis, .l { color: #6b5860; }
|
||||
.sub { margin: .4rem 0 0; font-size: .9rem; }
|
||||
section { margin: 2.25rem 0; }
|
||||
|
||||
.stats { display: flex; flex-wrap: wrap; gap: 1.25rem 2rem; margin: 2rem 0; }
|
||||
.tile { display: flex; flex-direction: column; min-width: 7rem; }
|
||||
.tile .v { font-size: 1.6rem; color: #b04a6a; line-height: 1.1; }
|
||||
.tile .l { font-size: .8rem; margin-top: .15rem; }
|
||||
|
||||
.chart svg { width: 100%%; height: auto; }
|
||||
.axis { font-size: 11px; fill: #6b5860; font-family: system-ui, sans-serif; }
|
||||
.note { font-size: 11px; fill: #463a3f; font-family: system-ui, sans-serif; }
|
||||
.gap { font-size: 10px; fill: #6b5860; font-family: system-ui, sans-serif; }
|
||||
.caption { font-size: .8rem; color: #6b5860; margin: .5rem 0 0; }
|
||||
|
||||
table { border-collapse: collapse; width: 100%%; font-size: .9rem; }
|
||||
th, td { text-align: left; padding: .45rem .6rem; border-bottom: 1px solid #f3cdd9; }
|
||||
th { color: #6b5860; font-weight: normal; font-size: .78rem;
|
||||
text-transform: uppercase; letter-spacing: .06em; }
|
||||
|
||||
.integrity { background: #fff2f6; border-left: 3px solid #f3b6c8;
|
||||
padding: 1rem 1.25rem; border-radius: .4rem; }
|
||||
.integrity.broken { border-left-color: #c2410c; }
|
||||
.integrity p { margin: 0; font-size: .92rem; }
|
||||
|
||||
.limits { font-size: .88rem; color: #6b5860; border-top: 1px solid #f3cdd9;
|
||||
padding-top: 1.25rem; }
|
||||
.limits strong { color: #463a3f; }
|
||||
.empty { color: #6b5860; }
|
||||
|
||||
@media print {
|
||||
body { margin: 0; max-width: none; }
|
||||
section, .chart svg, table { break-inside: avoid; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
`
|
||||
@@ -0,0 +1,376 @@
|
||||
package docs
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// chained builds a valid hash-chained snapshot run from (minutes-offset, words)
|
||||
// pairs, so tests can describe a writing history in terms a reader recognises
|
||||
// and get correct hashes for free.
|
||||
func chained(docID string, base time.Time, points ...[2]int) []db.DocumentVersion {
|
||||
var (
|
||||
out []db.DocumentVersion
|
||||
prev string
|
||||
)
|
||||
for i, p := range points {
|
||||
at := base.Add(time.Duration(p[0]) * time.Minute)
|
||||
text := strings.Repeat("word ", p[1])
|
||||
v := db.DocumentVersion{
|
||||
ID: fmt.Sprintf("v%d", i),
|
||||
DocID: docID,
|
||||
ContentText: text,
|
||||
WordCount: p[1],
|
||||
Kind: db.VersionKindAuto,
|
||||
CreatedAt: at,
|
||||
PrevHash: prev,
|
||||
}
|
||||
v.ContentHash = chainHash(prev, docID, at, p[1], text)
|
||||
prev = v.ContentHash
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestBuildPassportSessions(t *testing.T) {
|
||||
base := time.Date(2026, 3, 2, 9, 0, 0, 0, time.UTC)
|
||||
doc := db.Document{ID: "d1", Title: "Essay"}
|
||||
|
||||
// Two sittings: 09:00–09:30, then a three-hour break, then 12:30–13:00.
|
||||
vs := chained("d1", base,
|
||||
[2]int{0, 40}, [2]int{15, 120}, [2]int{30, 210},
|
||||
[2]int{210, 260}, [2]int{240, 330},
|
||||
)
|
||||
|
||||
d := buildPassport(doc, vs)
|
||||
|
||||
if len(d.Sessions) != 2 {
|
||||
t.Fatalf("sessions = %d, want 2", len(d.Sessions))
|
||||
}
|
||||
if got := d.Sessions[0].Duration(); got != 30*time.Minute {
|
||||
t.Errorf("session 1 duration = %v, want 30m", got)
|
||||
}
|
||||
if got := d.Sessions[1].Snapshots; got != 2 {
|
||||
t.Errorf("session 2 snapshots = %d, want 2", got)
|
||||
}
|
||||
if got := d.Span; got != 4*time.Hour {
|
||||
t.Errorf("span = %v, want 4h", got)
|
||||
}
|
||||
// Active time counts only time inside sessions, never the break.
|
||||
if got := d.ActiveTime; got != 60*time.Minute {
|
||||
t.Errorf("active time = %v, want 60m", got)
|
||||
}
|
||||
// Session 2 measures from session 1's final count (210 → 330).
|
||||
if got := d.Sessions[1].WordsAdded; got != 120 {
|
||||
t.Errorf("session 2 words = %d, want 120", got)
|
||||
}
|
||||
if d.ChainStatus != chainVerified {
|
||||
t.Errorf("chain = %q, want %q", d.ChainStatus, chainVerified)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPassportFlagsLargeJump(t *testing.T) {
|
||||
base := time.Date(2026, 3, 2, 9, 0, 0, 0, time.UTC)
|
||||
doc := db.Document{ID: "d1"}
|
||||
|
||||
t.Run("steady growth is not flagged", func(t *testing.T) {
|
||||
vs := chained("d1", base, [2]int{0, 100}, [2]int{5, 200}, [2]int{10, 300}, [2]int{15, 400})
|
||||
if d := buildPassport(doc, vs); d.NoteJump {
|
||||
t.Errorf("even growth flagged a jump of %d", d.LargestJump)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a paste-shaped jump is flagged", func(t *testing.T) {
|
||||
vs := chained("d1", base, [2]int{0, 20}, [2]int{5, 40}, [2]int{10, 900})
|
||||
d := buildPassport(doc, vs)
|
||||
if !d.NoteJump {
|
||||
t.Fatal("large jump not flagged")
|
||||
}
|
||||
if d.LargestJump != 860 {
|
||||
t.Errorf("largest jump = %d, want 860", d.LargestJump)
|
||||
}
|
||||
if !d.LargestJumpAt.Equal(base.Add(10 * time.Minute)) {
|
||||
t.Errorf("jump at %v, want +10m", d.LargestJumpAt)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestVerifyChain(t *testing.T) {
|
||||
base := time.Date(2026, 3, 2, 9, 0, 0, 0, time.UTC)
|
||||
good := func() []db.DocumentVersion {
|
||||
return chained("d1", base, [2]int{0, 50}, [2]int{5, 90}, [2]int{10, 160})
|
||||
}
|
||||
|
||||
t.Run("intact chain verifies", func(t *testing.T) {
|
||||
got, _ := verifyChain(db.Document{ID: "d1"}, good())
|
||||
if got != chainVerified {
|
||||
t.Errorf("got %q, want %q", got, chainVerified)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("edited content breaks it", func(t *testing.T) {
|
||||
vs := good()
|
||||
vs[1].ContentText = "something else entirely"
|
||||
if got, _ := verifyChain(db.Document{ID: "d1"}, vs); got != chainBroken {
|
||||
t.Errorf("got %q, want %q", got, chainBroken)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("backdating breaks it", func(t *testing.T) {
|
||||
vs := good()
|
||||
vs[2].CreatedAt = base.Add(-time.Hour)
|
||||
if got, _ := verifyChain(db.Document{ID: "d1"}, vs); got != chainBroken {
|
||||
t.Errorf("got %q, want %q", got, chainBroken)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a removed snapshot reads as a gap when pruning is allowed", func(t *testing.T) {
|
||||
vs := good()
|
||||
pruned := []db.DocumentVersion{vs[0], vs[2]} // middle snapshot gone
|
||||
got, _ := verifyChain(db.Document{ID: "d1"}, pruned)
|
||||
if got != chainGaps {
|
||||
t.Errorf("got %q, want %q", got, chainGaps)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a removed snapshot is tampering when history is preserved", func(t *testing.T) {
|
||||
vs := good()
|
||||
pruned := []db.DocumentVersion{vs[0], vs[2]}
|
||||
got, _ := verifyChain(db.Document{ID: "d1", PreserveHistory: true}, pruned)
|
||||
if got != chainBroken {
|
||||
t.Errorf("got %q, want %q", got, chainBroken)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("pre-chain snapshots are partial, not failures", func(t *testing.T) {
|
||||
vs := good()
|
||||
vs[0].ContentHash, vs[0].PrevHash = "", ""
|
||||
got, unhashed := verifyChain(db.Document{ID: "d1"}, vs)
|
||||
if got != chainPartial {
|
||||
t.Errorf("got %q, want %q", got, chainPartial)
|
||||
}
|
||||
if unhashed != 1 {
|
||||
t.Errorf("unhashed = %d, want 1", unhashed)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no hashes at all is unverifiable", func(t *testing.T) {
|
||||
vs := good()
|
||||
for i := range vs {
|
||||
vs[i].ContentHash, vs[i].PrevHash = "", ""
|
||||
}
|
||||
if got, _ := verifyChain(db.Document{ID: "d1"}, vs); got != chainUnverifiable {
|
||||
t.Errorf("got %q, want %q", got, chainUnverifiable)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The report must survive the degenerate histories — one snapshot, an empty
|
||||
// document — rather than dividing by a zero span or a zero maximum.
|
||||
func TestRenderPassportEdgeCases(t *testing.T) {
|
||||
base := time.Date(2026, 3, 2, 9, 0, 0, 0, time.UTC)
|
||||
|
||||
t.Run("no history", func(t *testing.T) {
|
||||
out := string(renderPassport(buildPassport(db.Document{Title: "Empty"}, nil)))
|
||||
if !strings.Contains(out, "no saved history") {
|
||||
t.Errorf("missing empty-state copy:\n%s", out)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("single snapshot", func(t *testing.T) {
|
||||
vs := chained("d1", base, [2]int{0, 12})
|
||||
out := string(renderPassport(buildPassport(db.Document{ID: "d1", Title: "One"}, vs)))
|
||||
if strings.Contains(out, "NaN") || strings.Contains(out, "+Inf") {
|
||||
t.Errorf("degenerate geometry leaked into output:\n%s", out)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty document", func(t *testing.T) {
|
||||
vs := chained("d1", base, [2]int{0, 0}, [2]int{5, 0})
|
||||
out := string(renderPassport(buildPassport(db.Document{ID: "d1"}, vs)))
|
||||
if strings.Contains(out, "NaN") {
|
||||
t.Errorf("zero word count produced NaN:\n%s", out)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("title is escaped", func(t *testing.T) {
|
||||
doc := db.Document{ID: "d1", Title: `<script>alert(1)</script>`}
|
||||
out := string(renderPassport(buildPassport(doc, chained("d1", base, [2]int{0, 5}))))
|
||||
if strings.Contains(out, "<script>") {
|
||||
t.Error("title was not escaped")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The chart must give its width to the writing, not to the gaps between
|
||||
// sittings. Three short sessions spread over three days is the case that a
|
||||
// wall-clock x axis renders as three vertical cliffs — visually identical to
|
||||
// pasted text, and wrong.
|
||||
func TestChartGivesWidthToWriting(t *testing.T) {
|
||||
base := time.Date(2026, 3, 2, 9, 0, 0, 0, time.UTC)
|
||||
|
||||
// ~30 minutes of work on each of three consecutive days.
|
||||
var pts [][2]int
|
||||
words := 0
|
||||
for day := 0; day < 3; day++ {
|
||||
for k := 0; k < 6; k++ {
|
||||
words += 50
|
||||
pts = append(pts, [2]int{day*1440 + k*5, words})
|
||||
}
|
||||
}
|
||||
|
||||
d := buildPassport(db.Document{ID: "d1"}, chained("d1", base, pts...))
|
||||
if len(d.Sessions) != 3 {
|
||||
t.Fatalf("sessions = %d, want 3", len(d.Sessions))
|
||||
}
|
||||
|
||||
out := renderChart(d)
|
||||
|
||||
// Every session band should be a substantial share of the plot, not a sliver.
|
||||
widths := regexp.MustCompile(`<rect [^>]*width="([0-9.]+)"`).FindAllStringSubmatch(out, -1)
|
||||
if len(widths) != 3 {
|
||||
t.Fatalf("session bands = %d, want 3", len(widths))
|
||||
}
|
||||
for i, m := range widths {
|
||||
w, err := strconv.ParseFloat(m[1], 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w < plotW*0.15 {
|
||||
t.Errorf("session %d band is %.1fpx of %dpx plot — writing got crushed", i+1, w, plotW)
|
||||
}
|
||||
}
|
||||
|
||||
// The compressed breaks must be stated, not silently removed.
|
||||
if got := strings.Count(out, `class="gap"`); got != 2 {
|
||||
t.Errorf("break labels = %d, want 2", got)
|
||||
}
|
||||
if !strings.Contains(out, "away") {
|
||||
t.Error("break labels do not name their duration")
|
||||
}
|
||||
}
|
||||
|
||||
// Chart coordinates must stay inside the viewBox whatever the history looks like.
|
||||
func TestChartStaysInBounds(t *testing.T) {
|
||||
base := time.Date(2026, 3, 2, 9, 0, 0, 0, time.UTC)
|
||||
|
||||
histories := map[string][][2]int{
|
||||
"single snapshot": {{0, 30}},
|
||||
"two sessions": {{0, 30}, {5, 90}, {600, 140}, {605, 210}},
|
||||
"words removed": {{0, 400}, {5, 380}, {10, 120}},
|
||||
"all zero": {{0, 0}, {5, 0}},
|
||||
"many snapshots": func() (p [][2]int) {
|
||||
for i := 0; i < 60; i++ {
|
||||
p = append(p, [2]int{i * 4, i * 20})
|
||||
}
|
||||
return
|
||||
}(),
|
||||
}
|
||||
|
||||
num := regexp.MustCompile(`(?:x|cx)="([0-9.-]+)"`)
|
||||
for name, pts := range histories {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
out := renderChart(buildPassport(db.Document{ID: "d1"}, chained("d1", base, pts...)))
|
||||
if strings.Contains(out, "NaN") || strings.Contains(out, "Inf") {
|
||||
t.Fatalf("degenerate geometry:\n%s", out)
|
||||
}
|
||||
for _, m := range num.FindAllStringSubmatch(out, -1) {
|
||||
v, err := strconv.ParseFloat(m[1], 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v < 0 || v > chartW {
|
||||
t.Errorf("x coordinate %.1f outside 0..%d", v, chartW)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanDuration(t *testing.T) {
|
||||
cases := []struct {
|
||||
in time.Duration
|
||||
want string
|
||||
}{
|
||||
{0, "under a minute"},
|
||||
{30 * time.Second, "under a minute"},
|
||||
{18 * time.Minute, "18m"},
|
||||
{2 * time.Hour, "2h"},
|
||||
{2*time.Hour + 40*time.Minute, "2h 40m"},
|
||||
{50 * time.Hour, "2d 2h"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := humanDuration(c.in); got != c.want {
|
||||
t.Errorf("humanDuration(%v) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- endpoint / persistence -------------------------------------------------
|
||||
|
||||
func TestPassportEndpoint(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
id := newDoc(t, srv)
|
||||
|
||||
do(t, srv, http.MethodPut, "/"+id,
|
||||
`{"content":"{}","content_text":"the first draft","word_count":3}`)
|
||||
|
||||
rec := do(t, srv, http.MethodGet, "/"+id+"/passport", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("passport: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/html") {
|
||||
t.Errorf("content-type = %q, want text/html", ct)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, "Writing passport") {
|
||||
t.Errorf("report body missing heading:\n%s", body)
|
||||
}
|
||||
// Snapshots written through the real insert path must verify.
|
||||
if !strings.Contains(body, "History intact") {
|
||||
t.Errorf("live-written history did not verify:\n%s", body)
|
||||
}
|
||||
|
||||
if rec := do(t, srv, http.MethodGet, "/does-not-exist/passport", ""); rec.Code != http.StatusNotFound {
|
||||
t.Errorf("missing doc: code = %d, want 404", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreserveHistoryExemptsFromPruning(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
id := newDoc(t, srv)
|
||||
|
||||
rec := do(t, srv, http.MethodPut, "/"+id, `{"preserve_history":true}`)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("set preserve_history: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
|
||||
decodeDoc := func(rec *httptest.ResponseRecorder) db.Document {
|
||||
t.Helper()
|
||||
var doc db.Document
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &doc); err != nil {
|
||||
t.Fatalf("decode doc: %v", err)
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
if !decodeDoc(rec).PreserveHistory {
|
||||
t.Fatal("preserve_history did not persist")
|
||||
}
|
||||
|
||||
// An ordinary body save must not clear the flag.
|
||||
rec = do(t, srv, http.MethodPut, "/"+id,
|
||||
`{"content":"{}","content_text":"hello there","word_count":2}`)
|
||||
if !decodeDoc(rec).PreserveHistory {
|
||||
t.Error("a normal save cleared preserve_history")
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
)
|
||||
@@ -50,12 +51,13 @@ func (h *Handler) SearchRoutes() chi.Router {
|
||||
return r
|
||||
}
|
||||
|
||||
// search runs a cross-document full-text search for the local user. Queries of
|
||||
// search runs a cross-document full-text search for the caller. Queries of
|
||||
// three or more runes use the trigram FTS index (fast, ranked); shorter queries
|
||||
// fall back to a LIKE scan so 2-character Chinese words still resolve. Either way
|
||||
// the snippet is built in Go from the original text, for clean word boundaries
|
||||
// and a uniform highlight format.
|
||||
func (h *Handler) search(w http.ResponseWriter, r *http.Request) {
|
||||
userID := auth.UserID(r.Context())
|
||||
q := strings.TrimSpace(r.URL.Query().Get("q"))
|
||||
if q == "" {
|
||||
httputil.WriteJSON(w, http.StatusOK, []searchResult{})
|
||||
@@ -80,7 +82,7 @@ func (h *Handler) search(w http.ResponseWriter, r *http.Request) {
|
||||
WHERE documents_fts MATCH ? AND d.user_id = ?
|
||||
ORDER BY rank
|
||||
LIMIT ?`,
|
||||
phrase, db.LocalUserID, maxSearchResults,
|
||||
phrase, userID, maxSearchResults,
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -110,7 +112,7 @@ func (h *Handler) search(w http.ResponseWriter, r *http.Request) {
|
||||
AND (title LIKE ? ESCAPE '\' OR content_text LIKE ? ESCAPE '\')
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT ?`,
|
||||
db.LocalUserID, like, like, maxSearchResults,
|
||||
userID, like, like, maxSearchResults,
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -144,7 +146,7 @@ func (h *Handler) search(w http.ResponseWriter, r *http.Request) {
|
||||
ids = append(ids, rw.id)
|
||||
}
|
||||
|
||||
byDoc, err := h.tagsByDoc(ids)
|
||||
byDoc, err := h.tagsByDoc(userID, ids)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
|
||||
+20
-17
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
)
|
||||
@@ -55,7 +56,7 @@ func (h *Handler) listTags(w http.ResponseWriter, r *http.Request) {
|
||||
WHERE t.user_id = ?
|
||||
GROUP BY t.id
|
||||
ORDER BY t.name COLLATE NOCASE`,
|
||||
db.LocalUserID,
|
||||
auth.UserID(r.Context()),
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -104,7 +105,7 @@ func (h *Handler) createTag(w http.ResponseWriter, r *http.Request) {
|
||||
`INSERT INTO tags (user_id, name, color) VALUES (?, ?, ?)
|
||||
ON CONFLICT(user_id, name) DO UPDATE SET name = excluded.name
|
||||
RETURNING id, name, color`,
|
||||
db.LocalUserID, name, normalizeColor(req.Color),
|
||||
auth.UserID(r.Context()), name, normalizeColor(req.Color),
|
||||
).Scan(&t.ID, &t.Name, &t.Color)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -117,6 +118,7 @@ func (h *Handler) createTag(w http.ResponseWriter, r *http.Request) {
|
||||
// a recolor needn't resend the name.
|
||||
func (h *Handler) updateTag(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
userID := auth.UserID(r.Context())
|
||||
|
||||
var req struct {
|
||||
Name *string `json:"name"`
|
||||
@@ -145,7 +147,7 @@ func (h *Handler) updateTag(w http.ResponseWriter, r *http.Request) {
|
||||
SET name = COALESCE(?, name),
|
||||
color = COALESCE(?, color)
|
||||
WHERE id = ? AND user_id = ?`,
|
||||
namePtr, colorPtr, id, db.LocalUserID,
|
||||
namePtr, colorPtr, id, userID,
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -159,7 +161,7 @@ func (h *Handler) updateTag(w http.ResponseWriter, r *http.Request) {
|
||||
var t db.Tag
|
||||
if err := h.DB.QueryRow(
|
||||
`SELECT id, name, color FROM tags WHERE id = ? AND user_id = ?`,
|
||||
id, db.LocalUserID,
|
||||
id, userID,
|
||||
).Scan(&t.ID, &t.Name, &t.Color); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
@@ -171,7 +173,7 @@ func (h *Handler) updateTag(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *Handler) deleteTag(w http.ResponseWriter, r *http.Request) {
|
||||
res, err := h.DB.Exec(
|
||||
`DELETE FROM tags WHERE id = ? AND user_id = ?`,
|
||||
chi.URLParam(r, "id"), db.LocalUserID,
|
||||
chi.URLParam(r, "id"), auth.UserID(r.Context()),
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -184,10 +186,11 @@ func (h *Handler) deleteTag(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// assignTag attaches a tag to a document. Both must belong to the local user;
|
||||
// assignTag attaches a tag to a document. Both must belong to the caller;
|
||||
// the assignment is idempotent (re-assigning is a no-op, not an error).
|
||||
func (h *Handler) assignTag(w http.ResponseWriter, r *http.Request) {
|
||||
docID := chi.URLParam(r, "id")
|
||||
userID := auth.UserID(r.Context())
|
||||
|
||||
var req struct {
|
||||
TagID string `json:"tag_id"`
|
||||
@@ -203,11 +206,11 @@ func (h *Handler) assignTag(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Verify both the doc and the tag belong to the user before linking, so a
|
||||
// stray id can't cross-link another account's rows.
|
||||
if !h.ownsDoc(docID) {
|
||||
if !h.ownsDoc(userID, docID) {
|
||||
notFound(w)
|
||||
return
|
||||
}
|
||||
if !h.ownsTag(req.TagID) {
|
||||
if !h.ownsTag(userID, req.TagID) {
|
||||
notFoundMsg(w, "tag not found")
|
||||
return
|
||||
}
|
||||
@@ -228,7 +231,7 @@ func (h *Handler) unassignTag(w http.ResponseWriter, r *http.Request) {
|
||||
docID := chi.URLParam(r, "id")
|
||||
tagID := chi.URLParam(r, "tagId")
|
||||
|
||||
if !h.ownsDoc(docID) {
|
||||
if !h.ownsDoc(auth.UserID(r.Context()), docID) {
|
||||
notFound(w)
|
||||
return
|
||||
}
|
||||
@@ -242,22 +245,22 @@ func (h *Handler) unassignTag(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// ownsDoc reports whether a document belongs to the local user.
|
||||
func (h *Handler) ownsDoc(docID string) bool {
|
||||
// ownsDoc reports whether a document belongs to the given user.
|
||||
func (h *Handler) ownsDoc(userID, docID string) bool {
|
||||
var exists bool
|
||||
_ = h.DB.QueryRow(
|
||||
`SELECT EXISTS(SELECT 1 FROM documents WHERE id = ? AND user_id = ?)`,
|
||||
docID, db.LocalUserID,
|
||||
docID, userID,
|
||||
).Scan(&exists)
|
||||
return exists
|
||||
}
|
||||
|
||||
// ownsTag reports whether a tag belongs to the local user.
|
||||
func (h *Handler) ownsTag(tagID string) bool {
|
||||
// ownsTag reports whether a tag belongs to the given user.
|
||||
func (h *Handler) ownsTag(userID, tagID string) bool {
|
||||
var exists bool
|
||||
_ = h.DB.QueryRow(
|
||||
`SELECT EXISTS(SELECT 1 FROM tags WHERE id = ? AND user_id = ?)`,
|
||||
tagID, db.LocalUserID,
|
||||
tagID, userID,
|
||||
).Scan(&exists)
|
||||
return exists
|
||||
}
|
||||
@@ -265,7 +268,7 @@ func (h *Handler) ownsTag(tagID string) bool {
|
||||
// tagsByDoc loads the tags for a set of documents in one query and groups them
|
||||
// by doc id. Used to decorate the document list and search results without an
|
||||
// N+1 of per-doc queries. Returns an empty (non-nil) map when ids is empty.
|
||||
func (h *Handler) tagsByDoc(ids []string) (map[string][]db.Tag, error) {
|
||||
func (h *Handler) tagsByDoc(userID string, ids []string) (map[string][]db.Tag, error) {
|
||||
out := map[string][]db.Tag{}
|
||||
if len(ids) == 0 {
|
||||
return out, nil
|
||||
@@ -277,7 +280,7 @@ func (h *Handler) tagsByDoc(ids []string) (map[string][]db.Tag, error) {
|
||||
for _, id := range ids {
|
||||
args = append(args, id)
|
||||
}
|
||||
args = append(args, db.LocalUserID)
|
||||
args = append(args, userID)
|
||||
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT dt.doc_id, t.id, t.name, t.color
|
||||
|
||||
@@ -27,7 +27,7 @@ func newFullServer(t *testing.T) http.Handler {
|
||||
r.Mount("/docs", h.Routes())
|
||||
r.Mount("/tags", h.TagRoutes())
|
||||
r.Mount("/search", h.SearchRoutes())
|
||||
return r
|
||||
return withAuth(r)
|
||||
}
|
||||
|
||||
// createDoc makes a document with the given title/body and returns its id.
|
||||
|
||||
+71
-15
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
)
|
||||
@@ -33,6 +34,7 @@ func (h *Handler) versionRoutes(r chi.Router) {
|
||||
r.Post("/{id}/versions", h.createVersion) // explicit "save a restore point"
|
||||
r.Get("/{id}/versions/{vid}", h.getVersion) // full body for preview
|
||||
r.Post("/{id}/versions/{vid}/restore", h.restoreVersion)
|
||||
r.Get("/{id}/passport", h.passport) // authorship report over that history
|
||||
}
|
||||
|
||||
// listVersions returns the document's snapshots, newest first, without the heavy
|
||||
@@ -48,7 +50,7 @@ func (h *Handler) listVersions(w http.ResponseWriter, r *http.Request) {
|
||||
JOIN documents d ON d.id = v.doc_id
|
||||
WHERE v.doc_id = ? AND d.user_id = ?
|
||||
ORDER BY v.created_at DESC`,
|
||||
docID, db.LocalUserID,
|
||||
docID, auth.UserID(r.Context()),
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -74,7 +76,7 @@ func (h *Handler) listVersions(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// getVersion returns one snapshot in full (including content) for preview.
|
||||
func (h *Handler) getVersion(w http.ResponseWriter, r *http.Request) {
|
||||
v, err := h.fetchVersion(chi.URLParam(r, "id"), chi.URLParam(r, "vid"))
|
||||
v, err := h.fetchVersion(auth.UserID(r.Context()), chi.URLParam(r, "id"), chi.URLParam(r, "vid"))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
notFoundMsg(w, "version not found")
|
||||
return
|
||||
@@ -91,7 +93,7 @@ func (h *Handler) getVersion(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *Handler) createVersion(w http.ResponseWriter, r *http.Request) {
|
||||
docID := chi.URLParam(r, "id")
|
||||
|
||||
doc, err := h.fetch(docID)
|
||||
doc, err := h.fetch(auth.UserID(r.Context()), docID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
notFound(w)
|
||||
return
|
||||
@@ -115,8 +117,9 @@ func (h *Handler) createVersion(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *Handler) restoreVersion(w http.ResponseWriter, r *http.Request) {
|
||||
docID := chi.URLParam(r, "id")
|
||||
vid := chi.URLParam(r, "vid")
|
||||
userID := auth.UserID(r.Context())
|
||||
|
||||
v, err := h.fetchVersion(docID, vid)
|
||||
v, err := h.fetchVersion(userID, docID, vid)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
notFoundMsg(w, "version not found")
|
||||
return
|
||||
@@ -126,7 +129,7 @@ func (h *Handler) restoreVersion(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
current, err := h.fetch(docID)
|
||||
current, err := h.fetch(userID, docID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
@@ -141,7 +144,7 @@ func (h *Handler) restoreVersion(w http.ResponseWriter, r *http.Request) {
|
||||
SET title = ?, content = ?, content_text = ?, word_count = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ? AND user_id = ?`,
|
||||
v.Title, v.Content, v.ContentText, v.WordCount, docID, db.LocalUserID,
|
||||
v.Title, v.Content, v.ContentText, v.WordCount, docID, userID,
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -152,7 +155,7 @@ func (h *Handler) restoreVersion(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := h.fetch(docID)
|
||||
doc, err := h.fetch(userID, docID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
@@ -202,20 +205,73 @@ func (h *Handler) maybeAutoSnapshot(doc db.Document) error {
|
||||
|
||||
// insertVersion writes a snapshot row of the given kind and returns it (without
|
||||
// the heavy content fields, matching the list shape).
|
||||
//
|
||||
// The row is linked into the document's hash chain: it carries the previous
|
||||
// snapshot's hash, and its own hash covers that link plus its content. The hash
|
||||
// can only be computed once the database has assigned created_at, so the insert
|
||||
// and the hash write share a transaction — a snapshot is never visible with a
|
||||
// hash that doesn't cover its own timestamp.
|
||||
func (h *Handler) insertVersion(doc db.Document, kind string) (db.DocumentVersion, error) {
|
||||
tx, err := h.DB.Begin()
|
||||
if err != nil {
|
||||
return db.DocumentVersion{}, err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op once committed
|
||||
|
||||
// Chain onto the newest existing snapshot. created_at has second
|
||||
// granularity, so rowid breaks ties in true insertion order; verification
|
||||
// walks the same ordering in reverse.
|
||||
var prevHash string
|
||||
err = tx.QueryRow(
|
||||
`SELECT content_hash FROM document_versions
|
||||
WHERE doc_id = ? ORDER BY created_at DESC, rowid DESC LIMIT 1`,
|
||||
doc.ID,
|
||||
).Scan(&prevHash)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return db.DocumentVersion{}, err
|
||||
}
|
||||
|
||||
var v db.DocumentVersion
|
||||
err := h.DB.QueryRow(
|
||||
`INSERT INTO document_versions (doc_id, title, content, content_text, word_count, kind)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
if err := tx.QueryRow(
|
||||
`INSERT INTO document_versions (doc_id, title, content, content_text, word_count, kind, prev_hash)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
RETURNING id, doc_id, title, word_count, kind, created_at`,
|
||||
doc.ID, doc.Title, doc.Content, doc.ContentText, doc.WordCount, kind,
|
||||
).Scan(&v.ID, &v.DocID, &v.Title, &v.WordCount, &v.Kind, &v.CreatedAt)
|
||||
return v, err
|
||||
doc.ID, doc.Title, doc.Content, doc.ContentText, doc.WordCount, kind, prevHash,
|
||||
).Scan(&v.ID, &v.DocID, &v.Title, &v.WordCount, &v.Kind, &v.CreatedAt); err != nil {
|
||||
return db.DocumentVersion{}, err
|
||||
}
|
||||
|
||||
v.PrevHash = prevHash
|
||||
v.ContentHash = chainHash(prevHash, v.DocID, v.CreatedAt, v.WordCount, doc.ContentText)
|
||||
if _, err := tx.Exec(
|
||||
`UPDATE document_versions SET content_hash = ? WHERE id = ?`, v.ContentHash, v.ID,
|
||||
); err != nil {
|
||||
return db.DocumentVersion{}, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return db.DocumentVersion{}, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// pruneAutoVersions trims a document's 'auto' snapshots to the newest
|
||||
// maxAutoVersions, leaving 'manual' and 'pre_restore' restore points intact.
|
||||
//
|
||||
// Documents flagged preserve_history are exempt entirely: their history is
|
||||
// authorship evidence, and evidence with the oldest entries dropped is exactly
|
||||
// the part a reader would want — the early, sparse, figuring-it-out edits that
|
||||
// distinguish writing from pasting.
|
||||
func (h *Handler) pruneAutoVersions(docID string) error {
|
||||
var preserve bool
|
||||
if err := h.DB.QueryRow(
|
||||
`SELECT preserve_history FROM documents WHERE id = ?`, docID,
|
||||
).Scan(&preserve); err != nil {
|
||||
return err
|
||||
}
|
||||
if preserve {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := h.DB.Exec(
|
||||
`DELETE FROM document_versions
|
||||
WHERE doc_id = ? AND kind = 'auto'
|
||||
@@ -230,14 +286,14 @@ func (h *Handler) pruneAutoVersions(docID string) error {
|
||||
}
|
||||
|
||||
// fetchVersion loads one full snapshot, scoped to its owner via the parent doc.
|
||||
func (h *Handler) fetchVersion(docID, vid string) (db.DocumentVersion, error) {
|
||||
func (h *Handler) fetchVersion(userID, docID, vid string) (db.DocumentVersion, error) {
|
||||
var v db.DocumentVersion
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT v.id, v.doc_id, v.title, v.content, v.content_text, v.word_count, v.kind, v.created_at
|
||||
FROM document_versions v
|
||||
JOIN documents d ON d.id = v.doc_id
|
||||
WHERE v.id = ? AND v.doc_id = ? AND d.user_id = ?`,
|
||||
vid, docID, db.LocalUserID,
|
||||
vid, docID, userID,
|
||||
).Scan(
|
||||
&v.ID, &v.DocID, &v.Title, &v.Content, &v.ContentText,
|
||||
&v.WordCount, &v.Kind, &v.CreatedAt,
|
||||
|
||||
@@ -35,3 +35,18 @@ func ServerError(w http.ResponseWriter, err error) {
|
||||
log.Printf("internal error: %v", err)
|
||||
ErrorJSON(w, http.StatusInternalServerError, "something went wrong")
|
||||
}
|
||||
|
||||
// UpstreamError is ServerError's counterpart for a dependency Petal calls out
|
||||
// to — the model, chiefly. Same discipline, and for a sharper reason: a dial
|
||||
// failure's error text contains the endpoint it failed to dial, so relaying it
|
||||
// hands anyone who can reach Petal the address of the inference box on the far
|
||||
// side of the VPN, along with which backend is running there.
|
||||
//
|
||||
// `what` names the pass for the operator's log ("checkpoint", "chat"). The
|
||||
// browser is told only that the helper is unreachable, which is all the client
|
||||
// ever did anything with: every LLM route's 502 renders as the same warm
|
||||
// "小助手在休息 · Petal's helper is resting".
|
||||
func UpstreamError(w http.ResponseWriter, what string, err error) {
|
||||
log.Printf("upstream error (%s): %v", what, err)
|
||||
ErrorJSON(w, http.StatusBadGateway, "Petal's helper is out of reach right now")
|
||||
}
|
||||
|
||||
+235
-13
@@ -2,26 +2,52 @@
|
||||
// images from the editor, they're saved to disk under the configured directory,
|
||||
// and served back by hashed filename. Content addressing means the same image
|
||||
// pasted twice is stored once, and URLs are stable and cacheable forever.
|
||||
//
|
||||
// Each stored file also has one row per owner in the `images` table, and a fetch
|
||||
// joins on the caller. Before that, the store was a flat directory with no
|
||||
// database presence at all: any authenticated user holding a sha256 could fetch
|
||||
// anyone else's image. Hashes aren't guessable, so it was never an emergency —
|
||||
// but "unguessable filename" is not access control, and images pasted into a
|
||||
// private journal are exactly the content that shouldn't depend on it.
|
||||
//
|
||||
// One row per owner (rather than one owner per file) is what keeps deduplication:
|
||||
// the same picture uploaded by two people is stored once and simply has two rows.
|
||||
// The file is removed only with its last row.
|
||||
package images
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
)
|
||||
|
||||
// maxUploadBytes caps a single image at 10 MiB — generous for a writing tool,
|
||||
// small enough to keep a careless paste from filling the disk.
|
||||
const maxUploadBytes = 10 << 20
|
||||
|
||||
// maxUserBytes caps what one account may keep stored, at 1 GiB. The per-upload
|
||||
// limit bounds a single careless paste; nothing bounded ten thousand of them,
|
||||
// and Petal's data directory is an 8 GiB encrypted volume shared with the
|
||||
// database, the backups and the TTS cache — the disk filling is the database
|
||||
// losing writes, not just images failing.
|
||||
//
|
||||
// A tenth of the volume per writer is far past any real use: a heavily
|
||||
// illustrated journal is tens of megabytes. It is a runaway backstop, and it is
|
||||
// deliberately generous enough that nobody writing normally will ever meet it.
|
||||
const maxUserBytes = 1 << 30
|
||||
|
||||
// extByContentType maps the image types we accept to a canonical extension. The
|
||||
// allowlist doubles as validation: anything not here is rejected.
|
||||
var extByContentType = map[string]string{
|
||||
@@ -32,25 +58,96 @@ var extByContentType = map[string]string{
|
||||
"image/svg+xml": ".svg",
|
||||
}
|
||||
|
||||
// Handler serves the upload + fetch endpoints, backed by a directory on disk.
|
||||
// Handler serves the upload + fetch endpoints, backed by a directory on disk and
|
||||
// an ownership table.
|
||||
type Handler struct {
|
||||
dir string
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// New constructs a Handler, ensuring the storage directory exists.
|
||||
func New(dir string) (*Handler, error) {
|
||||
// New constructs a Handler, ensuring the storage directory exists and that every
|
||||
// file already in it has an owner.
|
||||
func New(dir string, database *sql.DB, backfillOwner string) (*Handler, error) {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Handler{dir: dir}, nil
|
||||
h := &Handler{dir: dir, db: database}
|
||||
if err := h.backfill(backfillOwner); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// backfill claims pre-existing files for one user. Images uploaded before
|
||||
// ownership existed have no row, and a row is now what makes them fetchable —
|
||||
// so without this every picture already pasted into a document would 404.
|
||||
// Attributing them to the account that has been the only one until now is the
|
||||
// only answer the data supports. Idempotent: files that already have an owner
|
||||
// are left alone.
|
||||
func (h *Handler) backfill(owner string) error {
|
||||
if owner == "" {
|
||||
return nil
|
||||
}
|
||||
// The owner may not exist — after the `local` account has been migrated onto
|
||||
// a real one, it doesn't. Claiming for a missing user would violate the
|
||||
// foreign key, and this runs during startup, so the error would take the
|
||||
// whole app down. There is nothing left to claim in that case anyway: the
|
||||
// migration moves the image rows along with everything else.
|
||||
var ownerExists bool
|
||||
if err := h.db.QueryRow(
|
||||
`SELECT EXISTS(SELECT 1 FROM users WHERE id = ?)`, owner,
|
||||
).Scan(&ownerExists); err != nil {
|
||||
return err
|
||||
}
|
||||
if !ownerExists {
|
||||
return nil
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(h.dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
claimed := 0
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
var exists bool
|
||||
if err := h.db.QueryRow(
|
||||
`SELECT EXISTS(SELECT 1 FROM images WHERE name = ?)`, e.Name(),
|
||||
).Scan(&exists); err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
continue
|
||||
}
|
||||
var size int64
|
||||
if info, err := e.Info(); err == nil {
|
||||
size = info.Size()
|
||||
}
|
||||
if _, err := h.db.Exec(
|
||||
`INSERT INTO images (name, user_id, content_type, size) VALUES (?, ?, '', ?)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
e.Name(), owner, size,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
claimed++
|
||||
}
|
||||
if claimed > 0 {
|
||||
log.Printf("images: claimed %d pre-existing image(s) for %s", claimed, owner)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Routes mounts the image endpoints. Mount under "/images" so the full paths are
|
||||
// POST /api/images (upload) and GET /api/images/{name} (fetch).
|
||||
// POST /api/images (upload), GET /api/images/{name} (fetch) and
|
||||
// DELETE /api/images/{name} (drop your copy).
|
||||
func (h *Handler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Post("/", h.upload)
|
||||
r.Get("/{name}", h.serve)
|
||||
r.Delete("/{name}", h.remove)
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -79,7 +176,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
ext, ok := extByContentType[ct]
|
||||
if !ok {
|
||||
if looksLikeSVG(data) {
|
||||
ext, ok = ".svg", true
|
||||
ct, ext, ok = "image/svg+xml", ".svg", true
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
@@ -91,6 +188,19 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
name := hex.EncodeToString(sum[:])[:32] + ext
|
||||
path := filepath.Join(h.dir, name)
|
||||
|
||||
userID := auth.UserID(r.Context())
|
||||
within, err := h.withinQuota(userID, name, int64(len(data)))
|
||||
if err != nil {
|
||||
log.Printf("images: quota check failed for %s: %v", userID, err)
|
||||
http.Error(w, "could not store image", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !within {
|
||||
http.Error(w, "you've filled Petal's picture store — delete a few images and try again",
|
||||
http.StatusInsufficientStorage)
|
||||
return
|
||||
}
|
||||
|
||||
// Skip the write if this exact content is already stored.
|
||||
if _, statErr := os.Stat(path); errors.Is(statErr, os.ErrNotExist) {
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
@@ -99,28 +209,140 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// Record the caller as an owner. Re-uploading your own image is a no-op;
|
||||
// uploading someone else's identical image adds a second row over one file.
|
||||
if _, err := h.db.Exec(
|
||||
`INSERT INTO images (name, user_id, content_type, size) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (name, user_id) DO NOTHING`,
|
||||
name, userID, ct, len(data),
|
||||
); err != nil {
|
||||
log.Printf("images: could not record ownership of %s: %v", name, err)
|
||||
http.Error(w, "could not store image", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"url": "/api/images/" + name})
|
||||
}
|
||||
|
||||
// serve returns a stored image by its hashed filename. The filename is validated
|
||||
// to be a bare name (no path separators) so it can't escape the storage dir, and
|
||||
// served with a long-lived cache header since content-addressed URLs never change.
|
||||
// serve returns a stored image by its hashed filename, but only to someone who
|
||||
// owns it. The filename is validated to be a bare name (no path separators) so
|
||||
// it can't escape the storage dir, and served with a long-lived cache header
|
||||
// since content-addressed URLs never change.
|
||||
//
|
||||
// Someone else's image is a 404, not a 403: whether a hash exists is itself
|
||||
// information the caller has no business learning.
|
||||
func (h *Handler) serve(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
if name == "" || name != filepath.Base(name) || strings.ContainsAny(name, `/\`) {
|
||||
name, ok := safeName(chi.URLParam(r, "name"))
|
||||
if !ok || !h.owns(name, auth.UserID(r.Context())) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
path := filepath.Join(h.dir, filepath.Base(name))
|
||||
path := filepath.Join(h.dir, name)
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
|
||||
// Private: a shared cache must never hand one writer's image to another.
|
||||
w.Header().Set("Cache-Control", "private, max-age=31536000, immutable")
|
||||
|
||||
// SVG is a document format wearing an image's name: it can carry <script>,
|
||||
// and this route serves it from Petal's own origin. Rendered through an
|
||||
// <img> — the only way the editor ever shows one — that script never runs.
|
||||
// Navigated to directly, which is one "open image in new tab" away, it does,
|
||||
// and it runs with the API of whoever opened it.
|
||||
//
|
||||
// So every stored image answers with a CSP that permits nothing at all
|
||||
// except the inline styles an illustration legitimately carries. It costs
|
||||
// pasted SVGs nothing (an <img> was already a script-free context) and
|
||||
// leaves the direct-navigation case inert. nosniff is set at the edge, but
|
||||
// repeated here so the guarantee doesn't depend on Traefik's config.
|
||||
w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
http.ServeFile(w, r, path)
|
||||
}
|
||||
|
||||
// remove drops the caller's claim on an image, and deletes the file itself once
|
||||
// nobody is left holding it.
|
||||
func (h *Handler) remove(w http.ResponseWriter, r *http.Request) {
|
||||
name, ok := safeName(chi.URLParam(r, "name"))
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
res, err := h.db.Exec(`DELETE FROM images WHERE name = ? AND user_id = ?`,
|
||||
name, auth.UserID(r.Context()))
|
||||
if err != nil {
|
||||
http.Error(w, "could not delete image", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
var others bool
|
||||
if err := h.db.QueryRow(
|
||||
`SELECT EXISTS(SELECT 1 FROM images WHERE name = ?)`, name,
|
||||
).Scan(&others); err != nil {
|
||||
// The row is gone either way; leaving an orphaned file behind is a
|
||||
// wasted block, not a correctness problem.
|
||||
log.Printf("images: could not check remaining owners of %s: %v", name, err)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
if !others {
|
||||
if err := os.Remove(filepath.Join(h.dir, name)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
log.Printf("images: could not remove %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// withinQuota reports whether userID may store one more image of size bytes.
|
||||
//
|
||||
// An image the caller already owns is free: content addressing means re-pasting
|
||||
// the same picture stores nothing new, and charging for it would let a document
|
||||
// that merely repeats one illustration walk into the limit. Deduplication
|
||||
// across *accounts* is not credited the same way — two people each keep their
|
||||
// own claim on a shared file, because either of them deleting it must not
|
||||
// depend on what the other did.
|
||||
func (h *Handler) withinQuota(userID, name string, size int64) (bool, error) {
|
||||
var used, already sql.NullInt64
|
||||
if err := h.db.QueryRow(
|
||||
`SELECT (SELECT COALESCE(SUM(size), 0) FROM images WHERE user_id = ?),
|
||||
(SELECT size FROM images WHERE user_id = ? AND name = ?)`,
|
||||
userID, userID, name,
|
||||
).Scan(&used, &already); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if already.Valid {
|
||||
return true, nil // already stored for this account — costs nothing more
|
||||
}
|
||||
return used.Int64+size <= maxUserBytes, nil
|
||||
}
|
||||
|
||||
// owns reports whether userID has a claim on a stored image.
|
||||
func (h *Handler) owns(name, userID string) bool {
|
||||
var ok bool
|
||||
if err := h.db.QueryRow(
|
||||
`SELECT EXISTS(SELECT 1 FROM images WHERE name = ? AND user_id = ?)`, name, userID,
|
||||
).Scan(&ok); err != nil {
|
||||
log.Printf("images: ownership check failed for %s: %v", name, err)
|
||||
return false
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
// safeName rejects anything that isn't a bare filename, so a request can't walk
|
||||
// out of the storage directory.
|
||||
func safeName(name string) (string, bool) {
|
||||
if name == "" || name != filepath.Base(name) || strings.ContainsAny(name, `/\`) {
|
||||
return "", false
|
||||
}
|
||||
return name, true
|
||||
}
|
||||
|
||||
// looksLikeSVG does a cheap check for an <svg root tag near the start of the
|
||||
// file, since DetectContentType doesn't recognize SVG.
|
||||
func looksLikeSVG(data []byte) bool {
|
||||
|
||||
+258
-31
@@ -6,8 +6,13 @@ import (
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// a 1x1 transparent PNG.
|
||||
@@ -19,6 +24,39 @@ var pngBytes = []byte{
|
||||
0x42, 0x60, 0x82,
|
||||
}
|
||||
|
||||
// another 1x1 PNG, differing in one pixel byte, so it hashes elsewhere.
|
||||
var otherPNG = append(append([]byte{}, pngBytes[:len(pngBytes)-8]...),
|
||||
0x01, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44)
|
||||
|
||||
// newStore returns a handler over a fresh directory and database, plus a router
|
||||
// per user: identical but for who the auth middleware says is calling. Two users
|
||||
// over one store is the situation that ownership exists to handle.
|
||||
func newStore(t *testing.T) (dir string, alice, bob http.Handler) {
|
||||
t.Helper()
|
||||
dir = t.TempDir()
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
|
||||
if _, err := database.Exec(
|
||||
`INSERT INTO users (id, email, display_name) VALUES (?, ?, ?)`,
|
||||
"bob", "bob@petal.local", "Bob",
|
||||
); err != nil {
|
||||
t.Fatalf("seed second user: %v", err)
|
||||
}
|
||||
|
||||
h, err := New(dir, database.DB, db.LocalUserID)
|
||||
if err != nil {
|
||||
t.Fatalf("new store: %v", err)
|
||||
}
|
||||
mount := func(userID string) http.Handler {
|
||||
return auth.Middleware(auth.StaticResolver(userID))(h.Routes())
|
||||
}
|
||||
return dir, mount(db.LocalUserID), mount("bob")
|
||||
}
|
||||
|
||||
func uploadReq(t *testing.T, field string, data []byte) *http.Request {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
@@ -34,16 +72,11 @@ func uploadReq(t *testing.T, field string, data []byte) *http.Request {
|
||||
return req
|
||||
}
|
||||
|
||||
func TestUploadAndServe(t *testing.T) {
|
||||
h, err := New(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := h.Routes()
|
||||
|
||||
// Upload a PNG → expect a JSON url under /api/images/.
|
||||
// upload posts an image and returns its stored name.
|
||||
func upload(t *testing.T, h http.Handler, data []byte) string {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec, uploadReq(t, "image", pngBytes))
|
||||
h.ServeHTTP(rec, uploadReq(t, "image", data))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("upload code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
@@ -54,44 +87,238 @@ func TestUploadAndServe(t *testing.T) {
|
||||
if !strings.HasPrefix(resp.URL, "/api/images/") || !strings.HasSuffix(resp.URL, ".png") {
|
||||
t.Fatalf("unexpected url %q", resp.URL)
|
||||
}
|
||||
return strings.TrimPrefix(resp.URL, "/api/images/")
|
||||
}
|
||||
|
||||
func get(t *testing.T, h http.Handler, name string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/"+name, nil))
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestUploadAndServe(t *testing.T) {
|
||||
_, alice, _ := newStore(t)
|
||||
|
||||
name := upload(t, alice, pngBytes)
|
||||
|
||||
// The same content uploaded again dedupes to the same URL.
|
||||
rec2 := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec2, uploadReq(t, "image", pngBytes))
|
||||
var resp2 struct{ URL string }
|
||||
json.Unmarshal(rec2.Body.Bytes(), &resp2)
|
||||
if resp2.URL != resp.URL {
|
||||
t.Fatalf("expected dedup to same url, got %q vs %q", resp2.URL, resp.URL)
|
||||
if again := upload(t, alice, pngBytes); again != name {
|
||||
t.Fatalf("expected dedup to same name, got %q vs %q", again, name)
|
||||
}
|
||||
|
||||
// Fetch it back.
|
||||
name := strings.TrimPrefix(resp.URL, "/api/images/")
|
||||
rec3 := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec3, httptest.NewRequest(http.MethodGet, "/"+name, nil))
|
||||
if rec3.Code != http.StatusOK {
|
||||
t.Fatalf("serve code=%d", rec3.Code)
|
||||
rec := get(t, alice, name)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("serve code=%d", rec.Code)
|
||||
}
|
||||
if !bytes.Equal(rec3.Body.Bytes(), pngBytes) {
|
||||
if !bytes.Equal(rec.Body.Bytes(), pngBytes) {
|
||||
t.Fatal("served bytes differ from uploaded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRejectsNonImage(t *testing.T) {
|
||||
h, _ := New(t.TempDir())
|
||||
r := h.Routes()
|
||||
// The point of the ownership table: a hash is not a capability.
|
||||
func TestImageIsolation(t *testing.T) {
|
||||
_, alice, bob := newStore(t)
|
||||
name := upload(t, alice, pngBytes)
|
||||
|
||||
if rec := get(t, bob, name); rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("bob fetched alice's image: code=%d", rec.Code)
|
||||
}
|
||||
|
||||
// Nor can he delete it out from under her.
|
||||
rec := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec, uploadReq(t, "image", []byte("this is plainly not an image at all")))
|
||||
bob.ServeHTTP(rec, httptest.NewRequest(http.MethodDelete, "/"+name, nil))
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("bob deleted alice's image: code=%d", rec.Code)
|
||||
}
|
||||
if got := get(t, alice, name); got.Code != http.StatusOK {
|
||||
t.Fatalf("alice's image disappeared: code=%d", got.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplication has to survive ownership: one file, one row each.
|
||||
func TestDedupAcrossUsers(t *testing.T) {
|
||||
dir, alice, bob := newStore(t)
|
||||
|
||||
name := upload(t, alice, pngBytes)
|
||||
if bobName := upload(t, bob, pngBytes); bobName != name {
|
||||
t.Fatalf("expected the same stored name, got %q vs %q", bobName, name)
|
||||
}
|
||||
|
||||
entries, _ := os.ReadDir(dir)
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("expected 1 file on disk, found %d", len(entries))
|
||||
}
|
||||
for _, h := range []http.Handler{alice, bob} {
|
||||
if rec := get(t, h, name); rec.Code != http.StatusOK {
|
||||
t.Fatalf("owner could not fetch shared image: code=%d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Alice dropping her copy must not take Bob's picture away with it.
|
||||
rec := httptest.NewRecorder()
|
||||
alice.ServeHTTP(rec, httptest.NewRequest(http.MethodDelete, "/"+name, nil))
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("delete code=%d", rec.Code)
|
||||
}
|
||||
if got := get(t, alice, name); got.Code != http.StatusNotFound {
|
||||
t.Fatalf("alice still sees a deleted image: code=%d", got.Code)
|
||||
}
|
||||
if got := get(t, bob, name); got.Code != http.StatusOK {
|
||||
t.Fatalf("bob lost his image when alice deleted hers: code=%d", got.Code)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, name)); err != nil {
|
||||
t.Fatalf("file removed while still owned: %v", err)
|
||||
}
|
||||
|
||||
// The last owner leaving takes the file with them.
|
||||
rec2 := httptest.NewRecorder()
|
||||
bob.ServeHTTP(rec2, httptest.NewRequest(http.MethodDelete, "/"+name, nil))
|
||||
if rec2.Code != http.StatusNoContent {
|
||||
t.Fatalf("delete code=%d", rec2.Code)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, name)); !os.IsNotExist(err) {
|
||||
t.Fatalf("file survived its last owner: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Images that predate ownership must not vanish from documents that use them.
|
||||
func TestBackfillClaimsExistingFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
defer database.Close()
|
||||
|
||||
orphan := "deadbeefdeadbeefdeadbeefdeadbeef.png"
|
||||
if err := os.WriteFile(filepath.Join(dir, orphan), pngBytes, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h, err := New(dir, database.DB, db.LocalUserID)
|
||||
if err != nil {
|
||||
t.Fatalf("new store: %v", err)
|
||||
}
|
||||
alice := auth.Middleware(auth.StaticResolver(db.LocalUserID))(h.Routes())
|
||||
if rec := get(t, alice, orphan); rec.Code != http.StatusOK {
|
||||
t.Fatalf("pre-existing image not claimed: code=%d", rec.Code)
|
||||
}
|
||||
|
||||
// Re-running the backfill (i.e. a restart) must not double up or reassign.
|
||||
if _, err := New(dir, database.DB, "bob"); err != nil {
|
||||
t.Fatalf("second backfill: %v", err)
|
||||
}
|
||||
|
||||
// And an owner who no longer exists — which is what the `local` account
|
||||
// becomes once it has been migrated onto a real one — must be skipped, not
|
||||
// turned into a foreign-key error that takes startup down with it.
|
||||
if _, err := New(dir, database.DB, "nobody-at-all"); err != nil {
|
||||
t.Fatalf("backfill for a missing owner should be a no-op, got: %v", err)
|
||||
}
|
||||
var owners int
|
||||
if err := database.QueryRow(`SELECT COUNT(*) FROM images WHERE name = ?`, orphan).Scan(&owners); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if owners != 1 {
|
||||
t.Fatalf("expected the backfill to be idempotent, got %d owners", owners)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRejectsNonImage(t *testing.T) {
|
||||
_, alice, _ := newStore(t)
|
||||
rec := httptest.NewRecorder()
|
||||
alice.ServeHTTP(rec, uploadReq(t, "image", []byte("this is plainly not an image at all")))
|
||||
if rec.Code != http.StatusUnsupportedMediaType {
|
||||
t.Fatalf("expected 415, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeMissing(t *testing.T) {
|
||||
h, _ := New(t.TempDir())
|
||||
r := h.Routes()
|
||||
rec := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/deadbeef.png", nil))
|
||||
if rec.Code != http.StatusNotFound {
|
||||
_, alice, _ := newStore(t)
|
||||
if rec := get(t, alice, "deadbeef.png"); rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("expected 404, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// An SVG is a document, not a picture: it can carry <script>, and this route
|
||||
// serves it from Petal's own origin. Rendered through an <img> that script
|
||||
// never runs, but "open image in new tab" is one click away, and there it
|
||||
// would — with the API of whoever opened it. Every stored image therefore
|
||||
// answers with a CSP that permits nothing.
|
||||
func TestStoredImagesAreServedInert(t *testing.T) {
|
||||
_, alice, _ := newStore(t)
|
||||
|
||||
svg := []byte(`<svg xmlns="http://www.w3.org/2000/svg"><script>fetch('/api/docs')</script></svg>`)
|
||||
rec := httptest.NewRecorder()
|
||||
alice.ServeHTTP(rec, uploadReq(t, "image", svg))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("svg upload code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
var resp struct{ URL string }
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := strings.TrimPrefix(resp.URL, "/api/images/")
|
||||
|
||||
got := get(t, alice, name)
|
||||
if got.Code != http.StatusOK {
|
||||
t.Fatalf("serve code=%d", got.Code)
|
||||
}
|
||||
csp := got.Header().Get("Content-Security-Policy")
|
||||
if !strings.Contains(csp, "default-src 'none'") || !strings.Contains(csp, "sandbox") {
|
||||
t.Fatalf("CSP %q does not neutralize the response", csp)
|
||||
}
|
||||
if got.Header().Get("X-Content-Type-Options") != "nosniff" {
|
||||
t.Fatal("stored images must be served nosniff")
|
||||
}
|
||||
}
|
||||
|
||||
// A per-upload cap bounds one careless paste; nothing bounded ten thousand of
|
||||
// them, on the same volume the database lives on.
|
||||
func TestUploadQuota(t *testing.T) {
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
if _, err := database.Exec(
|
||||
`INSERT INTO users (id, email, display_name) VALUES (?, ?, ?)`,
|
||||
"bob", "bob@petal.local", "Bob",
|
||||
); err != nil {
|
||||
t.Fatalf("seed second user: %v", err)
|
||||
}
|
||||
h, err := New(t.TempDir(), database.DB, db.LocalUserID)
|
||||
if err != nil {
|
||||
t.Fatalf("new store: %v", err)
|
||||
}
|
||||
alice := auth.Middleware(auth.StaticResolver(db.LocalUserID))(h.Routes())
|
||||
bob := auth.Middleware(auth.StaticResolver("bob"))(h.Routes())
|
||||
|
||||
// Fill Alice's allowance by hand — uploading a gibibyte in a test would be
|
||||
// absurd, and what's under test is the accounting, not the arithmetic.
|
||||
name := upload(t, alice, pngBytes)
|
||||
if _, err := database.Exec(
|
||||
`UPDATE images SET size = ? WHERE user_id = ? AND name = ?`,
|
||||
int64(maxUserBytes), db.LocalUserID, name,
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Re-storing something she already has costs nothing, so it still works.
|
||||
if again := upload(t, alice, pngBytes); again != name {
|
||||
t.Fatalf("a re-upload of an owned image should dedupe, got %q", again)
|
||||
}
|
||||
|
||||
// Anything new does not.
|
||||
rec := httptest.NewRecorder()
|
||||
alice.ServeHTTP(rec, uploadReq(t, "image", otherPNG))
|
||||
if rec.Code != http.StatusInsufficientStorage {
|
||||
t.Fatalf("over-quota upload code=%d, want 507", rec.Code)
|
||||
}
|
||||
|
||||
// And it is *her* allowance, not the store's: Bob is unaffected.
|
||||
if got := upload(t, bob, otherPNG); got == "" {
|
||||
t.Fatal("one writer's quota must not stop another writing")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,3 +36,13 @@ var glossGz []byte
|
||||
//
|
||||
//go:embed data/phonetic.json.gz
|
||||
var phoneticGz []byte
|
||||
|
||||
// hanziGz is the gzipped Chinese→English map: simplified headword → [[pinyin,
|
||||
// senses], …]. Built from CC-CEDICT (scripts/build_cedict.py), unfiltered — the
|
||||
// word a learner stops on is the one they do not know, so this is the one
|
||||
// dataset here with no frequency gate. Loaded on its own sync.Once (see
|
||||
// hanzi.go), not with the four above, because only a learner-direction account
|
||||
// ever asks for it.
|
||||
//
|
||||
//go:embed data/hanzi.json.gz
|
||||
var hanziGz []byte
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,333 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/prosolis/dreamdict/dictionary"
|
||||
)
|
||||
|
||||
// DreamDict is a read-only handle on a built dict.db — one SQLite file holding
|
||||
// English, French, European Portuguese, Spanish and Mandarin.
|
||||
//
|
||||
// It is a second database beside petal.db and is never written to: the file is
|
||||
// built by DreamDict's own import CLI a few times a year, and Petal only reads
|
||||
// it. That is what makes importing the package the right shape rather than
|
||||
// running DreamDict as a service — a hover gloss should not depend on a second
|
||||
// process being up, still less on one reachable across a VPN.
|
||||
type DreamDict struct {
|
||||
d *dictionary.Dictionary
|
||||
}
|
||||
|
||||
// OpenDreamDict opens dict.db read-only.
|
||||
//
|
||||
// A missing file returns (nil, nil), not an error. Petal is expected to run
|
||||
// without dict.db — a laptop checkout has never had one, and the zh pair does
|
||||
// not need one — so "the file isn't there" is a deployment state the caller
|
||||
// handles by carrying on. A file that is *present but unusable* (corrupt, or
|
||||
// never imported) does return an error, because that one is a mistake someone
|
||||
// should hear about.
|
||||
func OpenDreamDict(path string) (*DreamDict, error) {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
d, err := dictionary.NewReadOnly(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &DreamDict{d: d}, nil
|
||||
}
|
||||
|
||||
// Close releases the dict.db handle. Safe on a nil DreamDict, so a caller that
|
||||
// never got one can defer it unconditionally.
|
||||
func (dd *DreamDict) Close() error {
|
||||
if dd == nil {
|
||||
return nil
|
||||
}
|
||||
return dd.d.Close()
|
||||
}
|
||||
|
||||
// Contents reports how many words the open dict.db holds per language, so
|
||||
// startup can log what it actually got.
|
||||
//
|
||||
// It counts rows rather than returning DreamDict's list of supported languages.
|
||||
// Those are not the same thing and the difference is the whole point: a
|
||||
// database built before Spanish existed still *supports* Spanish, and a log
|
||||
// line naming the supported set would have said so cheerfully while every
|
||||
// Spanish lookup came back empty. Counting rows is the question worth asking of
|
||||
// a file somebody had to copy onto the box by hand.
|
||||
func (dd *DreamDict) Contents() string {
|
||||
counts, err := dd.d.WordCount()
|
||||
if err != nil {
|
||||
return "unreadable: " + err.Error()
|
||||
}
|
||||
langs := make([]string, 0, len(counts))
|
||||
for lang := range counts {
|
||||
langs = append(langs, lang)
|
||||
}
|
||||
sort.Strings(langs)
|
||||
parts := make([]string, 0, len(langs))
|
||||
for _, lang := range langs {
|
||||
parts = append(parts, fmt.Sprintf("%s=%d", lang, counts[lang]))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "no words"
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// dreamProvider serves one writer: English lookups from dict.db, glossed into
|
||||
// native. The struct is a value, created per request by [Set.For] — it holds no
|
||||
// state beyond the shared handle and the language to translate into.
|
||||
type dreamProvider struct {
|
||||
dict *DreamDict
|
||||
native string // the writer's language, e.g. "pt-PT"
|
||||
}
|
||||
|
||||
// maxEtymology caps the free-form Wiktionary etymology. It is the one field
|
||||
// with no natural length: some entries are a clause, some are four paragraphs
|
||||
// tracing a word through three dead languages. The popover wants a line.
|
||||
const maxEtymology = 220
|
||||
|
||||
// Lookup fills a Result from dict.db.
|
||||
//
|
||||
// The word is de-inflected with the same [candidates] walk the embedded
|
||||
// datasets use, because dict.db stores headwords: "running" has no definitions
|
||||
// row of its own. The first candidate that *has* definitions becomes the
|
||||
// headword every other field is then read from, so a single popover never
|
||||
// mixes "running"'s frequency with "run"'s definitions.
|
||||
//
|
||||
// The gloss is walked separately. A word can be absent from the definitions
|
||||
// table and still have a translation (and vice versa), and the hover tooltip
|
||||
// asks for the gloss alone — so tying it to the definition headword would lose
|
||||
// glosses for no benefit.
|
||||
func (p dreamProvider) Lookup(word string) (Result, error) {
|
||||
res := Result{Word: word, Definitions: []Meaning{}, Synonyms: []string{}, Difficulty: unknownDifficulty}
|
||||
norm := strings.ToLower(strings.TrimSpace(word))
|
||||
if norm == "" {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
head := norm
|
||||
for _, c := range candidates(norm) {
|
||||
defs, err := p.dict.d.Define(c, langEN)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if len(defs) == 0 {
|
||||
continue
|
||||
}
|
||||
head = c
|
||||
for _, d := range defs {
|
||||
// DreamDict orders by source priority, so the curated senses
|
||||
// (WordNet, WOLF) are already ahead of the Wiktionary tail — taking
|
||||
// the first few is taking the best few.
|
||||
res.Definitions = append(res.Definitions, Meaning{PartOfSpeech: d.POS, Definition: d.Gloss})
|
||||
if len(res.Definitions) >= maxDefinitions {
|
||||
break
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
gloss, err := p.translate(norm)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
res.Gloss = gloss
|
||||
|
||||
syns, err := p.dict.d.Synonyms(head, langEN)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if len(syns) > maxSynonyms {
|
||||
syns = syns[:maxSynonyms]
|
||||
}
|
||||
res.Synonyms = append(res.Synonyms, syns...)
|
||||
|
||||
prons, err := p.dict.d.Pronunciation(head, langEN)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
res.Phonetic = pickIPA(prons)
|
||||
|
||||
if res.Frequency, err = p.dict.d.Frequency(head, langEN); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if res.Difficulty, err = p.dict.d.Difficulty(head, langEN); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
ety, err := p.dict.d.Etymology(head, langEN)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
res.Etymology = trimEtymology(ety)
|
||||
|
||||
if res.Reverse, err = p.reverse(norm); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// reverse reads the token as a word of the writer's own language, and returns
|
||||
// nil when it isn't one — which is the answer for almost every word she looks
|
||||
// up, since she is writing English.
|
||||
//
|
||||
// The English de-inflection walk is deliberately *not* applied here. [candidates]
|
||||
// knows about -s, -ed and -ing; running it over Portuguese would turn "vinhas"
|
||||
// into "vinha" by an English rule that happens to be right and "cantava" into
|
||||
// nothing by rules that are simply irrelevant. dict.db stores headwords, so an
|
||||
// inflected Portuguese form finds nothing and the card shows only the English
|
||||
// reading — the same outcome as today, rather than a confidently wrong one.
|
||||
func (p dreamProvider) reverse(norm string) (*Reverse, error) {
|
||||
back, err := p.dict.d.Equivalents(norm, p.native, langEN)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defs, err := p.dict.d.Define(norm, p.native)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(back) == 0 && len(defs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rev := &Reverse{Lang: p.native}
|
||||
if len(back) > maxGlossSenses {
|
||||
back = back[:maxGlossSenses]
|
||||
}
|
||||
rev.Gloss = strings.Join(back, "; ")
|
||||
for _, d := range defs {
|
||||
rev.Definitions = append(rev.Definitions, Meaning{PartOfSpeech: d.POS, Definition: d.Gloss})
|
||||
if len(rev.Definitions) >= maxReverseDefinitions {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
prons, err := p.dict.d.Pronunciation(norm, p.native)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rev.Phonetic = pickIPA(prons)
|
||||
|
||||
return rev, nil
|
||||
}
|
||||
|
||||
// maxReverseDefinitions is smaller than [maxDefinitions]: the reverse reading is
|
||||
// the second half of a card that already has an English one, and it is there to
|
||||
// say "this is also a Portuguese word, and here is what it means" rather than to
|
||||
// be a dictionary entry in its own right.
|
||||
const maxReverseDefinitions = 2
|
||||
|
||||
// Gloss returns the writer's-language translation alone — the hover tooltip's
|
||||
// fast path, one indexed query per candidate form and nothing else.
|
||||
func (p dreamProvider) Gloss(word string) (GlossResult, error) {
|
||||
norm := strings.ToLower(strings.TrimSpace(word))
|
||||
if norm == "" {
|
||||
return GlossResult{Word: word}, nil
|
||||
}
|
||||
gloss, err := p.translate(norm)
|
||||
if err != nil {
|
||||
return GlossResult{}, err
|
||||
}
|
||||
res := GlossResult{Word: word, Gloss: gloss}
|
||||
|
||||
// The tooltip carries only the reverse *gloss*, not the whole reading: it is
|
||||
// a one-line bubble under a resting pointer, and the popover is one click
|
||||
// away for anyone who wants the rest.
|
||||
back, err := p.dict.d.Equivalents(norm, p.native, langEN)
|
||||
if err != nil {
|
||||
return GlossResult{}, err
|
||||
}
|
||||
if len(back) > maxGlossSenses {
|
||||
back = back[:maxGlossSenses]
|
||||
}
|
||||
res.Reverse = strings.Join(back, "; ")
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// maxGlossSenses caps how many translations are strung together. One is often
|
||||
// too thin to disambiguate; the whole list is a wall of words in a tooltip.
|
||||
const maxGlossSenses = 3
|
||||
|
||||
// translate walks the candidate forms and returns the first that has an
|
||||
// equivalent in the writer's language, joined into one line.
|
||||
//
|
||||
// It asks for Equivalents rather than Translate on the strength of measuring
|
||||
// both against the real dict.db: Wiktionary's en→pt-PT translation table
|
||||
// answers for 17% of the 2,000 commonest English words, and the shared-synset
|
||||
// path answers for 62%. The plan assumed Translate would do — the database
|
||||
// says otherwise, and a gloss that is absent five times out of six is not a
|
||||
// gloss. Equivalents falls back to Translate internally, so nothing is lost.
|
||||
//
|
||||
// A language dict.db was built without simply has no rows, so this returns "" —
|
||||
// which is exactly what an unglossed word returns, and the popover already
|
||||
// renders that case. Spanish was precisely this until the database was rebuilt
|
||||
// with it on 2026-07-27; the code path did not change, the file did.
|
||||
func (p dreamProvider) translate(norm string) (string, error) {
|
||||
for _, c := range candidates(norm) {
|
||||
trs, err := p.dict.d.Equivalents(c, langEN, p.native)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(trs) == 0 {
|
||||
continue
|
||||
}
|
||||
if len(trs) > maxGlossSenses {
|
||||
trs = trs[:maxGlossSenses]
|
||||
}
|
||||
return strings.Join(trs, "; "), nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// pickIPA chooses what to show beside the read-aloud button. IPA is the only
|
||||
// form worth showing a learner — CMU's "IH0 F EH1 M ER0 AH0 L" is a machine
|
||||
// format, and printing it would be noise dressed up as help. If there's no IPA,
|
||||
// there's no phonetic line.
|
||||
func pickIPA(prons []dictionary.Pronunciation) string {
|
||||
for _, p := range prons {
|
||||
if strings.EqualFold(p.Format, "ipa") && strings.TrimSpace(p.Value) != "" {
|
||||
return strings.Trim(strings.TrimSpace(p.Value), "/[]")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// trimEtymology cuts Wiktionary's prose down to a line, preferring to stop at a
|
||||
// sentence boundary so the result reads as a finished thought rather than a
|
||||
// truncation.
|
||||
func trimEtymology(text string) string {
|
||||
text = strings.Join(strings.Fields(text), " ")
|
||||
if utf8.RuneCountInString(text) <= maxEtymology {
|
||||
return text
|
||||
}
|
||||
// Counted and cut in runes, not bytes. An etymology is the one field that
|
||||
// is *mostly* not English — ἐφήμερος, ephemerus, 短暫 — and a byte slice
|
||||
// through the middle of one of those characters is invalid UTF-8 in the
|
||||
// JSON response.
|
||||
cut := string([]rune(text)[:maxEtymology-1])
|
||||
// Stop at a sentence when one ends late enough to be worth keeping. An
|
||||
// early full stop ("From Latin. …") is not a summary, it's a discarded
|
||||
// paragraph, so that case falls through to the word-boundary cut.
|
||||
if i := strings.LastIndex(cut, ". "); i > len(cut)/2 {
|
||||
return cut[:i+1]
|
||||
}
|
||||
if i := strings.LastIndex(cut, " "); i > 0 {
|
||||
cut = cut[:i]
|
||||
}
|
||||
return cut + "…"
|
||||
}
|
||||
@@ -0,0 +1,664 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/prosolis/dreamdict/dictionary"
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// The fixture is a real dict.db on disk rather than an in-memory handle, so
|
||||
// these tests exercise the path production takes: stat the file, open it
|
||||
// read-only, find it seeded. A fake would have skipped every one of those.
|
||||
//
|
||||
// "ephemeral" is the worked example throughout: it has definitions only under
|
||||
// its own headword, translations into two languages, IPA alongside a CMU
|
||||
// pronunciation Petal must not show, a frequency, a difficulty and an etymology
|
||||
// long enough to need trimming.
|
||||
func writeFixture(t *testing.T, seeded bool) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "dict.db")
|
||||
sqldb, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatalf("open fixture: %v", err)
|
||||
}
|
||||
defer sqldb.Close()
|
||||
if err := dictionary.BootstrapSchema(sqldb); err != nil {
|
||||
t.Fatalf("bootstrap: %v", err)
|
||||
}
|
||||
if !seeded {
|
||||
return path
|
||||
}
|
||||
|
||||
exec := func(q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := sqldb.Exec(q, args...); err != nil {
|
||||
t.Fatalf("seed %q: %v", q, err)
|
||||
}
|
||||
}
|
||||
exec(`INSERT INTO meta (key, value) VALUES ('schema_version', '2')`)
|
||||
|
||||
exec(`INSERT INTO words (id, word, lang, pos, frequency, difficulty) VALUES
|
||||
(1, 'ephemeral', 'en', 'adjective', 50, 0.72),
|
||||
(2, 'run', 'en', 'verb', 900, 0.05),
|
||||
(3, 'plain', 'en', 'adjective', 0, NULL),
|
||||
(4, 'efémero', 'pt-PT', 'adjective', 12, 0.6)`)
|
||||
|
||||
exec(`INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES
|
||||
(1, 'adjective', 'lasting a very short time', 'wordnet', 10),
|
||||
(1, 'adjective', 'short-lived', 'wiktionary', 99),
|
||||
(1, 'adjective', 'transitory', 'wiktionary', 99),
|
||||
(1, 'adjective', 'fleeting', 'wiktionary', 99),
|
||||
(1, 'adjective', 'evanescent', 'wiktionary', 99),
|
||||
(2, 'verb', 'move fast on foot', 'wordnet', 10),
|
||||
(3, 'adjective', 'without decoration', 'wordnet', 10)`)
|
||||
|
||||
exec(`INSERT INTO synonyms (word_id, synonym, source) VALUES
|
||||
(1, 'fleeting', 'wordnet'), (1, 'transient', 'wordnet'),
|
||||
(2, 'sprint', 'wordnet')`)
|
||||
|
||||
exec(`INSERT INTO translations (word_id, translation, target_lang, source) VALUES
|
||||
(1, 'efémero', 'pt-PT', 'kaikki'),
|
||||
(1, 'passageiro','pt-PT', 'kaikki'),
|
||||
(1, 'éphémère', 'fr', 'kaikki'),
|
||||
(1, '短暂的', 'zh', 'cedict'),
|
||||
(2, 'correr', 'pt-PT', 'kaikki')`)
|
||||
|
||||
// CMU is listed first deliberately: picking the first row would show a
|
||||
// learner "IH0 F EH1 M ER0 AH0 L", which is a machine format, not help.
|
||||
exec(`INSERT INTO pronunciations (word_id, format, value, source) VALUES
|
||||
(1, 'cmu', 'IH0 F EH1 M ER0 AH0 L', 'cmudict'),
|
||||
(1, 'ipa', '/ɪˈfɛm.ər.əl/', 'wiktionary')`)
|
||||
|
||||
// "brief" carries no translation row at all — only a shared WordNet synset
|
||||
// with two pt-PT words. On the real database that is the *usual* case, not
|
||||
// the exotic one, so Petal must reach a gloss this way or the pt-PT pair
|
||||
// has almost no glosses. "breve" is the commoner of the two and leads.
|
||||
exec(`INSERT INTO words (id, word, lang, pos, frequency) VALUES
|
||||
(5, 'brief', 'en', 'adjective', 400),
|
||||
(6, 'breve', 'pt-PT', 'adjective', 300),
|
||||
(7, 'sucinto', 'pt-PT', 'adjective', 20)`)
|
||||
exec(`INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES
|
||||
(5, 'adjective', 'of short duration', 'wordnet', 10)`)
|
||||
exec(`INSERT INTO synsets (id, synset_id, pos) VALUES (1, '00751145-a', 'adjective')`)
|
||||
exec(`INSERT INTO word_synsets (word_id, synset_id, source) VALUES
|
||||
(5, 1, 'wordnet'), (6, 1, 'omw'), (7, 1, 'omw')`)
|
||||
|
||||
// "data" is the collision the Latin pairs create and the zh pair never did:
|
||||
// a real English word and a real Portuguese one, spelled identically and
|
||||
// meaning different things. There is no honest way to look at it in a mixed
|
||||
// document and know which was meant, so Petal shows both readings.
|
||||
exec(`INSERT INTO words (id, word, lang, pos, frequency) VALUES
|
||||
(8, 'data', 'en', 'noun', 800),
|
||||
(9, 'data', 'pt-PT', 'noun', 700),
|
||||
(10, 'date', 'en', 'noun', 750)`)
|
||||
exec(`INSERT INTO definitions (word_id, pos, gloss, source, priority) VALUES
|
||||
(8, 'noun', 'facts collected for reference', 'wordnet', 10),
|
||||
(9, 'noun', 'dia do mês', 'wiktionary', 20),
|
||||
(9, 'noun', 'momento no tempo', 'wiktionary', 30),
|
||||
(9, 'noun', 'um terceiro sentido', 'wiktionary', 40)`)
|
||||
exec(`INSERT INTO translations (word_id, translation, target_lang, source) VALUES
|
||||
(9, 'date', 'en', 'kaikki')`)
|
||||
exec(`INSERT INTO pronunciations (word_id, format, value, source) VALUES
|
||||
(9, 'ipa', '/ˈdatɐ/', 'wiktionary')`)
|
||||
|
||||
exec(`INSERT INTO etymology (word_id, text, source) VALUES
|
||||
(1, 'From Medieval Latin ephemerus, from Ancient Greek ἐφήμερος (ephḗmeros, "lasting only a day"), from ἐπί (epí, "upon") and ἡμέρα (hēméra, "day"). The sense of transience is attested in English from the late sixteenth century onwards.', 'wiktionary')`)
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
func openFixture(t *testing.T) *DreamDict {
|
||||
t.Helper()
|
||||
dd, err := OpenDreamDict(writeFixture(t, true))
|
||||
if err != nil {
|
||||
t.Fatalf("OpenDreamDict: %v", err)
|
||||
}
|
||||
if dd == nil {
|
||||
t.Fatal("OpenDreamDict returned no dictionary for a seeded file")
|
||||
}
|
||||
t.Cleanup(func() { dd.Close() })
|
||||
return dd
|
||||
}
|
||||
|
||||
func TestOpenMissingFileIsNotAnError(t *testing.T) {
|
||||
dd, err := OpenDreamDict(filepath.Join(t.TempDir(), "absent.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("a missing dict.db must not be an error: %v", err)
|
||||
}
|
||||
if dd != nil {
|
||||
t.Fatal("a missing dict.db must yield no dictionary")
|
||||
}
|
||||
// An unset path is the laptop default and must behave the same way.
|
||||
if dd, err := OpenDreamDict(""); err != nil || dd != nil {
|
||||
t.Fatalf(`OpenDreamDict("") = %v, %v; want nil, nil`, dd, err)
|
||||
}
|
||||
// Close on the nil handle is what main.go defers unconditionally.
|
||||
if err := dd.Close(); err != nil {
|
||||
t.Fatalf("Close on absent dictionary: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenPresentButUnseededIsAnError(t *testing.T) {
|
||||
// A file that exists but was never imported is somebody's mistake — a
|
||||
// half-finished deploy — and must be loud, unlike a file that isn't there.
|
||||
dd, err := OpenDreamDict(writeFixture(t, false))
|
||||
if err == nil {
|
||||
dd.Close()
|
||||
t.Fatal("an unseeded dict.db must report an error")
|
||||
}
|
||||
if dd != nil {
|
||||
t.Fatal("an unseeded dict.db must not yield a usable dictionary")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenUnreadablePathIsAnError(t *testing.T) {
|
||||
// Not a missing file: a directory where dict.db should be. Distinguishing
|
||||
// this from ErrNotExist is the whole point of the stat.
|
||||
dir := filepath.Join(t.TempDir(), "dict.db")
|
||||
if err := os.Mkdir(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := OpenDreamDict(dir); err == nil {
|
||||
t.Fatal("a directory in place of dict.db must report an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDreamLookupFillsEveryField(t *testing.T) {
|
||||
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||
res, err := p.Lookup("ephemeral")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if res.Gloss != "efémero; passageiro" {
|
||||
t.Errorf("Gloss = %q, want the pt-PT translations joined", res.Gloss)
|
||||
}
|
||||
if res.Phonetic != "ɪˈfɛm.ər.əl" {
|
||||
t.Errorf("Phonetic = %q, want the IPA without its slashes", res.Phonetic)
|
||||
}
|
||||
if len(res.Definitions) != maxDefinitions {
|
||||
t.Fatalf("Definitions = %d, want them capped at %d", len(res.Definitions), maxDefinitions)
|
||||
}
|
||||
if res.Definitions[0].Definition != "lasting a very short time" {
|
||||
t.Errorf("first definition = %q, want the curated (wordnet) sense first",
|
||||
res.Definitions[0].Definition)
|
||||
}
|
||||
if res.Definitions[0].PartOfSpeech != "adjective" {
|
||||
t.Errorf("part of speech = %q, want adjective", res.Definitions[0].PartOfSpeech)
|
||||
}
|
||||
if len(res.Synonyms) != 2 {
|
||||
t.Errorf("Synonyms = %v, want both", res.Synonyms)
|
||||
}
|
||||
if res.Frequency != 50 {
|
||||
t.Errorf("Frequency = %d, want 50", res.Frequency)
|
||||
}
|
||||
if res.Difficulty != 0.72 {
|
||||
t.Errorf("Difficulty = %v, want 0.72", res.Difficulty)
|
||||
}
|
||||
if !strings.HasPrefix(res.Etymology, "From Medieval Latin ephemerus") {
|
||||
t.Errorf("Etymology = %q, want the Wiktionary text", res.Etymology)
|
||||
}
|
||||
if len(res.Etymology) > maxEtymology {
|
||||
t.Errorf("Etymology not trimmed: %d chars", len(res.Etymology))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDreamGlossFollowsTheWriterNotTheWord(t *testing.T) {
|
||||
dd := openFixture(t)
|
||||
for lang, want := range map[string]string{
|
||||
"pt-PT": "efémero; passageiro",
|
||||
"fr": "éphémère",
|
||||
"es": "", // DreamDict supports Spanish; this database wasn't built with it
|
||||
"de": "", // never a Petal pair, and must not silently borrow another's
|
||||
} {
|
||||
got, err := dreamProvider{dict: dd, native: lang}.Gloss("ephemeral")
|
||||
if err != nil {
|
||||
t.Fatalf("Gloss(%s): %v", lang, err)
|
||||
}
|
||||
if got.Gloss != want {
|
||||
t.Errorf("Gloss for %s = %q, want %q", lang, got.Gloss, want)
|
||||
}
|
||||
if got.Word != "ephemeral" {
|
||||
t.Errorf("Word = %q, want the word as asked", got.Word)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDreamGlossesThroughSharedSynsets(t *testing.T) {
|
||||
// The measurement that drove this: on the real dict.db, Wiktionary's
|
||||
// en→pt-PT translation table answers for 17% of the 2,000 commonest English
|
||||
// words and the shared-synset path answers for 62%. A word with no
|
||||
// translation row must still get a gloss, commonest sense first.
|
||||
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||
res, err := p.Lookup("brief")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if res.Gloss != "breve; sucinto" {
|
||||
t.Errorf("Gloss = %q, want the synset equivalents, commonest first", res.Gloss)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDreamDeinflectsToTheHeadword(t *testing.T) {
|
||||
// dict.db stores headwords: "running" has no row of its own. The candidate
|
||||
// walk is what makes a right-click on real prose work at all.
|
||||
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||
res, err := p.Lookup("running")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if len(res.Definitions) == 0 || res.Definitions[0].Definition != "move fast on foot" {
|
||||
t.Fatalf("Definitions = %+v, want run's", res.Definitions)
|
||||
}
|
||||
// Every other field must come from the same headword — a popover that mixed
|
||||
// "running"'s (absent) frequency with "run"'s definitions would be lying.
|
||||
if res.Frequency != 900 {
|
||||
t.Errorf("Frequency = %d, want run's 900", res.Frequency)
|
||||
}
|
||||
if res.Difficulty != 0.05 {
|
||||
t.Errorf("Difficulty = %v, want run's 0.05", res.Difficulty)
|
||||
}
|
||||
if res.Synonyms[0] != "sprint" {
|
||||
t.Errorf("Synonyms = %v, want run's", res.Synonyms)
|
||||
}
|
||||
if res.Gloss != "correr" {
|
||||
t.Errorf("Gloss = %q, want run's", res.Gloss)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDreamMissIsAnEmptyResultNotAnError(t *testing.T) {
|
||||
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||
res, err := p.Lookup("zzzxqqq")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if len(res.Definitions) != 0 || len(res.Synonyms) != 0 || res.Gloss != "" {
|
||||
t.Errorf("expected an empty result, got %+v", res)
|
||||
}
|
||||
// The frontend renders [] and never null.
|
||||
if res.Definitions == nil || res.Synonyms == nil {
|
||||
t.Errorf("empty slices must be non-nil: %+v", res)
|
||||
}
|
||||
if res.Difficulty != unknownDifficulty {
|
||||
t.Errorf("Difficulty = %v, want the unknown sentinel", res.Difficulty)
|
||||
}
|
||||
// Empty input is a miss, not a crash.
|
||||
if res, err := p.Lookup(" "); err != nil || res.Gloss != "" {
|
||||
t.Errorf("Lookup(blank) = %+v, %v", res, err)
|
||||
}
|
||||
if res, err := p.Gloss(""); err != nil || res.Gloss != "" {
|
||||
t.Errorf("Gloss(empty) = %+v, %v", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDreamUnscoredWordKeepsTheUnknownSentinel(t *testing.T) {
|
||||
// "plain" is in the database with no frequency and a NULL difficulty. The
|
||||
// popover must be able to tell that apart from "difficulty 0.0, the easiest
|
||||
// word there is" — which is why the sentinel is -1 and not omitempty.
|
||||
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||
res, err := p.Lookup("plain")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if len(res.Definitions) == 0 {
|
||||
t.Fatal("expected plain to be found")
|
||||
}
|
||||
if res.Difficulty != unknownDifficulty {
|
||||
t.Errorf("Difficulty = %v, want the unknown sentinel for a NULL score", res.Difficulty)
|
||||
}
|
||||
if res.Frequency != 0 {
|
||||
t.Errorf("Frequency = %d, want 0 for no count", res.Frequency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickIPASkipsMachineFormats(t *testing.T) {
|
||||
if got := pickIPA([]dictionary.Pronunciation{{Format: "cmu", Value: "K AE1 T"}}); got != "" {
|
||||
t.Errorf("pickIPA on CMU alone = %q, want empty — CMU is not for a reader", got)
|
||||
}
|
||||
if got := pickIPA(nil); got != "" {
|
||||
t.Errorf("pickIPA(nil) = %q", got)
|
||||
}
|
||||
if got := pickIPA([]dictionary.Pronunciation{{Format: "IPA", Value: " [kæt] "}}); got != "kæt" {
|
||||
t.Errorf("pickIPA = %q, want the bare IPA regardless of case or brackets", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimEtymologyPrefersASentence(t *testing.T) {
|
||||
// A sentence that ends past halfway is the good cut: keep it, drop the rest.
|
||||
long := strings.Repeat("padding word ", 14) + "end. " + strings.Repeat("more ", 40)
|
||||
got := trimEtymology(long)
|
||||
if utf8.RuneCountInString(got) > maxEtymology {
|
||||
t.Errorf("not trimmed: %d runes", utf8.RuneCountInString(got))
|
||||
}
|
||||
if !strings.HasSuffix(got, "end.") {
|
||||
t.Errorf("trimEtymology = %q, want it to stop at the sentence", got)
|
||||
}
|
||||
|
||||
// An early full stop is not a summary — cutting there would throw away
|
||||
// almost the whole line — so this falls through to a word boundary.
|
||||
got = trimEtymology("From Latin. " + strings.Repeat("padding word ", 40))
|
||||
if strings.HasSuffix(got, "Latin.") {
|
||||
t.Errorf("trimEtymology = %q, want more than the first four words", got)
|
||||
}
|
||||
if !strings.HasSuffix(got, "…") {
|
||||
t.Errorf("trimEtymology = %q, want an ellipsis when cut mid-thought", got)
|
||||
}
|
||||
|
||||
// Multi-byte text must be cut on rune boundaries: a byte slice through
|
||||
// ἐφήμερος would put invalid UTF-8 in the JSON.
|
||||
greek := trimEtymology(strings.Repeat("ἐφήμερος ", 60))
|
||||
if !utf8.ValidString(greek) {
|
||||
t.Errorf("trimEtymology produced invalid UTF-8: %q", greek)
|
||||
}
|
||||
if n := utf8.RuneCountInString(greek); n > maxEtymology {
|
||||
t.Errorf("trimmed to %d runes, want at most %d", n, maxEtymology)
|
||||
}
|
||||
|
||||
if strings.Contains(got, " ") || strings.Contains(trimEtymology("a\n b"), "\n") {
|
||||
t.Error("whitespace should be collapsed to a single line")
|
||||
}
|
||||
// Short text passes through untouched.
|
||||
if got := trimEtymology("From Old English."); got != "From Old English." {
|
||||
t.Errorf("trimEtymology = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- the Set: which provider answers, and what happens when dict.db is absent
|
||||
|
||||
func TestSetRoutesByPairLanguage(t *testing.T) {
|
||||
set := NewSet(openFixture(t))
|
||||
if !set.HasDreamDict() {
|
||||
t.Fatal("HasDreamDict = false with a dictionary open")
|
||||
}
|
||||
// zh — and an empty column, which is what a pre-auth row reads as — stays
|
||||
// on the embedded datasets until the two have been compared on real
|
||||
// lookups. This test is the guard on that decision.
|
||||
for _, lang := range []string{"", LangZh} {
|
||||
if _, ok := set.For(lang).(*Lexicon); !ok {
|
||||
t.Errorf("For(%q) = %T, want the embedded Lexicon", lang, set.For(lang))
|
||||
}
|
||||
}
|
||||
for _, lang := range []string{"pt-PT", "fr", "es"} {
|
||||
p, ok := set.For(lang).(dreamProvider)
|
||||
if !ok {
|
||||
t.Fatalf("For(%q) = %T, want DreamDict", lang, set.For(lang))
|
||||
}
|
||||
if p.native != lang {
|
||||
t.Errorf("For(%q) glosses into %q", lang, p.native)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetWithoutDictKeepsTheEnglishHalf(t *testing.T) {
|
||||
// The interesting degradation: dict.db never got deployed. A pt-PT writer
|
||||
// should still get definitions, synonyms and phonetics — all compiled into
|
||||
// the binary and all correct for her — and lose only the translation.
|
||||
set := NewSet(nil)
|
||||
if set.HasDreamDict() {
|
||||
t.Fatal("HasDreamDict = true with no dictionary")
|
||||
}
|
||||
p := set.For("pt-PT")
|
||||
res, err := p.Lookup("happy")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if len(res.Definitions) == 0 || len(res.Synonyms) == 0 {
|
||||
t.Error("expected the embedded English half to survive a missing dict.db")
|
||||
}
|
||||
if res.Gloss != "" {
|
||||
t.Errorf("Gloss = %q — a pt-PT writer must never be handed the Chinese gloss", res.Gloss)
|
||||
}
|
||||
g, err := p.Gloss("happy")
|
||||
if err != nil {
|
||||
t.Fatalf("Gloss: %v", err)
|
||||
}
|
||||
if g.Gloss != "" {
|
||||
t.Errorf("Gloss = %q, want empty", g.Gloss)
|
||||
}
|
||||
if g.Word != "happy" {
|
||||
t.Errorf("Word = %q, want the word as asked", g.Word)
|
||||
}
|
||||
// The zh writer is untouched by any of this.
|
||||
zh, err := set.For(LangZh).Lookup("happy")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup(zh): %v", err)
|
||||
}
|
||||
if zh.Gloss == "" {
|
||||
t.Error("the zh pair must keep its embedded gloss with no dict.db")
|
||||
}
|
||||
}
|
||||
|
||||
// --- the handler: the pair language is read per request, from the caller's row
|
||||
|
||||
func mountLexicon(t *testing.T, set *Set) (*chi.Mux, *db.DB) {
|
||||
t.Helper()
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "petal.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("db.Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
|
||||
for _, u := range []struct{ id, lang string }{
|
||||
{"alice", LangZh}, {"bob", "pt-PT"},
|
||||
} {
|
||||
if _, err := database.Exec(
|
||||
`INSERT INTO users (id, email, display_name, pair_lang) VALUES (?, ?, ?, ?)`,
|
||||
u.id, u.id+"@example.com", u.id, u.lang,
|
||||
); err != nil {
|
||||
t.Fatalf("seed user: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
h := NewHandler(database.DB, set)
|
||||
r := chi.NewMux()
|
||||
r.Mount("/word", h.Routes())
|
||||
r.Mount("/gloss", h.GlossRoutes())
|
||||
return r, database
|
||||
}
|
||||
|
||||
func getAs(t *testing.T, r http.Handler, userID, path string) Result {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
req = req.WithContext(auth.WithUser(req.Context(), userID))
|
||||
rec := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("GET %s as %s = %d: %s", path, userID, rec.Code, rec.Body)
|
||||
}
|
||||
var res Result
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &res); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func TestHandlerGlossesInTheCallersLanguage(t *testing.T) {
|
||||
r, _ := mountLexicon(t, NewSet(openFixture(t)))
|
||||
|
||||
// Same URL, two writers, two languages. This is why the response is no
|
||||
// longer cacheable as `public`.
|
||||
bob := getAs(t, r, "bob", "/word/ephemeral")
|
||||
if bob.Gloss != "efémero; passageiro" {
|
||||
t.Errorf("bob's gloss = %q, want pt-PT", bob.Gloss)
|
||||
}
|
||||
alice := getAs(t, r, "alice", "/word/ephemeral")
|
||||
if !strings.ContainsAny(alice.Gloss, "短暂的") && alice.Gloss != "" {
|
||||
// alice is on the embedded ECDICT dataset, not the fixture's zh row —
|
||||
// what matters is that she is *not* served bob's Portuguese.
|
||||
t.Logf("alice's embedded gloss: %q", alice.Gloss)
|
||||
}
|
||||
if alice.Gloss == bob.Gloss && bob.Gloss != "" {
|
||||
t.Error("the zh writer was served the pt-PT gloss")
|
||||
}
|
||||
if strings.Contains(alice.Gloss, "efémero") {
|
||||
t.Errorf("alice's gloss = %q, want the embedded Chinese one", alice.Gloss)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerUnknownCallerFallsBackRatherThanFailing(t *testing.T) {
|
||||
// No session, or a user row that has gone: the lookup still answers, from
|
||||
// the embedded datasets. A dictionary that fails closed would be worse than
|
||||
// one that answers in the wrong language, because nothing at all is not a
|
||||
// dictionary.
|
||||
r, _ := mountLexicon(t, NewSet(openFixture(t)))
|
||||
res := getAs(t, r, "nobody", "/word/happy")
|
||||
if len(res.Definitions) == 0 {
|
||||
t.Error("expected the embedded fallback to answer for an unknown caller")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerCachesPrivately(t *testing.T) {
|
||||
r, _ := mountLexicon(t, NewSet(nil))
|
||||
req := httptest.NewRequest(http.MethodGet, "/gloss/happy", nil)
|
||||
req = req.WithContext(auth.WithUser(req.Context(), "alice"))
|
||||
rec := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec, req)
|
||||
if got := rec.Header().Get("Cache-Control"); !strings.HasPrefix(got, "private") {
|
||||
t.Errorf("Cache-Control = %q — a per-writer gloss must not go in a shared cache", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerDecodesPunctuatedWords(t *testing.T) {
|
||||
r, _ := mountLexicon(t, NewSet(openFixture(t)))
|
||||
res := getAs(t, r, "bob", "/word/"+"caf%C3%A9")
|
||||
if res.Word != "café" {
|
||||
t.Errorf("Word = %q, want the decoded word", res.Word)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentsCountsRowsNotSupportedLanguages(t *testing.T) {
|
||||
// The fixture is seeded with English and pt-PT only. DreamDict *supports*
|
||||
// French, Spanish and Chinese too — and a startup line that reported the
|
||||
// supported set would have named all five while every French lookup came
|
||||
// back empty. That is the failure this log line exists to catch, so it must
|
||||
// count rows.
|
||||
got := NewSet(openFixture(t))
|
||||
summary := got.Contents()
|
||||
if !strings.Contains(summary, "en=") || !strings.Contains(summary, "pt-PT=") {
|
||||
t.Errorf("Contents = %q, want the languages the fixture actually holds", summary)
|
||||
}
|
||||
for _, absent := range []string{"fr=", "es=", "zh="} {
|
||||
if strings.Contains(summary, absent) {
|
||||
t.Errorf("Contents = %q, must not name %q — no rows exist for it", summary, absent)
|
||||
}
|
||||
}
|
||||
// No dictionary at all still has to answer something printable.
|
||||
if s := NewSet(nil).Contents(); s == "" {
|
||||
t.Error("Contents with no dictionary must still say something")
|
||||
}
|
||||
}
|
||||
|
||||
// The Latin+Latin wrinkle (SUGGESTIONS.md §3a). An English+Chinese pair never
|
||||
// had to decide which language a word was in — the script decided. An
|
||||
// English+Portuguese pair has no script boundary, and "data", "sale", "comum"
|
||||
// and "tarde" are real words on both sides of it. Petal asks both directions
|
||||
// and shows whatever answers, which needs no language detector and therefore
|
||||
// cannot be wrong about somebody's writing.
|
||||
func TestDreamLookupShowsBothReadingsOnACollision(t *testing.T) {
|
||||
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||
res, err := p.Lookup("data")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
|
||||
// The English reading is unchanged and still leads.
|
||||
if len(res.Definitions) == 0 || res.Definitions[0].Definition != "facts collected for reference" {
|
||||
t.Fatalf("Definitions = %v, want the English sense first", res.Definitions)
|
||||
}
|
||||
|
||||
if res.Reverse == nil {
|
||||
t.Fatal("Reverse = nil; a word that exists in both languages must carry both readings")
|
||||
}
|
||||
if res.Reverse.Lang != "pt-PT" {
|
||||
t.Errorf("Reverse.Lang = %q, want the writer's language", res.Reverse.Lang)
|
||||
}
|
||||
if res.Reverse.Gloss != "date" {
|
||||
t.Errorf("Reverse.Gloss = %q, want the English meaning of the Portuguese word", res.Reverse.Gloss)
|
||||
}
|
||||
if res.Reverse.Phonetic != "ˈdatɐ" {
|
||||
t.Errorf("Reverse.Phonetic = %q, want the Portuguese IPA without slashes", res.Reverse.Phonetic)
|
||||
}
|
||||
// The reverse reading is a footnote on a card that already has an English
|
||||
// half, so it is capped harder than the main entry.
|
||||
if len(res.Reverse.Definitions) != maxReverseDefinitions {
|
||||
t.Fatalf("Reverse.Definitions = %d, want %d", len(res.Reverse.Definitions), maxReverseDefinitions)
|
||||
}
|
||||
if res.Reverse.Definitions[0].Definition != "dia do mês" {
|
||||
t.Errorf("Reverse.Definitions[0] = %q, want the Portuguese sense",
|
||||
res.Reverse.Definitions[0].Definition)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDreamLookupHasNoReverseForAnEnglishOnlyWord(t *testing.T) {
|
||||
// Which is almost every word she looks up: she is writing English. A
|
||||
// second block under every card would make the collision case invisible.
|
||||
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||
res, err := p.Lookup("ephemeral")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if res.Reverse != nil {
|
||||
t.Fatalf("Reverse = %+v, want none for a word that is only English", res.Reverse)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDreamGlossCarriesTheReverseReading(t *testing.T) {
|
||||
// The hover tooltip takes the same both-directions rule in one line less
|
||||
// space: the reverse *gloss* only, never the definitions.
|
||||
p := dreamProvider{dict: openFixture(t), native: "pt-PT"}
|
||||
|
||||
g, err := p.Gloss("data")
|
||||
if err != nil {
|
||||
t.Fatalf("Gloss: %v", err)
|
||||
}
|
||||
if g.Reverse != "date" {
|
||||
t.Errorf("Gloss.Reverse = %q, want the English meaning of the Portuguese word", g.Reverse)
|
||||
}
|
||||
|
||||
g, err = p.Gloss("ephemeral")
|
||||
if err != nil {
|
||||
t.Fatalf("Gloss: %v", err)
|
||||
}
|
||||
if g.Reverse != "" {
|
||||
t.Errorf("Gloss.Reverse = %q, want none for an English-only word", g.Reverse)
|
||||
}
|
||||
if g.Gloss != "efémero; passageiro" {
|
||||
t.Errorf("Gloss = %q, want the forward gloss untouched", g.Gloss)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReverseIsSilentForTheEmbeddedProviders(t *testing.T) {
|
||||
// The zh pair has no collisions and no DreamDict, and a writer with no
|
||||
// dict.db at all falls through to `glossless`. Neither may start emitting a
|
||||
// reverse block: the card would then claim a Chinese reading of an English
|
||||
// word, which is worse than saying nothing.
|
||||
set := NewSet(nil)
|
||||
|
||||
res, err := set.For(LangZh).Lookup("river")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if res.Reverse != nil {
|
||||
t.Errorf("embedded Reverse = %+v, want none", res.Reverse)
|
||||
}
|
||||
|
||||
res, err = set.For("pt-PT").Lookup("river")
|
||||
if err != nil {
|
||||
t.Fatalf("Lookup: %v", err)
|
||||
}
|
||||
if res.Reverse != nil {
|
||||
t.Errorf("glossless Reverse = %+v, want none", res.Reverse)
|
||||
}
|
||||
}
|
||||
+106
-38
@@ -1,20 +1,28 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
)
|
||||
|
||||
// Handler serves the word-lookup endpoint backed by a single shared Lexicon.
|
||||
// Handler serves the word-lookup endpoints. It holds the shared provider Set
|
||||
// and the database, because which provider answers depends on who is asking.
|
||||
type Handler struct {
|
||||
Lex *Lexicon
|
||||
Set *Set
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// New constructs a Handler with a fresh (lazily-loaded) Lexicon.
|
||||
func NewHandler() *Handler { return &Handler{Lex: New()} }
|
||||
// NewHandler constructs a Handler over a provider Set. db is used for one
|
||||
// thing: reading the caller's pair language.
|
||||
func NewHandler(db *sql.DB, set *Set) *Handler { return &Handler{Set: set, db: db} }
|
||||
|
||||
// Routes returns the router mounted at /api/word. The word is a path segment so
|
||||
// "/api/word/happy" reads naturally; it's URL-decoded to tolerate the rare
|
||||
@@ -26,56 +34,116 @@ func (h *Handler) Routes() chi.Router {
|
||||
}
|
||||
|
||||
// GlossRoutes returns the router mounted at /api/gloss — the lightweight
|
||||
// Chinese-only lookup behind the inline hover/select gloss. It shares the
|
||||
// Handler's Lexicon, so the datasets still load just once.
|
||||
// translation-only lookup behind the inline hover/select gloss. It shares the
|
||||
// Handler's Set, so the embedded datasets and dict.db are still opened once.
|
||||
func (h *Handler) GlossRoutes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/{word}", h.gloss)
|
||||
return r
|
||||
}
|
||||
|
||||
// lookup returns the definition + synonyms for one word. A word found in neither
|
||||
// dataset still returns 200 with empty lists, so the popover can show a friendly
|
||||
// "nothing found" rather than an error state.
|
||||
// HanziRoutes returns the router mounted at /api/hanzi — a Chinese word to its
|
||||
// pinyin and English senses, for a writer going the other way through the zh
|
||||
// pair (`users.direction = 'learning_pair'`).
|
||||
//
|
||||
// It does not go through [Handler.providerFor], and that is not an oversight.
|
||||
// providerFor picks a dictionary by the writer's *pair*, to answer "what does
|
||||
// this English word mean in her language" — a question whose answer differs per
|
||||
// pair. This endpoint asks the opposite question of exactly one language, and
|
||||
// [auth.SupportsLearnerDirection] already guarantees that language is Chinese.
|
||||
// Routing it through the pair would add a database read per hover to choose
|
||||
// between one option and itself.
|
||||
func (h *Handler) HanziRoutes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/{word}", h.hanzi)
|
||||
return r
|
||||
}
|
||||
|
||||
// hanzi answers a Chinese word lookup. Like the other two, a miss is a 200 with
|
||||
// empty lists — a hover that lands on a word the dictionary has never heard of
|
||||
// is an ordinary thing to happen while reading, and the tooltip simply doesn't
|
||||
// open.
|
||||
func (h *Handler) hanzi(w http.ResponseWriter, r *http.Request) {
|
||||
res, err := h.Set.Hanzi(pathWord(r))
|
||||
if err != nil {
|
||||
writeLookupErr(w, err)
|
||||
return
|
||||
}
|
||||
writeLookup(w, res)
|
||||
}
|
||||
|
||||
// providerFor returns the provider for the caller's language pair.
|
||||
//
|
||||
// The pair language is read here rather than threaded down because a word
|
||||
// lookup has no other query to piggyback on — unlike the document handlers,
|
||||
// which take pair_lang from the row-scoped query that already proves
|
||||
// ownership. It is one indexed primary-key read against a local SQLite file,
|
||||
// which costs less than encoding the response it feeds.
|
||||
//
|
||||
// A read that fails, or a caller with no user row, resolves to the empty
|
||||
// language, and [Set.For] maps that to today's embedded behaviour. Falling back
|
||||
// to a working dictionary beats failing the lookup.
|
||||
func (h *Handler) providerFor(ctx context.Context) Provider {
|
||||
var lang string
|
||||
if h.db != nil {
|
||||
_ = h.db.QueryRowContext(ctx,
|
||||
`SELECT COALESCE(pair_lang, '') FROM users WHERE id = ?`,
|
||||
auth.UserID(ctx),
|
||||
).Scan(&lang)
|
||||
}
|
||||
return h.Set.For(lang)
|
||||
}
|
||||
|
||||
// pathWord reads the {word} segment, URL-decoded.
|
||||
func pathWord(r *http.Request) string {
|
||||
word := chi.URLParam(r, "word")
|
||||
if decoded, err := url.PathUnescape(word); err == nil {
|
||||
word = decoded
|
||||
}
|
||||
return word
|
||||
}
|
||||
|
||||
// lookup returns the definition + synonyms for one word. A word found in no
|
||||
// dataset still returns 200 with empty lists, so the popover can show a
|
||||
// friendly "nothing found" rather than an error state.
|
||||
func (h *Handler) lookup(w http.ResponseWriter, r *http.Request) {
|
||||
word := chi.URLParam(r, "word")
|
||||
if decoded, err := url.PathUnescape(word); err == nil {
|
||||
word = decoded
|
||||
}
|
||||
|
||||
res, err := h.Lex.Lookup(word)
|
||||
res, err := h.providerFor(r.Context()).Lookup(pathWord(r))
|
||||
if err != nil {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
writeLookupErr(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
// Word lookups are static for the life of the build; let the browser cache
|
||||
// them so repeated right-clicks on the same word are instant.
|
||||
w.Header().Set("Cache-Control", "public, max-age=86400")
|
||||
_ = json.NewEncoder(w).Encode(res)
|
||||
writeLookup(w, res)
|
||||
}
|
||||
|
||||
// gloss returns just the Chinese translation for one word. Like lookup, a miss
|
||||
// is a 200 with an empty gloss so the hover tooltip can quietly skip rather than
|
||||
// error.
|
||||
// gloss returns just the translation for one word. Like lookup, a miss is a 200
|
||||
// with an empty gloss so the hover tooltip can quietly skip rather than error.
|
||||
func (h *Handler) gloss(w http.ResponseWriter, r *http.Request) {
|
||||
word := chi.URLParam(r, "word")
|
||||
if decoded, err := url.PathUnescape(word); err == nil {
|
||||
word = decoded
|
||||
}
|
||||
|
||||
res, err := h.Lex.Gloss(word)
|
||||
res, err := h.providerFor(r.Context()).Gloss(pathWord(r))
|
||||
if err != nil {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
writeLookupErr(w, err)
|
||||
return
|
||||
}
|
||||
writeLookup(w, res)
|
||||
}
|
||||
|
||||
// writeLookupErr answers a failed lookup. The real error is a dictionary or
|
||||
// database fault — a file path, a SQLite message — and belongs in the log, not
|
||||
// in a tooltip. The client treats any non-200 the same way, so nothing is lost
|
||||
// by saying less.
|
||||
func writeLookupErr(w http.ResponseWriter, err error) {
|
||||
log.Printf("lexicon: lookup failed: %v", err)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=86400")
|
||||
_ = json.NewEncoder(w).Encode(res)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": "lookup failed"})
|
||||
}
|
||||
|
||||
func writeLookup(w http.ResponseWriter, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
// A lookup is stable for the life of the deployment, so let the browser
|
||||
// keep it — repeated right-clicks on the same word are then instant. It is
|
||||
// `private` rather than `public` because the gloss is now in *her*
|
||||
// language: a shared cache keyed on the URL alone would hand one writer
|
||||
// another writer's language.
|
||||
w.Header().Set("Cache-Control", "private, max-age=86400")
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// The Chinese half of the lexicon: a word written in hanzi to its pinyin and
|
||||
// English senses. This is the mirror image of `gloss` — that one reads English
|
||||
// and answers in Chinese, for a Mandarin native practising English; this one
|
||||
// reads Chinese and answers in English, for the other direction of the same
|
||||
// pair (`users.direction = 'learning_pair'`).
|
||||
//
|
||||
// It is deliberately not folded into [Lexicon.load]. That method reads four
|
||||
// datasets on the first lookup of any kind, and this one is 3.1 MB gzipped that
|
||||
// only a learner-direction account will ever ask for — every other writer would
|
||||
// pay the decompression and the resident memory for a map they never touch. Its
|
||||
// own sync.Once means the cost lands on the first Chinese hover and nowhere
|
||||
// else.
|
||||
|
||||
// HanziReading is one pronunciation of a word and the senses it carries in that
|
||||
// pronunciation. A word usually has one; the ones that have two are why this is
|
||||
// a list rather than a pair of strings. 得 is dé, "to obtain", *and* de, the
|
||||
// particle that makes 说得很好 mean "speaks well" — a learner shown only the
|
||||
// first has been told something false about the sentence in front of them.
|
||||
type HanziReading struct {
|
||||
Pinyin string `json:"pinyin"`
|
||||
Senses string `json:"senses"`
|
||||
}
|
||||
|
||||
// HanziChar is one character of a word that the dictionary could not answer as
|
||||
// a whole. See [Lexicon.Hanzi].
|
||||
type HanziChar struct {
|
||||
Char string `json:"char"`
|
||||
Pinyin string `json:"pinyin"`
|
||||
Senses string `json:"senses"`
|
||||
}
|
||||
|
||||
// HanziResult is what a Chinese word lookup answers. Readings is empty for a
|
||||
// word the dictionary does not have, in which case Chars may carry the
|
||||
// character-by-character reading instead.
|
||||
type HanziResult struct {
|
||||
Word string `json:"word"`
|
||||
Readings []HanziReading `json:"readings"`
|
||||
Chars []HanziChar `json:"chars"`
|
||||
}
|
||||
|
||||
type hanziStore struct {
|
||||
once sync.Once
|
||||
err error
|
||||
// word → [[pinyin, senses], …], exactly as scripts/build_cedict.py writes it.
|
||||
entries map[string][][]string
|
||||
}
|
||||
|
||||
var hanzi hanziStore
|
||||
|
||||
func (h *hanziStore) load() {
|
||||
h.once.Do(func() {
|
||||
if err := gunzipJSON(hanziGz, &h.entries); err != nil {
|
||||
h.err = fmt.Errorf("load hanzi: %w", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// maxHanziChars caps the per-character fallback. A run longer than this is
|
||||
// almost certainly a phrase the segmenter split badly rather than a word, and
|
||||
// spelling out eight characters one at a time is a wall, not a hint.
|
||||
const maxHanziChars = 6
|
||||
|
||||
// Hanzi returns the pinyin and English senses of a Chinese word.
|
||||
//
|
||||
// There is no de-inflection walk here, and its absence is a fact about the
|
||||
// language rather than an omission: Chinese words do not inflect, so the
|
||||
// candidate forms [lookupGloss] tries for "running" → "run" have no analogue.
|
||||
// A lookup either hits the headword or it does not.
|
||||
//
|
||||
// What it does instead is fall back to the characters. The segmentation word
|
||||
// list is a superset of this dictionary — every glossable word can be
|
||||
// segmented, but jieba knows ordinary compounds CC-CEDICT has no entry for — so
|
||||
// a hover really can land on a word with nothing to say about it. Chinese
|
||||
// compounds are usually transparent from their parts (电脑 is "electric brain"),
|
||||
// which makes the character reading a genuinely useful second answer rather
|
||||
// than a consolation prize. It is returned as its own field so the surface can
|
||||
// say which of the two it is showing; a caller that only wants whole words can
|
||||
// ignore it.
|
||||
func (l *Lexicon) Hanzi(word string) (HanziResult, error) {
|
||||
hanzi.load()
|
||||
if hanzi.err != nil {
|
||||
return HanziResult{}, hanzi.err
|
||||
}
|
||||
|
||||
norm := strings.TrimSpace(word)
|
||||
res := HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}
|
||||
if norm == "" {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
if rows, ok := hanzi.entries[norm]; ok {
|
||||
res.Readings = toReadings(rows)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
chars := []rune(norm)
|
||||
if len(chars) < 2 || len(chars) > maxHanziChars {
|
||||
// A single character that missed has no parts to fall back to, and a long
|
||||
// run is not a word. Either way the honest answer is nothing.
|
||||
return res, nil
|
||||
}
|
||||
for _, r := range chars {
|
||||
if !unicode.Is(unicode.Han, r) {
|
||||
// Mixed input (a stray letter or digit inside the run) is not something
|
||||
// the character reading can explain, and guessing at the hanzi parts of
|
||||
// it would be worse than silence.
|
||||
return HanziResult{Word: word, Readings: []HanziReading{}, Chars: []HanziChar{}}, nil
|
||||
}
|
||||
rows, ok := hanzi.entries[string(r)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
first := toReadings(rows)
|
||||
if len(first) == 0 {
|
||||
continue
|
||||
}
|
||||
res.Chars = append(res.Chars, HanziChar{
|
||||
Char: string(r),
|
||||
Pinyin: first[0].Pinyin,
|
||||
Senses: first[0].Senses,
|
||||
})
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func toReadings(rows [][]string) []HanziReading {
|
||||
out := make([]HanziReading, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if len(row) < 2 {
|
||||
continue
|
||||
}
|
||||
out = append(out, HanziReading{Pinyin: row[0], Senses: row[1]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package lexicon
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
// The Chinese direction of the lexicon, against the real embedded asset — not a
|
||||
// fixture. The dataset is built by scripts/build_cedict.py, which asserts its
|
||||
// own invariants at build time; what these assert is that the *lookup* over it
|
||||
// behaves, including on the entries the build script goes out of its way to keep.
|
||||
|
||||
func TestHanziLookup(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
res, err := l.Hanzi("公园")
|
||||
if err != nil {
|
||||
t.Fatalf("lookup 公园: %v", err)
|
||||
}
|
||||
if len(res.Readings) == 0 {
|
||||
t.Fatal("公园 has no readings")
|
||||
}
|
||||
// Tone marks, not the numbered pinyin CC-CEDICT stores. The number is the
|
||||
// storage format; the marks are what a learner reads.
|
||||
if got := res.Readings[0].Pinyin; got != "gōngyuán" {
|
||||
t.Errorf("公园 pinyin = %q, want gōngyuán", got)
|
||||
}
|
||||
if !strings.Contains(res.Readings[0].Senses, "park") {
|
||||
t.Errorf("公园 senses = %q, want something about a park", res.Readings[0].Senses)
|
||||
}
|
||||
// A word answered whole says nothing about its characters — the fallback is
|
||||
// the other branch, and sending both would double the payload of the common
|
||||
// case to no purpose.
|
||||
if len(res.Chars) != 0 {
|
||||
t.Errorf("a whole-word hit also returned %d characters", len(res.Chars))
|
||||
}
|
||||
}
|
||||
|
||||
// 得 is the reason readings are a list. Answered with only dé "to obtain", a
|
||||
// learner hovering it in 说得很好 has been told something false about the
|
||||
// sentence they are looking at.
|
||||
func TestHanziParticleCarriesItsGrammaticalReading(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
for _, particle := range []string{"的", "地", "得"} {
|
||||
res, err := l.Hanzi(particle)
|
||||
if err != nil {
|
||||
t.Fatalf("lookup %s: %v", particle, err)
|
||||
}
|
||||
var found bool
|
||||
for _, r := range res.Readings {
|
||||
if r.Pinyin == "de" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s never reads as neutral \"de\": %+v", particle, res.Readings)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The fallback the segmentation gap makes necessary: jieba knows ordinary
|
||||
// compounds CC-CEDICT has no headword for, so a hover can land on a real word
|
||||
// with no entry. Chinese compounds are usually transparent from their parts, so
|
||||
// the characters are a real second answer.
|
||||
func TestHanziFallsBackToCharacters(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
// Constructed rather than borrowed from the corpus: a word that CC-CEDICT
|
||||
// *does* carry would test the other branch, and which compounds it happens to
|
||||
// omit is not something a test should pin.
|
||||
const made = "猫书"
|
||||
if _, ok := hanzi.entries[made]; ok {
|
||||
t.Skipf("%s has become a real headword; pick another compound", made)
|
||||
}
|
||||
res, err := l.Hanzi(made)
|
||||
if err != nil {
|
||||
t.Fatalf("lookup %s: %v", made, err)
|
||||
}
|
||||
if len(res.Readings) != 0 {
|
||||
t.Fatalf("%s answered as a whole word: %+v", made, res.Readings)
|
||||
}
|
||||
if len(res.Chars) != 2 {
|
||||
t.Fatalf("character fallback gave %d entries, want 2: %+v", len(res.Chars), res.Chars)
|
||||
}
|
||||
if res.Chars[0].Char != "猫" || !strings.Contains(res.Chars[0].Senses, "cat") {
|
||||
t.Errorf("first character = %+v, want 猫 ~ cat", res.Chars[0])
|
||||
}
|
||||
if res.Chars[0].Pinyin != "māo" {
|
||||
t.Errorf("猫 pinyin = %q, want māo", res.Chars[0].Pinyin)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHanziMisses(t *testing.T) {
|
||||
l := New()
|
||||
|
||||
for name, word := range map[string]string{
|
||||
// A single character with no entry has no parts to fall back to.
|
||||
"lone unknown character": "龥",
|
||||
"empty": "",
|
||||
"whitespace": " ",
|
||||
// Not Chinese at all: the English tokenizer owns these, and answering
|
||||
// would mean guessing.
|
||||
"english": "hello",
|
||||
"mixed": "猫cat",
|
||||
// Longer than a word: a bad segmentation, not something to spell out
|
||||
// character by character.
|
||||
"a whole clause": "我今天早上去公园跑步了",
|
||||
} {
|
||||
res, err := l.Hanzi(word)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if len(res.Readings) != 0 || len(res.Chars) != 0 {
|
||||
t.Errorf("%s (%q) answered with %+v / %+v", name, word, res.Readings, res.Chars)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHanziEndpoint(t *testing.T) {
|
||||
h := NewHandler(nil, NewSet(nil))
|
||||
r := chi.NewRouter()
|
||||
r.Mount("/hanzi", h.HanziRoutes())
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/"+"跑步", nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d", w.Code)
|
||||
}
|
||||
var got HanziResult
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.Word != "跑步" || len(got.Readings) == 0 || got.Readings[0].Pinyin != "pǎobù" {
|
||||
t.Fatalf("response = %+v", got)
|
||||
}
|
||||
|
||||
// A miss is a 200 with empty lists, like the other two lookups — the tooltip
|
||||
// quietly doesn't open rather than showing an error over her writing.
|
||||
w = httptest.NewRecorder()
|
||||
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/hanzi/zzz", nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("miss: status = %d, want 200", w.Code)
|
||||
}
|
||||
}
|
||||
@@ -29,14 +29,69 @@ type Result struct {
|
||||
Phonetic string `json:"phonetic"` // IPA for the English word; "" when absent
|
||||
Definitions []Meaning `json:"definitions"`
|
||||
Synonyms []string `json:"synonyms"`
|
||||
|
||||
// The fields below only ever come from DreamDict; the embedded datasets
|
||||
// leave them at their unknown values, and the popover hides them.
|
||||
|
||||
// Frequency is how common the word is (higher = more common). 0 means
|
||||
// unknown, which is DreamDict's own convention — a word it carries but has
|
||||
// no corpus count for is indistinguishable from a word it doesn't carry,
|
||||
// and the popover treats both the same way.
|
||||
Frequency int `json:"frequency"`
|
||||
// Difficulty runs 0.0 (easiest) to 1.0 (hardest); -1 means unknown. It is a
|
||||
// sentinel rather than an omitted field because 0.0 is a real, meaningful
|
||||
// score and `omitempty` would erase it.
|
||||
Difficulty float64 `json:"difficulty"`
|
||||
// Etymology is free-form Wiktionary prose, trimmed to a line. Where the
|
||||
// word came from is a real hook for a writer whose own language shares
|
||||
// Latin roots with English — "ephemeral" is much easier to keep once you
|
||||
// have seen efémero next to it.
|
||||
Etymology string `json:"etymology"`
|
||||
|
||||
// Reverse is the same token read as a word of the writer's own language,
|
||||
// present only when it is one. Absent for every writer whose pair is not
|
||||
// Latin-script, and for the overwhelming majority of words in one that is.
|
||||
Reverse *Reverse `json:"reverse,omitempty"`
|
||||
}
|
||||
|
||||
// Reverse is a lookup in the other direction: the token treated as a word of the
|
||||
// writer's language, translated into English.
|
||||
//
|
||||
// It exists because a Latin-script pair has no script boundary to tell the two
|
||||
// halves apart. In English+Chinese, "which language is this word?" answers
|
||||
// itself. In English+Portuguese it does not: *sale*, *casa*, *comum*, *tarde*
|
||||
// and *ali* are all real words on both sides, and *chat* and *pain* are the
|
||||
// French versions of the same trap.
|
||||
//
|
||||
// Petal does not guess. It asks both directions and shows whatever comes back,
|
||||
// which needs no detector, cannot be wrong about someone's writing, and — for a
|
||||
// learner — is more interesting than a correct guess would have been.
|
||||
type Reverse struct {
|
||||
// Lang is the language this reading is in, so the card can label it.
|
||||
Lang string `json:"lang"`
|
||||
// Gloss is the English meaning of the native-language word.
|
||||
Gloss string `json:"gloss"`
|
||||
// Definitions are the word's senses as written in the writer's own
|
||||
// language — the monolingual half, for when the English gloss isn't enough.
|
||||
Definitions []Meaning `json:"definitions,omitempty"`
|
||||
// Phonetic is IPA for the native-language pronunciation; "" when absent.
|
||||
Phonetic string `json:"phonetic,omitempty"`
|
||||
}
|
||||
|
||||
// unknownDifficulty is the [Result.Difficulty] value meaning "no score",
|
||||
// matching DreamDict's own -1 return.
|
||||
const unknownDifficulty = -1
|
||||
|
||||
// GlossResult is the lightweight payload for the inline hover/select gloss: just
|
||||
// the word and its Chinese translation, no definitions or synonyms. Kept small
|
||||
// so the hover tooltip is instant and trivially cacheable.
|
||||
type GlossResult struct {
|
||||
Word string `json:"word"`
|
||||
Gloss string `json:"gloss"`
|
||||
// Reverse is the English meaning of the word read as one of the writer's
|
||||
// own language — the tooltip's half of the both-directions rule (see
|
||||
// [Reverse]). Empty unless the token is a word in her language too.
|
||||
Reverse string `json:"reverse,omitempty"`
|
||||
}
|
||||
|
||||
// maxSynonyms caps how many synonyms we hand the popover, even though the dataset
|
||||
@@ -95,7 +150,7 @@ func (l *Lexicon) Lookup(word string) (Result, error) {
|
||||
}
|
||||
|
||||
norm := strings.ToLower(strings.TrimSpace(word))
|
||||
res := Result{Word: word, Definitions: []Meaning{}, Synonyms: []string{}}
|
||||
res := Result{Word: word, Definitions: []Meaning{}, Synonyms: []string{}, Difficulty: unknownDifficulty}
|
||||
if norm == "" {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package lexicon
|
||||
|
||||
// A word lookup used to mean exactly one thing: the embedded datasets, which
|
||||
// speak English and Mandarin and nothing else. That was fine while Petal had
|
||||
// one writer. It stops being fine the moment a pt-PT writer right-clicks a
|
||||
// word and gets a Chinese gloss.
|
||||
//
|
||||
// So the lookup becomes a seam. A [Provider] answers the same two questions the
|
||||
// popover and the hover tooltip have always asked; which provider answers them
|
||||
// depends on the writer's language pair, and [Set.For] is the only place that
|
||||
// decision is made.
|
||||
|
||||
// Provider answers word lookups for one writer. The embedded datasets and
|
||||
// DreamDict both satisfy it, and both treat a word they don't carry as an empty
|
||||
// result rather than an error — a miss is an ordinary outcome of looking a word
|
||||
// up, not a failure.
|
||||
type Provider interface {
|
||||
// Lookup returns the full popover payload: gloss, phonetic, definitions,
|
||||
// synonyms, and whatever extras the provider carries.
|
||||
Lookup(word string) (Result, error)
|
||||
// Gloss returns just the writer's-language translation. It is the hover
|
||||
// tooltip's fast path and skips everything else.
|
||||
Gloss(word string) (GlossResult, error)
|
||||
}
|
||||
|
||||
// LangZh is the one pair language still served by the embedded datasets. Every
|
||||
// other pair goes to DreamDict — see [Set.For] for why zh is held back.
|
||||
const LangZh = "zh"
|
||||
|
||||
// langEN is the language DreamDict is asked about for definitions, synonyms and
|
||||
// pronunciation. English is always the *target* language of the pair — what
|
||||
// varies is the language the gloss is written in.
|
||||
const langEN = "en"
|
||||
|
||||
// Set holds every provider Petal can serve a lookup from and picks between them
|
||||
// by pair language. One Set is shared by the whole process: the embedded
|
||||
// datasets load once, and dict.db is one read-only handle.
|
||||
type Set struct {
|
||||
embedded *Lexicon
|
||||
// dream is nil when dict.db was not deployed. That is a supported state,
|
||||
// not an error — see [Set.For].
|
||||
dream *DreamDict
|
||||
}
|
||||
|
||||
// NewSet returns a Set backed by the embedded datasets and, when dream is
|
||||
// non-nil, DreamDict. Passing a nil dream is how Petal runs without dict.db.
|
||||
func NewSet(dream *DreamDict) *Set {
|
||||
return &Set{embedded: New(), dream: dream}
|
||||
}
|
||||
|
||||
// HasDreamDict reports whether a dict.db is open. Only startup logging and
|
||||
// tests care; a handler never asks, because [Set.For] always returns something
|
||||
// usable.
|
||||
func (s *Set) HasDreamDict() bool { return s.dream != nil }
|
||||
|
||||
// Contents describes what the open dict.db actually holds, for the startup log.
|
||||
// With no dictionary it says so rather than returning an empty string, because
|
||||
// a blank in a log line is indistinguishable from a bug in the log line.
|
||||
func (s *Set) Contents() string {
|
||||
if s.dream == nil {
|
||||
return "no dict.db — embedded datasets only"
|
||||
}
|
||||
return s.dream.Contents()
|
||||
}
|
||||
|
||||
// For returns the provider that should answer lookups for a writer whose pair
|
||||
// language is lang.
|
||||
//
|
||||
// Three rules, in order:
|
||||
//
|
||||
// zh — and an empty code, which is what a pre-Phase-16 row reads as — stays on
|
||||
// the embedded ECDICT gloss. Not because DreamDict lacks Chinese (it has
|
||||
// CC-CEDICT), but because that path is in daily use by a real writer and the
|
||||
// two have not yet been compared on her actual lookups. Switching it is a
|
||||
// quality decision, and it hasn't been made.
|
||||
//
|
||||
// Any other pair goes to DreamDict, which is the only source that has pt-PT,
|
||||
// French or Spanish at all.
|
||||
//
|
||||
// If dict.db was never deployed, a non-zh writer falls back to the embedded
|
||||
// datasets with the gloss suppressed. This is the interesting case: the naive
|
||||
// "no data" answer would blank the popover entirely, when in fact the English
|
||||
// half of it — definitions, synonyms, phonetic — is compiled into the binary
|
||||
// and perfectly correct for her. Only the translation is missing, so only the
|
||||
// translation goes missing. A failed dictionary deploy costs her the gloss, not
|
||||
// the dictionary.
|
||||
func (s *Set) For(lang string) Provider {
|
||||
if lang == "" || lang == LangZh {
|
||||
return s.embedded
|
||||
}
|
||||
if s.dream != nil {
|
||||
return dreamProvider{dict: s.dream, native: lang}
|
||||
}
|
||||
return glossless{s.embedded}
|
||||
}
|
||||
|
||||
// glossless serves the embedded datasets with the Chinese gloss stripped, for a
|
||||
// writer who does not read Chinese. Handing her the zh gloss would be worse
|
||||
// than handing her nothing: an empty field reads as "not found", where the
|
||||
// wrong language reads as Petal being broken.
|
||||
type glossless struct{ inner Provider }
|
||||
|
||||
func (g glossless) Lookup(word string) (Result, error) {
|
||||
res, err := g.inner.Lookup(word)
|
||||
res.Gloss = ""
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (g glossless) Gloss(word string) (GlossResult, error) {
|
||||
return GlossResult{Word: word}, nil
|
||||
}
|
||||
|
||||
// Hanzi answers a Chinese-word lookup from the embedded CC-CEDICT map.
|
||||
//
|
||||
// It is on the Set rather than on [Provider] because it is not the same
|
||||
// question the other two ask. Lookup and Gloss vary by pair — which is why they
|
||||
// are behind an interface with two implementations — while this one is asked of
|
||||
// Chinese or not at all: the learner direction exists for exactly one pair (see
|
||||
// auth.learnerPairs), and DreamDict's own CC-CEDICT would be a second copy of
|
||||
// the same dictionary, chosen by a rule with one branch.
|
||||
func (s *Set) Hanzi(word string) (HanziResult, error) { return s.embedded.Hanzi(word) }
|
||||
@@ -41,7 +41,7 @@ type checkpointResponse struct {
|
||||
// RunCheckpoint sends the grammar checkpoint and parses the JSON result. It
|
||||
// applies the latency-guard truncation and the checkpoint sampling parameters
|
||||
// from the spec.
|
||||
func RunCheckpoint(ctx context.Context, client LLMClient, contentText, tone string) ([]RawSuggestion, error) {
|
||||
func RunCheckpoint(ctx context.Context, client LLMClient, contentText, tone string, _ Lang) ([]RawSuggestion, error) {
|
||||
raw, err := client.Complete(ctx, CompletionRequest{
|
||||
Messages: CheckpointMessages(TruncateDoc(contentText), tone),
|
||||
MaxTokens: checkpointMaxTokens,
|
||||
|
||||
@@ -18,10 +18,11 @@ const CollocationInterval = 25 * time.Second
|
||||
// reflect the full piece. Each flag carries a native replacement to apply.
|
||||
//
|
||||
// The tone argument is accepted for a uniform pass signature and passed through
|
||||
// to the prompt so a hint can prefer a register-appropriate pairing.
|
||||
func RunCollocation(ctx context.Context, client LLMClient, contentText, tone string) ([]RawSuggestion, error) {
|
||||
// to the prompt so a hint can prefer a register-appropriate pairing. `lang` is
|
||||
// the writer's pair language — the one each hint's short gloss is written in.
|
||||
func RunCollocation(ctx context.Context, client LLMClient, contentText, tone string, lang Lang) ([]RawSuggestion, error) {
|
||||
raw, err := client.Complete(ctx, CompletionRequest{
|
||||
Messages: CollocationMessages(contentText, tone),
|
||||
Messages: CollocationMessages(contentText, tone, lang),
|
||||
MaxTokens: 2048,
|
||||
Temperature: 0.3,
|
||||
RepetitionPenalty: 1.15,
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package llm
|
||||
|
||||
import "strings"
|
||||
|
||||
// The pair language, as the prompts need to talk about it.
|
||||
//
|
||||
// Three of Petal's prompts name the writer's first language rather than merely
|
||||
// being written in English: the collocation coach asks for a gloss in it, Ask
|
||||
// Petal offers to answer in it, and the explanation translator renders into it.
|
||||
// Before Phase 19 all three said "Simplified Chinese" outright, which made the
|
||||
// zh pair the only one that could ever work.
|
||||
//
|
||||
// A Lang is not a translation of the prompt — the instructions stay in English,
|
||||
// which is what the model follows best. It is the name the model should use for
|
||||
// her language, plus the one word it should watch for when she writes in it.
|
||||
type Lang struct {
|
||||
// Code matches users.pair_lang.
|
||||
Code string
|
||||
// Name is how the prompt refers to the language, spelled the way a model
|
||||
// recognises it. Regional precision matters here: "European Portuguese" is
|
||||
// not "Portuguese" to a model that has read far more pt-BR than pt-PT.
|
||||
Name string
|
||||
// Why asks the same thing she would ask in her own language. It goes into
|
||||
// the Ask Petal prompt as an example, so a model that answers only to
|
||||
// English "why" still recognises the question when she types it her way.
|
||||
Why string
|
||||
}
|
||||
|
||||
// langs holds every pair Petal can currently be a partner in. A language with a
|
||||
// frontend langpack but no entry here still works — it falls back to zh's
|
||||
// behaviour of the prompts, which is wrong but not broken — so keep the two in
|
||||
// step when a pair ships.
|
||||
var langs = map[string]Lang{
|
||||
"zh": {Code: "zh", Name: "Simplified Chinese (Mandarin)", Why: "为什么"},
|
||||
"pt-PT": {Code: "pt-PT", Name: "European Portuguese (pt-PT, never Brazilian Portuguese)", Why: "porquê"},
|
||||
"fr": {Code: "fr", Name: "French", Why: "pourquoi"},
|
||||
"es": {Code: "es", Name: "Spanish", Why: "por qué"},
|
||||
}
|
||||
|
||||
// DefaultLang is the pair assumed when none is known — the column's default, and
|
||||
// the only pair that existed before Phase 19.
|
||||
var DefaultLang = langs["zh"]
|
||||
|
||||
// LangFor resolves a users.pair_lang value. An empty or unrecognised code falls
|
||||
// back to the default rather than erroring: a prompt is not the place to
|
||||
// discover a configuration problem, and the writing still has to be checked.
|
||||
func LangFor(code string) Lang {
|
||||
if l, ok := langs[strings.TrimSpace(code)]; ok {
|
||||
return l
|
||||
}
|
||||
return DefaultLang
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLangForFallsBackToDefault(t *testing.T) {
|
||||
if got := LangFor("zh"); got.Code != "zh" {
|
||||
t.Fatalf("LangFor(zh) = %+v", got)
|
||||
}
|
||||
if got := LangFor("pt-PT"); got.Code != "pt-PT" {
|
||||
t.Fatalf("LangFor(pt-PT) = %+v", got)
|
||||
}
|
||||
// A blank column, a stray value, and stray whitespace all resolve rather
|
||||
// than erroring — a prompt is the wrong place to discover a config problem.
|
||||
for _, in := range []string{"", " ", "klingon", "ZH"} {
|
||||
if got := LangFor(in); got.Code != DefaultLang.Code {
|
||||
t.Fatalf("LangFor(%q) = %q, want the default %q", in, got.Code, DefaultLang.Code)
|
||||
}
|
||||
}
|
||||
if got := LangFor(" zh "); got.Code != "zh" {
|
||||
t.Fatalf("LangFor with padding = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The three prompts that name the writer's language must actually name *hers*.
|
||||
// Before Phase 19 all three said "Simplified Chinese" outright, which is the
|
||||
// bug this guards: a pt-PT writer asking "porquê" would have been answered in
|
||||
// Mandarin.
|
||||
func TestPromptsNameTheWritersLanguage(t *testing.T) {
|
||||
pt := LangFor("pt-PT")
|
||||
|
||||
collocation := CollocationMessages("The rain was strong.", "casual", pt)[0].Content
|
||||
if !strings.Contains(collocation, "European Portuguese") {
|
||||
t.Fatalf("collocation prompt doesn't ask for a pt-PT gloss:\n%s", collocation)
|
||||
}
|
||||
if strings.Contains(collocation, "Simplified Chinese") {
|
||||
t.Fatalf("collocation prompt still hardcodes Chinese:\n%s", collocation)
|
||||
}
|
||||
// The tone steering must survive alongside the language — they share one
|
||||
// format string, and getting the verbs in the wrong order silently drops one.
|
||||
if !strings.Contains(collocation, "relaxed, friendly, and conversational") {
|
||||
t.Fatalf("collocation prompt lost its tone guidance:\n%s", collocation)
|
||||
}
|
||||
|
||||
translate := TranslateMessages("Try a shorter sentence here.", pt)[0].Content
|
||||
if !strings.Contains(translate, "European Portuguese") || strings.Contains(translate, "Chinese") {
|
||||
t.Fatalf("translate prompt targets the wrong language:\n%s", translate)
|
||||
}
|
||||
|
||||
ask := AskPetalSystemPrompt("origin", "replacement", "grammar", "explanation", "paragraph", pt)
|
||||
if !strings.Contains(ask, "European Portuguese") || strings.Contains(ask, "Mandarin") {
|
||||
t.Fatalf("ask-petal prompt targets the wrong language:\n%s", ask)
|
||||
}
|
||||
if !strings.Contains(ask, "porquê") {
|
||||
t.Fatalf("ask-petal prompt doesn't recognise her word for \"why\":\n%s", ask)
|
||||
}
|
||||
// The suggestion context is positional in that template; a mis-numbered
|
||||
// verb would quietly blank one of these fields.
|
||||
for _, want := range []string{"origin", "replacement", "grammar", "explanation", "paragraph"} {
|
||||
if !strings.Contains(ask, want) {
|
||||
t.Fatalf("ask-petal prompt dropped %q:\n%s", want, ask)
|
||||
}
|
||||
}
|
||||
if strings.Contains(ask, "%!") {
|
||||
t.Fatalf("ask-petal prompt has a formatting error:\n%s", ask)
|
||||
}
|
||||
}
|
||||
|
||||
// The zh pair is in daily use and must be untouched by the extraction: its
|
||||
// prompts should read exactly as they did when they were hardcoded.
|
||||
func TestDefaultPairStillReadsAsBefore(t *testing.T) {
|
||||
zh := LangFor("zh")
|
||||
|
||||
if got := CollocationMessages("x", "", zh)[0].Content; !strings.Contains(got, "Simplified Chinese (Mandarin) gloss in parentheses") {
|
||||
t.Fatalf("zh collocation gloss changed:\n%s", got)
|
||||
}
|
||||
if got := TranslateMessages("x", zh)[0].Content; !strings.Contains(got, "natural, friendly Simplified Chinese (Mandarin)") {
|
||||
t.Fatalf("zh translate target changed:\n%s", got)
|
||||
}
|
||||
if got := AskPetalSystemPrompt("a", "b", "c", "d", "e", zh); !strings.Contains(got, "为什么") {
|
||||
t.Fatalf("zh ask-petal lost its Mandarin \"why\":\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// UX item 6: the Ask Petal answer is bilingual, pair language first, halves
|
||||
// separated by one blank line. That separator is not a stylistic preference —
|
||||
// AskPetal.tsx splits on it to render the two halves the way the companion
|
||||
// renders its two lines — so the instruction has to survive prompt edits.
|
||||
//
|
||||
// The direction the writer is learning in is deliberately not encoded: the pair
|
||||
// is (English + X), and an English speaker learning French needs the same two
|
||||
// halves a Mandarin speaker learning English does. The prompt asks for both and
|
||||
// lets the reader choose, so there is nothing here that names one half the
|
||||
// answer and the other a courtesy.
|
||||
func TestAskPetalAnswersInBothLanguages(t *testing.T) {
|
||||
for _, code := range []string{"zh", "pt-PT", "fr", "es"} {
|
||||
lang := LangFor(code)
|
||||
ask := AskPetalSystemPrompt("a", "b", "grammar", "d", "e", lang)
|
||||
|
||||
if !strings.Contains(ask, "BOTH languages") {
|
||||
t.Fatalf("%s: ask-petal no longer asks for both languages:\n%s", code, ask)
|
||||
}
|
||||
if !strings.Contains(ask, "single blank line") {
|
||||
t.Fatalf("%s: ask-petal lost the blank-line separator the client splits on:\n%s", code, ask)
|
||||
}
|
||||
// Order matters to the rendering: the pair language is the prominent
|
||||
// half, English the muted one beneath it.
|
||||
if !strings.Contains(ask, "first the whole answer in "+lang.Name) {
|
||||
t.Fatalf("%s: ask-petal doesn't put %s first:\n%s", code, lang.Name, ask)
|
||||
}
|
||||
// The instruction it replaced. Left in place it directly contradicts the
|
||||
// new one, and a model given both will pick one at random.
|
||||
if strings.Contains(ask, "Never mix languages") {
|
||||
t.Fatalf("%s: ask-petal still forbids the bilingual reply it now asks for:\n%s", code, ask)
|
||||
}
|
||||
if strings.Contains(ask, "%!") {
|
||||
t.Fatalf("%s: ask-petal prompt has a formatting error:\n%s", code, ask)
|
||||
}
|
||||
}
|
||||
}
|
||||
+57
-29
@@ -97,8 +97,8 @@ func VoiceMessages(contentText string) []Message {
|
||||
// just non-native ("do a decision" → "make a decision", "strong rain" → "heavy
|
||||
// rain"), and explicitly DEFERS real grammar/spelling errors to the grammar
|
||||
// checkpoint so the two families don't overlap. Every explanation is framed as a
|
||||
// warm "natives usually say…" note with a short Mandarin gloss — never
|
||||
// "error/wrong" — because these are stylistic, not mistakes. It is a distinct
|
||||
// warm "natives usually say…" note with a short gloss in the writer's own
|
||||
// language — never "error/wrong" — because these are stylistic, not mistakes. It is a distinct
|
||||
// pass from the grammar checkpoint (do not bundle them). `replacement` carries
|
||||
// the natural pairing the writer can accept in one tap.
|
||||
const collocationSystemPrompt = `You are a warm, encouraging writing assistant helping someone who speaks English as a second language. ` +
|
||||
@@ -113,7 +113,7 @@ Identify up to 5 such non-native word pairings. For each, give the natural pairi
|
||||
`Be gentle and specific. Do NOT flag grammar errors, spelling mistakes, or unclear sentences — those are handled ` +
|
||||
`elsewhere. Only flag word pairings that are correct but sound non-native.%s
|
||||
|
||||
Phrase every explanation warmly as "Natives usually say…" and include a brief Simplified Chinese gloss in parentheses. ` +
|
||||
Phrase every explanation warmly as "Natives usually say…" and include a brief %s gloss in parentheses. ` +
|
||||
`Never use the words "error", "wrong", or "mistake" — these are friendly polish, not corrections.
|
||||
|
||||
Respond ONLY with valid JSON. No preamble, no markdown fences. Format:
|
||||
@@ -132,10 +132,11 @@ If every pairing already sounds natural, return: {"suggestions": []}`
|
||||
|
||||
// CollocationMessages builds the message array for a collocation pass over the
|
||||
// WHOLE document (no truncation), gently steered toward the document's tone so a
|
||||
// hint can prefer a register-appropriate pairing.
|
||||
func CollocationMessages(contentText, tone string) []Message {
|
||||
// hint can prefer a register-appropriate pairing. The parenthetical gloss is
|
||||
// written in the writer's own language.
|
||||
func CollocationMessages(contentText, tone string, lang Lang) []Message {
|
||||
return []Message{
|
||||
{Role: "system", Content: fmt.Sprintf(collocationSystemPrompt, toneGuidance(tone))},
|
||||
{Role: "system", Content: fmt.Sprintf(collocationSystemPrompt, toneGuidance(tone), lang.Name)},
|
||||
{Role: "user", Content: contentText},
|
||||
}
|
||||
}
|
||||
@@ -143,30 +144,56 @@ func CollocationMessages(contentText, tone string) []Message {
|
||||
// askPetalSystemTemplate is the Ask Petal tutor prompt. The suggestion context
|
||||
// is interpolated in; the user's own messages are appended after this system
|
||||
// turn by the caller.
|
||||
//
|
||||
// The reply is bilingual, the pair language first. Until UX item 6 it mirrored
|
||||
// the language of the question instead — self-consistent, but it meant asking in
|
||||
// one language cost you the other, and the writer doesn't always know which one
|
||||
// the answer will be clearer in. Which half is the safety net and which is the
|
||||
// lesson depends on who is writing: the pair is (English + X) either way, and an
|
||||
// English speaker learning French wants the French half for the same reason a
|
||||
// Mandarin speaker learning English wants the English one. Petal cannot tell
|
||||
// them apart from a chat message, and doesn't need to — every other explanation
|
||||
// surface already gives both (the card's English body, the seeded bubble in the
|
||||
// pair language). The answer that goes deepest into the "why" was the one place
|
||||
// that didn't.
|
||||
//
|
||||
// The blank line between the halves is a contract with the client: AskPetal.tsx
|
||||
// splits on the first one to render her language prominently and the English
|
||||
// beneath it, mirroring the companion's bubble. A model that ignores the
|
||||
// instruction and writes one language degrades to a single plain block — the
|
||||
// answer is still readable, which is why the split is a rendering nicety and
|
||||
// never a parse the reply depends on.
|
||||
const askPetalSystemTemplate = `You are Petal, a warm and patient English writing tutor helping someone who is learning English ` +
|
||||
`as a second language. You are currently discussing a specific writing suggestion.
|
||||
|
||||
Suggestion context:
|
||||
- Original text: "%s"
|
||||
- Suggested replacement: "%s"
|
||||
- Issue type: %s
|
||||
- Initial explanation: "%s"
|
||||
- Surrounding paragraph: "%s"
|
||||
- Original text: "%[1]s"
|
||||
- Suggested replacement: "%[2]s"
|
||||
- Issue type: %[3]s
|
||||
- Initial explanation: "%[4]s"
|
||||
- Surrounding paragraph: "%[5]s"
|
||||
|
||||
The user wants to understand this suggestion better. Detect the language of the user's message ` +
|
||||
`and respond in that same language. If they write in Mandarin Chinese, respond entirely in ` +
|
||||
`Mandarin. If they write in English, respond in English. Never mix languages in a single response.
|
||||
The user wants to understand this suggestion better. Answer in BOTH languages, every time, ` +
|
||||
`whichever language they asked their question in: first the whole answer in %[6]s, then the ` +
|
||||
`same answer again in English. Separate the two with a single blank line. Do not label them, ` +
|
||||
`do not use a blank line anywhere else, and do not mix the two languages within one half — ` +
|
||||
`each half is complete on its own.
|
||||
|
||||
One of those two languages is the one they are surest in and the other is the one they are ` +
|
||||
`working in — you do not know which way round, so give both and let them choose. Both halves ` +
|
||||
`say the same thing: do not put a point in one that is missing from the other.
|
||||
|
||||
Explain clearly and kindly. Use simple language appropriate to the user's message. Give examples ` +
|
||||
`when helpful. If they ask "why" (or "为什么"), explain the grammar rule or idiom behind it. ` +
|
||||
`when helpful. If they ask "why" (or "%[7]s"), explain the grammar rule or idiom behind it. ` +
|
||||
`If they suggest an alternative phrasing, evaluate it honestly.
|
||||
|
||||
Keep responses concise (2-4 sentences). This is a chat, not an essay. Be encouraging — ` +
|
||||
Keep each half concise (2-3 sentences). This is a chat, not an essay. Be encouraging — ` +
|
||||
`learning a language is hard and they're doing great.`
|
||||
|
||||
// AskPetalSystemPrompt fills the tutor prompt with one suggestion's context.
|
||||
func AskPetalSystemPrompt(original, replacement, suggestionType, explanation, paragraph string) string {
|
||||
return fmt.Sprintf(askPetalSystemTemplate, original, replacement, suggestionType, explanation, paragraph)
|
||||
// AskPetalSystemPrompt fills the tutor prompt with one suggestion's context and
|
||||
// the writer's pair language, which is the one she may ask her question in.
|
||||
func AskPetalSystemPrompt(original, replacement, suggestionType, explanation, paragraph string, lang Lang) string {
|
||||
return fmt.Sprintf(askPetalSystemTemplate, original, replacement, suggestionType, explanation, paragraph, lang.Name, lang.Why)
|
||||
}
|
||||
|
||||
// rewriteSystemTemplate drives the "say it more naturally" / tone-rewrite tool.
|
||||
@@ -215,22 +242,23 @@ func RewriteMessages(text, style string) []Message {
|
||||
}
|
||||
|
||||
// translateSystemPrompt drives the explanation translator: it renders a
|
||||
// suggestion's English explanation into Simplified Chinese so an ESL reader sees
|
||||
// the "why" in her first language. Strict about returning ONLY the translation
|
||||
// (no quotes, no pinyin, no English echo) so it can drop straight into the chat
|
||||
// bubble. Kept warm and plain — these are short, friendly one-liners.
|
||||
// suggestion's English explanation into the writer's own language so an ESL
|
||||
// reader sees the "why" in her first language. Strict about returning ONLY the
|
||||
// translation (no quotes, no romanisation, no English echo) so it can drop
|
||||
// straight into the chat bubble. Kept warm and plain — these are short, friendly
|
||||
// one-liners.
|
||||
const translateSystemPrompt = `You are Petal, a warm writing assistant. Translate the English text the user ` +
|
||||
`sends into natural, friendly Simplified Chinese (Mandarin). It is a short explanation of a writing ` +
|
||||
`suggestion, written for a native Chinese speaker learning English.
|
||||
`sends into natural, friendly %[1]s. It is a short explanation of a writing ` +
|
||||
`suggestion, written for a native %[1]s speaker learning English.
|
||||
|
||||
Respond with ONLY the Simplified Chinese translation. No quotation marks, no pinyin, no English, no preamble — ` +
|
||||
Respond with ONLY the %[1]s translation. No quotation marks, no romanisation, no English, no preamble — ` +
|
||||
`just the translated sentence.`
|
||||
|
||||
// TranslateMessages builds the message array for translating one short English
|
||||
// explanation into Simplified Chinese.
|
||||
func TranslateMessages(text string) []Message {
|
||||
// explanation into the writer's own language.
|
||||
func TranslateMessages(text string, lang Lang) []Message {
|
||||
return []Message{
|
||||
{Role: "system", Content: translateSystemPrompt},
|
||||
{Role: "system", Content: fmt.Sprintf(translateSystemPrompt, lang.Name)},
|
||||
{Role: "user", Content: text},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,13 +4,13 @@ import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// RunTranslate renders a short English explanation into Simplified Chinese. It
|
||||
// is a one-shot Complete (the result seeds the Ask Petal bubble), kept at a low
|
||||
// temperature so the translation is faithful rather than creative. Output is
|
||||
// RunTranslate renders a short English explanation into the writer's own
|
||||
// language. It is a one-shot Complete (the result seeds the Ask Petal bubble),
|
||||
// kept at a low temperature so the translation is faithful rather than creative. Output is
|
||||
// trimmed of any stray surrounding quotes the model may add.
|
||||
func RunTranslate(ctx context.Context, client LLMClient, text string) (string, error) {
|
||||
func RunTranslate(ctx context.Context, client LLMClient, text string, lang Lang) (string, error) {
|
||||
out, err := client.Complete(ctx, CompletionRequest{
|
||||
Messages: TranslateMessages(text),
|
||||
Messages: TranslateMessages(text, lang),
|
||||
MaxTokens: 512,
|
||||
Temperature: 0.2,
|
||||
TopP: 0.9,
|
||||
|
||||
@@ -28,6 +28,12 @@ type vllmRequest struct {
|
||||
TopP float64 `json:"top_p"`
|
||||
Stop []string `json:"stop,omitempty"`
|
||||
Stream bool `json:"stream"`
|
||||
// ChatTemplateKwargs is a vLLM extension. Qwen3-family models reason by
|
||||
// default and prepend a plain-text preamble ("Here's a thinking process:")
|
||||
// ahead of the answer — not a <think> block, so it cannot be stripped after
|
||||
// the fact. Every Petal pass parses a JSON object out of the completion, so
|
||||
// an unsuppressed preamble fails the parse outright.
|
||||
ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"`
|
||||
}
|
||||
|
||||
func (c *VLLMClient) body(req CompletionRequest) vllmRequest {
|
||||
@@ -40,6 +46,8 @@ func (c *VLLMClient) body(req CompletionRequest) vllmRequest {
|
||||
TopP: req.TopP,
|
||||
Stop: req.Stop,
|
||||
Stream: req.Stream,
|
||||
|
||||
ChatTemplateKwargs: map[string]any{"enable_thinking": false},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ const VoiceInterval = 20 * time.Second
|
||||
// The tone argument is accepted for a uniform pass signature but ignored: voice
|
||||
// consistency is judged against the document's own established voice, not an
|
||||
// externally-chosen register.
|
||||
func RunVoice(ctx context.Context, client LLMClient, contentText, _ string) ([]RawSuggestion, error) {
|
||||
func RunVoice(ctx context.Context, client LLMClient, contentText, _ string, _ Lang) ([]RawSuggestion, error) {
|
||||
raw, err := client.Complete(ctx, CompletionRequest{
|
||||
Messages: VoiceMessages(contentText),
|
||||
MaxTokens: 2048,
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
// Package spell owns the personal spelling dictionary — the words a writer has
|
||||
// told Petal to stop flagging.
|
||||
//
|
||||
// It lived in the browser's localStorage until Phase 18, which was wrong twice
|
||||
// over: two people sharing a device shared one list (built from one person's
|
||||
// private writing), and one person writing on a laptop and a tablet had two
|
||||
// lists that never met. It is a small amount of state, but it is *her* state,
|
||||
// so it belongs to her account rather than to a browser profile.
|
||||
//
|
||||
// Everything here is scoped by `lang` as well as by user. That is the language
|
||||
// of the *dictionary* the word was accepted against, not the writer's own.
|
||||
//
|
||||
// Phase 18 justified that key by saying an en-US personal word must not silence
|
||||
// a pt-PT flag once the second pair shipped. Phase 21 shipped it and the
|
||||
// justification did not survive: under the both-dictionaries rule
|
||||
// (SUGGESTIONS.md §3a) a word is only ever flagged when *every* loaded
|
||||
// dictionary rejected it, so there is no such thing as a pt-PT flag an English
|
||||
// exception could silence. What the key is actually good for is narrower and
|
||||
// still worth having — the rows say which dictionary each acceptance was made
|
||||
// against, so a pair that later loses or gains a dictionary keeps a truthful
|
||||
// record instead of one merged list of unknown provenance. The browser writes a
|
||||
// row per loaded dictionary when she accepts a word; see useSpellChecker.
|
||||
package spell
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
)
|
||||
|
||||
// DefaultLang is the dictionary assumed when a caller doesn't name one. English
|
||||
// is in every pair, so it is the safe assumption; pt-PT is named explicitly by
|
||||
// the pt-PT pair's second dictionary.
|
||||
const DefaultLang = "en"
|
||||
|
||||
// MaxWordLen bounds a single entry. A personal dictionary holds words, and a
|
||||
// pasted paragraph is a bug (or an attempt to use the table as storage).
|
||||
const MaxWordLen = 80
|
||||
|
||||
// MaxBatch bounds one request. The only bulk caller is the one-time adoption of
|
||||
// a browser's pre-Phase-18 list, which is realistically tens of words.
|
||||
const MaxBatch = 500
|
||||
|
||||
// Handler owns the /api/spell routes.
|
||||
type Handler struct {
|
||||
DB *db.DB
|
||||
}
|
||||
|
||||
func New(database *db.DB) *Handler { return &Handler{DB: database} }
|
||||
|
||||
// Routes mounts the personal-dictionary endpoints under /api/spell.
|
||||
func (h *Handler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/words", h.list)
|
||||
r.Post("/words", h.add)
|
||||
r.Delete("/words", h.remove)
|
||||
return r
|
||||
}
|
||||
|
||||
type wordsResponse struct {
|
||||
Lang string `json:"lang"`
|
||||
Words []string `json:"words"`
|
||||
}
|
||||
|
||||
type addRequest struct {
|
||||
Lang string `json:"lang"`
|
||||
// Word and Words are both accepted so the everyday "add this one word" call
|
||||
// stays obvious while the one-shot migration of a browser's old list is a
|
||||
// single request rather than one per word.
|
||||
Word string `json:"word"`
|
||||
Words []string `json:"words"`
|
||||
}
|
||||
|
||||
// normLang keeps the dictionary tag in one canonical shape so "EN", "en" and a
|
||||
// missing value can never split one list into three.
|
||||
func normLang(lang string) string {
|
||||
lang = strings.ToLower(strings.TrimSpace(lang))
|
||||
if lang == "" {
|
||||
return DefaultLang
|
||||
}
|
||||
return lang
|
||||
}
|
||||
|
||||
// list returns the caller's words for one dictionary, alphabetically so the
|
||||
// order is stable between requests.
|
||||
func (h *Handler) list(w http.ResponseWriter, r *http.Request) {
|
||||
lang := normLang(r.URL.Query().Get("lang"))
|
||||
words, err := h.fetch(auth.UserID(r.Context()), lang)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, wordsResponse{Lang: lang, Words: words})
|
||||
}
|
||||
|
||||
// add inserts one or more words, idempotently, and answers with the resulting
|
||||
// full list — so the client never has to merge two views of the same set.
|
||||
func (h *Handler) add(w http.ResponseWriter, r *http.Request) {
|
||||
var req addRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
httputil.BadRequest(w, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
lang := normLang(req.Lang)
|
||||
|
||||
incoming := req.Words
|
||||
if req.Word != "" {
|
||||
incoming = append(incoming, req.Word)
|
||||
}
|
||||
clean := make([]string, 0, len(incoming))
|
||||
for _, word := range incoming {
|
||||
word = strings.TrimSpace(word)
|
||||
if word == "" || len([]rune(word)) > MaxWordLen {
|
||||
continue
|
||||
}
|
||||
clean = append(clean, word)
|
||||
}
|
||||
if len(clean) == 0 {
|
||||
httputil.BadRequest(w, "no word given")
|
||||
return
|
||||
}
|
||||
if len(clean) > MaxBatch {
|
||||
httputil.BadRequest(w, "too many words in one request")
|
||||
return
|
||||
}
|
||||
|
||||
userID := auth.UserID(r.Context())
|
||||
tx, err := h.DB.Begin()
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
for _, word := range clean {
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO personal_words (user_id, lang, word) VALUES (?, ?, ?)
|
||||
ON CONFLICT(user_id, lang, word) DO NOTHING`,
|
||||
userID, lang, word,
|
||||
); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
words, err := h.fetch(userID, lang)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, wordsResponse{Lang: lang, Words: words})
|
||||
}
|
||||
|
||||
// remove forgets one word. Deleting something that was never there is a success:
|
||||
// the caller's intent — "this word is not in my dictionary" — already holds.
|
||||
func (h *Handler) remove(w http.ResponseWriter, r *http.Request) {
|
||||
word := strings.TrimSpace(r.URL.Query().Get("word"))
|
||||
if word == "" {
|
||||
httputil.BadRequest(w, "no word given")
|
||||
return
|
||||
}
|
||||
lang := normLang(r.URL.Query().Get("lang"))
|
||||
userID := auth.UserID(r.Context())
|
||||
if _, err := h.DB.Exec(
|
||||
`DELETE FROM personal_words WHERE user_id = ? AND lang = ? AND word = ?`,
|
||||
userID, lang, word,
|
||||
); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
words, err := h.fetch(userID, lang)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, wordsResponse{Lang: lang, Words: words})
|
||||
}
|
||||
|
||||
// fetch reads one (user, dictionary) list. Both keys are always bound — an
|
||||
// unscoped read here would hand one writer another's private vocabulary.
|
||||
func (h *Handler) fetch(userID, lang string) ([]string, error) {
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT word FROM personal_words WHERE user_id = ? AND lang = ? ORDER BY word`,
|
||||
userID, lang,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
words := []string{} // never nil: the client expects a list, not null
|
||||
for rows.Next() {
|
||||
var word string
|
||||
if err := rows.Scan(&word); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
words = append(words, word)
|
||||
}
|
||||
return words, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package spell
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// newTestServer mounts the routes behind the same auth middleware main.go
|
||||
// installs — a bare router resolves no caller, so every scoped query would
|
||||
// silently match nothing.
|
||||
func newTestServer(t *testing.T) (http.Handler, *db.DB) {
|
||||
t.Helper()
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
r := chi.NewRouter()
|
||||
r.Mount("/spell", New(database).Routes())
|
||||
return auth.Middleware(auth.StaticResolver(db.LocalUserID))(r), database
|
||||
}
|
||||
|
||||
func do(t *testing.T, srv http.Handler, method, path, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
var r *http.Request
|
||||
if body != "" {
|
||||
r = httptest.NewRequest(method, path, bytes.NewBufferString(body))
|
||||
} else {
|
||||
r = httptest.NewRequest(method, path, nil)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
srv.ServeHTTP(rec, r)
|
||||
return rec
|
||||
}
|
||||
|
||||
func decodeWords(t *testing.T, rec *httptest.ResponseRecorder) wordsResponse {
|
||||
t.Helper()
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
var got wordsResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v (body %s)", err, rec.Body)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func equal(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// TestLifecycle walks add → list → re-add → delete, and asserts the two
|
||||
// properties the client relies on: adds are idempotent, and every response
|
||||
// carries the full resulting list so the browser never has to merge.
|
||||
func TestLifecycle(t *testing.T) {
|
||||
srv, _ := newTestServer(t)
|
||||
|
||||
// Empty to start, and a list, never null.
|
||||
got := decodeWords(t, do(t, srv, http.MethodGet, "/spell/words", ""))
|
||||
if got.Lang != "en" || len(got.Words) != 0 {
|
||||
t.Fatalf("fresh list = %+v, want empty en", got)
|
||||
}
|
||||
if !bytes.Contains(
|
||||
do(t, srv, http.MethodGet, "/spell/words", "").Body.Bytes(), []byte(`"words":[]`),
|
||||
) {
|
||||
t.Fatal("empty list encoded as null, not []")
|
||||
}
|
||||
|
||||
// One word, then a bulk add (the shape the browser's one-time adoption uses).
|
||||
got = decodeWords(t, do(t, srv, http.MethodPost, "/spell/words", `{"word":"Petal"}`))
|
||||
if !equal(got.Words, []string{"Petal"}) {
|
||||
t.Fatalf("after add = %v", got.Words)
|
||||
}
|
||||
got = decodeWords(t, do(t, srv, http.MethodPost, "/spell/words",
|
||||
`{"words":["hanfu","qipao","Petal"]}`))
|
||||
if !equal(got.Words, []string{"Petal", "hanfu", "qipao"}) {
|
||||
t.Fatalf("after bulk add = %v, want sorted and de-duplicated", got.Words)
|
||||
}
|
||||
|
||||
// Re-adding an existing word must not error or duplicate it.
|
||||
got = decodeWords(t, do(t, srv, http.MethodPost, "/spell/words", `{"word":"hanfu"}`))
|
||||
if !equal(got.Words, []string{"Petal", "hanfu", "qipao"}) {
|
||||
t.Fatalf("re-add changed the list: %v", got.Words)
|
||||
}
|
||||
|
||||
// Delete, then delete again — forgetting a word Petal never knew is a
|
||||
// success, since the caller's intent already holds.
|
||||
got = decodeWords(t, do(t, srv, http.MethodDelete, "/spell/words?word=qipao", ""))
|
||||
if !equal(got.Words, []string{"Petal", "hanfu"}) {
|
||||
t.Fatalf("after delete = %v", got.Words)
|
||||
}
|
||||
got = decodeWords(t, do(t, srv, http.MethodDelete, "/spell/words?word=qipao", ""))
|
||||
if !equal(got.Words, []string{"Petal", "hanfu"}) {
|
||||
t.Fatalf("repeat delete = %v", got.Words)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLanguagesDoNotMerge is the reason `lang` is in the primary key: a word the
|
||||
// writer excused in English must not silence the pt-PT dictionary too.
|
||||
func TestLanguagesDoNotMerge(t *testing.T) {
|
||||
srv, _ := newTestServer(t)
|
||||
|
||||
do(t, srv, http.MethodPost, "/spell/words", `{"word":"tarde"}`)
|
||||
got := decodeWords(t, do(t, srv, http.MethodPost, "/spell/words",
|
||||
`{"lang":"pt-PT","word":"tarde"}`))
|
||||
if !equal(got.Words, []string{"tarde"}) || got.Lang != "pt-pt" {
|
||||
t.Fatalf("pt list = %+v", got)
|
||||
}
|
||||
|
||||
// Removing it from one dictionary leaves the other alone.
|
||||
do(t, srv, http.MethodDelete, "/spell/words?lang=pt-PT&word=tarde", "")
|
||||
if got = decodeWords(t, do(t, srv, http.MethodGet, "/spell/words?lang=pt-PT", "")); len(got.Words) != 0 {
|
||||
t.Fatalf("pt list after delete = %v", got.Words)
|
||||
}
|
||||
if got = decodeWords(t, do(t, srv, http.MethodGet, "/spell/words", "")); !equal(got.Words, []string{"tarde"}) {
|
||||
t.Fatalf("en list collaterally damaged: %v", got.Words)
|
||||
}
|
||||
|
||||
// Case and whitespace in the tag must not split one list into three.
|
||||
got = decodeWords(t, do(t, srv, http.MethodGet, "/spell/words?lang=EN", ""))
|
||||
if !equal(got.Words, []string{"tarde"}) {
|
||||
t.Fatalf("uppercase lang tag saw a different list: %v", got.Words)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectsJunk(t *testing.T) {
|
||||
srv, _ := newTestServer(t)
|
||||
|
||||
cases := []struct{ name, method, path, body string }{
|
||||
{"empty word", http.MethodPost, "/spell/words", `{"word":" "}`},
|
||||
{"no word at all", http.MethodPost, "/spell/words", `{"lang":"en"}`},
|
||||
{"not json", http.MethodPost, "/spell/words", `nonsense`},
|
||||
{"delete without a word", http.MethodDelete, "/spell/words", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if rec := do(t, srv, c.method, c.path, c.body); rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("%s: code=%d, want 400", c.name, rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// An over-long entry is dropped rather than stored — a pasted paragraph is
|
||||
// not a word. Dropping the only entry leaves nothing to add, hence 400.
|
||||
long := `{"word":"` + string(bytes.Repeat([]byte("a"), MaxWordLen+1)) + `"}`
|
||||
if rec := do(t, srv, http.MethodPost, "/spell/words", long); rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("over-long word: code=%d, want 400", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTwoUsersDoNotShare is the standing rule for every user-scoped endpoint:
|
||||
// mount the same routes twice behind two resolvers over one database.
|
||||
func TestTwoUsersDoNotShare(t *testing.T) {
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
if _, err := database.Exec(
|
||||
`INSERT INTO users (id, email, display_name) VALUES (?, ?, ?)`,
|
||||
"bob", "bob@petal.local", "Bob",
|
||||
); err != nil {
|
||||
t.Fatalf("seed second user: %v", err)
|
||||
}
|
||||
mount := func(userID string) http.Handler {
|
||||
r := chi.NewRouter()
|
||||
r.Mount("/spell", New(database).Routes())
|
||||
return auth.Middleware(auth.StaticResolver(userID))(r)
|
||||
}
|
||||
alice, bob := mount(db.LocalUserID), mount("bob")
|
||||
|
||||
do(t, alice, http.MethodPost, "/spell/words", `{"words":["Xiaolan","hanfu"]}`)
|
||||
|
||||
// Bob sees none of it — a personal dictionary is built from private writing.
|
||||
if got := decodeWords(t, do(t, bob, http.MethodGet, "/spell/words", "")); len(got.Words) != 0 {
|
||||
t.Fatalf("bob sees alice's words: %v", got.Words)
|
||||
}
|
||||
|
||||
// Bob's own identical word is his own row, and deleting it leaves hers.
|
||||
do(t, bob, http.MethodPost, "/spell/words", `{"word":"hanfu"}`)
|
||||
do(t, bob, http.MethodDelete, "/spell/words?word=hanfu", "")
|
||||
if got := decodeWords(t, do(t, alice, http.MethodGet, "/spell/words", "")); !equal(got.Words, []string{"Xiaolan", "hanfu"}) {
|
||||
t.Fatalf("bob's delete reached alice's list: %v", got.Words)
|
||||
}
|
||||
|
||||
// Deleting the account takes the dictionary with it.
|
||||
if _, err := database.Exec(`DELETE FROM users WHERE id = 'bob'`); err != nil {
|
||||
t.Fatalf("delete user: %v", err)
|
||||
}
|
||||
var n int
|
||||
if err := database.QueryRow(
|
||||
`SELECT COUNT(*) FROM personal_words WHERE user_id = 'bob'`).Scan(&n); err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Fatalf("%d orphaned rows after the user was deleted", n)
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
)
|
||||
@@ -40,14 +40,17 @@ func (h *Handler) chat(w http.ResponseWriter, r *http.Request) {
|
||||
original, replacement, explanation, typ string
|
||||
fromPos int
|
||||
contentText string
|
||||
pairLang string
|
||||
)
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT s.original, s.replacement, s.explanation, s.type, s.from_pos, d.content_text
|
||||
`SELECT s.original, s.replacement, s.explanation, s.type, s.from_pos, d.content_text,
|
||||
COALESCE(u.pair_lang, '')
|
||||
FROM suggestions s
|
||||
JOIN documents d ON d.id = s.doc_id
|
||||
JOIN users u ON u.id = d.user_id
|
||||
WHERE s.id = ? AND d.user_id = ?`,
|
||||
sugID, db.LocalUserID,
|
||||
).Scan(&original, &replacement, &explanation, &typ, &fromPos, &contentText)
|
||||
sugID, auth.UserID(r.Context()),
|
||||
).Scan(&original, &replacement, &explanation, &typ, &fromPos, &contentText, &pairLang)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusNotFound, "suggestion not found")
|
||||
return
|
||||
@@ -58,7 +61,7 @@ func (h *Handler) chat(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
paragraph := surroundingParagraph(contentText, fromPos)
|
||||
systemPrompt := llm.AskPetalSystemPrompt(original, replacement, typ, explanation, paragraph)
|
||||
systemPrompt := llm.AskPetalSystemPrompt(original, replacement, typ, explanation, paragraph, llm.LangFor(pairLang))
|
||||
|
||||
// SSE requires an unbuffered, flushable writer. chi's middleware writers pass
|
||||
// Flush through; bail with a plain error if somehow they don't.
|
||||
@@ -72,7 +75,7 @@ func (h *Handler) chat(w http.ResponseWriter, r *http.Request) {
|
||||
if err != nil {
|
||||
// The stream never opened (e.g. LLM unreachable) — a normal JSON error is
|
||||
// still appropriate since we haven't written SSE headers yet.
|
||||
httputil.ErrorJSON(w, http.StatusBadGateway, "chat failed: "+err.Error())
|
||||
httputil.UpstreamError(w, "chat", err)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Chunking splits a document into sentence-sized units so a re-check can ask the
|
||||
// model only about the sentences that actually changed. Accepting one edit used
|
||||
// to re-run the whole document: every card vanished, came back with a new id and
|
||||
// a freshly-worded explanation, and spans re-merged into different shapes. The
|
||||
// sentences she didn't touch have nothing new to say about themselves, so their
|
||||
// suggestions are simply kept (see reconcilePending).
|
||||
//
|
||||
// A chunk's identity is its hash, not its position — she inserts a paragraph at
|
||||
// the top and every sentence below keeps its suggestions.
|
||||
|
||||
// chunk is one sentence of the document, with the hash that identifies it.
|
||||
type chunk struct {
|
||||
text string
|
||||
hash string
|
||||
}
|
||||
|
||||
// asciiTerminators end a sentence only when whitespace (or the end of the text)
|
||||
// follows, so "3.5" and "Ms." don't split mid-word — a wrong split costs only a
|
||||
// slightly smaller chunk, but a split inside a number would churn its hash on
|
||||
// every keystroke around it.
|
||||
const asciiTerminators = ".!?"
|
||||
|
||||
// cjkTerminators end a sentence outright: Chinese runs sentences together with
|
||||
// no space after 。, and she writes in both languages in one document.
|
||||
const cjkTerminators = "。!?"
|
||||
|
||||
// closers are swallowed into the sentence they close, so the quote mark travels
|
||||
// with the sentence rather than opening the next one.
|
||||
const closers = `)]}"'’”」』`
|
||||
|
||||
// splitChunks divides text into sentences, dropping whitespace-only runs.
|
||||
// Newlines always break a chunk, so a list or a line of dialogue is its own unit.
|
||||
//
|
||||
// `salt` distinguishes two *readings* of the same sentence. The grammar
|
||||
// checkpoint's advice depends on the document's tone — the same line gets
|
||||
// different notes as an academic essay than as a journal entry — so switching
|
||||
// tone must re-open every sentence rather than serve back advice written for the
|
||||
// old register.
|
||||
func splitChunks(text, salt string) []chunk {
|
||||
var out []chunk
|
||||
runes := []rune(text)
|
||||
start := 0
|
||||
add := func(end int) {
|
||||
if s := string(runes[start:end]); strings.TrimSpace(s) != "" {
|
||||
out = append(out, chunk{text: s, hash: hashChunk(s, salt)})
|
||||
}
|
||||
start = end
|
||||
}
|
||||
|
||||
for i := 0; i < len(runes); i++ {
|
||||
r := runes[i]
|
||||
if r == '\n' {
|
||||
add(i + 1)
|
||||
continue
|
||||
}
|
||||
cjk := strings.ContainsRune(cjkTerminators, r)
|
||||
if !cjk && !strings.ContainsRune(asciiTerminators, r) {
|
||||
continue
|
||||
}
|
||||
// Swallow a run of terminators ("?!", "…") and any closing punctuation.
|
||||
j := i + 1
|
||||
for j < len(runes) && (strings.ContainsRune(asciiTerminators+cjkTerminators+closers, runes[j])) {
|
||||
j++
|
||||
}
|
||||
if cjk || j >= len(runes) || unicode.IsSpace(runes[j]) {
|
||||
add(j)
|
||||
i = j - 1
|
||||
}
|
||||
}
|
||||
if start < len(runes) {
|
||||
add(len(runes))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hashChunk identifies a sentence by its content under the same normalization
|
||||
// the suppression logic uses: quote style and whitespace runs churn constantly
|
||||
// (the editor rewrites quotes as she types, a paragraph reflows) and none of
|
||||
// that changes what the sentence says, so none of it should cost a re-check.
|
||||
func hashChunk(s, salt string) string {
|
||||
sum := sha256.Sum256([]byte(salt + "\x00" + normalizeForDedup(s)))
|
||||
return hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
|
||||
// hashSet indexes chunks by hash — "is this sentence in the document?"
|
||||
func hashSet(chunks []chunk) map[string]bool {
|
||||
out := make(map[string]bool, len(chunks))
|
||||
for _, c := range chunks {
|
||||
out[c.hash] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changedChunks returns the chunks whose hash wasn't in the last checked set,
|
||||
// in document order and deduplicated — a sentence repeated verbatim is one
|
||||
// question, not two.
|
||||
func changedChunks(chunks []chunk, checked map[string]bool) []chunk {
|
||||
seen := make(map[string]bool, len(chunks))
|
||||
var out []chunk
|
||||
for _, c := range chunks {
|
||||
if checked[c.hash] || seen[c.hash] {
|
||||
continue
|
||||
}
|
||||
seen[c.hash] = true
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// joinChunks renders a chunk set as the text to hand the model: one sentence per
|
||||
// line, so two sentences pulled from opposite ends of the document don't read as
|
||||
// one run-on.
|
||||
func joinChunks(chunks []chunk) string {
|
||||
parts := make([]string, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
parts = append(parts, strings.TrimSpace(c.text))
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
// chunkFor names the sentence a suggestion belongs to: the first chunk whose
|
||||
// text contains the flagged span. Returns "" when the span straddles a sentence
|
||||
// boundary or the model paraphrased what it quoted — such a row is re-examined
|
||||
// on every pass rather than cached, which is the safe direction.
|
||||
func chunkFor(original string, chunks []chunk) string {
|
||||
o := normalizeForDedup(original)
|
||||
if o == "" {
|
||||
return ""
|
||||
}
|
||||
for _, c := range chunks {
|
||||
if strings.Contains(normalizeForDedup(c.text), o) {
|
||||
return c.hash
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func texts(chunks []chunk) []string {
|
||||
out := make([]string, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
out = append(out, strings.TrimSpace(c.text))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestSplitChunks(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "plain sentences",
|
||||
in: "I has two apple. She go to market yesterday! Why?",
|
||||
want: []string{"I has two apple.", "She go to market yesterday!", "Why?"},
|
||||
},
|
||||
{
|
||||
// A decimal must not split, or the sentence's identity would churn
|
||||
// while she types the number.
|
||||
name: "decimals stay whole",
|
||||
in: "It costs 3.50 today. Tomorrow, more.",
|
||||
want: []string{"It costs 3.50 today.", "Tomorrow, more."},
|
||||
},
|
||||
{
|
||||
name: "closing quote travels with its sentence",
|
||||
in: `He said "early," and left. She stayed.`,
|
||||
want: []string{`He said "early," and left.`, "She stayed."},
|
||||
},
|
||||
{
|
||||
// Chinese runs sentences together with no space after 。 — she writes
|
||||
// in both languages in one document.
|
||||
name: "cjk terminators split without a space",
|
||||
in: "我想说这句话。但是不知道用英语怎么说。",
|
||||
want: []string{"我想说这句话。", "但是不知道用英语怎么说。"},
|
||||
},
|
||||
{
|
||||
name: "newlines break chunks",
|
||||
in: "A list item\nAnother item\n",
|
||||
want: []string{"A list item", "Another item"},
|
||||
},
|
||||
{
|
||||
name: "blank runs are dropped",
|
||||
in: "\n\n \nOnly this.\n\n",
|
||||
want: []string{"Only this."},
|
||||
},
|
||||
{
|
||||
name: "trailing fragment is its own chunk",
|
||||
in: "Done. Still writing",
|
||||
want: []string{"Done.", "Still writing"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := texts(splitChunks(tc.in, ""))
|
||||
if len(got) != len(tc.want) {
|
||||
t.Fatalf("want %q, got %q", tc.want, got)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tc.want[i] {
|
||||
t.Fatalf("chunk %d: want %q, got %q", i, tc.want[i], got[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A sentence's identity survives the churn that doesn't change what it says:
|
||||
// the editor rewrites quotes as she types, and a paragraph reflows.
|
||||
func TestChunkIdentityIgnoresCosmeticChurn(t *testing.T) {
|
||||
a := splitChunks(`She said "hello" softly.`, "")
|
||||
b := splitChunks("She said “hello” softly.", "")
|
||||
if len(a) != 1 || len(b) != 1 {
|
||||
t.Fatalf("want one chunk each, got %d and %d", len(a), len(b))
|
||||
}
|
||||
if a[0].hash != b[0].hash {
|
||||
t.Fatalf("quote/whitespace churn changed the sentence's identity")
|
||||
}
|
||||
if same := splitChunks(`She said "hello" softly.`, "academic"); same[0].hash == a[0].hash {
|
||||
t.Fatalf("a different tone must be a different reading of the sentence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangedChunksAndLookup(t *testing.T) {
|
||||
chunks := splitChunks("One thing. Another thing. One thing.", "")
|
||||
if len(chunks) != 3 {
|
||||
t.Fatalf("want 3 chunks, got %d", len(chunks))
|
||||
}
|
||||
|
||||
// A repeated sentence is one question, not two.
|
||||
if got := changedChunks(chunks, nil); len(got) != 2 {
|
||||
t.Fatalf("want 2 distinct changed chunks, got %d", len(got))
|
||||
}
|
||||
|
||||
checked := hashSet(chunks[:1])
|
||||
changed := changedChunks(chunks, checked)
|
||||
if len(changed) != 1 || strings.TrimSpace(changed[0].text) != "Another thing." {
|
||||
t.Fatalf("want only the unread sentence, got %q", texts(changed))
|
||||
}
|
||||
|
||||
if chunkFor("Another", chunks) != chunks[1].hash {
|
||||
t.Fatalf("span was attributed to the wrong sentence")
|
||||
}
|
||||
// A span the document doesn't contain has no sentence, so it is never cached.
|
||||
if chunkFor("nowhere in here", chunks) != "" {
|
||||
t.Fatalf("unanchorable span should have no chunk")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
"gitea.parodia.dev/drwily/petal/internal/vocab"
|
||||
)
|
||||
|
||||
// The growth journal.
|
||||
//
|
||||
// The suggestions table already records everything this needs — it is purely a
|
||||
// read-side view, with no new capture and no model call. Two framing rules
|
||||
// decide what may appear here, and they are enforced in the SQL rather than left
|
||||
// to the copy:
|
||||
//
|
||||
// 1. It reports growth, never an error tally. Nothing counts what she got
|
||||
// wrong this month; the signals are things that *stopped* happening and
|
||||
// phrasing that *stuck*.
|
||||
// 2. It only ever compares the writer to her own past self. There is no
|
||||
// target, no average, no other user anywhere in these queries.
|
||||
//
|
||||
// A quiet month is quiet: every signal below is omitted rather than softened
|
||||
// when the data isn't there, because an invented milestone is worse than none.
|
||||
|
||||
// Journal is one writer's growth over the recent windows.
|
||||
type Journal struct {
|
||||
// Kept / KeptBefore are edits she took on board in the last 30 days and in
|
||||
// the 30 before that — her own past self, the only comparison offered.
|
||||
Kept int `json:"kept"`
|
||||
KeptBefore int `json:"kept_before"`
|
||||
// Stuck: phrasing she was given that now turns up across her own writing.
|
||||
Stuck []Chunk `json:"stuck"`
|
||||
// Faded: things she used to need fixing and hasn't, recently.
|
||||
Faded []Fade `json:"faded"`
|
||||
}
|
||||
|
||||
// Chunk is a phrase that has stuck: it appears in Docs of her documents now.
|
||||
type Chunk struct {
|
||||
Phrase string `json:"phrase"`
|
||||
Docs int `json:"docs"`
|
||||
}
|
||||
|
||||
// Fade is a pattern that has stopped appearing. Times is how often it came up
|
||||
// during the earlier window — context for "and not since", never a scoreboard.
|
||||
type Fade struct {
|
||||
Pattern string `json:"pattern"`
|
||||
Times int `json:"times"`
|
||||
}
|
||||
|
||||
// Journal windows, in days. `recent` is the month being reported on; `history`
|
||||
// reaches back far enough that a pattern's absence means something (one quiet
|
||||
// fortnight doesn't).
|
||||
const (
|
||||
recentDays = 30
|
||||
historyDays = 120
|
||||
maxSignals = 3 // per list: a journal is a couple of warm lines, not a report
|
||||
)
|
||||
|
||||
// growth serves GET /api/suggestions/growth.
|
||||
func (h *Handler) growth(w http.ResponseWriter, r *http.Request) {
|
||||
userID := auth.UserID(r.Context())
|
||||
j := Journal{Stuck: []Chunk{}, Faded: []Fade{}}
|
||||
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT
|
||||
sum(CASE WHEN s.resolved_at >= datetime('now', '-30 days') THEN 1 ELSE 0 END),
|
||||
sum(CASE WHEN s.resolved_at < datetime('now', '-30 days')
|
||||
AND s.resolved_at >= datetime('now', '-60 days') THEN 1 ELSE 0 END)
|
||||
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||
WHERE d.user_id = ? AND s.status = 'accepted' AND s.resolved_at IS NOT NULL`,
|
||||
userID,
|
||||
).Scan(&nullInt{&j.Kept}, &nullInt{&j.KeptBefore})
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
stuck, err := h.stuck(userID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
j.Stuck = stuck
|
||||
|
||||
faded, err := h.faded(userID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
j.Faded = faded
|
||||
|
||||
httputil.WriteJSON(w, http.StatusOK, j)
|
||||
}
|
||||
|
||||
// stuck finds accepted phrasing that now appears in more than one of her own
|
||||
// documents. One document is just the edit itself, still sitting where it was
|
||||
// applied; a second is her reaching for the phrase on her own, which is the
|
||||
// whole claim the line makes.
|
||||
func (h *Handler) stuck(userID string) ([]Chunk, error) {
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT DISTINCT s.replacement
|
||||
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||
WHERE d.user_id = ? AND s.status = 'accepted'
|
||||
AND s.resolved_at >= datetime('now', '-120 days')
|
||||
AND trim(s.replacement) != ''
|
||||
ORDER BY s.resolved_at DESC
|
||||
LIMIT 40`,
|
||||
userID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// vocab.PhraseKey is the same definition of "a learnable chunk" the garden
|
||||
// plants, so the journal and the garden can never disagree about what counts.
|
||||
var phrases []string
|
||||
for rows.Next() {
|
||||
var replacement string
|
||||
if err := rows.Scan(&replacement); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if key := vocab.PhraseKey(replacement); key != "" {
|
||||
phrases = append(phrases, key)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := []Chunk{}
|
||||
for _, p := range phrases {
|
||||
var docs int
|
||||
if err := h.DB.QueryRow(
|
||||
`SELECT count(*) FROM documents WHERE user_id = ? AND instr(lower(content_text), ?) > 0`,
|
||||
userID, p,
|
||||
).Scan(&docs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if docs >= 2 {
|
||||
out = append(out, Chunk{Phrase: p, Docs: docs})
|
||||
}
|
||||
}
|
||||
sortDesc(out, func(c Chunk) int { return c.Docs })
|
||||
return trim(out, maxSignals), nil
|
||||
}
|
||||
|
||||
// faded finds patterns she used to be corrected on during the earlier part of
|
||||
// the history window and hasn't been since.
|
||||
//
|
||||
// The guard that makes this honest: it says nothing at all unless she has
|
||||
// actually been writing lately. Without it, a month away from Petal would be
|
||||
// reported back to her as progress, which is the one way this feature could lie.
|
||||
func (h *Handler) faded(userID string) ([]Fade, error) {
|
||||
var wroteRecently int
|
||||
if err := h.DB.QueryRow(
|
||||
`SELECT count(*) FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||
WHERE d.user_id = ? AND s.resolved_at >= datetime('now', '-30 days')`,
|
||||
userID,
|
||||
).Scan(&wroteRecently); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if wroteRecently == 0 {
|
||||
return []Fade{}, nil
|
||||
}
|
||||
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT lower(trim(s.original)) AS pattern, count(*) AS times
|
||||
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||
WHERE d.user_id = ? AND s.status = 'accepted'
|
||||
AND s.resolved_at < datetime('now', '-30 days')
|
||||
AND s.resolved_at >= datetime('now', '-120 days')
|
||||
AND trim(s.original) != ''
|
||||
AND pattern NOT IN (
|
||||
SELECT lower(trim(s2.original))
|
||||
FROM suggestions s2 JOIN documents d2 ON d2.id = s2.doc_id
|
||||
WHERE d2.user_id = ? AND s2.status = 'accepted'
|
||||
AND s2.resolved_at >= datetime('now', '-30 days'))
|
||||
GROUP BY pattern
|
||||
HAVING times >= 2
|
||||
ORDER BY times DESC
|
||||
LIMIT 3`,
|
||||
userID, userID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []Fade{}
|
||||
for rows.Next() {
|
||||
var f Fade
|
||||
if err := rows.Scan(&f.Pattern, &f.Times); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// nullInt scans a possibly-NULL aggregate into an int (SUM over no rows is
|
||||
// NULL, which is a zero here, not an error).
|
||||
type nullInt struct{ dst *int }
|
||||
|
||||
func (n *nullInt) Scan(v any) error {
|
||||
switch t := v.(type) {
|
||||
case int64:
|
||||
*n.dst = int(t)
|
||||
case nil:
|
||||
*n.dst = 0
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortDesc[T any](s []T, key func(T) int) {
|
||||
for i := 1; i < len(s); i++ {
|
||||
for j := i; j > 0 && key(s[j]) > key(s[j-1]); j-- {
|
||||
s[j], s[j-1] = s[j-1], s[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func trim[T any](s []T, n int) []T {
|
||||
if len(s) > n {
|
||||
return s[:n]
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// resolved seeds one already-settled suggestion, dated `daysAgo` at the moment
|
||||
// she decided it (the journal reads decisions, not proposals).
|
||||
func resolved(t *testing.T, h *Handler, docID, status, original, replacement string, daysAgo int) {
|
||||
t.Helper()
|
||||
_, err := h.DB.Exec(
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at, resolved_at)
|
||||
VALUES (?, 0, 0, ?, ?, '', 'collocation', ?, datetime('now', ?), datetime('now', ?))`,
|
||||
docID, original, replacement, status,
|
||||
"-"+strconv.Itoa(daysAgo)+" days", "-"+strconv.Itoa(daysAgo)+" days",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("seed resolved suggestion: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func seedDoc(t *testing.T, h *Handler, userID, text string) string {
|
||||
t.Helper()
|
||||
var id string
|
||||
if err := h.DB.QueryRow(
|
||||
`INSERT INTO documents (user_id, content_text) VALUES (?, ?) RETURNING id`, userID, text,
|
||||
).Scan(&id); err != nil {
|
||||
t.Fatalf("seed doc: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func readJournal(t *testing.T, srv http.Handler) Journal {
|
||||
t.Helper()
|
||||
rec := do(t, srv, http.MethodGet, "/suggestions/growth", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("growth: got %d, want 200 (body %s)", rec.Code, rec.Body.String())
|
||||
}
|
||||
var j Journal
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &j); err != nil {
|
||||
t.Fatalf("decode journal: %v", err)
|
||||
}
|
||||
return j
|
||||
}
|
||||
|
||||
// TestJournalIsEmptyForANewWriter: nothing to report reports nothing. Empty
|
||||
// lists, not nulls, so the frontend never has to guess.
|
||||
func TestJournalIsEmptyForANewWriter(t *testing.T) {
|
||||
srv, _, _ := newTestServer(t, &stubClient{})
|
||||
j := readJournal(t, srv)
|
||||
if j.Kept != 0 || j.KeptBefore != 0 || len(j.Stuck) != 0 || len(j.Faded) != 0 {
|
||||
t.Fatalf("new writer got a journal: %+v", j)
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeptComparesHerToHerOwnPastSelf.
|
||||
func TestKeptCountsTwoWindows(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
for i := 0; i < 3; i++ {
|
||||
resolved(t, h, docID, "accepted", "do a decision", "make a decision", 5)
|
||||
}
|
||||
resolved(t, h, docID, "accepted", "big rain", "heavy rain", 40)
|
||||
resolved(t, h, docID, "rejected", "no thanks", "no, thank you", 5) // decisions kept only
|
||||
resolved(t, h, docID, "accepted", "long ago", "long since", 200) // outside both windows
|
||||
|
||||
j := readJournal(t, srv)
|
||||
if j.Kept != 3 {
|
||||
t.Errorf("Kept = %d, want 3", j.Kept)
|
||||
}
|
||||
if j.KeptBefore != 1 {
|
||||
t.Errorf("KeptBefore = %d, want 1", j.KeptBefore)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStuckNeedsASecondDocument: a phrase sitting in the one document it was
|
||||
// applied to has not stuck — it's just the edit, where she left it. A second
|
||||
// document is her reaching for it herself, which is the claim the line makes.
|
||||
func TestStuckNeedsASecondDocument(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
if _, err := h.DB.Exec(`UPDATE documents SET content_text = ? WHERE id = ?`,
|
||||
"I had to make a decision.", docID); err != nil {
|
||||
t.Fatalf("set content: %v", err)
|
||||
}
|
||||
resolved(t, h, docID, "accepted", "do a decision", "make a decision", 10)
|
||||
resolved(t, h, docID, "accepted", "do a photo", "take a photo", 10)
|
||||
|
||||
if j := readJournal(t, srv); len(j.Stuck) != 0 {
|
||||
t.Fatalf("one document counted as sticking: %+v", j.Stuck)
|
||||
}
|
||||
|
||||
// She uses it again, elsewhere, on her own.
|
||||
seedDoc(t, h, db.LocalUserID, "Later I had to Make A Decision about the flat.")
|
||||
j := readJournal(t, srv)
|
||||
if len(j.Stuck) != 1 {
|
||||
t.Fatalf("Stuck = %+v, want just the phrase she reused", j.Stuck)
|
||||
}
|
||||
if j.Stuck[0].Phrase != "make a decision" || j.Stuck[0].Docs != 2 {
|
||||
t.Errorf("Stuck[0] = %+v, want {make a decision 2} (case-insensitive)", j.Stuck[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestFadedNeedsRecentWriting is the guard that keeps this feature honest: a
|
||||
// month away from Petal must never be reported back as progress.
|
||||
func TestFadedNeedsRecentWriting(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 60)
|
||||
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 55)
|
||||
|
||||
if j := readJournal(t, srv); len(j.Faded) != 0 {
|
||||
t.Fatalf("silence reported as growth: %+v", j.Faded)
|
||||
}
|
||||
|
||||
// She has been writing again this month — now the absence means something.
|
||||
resolved(t, h, docID, "accepted", "big rain", "heavy rain", 3)
|
||||
j := readJournal(t, srv)
|
||||
if len(j.Faded) != 1 || j.Faded[0].Pattern != "在 the morning" || j.Faded[0].Times != 2 {
|
||||
t.Fatalf("Faded = %+v, want the pattern she stopped needing (twice, back then)", j.Faded)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFadedExcludesWhatStillHappens: a pattern corrected again this month has
|
||||
// not faded, however often it came up before.
|
||||
func TestFadedExcludesWhatStillHappens(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 60)
|
||||
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 55)
|
||||
resolved(t, h, docID, "accepted", "在 the morning", "in the morning", 2)
|
||||
|
||||
if j := readJournal(t, srv); len(j.Faded) != 0 {
|
||||
t.Fatalf("Faded = %+v, want empty — it still happens", j.Faded)
|
||||
}
|
||||
}
|
||||
|
||||
// TestJournalIsPerWriter: another account's learning is never anyone else's
|
||||
// journal, and the only comparison Petal draws is with her own past self.
|
||||
func TestJournalIsPerWriter(t *testing.T) {
|
||||
srv, _, h := newTestServer(t, &stubClient{})
|
||||
if _, err := h.DB.Exec(`INSERT INTO users (id, email) VALUES ('bob', 'bob@example.com')`); err != nil {
|
||||
t.Fatalf("seed user: %v", err)
|
||||
}
|
||||
bobDoc := seedDoc(t, h, "bob", "Bob had to make a decision.")
|
||||
seedDoc(t, h, "bob", "Bob will make a decision again.")
|
||||
resolved(t, h, bobDoc, "accepted", "do a decision", "make a decision", 5)
|
||||
resolved(t, h, bobDoc, "accepted", "big rain", "heavy rain", 60)
|
||||
resolved(t, h, bobDoc, "accepted", "big rain", "heavy rain", 55)
|
||||
|
||||
j := readJournal(t, srv)
|
||||
if j.Kept != 0 || j.KeptBefore != 0 || len(j.Stuck) != 0 || len(j.Faded) != 0 {
|
||||
t.Fatalf("bob's learning leaked into the local user's journal: %+v", j)
|
||||
}
|
||||
}
|
||||
+354
-105
@@ -11,14 +11,17 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
"gitea.parodia.dev/drwily/petal/internal/vocab"
|
||||
)
|
||||
|
||||
// Handler holds the dependencies for the checkpoint + suggestion routes. The
|
||||
@@ -53,12 +56,17 @@ func (h *Handler) RegisterDocRoutes(r chi.Router) {
|
||||
r.Post("/{id}/collocation", h.collocation)
|
||||
r.Post("/{id}/rewrite", h.rewrite)
|
||||
r.Get("/{id}/suggestions", h.listForDoc)
|
||||
r.Get("/{id}/settled", h.listSettled)
|
||||
}
|
||||
|
||||
// Routes returns the router mounted at /api/suggestions for per-suggestion
|
||||
// actions.
|
||||
func (h *Handler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
// The growth journal reads the same table these actions write, so it lives
|
||||
// here rather than growing its own mount. A literal segment, so it can never
|
||||
// be shadowed by an id.
|
||||
r.Get("/growth", h.growth)
|
||||
r.Post("/{id}/accept", h.accept)
|
||||
r.Post("/{id}/dismiss", h.dismiss)
|
||||
r.Post("/{id}/chat", h.chat)
|
||||
@@ -83,6 +91,22 @@ type mechanicsFinding struct {
|
||||
Original string `json:"original"`
|
||||
Replacement string `json:"replacement"`
|
||||
Explanation string `json:"explanation"`
|
||||
// Which family this offline finding belongs to. Empty (the historical shape)
|
||||
// means mechanics; the miscollocation rules send 'collocation' so a chunk the
|
||||
// rule pack caught is indistinguishable from one the coach caught — same
|
||||
// family, same rail, and the same planting into the garden on accept.
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
// localType maps a client-supplied family onto the two an offline rule may claim.
|
||||
// Anything else — including the empty string older clients send — is mechanics,
|
||||
// so a stray label can never smuggle a row into an LLM family and survive that
|
||||
// pass's DELETE.
|
||||
func localType(t string) string {
|
||||
if strings.ToLower(strings.TrimSpace(t)) == db.SuggestionTypeCollocation {
|
||||
return db.SuggestionTypeCollocation
|
||||
}
|
||||
return db.SuggestionTypeMechanics
|
||||
}
|
||||
|
||||
// maxMechanicsFindings caps a single submission so a runaway client can't flood
|
||||
@@ -98,11 +122,11 @@ const maxMechanicsFindings = 500
|
||||
func (h *Handler) mechanics(w http.ResponseWriter, r *http.Request) {
|
||||
docID := chi.URLParam(r, "id")
|
||||
|
||||
// Confirm the document exists (and is the local user's) for clean 404s.
|
||||
// Confirm the document exists (and belongs to the caller) for clean 404s.
|
||||
var exists bool
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT EXISTS(SELECT 1 FROM documents WHERE id = ? AND user_id = ?)`,
|
||||
docID, db.LocalUserID,
|
||||
docID, auth.UserID(r.Context()),
|
||||
).Scan(&exists)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -129,7 +153,7 @@ func (h *Handler) mechanics(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
out, err := h.fetchPending(docID)
|
||||
out, err := h.fetchPending(auth.UserID(r.Context()), docID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
@@ -137,10 +161,21 @@ func (h *Handler) mechanics(w http.ResponseWriter, r *http.Request) {
|
||||
httputil.WriteJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// replaceMechanics swaps the document's pending mechanics rows for the supplied
|
||||
// findings in one transaction, leaving the LLM families and actioned rows
|
||||
// untouched. Findings the user already accepted or dismissed are suppressed (the
|
||||
// detector has no memory between runs), and malformed spans are skipped.
|
||||
// replaceMechanics brings the document's pending offline rows in line with the
|
||||
// supplied findings in one transaction, leaving the LLM families and actioned
|
||||
// rows untouched. Findings the user already accepted or dismissed are suppressed
|
||||
// (the detector has no memory between runs), and malformed spans are skipped.
|
||||
//
|
||||
// A finding the detector still reports keeps its existing row — same id, same
|
||||
// created_at — and only its offsets move. This pass fires 250 ms after a
|
||||
// keystroke, so deleting and re-inserting the family would hand every card a new
|
||||
// identity several times a sentence: the rail would remount, a card expanded for
|
||||
// Ask Petal would collapse under her, and the arrival chime would re-fire.
|
||||
//
|
||||
// The scope is *source*, not type: the rule pack owns both the mechanics family
|
||||
// and its share of the collocation family, and every run is a full recompute of
|
||||
// the document. Scoping by type instead would strand offline collocations the
|
||||
// current text no longer warrants — the one row nobody would ever replace.
|
||||
func (h *Handler) replaceMechanics(docID string, findings []mechanicsFinding) error {
|
||||
tx, err := h.DB.Begin()
|
||||
if err != nil {
|
||||
@@ -148,18 +183,18 @@ func (h *Handler) replaceMechanics(docID string, findings []mechanicsFinding) er
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
if _, err := tx.Exec(
|
||||
`DELETE FROM suggestions WHERE doc_id = ? AND status = ? AND type = ?`,
|
||||
docID, db.SuggestionStatusPending, db.SuggestionTypeMechanics,
|
||||
); err != nil {
|
||||
existing, err := loadPending(tx, docID, "source = '"+db.SuggestionSourceLocal+"'")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
index := indexByEdit(existing)
|
||||
|
||||
sup, err := buildSuppressor(tx, docID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
kept := make(map[string]bool, len(existing))
|
||||
for _, f := range findings {
|
||||
if f.From < 0 || f.To <= f.From || strings.TrimSpace(f.Original) == "" {
|
||||
continue // malformed span — the client re-anchors by string anyway
|
||||
@@ -167,15 +202,34 @@ func (h *Handler) replaceMechanics(docID string, findings []mechanicsFinding) er
|
||||
if sup.suppressed(f.Original, f.Replacement) {
|
||||
continue
|
||||
}
|
||||
typ := localType(f.Type)
|
||||
if row, ok := index.take(f.Original, f.Replacement, f.From); ok {
|
||||
kept[row.id] = true
|
||||
if err := reposition(tx, row, f.From, f.To, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
docID, f.From, f.To, f.Original, f.Replacement, f.Explanation, db.SuggestionTypeMechanics,
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, source)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
docID, f.From, f.To, f.Original, f.Replacement, f.Explanation,
|
||||
typ, db.SuggestionSourceLocal,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Whatever the detector no longer reports, she has fixed.
|
||||
for _, row := range existing {
|
||||
if kept[row.id] {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, row.id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@@ -193,9 +247,12 @@ func (h *Handler) collocation(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// pass is the signature shared by the grammar checkpoint and the voice pass:
|
||||
// given the document text and the document's tone it returns the model's raw
|
||||
// suggestions. The voice pass ignores tone (see llm.RunVoice).
|
||||
type pass func(ctx context.Context, client llm.LLMClient, contentText, tone string) ([]llm.RawSuggestion, error)
|
||||
// given the document text, the document's tone and the writer's pair language it
|
||||
// returns the model's raw suggestions. The voice pass ignores both extras (see
|
||||
// llm.RunVoice) and the checkpoint ignores the language — only the collocation
|
||||
// coach writes a word of it — but one signature keeps runPass free of special
|
||||
// cases.
|
||||
type pass func(ctx context.Context, client llm.LLMClient, contentText, tone string, lang llm.Lang) ([]llm.RawSuggestion, error)
|
||||
|
||||
// runPass is the shared body for both LLM passes. It loads the document text,
|
||||
// enforces the pass's per-document rate limit, runs the model, swaps in the
|
||||
@@ -203,12 +260,19 @@ type pass func(ctx context.Context, client llm.LLMClient, contentText, tone stri
|
||||
// (both families) so the client always renders a unified picture.
|
||||
func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.RateLimiter, run pass, scope pendingScope) {
|
||||
docID := chi.URLParam(r, "id")
|
||||
userID := auth.UserID(r.Context())
|
||||
|
||||
var contentText, tone string
|
||||
// The writer's pair language rides along with the document rather than in a
|
||||
// second query: it is read from the same row-scoped lookup that already
|
||||
// proves she owns this document.
|
||||
var contentText, tone, pairLang string
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT content_text, tone FROM documents WHERE id = ? AND user_id = ?`,
|
||||
docID, db.LocalUserID,
|
||||
).Scan(&contentText, &tone)
|
||||
`SELECT d.content_text, d.tone, COALESCE(u.pair_lang, '')
|
||||
FROM documents d
|
||||
JOIN users u ON u.id = d.user_id
|
||||
WHERE d.id = ? AND d.user_id = ?`,
|
||||
docID, userID,
|
||||
).Scan(&contentText, &tone, &pairLang)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusNotFound, "document not found")
|
||||
return
|
||||
@@ -218,17 +282,70 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
||||
return
|
||||
}
|
||||
|
||||
// Nothing to analyze on an empty document — skip the LLM round-trip.
|
||||
// Nothing to analyze on an empty document — skip the LLM round-trip. The
|
||||
// family's rows go with the text they were about.
|
||||
if strings.TrimSpace(contentText) == "" {
|
||||
httputil.WriteJSON(w, http.StatusOK, []db.Suggestion{})
|
||||
if err := h.reconcilePending(docID, contentText, pairLang, nil, scope, nil, nil, false); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
out, err := h.fetchPending(userID, docID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, out)
|
||||
return
|
||||
}
|
||||
|
||||
// Decide what to ask about before spending anything: a chunked pass asks only
|
||||
// about the sentences that changed since it last read the document, and when
|
||||
// none did it doesn't call the model at all — nor consume its rate-limit slot,
|
||||
// so the next real edit isn't throttled by a check that had nothing to do.
|
||||
//
|
||||
// Only a chunked pass consults that record, so only it needs the tone folded
|
||||
// into a sentence's identity.
|
||||
salt := ""
|
||||
if scope.chunked {
|
||||
salt = tone
|
||||
}
|
||||
chunks := splitChunks(contentText, salt)
|
||||
askText, fresh := contentText, chunks
|
||||
if scope.chunked {
|
||||
checked, err := h.checkedChunks(docID, scope.family)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
changed := changedChunks(chunks, checked)
|
||||
if len(changed) == 0 {
|
||||
// Every sentence has already been read. Drop the rows whose sentence is
|
||||
// gone, keep the rest exactly as they are, and answer immediately.
|
||||
if err := h.reconcilePending(docID, contentText, pairLang, nil, scope, chunks, nil, false); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
out, err := h.fetchPending(userID, docID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, out)
|
||||
return
|
||||
}
|
||||
// When every sentence is new — a first pass, a paste, a tone switch — hand
|
||||
// over the document verbatim so the model reads it with its paragraphing
|
||||
// intact. Otherwise send just the delta, one sentence per line.
|
||||
if len(changed) < len(hashSet(chunks)) {
|
||||
askText, fresh = joinChunks(changed), changed
|
||||
}
|
||||
}
|
||||
|
||||
ok, _, slotAt := limiter.Allow(docID)
|
||||
if !ok {
|
||||
// Throttled: return the existing pending set unchanged rather than an
|
||||
// error, so the frontend keeps showing current suggestions.
|
||||
existing, err := h.fetchPending(docID)
|
||||
existing, err := h.fetchPending(userID, docID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
@@ -237,17 +354,19 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
||||
return
|
||||
}
|
||||
|
||||
raw, err := run(r.Context(), h.Client, contentText, tone)
|
||||
raw, err := run(r.Context(), h.Client, askText, tone, llm.LangFor(pairLang))
|
||||
if err != nil {
|
||||
// Allow ran before the model call, so a failed pass would otherwise hold
|
||||
// the per-document slot for the full interval — stranding the frontend's
|
||||
// auto-retry on the throttle path. Release it so a retry can re-run.
|
||||
limiter.Release(docID, slotAt)
|
||||
httputil.ErrorJSON(w, http.StatusBadGateway, "llm pass failed: "+err.Error())
|
||||
httputil.UpstreamError(w, "pass", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.replacePending(docID, contentText, raw, scope); err != nil {
|
||||
// A whole-document pass re-read everything, so every one of its rows is up for
|
||||
// re-proposal; a chunked pass only puts the sentences it asked about in play.
|
||||
if err := h.reconcilePending(docID, contentText, pairLang, raw, scope, chunks, fresh, !scope.chunked); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
@@ -255,7 +374,7 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
||||
// Return the unified pending set (grammar + voice), not just this batch, so
|
||||
// a grammar check never drops the voice highlights from the client and the
|
||||
// throttle path above stays consistent with the success path.
|
||||
out, err := h.fetchPending(docID)
|
||||
out, err := h.fetchPending(userID, docID)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
@@ -268,73 +387,49 @@ func (h *Handler) runPass(w http.ResponseWriter, r *http.Request, limiter *llm.R
|
||||
// inserts. The grammar checkpoint and voice pass each own a disjoint family, so
|
||||
// running one never disturbs the other's pending flags.
|
||||
type pendingScope struct {
|
||||
deleteWhere string // extra WHERE clause scoping the DELETE to this family
|
||||
deleteWhere string // extra WHERE clause scoping this pass to its own family
|
||||
forceType string // if set, every inserted row gets this type; else normalizeType
|
||||
// family keys the sentences this pass has already read (see checked_chunks).
|
||||
family string
|
||||
// chunked passes re-read only the sentences that changed. True for the typing-
|
||||
// cadence grammar checkpoint, which fires constantly and must feel still;
|
||||
// false for the explicit whole-document passes, where she pressed a button
|
||||
// asking for a fresh read of everything.
|
||||
chunked bool
|
||||
}
|
||||
|
||||
// Every scope below is confined to source='llm'. The offline rule pack owns its
|
||||
// own rows and recomputes them on each edit (see replaceMechanics); its findings
|
||||
// must survive all three model passes — including the collocation coach, which
|
||||
// now shares the collocation family with it.
|
||||
var (
|
||||
// grammarScope owns the grammar/phrasing/idiom/clarity flags — everything but
|
||||
// the other self-owned families (voice, collocation, mechanics), which run on
|
||||
// their own cadence/pass and must survive a grammar checkpoint. Notably the
|
||||
// deterministic mechanics pass writes its rows in the same /check request just
|
||||
// before this DELETE runs, so excluding it here is what keeps them alive.
|
||||
grammarScope = pendingScope{deleteWhere: "type NOT IN ('voice','collocation','mechanics')", forceType: ""}
|
||||
// voiceScope owns the voice flags only.
|
||||
voiceScope = pendingScope{deleteWhere: "type = 'voice'", forceType: db.SuggestionTypeVoice}
|
||||
// collocationScope owns the collocation flags only.
|
||||
collocationScope = pendingScope{deleteWhere: "type = 'collocation'", forceType: db.SuggestionTypeCollocation}
|
||||
// the other self-owned families (voice, collocation), which run on their own
|
||||
// cadence/pass and must survive a grammar checkpoint. Notably the offline pass
|
||||
// writes its rows in the same /check request just before this pass reconciles,
|
||||
// so the source clause is also what keeps them alive.
|
||||
grammarScope = pendingScope{
|
||||
deleteWhere: "source = 'llm' AND type NOT IN ('voice','collocation')",
|
||||
family: "grammar",
|
||||
chunked: true,
|
||||
}
|
||||
// voiceScope owns the model's voice flags only. Voice is a property of the
|
||||
// document as a whole — a sentence isn't inconsistent with itself — so this
|
||||
// pass always reads everything.
|
||||
voiceScope = pendingScope{
|
||||
deleteWhere: "source = 'llm' AND type = 'voice'",
|
||||
forceType: db.SuggestionTypeVoice,
|
||||
family: "voice",
|
||||
}
|
||||
// collocationScope owns the model's collocation flags only — the rule pack's
|
||||
// share of the same family is left standing.
|
||||
collocationScope = pendingScope{
|
||||
deleteWhere: "source = 'llm' AND type = 'collocation'",
|
||||
forceType: db.SuggestionTypeCollocation,
|
||||
family: "collocation",
|
||||
}
|
||||
)
|
||||
|
||||
// replacePending swaps a document's pending suggestions within one family for a
|
||||
// fresh batch in a single transaction. Accepted/rejected suggestions and the
|
||||
// other family's pending rows are left untouched.
|
||||
//
|
||||
// Suggestions touching a sentence the user already settled are suppressed from
|
||||
// the fresh batch (see suppressor): not just the identical edit re-proposed, but
|
||||
// reversals and re-polishing of the model's own just-accepted output — the
|
||||
// "fickle, keeps going back and forth on a few sentences" behavior. The model has
|
||||
// no memory between passes, so without this it re-opens resolved sentences every
|
||||
// checkpoint.
|
||||
func (h *Handler) replacePending(docID, contentText string, raw []llm.RawSuggestion, scope pendingScope) error {
|
||||
tx, err := h.DB.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
if _, err := tx.Exec(
|
||||
`DELETE FROM suggestions WHERE doc_id = ? AND status = ? AND `+scope.deleteWhere,
|
||||
docID, db.SuggestionStatusPending,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sup, err := buildSuppressor(tx, docID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, s := range raw {
|
||||
if sup.suppressed(s.Original, s.Replacement) {
|
||||
continue
|
||||
}
|
||||
typ := scope.forceType
|
||||
if typ == "" {
|
||||
typ = normalizeType(s.Type)
|
||||
}
|
||||
from, to := locate(contentText, s.Original)
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
docID, from, to, s.Original, s.Replacement, s.Explanation, typ,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// dedupQuoteReplacer folds every straight/curly single- and double-quote variant
|
||||
// (and backtick/acute accent) onto one canonical character. The editor and the
|
||||
// model both rewrite quotes between passes — a sentence accepted with "…" comes
|
||||
@@ -465,7 +560,7 @@ func buildSuppressor(tx *sql.Tx, docID string) (suppressor, error) {
|
||||
// listForDoc returns the document's current pending suggestions (used when the
|
||||
// editor loads a document, before any new checkpoint fires).
|
||||
func (h *Handler) listForDoc(w http.ResponseWriter, r *http.Request) {
|
||||
out, err := h.fetchPending(chi.URLParam(r, "id"))
|
||||
out, err := h.fetchPending(auth.UserID(r.Context()), chi.URLParam(r, "id"))
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
@@ -473,13 +568,89 @@ func (h *Handler) listForDoc(w http.ResponseWriter, r *http.Request) {
|
||||
httputil.WriteJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (h *Handler) fetchPending(docID string) ([]db.Suggestion, error) {
|
||||
// listSettled returns the normalized originals of every edit the user has
|
||||
// already accepted or dismissed on this document — the same spans buildSuppressor
|
||||
// drops on the server, handed to the client so its instant rule-pack pass can
|
||||
// drop them too.
|
||||
//
|
||||
// Without this the offline half of the loop has no memory. The rule pack detects
|
||||
// from the text alone and re-runs 250 ms after a keystroke, so a dismissed "the
|
||||
// the" comes straight back the moment she types anywhere in the document; the
|
||||
// server's reply then removes it again. That flicker is the visible symptom, but
|
||||
// the real one is worse: with the server unreachable — the case the rule pack
|
||||
// exists for — the reply never comes and a card she dismissed simply stays.
|
||||
//
|
||||
// Only the originals are sent. Replacements are the model's words, not hers, and
|
||||
// the client only needs to answer "has she settled this span?"
|
||||
func (h *Handler) listSettled(w http.ResponseWriter, r *http.Request) {
|
||||
out, err := h.fetchSettled(auth.UserID(r.Context()), chi.URLParam(r, "id"))
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
httputil.WriteJSON(w, http.StatusOK, settledResponse{Originals: out})
|
||||
}
|
||||
|
||||
// settledResponse wraps the list so the endpoint can grow a second field without
|
||||
// breaking a client that reads a bare array.
|
||||
type settledResponse struct {
|
||||
Originals []string `json:"originals"`
|
||||
}
|
||||
|
||||
// fetchSettled loads the distinct normalized originals of the document's actioned
|
||||
// rows. Scoped through documents for the same reason fetchPending is: an original
|
||||
// is a quotation of her writing.
|
||||
func (h *Handler) fetchSettled(userID, docID string) ([]string, error) {
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT id, doc_id, from_pos, to_pos, original, replacement, explanation, type, status, created_at
|
||||
FROM suggestions
|
||||
WHERE doc_id = ? AND status = ?
|
||||
ORDER BY from_pos ASC, created_at ASC`,
|
||||
docID, db.SuggestionStatusPending,
|
||||
`SELECT DISTINCT s.original
|
||||
FROM suggestions s
|
||||
JOIN documents d ON d.id = s.doc_id
|
||||
WHERE s.doc_id = ? AND d.user_id = ? AND s.status IN (?, ?)`,
|
||||
docID, userID, db.SuggestionStatusAccepted, db.SuggestionStatusRejected,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// DISTINCT is on the raw text; normalizing can collapse two rows into one, so
|
||||
// dedupe again on this side to keep the payload honest.
|
||||
seen := map[string]struct{}{}
|
||||
out := []string{}
|
||||
for rows.Next() {
|
||||
var original string
|
||||
if err := rows.Scan(&original); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
norm := normalizeForDedup(original)
|
||||
if norm == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[norm]; dup {
|
||||
continue
|
||||
}
|
||||
seen[norm] = struct{}{}
|
||||
out = append(out, norm)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// fetchPending loads a document's pending suggestions, joined through documents
|
||||
// so the rows are only reachable by the document's owner. A suggestion quotes the
|
||||
// sentence it corrects, so an unscoped read here would leak document text to
|
||||
// anyone holding a doc id.
|
||||
func (h *Handler) fetchPending(userID, docID string) ([]db.Suggestion, error) {
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT s.id, s.doc_id, s.from_pos, s.to_pos, s.original, s.replacement,
|
||||
s.explanation, s.type, s.status, s.source, s.created_at
|
||||
FROM suggestions s
|
||||
JOIN documents d ON d.id = s.doc_id
|
||||
WHERE s.doc_id = ? AND d.user_id = ? AND s.status = ?
|
||||
ORDER BY s.from_pos ASC, s.created_at ASC`,
|
||||
docID, userID, db.SuggestionStatusPending,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -491,7 +662,7 @@ func (h *Handler) fetchPending(docID string) ([]db.Suggestion, error) {
|
||||
var s db.Suggestion
|
||||
if err := rows.Scan(
|
||||
&s.ID, &s.DocID, &s.FromPos, &s.ToPos, &s.Original, &s.Replacement,
|
||||
&s.Explanation, &s.Type, &s.Status, &s.CreatedAt,
|
||||
&s.Explanation, &s.Type, &s.Status, &s.Source, &s.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -503,11 +674,14 @@ func (h *Handler) fetchPending(docID string) ([]db.Suggestion, error) {
|
||||
return dedupeSpans(out), nil
|
||||
}
|
||||
|
||||
// dedupeSpans resolves collisions between the deterministic mechanics family and
|
||||
// the LLM families: when a mechanics finding and an LLM suggestion fight over the
|
||||
// same characters, mechanics wins and the LLM card is dropped. Its span is exact
|
||||
// (the detector matched it), whereas the LLM positions are only advisory
|
||||
// (re-anchored by string at render), so the precise fix should own the span.
|
||||
// dedupeSpans resolves collisions between the offline rule pack and the model:
|
||||
// when a local finding and an LLM suggestion fight over the same characters, the
|
||||
// local one wins and the LLM card is dropped. Its span is exact (the detector
|
||||
// matched it), whereas the LLM positions are only advisory (re-anchored by string
|
||||
// at render), so the precise fix should own the span. This is why the split is by
|
||||
// source rather than by type — an offline miscollocation is as exact as an
|
||||
// offline comma, and the coach's fuzzy version of the same chunk shouldn't
|
||||
// double up next to it.
|
||||
//
|
||||
// This deliberately does NOT dedupe LLM-vs-LLM overlaps: voice (awareness-only,
|
||||
// no replacement) and collocation legitimately co-occupy the same span, and that
|
||||
@@ -517,7 +691,7 @@ func dedupeSpans(in []db.Suggestion) []db.Suggestion {
|
||||
type span struct{ from, to int }
|
||||
var claimed []span
|
||||
for _, s := range in {
|
||||
if s.Type == db.SuggestionTypeMechanics && s.FromPos >= 0 {
|
||||
if s.Source == db.SuggestionSourceLocal && s.FromPos >= 0 {
|
||||
claimed = append(claimed, span{s.FromPos, s.ToPos})
|
||||
}
|
||||
}
|
||||
@@ -527,7 +701,7 @@ func dedupeSpans(in []db.Suggestion) []db.Suggestion {
|
||||
|
||||
out := make([]db.Suggestion, 0, len(in))
|
||||
for _, s := range in {
|
||||
if s.Type != db.SuggestionTypeMechanics && s.FromPos >= 0 {
|
||||
if s.Source != db.SuggestionSourceLocal && s.FromPos >= 0 {
|
||||
overlaps := false
|
||||
for _, sp := range claimed {
|
||||
if s.FromPos < sp.to && sp.from < s.ToPos {
|
||||
@@ -536,7 +710,7 @@ func dedupeSpans(in []db.Suggestion) []db.Suggestion {
|
||||
}
|
||||
}
|
||||
if overlaps {
|
||||
continue // an exact mechanics fix owns these characters
|
||||
continue // an exact offline fix owns these characters
|
||||
}
|
||||
}
|
||||
out = append(out, s)
|
||||
@@ -554,10 +728,17 @@ func (h *Handler) dismiss(w http.ResponseWriter, r *http.Request) {
|
||||
h.setStatus(w, r, db.SuggestionStatusRejected)
|
||||
}
|
||||
|
||||
// setStatus accepts or dismisses one suggestion. The doc_id subquery scopes the
|
||||
// write to the caller's own documents, so a stray (or guessed) suggestion id
|
||||
// can't action a row belonging to another account; an unowned id simply affects
|
||||
// no rows and surfaces as a 404.
|
||||
func (h *Handler) setStatus(w http.ResponseWriter, r *http.Request, status string) {
|
||||
res, err := h.DB.Exec(
|
||||
`UPDATE suggestions SET status = ? WHERE id = ? AND status = ?`,
|
||||
`UPDATE suggestions SET status = ?, resolved_at = datetime('now')
|
||||
WHERE id = ? AND status = ?
|
||||
AND doc_id IN (SELECT id FROM documents WHERE user_id = ?)`,
|
||||
status, chi.URLParam(r, "id"), db.SuggestionStatusPending,
|
||||
auth.UserID(r.Context()),
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -567,9 +748,72 @@ func (h *Handler) setStatus(w http.ResponseWriter, r *http.Request, status strin
|
||||
httputil.ErrorJSON(w, http.StatusNotFound, "pending suggestion not found")
|
||||
return
|
||||
}
|
||||
if status == db.SuggestionStatusAccepted {
|
||||
h.plant(chi.URLParam(r, "id"), auth.UserID(r.Context()))
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// plant grows an accepted collocation into a vocabulary-garden phrase card. It
|
||||
// runs after the status write and swallows its own errors: accepting an edit is
|
||||
// the thing the writer asked for, and it must not fail — or even feel slower —
|
||||
// because a flashcard couldn't be made.
|
||||
//
|
||||
// Only collocations are planted. The other families correct *this* sentence
|
||||
// ("their" → "there", a comma, a clearer clause); a collocation is the one that
|
||||
// hands over a reusable chunk, which is the only thing worth reviewing in a week.
|
||||
func (h *Handler) plant(id, userID string) {
|
||||
var s db.Suggestion
|
||||
var contentText string
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT s.type, s.original, s.replacement, s.explanation, s.doc_id, d.content_text
|
||||
FROM suggestions s JOIN documents d ON d.id = s.doc_id
|
||||
WHERE s.id = ? AND d.user_id = ?`,
|
||||
id, userID,
|
||||
).Scan(&s.Type, &s.Original, &s.Replacement, &s.Explanation, &s.DocID, &contentText)
|
||||
if err != nil {
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
log.Printf("suggestions: could not read %s for planting: %v", id, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if s.Type != db.SuggestionTypeCollocation || strings.TrimSpace(s.Replacement) == "" {
|
||||
return
|
||||
}
|
||||
// The stored text is still the pre-accept draft — the client applies the
|
||||
// replacement in the editor. Correct the sentence here so the flashcard
|
||||
// quizzes the phrasing she is keeping, not the one she just left behind.
|
||||
docID := s.DocID
|
||||
if _, err := vocab.Plant(h.DB, userID, vocab.Phrase{
|
||||
Text: s.Replacement,
|
||||
Meaning: s.Explanation,
|
||||
Example: correctedSentence(contentText, s.Original, s.Replacement),
|
||||
DocID: &docID,
|
||||
}); err != nil {
|
||||
log.Printf("suggestions: could not plant %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// correctedSentence returns the sentence of contentText containing original,
|
||||
// with original swapped for replacement. Returns "" when the original isn't
|
||||
// found (the draft moved on) — a card with no example still reviews, just
|
||||
// without the cloze, so there's nothing to fall back to and nothing to guess.
|
||||
func correctedSentence(contentText, original, replacement string) string {
|
||||
idx := strings.Index(contentText, original)
|
||||
if original == "" || idx < 0 {
|
||||
return ""
|
||||
}
|
||||
start := strings.LastIndexAny(contentText[:idx], ".!?\n")
|
||||
end := strings.IndexAny(contentText[idx+len(original):], ".!?\n")
|
||||
if end < 0 {
|
||||
end = len(contentText)
|
||||
} else {
|
||||
end += idx + len(original) + 1 // keep the terminator
|
||||
}
|
||||
sentence := strings.TrimSpace(contentText[start+1 : end])
|
||||
return strings.Replace(sentence, original, replacement, 1)
|
||||
}
|
||||
|
||||
// locate finds the plaintext offsets of original within contentText. Returns
|
||||
// (-1, -1) when not found; the frontend anchors by string regardless, so a miss
|
||||
// here is non-fatal.
|
||||
@@ -583,6 +827,11 @@ func locate(contentText, original string) (int, int) {
|
||||
|
||||
// normalizeType maps the model's type string onto a valid suggestion type,
|
||||
// defaulting unknown values to grammar so a stray label never trips the CHECK.
|
||||
//
|
||||
// 'translate' is absent on purpose, and stays absent even though the type now
|
||||
// exists: it is decided from the span (see language.go), never taken from the
|
||||
// model. A model that volunteers the label anyway lands on grammar here and is
|
||||
// then promoted — or not — on the evidence.
|
||||
func normalizeType(t string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(t)) {
|
||||
case db.SuggestionTypeGrammar, db.SuggestionTypePhrasing, db.SuggestionTypeIdiom, db.SuggestionTypeClarity, db.SuggestionTypeCollocation:
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
)
|
||||
@@ -19,10 +20,17 @@ import (
|
||||
type stubClient struct {
|
||||
response string
|
||||
calls int
|
||||
// The full prompt of the most recent call, so a test can assert which
|
||||
// sentences a chunked pass actually asked about.
|
||||
lastPrompt string
|
||||
}
|
||||
|
||||
func (s *stubClient) Complete(_ context.Context, _ llm.CompletionRequest) (string, error) {
|
||||
func (s *stubClient) Complete(_ context.Context, req llm.CompletionRequest) (string, error) {
|
||||
s.calls++
|
||||
s.lastPrompt = ""
|
||||
for _, m := range req.Messages {
|
||||
s.lastPrompt += m.Content + "\n"
|
||||
}
|
||||
return s.response, nil
|
||||
}
|
||||
|
||||
@@ -56,7 +64,22 @@ func newTestServer(t *testing.T, client llm.LLMClient) (http.Handler, string, *H
|
||||
r := chi.NewRouter()
|
||||
r.Route("/docs", func(dr chi.Router) { h.RegisterDocRoutes(dr) })
|
||||
r.Mount("/suggestions", h.Routes())
|
||||
return r, docID, h
|
||||
|
||||
// Behind the same auth middleware main.go installs: handlers resolve the
|
||||
// caller from the request context, so a bare router would see no user.
|
||||
authed := auth.Middleware(auth.StaticResolver(db.LocalUserID))(r)
|
||||
return authed, docID, h
|
||||
}
|
||||
|
||||
// setDocText rewrites the seeded document, standing in for the writer editing.
|
||||
// The grammar checkpoint only asks the model about sentences that changed since
|
||||
// it last read the document, so a test that wants a second real pass has to
|
||||
// change something first — as she always has.
|
||||
func setDocText(t *testing.T, h *Handler, docID, text string) {
|
||||
t.Helper()
|
||||
if _, err := h.DB.Exec(`UPDATE documents SET content_text = ? WHERE id = ?`, text, docID); err != nil {
|
||||
t.Fatalf("update doc text: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func do(t *testing.T, srv http.Handler, method, path, body string) *httptest.ResponseRecorder {
|
||||
@@ -178,6 +201,7 @@ func TestFickleEditsSuppressed(t *testing.T) {
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
h.Limit = llm.NewRateLimiter(0)
|
||||
setDocText(t, h, docID, `He left "early," because of the rain. The cat always have a calm face.`)
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var got []db.Suggestion
|
||||
@@ -189,6 +213,10 @@ func TestFickleEditsSuppressed(t *testing.T) {
|
||||
do(t, srv, http.MethodPost, "/suggestions/"+s.ID+"/accept", "")
|
||||
}
|
||||
|
||||
// Both edits are now in the document, which is what re-opens those sentences
|
||||
// for a second reading.
|
||||
setDocText(t, h, docID, `He left "early," due to the rain. The cat always has a calm face.`)
|
||||
|
||||
// Reversal of the first accept (note the " → ' quote churn) and a re-polish of
|
||||
// the second accept must both be dropped; only the unrelated edit survives.
|
||||
client.response = `{"suggestions":[
|
||||
@@ -309,7 +337,9 @@ func TestCollocationPassCoexists(t *testing.T) {
|
||||
t.Fatalf("collocation response should carry all three families, got %+v", got)
|
||||
}
|
||||
|
||||
// A grammar checkpoint must NOT wipe the voice or collocation flags.
|
||||
// A grammar checkpoint must NOT wipe the voice or collocation flags. She fixes
|
||||
// the flagged sentence, so its own grammar row goes and nothing replaces it.
|
||||
setDocText(t, h, docID, "I have two apples.")
|
||||
client.response = `{"suggestions":[]}`
|
||||
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
)
|
||||
|
||||
// Suggestions are scoped indirectly: the table has no user_id of its own, only a
|
||||
// doc_id, so every access has to reach the owner through the parent document. A
|
||||
// forgotten join here is worse than it sounds — a suggestion quotes the sentence
|
||||
// it corrects, so listing another account's suggestions leaks their prose.
|
||||
|
||||
// newTwoUserSuggestionServer seeds one document owned by the local user and
|
||||
// returns routers for its owner and for a second, unrelated user.
|
||||
func newTwoUserSuggestionServer(t *testing.T, client llm.LLMClient) (owner, stranger http.Handler, docID string) {
|
||||
t.Helper()
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
|
||||
if _, err := database.Exec(
|
||||
`INSERT INTO users (id, email, display_name) VALUES (?, ?, ?)`,
|
||||
"bob", "bob@petal.local", "Bob",
|
||||
); err != nil {
|
||||
t.Fatalf("seed second user: %v", err)
|
||||
}
|
||||
|
||||
if err := database.QueryRow(
|
||||
`INSERT INTO documents (user_id, content_text) VALUES (?, ?) RETURNING id`,
|
||||
db.LocalUserID, "I has two apple.",
|
||||
).Scan(&docID); err != nil {
|
||||
t.Fatalf("seed doc: %v", err)
|
||||
}
|
||||
|
||||
mount := func(userID string) http.Handler {
|
||||
h := New(database, client)
|
||||
r := chi.NewRouter()
|
||||
r.Route("/docs", func(dr chi.Router) { h.RegisterDocRoutes(dr) })
|
||||
r.Mount("/suggestions", h.Routes())
|
||||
return auth.Middleware(auth.StaticResolver(userID))(r)
|
||||
}
|
||||
return mount(db.LocalUserID), mount("bob"), docID
|
||||
}
|
||||
|
||||
func TestSuggestionIsolation(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"I has","replacement":"I have","explanation":"subject-verb agreement","type":"grammar"}
|
||||
]}`}
|
||||
owner, stranger, docID := newTwoUserSuggestionServer(t, client)
|
||||
|
||||
// The owner runs a checkpoint so there is a real pending suggestion to guard.
|
||||
rec := do(t, owner, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("check: %d %s", rec.Code, rec.Body)
|
||||
}
|
||||
var pending []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &pending); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(pending) != 1 {
|
||||
t.Fatalf("owner has %d suggestions, want 1", len(pending))
|
||||
}
|
||||
sugID := pending[0].ID
|
||||
|
||||
t.Run("cannot list a stranger's suggestions", func(t *testing.T) {
|
||||
rec := do(t, stranger, http.MethodGet, "/docs/"+docID+"/suggestions", "")
|
||||
var out []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(out) != 0 {
|
||||
t.Fatalf("stranger read %d suggestions (leaking %q)", len(out), out[0].Original)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cannot run a pass on a stranger's document", func(t *testing.T) {
|
||||
rec := do(t, stranger, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("stranger check = %d, want 404", rec.Code)
|
||||
}
|
||||
})
|
||||
|
||||
// accept/dismiss take a bare suggestion id with no document in the path, so
|
||||
// the write has to scope itself through doc_id → documents.user_id.
|
||||
for _, action := range []string{"accept", "dismiss"} {
|
||||
t.Run("cannot "+action+" a stranger's suggestion", func(t *testing.T) {
|
||||
rec := do(t, stranger, http.MethodPost, "/suggestions/"+sugID+"/"+action, "")
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("stranger %s = %d, want 404 (body: %s)", action, rec.Code, rec.Body)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// After every attempt the suggestion must still be pending for its owner.
|
||||
rec = do(t, owner, http.MethodGet, "/docs/"+docID+"/suggestions", "")
|
||||
var after []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &after); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(after) != 1 || after[0].Status != db.SuggestionStatusPending {
|
||||
t.Fatalf("owner's suggestion was altered by the stranger: %+v", after)
|
||||
}
|
||||
|
||||
// And the owner can still action it — the scoping guards, it doesn't block.
|
||||
rec = do(t, owner, http.MethodPost, "/suggestions/"+sugID+"/accept", "")
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("owner accept = %d, want 204 (body: %s)", rec.Code, rec.Body)
|
||||
}
|
||||
|
||||
// That accept created a settled span, which is the other read of this table.
|
||||
// It carries originals only — but an original is a verbatim quotation of her
|
||||
// sentence, so it is the same leak as the pending list through a smaller hole.
|
||||
if got := getSettled(t, owner, docID); len(got) != 1 {
|
||||
t.Fatalf("owner should see their own settled span, got %v", got)
|
||||
}
|
||||
if got := getSettled(t, stranger, docID); len(got) != 0 {
|
||||
t.Fatalf("stranger read %d settled span(s) (leaking %q)", len(got), got[0])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Telling her language from English, well enough to label a card.
|
||||
//
|
||||
// When the checkpoint quotes a span she wrote in her own language and hands back
|
||||
// an English rendering, that is not a correction — nothing was wrong with what
|
||||
// she wrote — and it should not be filed under 'clarity'. The label is derived
|
||||
// here rather than asked of the model: a type is structural, and a model that
|
||||
// re-reasons every pass would drift between labels for the same sentence.
|
||||
//
|
||||
// The failure mode is deliberately cheap. Getting this wrong changes a card's
|
||||
// coloured pill and nothing else — the replacement, the explanation and the
|
||||
// Accept button are identical either way — so a heuristic is the right tool. It
|
||||
// is written to under-claim: a span it isn't sure about stays whatever the model
|
||||
// called it.
|
||||
//
|
||||
// The two pair families need genuinely different tests, and pretending otherwise
|
||||
// would be the bug:
|
||||
//
|
||||
// - zh is a different script. Counting Han runes is close to certain.
|
||||
// - pt-PT, fr and es share the Latin alphabet with English, where no such
|
||||
// signal exists. Those fall back to function words — the short, extremely
|
||||
// common words a sentence in that language can hardly avoid and an English
|
||||
// sentence has no reason to contain.
|
||||
|
||||
// isTranslation reports whether this edit is her own language rendered into
|
||||
// English, rather than a correction to her English. Both halves must hold: the
|
||||
// quoted span reads as the pair language, and what Petal offers back reads as
|
||||
// English. The second half matters — a Chinese span rewritten into different
|
||||
// Chinese is something else entirely, and Petal has no business calling it a
|
||||
// translation.
|
||||
func isTranslation(original, replacement, pairLang string) bool {
|
||||
if strings.TrimSpace(original) == "" || strings.TrimSpace(replacement) == "" {
|
||||
return false
|
||||
}
|
||||
return readsAsPairLang(original, pairLang) && readsAsEnglish(replacement)
|
||||
}
|
||||
|
||||
// readsAsPairLang reports whether s is predominantly in the writer's language.
|
||||
func readsAsPairLang(s, pairLang string) bool {
|
||||
switch normalizePairLang(pairLang) {
|
||||
case "zh":
|
||||
han, latin := scriptCounts(s)
|
||||
// Predominantly, not merely partly: one Chinese word inside an English
|
||||
// sentence is a vocabulary question, and the sentence around it is still
|
||||
// English prose with its own grammar to correct. Two runes is the floor
|
||||
// because a single Han character is as likely to be a stray keystroke.
|
||||
return han >= 2 && han > latin
|
||||
case "pt-PT", "fr", "es":
|
||||
return distinctMarkers(s, latinMarkers[normalizePairLang(pairLang)]) >= 2
|
||||
}
|
||||
// A pair Petal has no test for. Say no: an unlabelled card is a card that
|
||||
// reads as it did yesterday, and a wrongly-labelled one is a new defect.
|
||||
return false
|
||||
}
|
||||
|
||||
// readsAsEnglish reports whether s is English prose rather than more of her own
|
||||
// language. It is not a language identifier — it only has to separate "English"
|
||||
// from "the pair language", and it is only ever asked about text Petal itself
|
||||
// generated, so the bar is low on purpose: Latin letters present, and not
|
||||
// swamped by another script.
|
||||
func readsAsEnglish(s string) bool {
|
||||
han, latin := scriptCounts(s)
|
||||
return latin > 0 && latin > han
|
||||
}
|
||||
|
||||
// normalizePairLang folds the stored `users.pair_lang` into the codes below.
|
||||
// Empty (a document whose owner has no pair recorded) falls through to no test.
|
||||
func normalizePairLang(pairLang string) string {
|
||||
switch p := strings.ToLower(strings.TrimSpace(pairLang)); p {
|
||||
case "zh", "zh-cn", "zh-hans":
|
||||
return "zh"
|
||||
case "pt", "pt-pt":
|
||||
return "pt-PT"
|
||||
case "fr", "fr-fr":
|
||||
return "fr"
|
||||
case "es", "es-es":
|
||||
return "es"
|
||||
default:
|
||||
return p
|
||||
}
|
||||
}
|
||||
|
||||
// scriptCounts counts Han runes and ASCII letters. Everything else — digits,
|
||||
// punctuation, spaces, emoji — is ignored, so trailing 。or a stray comma
|
||||
// changes nothing.
|
||||
func scriptCounts(s string) (han, latin int) {
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case unicode.Is(unicode.Han, r):
|
||||
han++
|
||||
case r < unicode.MaxASCII && unicode.IsLetter(r):
|
||||
latin++
|
||||
}
|
||||
}
|
||||
return han, latin
|
||||
}
|
||||
|
||||
// distinctMarkers counts how many *different* marker words appear in s. Distinct
|
||||
// rather than total: "que ... que" is one writer's habit, while "eu quero" is two
|
||||
// independent pieces of evidence.
|
||||
func distinctMarkers(s string, markers map[string]bool) int {
|
||||
if len(markers) == 0 {
|
||||
return 0
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, w := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool {
|
||||
// Split on anything that isn't a letter, so punctuation and digits are
|
||||
// separators. Apostrophes included: French elision (j'ai, n'est) should
|
||||
// yield its parts.
|
||||
return !unicode.IsLetter(r)
|
||||
}) {
|
||||
if markers[w] {
|
||||
seen[w] = true
|
||||
}
|
||||
}
|
||||
return len(seen)
|
||||
}
|
||||
|
||||
// Function words that a sentence in each Latin pair can hardly avoid.
|
||||
//
|
||||
// Curated against English, not for coverage: every entry here is a word an
|
||||
// English sentence has essentially no reason to contain, which is why the lists
|
||||
// omit plenty of far more common words. Deliberately absent — each of them a
|
||||
// false positive waiting to happen — is anything that is *also* an English word:
|
||||
// the pan-Romance shorts (a, o, e, as, no, on, en, de, se, na, mi, son, era,
|
||||
// plus, pour, si, ma, ce, ne), Portuguese "do", Spanish "con", "ya" and "todo".
|
||||
// Dropping "con" costs the Spanish list one of its commonest words, and that is
|
||||
// the right trade — a marker that fires on English corroborates the wrong
|
||||
// answer, which is worse than a sentence Petal declines to label.
|
||||
//
|
||||
// A single marker is not enough (see readsAsPairLang), so these lists are read
|
||||
// as evidence to be corroborated rather than as a decision.
|
||||
var latinMarkers = map[string]map[string]bool{
|
||||
"fr": words(
|
||||
"je", "tu", "il", "elle", "ils", "elles", "nous", "vous", "est", "sont",
|
||||
"était", "étais", "une", "des", "les", "du", "dans", "avec", "que", "qui",
|
||||
"mais", "très", "être", "avoir", "pas", "cette", "cet", "ces", "mon",
|
||||
"mes", "notre", "votre", "leur", "aussi", "alors", "parce", "comme",
|
||||
"beaucoup", "toujours", "jamais", "quand", "bien", "chose", "temps",
|
||||
"moi", "toi", "lui", "peux", "veux", "sais", "faire", "dit", "aujourd",
|
||||
"hui", "quelque", "chez", "tout", "tous", "rien", "déjà", "encore",
|
||||
),
|
||||
"pt-PT": words(
|
||||
"eu", "você", "ele", "ela", "eles", "elas", "nós", "são", "uma", "os",
|
||||
"da", "dos", "das", "com", "que", "mas", "muito", "não", "meu",
|
||||
"minha", "seu", "sua", "isso", "este", "esta", "está", "estou", "quero",
|
||||
"também", "quando", "porque", "coisa", "tempo", "fazer", "sempre",
|
||||
"nunca", "bem", "obrigado", "obrigada", "gosto", "tenho", "tem", "foi",
|
||||
"ser", "ter", "mais", "já", "ainda", "aqui", "ali", "nada", "tudo",
|
||||
"todos", "para", "pela", "pelo", "sobre", "assim",
|
||||
),
|
||||
"es": words(
|
||||
"yo", "él", "ella", "ellos", "ellas", "nosotros", "una", "los", "las",
|
||||
"del", "que", "pero", "muy", "esto", "esta", "este", "está",
|
||||
"estoy", "quiero", "también", "cuando", "porque", "cosa", "tiempo",
|
||||
"hacer", "siempre", "nunca", "bien", "gracias", "tengo", "tiene", "fue",
|
||||
"ser", "tener", "más", "aquí", "allí", "nada", "todos",
|
||||
"para", "sobre", "así", "hola", "señor", "usted", "muchas",
|
||||
),
|
||||
}
|
||||
|
||||
func words(list ...string) map[string]bool {
|
||||
out := make(map[string]bool, len(list))
|
||||
for _, w := range list {
|
||||
out[w] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package suggestions
|
||||
|
||||
import "testing"
|
||||
|
||||
// The flagship case, and the ones next to it that must NOT become translations.
|
||||
func TestIsTranslation(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
original string
|
||||
replacement string
|
||||
pairLang string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
// The sentence from the UX review, verbatim.
|
||||
name: "whole Chinese sentence rendered into English",
|
||||
original: "我想说这句话但是不知道用英语怎么说。",
|
||||
replacement: "I want to say this but I don't know how to say it in English.",
|
||||
pairLang: "zh",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "ordinary English correction is not a translation",
|
||||
original: "She goes to market yesterday",
|
||||
replacement: "She went to the market yesterday",
|
||||
pairLang: "zh",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// One Chinese word inside English prose. The sentence around it is
|
||||
// still English with its own grammar to fix, and calling the card a
|
||||
// translation would mislabel a grammar fix.
|
||||
name: "single Chinese word inside an English sentence",
|
||||
original: "I bought a 苹果 at the store",
|
||||
replacement: "I bought an apple at the store",
|
||||
pairLang: "zh",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "a lone stray Han rune is not a sentence",
|
||||
original: "的",
|
||||
replacement: "of",
|
||||
pairLang: "zh",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// Chinese in, Chinese out: whatever this is, Petal is not translating.
|
||||
name: "Chinese rewritten as Chinese",
|
||||
original: "我想说这句话",
|
||||
replacement: "我要说这句话",
|
||||
pairLang: "zh",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// The same Chinese span, but the writer is on the French pair. Petal
|
||||
// has no business offering to translate a language she never claimed.
|
||||
name: "Chinese span on a non-zh pair",
|
||||
original: "我想说这句话但是不知道用英语怎么说。",
|
||||
replacement: "I want to say this in English.",
|
||||
pairLang: "fr",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "French sentence rendered into English",
|
||||
original: "Je ne sais pas comment le dire en anglais.",
|
||||
replacement: "I don't know how to say it in English.",
|
||||
pairLang: "fr",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "Portuguese sentence rendered into English",
|
||||
original: "Eu quero dizer isso mas não sei como.",
|
||||
replacement: "I want to say this but I don't know how.",
|
||||
pairLang: "pt-PT",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "Spanish sentence rendered into English",
|
||||
original: "Yo quiero decir esto pero no sé cómo.",
|
||||
replacement: "I want to say this but I don't know how.",
|
||||
pairLang: "es",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// A single marker is not evidence. "Que" appears in English writing
|
||||
// about other languages, in names, in quoted phrases.
|
||||
name: "one Latin marker is not enough",
|
||||
original: "The word que confused me",
|
||||
replacement: "The word que confuses me",
|
||||
pairLang: "pt-PT",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// The words most likely to sink this heuristic: English function words
|
||||
// that are also Romance function words. They are kept out of the lists
|
||||
// precisely so this sentence stays a grammar fix.
|
||||
name: "English full of pan-Romance lookalikes",
|
||||
original: "I do not know if a con man on the plus side as no era",
|
||||
replacement: "I do not know whether a con man, on the plus side, is no era",
|
||||
pairLang: "es",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "English with a borrowed French phrase stays English",
|
||||
original: "It was a pas de deux, more or less",
|
||||
replacement: "It was a pas de deux, more or less.",
|
||||
pairLang: "fr",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "empty replacement (an awareness-only finding)",
|
||||
original: "我想说这句话但是不知道用英语怎么说。",
|
||||
replacement: "",
|
||||
pairLang: "zh",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// A document whose owner has no pair recorded. No test, no label.
|
||||
name: "no pair language",
|
||||
original: "我想说这句话但是不知道用英语怎么说。",
|
||||
replacement: "I want to say this in English.",
|
||||
pairLang: "",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// An unshipped pair. Same rule: decline rather than guess.
|
||||
name: "unknown pair language",
|
||||
original: "Ich weiß nicht wie man das sagt.",
|
||||
replacement: "I don't know how to say that.",
|
||||
pairLang: "de",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := isTranslation(c.original, c.replacement, c.pairLang); got != c.want {
|
||||
t.Errorf("isTranslation(%q, %q, %q) = %v, want %v",
|
||||
c.original, c.replacement, c.pairLang, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// pair_lang is stored as the pack code, but a stored value has drifted before
|
||||
// (see the picker's history), so the fold is tested rather than assumed.
|
||||
func TestNormalizePairLang(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"zh": "zh", "zh-CN": "zh", "ZH": "zh",
|
||||
"pt": "pt-PT", "pt-PT": "pt-PT", "pt-pt": "pt-PT",
|
||||
"fr": "fr", "fr-FR": "fr",
|
||||
"es": "es", "es-ES": "es",
|
||||
" zh ": "zh",
|
||||
"": "",
|
||||
"de": "de",
|
||||
} {
|
||||
if got := normalizePairLang(in); got != want {
|
||||
t.Errorf("normalizePairLang(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// French elision must yield its parts, or "j'ai" and "n'est" — two of the
|
||||
// commonest shapes in the language — count for nothing.
|
||||
func TestElisionYieldsMarkers(t *testing.T) {
|
||||
if n := distinctMarkers("Je n'est pas", latinMarkers["fr"]); n < 3 {
|
||||
t.Errorf("elided French: got %d markers, want >= 3 (je, est, pas)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Distinct, not total: one word repeated is one piece of evidence.
|
||||
func TestRepeatedMarkerCountsOnce(t *testing.T) {
|
||||
if n := distinctMarkers("que que que", latinMarkers["pt-PT"]); n != 1 {
|
||||
t.Errorf("repeated marker: got %d, want 1", n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// The offline rule pack and the LLM now share the collocation family, which is
|
||||
// the point: the writer sees one rail and is never told which engine spoke. What
|
||||
// makes that safe is `source` — each pass replaces only its own rows. These tests
|
||||
// pin the two ways that could go wrong, both of which the old type-scoped DELETEs
|
||||
// would have hit.
|
||||
|
||||
// pendingOfType counts the pending rows of one family in a response body.
|
||||
func pendingOfType(got []db.Suggestion, typ string) []db.Suggestion {
|
||||
var out []db.Suggestion
|
||||
for _, s := range got {
|
||||
if s.Type == typ {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestOfflineCollocationFilesAsCollocation proves a miscollocation the rule pack
|
||||
// found is stored in the collocation family (so accepting it plants a garden
|
||||
// card, exactly as the coach's would) while still being marked as locally found.
|
||||
func TestOfflineCollocationFilesAsCollocation(t *testing.T) {
|
||||
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":0,"to":13,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"},
|
||||
{"from":20,"to":27,"original":"the the","replacement":"the","explanation":"doubled word","type":"mechanics"}
|
||||
]`)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want both findings, got %+v", got)
|
||||
}
|
||||
coll := pendingOfType(got, db.SuggestionTypeCollocation)
|
||||
if len(coll) != 1 {
|
||||
t.Fatalf("want 1 collocation, got %+v", got)
|
||||
}
|
||||
if coll[0].Source != db.SuggestionSourceLocal {
|
||||
t.Errorf("offline finding should be source=local, got %q", coll[0].Source)
|
||||
}
|
||||
if mech := pendingOfType(got, db.SuggestionTypeMechanics); len(mech) != 1 {
|
||||
t.Fatalf("want 1 mechanics finding, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnknownLocalTypeFallsBackToMechanics: a family the offline pass isn't
|
||||
// allowed to claim (or an older client sending none at all) must land in
|
||||
// mechanics. Otherwise a stray label would smuggle a row into an LLM family,
|
||||
// where nothing would ever replace it.
|
||||
func TestUnknownLocalTypeFallsBackToMechanics(t *testing.T) {
|
||||
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":0,"to":5,"original":"aaaaa","replacement":"bbbbb","explanation":"x","type":"voice"},
|
||||
{"from":6,"to":11,"original":"ccccc","replacement":"ddddd","explanation":"y"}
|
||||
]`)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want 2 findings, got %+v", got)
|
||||
}
|
||||
for _, s := range got {
|
||||
if s.Type != db.SuggestionTypeMechanics {
|
||||
t.Errorf("offline finding claimed family %q; only mechanics/collocation are allowed", s.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCoachDoesNotWipeOfflineCollocations is the collision the source column
|
||||
// exists for: the LLM collocation pass replaces the collocation family, and the
|
||||
// rule pack's share of that family has to survive it. Before `source`, running
|
||||
// the coach silently deleted every offline chunk on the page.
|
||||
func TestCoachDoesNotWipeOfflineCollocations(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"apple","replacement":"an apple","explanation":"article","type":"collocation"}
|
||||
]}`}
|
||||
srv, docID, _ := newTestServer(t, client)
|
||||
|
||||
// The seeded doc is "I has two apple." — the coach's flag anchors on "apple"
|
||||
// at [10,15], so the offline finding is given a span well clear of it. Two
|
||||
// findings fighting over the same characters is a different rule (see
|
||||
// TestOfflineCardWinsSpanCollision); this test is about the DELETE.
|
||||
postMechanics(t, srv, docID, `[
|
||||
{"from":0,"to":5,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||
]`)
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/collocation", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("collocation pass: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
var got []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
|
||||
var local, llm int
|
||||
for _, s := range pendingOfType(got, db.SuggestionTypeCollocation) {
|
||||
if s.Source == db.SuggestionSourceLocal {
|
||||
local++
|
||||
} else {
|
||||
llm++
|
||||
}
|
||||
}
|
||||
if local != 1 {
|
||||
t.Errorf("the coach wiped the offline collocation: local=%d, got %+v", local, got)
|
||||
}
|
||||
if llm != 1 {
|
||||
t.Errorf("want the coach's own flag alongside it: llm=%d, got %+v", llm, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOfflinePassReplacesItsOwnCollocations is the mirror: the rule pack
|
||||
// recomputes the whole document every run, so a chunk the current text no longer
|
||||
// warrants must go — and the coach's flags must stay. Scoping the offline DELETE
|
||||
// by type instead of source would have stranded the first row forever.
|
||||
func TestOfflinePassReplacesItsOwnCollocations(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"apple","replacement":"an apple","explanation":"article","type":"collocation"}
|
||||
]}`}
|
||||
srv, docID, _ := newTestServer(t, client)
|
||||
|
||||
// A coach flag, then an offline chunk, then a rerun that no longer finds it.
|
||||
do(t, srv, http.MethodPost, "/docs/"+docID+"/collocation", "")
|
||||
postMechanics(t, srv, docID, `[
|
||||
{"from":0,"to":13,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||
]`)
|
||||
got := postMechanics(t, srv, docID, `[]`)
|
||||
|
||||
for _, s := range got {
|
||||
if s.Source == db.SuggestionSourceLocal {
|
||||
t.Errorf("stale offline finding survived a recompute: %+v", s)
|
||||
}
|
||||
}
|
||||
if len(pendingOfType(got, db.SuggestionTypeCollocation)) != 1 {
|
||||
t.Fatalf("the coach's own flag should be untouched, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOfflineCollocationPlantsOnAccept closes the loop the family split was for:
|
||||
// a chunk the rule pack found, accepted, becomes a vocabulary-garden card — with
|
||||
// no model involved anywhere in the path.
|
||||
func TestOfflineCollocationPlantsOnAccept(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
if _, err := h.DB.Exec(
|
||||
`UPDATE documents SET content_text = ? WHERE id = ?`,
|
||||
"I had to do a decision about the job.", docID,
|
||||
); err != nil {
|
||||
t.Fatalf("set content: %v", err)
|
||||
}
|
||||
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":9,"to":22,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||
]`)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want the offline chunk, got %+v", got)
|
||||
}
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+got[0].ID+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
|
||||
cards := gardenCards(t, h)
|
||||
if len(cards) != 1 || cards[0].word != "make a decision" {
|
||||
t.Fatalf("want a planted phrase card, got %+v", cards)
|
||||
}
|
||||
// The example is the corrected sentence — the phrasing she kept, not the one
|
||||
// she just left behind.
|
||||
if cards[0].example != "I had to make a decision about the job." {
|
||||
t.Errorf("example should be the corrected sentence, got %q", cards[0].example)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOfflineCardWinsSpanCollision: the tiebreak is by engine, not by family. An
|
||||
// offline miscollocation has an exact span; the coach's overlapping flag is only
|
||||
// advisory, so it is the one that goes.
|
||||
func TestOfflineCardWinsSpanCollision(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"do a decision about","replacement":"decide about","explanation":"wordy","type":"collocation"}
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
if _, err := h.DB.Exec(
|
||||
`UPDATE documents SET content_text = ? WHERE id = ?`,
|
||||
"I had to do a decision about the job.", docID,
|
||||
); err != nil {
|
||||
t.Fatalf("set content: %v", err)
|
||||
}
|
||||
|
||||
do(t, srv, http.MethodPost, "/docs/"+docID+"/collocation", "")
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":9,"to":22,"original":"do a decision","replacement":"make a decision","explanation":"pairing","type":"collocation"}
|
||||
]`)
|
||||
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want the overlapping coach flag dropped, got %+v", got)
|
||||
}
|
||||
if got[0].Source != db.SuggestionSourceLocal {
|
||||
t.Errorf("the exact offline card should own the span, got %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestOfflineHanziFindingStaysMechanics: a 错别字 the Chinese rule pack found —
|
||||
// both halves written in hanzi — files as an ordinary mechanics row.
|
||||
//
|
||||
// The check is worth its own test because there is a rule one layer over that
|
||||
// would plausibly claim it. `isTranslation` re-labels an edit whose original
|
||||
// reads as the writer's language and whose replacement reads as English, which
|
||||
// is exactly how a zh-pair writer's quoted Chinese becomes a 'translate' card.
|
||||
// A wrong-character fix looks like the first half of that and nothing like the
|
||||
// second: 己经 → 已经 never leaves Chinese. It must stay a tidy-up in her own
|
||||
// sentence, on the same rail as a doubled word, with no rendering-into-English
|
||||
// implied anywhere.
|
||||
func TestOfflineHanziFindingStaysMechanics(t *testing.T) {
|
||||
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
|
||||
got := postMechanics(t, srv, docID, `[
|
||||
{"from":1,"to":3,"original":"己经","replacement":"已经","explanation":"已经 (already) takes 已","type":"mechanics"}
|
||||
]`)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want the one finding, got %+v", got)
|
||||
}
|
||||
if got[0].Type != db.SuggestionTypeMechanics {
|
||||
t.Errorf("hanzi fix filed as %q, want %q", got[0].Type, db.SuggestionTypeMechanics)
|
||||
}
|
||||
if got[0].Source != db.SuggestionSourceLocal {
|
||||
t.Errorf("source = %q, want %q", got[0].Source, db.SuggestionSourceLocal)
|
||||
}
|
||||
// The characters survive the round trip intact — a mangled span here would
|
||||
// replace the wrong characters in her document.
|
||||
if got[0].Original != "己经" || got[0].Replacement != "已经" {
|
||||
t.Errorf("round-tripped as %q → %q", got[0].Original, got[0].Replacement)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
)
|
||||
|
||||
// The langpack decides what Petal says in the browser; users.pair_lang has to
|
||||
// decide what the *model* says too, or a pt-PT writer gets a Mandarin gloss on
|
||||
// an otherwise Portuguese screen. These tests follow the value from the column
|
||||
// to the system prompt for each pass that names a language.
|
||||
//
|
||||
// This is the same failure mode the standing isolation rule guards against: the
|
||||
// column is read in a query the handler already ran, so nothing fails loudly if
|
||||
// the join is dropped — the prompt just quietly reverts to Mandarin.
|
||||
|
||||
// newPairServer seeds one writer on the given pair with a document of her own.
|
||||
func newPairServer(t *testing.T, client llm.LLMClient, pairLang string) (http.Handler, string, *db.DB) {
|
||||
t.Helper()
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "pair.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
|
||||
const userID = "writer-pt"
|
||||
if _, err := database.Exec(
|
||||
`INSERT INTO users (id, email, display_name, pair_lang) VALUES (?, ?, ?, ?)`,
|
||||
userID, "w@example.com", "Writer", pairLang,
|
||||
); err != nil {
|
||||
t.Fatalf("seed user: %v", err)
|
||||
}
|
||||
|
||||
var docID string
|
||||
if err := database.QueryRow(
|
||||
`INSERT INTO documents (user_id, content_text) VALUES (?, ?) RETURNING id`,
|
||||
userID, "The rain was strong yesterday.",
|
||||
).Scan(&docID); err != nil {
|
||||
t.Fatalf("seed doc: %v", err)
|
||||
}
|
||||
|
||||
h := New(database, client)
|
||||
r := chi.NewRouter()
|
||||
r.Route("/docs", func(dr chi.Router) { h.RegisterDocRoutes(dr) })
|
||||
r.Mount("/suggestions", h.Routes())
|
||||
return auth.Middleware(auth.StaticResolver(userID))(r), docID, database
|
||||
}
|
||||
|
||||
func TestCollocationPromptUsesTheWritersPair(t *testing.T) {
|
||||
client := &recordingClient{response: `{"suggestions":[]}`}
|
||||
srv, docID, _ := newPairServer(t, client, "pt-PT")
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/collocation", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("collocation: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
system := client.last.Messages[0].Content
|
||||
if !strings.Contains(system, "European Portuguese") {
|
||||
t.Fatalf("collocation prompt ignored pair_lang:\n%s", system)
|
||||
}
|
||||
if strings.Contains(system, "Simplified Chinese") {
|
||||
t.Fatalf("collocation prompt fell back to Mandarin:\n%s", system)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslatePromptUsesTheWritersPair(t *testing.T) {
|
||||
client := &recordingClient{response: "Chove muito."}
|
||||
srv, docID, database := newPairServer(t, client, "pt-PT")
|
||||
|
||||
var sugID string
|
||||
if err := database.QueryRow(
|
||||
`INSERT INTO suggestions (doc_id, original, replacement, explanation, type, from_pos, to_pos)
|
||||
VALUES (?, ?, ?, ?, ?, 0, 5) RETURNING id`,
|
||||
docID, "strong rain", "heavy rain", "Natives usually say heavy rain.", "collocation",
|
||||
).Scan(&sugID); err != nil {
|
||||
t.Fatalf("seed suggestion: %v", err)
|
||||
}
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/suggestions/"+sugID+"/translate", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("translate: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
system := client.last.Messages[0].Content
|
||||
if !strings.Contains(system, "European Portuguese") || strings.Contains(system, "Chinese") {
|
||||
t.Fatalf("translate prompt ignored pair_lang:\n%s", system)
|
||||
}
|
||||
}
|
||||
|
||||
// A writer whose column still holds the default — every account today — must be
|
||||
// answered exactly as before.
|
||||
func TestDefaultPairIsUnchanged(t *testing.T) {
|
||||
client := &recordingClient{response: `{"suggestions":[]}`}
|
||||
srv, docID, _ := newPairServer(t, client, "zh")
|
||||
|
||||
if rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/collocation", ""); rec.Code != http.StatusOK {
|
||||
t.Fatalf("collocation: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if system := client.last.Messages[0].Content; !strings.Contains(system, "Simplified Chinese (Mandarin) gloss") {
|
||||
t.Fatalf("zh writer no longer gets a Mandarin gloss:\n%s", system)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// seedSuggestion writes one pending suggestion against the seeded doc, after
|
||||
// replacing the doc's text so the sentence around `original` is under the test's
|
||||
// control.
|
||||
func seedSuggestion(t *testing.T, h *Handler, docID, text, sType, original, replacement, explanation string) string {
|
||||
t.Helper()
|
||||
if _, err := h.DB.Exec(`UPDATE documents SET content_text = ? WHERE id = ?`, text, docID); err != nil {
|
||||
t.Fatalf("set content: %v", err)
|
||||
}
|
||||
var id string
|
||||
err := h.DB.QueryRow(
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, status)
|
||||
VALUES (?, 0, 0, ?, ?, ?, ?, 'pending') RETURNING id`,
|
||||
docID, original, replacement, explanation, sType,
|
||||
).Scan(&id)
|
||||
if err != nil {
|
||||
t.Fatalf("seed suggestion: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
type card struct {
|
||||
word, definition, example string
|
||||
interval int
|
||||
}
|
||||
|
||||
func gardenCards(t *testing.T, h *Handler) []card {
|
||||
t.Helper()
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT word, definition, example, interval_days FROM vocab_words WHERE user_id = ? ORDER BY word`,
|
||||
db.LocalUserID,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("read garden: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []card
|
||||
for rows.Next() {
|
||||
var c card
|
||||
if err := rows.Scan(&c.word, &c.definition, &c.example, &c.interval); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestAcceptedCollocationIsPlanted walks the whole hand-over: a collocation the
|
||||
// writer accepts becomes a phrase card whose example is the *corrected*
|
||||
// sentence, so the flashcard quizzes the phrasing she kept.
|
||||
func TestAcceptedCollocationIsPlanted(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
id := seedSuggestion(t, h, docID,
|
||||
"Yesterday was hard. I had to do a decision about the job. Then I slept.",
|
||||
db.SuggestionTypeCollocation, "do a decision", "make a decision",
|
||||
"English pairs “make” with “decision”.")
|
||||
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept: got %d, want 204", rec.Code)
|
||||
}
|
||||
|
||||
cards := gardenCards(t, h)
|
||||
if len(cards) != 1 {
|
||||
t.Fatalf("garden has %d cards, want 1: %+v", len(cards), cards)
|
||||
}
|
||||
got := cards[0]
|
||||
if got.word != "make a decision" {
|
||||
t.Errorf("word = %q, want %q", got.word, "make a decision")
|
||||
}
|
||||
if got.example != "I had to make a decision about the job." {
|
||||
t.Errorf("example = %q — want the corrected sentence, bounded to its own sentence", got.example)
|
||||
}
|
||||
if got.definition != "English pairs “make” with “decision”." {
|
||||
t.Errorf("definition = %q, want the explanation", got.definition)
|
||||
}
|
||||
if got.interval != 1 {
|
||||
t.Errorf("interval_days = %d, want 1 (due tomorrow, like a fresh capture)", got.interval)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOnlyCollocationsArePlanted: the other families correct this sentence and
|
||||
// hand over nothing reusable. A dismissed collocation is not a lesson either.
|
||||
func TestOnlyCollocationsArePlanted(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
|
||||
grammar := seedSuggestion(t, h, docID, "I has two apples.",
|
||||
db.SuggestionTypeGrammar, "I has", "I have", "Subject–verb agreement.")
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+grammar+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept grammar: got %d", rec.Code)
|
||||
}
|
||||
|
||||
dismissed := seedSuggestion(t, h, docID, "We must take a photo of it.",
|
||||
db.SuggestionTypeCollocation, "do a photo", "take a photo", "Photos are taken.")
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+dismissed+"/dismiss", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("dismiss: got %d", rec.Code)
|
||||
}
|
||||
|
||||
if cards := gardenCards(t, h); len(cards) != 0 {
|
||||
t.Fatalf("garden grew %d card(s) from a grammar fix and a dismissal: %+v", len(cards), cards)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPlantingIsIdempotentAndNeverResets: accepting the same chunk again is
|
||||
// evidence it's still being learned — the worst possible response is to wipe the
|
||||
// card's first context and the schedule it has been climbing.
|
||||
func TestPlantingIsIdempotentAndNeverResets(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
first := seedSuggestion(t, h, docID, "I had to do a decision.",
|
||||
db.SuggestionTypeCollocation, "do a decision", "make a decision", "First explanation.")
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+first+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept: got %d", rec.Code)
|
||||
}
|
||||
// The card climbs a little.
|
||||
if _, err := h.DB.Exec(
|
||||
`UPDATE vocab_words SET reps = 3, interval_days = 7 WHERE user_id = ? AND word = 'make a decision'`,
|
||||
db.LocalUserID,
|
||||
); err != nil {
|
||||
t.Fatalf("advance card: %v", err)
|
||||
}
|
||||
|
||||
second := seedSuggestion(t, h, docID, "Later I must do a decision again.",
|
||||
db.SuggestionTypeCollocation, "do a decision", "make a decision", "Second explanation.")
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+second+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept again: got %d", rec.Code)
|
||||
}
|
||||
|
||||
cards := gardenCards(t, h)
|
||||
if len(cards) != 1 {
|
||||
t.Fatalf("garden has %d cards, want 1 (one chunk, one card)", len(cards))
|
||||
}
|
||||
if cards[0].definition != "First explanation." {
|
||||
t.Errorf("definition = %q — the existing card should win", cards[0].definition)
|
||||
}
|
||||
if cards[0].example != "I had to make a decision." {
|
||||
t.Errorf("example = %q — the first context should survive", cards[0].example)
|
||||
}
|
||||
if cards[0].interval != 7 {
|
||||
t.Errorf("interval_days = %d, want 7 — progress must not be reset", cards[0].interval)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSentenceRewriteIsNotAPhraseCard: a "collocation" long enough to be a
|
||||
// rewritten sentence makes a miserable flashcard, so it is dropped rather than
|
||||
// planted — and the accept still succeeds.
|
||||
func TestSentenceRewriteIsNotAPhraseCard(t *testing.T) {
|
||||
srv, docID, h := newTestServer(t, &stubClient{})
|
||||
long := "I would like to take this opportunity to thank you for everything"
|
||||
id := seedSuggestion(t, h, docID, "I want thank you for everything.",
|
||||
db.SuggestionTypeCollocation, "I want thank you for everything", long, "More natural.")
|
||||
if rec := do(t, srv, http.MethodPost, "/suggestions/"+id+"/accept", ""); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("accept: got %d, want 204 — a skipped card must never fail the accept", rec.Code)
|
||||
}
|
||||
if cards := gardenCards(t, h); len(cards) != 0 {
|
||||
t.Fatalf("planted a sentence as a phrase card: %+v", cards)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectedSentence(t *testing.T) {
|
||||
const text = "One thing. I had to do a decision fast! Another thing."
|
||||
cases := []struct {
|
||||
name, original, replacement, want string
|
||||
}{
|
||||
{"bounded to its sentence", "do a decision", "make a decision", "I had to make a decision fast!"},
|
||||
{"original no longer present", "do a choice", "make a choice", ""},
|
||||
{"empty original", "", "make a decision", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := correctedSentence(text, tc.original, tc.replacement); got != tc.want {
|
||||
t.Errorf("correctedSentence = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A document with no terminator at all is one sentence, and still works.
|
||||
if got := correctedSentence("i had to do a decision", "do a decision", "make a decision"); got != "i had to make a decision" {
|
||||
t.Errorf("unterminated doc: got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
)
|
||||
|
||||
// Reconciliation replaces the old "delete the family, insert the new batch"
|
||||
// shape of every pass. A suggestion the pass proposes again is the *same*
|
||||
// suggestion: it keeps its row, and therefore its id, its created_at and — most
|
||||
// visibly — the explanation it was first given. The model re-words its reasoning
|
||||
// every time it is asked, so re-inserting meant one unchanged mistake carried
|
||||
// three different explanations in a single sitting.
|
||||
//
|
||||
// The id is what the frontend keys its cards on, so a stable id is also what
|
||||
// keeps the rail from emptying and refilling, a card from collapsing mid-read,
|
||||
// and the arrival chime from re-firing for advice she has already seen.
|
||||
|
||||
// pendingRow is the part of an existing pending suggestion reconciliation cares
|
||||
// about.
|
||||
type pendingRow struct {
|
||||
id string
|
||||
original string
|
||||
replacement string
|
||||
chunkHash string
|
||||
from int
|
||||
}
|
||||
|
||||
// loadPending reads the pending rows a pass owns. `where` is the pass's own
|
||||
// scoping clause (by source, and for the model passes by family) — the same
|
||||
// fragment that used to scope its DELETE.
|
||||
func loadPending(tx *sql.Tx, docID, where string) ([]pendingRow, error) {
|
||||
rows, err := tx.Query(
|
||||
`SELECT id, original, replacement, chunk_hash, from_pos FROM suggestions
|
||||
WHERE doc_id = ? AND status = ? AND `+where,
|
||||
docID, db.SuggestionStatusPending,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []pendingRow
|
||||
for rows.Next() {
|
||||
var r pendingRow
|
||||
if err := rows.Scan(&r.id, &r.original, &r.replacement, &r.chunkHash, &r.from); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// editKey identifies an edit by what it proposes, not where: "this exact change
|
||||
// to this exact text". Normalized like the suppression comparisons, so the
|
||||
// editor's quote rewriting and a reflowed paragraph don't read as a new edit.
|
||||
func editKey(original, replacement string) string {
|
||||
return normalizeForDedup(original) + "\x00" + normalizeForDedup(replacement)
|
||||
}
|
||||
|
||||
// editIndex matches freshly proposed edits against the rows already standing.
|
||||
type editIndex struct {
|
||||
rows []pendingRow
|
||||
used []bool
|
||||
byKey map[string][]int
|
||||
}
|
||||
|
||||
func indexByEdit(rows []pendingRow) *editIndex {
|
||||
idx := &editIndex{rows: rows, used: make([]bool, len(rows)), byKey: map[string][]int{}}
|
||||
for i, r := range rows {
|
||||
k := editKey(r.original, r.replacement)
|
||||
idx.byKey[k] = append(idx.byKey[k], i)
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// take claims the standing row for this edit, if there is one. When a document
|
||||
// repeats the same mistake, `near` (the fresh span's start) picks the closest
|
||||
// standing row, so two identical cards keep their own identities instead of
|
||||
// trading them whenever the text between them grows.
|
||||
func (i *editIndex) take(original, replacement string, near int) (pendingRow, bool) {
|
||||
best, bestDist := -1, 0
|
||||
for _, n := range i.byKey[editKey(original, replacement)] {
|
||||
if i.used[n] {
|
||||
continue
|
||||
}
|
||||
d := i.rows[n].from - near
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
if best < 0 || d < bestDist {
|
||||
best, bestDist = n, d
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return pendingRow{}, false
|
||||
}
|
||||
i.used[best] = true
|
||||
return i.rows[best], true
|
||||
}
|
||||
|
||||
// reposition updates the advisory offsets (and the sentence a row belongs to)
|
||||
// without touching anything the writer can see. The frontend re-anchors by
|
||||
// string at render time, so these only matter for the local-vs-model span
|
||||
// arbitration in dedupeSpans.
|
||||
func reposition(tx *sql.Tx, row pendingRow, from, to int, chunkHash string) error {
|
||||
if row.from == from && row.chunkHash == chunkHash {
|
||||
return nil
|
||||
}
|
||||
_, err := tx.Exec(
|
||||
`UPDATE suggestions SET from_pos = ?, to_pos = ?, chunk_hash = ? WHERE id = ?`,
|
||||
from, to, chunkHash, row.id,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// reconcilePending brings a model pass's family in line with what it just
|
||||
// proposed, sentence by sentence:
|
||||
//
|
||||
// - A row on a sentence this pass didn't ask about is kept untouched — that
|
||||
// is the whole point of chunking. Only its offsets are refreshed.
|
||||
// - A row on a sentence that no longer exists in the document is dropped: she
|
||||
// rewrote or deleted it.
|
||||
// - A row on a sentence the pass *did* ask about survives only if the model
|
||||
// proposed the same edit again, in which case it keeps its identity.
|
||||
//
|
||||
// `fresh` names the sentences the model was asked about (nil when it wasn't
|
||||
// called at all). inPlayAll marks the whole-document passes — voice and the
|
||||
// collocation coach — where every row is up for re-proposal because the model
|
||||
// just re-read everything.
|
||||
//
|
||||
// `pairLang` is the writer's own language, needed only to type a finding that
|
||||
// turns out to be her language rendered into English (see language.go).
|
||||
func (h *Handler) reconcilePending(
|
||||
docID, contentText, pairLang string,
|
||||
raw []llm.RawSuggestion,
|
||||
scope pendingScope,
|
||||
chunks, fresh []chunk,
|
||||
inPlayAll bool,
|
||||
) error {
|
||||
tx, err := h.DB.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
existing, err := loadPending(tx, docID, scope.deleteWhere)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
present := hashSet(chunks)
|
||||
asked := hashSet(fresh)
|
||||
modelRan := inPlayAll || fresh != nil
|
||||
|
||||
// Sentences to hand back to the model next time, because a row we were
|
||||
// caching on them turned out to be unanchorable (see below).
|
||||
reopen := map[string]bool{}
|
||||
|
||||
var inPlay []pendingRow
|
||||
for _, r := range existing {
|
||||
switch {
|
||||
// A row whose sentence we can't name is never cached — it is re-examined
|
||||
// whenever the model speaks, and left alone when it doesn't.
|
||||
case inPlayAll, r.chunkHash == "" && modelRan, asked[r.chunkHash]:
|
||||
inPlay = append(inPlay, r)
|
||||
case r.chunkHash != "" && !present[r.chunkHash]:
|
||||
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, r.id); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
// Untouched sentence: keep the card exactly as she last saw it.
|
||||
from, to := locate(contentText, r.original)
|
||||
if from < 0 {
|
||||
// The sentence is unchanged in substance but the quoted span no
|
||||
// longer matches byte for byte — a quote mark the editor rewrote
|
||||
// inside it, say. The frontend anchors by that string, so this card
|
||||
// can't be shown; drop it and let the sentence be read again rather
|
||||
// than cache advice nobody can see.
|
||||
reopen[r.chunkHash] = true
|
||||
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, r.id); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := reposition(tx, r, from, to, r.chunkHash); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
for h := range reopen {
|
||||
delete(present, h)
|
||||
}
|
||||
|
||||
sup, err := buildSuppressor(tx, docID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
index := indexByEdit(inPlay)
|
||||
kept := make(map[string]bool, len(inPlay))
|
||||
for _, s := range raw {
|
||||
if sup.suppressed(s.Original, s.Replacement) {
|
||||
continue
|
||||
}
|
||||
from, to := locate(contentText, s.Original)
|
||||
// Attribute the finding to a sentence the model was actually shown before
|
||||
// falling back to the whole document: a short span ("the the") can occur in
|
||||
// two sentences, and crediting it to the cached one would drop it as advice
|
||||
// we already have.
|
||||
hash := chunkFor(s.Original, fresh)
|
||||
if hash == "" {
|
||||
hash = chunkFor(s.Original, chunks)
|
||||
}
|
||||
// A sentence we didn't ask about already has whatever advice it deserves.
|
||||
// The model can't normally quote one — it was only shown the delta — but if
|
||||
// it wanders there anyway, the cached card stands rather than gaining a
|
||||
// twin.
|
||||
if !inPlayAll && hash != "" && present[hash] && !asked[hash] {
|
||||
continue
|
||||
}
|
||||
if row, ok := index.take(s.Original, s.Replacement, from); ok {
|
||||
kept[row.id] = true
|
||||
if err := reposition(tx, row, from, to, hash); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
// A pass with a forced type owns its family outright and is never asked
|
||||
// about translation: voice reads whole paragraphs for tone, and the
|
||||
// collocation coach is about English word pairings. Only the open-typed
|
||||
// grammar checkpoint can turn out to have been handed her own language.
|
||||
typ := scope.forceType
|
||||
if typ == "" {
|
||||
typ = normalizeType(s.Type)
|
||||
if isTranslation(s.Original, s.Replacement, pairLang) {
|
||||
typ = db.SuggestionTypeTranslate
|
||||
}
|
||||
}
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO suggestions (doc_id, from_pos, to_pos, original, replacement, explanation, type, source, chunk_hash)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
docID, from, to, s.Original, s.Replacement, s.Explanation, typ, db.SuggestionSourceLLM, hash,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Asked about and not proposed again: the model has changed its mind, or she
|
||||
// has fixed it.
|
||||
for _, r := range inPlay {
|
||||
if kept[r.id] {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM suggestions WHERE id = ?`, r.id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Record the sentences this family has now read. Every sentence still in the
|
||||
// document has been read by *some* pass: the ones just asked about now, the
|
||||
// rest in an earlier round.
|
||||
if scope.chunked {
|
||||
if _, err := tx.Exec(
|
||||
`DELETE FROM checked_chunks WHERE doc_id = ? AND family = ?`, docID, scope.family,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
for h := range present {
|
||||
if _, err := tx.Exec(
|
||||
`INSERT INTO checked_chunks (doc_id, family, hash) VALUES (?, ?, ?)`,
|
||||
docID, scope.family, h,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// checkedChunks loads the sentences a family read on its last pass.
|
||||
func (h *Handler) checkedChunks(docID, family string) (map[string]bool, error) {
|
||||
rows, err := h.DB.Query(
|
||||
`SELECT hash FROM checked_chunks WHERE doc_id = ? AND family = ?`, docID, family,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var hash string
|
||||
if err := rows.Scan(&hash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[hash] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
)
|
||||
@@ -56,7 +56,7 @@ func (h *Handler) rewrite(w http.ResponseWriter, r *http.Request) {
|
||||
var exists int
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT 1 FROM documents WHERE id = ? AND user_id = ?`,
|
||||
docID, db.LocalUserID,
|
||||
docID, auth.UserID(r.Context()),
|
||||
).Scan(&exists)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusNotFound, "document not found")
|
||||
@@ -69,7 +69,7 @@ func (h *Handler) rewrite(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
out, err := llm.RunRewrite(r.Context(), h.Client, text, body.Style)
|
||||
if err != nil {
|
||||
httputil.ErrorJSON(w, http.StatusBadGateway, "rewrite failed: "+err.Error())
|
||||
httputil.UpstreamError(w, "rewrite", err)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// The settled endpoint exists for the offline half of the loop. The rule pack
|
||||
// detects from the text alone, 250 ms after a keystroke, and has no memory
|
||||
// between runs — so without the document's record of what she has already
|
||||
// answered, a dismissed finding is re-detected and re-rendered on the next
|
||||
// keystroke, and stays there for as long as the server can't be reached.
|
||||
|
||||
func getSettled(t *testing.T, srv http.Handler, docID string) []string {
|
||||
t.Helper()
|
||||
rec := do(t, srv, http.MethodGet, "/docs/"+docID+"/settled", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("settled: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
var out struct {
|
||||
Originals []string `json:"originals"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
return out.Originals
|
||||
}
|
||||
|
||||
func contains(list []string, want string) bool {
|
||||
for _, s := range list {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TestSettledListsActionedSpans proves the endpoint reports exactly the spans the
|
||||
// suppressor would drop: accepted and dismissed, never pending. A pending row
|
||||
// leaking in would be the damaging direction — the client would hide a card she
|
||||
// has never been shown an answer to.
|
||||
func TestSettledListsActionedSpans(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"},
|
||||
{"original":"two apple","replacement":"two apples","explanation":"plural","type":"grammar"}
|
||||
]}`}
|
||||
srv, docID, _ := newTestServer(t, client)
|
||||
|
||||
if got := getSettled(t, srv, docID); len(got) != 0 {
|
||||
t.Fatalf("nothing actioned yet, got %v", got)
|
||||
}
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var got []db.Suggestion
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &got)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("first pass: want 2, got %d", len(got))
|
||||
}
|
||||
|
||||
// One accepted, one still pending: only the accepted span is settled.
|
||||
do(t, srv, http.MethodPost, "/suggestions/"+got[0].ID+"/accept", "")
|
||||
settled := getSettled(t, srv, docID)
|
||||
if len(settled) != 1 || settled[0] != got[0].Original {
|
||||
t.Fatalf("want just %q settled, got %v", got[0].Original, settled)
|
||||
}
|
||||
|
||||
// A dismissal settles a span just as an accept does — the whole point of the
|
||||
// item: "you already decided about this one" doesn't mean "you agreed".
|
||||
do(t, srv, http.MethodPost, "/suggestions/"+got[1].ID+"/dismiss", "")
|
||||
settled = getSettled(t, srv, docID)
|
||||
if len(settled) != 2 || !contains(settled, got[1].Original) {
|
||||
t.Fatalf("dismissed span missing from %v", settled)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSettledNormalizesAndDedupes proves the payload is normalized server-side
|
||||
// and collapsed. The client compares its freshly-detected findings against these
|
||||
// strings, so the two sides have to agree on what "the same span" is — the
|
||||
// editor's quote churn is the case that breaks a byte-exact match, and it is why
|
||||
// normalizeForDedup exists at all.
|
||||
func TestSettledNormalizesAndDedupes(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
|
||||
// The same span twice, differing only in quote style and line breaks — one
|
||||
// accepted, one dismissed. Distinct rows; one settled span.
|
||||
a := seedSuggestion(t, h, docID, "text", db.SuggestionTypeGrammar,
|
||||
"She said \"hello\"\n to me", "She said 'hello' to me", "quotes")
|
||||
b := seedSuggestion(t, h, docID, "text", db.SuggestionTypeGrammar,
|
||||
"She said “hello” to me", "She said 'hello' to me", "quotes")
|
||||
do(t, srv, http.MethodPost, "/suggestions/"+a+"/accept", "")
|
||||
do(t, srv, http.MethodPost, "/suggestions/"+b+"/dismiss", "")
|
||||
|
||||
settled := getSettled(t, srv, docID)
|
||||
if len(settled) != 1 {
|
||||
t.Fatalf("two spellings of one span should collapse to one, got %v", settled)
|
||||
}
|
||||
if want := "She said 'hello' to me"; settled[0] != want {
|
||||
t.Fatalf("settled[0] = %q, want normalized %q", settled[0], want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNormalizeMatchesTheClient is the Go half of a pair. Every case here also
|
||||
// appears in web/src/lib/settled.test.ts, asserted against the TypeScript
|
||||
// reimplementation of this function. The two are compared across a network
|
||||
// boundary — the server normalizes what it sends, the client normalizes what it
|
||||
// checks against it — so they have to fold the same characters the same way, and
|
||||
// nothing but a shared list of cases can say so. Add to both or neither.
|
||||
func TestNormalizeMatchesTheClient(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"She said “hello”", "She said 'hello'"},
|
||||
{"She said \"hello\"", "She said 'hello'"},
|
||||
{"it‘s", "it's"},
|
||||
{"it’s", "it's"},
|
||||
{"`code´", "'code'"},
|
||||
{" a apple\n here ", "a apple here"},
|
||||
{"a\tapple", "a apple"},
|
||||
{" \n ", ""},
|
||||
{"我想说这句话", "我想说这句话"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := normalizeForDedup(c.in); got != c.want {
|
||||
t.Errorf("normalizeForDedup(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSettledEmptyIsAList guards the shape rather than the content: the client
|
||||
// spreads this array into its settled set, and a null would throw there. Go
|
||||
// marshals a nil slice as null, so this is one `[]string{}` away from breaking.
|
||||
func TestSettledEmptyIsAList(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[]}`}
|
||||
srv, docID, _ := newTestServer(t, client)
|
||||
|
||||
rec := do(t, srv, http.MethodGet, "/docs/"+docID+"/settled", "")
|
||||
if body := rec.Body.String(); body != "{\"originals\":[]}\n" && body != "{\"originals\":[]}" {
|
||||
t.Fatalf("empty settled body = %q, want an empty list", body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
)
|
||||
|
||||
// byOriginal indexes a pending set by the text each card flags.
|
||||
func byOriginal(in []db.Suggestion) map[string]db.Suggestion {
|
||||
out := map[string]db.Suggestion{}
|
||||
for _, s := range in {
|
||||
out[s.Original] = s
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestUntouchedSentencesKeepTheirCards is the heart of the stability work: she
|
||||
// edits one sentence, and the cards on every other sentence stay exactly as they
|
||||
// were — same id (so the rail keeps the card instead of remounting it), same
|
||||
// explanation (the model re-words its reasoning every time it is asked, and one
|
||||
// unchanged mistake used to carry three different explanations in a sitting).
|
||||
// The model is only asked about the sentence that changed.
|
||||
func TestUntouchedSentencesKeepTheirCards(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"I has two apple","replacement":"I have two apples","explanation":"first wording","type":"grammar"},
|
||||
{"original":"She go to market","replacement":"She goes to market","explanation":"agreement","type":"grammar"}
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
h.Limit = llm.NewRateLimiter(0)
|
||||
setDocText(t, h, docID, "I has two apple. She go to market yesterday.")
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var first []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &first); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(first) != 2 {
|
||||
t.Fatalf("first pass: want 2, got %d: %+v", len(first), first)
|
||||
}
|
||||
kept := byOriginal(first)["I has two apple"]
|
||||
|
||||
// She fixes only the second sentence. The model, asked again, re-words its
|
||||
// reasoning about the first — which it must never get the chance to do.
|
||||
setDocText(t, h, docID, "I has two apple. She goes to market yesterday.")
|
||||
client.response = `{"suggestions":[
|
||||
{"original":"I has two apple","replacement":"I have two apples","explanation":"REWORDED","type":"grammar"}
|
||||
]}`
|
||||
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var second []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &second); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
|
||||
if strings.Contains(client.lastPrompt, "I has two apple") {
|
||||
t.Fatalf("untouched sentence was sent to the model:\n%s", client.lastPrompt)
|
||||
}
|
||||
if !strings.Contains(client.lastPrompt, "She goes to market") {
|
||||
t.Fatalf("edited sentence was not sent to the model:\n%s", client.lastPrompt)
|
||||
}
|
||||
|
||||
now := byOriginal(second)["I has two apple"]
|
||||
if now.ID != kept.ID {
|
||||
t.Fatalf("card was remounted: id %q became %q", kept.ID, now.ID)
|
||||
}
|
||||
if now.Explanation != "first wording" {
|
||||
t.Fatalf("explanation drifted: %q", now.Explanation)
|
||||
}
|
||||
// The fixed sentence's card is gone, and the model's stray re-proposal for the
|
||||
// cached sentence did not become a second card.
|
||||
if len(second) != 1 {
|
||||
t.Fatalf("want exactly one card left, got %d: %+v", len(second), second)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnchangedDocumentSkipsTheModel proves a check with nothing new to read
|
||||
// costs nothing: no model call, and every card left standing untouched. This is
|
||||
// the doc-open and tone-less re-check path.
|
||||
func TestUnchangedDocumentSkipsTheModel(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"}
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
h.Limit = llm.NewRateLimiter(0)
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var first []db.Suggestion
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &first)
|
||||
if len(first) != 1 || client.calls != 1 {
|
||||
t.Fatalf("first pass: %d cards, %d calls", len(first), client.calls)
|
||||
}
|
||||
|
||||
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var second []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &second); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if client.calls != 1 {
|
||||
t.Fatalf("re-checking an unedited document called the model %d times", client.calls)
|
||||
}
|
||||
if len(second) != 1 || second[0].ID != first[0].ID {
|
||||
t.Fatalf("card did not survive an idle re-check: %+v", second)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeletedSentenceDropsItsCard covers the other half of the skip path: she
|
||||
// removes a flagged sentence outright, so nothing changed that the model could
|
||||
// be asked about — but its card must still go.
|
||||
func TestDeletedSentenceDropsItsCard(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"}
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
h.Limit = llm.NewRateLimiter(0)
|
||||
|
||||
do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
setDocText(t, h, docID, "")
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
var got []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("card outlived its sentence: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestToneChangeReopensEverySentence: the checkpoint's advice is written for the
|
||||
// document's tone, so switching from a journal to an academic essay has to
|
||||
// re-read sentences that haven't changed a character.
|
||||
func TestToneChangeReopensEverySentence(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"I has","replacement":"I have","explanation":"agreement","type":"grammar"}
|
||||
]}`}
|
||||
srv, docID, h := newTestServer(t, client)
|
||||
h.Limit = llm.NewRateLimiter(0)
|
||||
|
||||
do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if _, err := h.DB.Exec(`UPDATE documents SET tone = 'academic' WHERE id = ?`, docID); err != nil {
|
||||
t.Fatalf("set tone: %v", err)
|
||||
}
|
||||
do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
|
||||
if client.calls != 2 {
|
||||
t.Fatalf("tone change did not re-read the document: %d model calls", client.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMechanicsFindingsKeepTheirRows: the rule pack re-runs 250 ms after every
|
||||
// keystroke. A finding it still reports must keep its row, or the rail would
|
||||
// remount several times a sentence — collapsing a card she has open, and
|
||||
// re-firing the arrival chime for advice she is already reading.
|
||||
func TestMechanicsFindingsKeepTheirRows(t *testing.T) {
|
||||
srv, docID, _ := newTestServer(t, &stubClient{response: `{"suggestions":[]}`})
|
||||
body := `{"findings":[
|
||||
{"from":0,"to":5,"original":"I has","replacement":"I have","explanation":"agreement"},
|
||||
{"from":6,"to":15,"original":"two apple","replacement":"two apples","explanation":"plural"}
|
||||
]}`
|
||||
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/mechanics", body)
|
||||
var first []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &first); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(first) != 2 {
|
||||
t.Fatalf("want 2 rows, got %d", len(first))
|
||||
}
|
||||
|
||||
// She types elsewhere: same findings, shifted spans, one of them now fixed.
|
||||
rec = do(t, srv, http.MethodPost, "/docs/"+docID+"/mechanics", `{"findings":[
|
||||
{"from":20,"to":25,"original":"I has","replacement":"I have","explanation":"agreement"}
|
||||
]}`)
|
||||
var second []db.Suggestion
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &second); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(second) != 1 {
|
||||
t.Fatalf("want 1 row, got %d: %+v", len(second), second)
|
||||
}
|
||||
if second[0].ID != byOriginal(first)["I has"].ID {
|
||||
t.Fatalf("surviving finding was given a new identity: %+v", second[0])
|
||||
}
|
||||
if second[0].FromPos != 20 {
|
||||
t.Fatalf("span did not follow the text: %+v", second[0])
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
"gitea.parodia.dev/drwily/petal/internal/llm"
|
||||
)
|
||||
@@ -25,14 +25,15 @@ type translateResponse struct {
|
||||
func (h *Handler) translate(w http.ResponseWriter, r *http.Request) {
|
||||
sugID := chi.URLParam(r, "id")
|
||||
|
||||
var explanation string
|
||||
var explanation, pairLang string
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT s.explanation
|
||||
`SELECT s.explanation, COALESCE(u.pair_lang, '')
|
||||
FROM suggestions s
|
||||
JOIN documents d ON d.id = s.doc_id
|
||||
JOIN users u ON u.id = d.user_id
|
||||
WHERE s.id = ? AND d.user_id = ?`,
|
||||
sugID, db.LocalUserID,
|
||||
).Scan(&explanation)
|
||||
sugID, auth.UserID(r.Context()),
|
||||
).Scan(&explanation, &pairLang)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusNotFound, "suggestion not found")
|
||||
return
|
||||
@@ -48,9 +49,9 @@ func (h *Handler) translate(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
out, err := llm.RunTranslate(r.Context(), h.Client, explanation)
|
||||
out, err := llm.RunTranslate(r.Context(), h.Client, explanation, llm.LangFor(pairLang))
|
||||
if err != nil {
|
||||
httputil.ErrorJSON(w, http.StatusBadGateway, "translate failed: "+err.Error())
|
||||
httputil.UpstreamError(w, "translate", err)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package suggestions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
// The pair model's flagship moment, end to end: she reaches for a sentence in
|
||||
// her own language mid-document, and the card that comes back is labelled as a
|
||||
// translation rather than as a tidy-up of her Chinese.
|
||||
//
|
||||
// The label is asserted through the real /check path rather than against
|
||||
// isTranslation directly, because the point of the item was never the detector —
|
||||
// Petal already found these spans and already rendered them into English. What
|
||||
// was wrong was the type that reached the rail.
|
||||
func TestChineseSpanBecomesATranslateCard(t *testing.T) {
|
||||
// Note the model calls it "clarity", as the live build did. The type it
|
||||
// volunteers is not consulted.
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"我想说这句话但是不知道用英语怎么说。","replacement":"I want to say this but I don't know how to say it in English.","explanation":"这是英文说法 · Here is how to say it in English","type":"clarity"}
|
||||
]}`}
|
||||
srv, docID, database := newPairServer(t, client, "zh")
|
||||
setDocTextDB(t, database, docID, "My weekend was good. 我想说这句话但是不知道用英语怎么说。")
|
||||
|
||||
var out []db.Suggestion
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("check: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("want 1 card, got %d: %+v", len(out), out)
|
||||
}
|
||||
if out[0].Type != db.SuggestionTypeTranslate {
|
||||
t.Fatalf("card type = %q, want %q", out[0].Type, db.SuggestionTypeTranslate)
|
||||
}
|
||||
// The rendering and the reasoning are the model's, untouched — only the label
|
||||
// is Petal's.
|
||||
if out[0].Replacement != "I want to say this but I don't know how to say it in English." {
|
||||
t.Fatalf("replacement was rewritten: %q", out[0].Replacement)
|
||||
}
|
||||
}
|
||||
|
||||
// The other half of the same claim: an ordinary English correction on the same
|
||||
// writer's document keeps the type the model gave it. A relabel that fired on
|
||||
// everything would be no better than the label it replaced.
|
||||
func TestEnglishCorrectionKeepsItsType(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"My weekend was very good","replacement":"My weekend was wonderful","explanation":"stronger wording","type":"phrasing"}
|
||||
]}`}
|
||||
srv, docID, database := newPairServer(t, client, "zh")
|
||||
setDocTextDB(t, database, docID, "My weekend was very good.")
|
||||
|
||||
var out []db.Suggestion
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/check", "")
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("want 1 card, got %d: %+v", len(out), out)
|
||||
}
|
||||
if out[0].Type != db.SuggestionTypePhrasing {
|
||||
t.Fatalf("card type = %q, want %q", out[0].Type, db.SuggestionTypePhrasing)
|
||||
}
|
||||
}
|
||||
|
||||
// The voice pass reads whole paragraphs for tone and stamps its own family. A
|
||||
// Chinese paragraph must not be able to smuggle a translate row into it — voice
|
||||
// rows carry no replacement to accept, so a "translation" there would be a card
|
||||
// offering nothing.
|
||||
func TestVoicePassCannotProduceATranslateCard(t *testing.T) {
|
||||
client := &stubClient{response: `{"suggestions":[
|
||||
{"original":"我想说这句话但是不知道用英语怎么说。","replacement":"I want to say this in English.","explanation":"tone","type":"clarity"}
|
||||
]}`}
|
||||
srv, docID, database := newPairServer(t, client, "zh")
|
||||
setDocTextDB(t, database, docID, "A first paragraph.\n\n我想说这句话但是不知道用英语怎么说。")
|
||||
|
||||
var out []db.Suggestion
|
||||
rec := do(t, srv, http.MethodPost, "/docs/"+docID+"/voice", "")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("voice: code=%d body=%s", rec.Code, rec.Body)
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
for _, s := range out {
|
||||
if s.Type == db.SuggestionTypeTranslate {
|
||||
t.Fatalf("voice pass produced a translate card: %+v", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// setDocTextDB is setDocText for the pair harness, which hands back the DB
|
||||
// rather than the Handler.
|
||||
func setDocTextDB(t *testing.T, database *db.DB, docID, text string) {
|
||||
t.Helper()
|
||||
if _, err := database.Exec(
|
||||
`UPDATE documents SET content_text = ? WHERE id = ?`, text, docID,
|
||||
); err != nil {
|
||||
t.Fatalf("update doc text: %v", err)
|
||||
}
|
||||
}
|
||||
+144
-18
@@ -19,7 +19,9 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
@@ -36,6 +38,12 @@ const maxTextBytes = 4000
|
||||
// with the words, mirroring the old utterance.rate = 0.95. Higher = slower.
|
||||
const lengthScale = 1.1
|
||||
|
||||
// slowLengthScale is the "say it slower" replay (SUGGESTIONS §5e): roughly 0.75×
|
||||
// the normal pace, which is the speed listening drills have used for decades.
|
||||
// Piper stretches durations rather than resampling, so the voice keeps its pitch
|
||||
// instead of turning into a slowed tape.
|
||||
const slowLengthScale = lengthScale / 0.75
|
||||
|
||||
// audioFormat describes one output encoding: the cache-file extension, the
|
||||
// response Content-Type, and the ffmpeg args that turn Piper's WAV (on stdin)
|
||||
// into this format (on stdout). A nil ffmpegArgs means "serve the WAV as-is".
|
||||
@@ -56,6 +64,25 @@ var formats = map[string]audioFormat{
|
||||
ffmpegArgs: []string{"-f", "ogg", "-c:a", "libopus", "-b:a", "32k", "-ac", "1"}},
|
||||
}
|
||||
|
||||
// synthPath normalises TTS_PATH into a leading-slash path with no trailing
|
||||
// slash, so it concatenates cleanly onto a route's endpoint.
|
||||
//
|
||||
// Piper moved synthesis from `POST /` to `POST /synthesize` in 1.6.0, and the
|
||||
// request body is identical either side of that change. Rather than pinning
|
||||
// every deployment to one Piper release, the path is configuration: millenia
|
||||
// keeps the default `/` its installed server expects, and the containerised
|
||||
// 1.6.0 sidecars on the VPS set `/synthesize`.
|
||||
func synthPath(p string) string {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" || p == "/" {
|
||||
return "/"
|
||||
}
|
||||
if !strings.HasPrefix(p, "/") {
|
||||
p = "/" + p
|
||||
}
|
||||
return strings.TrimRight(p, "/")
|
||||
}
|
||||
|
||||
// route is the Piper instance and voice id serving one language. Each Piper
|
||||
// HTTP server loads exactly one model, so distinct languages mean distinct
|
||||
// endpoints (e.g. English on :5005, Chinese on :5006).
|
||||
@@ -67,9 +94,11 @@ type route struct {
|
||||
// Handler proxies synthesis to Piper and caches the result on disk.
|
||||
type Handler struct {
|
||||
routes map[string]route // base language (e.g. "en", "zh") -> Piper instance
|
||||
synthURI string // path Piper serves synthesis on (see TTSPath)
|
||||
cacheDir string
|
||||
format audioFormat
|
||||
client *http.Client
|
||||
writes atomic.Uint64 // cache writes since boot; drives the prune throttle
|
||||
}
|
||||
|
||||
// New builds a Handler from config. It returns (nil, false) when TTS_ENDPOINT is
|
||||
@@ -85,16 +114,14 @@ func New(cfg *config.Config) (*Handler, bool) {
|
||||
format = formats["mp3"]
|
||||
}
|
||||
|
||||
// Map by base language so en-US, en-GB, etc. all resolve to the English
|
||||
// instance (the client sends BCP-47 tags like the old Web Speech path did).
|
||||
// A language is only routable when both its endpoint and voice are set;
|
||||
// otherwise the client falls back to Web Speech for that language.
|
||||
// Keyed by base language so en-US, en-GB — and pt-PT, pt-BR, bare pt —
|
||||
// resolve to the one instance that has that language's model loaded (the
|
||||
// client sends BCP-47 tags, as the old Web Speech path did). Config has
|
||||
// already dropped any language configured by halves, so an unroutable
|
||||
// language reaches the client as a 404 and falls back to Web Speech.
|
||||
routes := map[string]route{}
|
||||
if cfg.TTSVoiceEN != "" {
|
||||
routes["en"] = route{strings.TrimRight(cfg.TTSEndpoint, "/"), cfg.TTSVoiceEN}
|
||||
}
|
||||
if cfg.TTSEndpointZH != "" && cfg.TTSVoiceZH != "" {
|
||||
routes["zh"] = route{strings.TrimRight(cfg.TTSEndpointZH, "/"), cfg.TTSVoiceZH}
|
||||
for lang, v := range cfg.TTSVoices {
|
||||
routes[lang] = route{endpoint: v.Endpoint, voice: v.Voice}
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(cfg.TTSCacheDir, 0o755); err != nil {
|
||||
@@ -105,12 +132,26 @@ func New(cfg *config.Config) (*Handler, bool) {
|
||||
|
||||
return &Handler{
|
||||
routes: routes,
|
||||
synthURI: synthPath(cfg.TTSPath),
|
||||
cacheDir: cfg.TTSCacheDir,
|
||||
format: format,
|
||||
client: &http.Client{Timeout: cfg.TTSTimeout},
|
||||
}, true
|
||||
}
|
||||
|
||||
// Languages lists the base language tags this handler can synthesize, sorted,
|
||||
// each with the voice serving it — for the startup line, so a deployment says
|
||||
// which sidecars it actually reached rather than which ones it was configured
|
||||
// to want.
|
||||
func (h *Handler) Languages() []string {
|
||||
out := make([]string, 0, len(h.routes))
|
||||
for lang, rt := range h.routes {
|
||||
out = append(out, lang+"="+rt.voice)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Routes mounts the synthesis endpoint. Mount under "/tts" so the full path is
|
||||
// POST /api/tts.
|
||||
func (h *Handler) Routes() chi.Router {
|
||||
@@ -119,11 +160,13 @@ func (h *Handler) Routes() chi.Router {
|
||||
return r
|
||||
}
|
||||
|
||||
// synthRequest is the body the editor posts: a passage and the BCP-47 language
|
||||
// tag it's written in (e.g. "en-US", "zh-CN").
|
||||
// synthRequest is the body the editor posts: a passage, the BCP-47 language tag
|
||||
// it's written in (e.g. "en-US", "zh-CN", "pt-PT"), and whether to say it slowly
|
||||
// — the replay a learner reaches for when the sentence went past too fast.
|
||||
type synthRequest struct {
|
||||
Text string `json:"text"`
|
||||
Lang string `json:"lang"`
|
||||
Slow bool `json:"slow"`
|
||||
}
|
||||
|
||||
// synth resolves a voice for the requested language, returns cached audio when
|
||||
@@ -159,9 +202,17 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Content-addressed: identical (voice, text) → identical clip. The format
|
||||
// extension keeps encodings from colliding in the same dir.
|
||||
sum := sha256.Sum256([]byte(rt.voice + "\n" + text))
|
||||
scale := lengthScale
|
||||
if req.Slow {
|
||||
scale = slowLengthScale
|
||||
}
|
||||
|
||||
// Content-addressed: identical (voice, pace, text) → identical clip. The pace
|
||||
// belongs in the key — without it the slow replay of a word already heard at
|
||||
// normal speed would be served from cache at normal speed, which is the one
|
||||
// request where the difference is the whole point. The format extension keeps
|
||||
// encodings from colliding in the same dir.
|
||||
sum := sha256.Sum256([]byte(fmt.Sprintf("%s\n%.3f\n%s", rt.voice, scale, text)))
|
||||
name := hex.EncodeToString(sum[:])[:32] + h.format.ext
|
||||
path := filepath.Join(h.cacheDir, name)
|
||||
|
||||
@@ -170,7 +221,7 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
audio, err := h.synthesize(r.Context(), rt, text)
|
||||
audio, err := h.synthesize(r.Context(), rt, text, scale)
|
||||
if err != nil {
|
||||
http.Error(w, "synthesis failed", http.StatusBadGateway)
|
||||
fmt.Fprintf(os.Stderr, "tts: synthesize: %v\n", err)
|
||||
@@ -184,6 +235,7 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
||||
if err := os.WriteFile(tmp, audio, 0o644); err == nil {
|
||||
_ = os.Rename(tmp, path)
|
||||
}
|
||||
h.pruneCache()
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", h.format.contentType)
|
||||
@@ -191,6 +243,80 @@ func (h *Handler) synth(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write(audio)
|
||||
}
|
||||
|
||||
// maxCacheBytes bounds the whole clip cache at 512 MiB.
|
||||
//
|
||||
// Each clip is small, so nothing about ordinary reading approaches this — a
|
||||
// year of tapping words is tens of megabytes. What it bounds is the shape of
|
||||
// the endpoint: the cache key is the *text*, so a client asking for four
|
||||
// thousand distinct characters at a time writes a new file every request, for
|
||||
// as long as it cares to. That is an authenticated writer filling the same
|
||||
// encrypted volume the database lives on, and a full disk is SQLite failing to
|
||||
// write, not merely read-aloud getting slower.
|
||||
const maxCacheBytes = 512 << 20
|
||||
|
||||
// pruneEvery throttles the sweep: checking the directory on every synthesis
|
||||
// would stat the whole cache for each new word. Synthesis is already the slow
|
||||
// path and misses are rare once a writer settles, so one sweep per this many
|
||||
// cache writes keeps the cost invisible while still converging long before the
|
||||
// limit means anything.
|
||||
const pruneEvery = 64
|
||||
|
||||
// pruneCache trims the cache back under maxCacheBytes, oldest-first, and is a
|
||||
// no-op the great majority of the time it is called.
|
||||
//
|
||||
// Oldest by modification time is a fair approximation of least-recently-useful
|
||||
// here: a clip is written once and only ever read afterwards, so its age is how
|
||||
// long ago someone wanted it. Evicting one costs a re-synthesis, never data —
|
||||
// which is why this can be as approximate as it likes, and why every error
|
||||
// along the way is simply given up on.
|
||||
func (h *Handler) pruneCache() {
|
||||
if n := h.writes.Add(1); n%pruneEvery != 0 {
|
||||
return
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(h.cacheDir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
type clip struct {
|
||||
path string
|
||||
size int64
|
||||
mod time.Time
|
||||
}
|
||||
var clips []clip
|
||||
var total int64
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
clips = append(clips, clip{filepath.Join(h.cacheDir, e.Name()), info.Size(), info.ModTime()})
|
||||
total += info.Size()
|
||||
}
|
||||
if total <= maxCacheBytes {
|
||||
return
|
||||
}
|
||||
|
||||
sort.Slice(clips, func(i, j int) bool { return clips[i].mod.Before(clips[j].mod) })
|
||||
// Drop to 80% rather than exactly to the line, so the next few hundred
|
||||
// clips don't each trigger another sweep.
|
||||
target := int64(maxCacheBytes / 100 * 80)
|
||||
removed := 0
|
||||
for _, c := range clips {
|
||||
if total <= target {
|
||||
break
|
||||
}
|
||||
if os.Remove(c.path) == nil {
|
||||
total -= c.size
|
||||
removed++
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "tts: cache over %d bytes — evicted %d oldest clip(s)\n", int64(maxCacheBytes), removed)
|
||||
}
|
||||
|
||||
// serve streams a cached clip with a long-lived immutable cache header (the URL
|
||||
// is content-addressed, so the bytes never change for a given request).
|
||||
func (h *Handler) serve(w http.ResponseWriter, r *http.Request, path string) {
|
||||
@@ -201,13 +327,13 @@ func (h *Handler) serve(w http.ResponseWriter, r *http.Request, path string) {
|
||||
|
||||
// synthesize POSTs to the route's Piper instance, then transcodes the returned
|
||||
// WAV when the configured format calls for it.
|
||||
func (h *Handler) synthesize(ctx context.Context, rt route, text string) ([]byte, error) {
|
||||
func (h *Handler) synthesize(ctx context.Context, rt route, text string, scale float64) ([]byte, error) {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"text": text,
|
||||
"voice": rt.voice,
|
||||
"length_scale": lengthScale,
|
||||
"length_scale": scale,
|
||||
})
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, rt.endpoint+"/", bytes.NewReader(body))
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, rt.endpoint+h.synthURI, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ func newStubPiper(t *testing.T, body []byte) (*httptest.Server, *int32, *synthEc
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
last.voice, _ = req["voice"].(string)
|
||||
last.text, _ = req["text"].(string)
|
||||
last.scale, _ = req["length_scale"].(float64)
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
_, _ = w.Write(body)
|
||||
}))
|
||||
@@ -29,22 +30,74 @@ func newStubPiper(t *testing.T, body []byte) (*httptest.Server, *int32, *synthEc
|
||||
return srv, &calls, last
|
||||
}
|
||||
|
||||
type synthEcho struct{ voice, text string }
|
||||
type synthEcho struct {
|
||||
voice, text string
|
||||
scale float64
|
||||
}
|
||||
|
||||
// newHandler builds a wav-format handler (no ffmpeg) pointed at a stub server.
|
||||
func newHandler(t *testing.T, endpoint string) *Handler {
|
||||
t.Helper()
|
||||
return &Handler{
|
||||
routes: map[string]route{"en": {strings.TrimRight(endpoint, "/"), "en_US-amy-medium"}},
|
||||
synthURI: synthPath("/"),
|
||||
cacheDir: t.TempDir(),
|
||||
format: formats["wav"],
|
||||
client: http.DefaultClient,
|
||||
}
|
||||
}
|
||||
|
||||
// Piper 1.6.0 serves synthesis on /synthesize and 405s on /. The path is
|
||||
// configuration (TTS_PATH) so one Petal build talks to either server version;
|
||||
// this asserts the configured path is the one actually requested.
|
||||
func TestSynthUsesConfiguredPath(t *testing.T) {
|
||||
var gotPath string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if r.URL.Path != "/synthesize" {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
_, _ = w.Write([]byte("RIFF....fake-wav"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
h := newHandler(t, srv.URL)
|
||||
h.synthURI = synthPath("/synthesize")
|
||||
|
||||
if rr := post(t, h, "hello there", "en-US"); rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200 (piper saw path %q)", rr.Code, gotPath)
|
||||
}
|
||||
if gotPath != "/synthesize" {
|
||||
t.Errorf("piper path = %q, want /synthesize", gotPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSynthPathNormalisation(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"": "/",
|
||||
"/": "/",
|
||||
"synthesize": "/synthesize",
|
||||
"/synthesize": "/synthesize",
|
||||
"/synthesize/": "/synthesize",
|
||||
" /v1/tts ": "/v1/tts",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := synthPath(in); got != want {
|
||||
t.Errorf("synthPath(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func post(t *testing.T, h *Handler, text, lang string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
b, _ := json.Marshal(synthRequest{Text: text, Lang: lang})
|
||||
return postReq(t, h, synthRequest{Text: text, Lang: lang})
|
||||
}
|
||||
|
||||
func postReq(t *testing.T, h *Handler, body synthRequest) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
b, _ := json.Marshal(body)
|
||||
req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(b))
|
||||
rr := httptest.NewRecorder()
|
||||
h.synth(rr, req)
|
||||
@@ -148,8 +201,87 @@ func TestTextIsCapped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The slow replay is the whole of SUGGESTIONS §5e: same text, same voice, more
|
||||
// time per phoneme.
|
||||
func TestSlowRequestStretchesTheVoice(t *testing.T) {
|
||||
srv, _, last := newStubPiper(t, []byte("RIFF....fake-wav"))
|
||||
h := newHandler(t, srv.URL)
|
||||
|
||||
if rr := postReq(t, h, synthRequest{Text: "reception", Lang: "en-US"}); rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
if last.scale != lengthScale {
|
||||
t.Fatalf("normal length_scale = %v, want %v", last.scale, lengthScale)
|
||||
}
|
||||
|
||||
if rr := postReq(t, h, synthRequest{Text: "reception", Lang: "en-US", Slow: true}); rr.Code != http.StatusOK {
|
||||
t.Fatalf("slow status = %d, want 200", rr.Code)
|
||||
}
|
||||
if last.scale != slowLengthScale {
|
||||
t.Fatalf("slow length_scale = %v, want %v", last.scale, slowLengthScale)
|
||||
}
|
||||
if slowLengthScale <= lengthScale {
|
||||
t.Fatalf("slowLengthScale %v is not slower than %v", slowLengthScale, lengthScale)
|
||||
}
|
||||
}
|
||||
|
||||
// The pace has to be part of the cache key. Without it, asking for the slow
|
||||
// replay of a word already heard at normal speed serves the normal clip — the
|
||||
// one request where hearing the difference is the entire point.
|
||||
func TestSlowClipIsNotServedFromTheNormalCache(t *testing.T) {
|
||||
srv, calls, last := newStubPiper(t, []byte("RIFF....fake-wav"))
|
||||
h := newHandler(t, srv.URL)
|
||||
|
||||
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US"})
|
||||
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US", Slow: true})
|
||||
if *calls != 2 {
|
||||
t.Fatalf("piper calls = %d, want 2 (the slow clip is a different clip)", *calls)
|
||||
}
|
||||
if last.scale != slowLengthScale {
|
||||
t.Fatalf("second call length_scale = %v, want the slow one", last.scale)
|
||||
}
|
||||
|
||||
// …and each pace still caches on its own.
|
||||
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US", Slow: true})
|
||||
postReq(t, h, synthRequest{Text: "reception", Lang: "en-US"})
|
||||
if *calls != 2 {
|
||||
t.Fatalf("piper calls = %d, want 2 (both paces now cached)", *calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A Portuguese request must reach the Portuguese instance on the base tag alone:
|
||||
// env var names cannot hold the hyphen in pt-PT, so config keys the map on "pt"
|
||||
// and the handler has to meet it there. pt-BR resolves to the same instance
|
||||
// because there is only one Portuguese voice loaded — and it is the European one.
|
||||
func TestPortugueseRoutesOnTheBaseTag(t *testing.T) {
|
||||
enSrv, enCalls, _ := newStubPiper(t, []byte("EN-wav"))
|
||||
ptSrv, ptCalls, ptLast := newStubPiper(t, []byte("PT-wav"))
|
||||
h := &Handler{
|
||||
routes: map[string]route{
|
||||
"en": {strings.TrimRight(enSrv.URL, "/"), "en_US-amy-medium"},
|
||||
"pt": {strings.TrimRight(ptSrv.URL, "/"), "pt_PT-tugão-medium"},
|
||||
},
|
||||
cacheDir: t.TempDir(),
|
||||
format: formats["wav"],
|
||||
client: http.DefaultClient,
|
||||
}
|
||||
|
||||
// Distinct text per tag, so a cache hit can't stand in for a route.
|
||||
for i, tag := range []string{"pt-PT", "pt", "pt-BR"} {
|
||||
if rr := post(t, h, strings.Repeat("receção ", i+1), tag); rr.Code != http.StatusOK {
|
||||
t.Fatalf("%s status = %d, want 200", tag, rr.Code)
|
||||
}
|
||||
}
|
||||
if *ptCalls != 3 || *enCalls != 0 {
|
||||
t.Fatalf("calls en=%d pt=%d, want en=0 pt=3", *enCalls, *ptCalls)
|
||||
}
|
||||
if ptLast.voice != "pt_PT-tugão-medium" {
|
||||
t.Fatalf("pt voice = %q, want the European Portuguese voice", ptLast.voice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBaseLang(t *testing.T) {
|
||||
cases := map[string]string{"en-US": "en", "EN_gb": "en", "zh-CN": "zh", "en": "en", "": ""}
|
||||
cases := map[string]string{"en-US": "en", "EN_gb": "en", "zh-CN": "zh", "pt-PT": "pt", "en": "en", "": ""}
|
||||
for in, want := range cases {
|
||||
if got := baseLang(in); got != want {
|
||||
t.Errorf("baseLang(%q) = %q, want %q", in, got, want)
|
||||
|
||||
+17
-13
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
"gitea.parodia.dev/drwily/petal/internal/httputil"
|
||||
)
|
||||
@@ -68,15 +69,15 @@ func scanWord(s interface {
|
||||
}
|
||||
|
||||
// list returns the full garden, newest blossoms first.
|
||||
func (h *Handler) list(w http.ResponseWriter, _ *http.Request) {
|
||||
func (h *Handler) list(w http.ResponseWriter, r *http.Request) {
|
||||
h.queryList(w, `SELECT `+vocabColumns+` FROM vocab_words
|
||||
WHERE user_id = ? ORDER BY created_at DESC`, db.LocalUserID)
|
||||
WHERE user_id = ? ORDER BY created_at DESC`, auth.UserID(r.Context()))
|
||||
}
|
||||
|
||||
// due returns only the cards whose review time has arrived, soonest first.
|
||||
func (h *Handler) due(w http.ResponseWriter, _ *http.Request) {
|
||||
func (h *Handler) due(w http.ResponseWriter, r *http.Request) {
|
||||
h.queryList(w, `SELECT `+vocabColumns+` FROM vocab_words
|
||||
WHERE user_id = ? AND due_at <= datetime('now') ORDER BY due_at ASC`, db.LocalUserID)
|
||||
WHERE user_id = ? AND due_at <= datetime('now') ORDER BY due_at ASC`, auth.UserID(r.Context()))
|
||||
}
|
||||
|
||||
func (h *Handler) queryList(w http.ResponseWriter, query string, args ...any) {
|
||||
@@ -138,6 +139,8 @@ func clamp(s string, max int) string {
|
||||
// schedule untouched but refreshes its gloss/phonetic/example/doc_id so the most
|
||||
// recent context wins. Looking words up IS the data source — no extra effort.
|
||||
func (h *Handler) capture(w http.ResponseWriter, r *http.Request) {
|
||||
userID := auth.UserID(r.Context())
|
||||
|
||||
var req captureRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
httputil.ErrorJSON(w, http.StatusBadRequest, "invalid body")
|
||||
@@ -169,7 +172,7 @@ func (h *Handler) capture(w http.ResponseWriter, r *http.Request) {
|
||||
var ok int
|
||||
err := h.DB.QueryRow(
|
||||
`SELECT 1 FROM documents WHERE id = ? AND user_id = ?`,
|
||||
*req.DocID, db.LocalUserID,
|
||||
*req.DocID, userID,
|
||||
).Scan(&ok)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusBadRequest, "unknown doc_id")
|
||||
@@ -194,14 +197,14 @@ func (h *Handler) capture(w http.ResponseWriter, r *http.Request) {
|
||||
phonetic = excluded.phonetic,
|
||||
example = CASE WHEN excluded.example != '' THEN excluded.example ELSE vocab_words.example END,
|
||||
doc_id = COALESCE(excluded.doc_id, vocab_words.doc_id)`,
|
||||
db.LocalUserID, word, req.Gloss, req.Definition, req.Phonetic, req.Example, req.DocID,
|
||||
userID, word, req.Gloss, req.Definition, req.Phonetic, req.Example, req.DocID,
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
out, err := h.fetch(word)
|
||||
out, err := h.fetch(userID, word)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
@@ -210,10 +213,10 @@ func (h *Handler) capture(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// fetch loads one word row by its (user, word) key.
|
||||
func (h *Handler) fetch(word string) (Word, error) {
|
||||
func (h *Handler) fetch(userID, word string) (Word, error) {
|
||||
return scanWord(h.DB.QueryRow(
|
||||
`SELECT `+vocabColumns+` FROM vocab_words WHERE user_id = ? AND word = ?`,
|
||||
db.LocalUserID, word,
|
||||
userID, word,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -225,6 +228,7 @@ type reviewRequest struct {
|
||||
// scheduler; the new interval is applied as `due_at = now + interval days`.
|
||||
func (h *Handler) review(w http.ResponseWriter, r *http.Request) {
|
||||
id := chi.URLParam(r, "id")
|
||||
userID := auth.UserID(r.Context())
|
||||
var req reviewRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
httputil.ErrorJSON(w, http.StatusBadRequest, "invalid body")
|
||||
@@ -249,7 +253,7 @@ func (h *Handler) review(w http.ResponseWriter, r *http.Request) {
|
||||
var cur State
|
||||
err = tx.QueryRow(
|
||||
`SELECT reps, interval_days, ease, lapses FROM vocab_words WHERE id = ? AND user_id = ?`,
|
||||
id, db.LocalUserID,
|
||||
id, userID,
|
||||
).Scan(&cur.Reps, &cur.Interval, &cur.Ease, &cur.Lapses)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httputil.ErrorJSON(w, http.StatusNotFound, "word not found")
|
||||
@@ -269,14 +273,14 @@ func (h *Handler) review(w http.ResponseWriter, r *http.Request) {
|
||||
reps = ?, interval_days = ?, ease = ?, lapses = ?,
|
||||
last_reviewed = datetime('now'), due_at = datetime('now', ?)
|
||||
WHERE id = ? AND user_id = ?`,
|
||||
nxt.Reps, nxt.Interval, nxt.Ease, nxt.Lapses, offset, id, db.LocalUserID,
|
||||
nxt.Reps, nxt.Interval, nxt.Ease, nxt.Lapses, offset, id, userID,
|
||||
); err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
out, err := scanWord(tx.QueryRow(
|
||||
`SELECT `+vocabColumns+` FROM vocab_words WHERE id = ? AND user_id = ?`, id, db.LocalUserID,
|
||||
`SELECT `+vocabColumns+` FROM vocab_words WHERE id = ? AND user_id = ?`, id, userID,
|
||||
))
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
@@ -293,7 +297,7 @@ func (h *Handler) review(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *Handler) remove(w http.ResponseWriter, r *http.Request) {
|
||||
res, err := h.DB.Exec(
|
||||
`DELETE FROM vocab_words WHERE id = ? AND user_id = ?`,
|
||||
chi.URLParam(r, "id"), db.LocalUserID,
|
||||
chi.URLParam(r, "id"), auth.UserID(r.Context()),
|
||||
)
|
||||
if err != nil {
|
||||
httputil.ServerError(w, err)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/auth"
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
@@ -23,7 +24,12 @@ func newTestServer(t *testing.T) (http.Handler, *db.DB) {
|
||||
t.Cleanup(func() { database.Close() })
|
||||
r := chi.NewRouter()
|
||||
r.Mount("/vocab", New(database).Routes())
|
||||
return r, database
|
||||
|
||||
// Behind the same auth middleware main.go installs: handlers resolve the
|
||||
// caller from the request context, so a bare router would see no user and
|
||||
// every user-scoped query would match nothing.
|
||||
authed := auth.Middleware(auth.StaticResolver(db.LocalUserID))(r)
|
||||
return authed, database
|
||||
}
|
||||
|
||||
func do(t *testing.T, srv http.Handler, method, path, body string) *httptest.ResponseRecorder {
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package vocab
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Planting: the garden's second source.
|
||||
//
|
||||
// Capture (handlers.go) records words the writer *sought out*. Planting records
|
||||
// phrasing she was gently *given* — an accepted collocation like "make a
|
||||
// decision" is a learnable chunk exactly like a looked-up word, and the SM-2-lite
|
||||
// scheduler doesn't care that it's three words rather than one. Together the two
|
||||
// halves make the garden a record of both sides of learning.
|
||||
//
|
||||
// Everything here is best-effort by design: planting hangs off accepting a
|
||||
// suggestion, and that accept must succeed whether or not a card comes of it.
|
||||
|
||||
// Execer is the slice of *sql.DB (or *sql.Tx) that planting needs.
|
||||
type Execer interface {
|
||||
Exec(query string, args ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
// Phrase is one chunk to plant.
|
||||
type Phrase struct {
|
||||
Text string // the corrected phrasing, e.g. "make a decision"
|
||||
Meaning string // why it's better — the suggestion's explanation
|
||||
Example string // the sentence she met it in, already corrected
|
||||
DocID *string // where, so "where did I see this?" stays one tap
|
||||
}
|
||||
|
||||
// Phrase-card caps. A collocation is a short chunk; anything longer is a
|
||||
// rewritten sentence wearing a collocation's label, and a sentence makes a
|
||||
// miserable flashcard. Both bounds are deliberately tight — the cost of
|
||||
// skipping a real chunk is one missing card, the cost of planting a sentence is
|
||||
// a garden the writer stops trusting.
|
||||
const (
|
||||
maxPhraseRunes = 60
|
||||
maxPhraseWords = 6
|
||||
minPhraseWords = 2
|
||||
)
|
||||
|
||||
// PhraseKey normalizes a replacement into a garden key, or returns "" when the
|
||||
// text isn't a plantable chunk.
|
||||
//
|
||||
// Lowercasing matches capture's normalization, so a phrase and a looked-up word
|
||||
// share one UNIQUE(user_id, word) namespace rather than colliding sideways.
|
||||
// Single words are rejected on purpose: a one-word fix is word choice, and word
|
||||
// choice already reaches the garden through lookup — planting it here would give
|
||||
// it a card with no gloss and no phonetic, which reviews badly.
|
||||
func PhraseKey(text string) string {
|
||||
// Collapse all whitespace (a replacement can carry a newline from the
|
||||
// editor) so the key is stable and the word count is honest.
|
||||
s := strings.Join(strings.Fields(strings.ToLower(text)), " ")
|
||||
// Trim the punctuation a phrase picks up from the sentence around it, but
|
||||
// leave inner marks alone: "can't afford" and "in one's own time" are chunks.
|
||||
s = strings.Trim(s, `.,;:!?…"'“”‘’()[]`)
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || len([]rune(s)) > maxPhraseRunes {
|
||||
return ""
|
||||
}
|
||||
n := len(strings.Fields(s))
|
||||
if n < minPhraseWords || n > maxPhraseWords {
|
||||
return ""
|
||||
}
|
||||
// A chunk of pure digits or symbols ("12 000", "-- --") isn't vocabulary.
|
||||
if !strings.ContainsFunc(s, unicode.IsLetter) {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Plant adds a phrase card to the garden, due tomorrow like any fresh capture.
|
||||
// It reports whether a new card was created.
|
||||
//
|
||||
// ON CONFLICT DO NOTHING, unlike capture's refresh-the-context upsert: accepting
|
||||
// the same collocation again months later is evidence the chunk is still being
|
||||
// learned, and the last thing that should do is overwrite the card's first
|
||||
// context or disturb a schedule it has been climbing. An existing card wins.
|
||||
func Plant(ex Execer, userID string, p Phrase) (bool, error) {
|
||||
key := PhraseKey(p.Text)
|
||||
if key == "" {
|
||||
return false, nil
|
||||
}
|
||||
res, err := ex.Exec(
|
||||
`INSERT INTO vocab_words (user_id, word, gloss, definition, phonetic, example, doc_id, due_at, interval_days)
|
||||
VALUES (?, ?, '', ?, '', ?, ?, datetime('now', '+1 day'), 1)
|
||||
ON CONFLICT(user_id, word) DO NOTHING`,
|
||||
userID, key,
|
||||
clamp(strings.TrimSpace(p.Meaning), maxDefinitionLen),
|
||||
clamp(strings.TrimSpace(p.Example), maxExampleLen),
|
||||
p.DocID,
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
return n > 0, err
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package vocab
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.parodia.dev/drwily/petal/internal/db"
|
||||
)
|
||||
|
||||
func TestPhraseKey(t *testing.T) {
|
||||
cases := []struct {
|
||||
in, want string
|
||||
}{
|
||||
{"make a decision", "make a decision"},
|
||||
{"Make A Decision", "make a decision"}, // shares one namespace with lookups
|
||||
{" make a\ndecision ", "make a decision"}, // the editor's whitespace
|
||||
{"“make a decision.”", "make a decision"}, // punctuation from the sentence around it
|
||||
{"can’t afford it", "can’t afford it"}, // inner marks are part of the chunk
|
||||
{"decision", ""}, // word choice, not a chunk — lookup's job
|
||||
{"", ""}, //
|
||||
{"...", ""}, //
|
||||
{"12 000", ""}, // digits aren't vocabulary
|
||||
{"a b c d e f g", ""}, // a clause wearing a chunk's label
|
||||
{"in one’s own good time again", "in one’s own good time again"}, // six words is still a chunk
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := PhraseKey(tc.in); got != tc.want {
|
||||
t.Errorf("PhraseKey(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
// The length cap counts runes, not bytes — otherwise a Portuguese chunk well
|
||||
// inside the limit would be dropped for being accented.
|
||||
accented := "ãããããã ãããããã ãããããã ãããããã ãããããã" // 34 runes, 64 bytes
|
||||
if got := PhraseKey(accented); got != accented {
|
||||
t.Errorf("PhraseKey(%d runes / %d bytes) = %q, want it kept", len([]rune(accented)), len(accented), got)
|
||||
}
|
||||
long := "ãããããããããããã ãããããããããããã ãããããããããããã ãããããããããããã ãããããããããããã ãããããããããããã"
|
||||
if got := PhraseKey(long); got != "" {
|
||||
t.Errorf("PhraseKey(%d runes) = %q, want \"\"", len([]rune(long)), got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlantCreatesOnceAndReportsIt(t *testing.T) {
|
||||
database, err := db.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { database.Close() })
|
||||
|
||||
p := Phrase{Text: "make a decision", Meaning: "why", Example: "I had to make a decision."}
|
||||
created, err := Plant(database, db.LocalUserID, p)
|
||||
if err != nil || !created {
|
||||
t.Fatalf("first plant: created=%v err=%v", created, err)
|
||||
}
|
||||
created, err = Plant(database, db.LocalUserID, p)
|
||||
if err != nil || created {
|
||||
t.Fatalf("second plant: created=%v err=%v, want false", created, err)
|
||||
}
|
||||
|
||||
// A card that isn't plantable is a silent no-op, not an error: planting hangs
|
||||
// off accepting an edit, and that accept must never fail for a flashcard.
|
||||
created, err = Plant(database, db.LocalUserID, Phrase{Text: "decision"})
|
||||
if err != nil || created {
|
||||
t.Fatalf("unplantable: created=%v err=%v", created, err)
|
||||
}
|
||||
|
||||
var n int
|
||||
if err := database.QueryRow(`SELECT count(*) FROM vocab_words WHERE user_id = ?`, db.LocalUserID).Scan(&n); err != nil {
|
||||
t.Fatalf("count: %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("garden has %d cards, want 1", n)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user