diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d15bb58 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,33 @@ +# Keep the build context small and the image reproducible. Anything the build +# needs but that is gitignored (web/dist) is produced inside the image instead. + +.git +.gitignore +.env + +# Built by stage 1 — never copy a stale local build into the image. +web/dist +web/node_modules + +# Local runtime state: the live database, images and TTS cache must never end +# up baked into an image layer. +data/ +*.db +*.db-shm +*.db-wal +backups/ + +# Local build outputs +/petal +*.test +*.out +*.log + +# Docs and tooling that don't affect the binary +*.md +!web/**/*.md +deploy/ +scripts/ +.vscode/ +.idea/ +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8ecd37a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,66 @@ +# Petal — multi-stage build producing the single self-contained binary. +# +# Stage 1 builds the frontend; stage 2 compiles the Go server with web/dist +# embedded (go:embed), so the runtime image carries one executable and no +# assets. modernc's SQLite is pure Go, so CGO stays off and the binary is +# static — the runtime layer exists only for ffmpeg (read-aloud transcodes +# Piper's WAV to mp3) and CA certificates. + +# ---------- stage 1: frontend ---------- +FROM node:22-alpine AS web + +WORKDIR /src/web + +# Install deps against the lockfile alone so this layer caches across source +# edits. The Hunspell dictionaries come from a devDependency, so a plain +# `npm ci` (not --omit=dev) is required for the spell checker to ship. +COPY web/package.json web/package-lock.json ./ +RUN npm ci + +COPY web/ ./ +RUN npm run build + +# ---------- stage 2: server ---------- +FROM golang:1.25-alpine AS build + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +# The build context's web/dist is gitignored and excluded by .dockerignore; +# take the freshly built one from stage 1 so go:embed picks it up. +COPY --from=web /src/web/dist ./web/dist + +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/petal ./cmd/server + +# ---------- stage 3: runtime ---------- +FROM alpine:3.21 + +# ffmpeg: read-aloud pipes Piper's WAV through it to mp3/opus. tzdata: the +# companion's bedtime nag and night mode read the local clock, so the container +# needs a real timezone rather than bare UTC. +RUN apk add --no-cache ca-certificates ffmpeg tzdata \ + && adduser -D -u 10001 petal + +WORKDIR /app +COPY --from=build /out/petal /app/petal + +# Mount point for petal.db (+ -wal/-shm), the image store and the TTS cache. +RUN mkdir -p /data && chown -R petal:petal /data +VOLUME ["/data"] + +USER petal +EXPOSE 8080 + +ENV PORT=8080 \ + DATABASE_PATH=/data/petal.db \ + IMAGE_DIR=/data/images \ + TTS_CACHE_DIR=/data/tts + +# Same endpoint Traefik and the uptime probe use; needs no session by design. +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD wget -qO- http://127.0.0.1:8080/api/health || exit 1 + +ENTRYPOINT ["/app/petal"] diff --git a/cmd/server/main.go b/cmd/server/main.go index c1739da..2370f6c 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "flag" "io/fs" "log" "net/http" @@ -26,8 +27,25 @@ import ( ) func main() { + backupTo := flag.String("backup", "", + "write a consistent copy of the database to this path and exit (no server)") + flag.Parse() + cfg := config.Load() + // Backup mode short-circuits before anything else starts: no migrations, no + // seed, no listener. It runs against the live database safely (VACUUM INTO + // takes only a read transaction), so the nightly job is + // docker compose exec petal /app/petal -backup /data/backups/.db + // against the running container rather than a copy of three WAL files. + if *backupTo != "" { + if err := db.Backup(cfg.DatabasePath, *backupTo); err != nil { + log.Fatalf("backup: %v", err) + } + log.Printf("backup written to %s", *backupTo) + return + } + database, err := db.Open(cfg.DatabasePath) if err != nil { log.Fatalf("database: %v", err) diff --git a/deploy/backup-petal.sh b/deploy/backup-petal.sh new file mode 100755 index 0000000..98c9521 --- /dev/null +++ b/deploy/backup-petal.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Nightly off-VPS backup of Petal's database. +# +# ./backup-petal.sh # snapshot, compress, push off-box, prune +# ./backup-petal.sh --local-only # snapshot + prune, skip the remote push +# +# Run it from cron on the VPS (see deploy/README.md). The snapshot itself goes +# through `petal -backup`, which uses SQLite's VACUUM INTO: one coherent file +# including anything still in the WAL, taken without a write lock, so it is +# safe against the live running app. That is why this script never touches +# petal.db / -wal / -shm directly — copying those three separately can capture +# a torn mid-checkpoint state. +# +# Everything below is overridable from the environment. +set -euo pipefail + +# Stack directory (holds docker-compose.yml and ./data). +STACK_DIR="${STACK_DIR:-$HOME/petal}" +# Where snapshots land on the VPS before being pushed off-box. Inside ./data so +# the container can write it through the existing bind mount. +LOCAL_DIR="${LOCAL_DIR:-$STACK_DIR/data/backups}" +# Off-VPS destination: millenia over headscale. Empty disables the push. +REMOTE_HOST="${REMOTE_HOST:-100.64.0.2}" +REMOTE_USER="${REMOTE_USER:-}" +REMOTE_DIR="${REMOTE_DIR:-petal-backups}" +# Retention, in days, on each side. +KEEP_LOCAL_DAYS="${KEEP_LOCAL_DAYS:-7}" +KEEP_REMOTE_DAYS="${KEEP_REMOTE_DAYS:-30}" + +local_only=0 +[ "${1:-}" = "--local-only" ] && local_only=1 + +stamp="$(date -u +%Y%m%dT%H%M%SZ)" +name="petal-${stamp}.db" + +cd "$STACK_DIR" + +echo ">> snapshotting to data/backups/${name}" +# The container writes to its own /data mount; ./data/backups is the same +# directory seen from the host. +docker compose exec -T petal /app/petal -backup "/data/backups/${name}" + +snapshot="${LOCAL_DIR}/${name}" +[ -s "$snapshot" ] || { echo "snapshot missing or empty: $snapshot" >&2; exit 1; } + +echo ">> compressing" +gzip -9 "$snapshot" +archive="${snapshot}.gz" +echo " $(du -h "$archive" | cut -f1) ${archive}" + +if [ "$local_only" -eq 0 ] && [ -n "$REMOTE_HOST" ]; then + target="${REMOTE_HOST}" + [ -n "$REMOTE_USER" ] && target="${REMOTE_USER}@${REMOTE_HOST}" + + echo ">> pushing to ${target}:${REMOTE_DIR}/" + ssh -o BatchMode=yes "$target" "mkdir -p '${REMOTE_DIR}'" + scp -q -o BatchMode=yes "$archive" "${target}:${REMOTE_DIR}/" + + # Verify by size rather than trusting scp's exit code alone — a truncated + # transfer that still exits 0 would leave a backup that only looks fine. + local_size="$(stat -c%s "$archive")" + remote_size="$(ssh -o BatchMode=yes "$target" "stat -c%s '${REMOTE_DIR}/$(basename "$archive")'")" + if [ "$local_size" != "$remote_size" ]; then + echo "size mismatch after transfer: local ${local_size}, remote ${remote_size}" >&2 + exit 1 + fi + echo " verified ${remote_size} bytes" + + echo ">> pruning remote copies older than ${KEEP_REMOTE_DAYS} days" + ssh -o BatchMode=yes "$target" \ + "find '${REMOTE_DIR}' -name 'petal-*.db.gz' -type f -mtime +${KEEP_REMOTE_DAYS} -delete" +elif [ "$local_only" -eq 1 ]; then + echo ">> --local-only: skipping the remote push" +else + echo ">> REMOTE_HOST is empty: skipping the remote push" >&2 +fi + +echo ">> pruning local copies older than ${KEEP_LOCAL_DAYS} days" +find "$LOCAL_DIR" -name 'petal-*.db.gz' -type f -mtime "+${KEEP_LOCAL_DAYS}" -delete + +echo ">> done" diff --git a/deploy/petal.env.example b/deploy/petal.env.example new file mode 100644 index 0000000..372439f --- /dev/null +++ b/deploy/petal.env.example @@ -0,0 +1,49 @@ +# Petal — production environment for the parodia.dev VPS. +# Copy to the stack directory as `.env` (docker-compose.yml reads it via +# env_file) and fill in the model names. Values the image already fixes +# (PORT, DATABASE_PATH, IMAGE_DIR, TTS_CACHE_DIR, TTS endpoints) are set in +# docker-compose.yml, not here. + +# --- Routing ----------------------------------------------------------------- +# Must match the DNS A record and the Traefik Host() rule. +PETAL_HOST=petal.parodia.dev +# Absolute origin the app knows itself by. Phase 16's OIDC redirect URI is +# built from this, so it has to be the real public HTTPS origin. +BASE_URL=https://petal.parodia.dev + +# The companion's bedtime nag and the night theme read the container clock. +TZ=Europe/Lisbon + +# --- LLM (millenia, over headscale) ------------------------------------------ +# The only cross-VPN dependency. Petal degrades warmly when it's unreachable: +# spell check, gloss, garden, search, export and read-aloud all keep working and +# the status bar shows 小助手在休息 · Petal's helper is resting. +# +# 100.64.0.2 is millenia on the headscale network. vLLM must be bound to that +# interface (NOT 0.0.0.0 — this host is public); see deploy/README.md. +LLM_BACKEND=vllm +LLM_ENDPOINT=http://100.64.0.2:8000 +LLM_MODEL= +LLM_CHAT_MODEL= +# 30s is the local-network default. Over WAN + VPN, with the voice and +# collocation passes sending a whole document, that truncates real work — the +# request is a hard deadline on Complete, and a timeout surfaces as the same +# warm 502 as an unreachable model. 90s leaves headroom without letting a +# genuinely wedged backend hang the pass forever. +LLM_TIMEOUT=90s + +# --- Read-aloud (Piper sidecars) --------------------------------------------- +# Endpoints are wired in docker-compose.yml; these pick the voice each sidecar +# loads. Changing one means recreating that container so it downloads the model. +TTS_VOICE_EN=en_US-amy-medium +TTS_VOICE_ZH=zh_CN-huayan-medium +TTS_AUDIO_FORMAT=mp3 +TTS_TIMEOUT=15s + +# --- Auth (Phase 16 — not wired yet) ----------------------------------------- +# Authentik already runs on this host. Filled in when the OIDC flow lands. +# SESSION_SECRET= +# AUTHENTIK_URL=https://auth.parodia.dev +# AUTHENTIK_CLIENT_ID=petal +# AUTHENTIK_CLIENT_SECRET= +# PETAL_ALLOWED_SUBS= diff --git a/deploy/piper/Dockerfile b/deploy/piper/Dockerfile new file mode 100644 index 0000000..aa25267 --- /dev/null +++ b/deploy/piper/Dockerfile @@ -0,0 +1,37 @@ +# Piper neural-TTS HTTP server — the read-aloud backend Petal proxies to. +# +# One image, any voice: the model is named by PIPER_VOICE at runtime and +# downloaded into the shared /voices volume on first start. Each Piper server +# loads exactly one voice, so a new language is a new service in +# docker-compose.yml, not a new image (English and Chinese today; pt-PT lands +# with the Portuguese pair). +# +# python:3.12 rather than 3.13 — piper-tts pulls onnxruntime, whose wheel +# coverage for 3.13 still lags. +FROM python:3.12-slim + +RUN pip install --no-cache-dir "piper-tts[http]" \ + && useradd -m -u 10002 piper + +ENV PIPER_VOICE=en_US-amy-medium \ + PIPER_DATA_DIR=/voices \ + PIPER_PORT=5000 + +RUN mkdir -p /voices && chown piper:piper /voices +VOLUME ["/voices"] + +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +USER piper +EXPOSE 5000 + +# The server has no dedicated health route, so synthesizing a single word is +# the honest check: it proves the model loaded, not just that a port is open. +HEALTHCHECK --interval=60s --timeout=20s --start-period=180s --retries=3 \ + CMD python -c "import os,urllib.request,json; \ +urllib.request.urlopen(urllib.request.Request('http://127.0.0.1:'+os.environ['PIPER_PORT']+'/', \ +data=json.dumps({'text':'ok','voice':os.environ['PIPER_VOICE']}).encode(), \ +headers={'Content-Type':'application/json'}), timeout=15).read(1)" + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/deploy/piper/entrypoint.sh b/deploy/piper/entrypoint.sh new file mode 100755 index 0000000..dc84ce4 --- /dev/null +++ b/deploy/piper/entrypoint.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Fetch the configured voice if the shared volume doesn't have it yet, then +# serve it. The download is the only step that needs the internet, and it runs +# once per voice for the life of the volume — Petal itself stays offline-first. +set -euo pipefail + +voice="${PIPER_VOICE:?PIPER_VOICE must be set}" +data_dir="${PIPER_DATA_DIR:-/voices}" +port="${PIPER_PORT:-5000}" + +if [ ! -f "${data_dir}/${voice}.onnx" ]; then + echo ">> downloading voice ${voice} into ${data_dir}" + python -m piper.download_voices "${voice}" --data-dir "${data_dir}" +fi + +echo ">> serving ${voice} on :${port}" +# 0.0.0.0 is safe here: the container sits on Petal's internal compose network +# with no published ports, so only Petal can reach it. +exec python -m piper.http_server \ + -m "${voice}" \ + --data-dir "${data_dir}" \ + --host 0.0.0.0 --port "${port}" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c62d83b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,103 @@ +# Petal on the parodia.dev VPS. +# +# docker compose up -d --build +# +# Fronted by the host's existing Traefik (external `traefik` network, the +# `web-secure` entrypoint and the `default` cert resolver — same convention the +# other services on this box use). Petal itself never binds a host port; the +# only way in is through Traefik over HTTPS. +# +# Read-aloud runs as two sibling containers rather than host systemd services: +# each Piper HTTP server loads exactly one voice, the host has no lingering +# user session to keep systemd units alive, and keeping them on the internal +# network means the TTS ports are unreachable from anywhere but Petal. +# +# Copy deploy/petal.env.example to .env before the first `up`. + +name: petal + +services: + petal: + build: + context: . + dockerfile: Dockerfile + image: petal:local + container_name: petal + restart: unless-stopped + env_file: .env + environment: + # Fixed by the image layout; kept here so they're visible at a glance. + PORT: "8080" + DATABASE_PATH: /data/petal.db + IMAGE_DIR: /data/images + TTS_CACHE_DIR: /data/tts + # Piper sidecars. Each server loads one voice, so English and Chinese are + # separate containers; the handler maps language → instance from config. + TTS_ENDPOINT: http://piper-en:5000 + TTS_ENDPOINT_ZH: http://piper-zh:5000 + # The companion's bedtime nag and night mode read the local clock. + TZ: ${TZ:-Europe/Lisbon} + volumes: + # A bind mount, not a named volume: petal.db must be trivially reachable + # from the host for the nightly backup and for a restore. + - ./data:/data + networks: + - traefik + - internal + depends_on: + - piper-en + - piper-zh + labels: + traefik.enable: "true" + traefik.docker.network: traefik + traefik.http.routers.petal.rule: Host(`${PETAL_HOST:-petal.parodia.dev}`) + traefik.http.routers.petal.entrypoints: web-secure + traefik.http.routers.petal.tls: "true" + traefik.http.routers.petal.tls.certResolver: default + traefik.http.routers.petal.service: petal + traefik.http.routers.petal.middlewares: compression@file,petal-headers + traefik.http.services.petal.loadbalancer.server.port: "8080" + # Petal is a private writing space: no framing, no sniffing, HSTS on. + traefik.http.middlewares.petal-headers.headers.customresponseheaders.Content-Security-Policy: frame-ancestors 'self' + traefik.http.middlewares.petal-headers.headers.customresponseheaders.Strict-Transport-Security: max-age=31536000; includeSubDomains + traefik.http.middlewares.petal-headers.headers.customresponseheaders.X-Content-Type-Options: nosniff + traefik.http.middlewares.petal-headers.headers.customresponseheaders.Referrer-Policy: same-origin + + piper-en: + build: + context: deploy/piper + image: petal-piper:local + container_name: petal-piper-en + restart: unless-stopped + environment: + PIPER_VOICE: ${TTS_VOICE_EN:-en_US-amy-medium} + volumes: + - piper-voices:/voices + networks: + - internal + + piper-zh: + build: + context: deploy/piper + image: petal-piper:local + container_name: petal-piper-zh + restart: unless-stopped + environment: + PIPER_VOICE: ${TTS_VOICE_ZH:-zh_CN-huayan-medium} + volumes: + - piper-voices:/voices + networks: + - internal + +networks: + # Created and owned by the host's Traefik stack. + traefik: + external: true + # Petal ↔ Piper only. Not reachable from the internet or the other stacks. + internal: + driver: bridge + +volumes: + # Downloaded voice models, shared read-mostly by both Piper instances so the + # same model is never fetched twice. + piper-voices: diff --git a/internal/db/backup.go b/internal/db/backup.go new file mode 100644 index 0000000..bd77d66 --- /dev/null +++ b/internal/db/backup.go @@ -0,0 +1,79 @@ +package db + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" +) + +// Backup writes a consistent copy of the database at srcPath to destPath using +// SQLite's `VACUUM INTO`. +// +// Why not copy the file: Petal runs in WAL mode, so at any instant the newest +// committed pages may live in petal.db-wal rather than petal.db. Copying the +// three files separately can capture them mid-checkpoint and produce a backup +// that is subtly torn. `VACUUM INTO` runs inside a read transaction, so it sees +// one coherent snapshot including the WAL, and emits a single defragmented file +// with no -wal/-shm companions — exactly what you want to ship off-box. +// +// It takes no write lock, so this is safe to run against the live database +// while someone is writing. +// +// destPath must not already exist: SQLite refuses to overwrite, which keeps a +// failed run from destroying the previous good backup. +func Backup(srcPath, destPath string) error { + if _, err := os.Stat(srcPath); err != nil { + return fmt.Errorf("source database: %w", err) + } + if _, err := os.Stat(destPath); err == nil { + return fmt.Errorf("destination %s already exists", destPath) + } + if dir := filepath.Dir(destPath); dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("create backup dir: %w", err) + } + } + + // Opened directly rather than through Open: a backup must never migrate or + // seed the database it is copying. + conn, err := sql.Open("sqlite", dsn(srcPath)) + if err != nil { + return fmt.Errorf("open source: %w", err) + } + defer conn.Close() + conn.SetMaxOpenConns(1) + + if err := conn.Ping(); err != nil { + return fmt.Errorf("ping source: %w", err) + } + + // The path is interpolated because VACUUM INTO takes a literal, not a bound + // parameter. Quotes are doubled so a path containing one can't break out. + quoted := "'" + escapeSQLiteString(destPath) + "'" + if _, err := conn.Exec("VACUUM INTO " + quoted); err != nil { + return fmt.Errorf("vacuum into %s: %w", destPath, err) + } + + // A zero-byte result would mean the vacuum silently produced nothing; catch + // it here rather than discovering it during a restore. + info, err := os.Stat(destPath) + if err != nil { + return fmt.Errorf("stat backup: %w", err) + } + if info.Size() == 0 { + return fmt.Errorf("backup %s is empty", destPath) + } + return nil +} + +func escapeSQLiteString(s string) string { + out := make([]byte, 0, len(s)) + for i := 0; i < len(s); i++ { + if s[i] == '\'' { + out = append(out, '\'') + } + out = append(out, s[i]) + } + return string(out) +} diff --git a/internal/db/backup_test.go b/internal/db/backup_test.go new file mode 100644 index 0000000..894e204 --- /dev/null +++ b/internal/db/backup_test.go @@ -0,0 +1,108 @@ +package db + +import ( + "database/sql" + "os" + "path/filepath" + "testing" +) + +// The point of VACUUM INTO over a file copy is that it captures rows still +// sitting in the WAL. This writes with the source connection open (so the WAL +// is hot and unlikely to have been checkpointed) and asserts the backup has +// them. +func TestBackupCapturesLiveWrites(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "petal.db") + dest := filepath.Join(dir, "backups", "petal-backup.db") + + d, err := Open(src) + if err != nil { + t.Fatalf("open source: %v", err) + } + defer d.Close() + + if _, err := d.Exec( + `INSERT INTO documents (id, user_id, title, content_text) VALUES ('d1', ?, '春天', 'hello 春天')`, + LocalUserID, + ); err != nil { + t.Fatalf("insert: %v", err) + } + + if err := Backup(src, dest); err != nil { + t.Fatalf("backup: %v", err) + } + + // VACUUM INTO emits a single self-contained file — no -wal/-shm to ship + // alongside it. + for _, suffix := range []string{"-wal", "-shm"} { + if _, err := os.Stat(dest + suffix); err == nil { + t.Errorf("backup left a %s companion file behind", suffix) + } + } + + copyConn, err := sql.Open("sqlite", dsn(dest)) + if err != nil { + t.Fatalf("open backup: %v", err) + } + defer copyConn.Close() + + var title string + if err := copyConn.QueryRow(`SELECT title FROM documents WHERE id = 'd1'`).Scan(&title); err != nil { + t.Fatalf("row missing from backup: %v", err) + } + if title != "春天" { + t.Errorf("title = %q, want 春天", title) + } + + // The seeded user has to come across too, or a restore would orphan every + // document's foreign key. + var users int + if err := copyConn.QueryRow(`SELECT COUNT(*) FROM users WHERE id = ?`, LocalUserID).Scan(&users); err != nil { + t.Fatalf("count users: %v", err) + } + if users != 1 { + t.Errorf("users in backup = %d, want 1", users) + } +} + +// A second run to the same path must fail loudly rather than clobber or +// half-write the previous good backup. +func TestBackupRefusesExistingDestination(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "petal.db") + dest := filepath.Join(dir, "petal-backup.db") + + d, err := Open(src) + if err != nil { + t.Fatalf("open source: %v", err) + } + defer d.Close() + + if err := Backup(src, dest); err != nil { + t.Fatalf("first backup: %v", err) + } + before, err := os.ReadFile(dest) + if err != nil { + t.Fatalf("read backup: %v", err) + } + + if err := Backup(src, dest); err == nil { + t.Fatal("second backup to the same path succeeded; want an error") + } + + after, err := os.ReadFile(dest) + if err != nil { + t.Fatalf("re-read backup: %v", err) + } + if len(before) != len(after) { + t.Errorf("existing backup was modified: %d bytes → %d", len(before), len(after)) + } +} + +func TestBackupMissingSource(t *testing.T) { + dir := t.TempDir() + if err := Backup(filepath.Join(dir, "nope.db"), filepath.Join(dir, "out.db")); err == nil { + t.Fatal("backup of a nonexistent database succeeded; want an error") + } +}