From e65ffb13732d3d2cf07cbe20f05a7ab1d9eda534 Mon Sep 17 00:00:00 2001
From: prosolis <5590409+prosolis@users.noreply.github.com>
Date: Mon, 22 Jun 2026 01:54:34 -0700
Subject: [PATCH] Add server-side web usage metrics with !petestats command
Track per-page/per-channel view counts and a privacy-preserving daily
unique-visitor estimate (salted IP+UA hash, salt rotated daily and never
persisted). No third-party analytics, no JS beacon. Surfaced via the
admin-gated !petestats Matrix command (named to avoid an existing !stats
bot in the rooms).
---
internal/storage/db.go | 5 ++
internal/storage/metrics.go | 114 +++++++++++++++++++++++++++++++
internal/storage/metrics_test.go | 42 ++++++++++++
internal/storage/schema.go | 22 ++++++
internal/web/handlers.go | 2 +
internal/web/metrics.go | 93 +++++++++++++++++++++++++
internal/web/server.go | 7 ++
main.go | 85 ++++++++++++++++++++++-
8 files changed, 367 insertions(+), 3 deletions(-)
create mode 100644 internal/storage/metrics.go
create mode 100644 internal/storage/metrics_test.go
create mode 100644 internal/web/metrics.go
diff --git a/internal/storage/db.go b/internal/storage/db.go
index 2e71039..5040344 100644
--- a/internal/storage/db.go
+++ b/internal/storage/db.go
@@ -116,6 +116,11 @@ func RunMaintenance() {
exec("prune old reactions",
`DELETE FROM reactions WHERE reacted_at < ?`, storyCutoff)
+ // Daily unique tokens are only useful for the recent window; their salts are
+ // long gone. page_views is kept forever (tiny aggregate, all-time totals).
+ exec("prune old daily_visitors",
+ `DELETE FROM daily_visitors WHERE day < ?`, unixDay()-30)
+
exec("wal checkpoint", "PRAGMA wal_checkpoint(TRUNCATE)")
exec("optimize", "PRAGMA optimize")
}
diff --git a/internal/storage/metrics.go b/internal/storage/metrics.go
new file mode 100644
index 0000000..abcc355
--- /dev/null
+++ b/internal/storage/metrics.go
@@ -0,0 +1,114 @@
+package storage
+
+import (
+ "log/slog"
+ "sort"
+)
+
+const secondsPerDay = 86400
+
+// unixDay returns the current UTC day number (floor(unix / 86400)).
+func unixDay() int64 { return nowUnix() / secondsPerDay }
+
+// RecordPageView increments the all-time and per-day view counter for a coarse
+// path label ("home", a channel slug, …). Fire-and-forget: a failure here must
+// never affect serving a page, so errors are logged and swallowed.
+func RecordPageView(path string) {
+ exec("record page view",
+ `INSERT INTO page_views (path, day, views) VALUES (?, ?, 1)
+ ON CONFLICT(path, day) DO UPDATE SET views = views + 1`,
+ path, unixDay())
+}
+
+// RecordVisitor records a (already-hashed, salted) visitor token for today's
+// unique estimate. Duplicate tokens within the same day are ignored.
+func RecordVisitor(token string) {
+ exec("record visitor",
+ `INSERT OR IGNORE INTO daily_visitors (day, visitor) VALUES (?, ?)`,
+ unixDay(), token)
+}
+
+// PathStat is one row of the per-page usage breakdown.
+type PathStat struct {
+ Path string
+ Total int // all-time views
+ Today int // views since the start of the current UTC day
+}
+
+// DayStat is one day's unique-visitor estimate.
+type DayStat struct {
+ Day int64 // unix day number
+ Uniques int
+}
+
+// MetricsSummary is the aggregate usage readout surfaced by !petestats.
+type MetricsSummary struct {
+ Pages []PathStat // per-path, busiest first
+ TotalViews int // all-time, all paths
+ ViewsToday int // all paths, current UTC day
+ UniquesToday int // distinct visitor tokens, current UTC day
+ Last7Days []DayStat // daily uniques, oldest → newest (note: salt rotates
+ // daily, so these CANNOT be summed into a cross-day unique count)
+}
+
+// GetMetricsSummary assembles the usage readout. Best-effort: any sub-query
+// failure leaves that field at its zero value rather than failing the whole
+// command.
+func GetMetricsSummary() MetricsSummary {
+ today := unixDay()
+ var m MetricsSummary
+
+ rows, err := Get().Query(
+ `SELECT path,
+ SUM(views) AS total,
+ COALESCE(SUM(CASE WHEN day = ? THEN views END), 0) AS today
+ FROM page_views
+ GROUP BY path`, today)
+ if err != nil {
+ slog.Error("metrics: page_views query failed", "err", err)
+ } else {
+ defer rows.Close()
+ for rows.Next() {
+ var p PathStat
+ if err := rows.Scan(&p.Path, &p.Total, &p.Today); err != nil {
+ slog.Error("metrics: scan page stat failed", "err", err)
+ continue
+ }
+ m.Pages = append(m.Pages, p)
+ m.TotalViews += p.Total
+ m.ViewsToday += p.Today
+ }
+ }
+ sort.Slice(m.Pages, func(i, j int) bool {
+ if m.Pages[i].Total != m.Pages[j].Total {
+ return m.Pages[i].Total > m.Pages[j].Total
+ }
+ return m.Pages[i].Path < m.Pages[j].Path
+ })
+
+ if err := Get().QueryRow(
+ `SELECT COUNT(*) FROM daily_visitors WHERE day = ?`, today,
+ ).Scan(&m.UniquesToday); err != nil {
+ slog.Error("metrics: uniques-today query failed", "err", err)
+ }
+
+ since := today - 6 // inclusive 7-day window
+ vrows, err := Get().Query(
+ `SELECT day, COUNT(*) FROM daily_visitors
+ WHERE day >= ? GROUP BY day ORDER BY day`, since)
+ if err != nil {
+ slog.Error("metrics: 7-day uniques query failed", "err", err)
+ } else {
+ defer vrows.Close()
+ for vrows.Next() {
+ var d DayStat
+ if err := vrows.Scan(&d.Day, &d.Uniques); err != nil {
+ slog.Error("metrics: scan day stat failed", "err", err)
+ continue
+ }
+ m.Last7Days = append(m.Last7Days, d)
+ }
+ }
+
+ return m
+}
diff --git a/internal/storage/metrics_test.go b/internal/storage/metrics_test.go
new file mode 100644
index 0000000..7ddca91
--- /dev/null
+++ b/internal/storage/metrics_test.go
@@ -0,0 +1,42 @@
+package storage
+
+import "testing"
+
+func TestMetrics_ViewsAndUniques(t *testing.T) {
+ setupTestDB(t)
+
+ // Three views on home, two on gaming.
+ RecordPageView("home")
+ RecordPageView("home")
+ RecordPageView("home")
+ RecordPageView("gaming")
+ RecordPageView("gaming")
+
+ // Two distinct visitors today; the repeat token must not double-count.
+ RecordVisitor("tok-a")
+ RecordVisitor("tok-a")
+ RecordVisitor("tok-b")
+
+ m := GetMetricsSummary()
+
+ if m.TotalViews != 5 {
+ t.Errorf("TotalViews = %d, want 5", m.TotalViews)
+ }
+ if m.ViewsToday != 5 {
+ t.Errorf("ViewsToday = %d, want 5", m.ViewsToday)
+ }
+ if m.UniquesToday != 2 {
+ t.Errorf("UniquesToday = %d, want 2 (deduped)", m.UniquesToday)
+ }
+ if len(m.Pages) == 0 || m.Pages[0].Path != "home" || m.Pages[0].Total != 3 {
+ t.Errorf("Pages[0] = %+v, want home with 3 views first", m.Pages)
+ }
+}
+
+func TestMetrics_EmptyIsZero(t *testing.T) {
+ setupTestDB(t)
+ m := GetMetricsSummary()
+ if m.TotalViews != 0 || m.ViewsToday != 0 || m.UniquesToday != 0 || len(m.Pages) != 0 {
+ t.Errorf("expected zero-value summary, got %+v", m)
+ }
+}
diff --git a/internal/storage/schema.go b/internal/storage/schema.go
index 6a58aa3..5c5463f 100644
--- a/internal/storage/schema.go
+++ b/internal/storage/schema.go
@@ -53,6 +53,26 @@ CREATE TABLE IF NOT EXISTS user_preferences (
updated_at INTEGER NOT NULL
);
+-- Aggregate web usage. page_views holds running view counts keyed by a coarse
+-- path label ("home", channel slug, …) and the UTC day, so we can report both
+-- all-time totals and per-day breakdowns without storing any per-request rows.
+CREATE TABLE IF NOT EXISTS page_views (
+ path TEXT NOT NULL,
+ day INTEGER NOT NULL, -- unix day (floor(unix / 86400)), UTC
+ views INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (path, day)
+);
+
+-- Privacy-preserving daily unique estimate. visitor is a salted hash of
+-- IP+User-Agent; the salt rotates every UTC day and is never persisted, so the
+-- hashes are irreversible and cannot be linked across days. We keep only enough
+-- to dedup within a single day, then prune.
+CREATE TABLE IF NOT EXISTS daily_visitors (
+ day INTEGER NOT NULL,
+ visitor TEXT NOT NULL,
+ PRIMARY KEY (day, visitor)
+);
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_post_log_guid_channel ON post_log(guid, channel);
CREATE INDEX IF NOT EXISTS idx_post_log_event_id ON post_log(event_id);
CREATE INDEX IF NOT EXISTS idx_post_log_channel_posted ON post_log(channel, posted_at);
@@ -65,6 +85,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_url_canonical ON stories(url_canon
CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_source_headline_norm ON stories(source, headline_norm) WHERE headline_norm IS NOT NULL AND headline_norm <> '';
CREATE INDEX IF NOT EXISTS idx_reactions_post_guid ON reactions(post_guid);
CREATE UNIQUE INDEX IF NOT EXISTS idx_reactions_event_id ON reactions(event_id);
+CREATE INDEX IF NOT EXISTS idx_page_views_day ON page_views(day);
+CREATE INDEX IF NOT EXISTS idx_daily_visitors_day ON daily_visitors(day);
`
const ftsSchema = `
diff --git a/internal/web/handlers.go b/internal/web/handlers.go
index d54a9a8..e111369 100644
--- a/internal/web/handlers.go
+++ b/internal/web/handlers.go
@@ -123,6 +123,7 @@ func (s *Server) base(r *http.Request) pageData {
}
func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) {
+ s.track(r, "home")
const (
justPostedLimit = 6
latestLimit = 16
@@ -177,6 +178,7 @@ func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) {
}
func (s *Server) handleChannel(w http.ResponseWriter, r *http.Request, ch Channel) {
+ s.track(r, ch.Slug)
page := 1
if p := r.URL.Query().Get("page"); p != "" {
if n, err := strconv.Atoi(p); err == nil && n > 0 {
diff --git a/internal/web/metrics.go b/internal/web/metrics.go
new file mode 100644
index 0000000..8177b6a
--- /dev/null
+++ b/internal/web/metrics.go
@@ -0,0 +1,93 @@
+package web
+
+import (
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/hex"
+ "log/slog"
+ "net"
+ "net/http"
+ "strings"
+ "time"
+
+ "pete/internal/storage"
+)
+
+// track records one page view plus a privacy-preserving unique-visitor token
+// for the given coarse label ("home", a channel slug, …). It is deliberately
+// cheap and best-effort: the visitor token is derived synchronously (so it sees
+// the live request) but the DB writes are fired in the background so page
+// rendering never waits on them.
+func (s *Server) track(r *http.Request, label string) {
+ token := s.visitorToken(r)
+ go func() {
+ storage.RecordPageView(label)
+ if token != "" {
+ storage.RecordVisitor(token)
+ }
+ }()
+}
+
+// visitorToken returns an irreversible per-day token for the requester: a
+// SHA-256 of (daily salt || client IP || User-Agent), truncated. Because the
+// salt rotates each UTC day and is never stored, tokens cannot be reversed back
+// to an IP or linked across days. Returns "" if no client IP can be determined.
+func (s *Server) visitorToken(r *http.Request) string {
+ ip := clientIP(r)
+ if ip == "" {
+ return ""
+ }
+ salt := s.dailySalt()
+ h := sha256.New()
+ h.Write(salt[:])
+ h.Write([]byte(ip))
+ h.Write([]byte{0})
+ h.Write([]byte(r.UserAgent()))
+ return hex.EncodeToString(h.Sum(nil)[:16])
+}
+
+// dailySalt returns the current day's salt, generating a fresh random one
+// whenever the UTC day rolls over. The previous salt is discarded, which is
+// what makes cross-day correlation impossible.
+func (s *Server) dailySalt() [16]byte {
+ day := time.Now().UTC().Unix() / 86400
+ s.metricsMu.Lock()
+ defer s.metricsMu.Unlock()
+ if day != s.saltDay || isZero(s.salt) {
+ if _, err := rand.Read(s.salt[:]); err != nil {
+ // crypto/rand should never fail; if it does, fall back to a
+ // day-derived constant so we still dedup within the day.
+ slog.Error("web: salt generation failed; using weak fallback", "err", err)
+ for i := range s.salt {
+ s.salt[i] = byte(day >> (uint(i%8) * 8))
+ }
+ }
+ s.saltDay = day
+ }
+ return s.salt
+}
+
+func isZero(b [16]byte) bool {
+ for _, x := range b {
+ if x != 0 {
+ return false
+ }
+ }
+ return true
+}
+
+// clientIP extracts the requester's IP, honoring the leftmost X-Forwarded-For
+// entry when present (Pete runs behind a reverse proxy) and otherwise falling
+// back to RemoteAddr. Only used to derive a one-way hash, never stored.
+func clientIP(r *http.Request) string {
+ if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
+ if first := strings.TrimSpace(strings.Split(xff, ",")[0]); first != "" {
+ return first
+ }
+ }
+ host, _, err := net.SplitHostPort(r.RemoteAddr)
+ if err != nil {
+ return r.RemoteAddr
+ }
+ return host
+}
diff --git a/internal/web/server.go b/internal/web/server.go
index 0ab7e3d..4439daf 100644
--- a/internal/web/server.go
+++ b/internal/web/server.go
@@ -9,6 +9,7 @@ import (
"io/fs"
"log/slog"
"net/http"
+ "sync"
"time"
"pete/internal/config"
@@ -56,6 +57,12 @@ type Server struct {
srv *http.Server
tpls map[string]*template.Template // keyed by page name (e.g. "index", "channel")
auth *Authenticator // nil when sign-in is disabled or unavailable
+
+ // Daily-rotated salt for the privacy-preserving unique-visitor estimate.
+ // Guarded by metricsMu; never persisted (see metrics.go).
+ metricsMu sync.Mutex
+ saltDay int64
+ salt [16]byte
}
// New builds the server. Templates are parsed once at startup. Each page
diff --git a/main.go b/main.go
index ce30e3f..f174ead 100644
--- a/main.go
+++ b/main.go
@@ -3,9 +3,12 @@ package main
import (
"context"
"flag"
+ "fmt"
+ "html/template"
"log/slog"
"os"
"os/signal"
+ "strconv"
"strings"
"syscall"
"time"
@@ -98,15 +101,28 @@ func main() {
}
}
- // Wire !post command: force-publish next queued story for the room's channel.
+ // Wire in-room commands. All are gated to the admin allowlist; the room
+ // rooms are public so anything not from an admin is silently ignored.
mx.SetMessageHandler(func(channel string, roomID id.RoomID, eventID id.EventID, sender id.UserID, body string) {
- if strings.TrimSpace(body) != "!post" {
+ cmd := strings.TrimSpace(body)
+ if cmd != "!post" && cmd != "!petestats" {
return
}
if !admins[sender] {
- slog.Warn("!post ignored: sender not in matrix.admins allowlist", "channel", channel, "sender", sender)
+ slog.Warn("command ignored: sender not in matrix.admins allowlist", "cmd", cmd, "channel", channel, "sender", sender)
return
}
+
+ if cmd == "!petestats" {
+ slog.Info("!petestats requested", "channel", channel, "sender", sender)
+ plain, htmlBody := formatStats(storage.GetMetricsSummary())
+ if err := mx.PostThreadedReply(channel, eventID, plain, htmlBody); err != nil {
+ slog.Warn("!petestats: failed to send reply", "err", err)
+ }
+ return
+ }
+
+ // !post: force-publish the next queued story for the room's channel.
slog.Info("!post requested", "channel", channel, "sender", sender)
if queue.ForcePost(channel) {
return
@@ -236,6 +252,69 @@ func main() {
slog.Info("pete stopped")
}
+// formatStats renders a usage summary for the !petestats reply, returning a
+// plain-text and an HTML body. Daily uniques are reported per-day only: the
+// privacy salt rotates each day, so they cannot be honestly summed across days.
+func formatStats(m storage.MetricsSummary) (plain, htmlBody string) {
+ const topN = 8
+
+ var p, h strings.Builder
+ p.WriteString("📊 Pete usage\n")
+ h.WriteString("📊 Pete usage
")
+
+ fmt.Fprintf(&p, "Today: %s views · ~%s unique visitors\n",
+ comma(m.ViewsToday), comma(m.UniquesToday))
+ fmt.Fprintf(&h, "Today: %s views · ~%s unique visitors
",
+ comma(m.ViewsToday), comma(m.UniquesToday))
+
+ fmt.Fprintf(&p, "All time: %s views\n", comma(m.TotalViews))
+ fmt.Fprintf(&h, "All time: %s views
", comma(m.TotalViews))
+
+ if len(m.Pages) > 0 {
+ pages := m.Pages
+ if len(pages) > topN {
+ pages = pages[:topN]
+ }
+ parts := make([]string, 0, len(pages))
+ for _, pg := range pages {
+ parts = append(parts, fmt.Sprintf("%s %s", pg.Path, comma(pg.Total)))
+ }
+ fmt.Fprintf(&p, "Top pages (all time): %s\n", strings.Join(parts, " · "))
+ fmt.Fprintf(&h, "Top pages (all time): %s
",
+ template.HTMLEscapeString(strings.Join(parts, " · ")))
+ }
+
+ if len(m.Last7Days) > 0 {
+ parts := make([]string, 0, len(m.Last7Days))
+ for _, d := range m.Last7Days {
+ label := time.Unix(d.Day*86400, 0).UTC().Format("Mon")
+ parts = append(parts, fmt.Sprintf("%s %s", label, comma(d.Uniques)))
+ }
+ fmt.Fprintf(&p, "Daily uniques (7d): %s", strings.Join(parts, " · "))
+ fmt.Fprintf(&h, "Daily uniques (7d): %s",
+ template.HTMLEscapeString(strings.Join(parts, " · ")))
+ }
+
+ return p.String(), h.String()
+}
+
+// comma formats a non-negative int with thousands separators (e.g. 12503 →
+// "12,503").
+func comma(n int) string {
+ s := strconv.Itoa(n)
+ if n < 0 {
+ return s // not expected for counts; leave untouched
+ }
+ var b strings.Builder
+ for i, c := range s {
+ if i > 0 && (len(s)-i)%3 == 0 {
+ b.WriteByte(',')
+ }
+ b.WriteRune(c)
+ }
+ return b.String()
+}
+
func runLocal(cfg *config.Config) {
cfg.Web.Enabled = true