Add server-side web usage metrics with !petestats command

Track per-page/per-channel view counts and a privacy-preserving daily
unique-visitor estimate (salted IP+UA hash, salt rotated daily and never
persisted). No third-party analytics, no JS beacon. Surfaced via the
admin-gated !petestats Matrix command (named to avoid an existing !stats
bot in the rooms).
This commit is contained in:
prosolis
2026-06-22 01:54:34 -07:00
parent aaf6e551a0
commit e65ffb1373
8 changed files with 367 additions and 3 deletions

View File

@@ -116,6 +116,11 @@ func RunMaintenance() {
exec("prune old reactions",
`DELETE FROM reactions WHERE reacted_at < ?`, storyCutoff)
// Daily unique tokens are only useful for the recent window; their salts are
// long gone. page_views is kept forever (tiny aggregate, all-time totals).
exec("prune old daily_visitors",
`DELETE FROM daily_visitors WHERE day < ?`, unixDay()-30)
exec("wal checkpoint", "PRAGMA wal_checkpoint(TRUNCATE)")
exec("optimize", "PRAGMA optimize")
}

114
internal/storage/metrics.go Normal file
View File

@@ -0,0 +1,114 @@
package storage
import (
"log/slog"
"sort"
)
const secondsPerDay = 86400
// unixDay returns the current UTC day number (floor(unix / 86400)).
func unixDay() int64 { return nowUnix() / secondsPerDay }
// RecordPageView increments the all-time and per-day view counter for a coarse
// path label ("home", a channel slug, …). Fire-and-forget: a failure here must
// never affect serving a page, so errors are logged and swallowed.
func RecordPageView(path string) {
exec("record page view",
`INSERT INTO page_views (path, day, views) VALUES (?, ?, 1)
ON CONFLICT(path, day) DO UPDATE SET views = views + 1`,
path, unixDay())
}
// RecordVisitor records a (already-hashed, salted) visitor token for today's
// unique estimate. Duplicate tokens within the same day are ignored.
func RecordVisitor(token string) {
exec("record visitor",
`INSERT OR IGNORE INTO daily_visitors (day, visitor) VALUES (?, ?)`,
unixDay(), token)
}
// PathStat is one row of the per-page usage breakdown.
type PathStat struct {
Path string
Total int // all-time views
Today int // views since the start of the current UTC day
}
// DayStat is one day's unique-visitor estimate.
type DayStat struct {
Day int64 // unix day number
Uniques int
}
// MetricsSummary is the aggregate usage readout surfaced by !petestats.
type MetricsSummary struct {
Pages []PathStat // per-path, busiest first
TotalViews int // all-time, all paths
ViewsToday int // all paths, current UTC day
UniquesToday int // distinct visitor tokens, current UTC day
Last7Days []DayStat // daily uniques, oldest → newest (note: salt rotates
// daily, so these CANNOT be summed into a cross-day unique count)
}
// GetMetricsSummary assembles the usage readout. Best-effort: any sub-query
// failure leaves that field at its zero value rather than failing the whole
// command.
func GetMetricsSummary() MetricsSummary {
today := unixDay()
var m MetricsSummary
rows, err := Get().Query(
`SELECT path,
SUM(views) AS total,
COALESCE(SUM(CASE WHEN day = ? THEN views END), 0) AS today
FROM page_views
GROUP BY path`, today)
if err != nil {
slog.Error("metrics: page_views query failed", "err", err)
} else {
defer rows.Close()
for rows.Next() {
var p PathStat
if err := rows.Scan(&p.Path, &p.Total, &p.Today); err != nil {
slog.Error("metrics: scan page stat failed", "err", err)
continue
}
m.Pages = append(m.Pages, p)
m.TotalViews += p.Total
m.ViewsToday += p.Today
}
}
sort.Slice(m.Pages, func(i, j int) bool {
if m.Pages[i].Total != m.Pages[j].Total {
return m.Pages[i].Total > m.Pages[j].Total
}
return m.Pages[i].Path < m.Pages[j].Path
})
if err := Get().QueryRow(
`SELECT COUNT(*) FROM daily_visitors WHERE day = ?`, today,
).Scan(&m.UniquesToday); err != nil {
slog.Error("metrics: uniques-today query failed", "err", err)
}
since := today - 6 // inclusive 7-day window
vrows, err := Get().Query(
`SELECT day, COUNT(*) FROM daily_visitors
WHERE day >= ? GROUP BY day ORDER BY day`, since)
if err != nil {
slog.Error("metrics: 7-day uniques query failed", "err", err)
} else {
defer vrows.Close()
for vrows.Next() {
var d DayStat
if err := vrows.Scan(&d.Day, &d.Uniques); err != nil {
slog.Error("metrics: scan day stat failed", "err", err)
continue
}
m.Last7Days = append(m.Last7Days, d)
}
}
return m
}

View File

@@ -0,0 +1,42 @@
package storage
import "testing"
func TestMetrics_ViewsAndUniques(t *testing.T) {
setupTestDB(t)
// Three views on home, two on gaming.
RecordPageView("home")
RecordPageView("home")
RecordPageView("home")
RecordPageView("gaming")
RecordPageView("gaming")
// Two distinct visitors today; the repeat token must not double-count.
RecordVisitor("tok-a")
RecordVisitor("tok-a")
RecordVisitor("tok-b")
m := GetMetricsSummary()
if m.TotalViews != 5 {
t.Errorf("TotalViews = %d, want 5", m.TotalViews)
}
if m.ViewsToday != 5 {
t.Errorf("ViewsToday = %d, want 5", m.ViewsToday)
}
if m.UniquesToday != 2 {
t.Errorf("UniquesToday = %d, want 2 (deduped)", m.UniquesToday)
}
if len(m.Pages) == 0 || m.Pages[0].Path != "home" || m.Pages[0].Total != 3 {
t.Errorf("Pages[0] = %+v, want home with 3 views first", m.Pages)
}
}
func TestMetrics_EmptyIsZero(t *testing.T) {
setupTestDB(t)
m := GetMetricsSummary()
if m.TotalViews != 0 || m.ViewsToday != 0 || m.UniquesToday != 0 || len(m.Pages) != 0 {
t.Errorf("expected zero-value summary, got %+v", m)
}
}

View File

@@ -53,6 +53,26 @@ CREATE TABLE IF NOT EXISTS user_preferences (
updated_at INTEGER NOT NULL
);
-- Aggregate web usage. page_views holds running view counts keyed by a coarse
-- path label ("home", channel slug, …) and the UTC day, so we can report both
-- all-time totals and per-day breakdowns without storing any per-request rows.
CREATE TABLE IF NOT EXISTS page_views (
path TEXT NOT NULL,
day INTEGER NOT NULL, -- unix day (floor(unix / 86400)), UTC
views INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (path, day)
);
-- Privacy-preserving daily unique estimate. visitor is a salted hash of
-- IP+User-Agent; the salt rotates every UTC day and is never persisted, so the
-- hashes are irreversible and cannot be linked across days. We keep only enough
-- to dedup within a single day, then prune.
CREATE TABLE IF NOT EXISTS daily_visitors (
day INTEGER NOT NULL,
visitor TEXT NOT NULL,
PRIMARY KEY (day, visitor)
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_post_log_guid_channel ON post_log(guid, channel);
CREATE INDEX IF NOT EXISTS idx_post_log_event_id ON post_log(event_id);
CREATE INDEX IF NOT EXISTS idx_post_log_channel_posted ON post_log(channel, posted_at);
@@ -65,6 +85,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_url_canonical ON stories(url_canon
CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_source_headline_norm ON stories(source, headline_norm) WHERE headline_norm IS NOT NULL AND headline_norm <> '';
CREATE INDEX IF NOT EXISTS idx_reactions_post_guid ON reactions(post_guid);
CREATE UNIQUE INDEX IF NOT EXISTS idx_reactions_event_id ON reactions(event_id);
CREATE INDEX IF NOT EXISTS idx_page_views_day ON page_views(day);
CREATE INDEX IF NOT EXISTS idx_daily_visitors_day ON daily_visitors(day);
`
const ftsSchema = `