Add server-side web usage metrics with !petestats command
Track per-page/per-channel view counts and a privacy-preserving daily unique-visitor estimate (salted IP+UA hash, salt rotated daily and never persisted). No third-party analytics, no JS beacon. Surfaced via the admin-gated !petestats Matrix command (named to avoid an existing !stats bot in the rooms).
This commit is contained in:
@@ -116,6 +116,11 @@ func RunMaintenance() {
|
||||
exec("prune old reactions",
|
||||
`DELETE FROM reactions WHERE reacted_at < ?`, storyCutoff)
|
||||
|
||||
// Daily unique tokens are only useful for the recent window; their salts are
|
||||
// long gone. page_views is kept forever (tiny aggregate, all-time totals).
|
||||
exec("prune old daily_visitors",
|
||||
`DELETE FROM daily_visitors WHERE day < ?`, unixDay()-30)
|
||||
|
||||
exec("wal checkpoint", "PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
exec("optimize", "PRAGMA optimize")
|
||||
}
|
||||
|
||||
114
internal/storage/metrics.go
Normal file
114
internal/storage/metrics.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"sort"
|
||||
)
|
||||
|
||||
const secondsPerDay = 86400
|
||||
|
||||
// unixDay returns the current UTC day number (floor(unix / 86400)).
|
||||
func unixDay() int64 { return nowUnix() / secondsPerDay }
|
||||
|
||||
// RecordPageView increments the all-time and per-day view counter for a coarse
|
||||
// path label ("home", a channel slug, …). Fire-and-forget: a failure here must
|
||||
// never affect serving a page, so errors are logged and swallowed.
|
||||
func RecordPageView(path string) {
|
||||
exec("record page view",
|
||||
`INSERT INTO page_views (path, day, views) VALUES (?, ?, 1)
|
||||
ON CONFLICT(path, day) DO UPDATE SET views = views + 1`,
|
||||
path, unixDay())
|
||||
}
|
||||
|
||||
// RecordVisitor records a (already-hashed, salted) visitor token for today's
|
||||
// unique estimate. Duplicate tokens within the same day are ignored.
|
||||
func RecordVisitor(token string) {
|
||||
exec("record visitor",
|
||||
`INSERT OR IGNORE INTO daily_visitors (day, visitor) VALUES (?, ?)`,
|
||||
unixDay(), token)
|
||||
}
|
||||
|
||||
// PathStat is one row of the per-page usage breakdown.
|
||||
type PathStat struct {
|
||||
Path string
|
||||
Total int // all-time views
|
||||
Today int // views since the start of the current UTC day
|
||||
}
|
||||
|
||||
// DayStat is one day's unique-visitor estimate.
|
||||
type DayStat struct {
|
||||
Day int64 // unix day number
|
||||
Uniques int
|
||||
}
|
||||
|
||||
// MetricsSummary is the aggregate usage readout surfaced by !petestats.
|
||||
type MetricsSummary struct {
|
||||
Pages []PathStat // per-path, busiest first
|
||||
TotalViews int // all-time, all paths
|
||||
ViewsToday int // all paths, current UTC day
|
||||
UniquesToday int // distinct visitor tokens, current UTC day
|
||||
Last7Days []DayStat // daily uniques, oldest → newest (note: salt rotates
|
||||
// daily, so these CANNOT be summed into a cross-day unique count)
|
||||
}
|
||||
|
||||
// GetMetricsSummary assembles the usage readout. Best-effort: any sub-query
|
||||
// failure leaves that field at its zero value rather than failing the whole
|
||||
// command.
|
||||
func GetMetricsSummary() MetricsSummary {
|
||||
today := unixDay()
|
||||
var m MetricsSummary
|
||||
|
||||
rows, err := Get().Query(
|
||||
`SELECT path,
|
||||
SUM(views) AS total,
|
||||
COALESCE(SUM(CASE WHEN day = ? THEN views END), 0) AS today
|
||||
FROM page_views
|
||||
GROUP BY path`, today)
|
||||
if err != nil {
|
||||
slog.Error("metrics: page_views query failed", "err", err)
|
||||
} else {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var p PathStat
|
||||
if err := rows.Scan(&p.Path, &p.Total, &p.Today); err != nil {
|
||||
slog.Error("metrics: scan page stat failed", "err", err)
|
||||
continue
|
||||
}
|
||||
m.Pages = append(m.Pages, p)
|
||||
m.TotalViews += p.Total
|
||||
m.ViewsToday += p.Today
|
||||
}
|
||||
}
|
||||
sort.Slice(m.Pages, func(i, j int) bool {
|
||||
if m.Pages[i].Total != m.Pages[j].Total {
|
||||
return m.Pages[i].Total > m.Pages[j].Total
|
||||
}
|
||||
return m.Pages[i].Path < m.Pages[j].Path
|
||||
})
|
||||
|
||||
if err := Get().QueryRow(
|
||||
`SELECT COUNT(*) FROM daily_visitors WHERE day = ?`, today,
|
||||
).Scan(&m.UniquesToday); err != nil {
|
||||
slog.Error("metrics: uniques-today query failed", "err", err)
|
||||
}
|
||||
|
||||
since := today - 6 // inclusive 7-day window
|
||||
vrows, err := Get().Query(
|
||||
`SELECT day, COUNT(*) FROM daily_visitors
|
||||
WHERE day >= ? GROUP BY day ORDER BY day`, since)
|
||||
if err != nil {
|
||||
slog.Error("metrics: 7-day uniques query failed", "err", err)
|
||||
} else {
|
||||
defer vrows.Close()
|
||||
for vrows.Next() {
|
||||
var d DayStat
|
||||
if err := vrows.Scan(&d.Day, &d.Uniques); err != nil {
|
||||
slog.Error("metrics: scan day stat failed", "err", err)
|
||||
continue
|
||||
}
|
||||
m.Last7Days = append(m.Last7Days, d)
|
||||
}
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
42
internal/storage/metrics_test.go
Normal file
42
internal/storage/metrics_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package storage
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMetrics_ViewsAndUniques(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
// Three views on home, two on gaming.
|
||||
RecordPageView("home")
|
||||
RecordPageView("home")
|
||||
RecordPageView("home")
|
||||
RecordPageView("gaming")
|
||||
RecordPageView("gaming")
|
||||
|
||||
// Two distinct visitors today; the repeat token must not double-count.
|
||||
RecordVisitor("tok-a")
|
||||
RecordVisitor("tok-a")
|
||||
RecordVisitor("tok-b")
|
||||
|
||||
m := GetMetricsSummary()
|
||||
|
||||
if m.TotalViews != 5 {
|
||||
t.Errorf("TotalViews = %d, want 5", m.TotalViews)
|
||||
}
|
||||
if m.ViewsToday != 5 {
|
||||
t.Errorf("ViewsToday = %d, want 5", m.ViewsToday)
|
||||
}
|
||||
if m.UniquesToday != 2 {
|
||||
t.Errorf("UniquesToday = %d, want 2 (deduped)", m.UniquesToday)
|
||||
}
|
||||
if len(m.Pages) == 0 || m.Pages[0].Path != "home" || m.Pages[0].Total != 3 {
|
||||
t.Errorf("Pages[0] = %+v, want home with 3 views first", m.Pages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetrics_EmptyIsZero(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
m := GetMetricsSummary()
|
||||
if m.TotalViews != 0 || m.ViewsToday != 0 || m.UniquesToday != 0 || len(m.Pages) != 0 {
|
||||
t.Errorf("expected zero-value summary, got %+v", m)
|
||||
}
|
||||
}
|
||||
@@ -53,6 +53,26 @@ CREATE TABLE IF NOT EXISTS user_preferences (
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Aggregate web usage. page_views holds running view counts keyed by a coarse
|
||||
-- path label ("home", channel slug, …) and the UTC day, so we can report both
|
||||
-- all-time totals and per-day breakdowns without storing any per-request rows.
|
||||
CREATE TABLE IF NOT EXISTS page_views (
|
||||
path TEXT NOT NULL,
|
||||
day INTEGER NOT NULL, -- unix day (floor(unix / 86400)), UTC
|
||||
views INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (path, day)
|
||||
);
|
||||
|
||||
-- Privacy-preserving daily unique estimate. visitor is a salted hash of
|
||||
-- IP+User-Agent; the salt rotates every UTC day and is never persisted, so the
|
||||
-- hashes are irreversible and cannot be linked across days. We keep only enough
|
||||
-- to dedup within a single day, then prune.
|
||||
CREATE TABLE IF NOT EXISTS daily_visitors (
|
||||
day INTEGER NOT NULL,
|
||||
visitor TEXT NOT NULL,
|
||||
PRIMARY KEY (day, visitor)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_post_log_guid_channel ON post_log(guid, channel);
|
||||
CREATE INDEX IF NOT EXISTS idx_post_log_event_id ON post_log(event_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_post_log_channel_posted ON post_log(channel, posted_at);
|
||||
@@ -65,6 +85,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_url_canonical ON stories(url_canon
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_stories_source_headline_norm ON stories(source, headline_norm) WHERE headline_norm IS NOT NULL AND headline_norm <> '';
|
||||
CREATE INDEX IF NOT EXISTS idx_reactions_post_guid ON reactions(post_guid);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_reactions_event_id ON reactions(event_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_page_views_day ON page_views(day);
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_visitors_day ON daily_visitors(day);
|
||||
`
|
||||
|
||||
const ftsSchema = `
|
||||
|
||||
Reference in New Issue
Block a user