Add personalization, outbound feeds, and PWA/push to the web UI

A multi-session build turning Pete's read-only web UI into something people
return to. Five phases, signed-in features keyed off the OIDC subject; anonymous
visitors keep the reverse-chron feed and localStorage-only state.

Phase 1 — per-user read + bookmark state: user_story_state table +
storage/userstate.go; auth-gated /api/read, /api/bookmark, /api/state and a
/bookmarks page; reader.js syncs state server-side for signed-in users. Also
hides the Matrix-posting UI when posting.enabled=false (web-only mode).

Phase 2 — outbound feeds: storage.ListForFeed + web/feed.go hand-build RSS 2.0
(content:encoded) and JSON Feed 1.1 (no new dep); /feed.xml, /feed.json and
per-channel variants; <link rel=alternate> discovery tags.

Phase 3 — "For you" + related: storage/rank.go scores recent unread candidates
by channel/source affinity + recency decay; RelatedStories via FTS5. ForYou rail
+ /for-you page; public /api/related feeds the reader's "You might also like".

Phase 4 — source-health dashboard: source_health table + storage/sourcehealth.go
(RecordPollResult, ListSourceHealth, SourceContentStats), written by the poller;
admin-gated /status page behind web.admin_subs.

Phase 5 — PWA + offline reader + Web Push: root-scoped manifest.webmanifest and
sw.js (app-shell precache, /api/article runtime cache for offline reading,
offline fallback, push/notificationclick handlers); PNG icons from pete.avif;
pwa.js registers the SW and drives a notifications toggle. Web Push adds
webpush-go, a [web.push] config block (pete -genvapid mints VAPID keys), a
push_subscriptions table, auth-gated subscribe/unsubscribe endpoints, and a
digest sender that pings each subscriber "N new stories" past their watermark,
honoring disabled-sources and pruning gone endpoints.

Tests added beside each new storage/web file; go test ./... and go vet clean.
This commit is contained in:
prosolis
2026-07-07 00:07:19 -07:00
parent 55aa167151
commit 71f7050f41
45 changed files with 3622 additions and 36 deletions

View File

@@ -120,6 +120,11 @@ func RunMaintenance() {
exec("prune old reactions",
`DELETE FROM reactions WHERE reacted_at < ?`, storyCutoff)
// Drop per-user read/bookmark rows whose story has been pruned above, so the
// table can't accumulate dangling references as stories age out.
exec("prune orphan user_story_state",
`DELETE FROM user_story_state WHERE story_id NOT IN (SELECT id FROM stories)`)
// Daily unique tokens are only useful for the recent window; their salts are
// long gone. page_views is kept forever (tiny aggregate, all-time totals).
exec("prune old daily_visitors",

101
internal/storage/push.go Normal file
View File

@@ -0,0 +1,101 @@
package storage
import "fmt"
// PushSubscription is one browser/device endpoint a signed-in user has opted in
// for Web Push digests. See the push_subscriptions schema for the field roles.
type PushSubscription struct {
Endpoint string
UserSub string
P256dh string
Auth string
CreatedAt int64
LastNotifiedAt int64
}
// AddPushSubscription records (or refreshes) a push endpoint for a user. The
// endpoint is the primary key, so a re-subscribe from the same browser updates
// the keys and resets the digest watermark to now — the user shouldn't be
// paged for everything published before they opted in.
func AddPushSubscription(sub, endpoint, p256dh, auth string) error {
now := nowUnix()
_, err := Get().Exec(`
INSERT INTO push_subscriptions (endpoint, user_sub, p256dh, auth, created_at, last_notified_at)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(endpoint) DO UPDATE SET
user_sub = excluded.user_sub,
p256dh = excluded.p256dh,
auth = excluded.auth,
last_notified_at = excluded.last_notified_at`,
endpoint, sub, p256dh, auth, now, now)
if err != nil {
return fmt.Errorf("add push subscription: %w", err)
}
return nil
}
// RemovePushSubscription drops one endpoint. Used both for an explicit opt-out
// and when a push service reports the endpoint is gone (404/410).
func RemovePushSubscription(endpoint string) error {
_, err := Get().Exec(`DELETE FROM push_subscriptions WHERE endpoint = ?`, endpoint)
if err != nil {
return fmt.Errorf("remove push subscription: %w", err)
}
return nil
}
// ListPushSubscriptions returns every stored subscription, for the digest sender.
func ListPushSubscriptions() ([]PushSubscription, error) {
rows, err := Get().Query(
`SELECT endpoint, user_sub, p256dh, auth, created_at, last_notified_at
FROM push_subscriptions`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []PushSubscription
for rows.Next() {
var p PushSubscription
if err := rows.Scan(&p.Endpoint, &p.UserSub, &p.P256dh, &p.Auth, &p.CreatedAt, &p.LastNotifiedAt); err != nil {
return nil, err
}
out = append(out, p)
}
return out, rows.Err()
}
// TouchPushSubscription advances an endpoint's digest watermark so its next
// digest only considers stories seen after ts.
func TouchPushSubscription(endpoint string, ts int64) error {
_, err := Get().Exec(
`UPDATE push_subscriptions SET last_notified_at = ? WHERE endpoint = ?`, ts, endpoint)
if err != nil {
return fmt.Errorf("touch push subscription: %w", err)
}
return nil
}
// NewClassifiedSince returns classified stories first seen after sinceUnix,
// newest first, capped at limit. The digest sender uses it to count and preview
// what's new for a subscriber; it carries just the fields a digest needs.
func NewClassifiedSince(sinceUnix int64, limit int) ([]Story, error) {
rows, err := Get().Query(
`SELECT id, headline, source, channel, seen_at
FROM stories
WHERE classified = 1 AND channel <> '_discarded' AND seen_at > ?
ORDER BY seen_at DESC
LIMIT ?`, sinceUnix, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.ID, &s.Headline, &s.Source, &s.Channel, &s.SeenAt); err != nil {
return nil, err
}
out = append(out, s)
}
return out, rows.Err()
}

View File

@@ -0,0 +1,105 @@
package storage
import "testing"
func TestPushSubscriptionLifecycle(t *testing.T) {
setupTestDB(t)
if err := AddPushSubscription("sub-1", "https://push.example/ep-a", "p256-a", "auth-a"); err != nil {
t.Fatal(err)
}
// A second endpoint for the same user (e.g. a second device).
if err := AddPushSubscription("sub-1", "https://push.example/ep-b", "p256-b", "auth-b"); err != nil {
t.Fatal(err)
}
// A different user.
if err := AddPushSubscription("sub-2", "https://push.example/ep-c", "p256-c", "auth-c"); err != nil {
t.Fatal(err)
}
subs, err := ListPushSubscriptions()
if err != nil {
t.Fatal(err)
}
if len(subs) != 3 {
t.Fatalf("got %d subscriptions, want 3", len(subs))
}
// Re-subscribing the same endpoint updates keys in place, not a new row.
if err := AddPushSubscription("sub-1", "https://push.example/ep-a", "p256-a2", "auth-a2"); err != nil {
t.Fatal(err)
}
subs, _ = ListPushSubscriptions()
if len(subs) != 3 {
t.Fatalf("after re-subscribe got %d rows, want 3 (upsert, not insert)", len(subs))
}
var epA PushSubscription
for _, s := range subs {
if s.Endpoint == "https://push.example/ep-a" {
epA = s
}
}
if epA.P256dh != "p256-a2" || epA.Auth != "auth-a2" {
t.Errorf("re-subscribe did not refresh keys: %+v", epA)
}
if err := RemovePushSubscription("https://push.example/ep-a"); err != nil {
t.Fatal(err)
}
subs, _ = ListPushSubscriptions()
if len(subs) != 2 {
t.Fatalf("after remove got %d rows, want 2", len(subs))
}
}
func TestTouchPushSubscriptionAdvancesWatermark(t *testing.T) {
setupTestDB(t)
if err := AddPushSubscription("sub-1", "https://push.example/ep", "p", "a"); err != nil {
t.Fatal(err)
}
subs, _ := ListPushSubscriptions()
orig := subs[0].LastNotifiedAt
want := orig + 5000
if err := TouchPushSubscription("https://push.example/ep", want); err != nil {
t.Fatal(err)
}
subs, _ = ListPushSubscriptions()
if subs[0].LastNotifiedAt != want {
t.Errorf("watermark = %d, want %d", subs[0].LastNotifiedAt, want)
}
}
func TestNewClassifiedSince(t *testing.T) {
setupTestDB(t)
now := nowUnix()
insert := func(guid, headline, source, channel string, classified bool, seenAt int64) {
if err := InsertStory(&Story{
GUID: guid, Headline: headline, ArticleURL: "https://x/" + guid,
Source: source, Channel: channel, Classified: classified, SeenAt: seenAt,
}); err != nil {
t.Fatal(err)
}
}
insert("old", "Old news", "Feed A", "tech", true, now-1000)
insert("fresh1", "Fresh one", "Feed A", "tech", true, now-100)
insert("fresh2", "Fresh two", "Feed B", "gaming", true, now-50)
insert("unclassified", "Pending", "Feed A", "", false, now-10)
insert("discarded", "Junk", "Feed A", "_discarded", true, now-10)
got, err := NewClassifiedSince(now-500, 10)
if err != nil {
t.Fatal(err)
}
if len(got) != 2 {
t.Fatalf("got %d stories, want 2 (fresh classified, non-discarded, after watermark)", len(got))
}
// Newest first.
if got[0].GUID != "" && got[0].Headline != "Fresh two" {
t.Errorf("first result = %q, want newest 'Fresh two'", got[0].Headline)
}
if got[0].Source != "Feed B" || got[1].Source != "Feed A" {
t.Errorf("unexpected order/sources: %q then %q", got[0].Source, got[1].Source)
}
}

View File

@@ -300,6 +300,55 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
return out, rows.Err()
}
// ListForFeed returns up to `limit` classified stories for the outbound RSS and
// JSON feeds, newest first. Unlike the web list queries it also selects the full
// `content` and `published_at`, which the feeds need for content:encoded bodies
// and real pubDates. channel == "" spans all real channels; otherwise it scopes
// to that one channel. Sentinel channels are always excluded.
func ListForFeed(channel string, limit int) ([]Story, error) {
const cols = `s.id, s.guid, s.headline, s.lede, s.content, s.image_url, s.article_url, s.url_canonical, s.source, s.channel, s.seen_at, s.published_at`
var (
rows *sql.Rows
err error
)
if channel == "" {
rows, err = Get().Query(
`SELECT `+cols+`
FROM stories s
WHERE s.classified = 1
AND s.channel IS NOT NULL
AND s.channel NOT IN ('_discarded', '_duplicate')
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
LIMIT ?`, limit)
} else {
rows, err = Get().Query(
`SELECT `+cols+`
FROM stories s
WHERE s.classified = 1
AND s.channel = ?
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
LIMIT ?`, channel, limit)
}
if err != nil {
return nil, err
}
defer rows.Close()
var out []Story
for rows.Next() {
var s Story
var content, canonical sql.NullString
var published sql.NullInt64
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &content, &s.ImageURL, &s.ArticleURL, &canonical, &s.Source, &s.Channel, &s.SeenAt, &published); err != nil {
return nil, err
}
s.Content = content.String
s.URLCanonical = canonical.String
s.PublishedAt = published.Int64
out = append(out, s)
}
return out, rows.Err()
}
// ListRecentlyPosted returns up to `limit` stories that have been posted to
// Matrix, ordered by post time (newest first). Posted is always true.
func ListRecentlyPosted(limit int) ([]Story, error) {

275
internal/storage/rank.go Normal file
View File

@@ -0,0 +1,275 @@
package storage
import (
"database/sql"
"errors"
"math"
"sort"
"strings"
)
// Ranking weights for the "For you" feed. Bookmarks are a stronger signal of
// interest than a plain read, and channel affinity outweighs source affinity
// (a reader follows topics more than outlets). Recency keeps the feed fresh so
// a heavily-read channel doesn't surface week-old stories over today's news.
const (
affinityReadWeight = 1.0
affinityBookmarkWeight = 3.0
forYouChannelWeight = 1.0
forYouSourceWeight = 0.7
forYouRecencyWeight = 0.9
// forYouCandidatePool bounds how many recent unread stories we score in Go.
forYouCandidatePool = 400
// forYouRecencyHalfLifeHours sets how fast the recency term decays.
forYouRecencyHalfLifeHours = 48.0
)
// userAffinity builds normalized channel and source affinity scores (each 0..1)
// for a signed-in user from their read + bookmark history. Bookmarks count for
// more than plain reads. total is the number of signal-bearing rows; when it is
// zero the user has no history yet and both maps are empty.
func userAffinity(sub string) (channel, source map[string]float64, total int, err error) {
channel = make(map[string]float64)
source = make(map[string]float64)
if sub == "" {
return channel, source, 0, nil
}
rows, err := Get().Query(
`SELECT s.channel, s.source, u.read_at, u.bookmarked_at
FROM user_story_state u
JOIN stories s ON s.id = u.story_id
WHERE u.user_sub = ?
AND s.channel IS NOT NULL
AND s.channel NOT IN ('_discarded', '_duplicate')`, sub)
if err != nil {
return nil, nil, 0, err
}
defer rows.Close()
for rows.Next() {
var ch, src string
var readAt, markAt sql.NullInt64
if err := rows.Scan(&ch, &src, &readAt, &markAt); err != nil {
return nil, nil, 0, err
}
w := 0.0
if readAt.Valid {
w += affinityReadWeight
}
if markAt.Valid {
w += affinityBookmarkWeight
}
if w == 0 {
continue
}
channel[ch] += w
source[src] += w
total++
}
if err := rows.Err(); err != nil {
return nil, nil, 0, err
}
normalizeMax(channel)
normalizeMax(source)
return channel, source, total, nil
}
// normalizeMax scales a map's values so the largest becomes 1.0, leaving an
// all-zero (or empty) map untouched.
func normalizeMax(m map[string]float64) {
var max float64
for _, v := range m {
if v > max {
max = v
}
}
if max == 0 {
return
}
for k := range m {
m[k] /= max
}
}
// ForYou returns a personalized ranking of recent, unread stories for a
// signed-in user, blending channel affinity, source affinity, and recency. It
// returns (nil, nil) when the user has no read/bookmark history yet, so callers
// can fall back to the plain latest feed.
func ForYou(sub string, limit int) ([]Story, error) {
if limit <= 0 {
return nil, nil
}
chAff, srcAff, total, err := userAffinity(sub)
if err != nil {
return nil, err
}
if total == 0 {
return nil, nil
}
rows, err := Get().Query(
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at, COALESCE(s.published_at, s.seen_at),
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories s
WHERE s.classified = 1
AND s.channel IS NOT NULL
AND s.channel NOT IN ('_discarded', '_duplicate')
AND s.id NOT IN (
SELECT story_id FROM user_story_state
WHERE user_sub = ? AND read_at IS NOT NULL)
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
LIMIT ?`, sub, forYouCandidatePool)
if err != nil {
return nil, err
}
defer rows.Close()
now := float64(nowUnix())
type scored struct {
s Story
score float64
}
var cands []scored
for rows.Next() {
var s Story
var effectiveAt int64
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &effectiveAt, &s.Posted); err != nil {
return nil, err
}
ageHours := (now - float64(effectiveAt)) / 3600.0
if ageHours < 0 {
ageHours = 0
}
recency := math.Exp(-ageHours / forYouRecencyHalfLifeHours)
score := forYouChannelWeight*chAff[s.Channel] +
forYouSourceWeight*srcAff[s.Source] +
forYouRecencyWeight*recency
cands = append(cands, scored{s, score})
}
if err := rows.Err(); err != nil {
return nil, err
}
// Candidates arrive newest-first, and a stable sort keeps that order for
// equal scores, so ties break toward the more recent story.
sort.SliceStable(cands, func(i, j int) bool { return cands[i].score > cands[j].score })
if len(cands) > limit {
cands = cands[:limit]
}
out := make([]Story, 0, len(cands))
for _, c := range cands {
out = append(out, c.s)
}
return out, nil
}
// relatedRecencyWindowSeconds bounds "related" results to roughly the same
// window the story feed lives in (stories are pruned at 30 days anyway).
const relatedRecencyWindowSeconds = 30 * 86400
// relatedStopwords are high-frequency, low-signal tokens dropped when building
// the "related" FTS query so matches key off the story's actual subject matter.
var relatedStopwords = map[string]bool{
"the": true, "and": true, "for": true, "with": true, "that": true,
"this": true, "from": true, "has": true, "have": true, "are": true,
"was": true, "were": true, "will": true, "its": true, "into": true,
"out": true, "how": true, "why": true, "what": true, "who": true,
"you": true, "your": true, "not": true, "but": true, "all": true,
"new": true, "now": true, "more": true, "after": true, "over": true,
"about": true, "says": true, "said": true, "of": true, "to": true,
"in": true, "is": true, "on": true, "at": true, "by": true, "as": true,
"an": true, "be": true, "it": true, "or": true, "if": true, "we": true,
"he": true, "do": true, "up": true, "so": true, "no": true, "my": true,
}
// RelatedStories returns classified stories textually similar to the given
// story, ranked by FTS5 relevance and excluding the story itself. The match
// query is an OR of significant tokens from the seed story's headline and lede,
// so it surfaces stories that share subject matter rather than requiring every
// term (which is what SearchStories' implicit-AND query would demand).
func RelatedStories(storyID int64, limit int) ([]Story, error) {
if storyID <= 0 || limit <= 0 {
return nil, nil
}
var headline, lede sql.NullString
err := Get().QueryRow(`SELECT headline, lede FROM stories WHERE id = ?`, storyID).Scan(&headline, &lede)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
return nil, err
}
fts := buildRelatedFTSQuery(headline.String + " " + lede.String)
if fts == "" {
return nil, nil
}
cutoff := nowUnix() - relatedRecencyWindowSeconds
rows, err := Get().Query(
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM stories_fts f
JOIN stories s ON s.id = f.rowid
WHERE f.stories_fts MATCH ?
AND s.id != ?
AND s.classified = 1
AND s.channel IS NOT NULL
AND s.channel NOT IN ('_discarded', '_duplicate')
AND s.seen_at >= ?
ORDER BY bm25(stories_fts) ASC, s.seen_at DESC
LIMIT ?`, fts, storyID, cutoff, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
}
return out, rows.Err()
}
// buildRelatedFTSQuery tokenizes a story's headline+lede into a de-duplicated,
// stopword-filtered OR query of quoted terms, capped so a long lede can't build
// a pathological query. Returns "" when nothing usable remains.
func buildRelatedFTSQuery(raw string) string {
var tokens []string
var cur strings.Builder
seen := make(map[string]bool)
flush := func() {
if cur.Len() == 0 {
return
}
t := strings.ToLower(cur.String())
cur.Reset()
if len([]rune(t)) < 2 || relatedStopwords[t] || seen[t] {
return
}
seen[t] = true
tokens = append(tokens, t)
}
for _, r := range raw {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
cur.WriteRune(r)
case r > 127:
cur.WriteRune(r)
default:
flush()
}
}
flush()
if len(tokens) == 0 {
return ""
}
const maxTokens = 16
if len(tokens) > maxTokens {
tokens = tokens[:maxTokens]
}
for i, t := range tokens {
tokens[i] = `"` + t + `"`
}
return strings.Join(tokens, " OR ")
}

View File

@@ -0,0 +1,141 @@
package storage
import (
"strings"
"testing"
)
// seedStoryFull inserts a story with an explicit channel + source so ranking
// tests can build a skewed affinity profile.
func seedStoryFull(t *testing.T, guid, channel, source, headline, lede string) int64 {
t.Helper()
s := &Story{
GUID: guid,
Headline: headline,
Lede: lede,
ArticleURL: "https://example.com/" + guid,
Source: source,
Channel: channel,
Classified: true,
SeenAt: nowUnix(),
}
if err := InsertStory(s); err != nil {
t.Fatalf("insert story %s: %v", guid, err)
}
var id int64
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, guid).Scan(&id); err != nil {
t.Fatalf("lookup id %s: %v", guid, err)
}
return id
}
func TestForYou_LeansToAffinityAndExcludesRead(t *testing.T) {
setupTestDB(t)
const sub = "sub-1"
// A skewed history: the user reads/bookmarks gaming stories.
g1 := seedStoryFull(t, "g1", "gaming", "GameSite", "Game one", "")
g2 := seedStoryFull(t, "g2", "gaming", "GameSite", "Game two", "")
// Candidates to rank (all unread until we mark some).
gc := seedStoryFull(t, "gc", "gaming", "GameSite", "Fresh gaming story", "")
tc := seedStoryFull(t, "tc", "tech", "TechSite", "Fresh tech story", "")
if err := SetRead(sub, g1, true); err != nil {
t.Fatalf("read g1: %v", err)
}
if err := SetBookmark(sub, g2, true); err != nil {
t.Fatalf("bookmark g2: %v", err)
}
got, err := ForYou(sub, 10)
if err != nil {
t.Fatalf("ForYou: %v", err)
}
if len(got) == 0 {
t.Fatal("expected results")
}
// Affinity leans gaming, so a gaming story tops the list and the tech story
// sinks to the bottom. (gc and the unread-but-bookmarked g2 both qualify and
// tie on score, so we assert on channel rather than a specific id.)
if got[0].Channel != "gaming" {
t.Fatalf("expected a gaming story first, got channel %q (id %d)", got[0].Channel, got[0].ID)
}
if got[len(got)-1].ID != tc {
t.Fatalf("expected tech candidate ranked last, got id %d", got[len(got)-1].ID)
}
// Read stories must never appear.
for _, s := range got {
if s.ID == g1 {
t.Fatalf("read story g1 leaked into ForYou")
}
}
// Sanity: the gaming candidate that was never touched should be present.
var sawGC bool
for _, s := range got {
if s.ID == gc {
sawGC = true
}
}
if !sawGC {
t.Fatalf("expected unread gaming candidate gc in results")
}
}
func TestForYou_NoHistoryReturnsNil(t *testing.T) {
setupTestDB(t)
seedStoryFull(t, "a", "tech", "TechSite", "Something", "")
got, err := ForYou("nobody", 10)
if err != nil {
t.Fatalf("ForYou: %v", err)
}
if got != nil {
t.Fatalf("expected nil for a user with no history, got %d rows", len(got))
}
}
func TestRelatedStories_OnTopicExcludesSelf(t *testing.T) {
setupTestDB(t)
seed := seedStoryFull(t, "seed", "tech", "TechSite",
"Apple unveils new iPhone camera", "The new camera sensor is larger.")
rel := seedStoryFull(t, "rel", "tech", "OtherSite",
"iPhone camera teardown reveals sensor", "A look at the new iPhone camera.")
seedStoryFull(t, "off", "gaming", "GameSite",
"Nintendo announces handheld", "A brand new portable console.")
got, err := RelatedStories(seed, 5)
if err != nil {
t.Fatalf("RelatedStories: %v", err)
}
if len(got) == 0 {
t.Fatal("expected at least one related story")
}
for _, s := range got {
if s.ID == seed {
t.Fatal("seed story returned as its own related")
}
}
if got[0].ID != rel {
t.Fatalf("expected the on-topic iPhone story first, got id %d", got[0].ID)
}
}
func TestBuildRelatedFTSQuery(t *testing.T) {
// Stopwords and 1-char tokens drop out; the rest become an OR of quoted terms.
q := buildRelatedFTSQuery("The new iPhone camera is a big deal")
if q == "" {
t.Fatal("expected a non-empty query")
}
for _, bad := range []string{`"the"`, `"is"`, `"new"`} {
if strings.Contains(q, bad) {
t.Fatalf("stopword leaked into query: %s in %q", bad, q)
}
}
for _, want := range []string{`"iphone"`, `"camera"`} {
if !strings.Contains(q, want) {
t.Fatalf("expected %s in query %q", want, q)
}
}
if buildRelatedFTSQuery("the a of to") != "" {
t.Fatal("expected empty query when only stopwords remain")
}
}

View File

@@ -54,6 +54,18 @@ CREATE TABLE IF NOT EXISTS user_preferences (
updated_at INTEGER NOT NULL
);
-- Per-user read + bookmark state for signed-in visitors, keyed by OIDC subject.
-- One row carries both signals; a NULL timestamp means "not set". A row with
-- both timestamps NULL is meaningless and is pruned, so presence of a row means
-- the story is read, bookmarked, or both.
CREATE TABLE IF NOT EXISTS user_story_state (
user_sub TEXT NOT NULL,
story_id INTEGER NOT NULL,
read_at INTEGER,
bookmarked_at INTEGER,
PRIMARY KEY (user_sub, story_id)
);
-- Aggregate web usage. page_views holds running view counts keyed by a coarse
-- path label ("home", channel slug, …) and the UTC day, so we can report both
-- all-time totals and per-day breakdowns without storing any per-request rows.
@@ -64,6 +76,35 @@ CREATE TABLE IF NOT EXISTS page_views (
PRIMARY KEY (path, day)
);
-- Per-source poll health, one row per configured feed (keyed by source name).
-- Written on every poll (success and failure) so the owner-facing dashboard can
-- show which feeds are healthy without keeping the poller's in-memory state.
-- last_success_at / last_item_count survive failures so a broken feed still
-- shows when it last worked and how much it last returned.
CREATE TABLE IF NOT EXISTS source_health (
source TEXT PRIMARY KEY,
last_poll_at INTEGER, -- unix, most recent poll attempt
last_success_at INTEGER, -- unix, most recent successful fetch
last_error TEXT, -- last failure message ('' when healthy)
consecutive_failures INTEGER NOT NULL DEFAULT 0,
last_item_count INTEGER NOT NULL DEFAULT 0, -- items in the last successful fetch
updated_at INTEGER NOT NULL
);
-- Web Push subscriptions for signed-in users, one row per browser/device
-- endpoint. p256dh + auth are the client's encryption keys (RFC 8291); the
-- server needs them to encrypt each push. last_notified_at is the per-endpoint
-- digest watermark: the sender only counts stories seen after it. A user can
-- have several endpoints (phone, desktop) — each is notified independently.
CREATE TABLE IF NOT EXISTS push_subscriptions (
endpoint TEXT PRIMARY KEY,
user_sub TEXT NOT NULL,
p256dh TEXT NOT NULL,
auth TEXT NOT NULL,
created_at INTEGER NOT NULL,
last_notified_at INTEGER NOT NULL
);
-- Privacy-preserving daily unique estimate. visitor is a salted hash of
-- IP+User-Agent; the salt rotates every UTC day and is never persisted, so the
-- hashes are irreversible and cannot be linked across days. We keep only enough
@@ -88,6 +129,9 @@ CREATE INDEX IF NOT EXISTS idx_reactions_post_guid ON reactions(post_guid);
CREATE UNIQUE INDEX IF NOT EXISTS idx_reactions_event_id ON reactions(event_id);
CREATE INDEX IF NOT EXISTS idx_page_views_day ON page_views(day);
CREATE INDEX IF NOT EXISTS idx_daily_visitors_day ON daily_visitors(day);
CREATE INDEX IF NOT EXISTS idx_user_state_bookmarks ON user_story_state(user_sub, bookmarked_at) WHERE bookmarked_at IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_user_state_reads ON user_story_state(user_sub, read_at) WHERE read_at IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_push_sub_user ON push_subscriptions(user_sub);
`
const ftsSchema = `

View File

@@ -0,0 +1,149 @@
package storage
import (
"database/sql"
"fmt"
)
// SourceHealth is one source's persisted poll health, as written by the poller.
type SourceHealth struct {
Source string
LastPollAt int64 // 0 = never polled
LastSuccessAt int64 // 0 = never succeeded
LastError string
ConsecutiveFailures int
LastItemCount int
UpdatedAt int64
}
// SourceContentStat is per-source content derived from the stories table (plus
// post_log for last-posted), independent of poll health.
type SourceContentStat struct {
Source string
Total int // stories currently retained for this source
Classified int // of those, how many are classified
Paywalled int // of those, how many are gated
LastSeenAt int64 // MAX(seen_at); 0 = none
LastPostedAt int64 // MAX(post_log.posted_at) joined by guid; 0 = never posted
}
// RecordPollResult upserts a source's health row after a poll attempt. On
// success it clears the error and failure counter and records the item count;
// on failure it bumps consecutive_failures and stores the message while
// preserving the last successful timestamp and item count. Fire-and-forget:
// a failure here must never disrupt polling, so errors are logged and swallowed.
func RecordPollResult(source string, ok bool, itemCount int, pollErr error) {
now := nowUnix()
if ok {
exec("record poll success",
`INSERT INTO source_health
(source, last_poll_at, last_success_at, last_error, consecutive_failures, last_item_count, updated_at)
VALUES (?, ?, ?, '', 0, ?, ?)
ON CONFLICT(source) DO UPDATE SET
last_poll_at = excluded.last_poll_at,
last_success_at = excluded.last_success_at,
last_error = '',
consecutive_failures = 0,
last_item_count = excluded.last_item_count,
updated_at = excluded.updated_at`,
source, now, now, itemCount, now)
return
}
msg := ""
if pollErr != nil {
msg = pollErr.Error()
}
exec("record poll failure",
`INSERT INTO source_health
(source, last_poll_at, last_success_at, last_error, consecutive_failures, last_item_count, updated_at)
VALUES (?, ?, NULL, ?, 1, 0, ?)
ON CONFLICT(source) DO UPDATE SET
last_poll_at = excluded.last_poll_at,
last_error = excluded.last_error,
consecutive_failures = source_health.consecutive_failures + 1,
updated_at = excluded.updated_at`,
source, now, msg, now)
}
// ListSourceHealth returns the poll-health row for every source that has been
// polled at least once, keyed by source name.
func ListSourceHealth() (map[string]SourceHealth, error) {
rows, err := Get().Query(
`SELECT source, last_poll_at, last_success_at, last_error,
consecutive_failures, last_item_count, updated_at
FROM source_health`)
if err != nil {
return nil, fmt.Errorf("list source health: %w", err)
}
defer rows.Close()
out := make(map[string]SourceHealth)
for rows.Next() {
var h SourceHealth
var lastPoll, lastSuccess sql.NullInt64
var lastErr sql.NullString
if err := rows.Scan(&h.Source, &lastPoll, &lastSuccess, &lastErr,
&h.ConsecutiveFailures, &h.LastItemCount, &h.UpdatedAt); err != nil {
return nil, err
}
h.LastPollAt = lastPoll.Int64
h.LastSuccessAt = lastSuccess.Int64
h.LastError = lastErr.String
out[h.Source] = h
}
return out, rows.Err()
}
// SourceContentStats derives per-source content counts from the stories table,
// with last-posted joined from post_log by guid. Keyed by source name. Only
// sources with at least one retained story appear; the caller pads out the rest
// from its configured source list.
func SourceContentStats() (map[string]SourceContentStat, error) {
out := make(map[string]SourceContentStat)
rows, err := Get().Query(
`SELECT source,
COUNT(*),
COALESCE(SUM(classified), 0),
COALESCE(SUM(paywalled), 0),
COALESCE(MAX(seen_at), 0)
FROM stories
GROUP BY source`)
if err != nil {
return nil, fmt.Errorf("source content stats: %w", err)
}
defer rows.Close()
for rows.Next() {
var st SourceContentStat
if err := rows.Scan(&st.Source, &st.Total, &st.Classified, &st.Paywalled, &st.LastSeenAt); err != nil {
return nil, err
}
out[st.Source] = st
}
if err := rows.Err(); err != nil {
return nil, err
}
// Last-posted per source, joined by guid. Kept separate so sources with
// stories but no posts still appear above with a zero last-posted.
prows, err := Get().Query(
`SELECT s.source, MAX(p.posted_at)
FROM post_log p
JOIN stories s ON s.guid = p.guid
GROUP BY s.source`)
if err != nil {
return nil, fmt.Errorf("source last-posted: %w", err)
}
defer prows.Close()
for prows.Next() {
var source string
var lastPosted int64
if err := prows.Scan(&source, &lastPosted); err != nil {
return nil, err
}
st := out[source]
st.Source = source
st.LastPostedAt = lastPosted
out[source] = st
}
return out, prows.Err()
}

View File

@@ -0,0 +1,104 @@
package storage
import (
"errors"
"testing"
)
func TestRecordPollResult_SuccessThenFailure(t *testing.T) {
setupTestDB(t)
// First a successful poll returning 7 items.
RecordPollResult("Feed A", true, 7, nil)
h, err := ListSourceHealth()
if err != nil {
t.Fatal(err)
}
a, ok := h["Feed A"]
if !ok {
t.Fatal("expected a health row for Feed A")
}
if a.LastItemCount != 7 || a.ConsecutiveFailures != 0 || a.LastError != "" {
t.Errorf("after success: items=%d fails=%d err=%q", a.LastItemCount, a.ConsecutiveFailures, a.LastError)
}
if a.LastPollAt == 0 || a.LastSuccessAt == 0 {
t.Errorf("after success: last_poll=%d last_success=%d, want both set", a.LastPollAt, a.LastSuccessAt)
}
successTS := a.LastSuccessAt
// Two failures in a row: failures accumulate, but the last-success timestamp
// and item count are preserved so the dashboard still shows when it last worked.
RecordPollResult("Feed A", false, 0, errors.New("dial tcp: timeout"))
RecordPollResult("Feed A", false, 0, errors.New("502 bad gateway"))
h, _ = ListSourceHealth()
a = h["Feed A"]
if a.ConsecutiveFailures != 2 {
t.Errorf("consecutive failures = %d, want 2", a.ConsecutiveFailures)
}
if a.LastError != "502 bad gateway" {
t.Errorf("last error = %q, want %q", a.LastError, "502 bad gateway")
}
if a.LastSuccessAt != successTS {
t.Errorf("last success = %d, want preserved %d", a.LastSuccessAt, successTS)
}
if a.LastItemCount != 7 {
t.Errorf("last item count = %d, want preserved 7", a.LastItemCount)
}
// Recovery clears the error and resets the counter.
RecordPollResult("Feed A", true, 3, nil)
h, _ = ListSourceHealth()
a = h["Feed A"]
if a.ConsecutiveFailures != 0 || a.LastError != "" || a.LastItemCount != 3 {
t.Errorf("after recovery: fails=%d err=%q items=%d", a.ConsecutiveFailures, a.LastError, a.LastItemCount)
}
}
func TestSourceContentStats(t *testing.T) {
setupTestDB(t)
// Source X: two classified stories, one paywalled; one posted.
InsertStory(&Story{GUID: "x1", Headline: "X one", ArticleURL: "https://x.com/1", Source: "X", Channel: "tech", Classified: true, SeenAt: 100})
InsertStory(&Story{GUID: "x2", Headline: "X two", ArticleURL: "https://x.com/2", Source: "X", Channel: "tech", Classified: true, Paywalled: true, SeenAt: 200})
// Source Y: one unclassified story, never posted.
InsertStory(&Story{GUID: "y1", Headline: "Y one", ArticleURL: "https://y.com/1", Source: "Y", SeenAt: 150})
InsertPostLog("x1", "tech", "$e1", "", false)
// Backdate/forward the post so we can assert MAX(posted_at).
Get().Exec(`UPDATE post_log SET posted_at = ? WHERE guid = ?`, int64(500), "x1")
stats, err := SourceContentStats()
if err != nil {
t.Fatal(err)
}
x := stats["X"]
if x.Total != 2 || x.Classified != 2 || x.Paywalled != 1 {
t.Errorf("X: total=%d classified=%d paywalled=%d, want 2/2/1", x.Total, x.Classified, x.Paywalled)
}
if x.LastSeenAt != 200 {
t.Errorf("X last seen = %d, want 200", x.LastSeenAt)
}
if x.LastPostedAt != 500 {
t.Errorf("X last posted = %d, want 500", x.LastPostedAt)
}
y := stats["Y"]
if y.Total != 1 || y.Classified != 0 || y.Paywalled != 0 {
t.Errorf("Y: total=%d classified=%d paywalled=%d, want 1/0/0", y.Total, y.Classified, y.Paywalled)
}
if y.LastPostedAt != 0 {
t.Errorf("Y last posted = %d, want 0 (never posted)", y.LastPostedAt)
}
}
func TestListSourceHealth_Empty(t *testing.T) {
setupTestDB(t)
h, err := ListSourceHealth()
if err != nil {
t.Fatal(err)
}
if len(h) != 0 {
t.Errorf("expected no health rows, got %d", len(h))
}
}

View File

@@ -0,0 +1,139 @@
package storage
import (
"database/sql"
"fmt"
"strings"
)
// SetRead marks (read=true) or clears (read=false) the read flag for one story
// and one signed-in user (keyed by OIDC subject). Read and bookmark state share
// a row; clearing the last remaining flag removes the row.
func SetRead(sub string, storyID int64, read bool) error {
var ts any
if read {
ts = nowUnix()
}
_, err := Get().Exec(`
INSERT INTO user_story_state (user_sub, story_id, read_at, bookmarked_at)
VALUES (?, ?, ?, NULL)
ON CONFLICT(user_sub, story_id) DO UPDATE SET read_at = excluded.read_at`,
sub, storyID, ts)
if err != nil {
return fmt.Errorf("set read: %w", err)
}
return pruneEmptyState(sub, storyID)
}
// SetBookmark adds (on=true) or removes (on=false) a bookmark for one story and
// one signed-in user. See SetRead for the shared-row semantics.
func SetBookmark(sub string, storyID int64, on bool) error {
var ts any
if on {
ts = nowUnix()
}
_, err := Get().Exec(`
INSERT INTO user_story_state (user_sub, story_id, read_at, bookmarked_at)
VALUES (?, ?, NULL, ?)
ON CONFLICT(user_sub, story_id) DO UPDATE SET bookmarked_at = excluded.bookmarked_at`,
sub, storyID, ts)
if err != nil {
return fmt.Errorf("set bookmark: %w", err)
}
return pruneEmptyState(sub, storyID)
}
// pruneEmptyState drops a row once neither flag is set, keeping the table to
// only meaningful state.
func pruneEmptyState(sub string, storyID int64) error {
_, err := Get().Exec(
`DELETE FROM user_story_state
WHERE user_sub = ? AND story_id = ? AND read_at IS NULL AND bookmarked_at IS NULL`,
sub, storyID)
if err != nil {
return fmt.Errorf("prune user state: %w", err)
}
return nil
}
// UserStoryState reports, for the given story ids, which are read and which are
// bookmarked by this user. Both maps contain only ids whose flag is set, so a
// missing key means false. An empty sub or id list returns empty maps.
func UserStoryState(sub string, ids []int64) (read, bookmarked map[int64]bool, err error) {
read = make(map[int64]bool)
bookmarked = make(map[int64]bool)
if sub == "" || len(ids) == 0 {
return read, bookmarked, nil
}
q := `SELECT story_id, read_at, bookmarked_at FROM user_story_state
WHERE user_sub = ? AND story_id IN (` + placeholders(len(ids)) + `)`
args := make([]any, 0, len(ids)+1)
args = append(args, sub)
for _, id := range ids {
args = append(args, id)
}
rows, err := Get().Query(q, args...)
if err != nil {
return nil, nil, fmt.Errorf("user story state: %w", err)
}
defer rows.Close()
for rows.Next() {
var id int64
var r, b sql.NullInt64
if err := rows.Scan(&id, &r, &b); err != nil {
return nil, nil, err
}
if r.Valid {
read[id] = true
}
if b.Valid {
bookmarked[id] = true
}
}
return read, bookmarked, rows.Err()
}
// ListBookmarks returns the user's bookmarked stories, most recently bookmarked
// first, restricted to still-classified stories.
func ListBookmarks(sub string, limit, offset int) ([]Story, error) {
rows, err := Get().Query(
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
FROM user_story_state u
JOIN stories s ON s.id = u.story_id
WHERE u.user_sub = ?
AND u.bookmarked_at IS NOT NULL
AND s.classified = 1
ORDER BY u.bookmarked_at DESC, u.story_id DESC
LIMIT ? OFFSET ?`, sub, limit, offset)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Story
for rows.Next() {
var s Story
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
return nil, err
}
out = append(out, s)
}
return out, rows.Err()
}
// CountBookmarks returns how many stories the user has bookmarked.
func CountBookmarks(sub string) (int, error) {
var n int
err := Get().QueryRow(
`SELECT COUNT(*) FROM user_story_state WHERE user_sub = ? AND bookmarked_at IS NOT NULL`,
sub).Scan(&n)
return n, err
}
// placeholders returns "?, ?, …" with n slots for an IN clause.
func placeholders(n int) string {
if n <= 0 {
return ""
}
return strings.Repeat("?, ", n-1) + "?"
}

View File

@@ -0,0 +1,126 @@
package storage
import "testing"
func seedStory(t *testing.T, guid string) int64 {
t.Helper()
s := &Story{
GUID: guid,
Headline: "Headline " + guid,
Lede: "Lede.",
ArticleURL: "https://example.com/" + guid,
Source: "Test Source",
Channel: "tech",
Classified: true,
SeenAt: 1700000000,
}
if err := InsertStory(s); err != nil {
t.Fatalf("insert story %s: %v", guid, err)
}
var id int64
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, guid).Scan(&id); err != nil {
t.Fatalf("lookup id %s: %v", guid, err)
}
return id
}
func TestUserStoryState_ReadAndBookmark(t *testing.T) {
setupTestDB(t)
id1 := seedStory(t, "s1")
id2 := seedStory(t, "s2")
const sub = "ak-sub-1"
// Nothing set yet.
read, marks, err := UserStoryState(sub, []int64{id1, id2})
if err != nil {
t.Fatalf("initial state: %v", err)
}
if len(read) != 0 || len(marks) != 0 {
t.Fatalf("expected empty state, got read=%v bookmarked=%v", read, marks)
}
// Mark id1 read, bookmark id1 and id2.
if err := SetRead(sub, id1, true); err != nil {
t.Fatalf("set read: %v", err)
}
if err := SetBookmark(sub, id1, true); err != nil {
t.Fatalf("set bookmark id1: %v", err)
}
if err := SetBookmark(sub, id2, true); err != nil {
t.Fatalf("set bookmark id2: %v", err)
}
read, marks, _ = UserStoryState(sub, []int64{id1, id2})
if !read[id1] || read[id2] {
t.Fatalf("read map wrong: %v", read)
}
if !marks[id1] || !marks[id2] {
t.Fatalf("bookmark map wrong: %v", marks)
}
// id1 carries both flags in a single row.
if !read[id1] || !marks[id1] {
t.Fatalf("expected id1 read+bookmarked, read=%v marks=%v", read, marks)
}
// ListBookmarks orders newest bookmark first, tiebreaking on story id so the
// result is deterministic within a single second.
bm, err := ListBookmarks(sub, 10, 0)
if err != nil {
t.Fatalf("list bookmarks: %v", err)
}
if len(bm) != 2 {
t.Fatalf("expected 2 bookmarks, got %d", len(bm))
}
if bm[0].ID != id2 || bm[1].ID != id1 {
t.Fatalf("bookmark order wrong: %d then %d", bm[0].ID, bm[1].ID)
}
if n, _ := CountBookmarks(sub); n != 2 {
t.Fatalf("count bookmarks = %d, want 2", n)
}
}
func TestUserStoryState_ClearRemovesRow(t *testing.T) {
setupTestDB(t)
id := seedStory(t, "s1")
const sub = "ak-sub-1"
if err := SetRead(sub, id, true); err != nil {
t.Fatalf("set read: %v", err)
}
if err := SetRead(sub, id, false); err != nil {
t.Fatalf("unset read: %v", err)
}
// Row should be gone since neither flag is set.
var n int
if err := Get().QueryRow(`SELECT COUNT(*) FROM user_story_state WHERE user_sub = ?`, sub).Scan(&n); err != nil {
t.Fatalf("count rows: %v", err)
}
if n != 0 {
t.Fatalf("expected row pruned, found %d", n)
}
// Setting read then bookmark then clearing only read must keep the row.
_ = SetRead(sub, id, true)
_ = SetBookmark(sub, id, true)
_ = SetRead(sub, id, false)
read, marks, _ := UserStoryState(sub, []int64{id})
if read[id] {
t.Fatalf("expected not read")
}
if !marks[id] {
t.Fatalf("expected still bookmarked after clearing read")
}
}
func TestUserStoryState_ScopedBySub(t *testing.T) {
setupTestDB(t)
id := seedStory(t, "s1")
_ = SetBookmark("user-a", id, true)
read, marks, _ := UserStoryState("user-b", []int64{id})
if len(read) != 0 || len(marks) != 0 {
t.Fatalf("user-b should see no state, got read=%v marks=%v", read, marks)
}
if n, _ := CountBookmarks("user-b"); n != 0 {
t.Fatalf("user-b count = %d, want 0", n)
}
}