Add personalization, outbound feeds, and PWA/push to the web UI
A multi-session build turning Pete's read-only web UI into something people return to. Five phases, signed-in features keyed off the OIDC subject; anonymous visitors keep the reverse-chron feed and localStorage-only state. Phase 1 — per-user read + bookmark state: user_story_state table + storage/userstate.go; auth-gated /api/read, /api/bookmark, /api/state and a /bookmarks page; reader.js syncs state server-side for signed-in users. Also hides the Matrix-posting UI when posting.enabled=false (web-only mode). Phase 2 — outbound feeds: storage.ListForFeed + web/feed.go hand-build RSS 2.0 (content:encoded) and JSON Feed 1.1 (no new dep); /feed.xml, /feed.json and per-channel variants; <link rel=alternate> discovery tags. Phase 3 — "For you" + related: storage/rank.go scores recent unread candidates by channel/source affinity + recency decay; RelatedStories via FTS5. ForYou rail + /for-you page; public /api/related feeds the reader's "You might also like". Phase 4 — source-health dashboard: source_health table + storage/sourcehealth.go (RecordPollResult, ListSourceHealth, SourceContentStats), written by the poller; admin-gated /status page behind web.admin_subs. Phase 5 — PWA + offline reader + Web Push: root-scoped manifest.webmanifest and sw.js (app-shell precache, /api/article runtime cache for offline reading, offline fallback, push/notificationclick handlers); PNG icons from pete.avif; pwa.js registers the SW and drives a notifications toggle. Web Push adds webpush-go, a [web.push] config block (pete -genvapid mints VAPID keys), a push_subscriptions table, auth-gated subscribe/unsubscribe endpoints, and a digest sender that pings each subscriber "N new stories" past their watermark, honoring disabled-sources and pruning gone endpoints. Tests added beside each new storage/web file; go test ./... and go vet clean.
This commit is contained in:
@@ -120,6 +120,11 @@ func RunMaintenance() {
|
||||
exec("prune old reactions",
|
||||
`DELETE FROM reactions WHERE reacted_at < ?`, storyCutoff)
|
||||
|
||||
// Drop per-user read/bookmark rows whose story has been pruned above, so the
|
||||
// table can't accumulate dangling references as stories age out.
|
||||
exec("prune orphan user_story_state",
|
||||
`DELETE FROM user_story_state WHERE story_id NOT IN (SELECT id FROM stories)`)
|
||||
|
||||
// Daily unique tokens are only useful for the recent window; their salts are
|
||||
// long gone. page_views is kept forever (tiny aggregate, all-time totals).
|
||||
exec("prune old daily_visitors",
|
||||
|
||||
101
internal/storage/push.go
Normal file
101
internal/storage/push.go
Normal file
@@ -0,0 +1,101 @@
|
||||
package storage
|
||||
|
||||
import "fmt"
|
||||
|
||||
// PushSubscription is one browser/device endpoint a signed-in user has opted in
|
||||
// for Web Push digests. See the push_subscriptions schema for the field roles.
|
||||
type PushSubscription struct {
|
||||
Endpoint string
|
||||
UserSub string
|
||||
P256dh string
|
||||
Auth string
|
||||
CreatedAt int64
|
||||
LastNotifiedAt int64
|
||||
}
|
||||
|
||||
// AddPushSubscription records (or refreshes) a push endpoint for a user. The
|
||||
// endpoint is the primary key, so a re-subscribe from the same browser updates
|
||||
// the keys and resets the digest watermark to now — the user shouldn't be
|
||||
// paged for everything published before they opted in.
|
||||
func AddPushSubscription(sub, endpoint, p256dh, auth string) error {
|
||||
now := nowUnix()
|
||||
_, err := Get().Exec(`
|
||||
INSERT INTO push_subscriptions (endpoint, user_sub, p256dh, auth, created_at, last_notified_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(endpoint) DO UPDATE SET
|
||||
user_sub = excluded.user_sub,
|
||||
p256dh = excluded.p256dh,
|
||||
auth = excluded.auth,
|
||||
last_notified_at = excluded.last_notified_at`,
|
||||
endpoint, sub, p256dh, auth, now, now)
|
||||
if err != nil {
|
||||
return fmt.Errorf("add push subscription: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemovePushSubscription drops one endpoint. Used both for an explicit opt-out
|
||||
// and when a push service reports the endpoint is gone (404/410).
|
||||
func RemovePushSubscription(endpoint string) error {
|
||||
_, err := Get().Exec(`DELETE FROM push_subscriptions WHERE endpoint = ?`, endpoint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("remove push subscription: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListPushSubscriptions returns every stored subscription, for the digest sender.
|
||||
func ListPushSubscriptions() ([]PushSubscription, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT endpoint, user_sub, p256dh, auth, created_at, last_notified_at
|
||||
FROM push_subscriptions`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []PushSubscription
|
||||
for rows.Next() {
|
||||
var p PushSubscription
|
||||
if err := rows.Scan(&p.Endpoint, &p.UserSub, &p.P256dh, &p.Auth, &p.CreatedAt, &p.LastNotifiedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// TouchPushSubscription advances an endpoint's digest watermark so its next
|
||||
// digest only considers stories seen after ts.
|
||||
func TouchPushSubscription(endpoint string, ts int64) error {
|
||||
_, err := Get().Exec(
|
||||
`UPDATE push_subscriptions SET last_notified_at = ? WHERE endpoint = ?`, ts, endpoint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("touch push subscription: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewClassifiedSince returns classified stories first seen after sinceUnix,
|
||||
// newest first, capped at limit. The digest sender uses it to count and preview
|
||||
// what's new for a subscriber; it carries just the fields a digest needs.
|
||||
func NewClassifiedSince(sinceUnix int64, limit int) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT id, headline, source, channel, seen_at
|
||||
FROM stories
|
||||
WHERE classified = 1 AND channel <> '_discarded' AND seen_at > ?
|
||||
ORDER BY seen_at DESC
|
||||
LIMIT ?`, sinceUnix, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.ID, &s.Headline, &s.Source, &s.Channel, &s.SeenAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
105
internal/storage/push_test.go
Normal file
105
internal/storage/push_test.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package storage
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestPushSubscriptionLifecycle(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
if err := AddPushSubscription("sub-1", "https://push.example/ep-a", "p256-a", "auth-a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A second endpoint for the same user (e.g. a second device).
|
||||
if err := AddPushSubscription("sub-1", "https://push.example/ep-b", "p256-b", "auth-b"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A different user.
|
||||
if err := AddPushSubscription("sub-2", "https://push.example/ep-c", "p256-c", "auth-c"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
subs, err := ListPushSubscriptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(subs) != 3 {
|
||||
t.Fatalf("got %d subscriptions, want 3", len(subs))
|
||||
}
|
||||
|
||||
// Re-subscribing the same endpoint updates keys in place, not a new row.
|
||||
if err := AddPushSubscription("sub-1", "https://push.example/ep-a", "p256-a2", "auth-a2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subs, _ = ListPushSubscriptions()
|
||||
if len(subs) != 3 {
|
||||
t.Fatalf("after re-subscribe got %d rows, want 3 (upsert, not insert)", len(subs))
|
||||
}
|
||||
var epA PushSubscription
|
||||
for _, s := range subs {
|
||||
if s.Endpoint == "https://push.example/ep-a" {
|
||||
epA = s
|
||||
}
|
||||
}
|
||||
if epA.P256dh != "p256-a2" || epA.Auth != "auth-a2" {
|
||||
t.Errorf("re-subscribe did not refresh keys: %+v", epA)
|
||||
}
|
||||
|
||||
if err := RemovePushSubscription("https://push.example/ep-a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subs, _ = ListPushSubscriptions()
|
||||
if len(subs) != 2 {
|
||||
t.Fatalf("after remove got %d rows, want 2", len(subs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTouchPushSubscriptionAdvancesWatermark(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
if err := AddPushSubscription("sub-1", "https://push.example/ep", "p", "a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subs, _ := ListPushSubscriptions()
|
||||
orig := subs[0].LastNotifiedAt
|
||||
|
||||
want := orig + 5000
|
||||
if err := TouchPushSubscription("https://push.example/ep", want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subs, _ = ListPushSubscriptions()
|
||||
if subs[0].LastNotifiedAt != want {
|
||||
t.Errorf("watermark = %d, want %d", subs[0].LastNotifiedAt, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClassifiedSince(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
now := nowUnix()
|
||||
|
||||
insert := func(guid, headline, source, channel string, classified bool, seenAt int64) {
|
||||
if err := InsertStory(&Story{
|
||||
GUID: guid, Headline: headline, ArticleURL: "https://x/" + guid,
|
||||
Source: source, Channel: channel, Classified: classified, SeenAt: seenAt,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
insert("old", "Old news", "Feed A", "tech", true, now-1000)
|
||||
insert("fresh1", "Fresh one", "Feed A", "tech", true, now-100)
|
||||
insert("fresh2", "Fresh two", "Feed B", "gaming", true, now-50)
|
||||
insert("unclassified", "Pending", "Feed A", "", false, now-10)
|
||||
insert("discarded", "Junk", "Feed A", "_discarded", true, now-10)
|
||||
|
||||
got, err := NewClassifiedSince(now-500, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d stories, want 2 (fresh classified, non-discarded, after watermark)", len(got))
|
||||
}
|
||||
// Newest first.
|
||||
if got[0].GUID != "" && got[0].Headline != "Fresh two" {
|
||||
t.Errorf("first result = %q, want newest 'Fresh two'", got[0].Headline)
|
||||
}
|
||||
if got[0].Source != "Feed B" || got[1].Source != "Feed A" {
|
||||
t.Errorf("unexpected order/sources: %q then %q", got[0].Source, got[1].Source)
|
||||
}
|
||||
}
|
||||
@@ -300,6 +300,55 @@ func ListAllClassified(limit, offset int) ([]Story, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListForFeed returns up to `limit` classified stories for the outbound RSS and
|
||||
// JSON feeds, newest first. Unlike the web list queries it also selects the full
|
||||
// `content` and `published_at`, which the feeds need for content:encoded bodies
|
||||
// and real pubDates. channel == "" spans all real channels; otherwise it scopes
|
||||
// to that one channel. Sentinel channels are always excluded.
|
||||
func ListForFeed(channel string, limit int) ([]Story, error) {
|
||||
const cols = `s.id, s.guid, s.headline, s.lede, s.content, s.image_url, s.article_url, s.url_canonical, s.source, s.channel, s.seen_at, s.published_at`
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if channel == "" {
|
||||
rows, err = Get().Query(
|
||||
`SELECT `+cols+`
|
||||
FROM stories s
|
||||
WHERE s.classified = 1
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')
|
||||
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
|
||||
LIMIT ?`, limit)
|
||||
} else {
|
||||
rows, err = Get().Query(
|
||||
`SELECT `+cols+`
|
||||
FROM stories s
|
||||
WHERE s.classified = 1
|
||||
AND s.channel = ?
|
||||
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
|
||||
LIMIT ?`, channel, limit)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
var content, canonical sql.NullString
|
||||
var published sql.NullInt64
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &content, &s.ImageURL, &s.ArticleURL, &canonical, &s.Source, &s.Channel, &s.SeenAt, &published); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Content = content.String
|
||||
s.URLCanonical = canonical.String
|
||||
s.PublishedAt = published.Int64
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ListRecentlyPosted returns up to `limit` stories that have been posted to
|
||||
// Matrix, ordered by post time (newest first). Posted is always true.
|
||||
func ListRecentlyPosted(limit int) ([]Story, error) {
|
||||
|
||||
275
internal/storage/rank.go
Normal file
275
internal/storage/rank.go
Normal file
@@ -0,0 +1,275 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Ranking weights for the "For you" feed. Bookmarks are a stronger signal of
|
||||
// interest than a plain read, and channel affinity outweighs source affinity
|
||||
// (a reader follows topics more than outlets). Recency keeps the feed fresh so
|
||||
// a heavily-read channel doesn't surface week-old stories over today's news.
|
||||
const (
|
||||
affinityReadWeight = 1.0
|
||||
affinityBookmarkWeight = 3.0
|
||||
|
||||
forYouChannelWeight = 1.0
|
||||
forYouSourceWeight = 0.7
|
||||
forYouRecencyWeight = 0.9
|
||||
|
||||
// forYouCandidatePool bounds how many recent unread stories we score in Go.
|
||||
forYouCandidatePool = 400
|
||||
// forYouRecencyHalfLifeHours sets how fast the recency term decays.
|
||||
forYouRecencyHalfLifeHours = 48.0
|
||||
)
|
||||
|
||||
// userAffinity builds normalized channel and source affinity scores (each 0..1)
|
||||
// for a signed-in user from their read + bookmark history. Bookmarks count for
|
||||
// more than plain reads. total is the number of signal-bearing rows; when it is
|
||||
// zero the user has no history yet and both maps are empty.
|
||||
func userAffinity(sub string) (channel, source map[string]float64, total int, err error) {
|
||||
channel = make(map[string]float64)
|
||||
source = make(map[string]float64)
|
||||
if sub == "" {
|
||||
return channel, source, 0, nil
|
||||
}
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.channel, s.source, u.read_at, u.bookmarked_at
|
||||
FROM user_story_state u
|
||||
JOIN stories s ON s.id = u.story_id
|
||||
WHERE u.user_sub = ?
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')`, sub)
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var ch, src string
|
||||
var readAt, markAt sql.NullInt64
|
||||
if err := rows.Scan(&ch, &src, &readAt, &markAt); err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
w := 0.0
|
||||
if readAt.Valid {
|
||||
w += affinityReadWeight
|
||||
}
|
||||
if markAt.Valid {
|
||||
w += affinityBookmarkWeight
|
||||
}
|
||||
if w == 0 {
|
||||
continue
|
||||
}
|
||||
channel[ch] += w
|
||||
source[src] += w
|
||||
total++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
normalizeMax(channel)
|
||||
normalizeMax(source)
|
||||
return channel, source, total, nil
|
||||
}
|
||||
|
||||
// normalizeMax scales a map's values so the largest becomes 1.0, leaving an
|
||||
// all-zero (or empty) map untouched.
|
||||
func normalizeMax(m map[string]float64) {
|
||||
var max float64
|
||||
for _, v := range m {
|
||||
if v > max {
|
||||
max = v
|
||||
}
|
||||
}
|
||||
if max == 0 {
|
||||
return
|
||||
}
|
||||
for k := range m {
|
||||
m[k] /= max
|
||||
}
|
||||
}
|
||||
|
||||
// ForYou returns a personalized ranking of recent, unread stories for a
|
||||
// signed-in user, blending channel affinity, source affinity, and recency. It
|
||||
// returns (nil, nil) when the user has no read/bookmark history yet, so callers
|
||||
// can fall back to the plain latest feed.
|
||||
func ForYou(sub string, limit int) ([]Story, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
chAff, srcAff, total, err := userAffinity(sub)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if total == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at, COALESCE(s.published_at, s.seen_at),
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM stories s
|
||||
WHERE s.classified = 1
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')
|
||||
AND s.id NOT IN (
|
||||
SELECT story_id FROM user_story_state
|
||||
WHERE user_sub = ? AND read_at IS NOT NULL)
|
||||
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
|
||||
LIMIT ?`, sub, forYouCandidatePool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
now := float64(nowUnix())
|
||||
type scored struct {
|
||||
s Story
|
||||
score float64
|
||||
}
|
||||
var cands []scored
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
var effectiveAt int64
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &effectiveAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ageHours := (now - float64(effectiveAt)) / 3600.0
|
||||
if ageHours < 0 {
|
||||
ageHours = 0
|
||||
}
|
||||
recency := math.Exp(-ageHours / forYouRecencyHalfLifeHours)
|
||||
score := forYouChannelWeight*chAff[s.Channel] +
|
||||
forYouSourceWeight*srcAff[s.Source] +
|
||||
forYouRecencyWeight*recency
|
||||
cands = append(cands, scored{s, score})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Candidates arrive newest-first, and a stable sort keeps that order for
|
||||
// equal scores, so ties break toward the more recent story.
|
||||
sort.SliceStable(cands, func(i, j int) bool { return cands[i].score > cands[j].score })
|
||||
if len(cands) > limit {
|
||||
cands = cands[:limit]
|
||||
}
|
||||
out := make([]Story, 0, len(cands))
|
||||
for _, c := range cands {
|
||||
out = append(out, c.s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// relatedRecencyWindowSeconds bounds "related" results to roughly the same
|
||||
// window the story feed lives in (stories are pruned at 30 days anyway).
|
||||
const relatedRecencyWindowSeconds = 30 * 86400
|
||||
|
||||
// relatedStopwords are high-frequency, low-signal tokens dropped when building
|
||||
// the "related" FTS query so matches key off the story's actual subject matter.
|
||||
var relatedStopwords = map[string]bool{
|
||||
"the": true, "and": true, "for": true, "with": true, "that": true,
|
||||
"this": true, "from": true, "has": true, "have": true, "are": true,
|
||||
"was": true, "were": true, "will": true, "its": true, "into": true,
|
||||
"out": true, "how": true, "why": true, "what": true, "who": true,
|
||||
"you": true, "your": true, "not": true, "but": true, "all": true,
|
||||
"new": true, "now": true, "more": true, "after": true, "over": true,
|
||||
"about": true, "says": true, "said": true, "of": true, "to": true,
|
||||
"in": true, "is": true, "on": true, "at": true, "by": true, "as": true,
|
||||
"an": true, "be": true, "it": true, "or": true, "if": true, "we": true,
|
||||
"he": true, "do": true, "up": true, "so": true, "no": true, "my": true,
|
||||
}
|
||||
|
||||
// RelatedStories returns classified stories textually similar to the given
|
||||
// story, ranked by FTS5 relevance and excluding the story itself. The match
|
||||
// query is an OR of significant tokens from the seed story's headline and lede,
|
||||
// so it surfaces stories that share subject matter rather than requiring every
|
||||
// term (which is what SearchStories' implicit-AND query would demand).
|
||||
func RelatedStories(storyID int64, limit int) ([]Story, error) {
|
||||
if storyID <= 0 || limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var headline, lede sql.NullString
|
||||
err := Get().QueryRow(`SELECT headline, lede FROM stories WHERE id = ?`, storyID).Scan(&headline, &lede)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
fts := buildRelatedFTSQuery(headline.String + " " + lede.String)
|
||||
if fts == "" {
|
||||
return nil, nil
|
||||
}
|
||||
cutoff := nowUnix() - relatedRecencyWindowSeconds
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM stories_fts f
|
||||
JOIN stories s ON s.id = f.rowid
|
||||
WHERE f.stories_fts MATCH ?
|
||||
AND s.id != ?
|
||||
AND s.classified = 1
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')
|
||||
AND s.seen_at >= ?
|
||||
ORDER BY bm25(stories_fts) ASC, s.seen_at DESC
|
||||
LIMIT ?`, fts, storyID, cutoff, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// buildRelatedFTSQuery tokenizes a story's headline+lede into a de-duplicated,
|
||||
// stopword-filtered OR query of quoted terms, capped so a long lede can't build
|
||||
// a pathological query. Returns "" when nothing usable remains.
|
||||
func buildRelatedFTSQuery(raw string) string {
|
||||
var tokens []string
|
||||
var cur strings.Builder
|
||||
seen := make(map[string]bool)
|
||||
flush := func() {
|
||||
if cur.Len() == 0 {
|
||||
return
|
||||
}
|
||||
t := strings.ToLower(cur.String())
|
||||
cur.Reset()
|
||||
if len([]rune(t)) < 2 || relatedStopwords[t] || seen[t] {
|
||||
return
|
||||
}
|
||||
seen[t] = true
|
||||
tokens = append(tokens, t)
|
||||
}
|
||||
for _, r := range raw {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
|
||||
cur.WriteRune(r)
|
||||
case r > 127:
|
||||
cur.WriteRune(r)
|
||||
default:
|
||||
flush()
|
||||
}
|
||||
}
|
||||
flush()
|
||||
if len(tokens) == 0 {
|
||||
return ""
|
||||
}
|
||||
const maxTokens = 16
|
||||
if len(tokens) > maxTokens {
|
||||
tokens = tokens[:maxTokens]
|
||||
}
|
||||
for i, t := range tokens {
|
||||
tokens[i] = `"` + t + `"`
|
||||
}
|
||||
return strings.Join(tokens, " OR ")
|
||||
}
|
||||
141
internal/storage/rank_test.go
Normal file
141
internal/storage/rank_test.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// seedStoryFull inserts a story with an explicit channel + source so ranking
|
||||
// tests can build a skewed affinity profile.
|
||||
func seedStoryFull(t *testing.T, guid, channel, source, headline, lede string) int64 {
|
||||
t.Helper()
|
||||
s := &Story{
|
||||
GUID: guid,
|
||||
Headline: headline,
|
||||
Lede: lede,
|
||||
ArticleURL: "https://example.com/" + guid,
|
||||
Source: source,
|
||||
Channel: channel,
|
||||
Classified: true,
|
||||
SeenAt: nowUnix(),
|
||||
}
|
||||
if err := InsertStory(s); err != nil {
|
||||
t.Fatalf("insert story %s: %v", guid, err)
|
||||
}
|
||||
var id int64
|
||||
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, guid).Scan(&id); err != nil {
|
||||
t.Fatalf("lookup id %s: %v", guid, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestForYou_LeansToAffinityAndExcludesRead(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
const sub = "sub-1"
|
||||
|
||||
// A skewed history: the user reads/bookmarks gaming stories.
|
||||
g1 := seedStoryFull(t, "g1", "gaming", "GameSite", "Game one", "")
|
||||
g2 := seedStoryFull(t, "g2", "gaming", "GameSite", "Game two", "")
|
||||
// Candidates to rank (all unread until we mark some).
|
||||
gc := seedStoryFull(t, "gc", "gaming", "GameSite", "Fresh gaming story", "")
|
||||
tc := seedStoryFull(t, "tc", "tech", "TechSite", "Fresh tech story", "")
|
||||
|
||||
if err := SetRead(sub, g1, true); err != nil {
|
||||
t.Fatalf("read g1: %v", err)
|
||||
}
|
||||
if err := SetBookmark(sub, g2, true); err != nil {
|
||||
t.Fatalf("bookmark g2: %v", err)
|
||||
}
|
||||
|
||||
got, err := ForYou(sub, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ForYou: %v", err)
|
||||
}
|
||||
if len(got) == 0 {
|
||||
t.Fatal("expected results")
|
||||
}
|
||||
// Affinity leans gaming, so a gaming story tops the list and the tech story
|
||||
// sinks to the bottom. (gc and the unread-but-bookmarked g2 both qualify and
|
||||
// tie on score, so we assert on channel rather than a specific id.)
|
||||
if got[0].Channel != "gaming" {
|
||||
t.Fatalf("expected a gaming story first, got channel %q (id %d)", got[0].Channel, got[0].ID)
|
||||
}
|
||||
if got[len(got)-1].ID != tc {
|
||||
t.Fatalf("expected tech candidate ranked last, got id %d", got[len(got)-1].ID)
|
||||
}
|
||||
// Read stories must never appear.
|
||||
for _, s := range got {
|
||||
if s.ID == g1 {
|
||||
t.Fatalf("read story g1 leaked into ForYou")
|
||||
}
|
||||
}
|
||||
// Sanity: the gaming candidate that was never touched should be present.
|
||||
var sawGC bool
|
||||
for _, s := range got {
|
||||
if s.ID == gc {
|
||||
sawGC = true
|
||||
}
|
||||
}
|
||||
if !sawGC {
|
||||
t.Fatalf("expected unread gaming candidate gc in results")
|
||||
}
|
||||
}
|
||||
|
||||
func TestForYou_NoHistoryReturnsNil(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
seedStoryFull(t, "a", "tech", "TechSite", "Something", "")
|
||||
got, err := ForYou("nobody", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("ForYou: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Fatalf("expected nil for a user with no history, got %d rows", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRelatedStories_OnTopicExcludesSelf(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
seed := seedStoryFull(t, "seed", "tech", "TechSite",
|
||||
"Apple unveils new iPhone camera", "The new camera sensor is larger.")
|
||||
rel := seedStoryFull(t, "rel", "tech", "OtherSite",
|
||||
"iPhone camera teardown reveals sensor", "A look at the new iPhone camera.")
|
||||
seedStoryFull(t, "off", "gaming", "GameSite",
|
||||
"Nintendo announces handheld", "A brand new portable console.")
|
||||
|
||||
got, err := RelatedStories(seed, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("RelatedStories: %v", err)
|
||||
}
|
||||
if len(got) == 0 {
|
||||
t.Fatal("expected at least one related story")
|
||||
}
|
||||
for _, s := range got {
|
||||
if s.ID == seed {
|
||||
t.Fatal("seed story returned as its own related")
|
||||
}
|
||||
}
|
||||
if got[0].ID != rel {
|
||||
t.Fatalf("expected the on-topic iPhone story first, got id %d", got[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRelatedFTSQuery(t *testing.T) {
|
||||
// Stopwords and 1-char tokens drop out; the rest become an OR of quoted terms.
|
||||
q := buildRelatedFTSQuery("The new iPhone camera is a big deal")
|
||||
if q == "" {
|
||||
t.Fatal("expected a non-empty query")
|
||||
}
|
||||
for _, bad := range []string{`"the"`, `"is"`, `"new"`} {
|
||||
if strings.Contains(q, bad) {
|
||||
t.Fatalf("stopword leaked into query: %s in %q", bad, q)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{`"iphone"`, `"camera"`} {
|
||||
if !strings.Contains(q, want) {
|
||||
t.Fatalf("expected %s in query %q", want, q)
|
||||
}
|
||||
}
|
||||
if buildRelatedFTSQuery("the a of to") != "" {
|
||||
t.Fatal("expected empty query when only stopwords remain")
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,18 @@ CREATE TABLE IF NOT EXISTS user_preferences (
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Per-user read + bookmark state for signed-in visitors, keyed by OIDC subject.
|
||||
-- One row carries both signals; a NULL timestamp means "not set". A row with
|
||||
-- both timestamps NULL is meaningless and is pruned, so presence of a row means
|
||||
-- the story is read, bookmarked, or both.
|
||||
CREATE TABLE IF NOT EXISTS user_story_state (
|
||||
user_sub TEXT NOT NULL,
|
||||
story_id INTEGER NOT NULL,
|
||||
read_at INTEGER,
|
||||
bookmarked_at INTEGER,
|
||||
PRIMARY KEY (user_sub, story_id)
|
||||
);
|
||||
|
||||
-- Aggregate web usage. page_views holds running view counts keyed by a coarse
|
||||
-- path label ("home", channel slug, …) and the UTC day, so we can report both
|
||||
-- all-time totals and per-day breakdowns without storing any per-request rows.
|
||||
@@ -64,6 +76,35 @@ CREATE TABLE IF NOT EXISTS page_views (
|
||||
PRIMARY KEY (path, day)
|
||||
);
|
||||
|
||||
-- Per-source poll health, one row per configured feed (keyed by source name).
|
||||
-- Written on every poll (success and failure) so the owner-facing dashboard can
|
||||
-- show which feeds are healthy without keeping the poller's in-memory state.
|
||||
-- last_success_at / last_item_count survive failures so a broken feed still
|
||||
-- shows when it last worked and how much it last returned.
|
||||
CREATE TABLE IF NOT EXISTS source_health (
|
||||
source TEXT PRIMARY KEY,
|
||||
last_poll_at INTEGER, -- unix, most recent poll attempt
|
||||
last_success_at INTEGER, -- unix, most recent successful fetch
|
||||
last_error TEXT, -- last failure message ('' when healthy)
|
||||
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||
last_item_count INTEGER NOT NULL DEFAULT 0, -- items in the last successful fetch
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Web Push subscriptions for signed-in users, one row per browser/device
|
||||
-- endpoint. p256dh + auth are the client's encryption keys (RFC 8291); the
|
||||
-- server needs them to encrypt each push. last_notified_at is the per-endpoint
|
||||
-- digest watermark: the sender only counts stories seen after it. A user can
|
||||
-- have several endpoints (phone, desktop) — each is notified independently.
|
||||
CREATE TABLE IF NOT EXISTS push_subscriptions (
|
||||
endpoint TEXT PRIMARY KEY,
|
||||
user_sub TEXT NOT NULL,
|
||||
p256dh TEXT NOT NULL,
|
||||
auth TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
last_notified_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Privacy-preserving daily unique estimate. visitor is a salted hash of
|
||||
-- IP+User-Agent; the salt rotates every UTC day and is never persisted, so the
|
||||
-- hashes are irreversible and cannot be linked across days. We keep only enough
|
||||
@@ -88,6 +129,9 @@ CREATE INDEX IF NOT EXISTS idx_reactions_post_guid ON reactions(post_guid);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_reactions_event_id ON reactions(event_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_page_views_day ON page_views(day);
|
||||
CREATE INDEX IF NOT EXISTS idx_daily_visitors_day ON daily_visitors(day);
|
||||
CREATE INDEX IF NOT EXISTS idx_user_state_bookmarks ON user_story_state(user_sub, bookmarked_at) WHERE bookmarked_at IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_user_state_reads ON user_story_state(user_sub, read_at) WHERE read_at IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_push_sub_user ON push_subscriptions(user_sub);
|
||||
`
|
||||
|
||||
const ftsSchema = `
|
||||
|
||||
149
internal/storage/sourcehealth.go
Normal file
149
internal/storage/sourcehealth.go
Normal file
@@ -0,0 +1,149 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// SourceHealth is one source's persisted poll health, as written by the poller.
|
||||
type SourceHealth struct {
|
||||
Source string
|
||||
LastPollAt int64 // 0 = never polled
|
||||
LastSuccessAt int64 // 0 = never succeeded
|
||||
LastError string
|
||||
ConsecutiveFailures int
|
||||
LastItemCount int
|
||||
UpdatedAt int64
|
||||
}
|
||||
|
||||
// SourceContentStat is per-source content derived from the stories table (plus
|
||||
// post_log for last-posted), independent of poll health.
|
||||
type SourceContentStat struct {
|
||||
Source string
|
||||
Total int // stories currently retained for this source
|
||||
Classified int // of those, how many are classified
|
||||
Paywalled int // of those, how many are gated
|
||||
LastSeenAt int64 // MAX(seen_at); 0 = none
|
||||
LastPostedAt int64 // MAX(post_log.posted_at) joined by guid; 0 = never posted
|
||||
}
|
||||
|
||||
// RecordPollResult upserts a source's health row after a poll attempt. On
|
||||
// success it clears the error and failure counter and records the item count;
|
||||
// on failure it bumps consecutive_failures and stores the message while
|
||||
// preserving the last successful timestamp and item count. Fire-and-forget:
|
||||
// a failure here must never disrupt polling, so errors are logged and swallowed.
|
||||
func RecordPollResult(source string, ok bool, itemCount int, pollErr error) {
|
||||
now := nowUnix()
|
||||
if ok {
|
||||
exec("record poll success",
|
||||
`INSERT INTO source_health
|
||||
(source, last_poll_at, last_success_at, last_error, consecutive_failures, last_item_count, updated_at)
|
||||
VALUES (?, ?, ?, '', 0, ?, ?)
|
||||
ON CONFLICT(source) DO UPDATE SET
|
||||
last_poll_at = excluded.last_poll_at,
|
||||
last_success_at = excluded.last_success_at,
|
||||
last_error = '',
|
||||
consecutive_failures = 0,
|
||||
last_item_count = excluded.last_item_count,
|
||||
updated_at = excluded.updated_at`,
|
||||
source, now, now, itemCount, now)
|
||||
return
|
||||
}
|
||||
msg := ""
|
||||
if pollErr != nil {
|
||||
msg = pollErr.Error()
|
||||
}
|
||||
exec("record poll failure",
|
||||
`INSERT INTO source_health
|
||||
(source, last_poll_at, last_success_at, last_error, consecutive_failures, last_item_count, updated_at)
|
||||
VALUES (?, ?, NULL, ?, 1, 0, ?)
|
||||
ON CONFLICT(source) DO UPDATE SET
|
||||
last_poll_at = excluded.last_poll_at,
|
||||
last_error = excluded.last_error,
|
||||
consecutive_failures = source_health.consecutive_failures + 1,
|
||||
updated_at = excluded.updated_at`,
|
||||
source, now, msg, now)
|
||||
}
|
||||
|
||||
// ListSourceHealth returns the poll-health row for every source that has been
|
||||
// polled at least once, keyed by source name.
|
||||
func ListSourceHealth() (map[string]SourceHealth, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT source, last_poll_at, last_success_at, last_error,
|
||||
consecutive_failures, last_item_count, updated_at
|
||||
FROM source_health`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list source health: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]SourceHealth)
|
||||
for rows.Next() {
|
||||
var h SourceHealth
|
||||
var lastPoll, lastSuccess sql.NullInt64
|
||||
var lastErr sql.NullString
|
||||
if err := rows.Scan(&h.Source, &lastPoll, &lastSuccess, &lastErr,
|
||||
&h.ConsecutiveFailures, &h.LastItemCount, &h.UpdatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h.LastPollAt = lastPoll.Int64
|
||||
h.LastSuccessAt = lastSuccess.Int64
|
||||
h.LastError = lastErr.String
|
||||
out[h.Source] = h
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SourceContentStats derives per-source content counts from the stories table,
|
||||
// with last-posted joined from post_log by guid. Keyed by source name. Only
|
||||
// sources with at least one retained story appear; the caller pads out the rest
|
||||
// from its configured source list.
|
||||
func SourceContentStats() (map[string]SourceContentStat, error) {
|
||||
out := make(map[string]SourceContentStat)
|
||||
|
||||
rows, err := Get().Query(
|
||||
`SELECT source,
|
||||
COUNT(*),
|
||||
COALESCE(SUM(classified), 0),
|
||||
COALESCE(SUM(paywalled), 0),
|
||||
COALESCE(MAX(seen_at), 0)
|
||||
FROM stories
|
||||
GROUP BY source`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("source content stats: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var st SourceContentStat
|
||||
if err := rows.Scan(&st.Source, &st.Total, &st.Classified, &st.Paywalled, &st.LastSeenAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[st.Source] = st
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Last-posted per source, joined by guid. Kept separate so sources with
|
||||
// stories but no posts still appear above with a zero last-posted.
|
||||
prows, err := Get().Query(
|
||||
`SELECT s.source, MAX(p.posted_at)
|
||||
FROM post_log p
|
||||
JOIN stories s ON s.guid = p.guid
|
||||
GROUP BY s.source`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("source last-posted: %w", err)
|
||||
}
|
||||
defer prows.Close()
|
||||
for prows.Next() {
|
||||
var source string
|
||||
var lastPosted int64
|
||||
if err := prows.Scan(&source, &lastPosted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
st := out[source]
|
||||
st.Source = source
|
||||
st.LastPostedAt = lastPosted
|
||||
out[source] = st
|
||||
}
|
||||
return out, prows.Err()
|
||||
}
|
||||
104
internal/storage/sourcehealth_test.go
Normal file
104
internal/storage/sourcehealth_test.go
Normal file
@@ -0,0 +1,104 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRecordPollResult_SuccessThenFailure(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
// First a successful poll returning 7 items.
|
||||
RecordPollResult("Feed A", true, 7, nil)
|
||||
h, err := ListSourceHealth()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, ok := h["Feed A"]
|
||||
if !ok {
|
||||
t.Fatal("expected a health row for Feed A")
|
||||
}
|
||||
if a.LastItemCount != 7 || a.ConsecutiveFailures != 0 || a.LastError != "" {
|
||||
t.Errorf("after success: items=%d fails=%d err=%q", a.LastItemCount, a.ConsecutiveFailures, a.LastError)
|
||||
}
|
||||
if a.LastPollAt == 0 || a.LastSuccessAt == 0 {
|
||||
t.Errorf("after success: last_poll=%d last_success=%d, want both set", a.LastPollAt, a.LastSuccessAt)
|
||||
}
|
||||
successTS := a.LastSuccessAt
|
||||
|
||||
// Two failures in a row: failures accumulate, but the last-success timestamp
|
||||
// and item count are preserved so the dashboard still shows when it last worked.
|
||||
RecordPollResult("Feed A", false, 0, errors.New("dial tcp: timeout"))
|
||||
RecordPollResult("Feed A", false, 0, errors.New("502 bad gateway"))
|
||||
h, _ = ListSourceHealth()
|
||||
a = h["Feed A"]
|
||||
if a.ConsecutiveFailures != 2 {
|
||||
t.Errorf("consecutive failures = %d, want 2", a.ConsecutiveFailures)
|
||||
}
|
||||
if a.LastError != "502 bad gateway" {
|
||||
t.Errorf("last error = %q, want %q", a.LastError, "502 bad gateway")
|
||||
}
|
||||
if a.LastSuccessAt != successTS {
|
||||
t.Errorf("last success = %d, want preserved %d", a.LastSuccessAt, successTS)
|
||||
}
|
||||
if a.LastItemCount != 7 {
|
||||
t.Errorf("last item count = %d, want preserved 7", a.LastItemCount)
|
||||
}
|
||||
|
||||
// Recovery clears the error and resets the counter.
|
||||
RecordPollResult("Feed A", true, 3, nil)
|
||||
h, _ = ListSourceHealth()
|
||||
a = h["Feed A"]
|
||||
if a.ConsecutiveFailures != 0 || a.LastError != "" || a.LastItemCount != 3 {
|
||||
t.Errorf("after recovery: fails=%d err=%q items=%d", a.ConsecutiveFailures, a.LastError, a.LastItemCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceContentStats(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
// Source X: two classified stories, one paywalled; one posted.
|
||||
InsertStory(&Story{GUID: "x1", Headline: "X one", ArticleURL: "https://x.com/1", Source: "X", Channel: "tech", Classified: true, SeenAt: 100})
|
||||
InsertStory(&Story{GUID: "x2", Headline: "X two", ArticleURL: "https://x.com/2", Source: "X", Channel: "tech", Classified: true, Paywalled: true, SeenAt: 200})
|
||||
// Source Y: one unclassified story, never posted.
|
||||
InsertStory(&Story{GUID: "y1", Headline: "Y one", ArticleURL: "https://y.com/1", Source: "Y", SeenAt: 150})
|
||||
|
||||
InsertPostLog("x1", "tech", "$e1", "", false)
|
||||
// Backdate/forward the post so we can assert MAX(posted_at).
|
||||
Get().Exec(`UPDATE post_log SET posted_at = ? WHERE guid = ?`, int64(500), "x1")
|
||||
|
||||
stats, err := SourceContentStats()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
x := stats["X"]
|
||||
if x.Total != 2 || x.Classified != 2 || x.Paywalled != 1 {
|
||||
t.Errorf("X: total=%d classified=%d paywalled=%d, want 2/2/1", x.Total, x.Classified, x.Paywalled)
|
||||
}
|
||||
if x.LastSeenAt != 200 {
|
||||
t.Errorf("X last seen = %d, want 200", x.LastSeenAt)
|
||||
}
|
||||
if x.LastPostedAt != 500 {
|
||||
t.Errorf("X last posted = %d, want 500", x.LastPostedAt)
|
||||
}
|
||||
|
||||
y := stats["Y"]
|
||||
if y.Total != 1 || y.Classified != 0 || y.Paywalled != 0 {
|
||||
t.Errorf("Y: total=%d classified=%d paywalled=%d, want 1/0/0", y.Total, y.Classified, y.Paywalled)
|
||||
}
|
||||
if y.LastPostedAt != 0 {
|
||||
t.Errorf("Y last posted = %d, want 0 (never posted)", y.LastPostedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSourceHealth_Empty(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
h, err := ListSourceHealth()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(h) != 0 {
|
||||
t.Errorf("expected no health rows, got %d", len(h))
|
||||
}
|
||||
}
|
||||
139
internal/storage/userstate.go
Normal file
139
internal/storage/userstate.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// SetRead marks (read=true) or clears (read=false) the read flag for one story
|
||||
// and one signed-in user (keyed by OIDC subject). Read and bookmark state share
|
||||
// a row; clearing the last remaining flag removes the row.
|
||||
func SetRead(sub string, storyID int64, read bool) error {
|
||||
var ts any
|
||||
if read {
|
||||
ts = nowUnix()
|
||||
}
|
||||
_, err := Get().Exec(`
|
||||
INSERT INTO user_story_state (user_sub, story_id, read_at, bookmarked_at)
|
||||
VALUES (?, ?, ?, NULL)
|
||||
ON CONFLICT(user_sub, story_id) DO UPDATE SET read_at = excluded.read_at`,
|
||||
sub, storyID, ts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("set read: %w", err)
|
||||
}
|
||||
return pruneEmptyState(sub, storyID)
|
||||
}
|
||||
|
||||
// SetBookmark adds (on=true) or removes (on=false) a bookmark for one story and
|
||||
// one signed-in user. See SetRead for the shared-row semantics.
|
||||
func SetBookmark(sub string, storyID int64, on bool) error {
|
||||
var ts any
|
||||
if on {
|
||||
ts = nowUnix()
|
||||
}
|
||||
_, err := Get().Exec(`
|
||||
INSERT INTO user_story_state (user_sub, story_id, read_at, bookmarked_at)
|
||||
VALUES (?, ?, NULL, ?)
|
||||
ON CONFLICT(user_sub, story_id) DO UPDATE SET bookmarked_at = excluded.bookmarked_at`,
|
||||
sub, storyID, ts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("set bookmark: %w", err)
|
||||
}
|
||||
return pruneEmptyState(sub, storyID)
|
||||
}
|
||||
|
||||
// pruneEmptyState drops a row once neither flag is set, keeping the table to
|
||||
// only meaningful state.
|
||||
func pruneEmptyState(sub string, storyID int64) error {
|
||||
_, err := Get().Exec(
|
||||
`DELETE FROM user_story_state
|
||||
WHERE user_sub = ? AND story_id = ? AND read_at IS NULL AND bookmarked_at IS NULL`,
|
||||
sub, storyID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prune user state: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UserStoryState reports, for the given story ids, which are read and which are
|
||||
// bookmarked by this user. Both maps contain only ids whose flag is set, so a
|
||||
// missing key means false. An empty sub or id list returns empty maps.
|
||||
func UserStoryState(sub string, ids []int64) (read, bookmarked map[int64]bool, err error) {
|
||||
read = make(map[int64]bool)
|
||||
bookmarked = make(map[int64]bool)
|
||||
if sub == "" || len(ids) == 0 {
|
||||
return read, bookmarked, nil
|
||||
}
|
||||
q := `SELECT story_id, read_at, bookmarked_at FROM user_story_state
|
||||
WHERE user_sub = ? AND story_id IN (` + placeholders(len(ids)) + `)`
|
||||
args := make([]any, 0, len(ids)+1)
|
||||
args = append(args, sub)
|
||||
for _, id := range ids {
|
||||
args = append(args, id)
|
||||
}
|
||||
rows, err := Get().Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("user story state: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var r, b sql.NullInt64
|
||||
if err := rows.Scan(&id, &r, &b); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if r.Valid {
|
||||
read[id] = true
|
||||
}
|
||||
if b.Valid {
|
||||
bookmarked[id] = true
|
||||
}
|
||||
}
|
||||
return read, bookmarked, rows.Err()
|
||||
}
|
||||
|
||||
// ListBookmarks returns the user's bookmarked stories, most recently bookmarked
|
||||
// first, restricted to still-classified stories.
|
||||
func ListBookmarks(sub string, limit, offset int) ([]Story, error) {
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM user_story_state u
|
||||
JOIN stories s ON s.id = u.story_id
|
||||
WHERE u.user_sub = ?
|
||||
AND u.bookmarked_at IS NOT NULL
|
||||
AND s.classified = 1
|
||||
ORDER BY u.bookmarked_at DESC, u.story_id DESC
|
||||
LIMIT ? OFFSET ?`, sub, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// CountBookmarks returns how many stories the user has bookmarked.
|
||||
func CountBookmarks(sub string) (int, error) {
|
||||
var n int
|
||||
err := Get().QueryRow(
|
||||
`SELECT COUNT(*) FROM user_story_state WHERE user_sub = ? AND bookmarked_at IS NOT NULL`,
|
||||
sub).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// placeholders returns "?, ?, …" with n slots for an IN clause.
|
||||
func placeholders(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Repeat("?, ", n-1) + "?"
|
||||
}
|
||||
126
internal/storage/userstate_test.go
Normal file
126
internal/storage/userstate_test.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package storage
|
||||
|
||||
import "testing"
|
||||
|
||||
func seedStory(t *testing.T, guid string) int64 {
|
||||
t.Helper()
|
||||
s := &Story{
|
||||
GUID: guid,
|
||||
Headline: "Headline " + guid,
|
||||
Lede: "Lede.",
|
||||
ArticleURL: "https://example.com/" + guid,
|
||||
Source: "Test Source",
|
||||
Channel: "tech",
|
||||
Classified: true,
|
||||
SeenAt: 1700000000,
|
||||
}
|
||||
if err := InsertStory(s); err != nil {
|
||||
t.Fatalf("insert story %s: %v", guid, err)
|
||||
}
|
||||
var id int64
|
||||
if err := Get().QueryRow(`SELECT id FROM stories WHERE guid = ?`, guid).Scan(&id); err != nil {
|
||||
t.Fatalf("lookup id %s: %v", guid, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestUserStoryState_ReadAndBookmark(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
id1 := seedStory(t, "s1")
|
||||
id2 := seedStory(t, "s2")
|
||||
const sub = "ak-sub-1"
|
||||
|
||||
// Nothing set yet.
|
||||
read, marks, err := UserStoryState(sub, []int64{id1, id2})
|
||||
if err != nil {
|
||||
t.Fatalf("initial state: %v", err)
|
||||
}
|
||||
if len(read) != 0 || len(marks) != 0 {
|
||||
t.Fatalf("expected empty state, got read=%v bookmarked=%v", read, marks)
|
||||
}
|
||||
|
||||
// Mark id1 read, bookmark id1 and id2.
|
||||
if err := SetRead(sub, id1, true); err != nil {
|
||||
t.Fatalf("set read: %v", err)
|
||||
}
|
||||
if err := SetBookmark(sub, id1, true); err != nil {
|
||||
t.Fatalf("set bookmark id1: %v", err)
|
||||
}
|
||||
if err := SetBookmark(sub, id2, true); err != nil {
|
||||
t.Fatalf("set bookmark id2: %v", err)
|
||||
}
|
||||
read, marks, _ = UserStoryState(sub, []int64{id1, id2})
|
||||
if !read[id1] || read[id2] {
|
||||
t.Fatalf("read map wrong: %v", read)
|
||||
}
|
||||
if !marks[id1] || !marks[id2] {
|
||||
t.Fatalf("bookmark map wrong: %v", marks)
|
||||
}
|
||||
// id1 carries both flags in a single row.
|
||||
if !read[id1] || !marks[id1] {
|
||||
t.Fatalf("expected id1 read+bookmarked, read=%v marks=%v", read, marks)
|
||||
}
|
||||
|
||||
// ListBookmarks orders newest bookmark first, tiebreaking on story id so the
|
||||
// result is deterministic within a single second.
|
||||
bm, err := ListBookmarks(sub, 10, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("list bookmarks: %v", err)
|
||||
}
|
||||
if len(bm) != 2 {
|
||||
t.Fatalf("expected 2 bookmarks, got %d", len(bm))
|
||||
}
|
||||
if bm[0].ID != id2 || bm[1].ID != id1 {
|
||||
t.Fatalf("bookmark order wrong: %d then %d", bm[0].ID, bm[1].ID)
|
||||
}
|
||||
if n, _ := CountBookmarks(sub); n != 2 {
|
||||
t.Fatalf("count bookmarks = %d, want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserStoryState_ClearRemovesRow(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
id := seedStory(t, "s1")
|
||||
const sub = "ak-sub-1"
|
||||
|
||||
if err := SetRead(sub, id, true); err != nil {
|
||||
t.Fatalf("set read: %v", err)
|
||||
}
|
||||
if err := SetRead(sub, id, false); err != nil {
|
||||
t.Fatalf("unset read: %v", err)
|
||||
}
|
||||
// Row should be gone since neither flag is set.
|
||||
var n int
|
||||
if err := Get().QueryRow(`SELECT COUNT(*) FROM user_story_state WHERE user_sub = ?`, sub).Scan(&n); err != nil {
|
||||
t.Fatalf("count rows: %v", err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Fatalf("expected row pruned, found %d", n)
|
||||
}
|
||||
|
||||
// Setting read then bookmark then clearing only read must keep the row.
|
||||
_ = SetRead(sub, id, true)
|
||||
_ = SetBookmark(sub, id, true)
|
||||
_ = SetRead(sub, id, false)
|
||||
read, marks, _ := UserStoryState(sub, []int64{id})
|
||||
if read[id] {
|
||||
t.Fatalf("expected not read")
|
||||
}
|
||||
if !marks[id] {
|
||||
t.Fatalf("expected still bookmarked after clearing read")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserStoryState_ScopedBySub(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
id := seedStory(t, "s1")
|
||||
_ = SetBookmark("user-a", id, true)
|
||||
|
||||
read, marks, _ := UserStoryState("user-b", []int64{id})
|
||||
if len(read) != 0 || len(marks) != 0 {
|
||||
t.Fatalf("user-b should see no state, got read=%v marks=%v", read, marks)
|
||||
}
|
||||
if n, _ := CountBookmarks("user-b"); n != 0 {
|
||||
t.Fatalf("user-b count = %d, want 0", n)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user