Add personalization, outbound feeds, and PWA/push to the web UI
A multi-session build turning Pete's read-only web UI into something people return to. Five phases, signed-in features keyed off the OIDC subject; anonymous visitors keep the reverse-chron feed and localStorage-only state. Phase 1 — per-user read + bookmark state: user_story_state table + storage/userstate.go; auth-gated /api/read, /api/bookmark, /api/state and a /bookmarks page; reader.js syncs state server-side for signed-in users. Also hides the Matrix-posting UI when posting.enabled=false (web-only mode). Phase 2 — outbound feeds: storage.ListForFeed + web/feed.go hand-build RSS 2.0 (content:encoded) and JSON Feed 1.1 (no new dep); /feed.xml, /feed.json and per-channel variants; <link rel=alternate> discovery tags. Phase 3 — "For you" + related: storage/rank.go scores recent unread candidates by channel/source affinity + recency decay; RelatedStories via FTS5. ForYou rail + /for-you page; public /api/related feeds the reader's "You might also like". Phase 4 — source-health dashboard: source_health table + storage/sourcehealth.go (RecordPollResult, ListSourceHealth, SourceContentStats), written by the poller; admin-gated /status page behind web.admin_subs. Phase 5 — PWA + offline reader + Web Push: root-scoped manifest.webmanifest and sw.js (app-shell precache, /api/article runtime cache for offline reading, offline fallback, push/notificationclick handlers); PNG icons from pete.avif; pwa.js registers the SW and drives a notifications toggle. Web Push adds webpush-go, a [web.push] config block (pete -genvapid mints VAPID keys), a push_subscriptions table, auth-gated subscribe/unsubscribe endpoints, and a digest sender that pings each subscriber "N new stories" past their watermark, honoring disabled-sources and pruning gone endpoints. Tests added beside each new storage/web file; go test ./... and go vet clean.
This commit is contained in:
prosolis
275
internal/storage/rank.go
Normal file
275
internal/storage/rank.go
Normal file
@@ -0,0 +1,275 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Ranking weights for the "For you" feed. Bookmarks are a stronger signal of
|
||||
// interest than a plain read, and channel affinity outweighs source affinity
|
||||
// (a reader follows topics more than outlets). Recency keeps the feed fresh so
|
||||
// a heavily-read channel doesn't surface week-old stories over today's news.
|
||||
const (
|
||||
affinityReadWeight = 1.0
|
||||
affinityBookmarkWeight = 3.0
|
||||
|
||||
forYouChannelWeight = 1.0
|
||||
forYouSourceWeight = 0.7
|
||||
forYouRecencyWeight = 0.9
|
||||
|
||||
// forYouCandidatePool bounds how many recent unread stories we score in Go.
|
||||
forYouCandidatePool = 400
|
||||
// forYouRecencyHalfLifeHours sets how fast the recency term decays.
|
||||
forYouRecencyHalfLifeHours = 48.0
|
||||
)
|
||||
|
||||
// userAffinity builds normalized channel and source affinity scores (each 0..1)
|
||||
// for a signed-in user from their read + bookmark history. Bookmarks count for
|
||||
// more than plain reads. total is the number of signal-bearing rows; when it is
|
||||
// zero the user has no history yet and both maps are empty.
|
||||
func userAffinity(sub string) (channel, source map[string]float64, total int, err error) {
|
||||
channel = make(map[string]float64)
|
||||
source = make(map[string]float64)
|
||||
if sub == "" {
|
||||
return channel, source, 0, nil
|
||||
}
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.channel, s.source, u.read_at, u.bookmarked_at
|
||||
FROM user_story_state u
|
||||
JOIN stories s ON s.id = u.story_id
|
||||
WHERE u.user_sub = ?
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')`, sub)
|
||||
if err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var ch, src string
|
||||
var readAt, markAt sql.NullInt64
|
||||
if err := rows.Scan(&ch, &src, &readAt, &markAt); err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
w := 0.0
|
||||
if readAt.Valid {
|
||||
w += affinityReadWeight
|
||||
}
|
||||
if markAt.Valid {
|
||||
w += affinityBookmarkWeight
|
||||
}
|
||||
if w == 0 {
|
||||
continue
|
||||
}
|
||||
channel[ch] += w
|
||||
source[src] += w
|
||||
total++
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, nil, 0, err
|
||||
}
|
||||
normalizeMax(channel)
|
||||
normalizeMax(source)
|
||||
return channel, source, total, nil
|
||||
}
|
||||
|
||||
// normalizeMax scales a map's values so the largest becomes 1.0, leaving an
|
||||
// all-zero (or empty) map untouched.
|
||||
func normalizeMax(m map[string]float64) {
|
||||
var max float64
|
||||
for _, v := range m {
|
||||
if v > max {
|
||||
max = v
|
||||
}
|
||||
}
|
||||
if max == 0 {
|
||||
return
|
||||
}
|
||||
for k := range m {
|
||||
m[k] /= max
|
||||
}
|
||||
}
|
||||
|
||||
// ForYou returns a personalized ranking of recent, unread stories for a
|
||||
// signed-in user, blending channel affinity, source affinity, and recency. It
|
||||
// returns (nil, nil) when the user has no read/bookmark history yet, so callers
|
||||
// can fall back to the plain latest feed.
|
||||
func ForYou(sub string, limit int) ([]Story, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
chAff, srcAff, total, err := userAffinity(sub)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if total == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at, COALESCE(s.published_at, s.seen_at),
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM stories s
|
||||
WHERE s.classified = 1
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')
|
||||
AND s.id NOT IN (
|
||||
SELECT story_id FROM user_story_state
|
||||
WHERE user_sub = ? AND read_at IS NOT NULL)
|
||||
ORDER BY COALESCE(s.published_at, s.seen_at) DESC
|
||||
LIMIT ?`, sub, forYouCandidatePool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
now := float64(nowUnix())
|
||||
type scored struct {
|
||||
s Story
|
||||
score float64
|
||||
}
|
||||
var cands []scored
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
var effectiveAt int64
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &effectiveAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ageHours := (now - float64(effectiveAt)) / 3600.0
|
||||
if ageHours < 0 {
|
||||
ageHours = 0
|
||||
}
|
||||
recency := math.Exp(-ageHours / forYouRecencyHalfLifeHours)
|
||||
score := forYouChannelWeight*chAff[s.Channel] +
|
||||
forYouSourceWeight*srcAff[s.Source] +
|
||||
forYouRecencyWeight*recency
|
||||
cands = append(cands, scored{s, score})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Candidates arrive newest-first, and a stable sort keeps that order for
|
||||
// equal scores, so ties break toward the more recent story.
|
||||
sort.SliceStable(cands, func(i, j int) bool { return cands[i].score > cands[j].score })
|
||||
if len(cands) > limit {
|
||||
cands = cands[:limit]
|
||||
}
|
||||
out := make([]Story, 0, len(cands))
|
||||
for _, c := range cands {
|
||||
out = append(out, c.s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// relatedRecencyWindowSeconds bounds "related" results to roughly the same
|
||||
// window the story feed lives in (stories are pruned at 30 days anyway).
|
||||
const relatedRecencyWindowSeconds = 30 * 86400
|
||||
|
||||
// relatedStopwords are high-frequency, low-signal tokens dropped when building
|
||||
// the "related" FTS query so matches key off the story's actual subject matter.
|
||||
var relatedStopwords = map[string]bool{
|
||||
"the": true, "and": true, "for": true, "with": true, "that": true,
|
||||
"this": true, "from": true, "has": true, "have": true, "are": true,
|
||||
"was": true, "were": true, "will": true, "its": true, "into": true,
|
||||
"out": true, "how": true, "why": true, "what": true, "who": true,
|
||||
"you": true, "your": true, "not": true, "but": true, "all": true,
|
||||
"new": true, "now": true, "more": true, "after": true, "over": true,
|
||||
"about": true, "says": true, "said": true, "of": true, "to": true,
|
||||
"in": true, "is": true, "on": true, "at": true, "by": true, "as": true,
|
||||
"an": true, "be": true, "it": true, "or": true, "if": true, "we": true,
|
||||
"he": true, "do": true, "up": true, "so": true, "no": true, "my": true,
|
||||
}
|
||||
|
||||
// RelatedStories returns classified stories textually similar to the given
|
||||
// story, ranked by FTS5 relevance and excluding the story itself. The match
|
||||
// query is an OR of significant tokens from the seed story's headline and lede,
|
||||
// so it surfaces stories that share subject matter rather than requiring every
|
||||
// term (which is what SearchStories' implicit-AND query would demand).
|
||||
func RelatedStories(storyID int64, limit int) ([]Story, error) {
|
||||
if storyID <= 0 || limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var headline, lede sql.NullString
|
||||
err := Get().QueryRow(`SELECT headline, lede FROM stories WHERE id = ?`, storyID).Scan(&headline, &lede)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
fts := buildRelatedFTSQuery(headline.String + " " + lede.String)
|
||||
if fts == "" {
|
||||
return nil, nil
|
||||
}
|
||||
cutoff := nowUnix() - relatedRecencyWindowSeconds
|
||||
rows, err := Get().Query(
|
||||
`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
|
||||
EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
|
||||
FROM stories_fts f
|
||||
JOIN stories s ON s.id = f.rowid
|
||||
WHERE f.stories_fts MATCH ?
|
||||
AND s.id != ?
|
||||
AND s.classified = 1
|
||||
AND s.channel IS NOT NULL
|
||||
AND s.channel NOT IN ('_discarded', '_duplicate')
|
||||
AND s.seen_at >= ?
|
||||
ORDER BY bm25(stories_fts) ASC, s.seen_at DESC
|
||||
LIMIT ?`, fts, storyID, cutoff, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Story
|
||||
for rows.Next() {
|
||||
var s Story
|
||||
if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// buildRelatedFTSQuery tokenizes a story's headline+lede into a de-duplicated,
|
||||
// stopword-filtered OR query of quoted terms, capped so a long lede can't build
|
||||
// a pathological query. Returns "" when nothing usable remains.
|
||||
func buildRelatedFTSQuery(raw string) string {
|
||||
var tokens []string
|
||||
var cur strings.Builder
|
||||
seen := make(map[string]bool)
|
||||
flush := func() {
|
||||
if cur.Len() == 0 {
|
||||
return
|
||||
}
|
||||
t := strings.ToLower(cur.String())
|
||||
cur.Reset()
|
||||
if len([]rune(t)) < 2 || relatedStopwords[t] || seen[t] {
|
||||
return
|
||||
}
|
||||
seen[t] = true
|
||||
tokens = append(tokens, t)
|
||||
}
|
||||
for _, r := range raw {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
|
||||
cur.WriteRune(r)
|
||||
case r > 127:
|
||||
cur.WriteRune(r)
|
||||
default:
|
||||
flush()
|
||||
}
|
||||
}
|
||||
flush()
|
||||
if len(tokens) == 0 {
|
||||
return ""
|
||||
}
|
||||
const maxTokens = 16
|
||||
if len(tokens) > maxTokens {
|
||||
tokens = tokens[:maxTokens]
|
||||
}
|
||||
for i, t := range tokens {
|
||||
tokens[i] = `"` + t + `"`
|
||||
}
|
||||
return strings.Join(tokens, " OR ")
|
||||
}
|
||||
Reference in New Issue
Block a user