Pete/internal/storage/rank.go

package storage

import (
	"database/sql"
	"errors"
	"math"
	"sort"
	"strings"
)

// Ranking weights for the "For you" feed. Bookmarks are a stronger signal of
// interest than a plain read, and channel affinity outweighs source affinity
// (a reader follows topics more than outlets). Recency keeps the feed fresh so
// a heavily-read channel doesn't surface week-old stories over today's news.
const (
	affinityReadWeight     = 1.0
	affinityBookmarkWeight = 3.0

	forYouChannelWeight = 1.0
	forYouSourceWeight  = 0.7
	forYouRecencyWeight = 0.9

	// forYouCandidatePool bounds how many recent unread stories we score in Go.
	forYouCandidatePool = 400
	// forYouRecencyHalfLifeHours sets how fast the recency term decays.
	forYouRecencyHalfLifeHours = 48.0
)

// userAffinity builds normalized channel and source affinity scores (each 0..1)
// for a signed-in user from their read + bookmark history. Bookmarks count for
// more than plain reads. total is the number of signal-bearing rows; when it is
// zero the user has no history yet and both maps are empty.
func userAffinity(sub string) (channel, source map[string]float64, total int, err error) {
	channel = make(map[string]float64)
	source = make(map[string]float64)
	if sub == "" {
		return channel, source, 0, nil
	}
	rows, err := Get().Query(
		`SELECT s.channel, s.source, u.read_at, u.bookmarked_at
		 FROM user_story_state u
		 JOIN stories s ON s.id = u.story_id
		 WHERE u.user_sub = ?
		   AND s.channel IS NOT NULL
		   AND s.channel NOT IN ('_discarded', '_duplicate')`, sub)
	if err != nil {
		return nil, nil, 0, err
	}
	defer rows.Close()
	for rows.Next() {
		var ch, src string
		var readAt, markAt sql.NullInt64
		if err := rows.Scan(&ch, &src, &readAt, &markAt); err != nil {
			return nil, nil, 0, err
		}
		w := 0.0
		if readAt.Valid {
			w += affinityReadWeight
		}
		if markAt.Valid {
			w += affinityBookmarkWeight
		}
		if w == 0 {
			continue
		}
		channel[ch] += w
		source[src] += w
		total++
	}
	if err := rows.Err(); err != nil {
		return nil, nil, 0, err
	}
	normalizeMax(channel)
	normalizeMax(source)
	return channel, source, total, nil
}

// normalizeMax scales a map's values so the largest becomes 1.0, leaving an
// all-zero (or empty) map untouched.
func normalizeMax(m map[string]float64) {
	var max float64
	for _, v := range m {
		if v > max {
			max = v
		}
	}
	if max == 0 {
		return
	}
	for k := range m {
		m[k] /= max
	}
}

// ForYou returns a personalized ranking of recent, unread stories for a
// signed-in user, blending channel affinity, source affinity, and recency. It
// returns (nil, nil) when the user has no read/bookmark history yet, so callers
// can fall back to the plain latest feed.
func ForYou(sub string, limit int) ([]Story, error) {
	if limit <= 0 {
		return nil, nil
	}
	chAff, srcAff, total, err := userAffinity(sub)
	if err != nil {
		return nil, err
	}
	if total == 0 {
		return nil, nil
	}
	rows, err := Get().Query(
		`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at, COALESCE(s.published_at, s.seen_at),
		        EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
		 FROM stories s
		 WHERE s.classified = 1
		   AND s.channel IS NOT NULL
		   AND s.channel NOT IN ('_discarded', '_duplicate')
		   AND s.id NOT IN (
		        SELECT story_id FROM user_story_state
		        WHERE user_sub = ? AND read_at IS NOT NULL)
		 ORDER BY COALESCE(s.published_at, s.seen_at) DESC
		 LIMIT ?`, sub, forYouCandidatePool)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	now := float64(nowUnix())
	type scored struct {
		s     Story
		score float64
	}
	var cands []scored
	for rows.Next() {
		var s Story
		var effectiveAt int64
		if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &effectiveAt, &s.Posted); err != nil {
			return nil, err
		}
		ageHours := (now - float64(effectiveAt)) / 3600.0
		if ageHours < 0 {
			ageHours = 0
		}
		recency := math.Exp(-ageHours / forYouRecencyHalfLifeHours)
		score := forYouChannelWeight*chAff[s.Channel] +
			forYouSourceWeight*srcAff[s.Source] +
			forYouRecencyWeight*recency
		cands = append(cands, scored{s, score})
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	// Candidates arrive newest-first, and a stable sort keeps that order for
	// equal scores, so ties break toward the more recent story.
	sort.SliceStable(cands, func(i, j int) bool { return cands[i].score > cands[j].score })
	if len(cands) > limit {
		cands = cands[:limit]
	}
	out := make([]Story, 0, len(cands))
	for _, c := range cands {
		out = append(out, c.s)
	}
	return out, nil
}

// relatedRecencyWindowSeconds bounds "related" results to roughly the same
// window the story feed lives in (stories are pruned at 30 days anyway).
const relatedRecencyWindowSeconds = 30 * 86400

// relatedStopwords are high-frequency, low-signal tokens dropped when building
// the "related" FTS query so matches key off the story's actual subject matter.
var relatedStopwords = map[string]bool{
	"the": true, "and": true, "for": true, "with": true, "that": true,
	"this": true, "from": true, "has": true, "have": true, "are": true,
	"was": true, "were": true, "will": true, "its": true, "into": true,
	"out": true, "how": true, "why": true, "what": true, "who": true,
	"you": true, "your": true, "not": true, "but": true, "all": true,
	"new": true, "now": true, "more": true, "after": true, "over": true,
	"about": true, "says": true, "said": true, "of": true, "to": true,
	"in": true, "is": true, "on": true, "at": true, "by": true, "as": true,
	"an": true, "be": true, "it": true, "or": true, "if": true, "we": true,
	"he": true, "do": true, "up": true, "so": true, "no": true, "my": true,
}

// RelatedStories returns classified stories textually similar to the given
// story, ranked by FTS5 relevance and excluding the story itself. The match
// query is an OR of significant tokens from the seed story's headline and lede,
// so it surfaces stories that share subject matter rather than requiring every
// term (which is what SearchStories' implicit-AND query would demand).
func RelatedStories(storyID int64, limit int) ([]Story, error) {
	if storyID <= 0 || limit <= 0 {
		return nil, nil
	}
	var headline, lede sql.NullString
	err := Get().QueryRow(`SELECT headline, lede FROM stories WHERE id = ?`, storyID).Scan(&headline, &lede)
	if err != nil {
		if errors.Is(err, sql.ErrNoRows) {
			return nil, nil
		}
		return nil, err
	}
	fts := buildRelatedFTSQuery(headline.String + " " + lede.String)
	if fts == "" {
		return nil, nil
	}
	cutoff := nowUnix() - relatedRecencyWindowSeconds
	rows, err := Get().Query(
		`SELECT s.id, s.guid, s.headline, s.lede, s.image_url, s.article_url, s.source, s.platforms, s.channel, s.paywalled, s.seen_at,
		        EXISTS(SELECT 1 FROM post_log p WHERE p.guid = s.guid) AS posted
		 FROM stories_fts f
		 JOIN stories s ON s.id = f.rowid
		 WHERE f.stories_fts MATCH ?
		   AND s.id != ?
		   AND s.classified = 1
		   AND s.channel IS NOT NULL
		   AND s.channel NOT IN ('_discarded', '_duplicate')
		   AND s.seen_at >= ?
		 ORDER BY bm25(stories_fts) ASC, s.seen_at DESC
		 LIMIT ?`, fts, storyID, cutoff, limit)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var out []Story
	for rows.Next() {
		var s Story
		if err := rows.Scan(&s.ID, &s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.Platforms, &s.Channel, &s.Paywalled, &s.SeenAt, &s.Posted); err != nil {
			return nil, err
		}
		out = append(out, s)
	}
	return out, rows.Err()
}

// buildRelatedFTSQuery tokenizes a story's headline+lede into a de-duplicated,
// stopword-filtered OR query of quoted terms, capped so a long lede can't build
// a pathological query. Returns "" when nothing usable remains.
func buildRelatedFTSQuery(raw string) string {
	var tokens []string
	var cur strings.Builder
	seen := make(map[string]bool)
	flush := func() {
		if cur.Len() == 0 {
			return
		}
		t := strings.ToLower(cur.String())
		cur.Reset()
		if len([]rune(t)) < 2 || relatedStopwords[t] || seen[t] {
			return
		}
		seen[t] = true
		tokens = append(tokens, t)
	}
	for _, r := range raw {
		switch {
		case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
			cur.WriteRune(r)
		case r > 127:
			cur.WriteRune(r)
		default:
			flush()
		}
	}
	flush()
	if len(tokens) == 0 {
		return ""
	}
	const maxTokens = 16
	if len(tokens) > maxTokens {
		tokens = tokens[:maxTokens]
	}
	for i, t := range tokens {
		tokens[i] = `"` + t + `"`
	}
	return strings.Join(tokens, " OR ")
}