Initial commit

internal/storage/queries.go

@@ -0,0 +1,225 @@
+package storage
+
+import (
+	"encoding/json"
+	"log/slog"
+	"time"
+)
+
+func nowUnix() int64 {
+	return time.Now().Unix()
+}
+
+// IsGUIDSeen checks if a GUID has already been ingested.
+func IsGUIDSeen(guid string) bool {
+	var count int
+	if err := Get().QueryRow(`SELECT COUNT(*) FROM stories WHERE guid = ?`, guid).Scan(&count); err != nil {
+		slog.Error("IsGUIDSeen query failed", "guid", guid, "err", err)
+		return true // fail closed: assume seen to prevent duplicates
+	}
+	return count > 0
+}
+
+// InsertStory inserts a new story record.
+func InsertStory(s *Story) error {
+	classified := 0
+	if s.Classified {
+		classified = 1
+	}
+	_, err := Get().Exec(
+		`INSERT INTO stories (guid, headline, lede, image_url, article_url, url_canonical, headline_norm, source, feed_hint, platforms, channel, classified, seen_at)
+		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		s.GUID, s.Headline, s.Lede, s.ImageURL, s.ArticleURL, nullIfEmpty(s.URLCanonical), nullIfEmpty(s.HeadlineNorm), s.Source, s.FeedHint, s.Platforms, s.Channel, classified, s.SeenAt,
+	)
+	return err
+}
+
+// nullIfEmpty returns a SQL NULL for empty strings so partial unique indexes
+// (WHERE col IS NOT NULL) can hold multiple "unknown" rows without conflict.
+func nullIfEmpty(s string) any {
+	if s == "" {
+		return nil
+	}
+	return s
+}
+
+// IsCanonicalSeen reports whether any story with this canonical URL exists.
+// Empty input always returns false (no canonical = no dedup possible).
+func IsCanonicalSeen(canonical string) bool {
+	if canonical == "" {
+		return false
+	}
+	var n int
+	if err := Get().QueryRow(`SELECT COUNT(*) FROM stories WHERE url_canonical = ?`, canonical).Scan(&n); err != nil {
+		slog.Error("IsCanonicalSeen query failed", "err", err)
+		return true // fail closed
+	}
+	return n > 0
+}
+
+// IsHeadlineSeen reports whether a same-source story with this normalized
+// headline already exists. Empty inputs return false.
+func IsHeadlineSeen(source, headlineNorm string) bool {
+	if source == "" || headlineNorm == "" {
+		return false
+	}
+	var n int
+	if err := Get().QueryRow(
+		`SELECT COUNT(*) FROM stories WHERE source = ? AND headline_norm = ?`,
+		source, headlineNorm).Scan(&n); err != nil {
+		slog.Error("IsHeadlineSeen query failed", "err", err)
+		return true
+	}
+	return n > 0
+}
+
+// WasCanonicalPostedRecently reports whether the canonical URL was posted to
+// the given channel within `cooldownSeconds`. Empty canonical returns false.
+func WasCanonicalPostedRecently(canonical, channel string, cooldownSeconds int64) bool {
+	if canonical == "" || cooldownSeconds <= 0 {
+		return false
+	}
+	cutoff := nowUnix() - cooldownSeconds
+	var n int
+	if err := Get().QueryRow(
+		`SELECT COUNT(*) FROM post_log WHERE url_canonical = ? AND channel = ? AND posted_at >= ?`,
+		canonical, channel, cutoff).Scan(&n); err != nil {
+		slog.Error("WasCanonicalPostedRecently query failed", "err", err)
+		return true // fail closed
+	}
+	return n > 0
+}
+
+// MarkClassified marks a story as successfully classified and sets its channel.
+func MarkClassified(guid, channel, platforms string) {
+	exec("mark classified",
+		`UPDATE stories SET classified = 1, channel = ?, platforms = ? WHERE guid = ?`,
+		channel, platforms, guid)
+}
+
+// GetUnclassifiedStories returns stories from a specific source that were ingested but not yet classified.
+func GetUnclassifiedStories(source string) ([]Story, error) {
+	rows, err := Get().Query(
+		`SELECT guid, headline, lede, image_url, article_url, source, feed_hint, platforms, seen_at
+		 FROM stories WHERE classified = 0 AND source = ? ORDER BY seen_at ASC LIMIT 20`, source)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var stories []Story
+	for rows.Next() {
+		var s Story
+		if err := rows.Scan(&s.GUID, &s.Headline, &s.Lede, &s.ImageURL, &s.ArticleURL, &s.Source, &s.FeedHint, &s.Platforms, &s.SeenAt); err != nil {
+			return nil, err
+		}
+		stories = append(stories, s)
+	}
+	return stories, rows.Err()
+}
+
+// InsertRecentHeadline adds a headline to the dedup context window.
+func InsertRecentHeadline(guid, headline, source string) {
+	exec("insert recent_headline",
+		`INSERT OR REPLACE INTO recent_headlines (guid, headline, source, seen_at) VALUES (?, ?, ?, ?)`,
+		guid, headline, source, nowUnix())
+}
+
+// GetRecentHeadlines returns the most recent headlines for LLM dedup context, capped at limit.
+func GetRecentHeadlines(limit int) ([]RecentHeadline, error) {
+	rows, err := Get().Query(
+		`SELECT guid, headline, source, seen_at FROM recent_headlines ORDER BY seen_at DESC LIMIT ?`, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var headlines []RecentHeadline
+	for rows.Next() {
+		var h RecentHeadline
+		if err := rows.Scan(&h.GUID, &h.Headline, &h.Source, &h.SeenAt); err != nil {
+			return nil, err
+		}
+		headlines = append(headlines, h)
+	}
+	return headlines, rows.Err()
+}
+
+// InsertClassificationLog stores a classification result for tuning review.
+func InsertClassificationLog(guid, resultJSON string) {
+	exec("insert classification_log",
+		`INSERT OR REPLACE INTO classification_log (guid, result, logged_at) VALUES (?, ?, ?)`,
+		guid, resultJSON, nowUnix())
+}
+
+// InsertPostLog records that a story was posted to a channel.
+// Uses OR IGNORE to prevent duplicate posts for the same story+channel.
+func InsertPostLog(guid, channel, eventID, urlCanonical string) {
+	exec("insert post_log",
+		`INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at) VALUES (?, ?, ?, ?, ?)`,
+		guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix())
+}
+
+// GetLastPostTime returns the unix timestamp of the most recent post to a channel.
+func GetLastPostTime(channel string) int64 {
+	var t int64
+	if err := Get().QueryRow(`SELECT COALESCE(MAX(posted_at), 0) FROM post_log WHERE channel = ?`, channel).Scan(&t); err != nil {
+		slog.Error("GetLastPostTime query failed", "channel", channel, "err", err)
+		return nowUnix() // fail closed: pretend we just posted to prevent flooding
+	}
+	return t
+}
+
+// CountPostsInWindow counts posts to a channel within a time window.
+func CountPostsInWindow(channel string, windowStart int64) int {
+	var count int
+	if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE channel = ? AND posted_at >= ?`, channel, windowStart).Scan(&count); err != nil {
+		slog.Error("CountPostsInWindow query failed", "channel", channel, "err", err)
+		return 999 // fail closed: pretend burst cap reached to prevent flooding
+	}
+	return count
+}
+
+// LookupPostGUID finds the story GUID for a given Matrix event ID.
+func LookupPostGUID(eventID string) (guid, channel string, found bool) {
+	err := Get().QueryRow(
+		`SELECT guid, channel FROM post_log WHERE event_id = ? LIMIT 1`, eventID).Scan(&guid, &channel)
+	if err != nil {
+		return "", "", false
+	}
+	return guid, channel, true
+}
+
+// InsertReaction records a reaction on a posted story.
+// Uses OR IGNORE to deduplicate if Matrix delivers the same reaction twice.
+func InsertReaction(postGUID, channel, eventID, emoji, userID string, reactedAt int64) {
+	exec("insert reaction",
+		`INSERT OR IGNORE INTO reactions (post_guid, channel, event_id, emoji, user_id, reacted_at) VALUES (?, ?, ?, ?, ?, ?)`,
+		postGUID, channel, eventID, emoji, userID, reactedAt)
+}
+
+// MarshalPlatforms converts a string slice to a JSON array string for storage.
+func MarshalPlatforms(platforms []string) string {
+	if len(platforms) == 0 {
+		return "[]"
+	}
+	data, err := json.Marshal(platforms)
+	if err != nil {
+		slog.Error("marshal platforms failed", "err", err)
+		return "[]"
+	}
+	return string(data)
+}
+
+// UnmarshalPlatforms converts a JSON array string back to a string slice.
+func UnmarshalPlatforms(raw string) []string {
+	if raw == "" || raw == "[]" {
+		return nil
+	}
+	var platforms []string
+	if err := json.Unmarshal([]byte(raw), &platforms); err != nil {
+		slog.Error("unmarshal platforms failed", "err", err, "raw", raw)
+		return nil
+	}
+	return platforms
+}