Initial commit

internal/poster/queue.go

@@ -0,0 +1,229 @@
+package poster
+
+import (
+	"context"
+	"log/slog"
+	"sync"
+	"time"
+
+	"pete/internal/config"
+	"pete/internal/dedup"
+	"pete/internal/matrix"
+	"pete/internal/storage"
+)
+
+const maxRetries = 3
+
+// QueueItem represents a story ready to be posted.
+type QueueItem struct {
+	GUID       string
+	Headline   string
+	Lede       string
+	ImageURL   string
+	ArticleURL string
+	Source     string
+	Channel    string
+	Platforms  []string
+	retries    int
+}
+
+// Queue manages per-channel metered release of stories.
+type Queue struct {
+	mu       sync.Mutex
+	queues   map[string][]QueueItem // channel -> items
+	config   config.PostingConfig
+	mx       *matrix.Client
+	done     chan struct{}
+}
+
+// NewQueue creates a new metered release queue.
+func NewQueue(cfg config.PostingConfig, mx *matrix.Client) *Queue {
+	return &Queue{
+		queues: make(map[string][]QueueItem),
+		config: cfg,
+		mx:     mx,
+		done:   make(chan struct{}),
+	}
+}
+
+// Enqueue adds a story to the appropriate channel queue.
+// Stories for unconfigured channels are dropped with a warning.
+func (q *Queue) Enqueue(item QueueItem) {
+	if _, ok := q.mx.ChannelRoomID(item.Channel); !ok {
+		slog.Warn("dropping story for unconfigured channel",
+			"guid", item.GUID,
+			"channel", item.Channel,
+		)
+		return
+	}
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	q.queues[item.Channel] = append(q.queues[item.Channel], item)
+	slog.Info("story queued for posting",
+		"guid", item.GUID,
+		"channel", item.Channel,
+		"queue_depth", len(q.queues[item.Channel]),
+	)
+}
+
+// Start runs the queue drain ticker. Blocks until ctx is cancelled, then drains remaining items.
+func (q *Queue) Start(ctx context.Context) {
+	defer close(q.done)
+	ticker := time.NewTicker(10 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			// Graceful drain: attempt to post remaining queued items
+			q.drainAll()
+			return
+		case <-ticker.C:
+			q.drain()
+		}
+	}
+}
+
+// Wait blocks until the queue goroutine has finished (including graceful drain).
+func (q *Queue) Wait() {
+	<-q.done
+}
+
+func (q *Queue) drain() {
+	q.mu.Lock()
+	channels := make([]string, 0, len(q.queues))
+	for ch := range q.queues {
+		channels = append(channels, ch)
+	}
+	q.mu.Unlock()
+
+	for _, ch := range channels {
+		q.drainChannel(ch)
+	}
+}
+
+// drainAll posts all remaining items ignoring rate limits (shutdown path).
+func (q *Queue) drainAll() {
+	q.mu.Lock()
+	channels := make([]string, 0, len(q.queues))
+	for ch := range q.queues {
+		channels = append(channels, ch)
+	}
+	q.mu.Unlock()
+
+	for _, ch := range channels {
+		for {
+			q.mu.Lock()
+			items := q.queues[ch]
+			if len(items) == 0 {
+				q.mu.Unlock()
+				break
+			}
+			item := items[0]
+			q.queues[ch] = items[1:]
+			q.mu.Unlock()
+
+			q.postItem(item)
+		}
+	}
+}
+
+func (q *Queue) drainChannel(channel string) {
+	q.mu.Lock()
+	items := q.queues[channel]
+	if len(items) == 0 {
+		q.mu.Unlock()
+		return
+	}
+	q.mu.Unlock()
+
+	now := time.Now().Unix()
+	minInterval := int64(q.config.MinIntervalSeconds)
+	burstWindow := int64(q.config.BurstCapWindowSeconds)
+
+	// Check minimum interval since last post
+	lastPost := storage.GetLastPostTime(channel)
+	if now-lastPost < minInterval {
+		return
+	}
+
+	// Check burst cap
+	windowStart := now - burstWindow
+	postsInWindow := storage.CountPostsInWindow(channel, windowStart)
+	if postsInWindow >= q.config.BurstCapCount {
+		return
+	}
+
+	// Dequeue one item
+	q.mu.Lock()
+	items = q.queues[channel]
+	if len(items) == 0 {
+		q.mu.Unlock()
+		return
+	}
+	item := items[0]
+	q.queues[channel] = items[1:]
+	q.mu.Unlock()
+
+	q.postItem(item)
+}
+
+func (q *Queue) postItem(item QueueItem) {
+	// Last-mile dedup: if this canonical URL was already posted to this channel
+	// within the cooldown window, drop silently. Catches "same article, different
+	// GUID across feeds" and any race where two items slipped past ingest dedup.
+	canonical := dedup.CanonicalURL(item.ArticleURL)
+	cooldownSec := int64(q.config.DedupCooldownHours) * 3600
+	if storage.WasCanonicalPostedRecently(canonical, item.Channel, cooldownSec) {
+		slog.Info("dropping duplicate post (canonical URL posted within cooldown)",
+			"guid", item.GUID,
+			"channel", item.Channel,
+			"url_canonical", canonical,
+			"cooldown_hours", q.config.DedupCooldownHours,
+		)
+		return
+	}
+
+	story := &matrix.PostableStory{
+		ImageURL:   item.ImageURL,
+		Headline:   item.Headline,
+		ArticleURL: item.ArticleURL,
+		Lede:       item.Lede,
+		Source:     item.Source,
+		Channel:    item.Channel,
+		Platforms:  item.Platforms,
+	}
+
+	eventID, err := q.mx.PostStory(item.Channel, story)
+	if err != nil {
+		item.retries++
+		if item.retries >= maxRetries {
+			slog.Error("story dead-lettered after max retries",
+				"guid", item.GUID,
+				"channel", item.Channel,
+				"err", err,
+			)
+			return
+		}
+		slog.Error("failed to post story, will retry",
+			"guid", item.GUID,
+			"channel", item.Channel,
+			"attempt", item.retries,
+			"err", err,
+		)
+		// Re-queue at front for retry
+		q.mu.Lock()
+		q.queues[item.Channel] = append([]QueueItem{item}, q.queues[item.Channel]...)
+		q.mu.Unlock()
+		return
+	}
+
+	// Record in post log (INSERT OR IGNORE via unique index)
+	storage.InsertPostLog(item.GUID, item.Channel, string(eventID), canonical)
+
+	slog.Info("story posted",
+		"guid", item.GUID,
+		"channel", item.Channel,
+		"event_id", eventID,
+	)
+}