Initial commit
This commit is contained in:
prosolis
229
internal/poster/queue.go
Normal file
229
internal/poster/queue.go
Normal file
@@ -0,0 +1,229 @@
|
||||
package poster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
"pete/internal/dedup"
|
||||
"pete/internal/matrix"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
const maxRetries = 3
|
||||
|
||||
// QueueItem represents a story ready to be posted.
|
||||
type QueueItem struct {
|
||||
GUID string
|
||||
Headline string
|
||||
Lede string
|
||||
ImageURL string
|
||||
ArticleURL string
|
||||
Source string
|
||||
Channel string
|
||||
Platforms []string
|
||||
retries int
|
||||
}
|
||||
|
||||
// Queue manages per-channel metered release of stories.
|
||||
type Queue struct {
|
||||
mu sync.Mutex
|
||||
queues map[string][]QueueItem // channel -> items
|
||||
config config.PostingConfig
|
||||
mx *matrix.Client
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// NewQueue creates a new metered release queue.
|
||||
func NewQueue(cfg config.PostingConfig, mx *matrix.Client) *Queue {
|
||||
return &Queue{
|
||||
queues: make(map[string][]QueueItem),
|
||||
config: cfg,
|
||||
mx: mx,
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Enqueue adds a story to the appropriate channel queue.
|
||||
// Stories for unconfigured channels are dropped with a warning.
|
||||
func (q *Queue) Enqueue(item QueueItem) {
|
||||
if _, ok := q.mx.ChannelRoomID(item.Channel); !ok {
|
||||
slog.Warn("dropping story for unconfigured channel",
|
||||
"guid", item.GUID,
|
||||
"channel", item.Channel,
|
||||
)
|
||||
return
|
||||
}
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
q.queues[item.Channel] = append(q.queues[item.Channel], item)
|
||||
slog.Info("story queued for posting",
|
||||
"guid", item.GUID,
|
||||
"channel", item.Channel,
|
||||
"queue_depth", len(q.queues[item.Channel]),
|
||||
)
|
||||
}
|
||||
|
||||
// Start runs the queue drain ticker. Blocks until ctx is cancelled, then drains remaining items.
|
||||
func (q *Queue) Start(ctx context.Context) {
|
||||
defer close(q.done)
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// Graceful drain: attempt to post remaining queued items
|
||||
q.drainAll()
|
||||
return
|
||||
case <-ticker.C:
|
||||
q.drain()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Wait blocks until the queue goroutine has finished (including graceful drain).
|
||||
func (q *Queue) Wait() {
|
||||
<-q.done
|
||||
}
|
||||
|
||||
func (q *Queue) drain() {
|
||||
q.mu.Lock()
|
||||
channels := make([]string, 0, len(q.queues))
|
||||
for ch := range q.queues {
|
||||
channels = append(channels, ch)
|
||||
}
|
||||
q.mu.Unlock()
|
||||
|
||||
for _, ch := range channels {
|
||||
q.drainChannel(ch)
|
||||
}
|
||||
}
|
||||
|
||||
// drainAll posts all remaining items ignoring rate limits (shutdown path).
|
||||
func (q *Queue) drainAll() {
|
||||
q.mu.Lock()
|
||||
channels := make([]string, 0, len(q.queues))
|
||||
for ch := range q.queues {
|
||||
channels = append(channels, ch)
|
||||
}
|
||||
q.mu.Unlock()
|
||||
|
||||
for _, ch := range channels {
|
||||
for {
|
||||
q.mu.Lock()
|
||||
items := q.queues[ch]
|
||||
if len(items) == 0 {
|
||||
q.mu.Unlock()
|
||||
break
|
||||
}
|
||||
item := items[0]
|
||||
q.queues[ch] = items[1:]
|
||||
q.mu.Unlock()
|
||||
|
||||
q.postItem(item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (q *Queue) drainChannel(channel string) {
|
||||
q.mu.Lock()
|
||||
items := q.queues[channel]
|
||||
if len(items) == 0 {
|
||||
q.mu.Unlock()
|
||||
return
|
||||
}
|
||||
q.mu.Unlock()
|
||||
|
||||
now := time.Now().Unix()
|
||||
minInterval := int64(q.config.MinIntervalSeconds)
|
||||
burstWindow := int64(q.config.BurstCapWindowSeconds)
|
||||
|
||||
// Check minimum interval since last post
|
||||
lastPost := storage.GetLastPostTime(channel)
|
||||
if now-lastPost < minInterval {
|
||||
return
|
||||
}
|
||||
|
||||
// Check burst cap
|
||||
windowStart := now - burstWindow
|
||||
postsInWindow := storage.CountPostsInWindow(channel, windowStart)
|
||||
if postsInWindow >= q.config.BurstCapCount {
|
||||
return
|
||||
}
|
||||
|
||||
// Dequeue one item
|
||||
q.mu.Lock()
|
||||
items = q.queues[channel]
|
||||
if len(items) == 0 {
|
||||
q.mu.Unlock()
|
||||
return
|
||||
}
|
||||
item := items[0]
|
||||
q.queues[channel] = items[1:]
|
||||
q.mu.Unlock()
|
||||
|
||||
q.postItem(item)
|
||||
}
|
||||
|
||||
func (q *Queue) postItem(item QueueItem) {
|
||||
// Last-mile dedup: if this canonical URL was already posted to this channel
|
||||
// within the cooldown window, drop silently. Catches "same article, different
|
||||
// GUID across feeds" and any race where two items slipped past ingest dedup.
|
||||
canonical := dedup.CanonicalURL(item.ArticleURL)
|
||||
cooldownSec := int64(q.config.DedupCooldownHours) * 3600
|
||||
if storage.WasCanonicalPostedRecently(canonical, item.Channel, cooldownSec) {
|
||||
slog.Info("dropping duplicate post (canonical URL posted within cooldown)",
|
||||
"guid", item.GUID,
|
||||
"channel", item.Channel,
|
||||
"url_canonical", canonical,
|
||||
"cooldown_hours", q.config.DedupCooldownHours,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
story := &matrix.PostableStory{
|
||||
ImageURL: item.ImageURL,
|
||||
Headline: item.Headline,
|
||||
ArticleURL: item.ArticleURL,
|
||||
Lede: item.Lede,
|
||||
Source: item.Source,
|
||||
Channel: item.Channel,
|
||||
Platforms: item.Platforms,
|
||||
}
|
||||
|
||||
eventID, err := q.mx.PostStory(item.Channel, story)
|
||||
if err != nil {
|
||||
item.retries++
|
||||
if item.retries >= maxRetries {
|
||||
slog.Error("story dead-lettered after max retries",
|
||||
"guid", item.GUID,
|
||||
"channel", item.Channel,
|
||||
"err", err,
|
||||
)
|
||||
return
|
||||
}
|
||||
slog.Error("failed to post story, will retry",
|
||||
"guid", item.GUID,
|
||||
"channel", item.Channel,
|
||||
"attempt", item.retries,
|
||||
"err", err,
|
||||
)
|
||||
// Re-queue at front for retry
|
||||
q.mu.Lock()
|
||||
q.queues[item.Channel] = append([]QueueItem{item}, q.queues[item.Channel]...)
|
||||
q.mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
// Record in post log (INSERT OR IGNORE via unique index)
|
||||
storage.InsertPostLog(item.GUID, item.Channel, string(eventID), canonical)
|
||||
|
||||
slog.Info("story posted",
|
||||
"guid", item.GUID,
|
||||
"channel", item.Channel,
|
||||
"event_id", eventID,
|
||||
)
|
||||
}
|
||||
145
internal/poster/queue_test.go
Normal file
145
internal/poster/queue_test.go
Normal file
@@ -0,0 +1,145 @@
|
||||
package poster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
// mockMatrix implements the subset of matrix.Client that Queue needs.
|
||||
// We test via the public Enqueue/Start interface and check storage state.
|
||||
|
||||
func setupTestEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
if err := storage.Init(dbPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { storage.Close() })
|
||||
}
|
||||
|
||||
func TestQueue_EnqueueAndDrain(t *testing.T) {
|
||||
setupTestEnv(t)
|
||||
|
||||
q := &Queue{
|
||||
queues: make(map[string][]QueueItem),
|
||||
config: config.PostingConfig{
|
||||
MinIntervalSeconds: 0,
|
||||
BurstCapCount: 10,
|
||||
BurstCapWindowSeconds: 1800,
|
||||
},
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
|
||||
// Enqueue directly (bypass ChannelRoomID check)
|
||||
q.mu.Lock()
|
||||
q.queues["tech"] = append(q.queues["tech"], QueueItem{
|
||||
GUID: "g1",
|
||||
Channel: "tech",
|
||||
})
|
||||
q.mu.Unlock()
|
||||
|
||||
// Verify item is in queue
|
||||
q.mu.Lock()
|
||||
if len(q.queues["tech"]) != 1 {
|
||||
t.Fatalf("expected 1 item in queue, got %d", len(q.queues["tech"]))
|
||||
}
|
||||
q.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestQueueItem_MaxRetries(t *testing.T) {
|
||||
// Verify the retry counter works
|
||||
item := QueueItem{GUID: "g1", Channel: "tech", retries: 0}
|
||||
item.retries++
|
||||
if item.retries != 1 {
|
||||
t.Errorf("expected retries=1, got %d", item.retries)
|
||||
}
|
||||
item.retries++
|
||||
item.retries++
|
||||
if item.retries < maxRetries {
|
||||
t.Errorf("expected retries >= maxRetries after 3 increments, got %d", item.retries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueue_StartAndWait(t *testing.T) {
|
||||
setupTestEnv(t)
|
||||
|
||||
q := &Queue{
|
||||
queues: make(map[string][]QueueItem),
|
||||
config: config.PostingConfig{
|
||||
MinIntervalSeconds: 0,
|
||||
BurstCapCount: 10,
|
||||
BurstCapWindowSeconds: 1800,
|
||||
},
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
q.Start(ctx)
|
||||
}()
|
||||
|
||||
// Cancel immediately — Start should return promptly
|
||||
cancel()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
q.Wait()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// good
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("queue.Wait() did not return after cancel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueue_BurstCap(t *testing.T) {
|
||||
setupTestEnv(t)
|
||||
|
||||
cfg := config.PostingConfig{
|
||||
MinIntervalSeconds: 0,
|
||||
BurstCapCount: 2,
|
||||
BurstCapWindowSeconds: 3600,
|
||||
}
|
||||
|
||||
q := &Queue{
|
||||
queues: make(map[string][]QueueItem),
|
||||
config: cfg,
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
|
||||
// Simulate 2 posts already in window
|
||||
now := time.Now().Unix()
|
||||
storage.Get().Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`,
|
||||
"prev1", "tech", "$e1", now-100)
|
||||
storage.Get().Exec(`INSERT INTO post_log (guid, channel, event_id, posted_at) VALUES (?, ?, ?, ?)`,
|
||||
"prev2", "tech", "$e2", now-50)
|
||||
|
||||
// Queue a new item
|
||||
q.mu.Lock()
|
||||
q.queues["tech"] = []QueueItem{{GUID: "g3", Channel: "tech"}}
|
||||
q.mu.Unlock()
|
||||
|
||||
// drainChannel should not post (burst cap reached)
|
||||
q.drainChannel("tech")
|
||||
|
||||
q.mu.Lock()
|
||||
remaining := len(q.queues["tech"])
|
||||
q.mu.Unlock()
|
||||
|
||||
if remaining != 1 {
|
||||
t.Errorf("expected item to remain in queue (burst cap), got %d remaining", remaining)
|
||||
}
|
||||
}
|
||||
27
internal/poster/tracker.go
Normal file
27
internal/poster/tracker.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package poster
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"pete/internal/storage"
|
||||
|
||||
"maunium.net/go/mautrix/id"
|
||||
)
|
||||
|
||||
// HandleReaction processes a reaction event by mapping it back to the story GUID.
|
||||
func HandleReaction(roomID id.RoomID, eventID id.EventID, targetEventID id.EventID, emoji string, userID id.UserID) {
|
||||
guid, channel, found := storage.LookupPostGUID(string(targetEventID))
|
||||
if !found {
|
||||
// Reaction on a message we didn't post — ignore
|
||||
return
|
||||
}
|
||||
|
||||
storage.InsertReaction(guid, channel, string(eventID), emoji, string(userID), time.Now().Unix())
|
||||
slog.Debug("reaction recorded",
|
||||
"guid", guid,
|
||||
"channel", channel,
|
||||
"emoji", emoji,
|
||||
"user", userID,
|
||||
)
|
||||
}
|
||||
89
internal/poster/tracker_test.go
Normal file
89
internal/poster/tracker_test.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package poster
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"pete/internal/storage"
|
||||
|
||||
"maunium.net/go/mautrix/id"
|
||||
)
|
||||
|
||||
func setupTrackerTestDB(t *testing.T) {
|
||||
t.Helper()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
if err := storage.Init(dbPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { storage.Close() })
|
||||
}
|
||||
|
||||
func TestHandleReaction_KnownPost(t *testing.T) {
|
||||
setupTrackerTestDB(t)
|
||||
|
||||
// Insert a post log entry
|
||||
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "")
|
||||
|
||||
// Handle a reaction to that post
|
||||
HandleReaction(
|
||||
id.RoomID("!room:example.org"),
|
||||
id.EventID("$react1:example.org"),
|
||||
id.EventID("$post1:example.org"),
|
||||
"👍",
|
||||
id.UserID("@user:example.org"),
|
||||
)
|
||||
|
||||
// Verify reaction was recorded
|
||||
var count int
|
||||
storage.Get().QueryRow(`SELECT COUNT(*) FROM reactions WHERE post_guid = ?`, "story-1").Scan(&count)
|
||||
if count != 1 {
|
||||
t.Errorf("expected 1 reaction, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleReaction_UnknownPost(t *testing.T) {
|
||||
setupTrackerTestDB(t)
|
||||
|
||||
// Handle a reaction to an unknown event — should not crash or insert
|
||||
HandleReaction(
|
||||
id.RoomID("!room:example.org"),
|
||||
id.EventID("$react1:example.org"),
|
||||
id.EventID("$unknown:example.org"),
|
||||
"👍",
|
||||
id.UserID("@user:example.org"),
|
||||
)
|
||||
|
||||
var count int
|
||||
storage.Get().QueryRow(`SELECT COUNT(*) FROM reactions`).Scan(&count)
|
||||
if count != 0 {
|
||||
t.Errorf("expected 0 reactions for unknown post, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleReaction_DuplicateIgnored(t *testing.T) {
|
||||
setupTrackerTestDB(t)
|
||||
|
||||
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "")
|
||||
|
||||
// Send same reaction twice
|
||||
HandleReaction(
|
||||
id.RoomID("!room:example.org"),
|
||||
id.EventID("$react1:example.org"),
|
||||
id.EventID("$post1:example.org"),
|
||||
"👍",
|
||||
id.UserID("@user:example.org"),
|
||||
)
|
||||
HandleReaction(
|
||||
id.RoomID("!room:example.org"),
|
||||
id.EventID("$react1:example.org"),
|
||||
id.EventID("$post1:example.org"),
|
||||
"👍",
|
||||
id.UserID("@user:example.org"),
|
||||
)
|
||||
|
||||
var count int
|
||||
storage.Get().QueryRow(`SELECT COUNT(*) FROM reactions`).Scan(&count)
|
||||
if count != 1 {
|
||||
t.Errorf("expected 1 reaction (deduped), got %d", count)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user