Exclude !post from the daily cap

Manual !post overrides were counted toward daily_cap_total, so a few
forced posts could starve the round-robin rotation for the rest of the
day. Tag forced rows in post_log and skip them in CountAllPostsInWindow
so the cap only meters the auto-rotation.
This commit is contained in:
prosolis
2026-05-26 17:15:53 -07:00
parent a15025089d
commit b617d403b7
7 changed files with 30 additions and 19 deletions

View File

@@ -109,7 +109,7 @@ func (q *Queue) ForcePost(channel string) bool {
slog.Info("force-posting story on demand",
"guid", item.GUID, "channel", channel)
return q.postItem(item)
return q.postItem(item, true)
}
func (q *Queue) drain() {
@@ -194,21 +194,23 @@ func (q *Queue) drainChannel(channel string) {
q.queues[channel] = items[1:]
q.mu.Unlock()
q.postItem(item)
q.postItem(item, false)
}
// PostNow sends a story immediately, bypassing the in-memory queue and all
// pacing limits. Last-mile canonical-URL dedup still applies. Used by !post
// to satisfy on-demand requests with stories pulled directly from storage.
func (q *Queue) PostNow(item QueueItem) {
q.postItem(item)
q.postItem(item, true)
}
// postItem returns true when a Matrix event was actually sent, false on
// any non-success path (dedup-skip, transport failure with or without
// retry, dead-letter). ForcePost uses the return value to decide whether
// to acknowledge the user-initiated !post or fall back to a DB lookup.
func (q *Queue) postItem(item QueueItem) bool {
// forced=true marks the resulting post_log row so it's excluded from the
// global daily cap (manual overrides shouldn't steal the rotation budget).
func (q *Queue) postItem(item QueueItem, forced bool) bool {
// Last-mile dedup: if this canonical URL was already posted to this channel
// within the cooldown window, drop silently. Catches "same article, different
// GUID across feeds" and any race where two items slipped past ingest dedup.
@@ -264,7 +266,7 @@ func (q *Queue) postItem(item QueueItem) bool {
}
// Record in post log (INSERT OR IGNORE via unique index)
storage.InsertPostLog(item.GUID, item.Channel, string(eventID), canonical)
storage.InsertPostLog(item.GUID, item.Channel, string(eventID), canonical, forced)
slog.Info("story posted",
"guid", item.GUID,

View File

@@ -22,7 +22,7 @@ func TestHandleReaction_KnownPost(t *testing.T) {
setupTrackerTestDB(t)
// Insert a post log entry
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "")
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "", false)
// Handle a reaction to that post
HandleReaction(
@@ -63,7 +63,7 @@ func TestHandleReaction_UnknownPost(t *testing.T) {
func TestHandleReaction_DuplicateIgnored(t *testing.T) {
setupTrackerTestDB(t)
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "")
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "", false)
// Send same reaction twice
HandleReaction(

View File

@@ -156,7 +156,7 @@ func TestTick_AlreadyPostedExcluded(t *testing.T) {
setupTestDB(t)
insertPostable(t, "t-posted", "GuardianTech", "tech", 200)
insertPostable(t, "t-fresh", "GuardianTech", "tech", 100)
storage.InsertPostLog("t-posted", "tech", "$evt1", "https://example.com/t-posted")
storage.InsertPostLog("t-posted", "tech", "$evt1", "https://example.com/t-posted", false)
fq := &fakeQueue{}
rr := New([]string{"tech"}, 4, fq)

View File

@@ -83,6 +83,7 @@ func runMigrations(d *sql.DB) error {
addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
addColumnIfMissing(d, "stories", "published_at", "INTEGER")
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
addColumnIfMissing(d, "post_log", "forced", "INTEGER NOT NULL DEFAULT 0")
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")
// FTS5 virtual tables don't support IF NOT EXISTS reliably.

View File

@@ -122,10 +122,16 @@ func GetStoryByGUID(guid string) (*Story, error) {
// InsertPostLog records that a story was posted to a channel.
// Uses OR IGNORE to prevent duplicate posts for the same story+channel.
func InsertPostLog(guid, channel, eventID, urlCanonical string) {
// forced=true marks the row as a manual !post override so it doesn't count
// against the global daily cap.
func InsertPostLog(guid, channel, eventID, urlCanonical string, forced bool) {
f := 0
if forced {
f = 1
}
exec("insert post_log",
`INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at) VALUES (?, ?, ?, ?, ?)`,
guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix())
`INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at, forced) VALUES (?, ?, ?, ?, ?, ?)`,
guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix(), f)
}
// GetLastPostTime returns the unix timestamp of the most recent post to a channel.
@@ -139,10 +145,11 @@ func GetLastPostTime(channel string) int64 {
}
// CountAllPostsInWindow counts posts across ALL channels within a time window.
// Used for the global daily cap.
// Used for the global daily cap. Excludes forced (!post) entries so manual
// overrides don't eat into the auto-rotation budget.
func CountAllPostsInWindow(windowStart int64) int {
var count int
if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE posted_at >= ?`, windowStart).Scan(&count); err != nil {
if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE posted_at >= ? AND forced = 0`, windowStart).Scan(&count); err != nil {
slog.Error("CountAllPostsInWindow query failed", "err", err)
return 1<<31 - 1 // fail closed: huge number prevents posting
}

View File

@@ -25,7 +25,8 @@ CREATE TABLE IF NOT EXISTS post_log (
channel TEXT NOT NULL,
event_id TEXT,
url_canonical TEXT,
posted_at INTEGER NOT NULL
posted_at INTEGER NOT NULL,
forced INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS round_robin_state (

View File

@@ -98,8 +98,8 @@ func TestMarkClassified(t *testing.T) {
func TestPostLogAndLookup(t *testing.T) {
setupTestDB(t)
InsertPostLog("story-1", "tech", "$event1:example.org", "")
InsertPostLog("story-2", "politics", "$event2:example.org", "")
InsertPostLog("story-1", "tech", "$event1:example.org", "", false)
InsertPostLog("story-2", "politics", "$event2:example.org", "", false)
guid, channel, found := LookupPostGUID("$event1:example.org")
if !found {
@@ -193,9 +193,9 @@ func TestMarshalUnmarshalPlatforms(t *testing.T) {
func TestInsertPostLog_DuplicateIgnored(t *testing.T) {
setupTestDB(t)
InsertPostLog("story-1", "tech", "$event1:example.org", "")
InsertPostLog("story-1", "tech", "$event1:example.org", "", false)
// Same guid+channel should be silently ignored
InsertPostLog("story-1", "tech", "$event1-retry:example.org", "")
InsertPostLog("story-1", "tech", "$event1-retry:example.org", "", false)
var count int
Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ? AND channel = ?`, "story-1", "tech").Scan(&count)
@@ -204,7 +204,7 @@ func TestInsertPostLog_DuplicateIgnored(t *testing.T) {
}
// Different channel should be allowed
InsertPostLog("story-1", "politics", "$event2:example.org", "")
InsertPostLog("story-1", "politics", "$event2:example.org", "", false)
Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ?`, "story-1").Scan(&count)
if count != 2 {
t.Errorf("expected 2 post_log entries across channels, got %d", count)