Exclude !post from the daily cap

Manual !post overrides were counted toward daily_cap_total, so a few
forced posts could starve the round-robin rotation for the rest of the
day. Tag forced rows in post_log and skip them in CountAllPostsInWindow
so the cap only meters the auto-rotation.
This commit is contained in:
prosolis
2026-05-26 17:15:53 -07:00
parent a15025089d
commit b617d403b7
7 changed files with 30 additions and 19 deletions

View File

@@ -109,7 +109,7 @@ func (q *Queue) ForcePost(channel string) bool {
slog.Info("force-posting story on demand", slog.Info("force-posting story on demand",
"guid", item.GUID, "channel", channel) "guid", item.GUID, "channel", channel)
return q.postItem(item) return q.postItem(item, true)
} }
func (q *Queue) drain() { func (q *Queue) drain() {
@@ -194,21 +194,23 @@ func (q *Queue) drainChannel(channel string) {
q.queues[channel] = items[1:] q.queues[channel] = items[1:]
q.mu.Unlock() q.mu.Unlock()
q.postItem(item) q.postItem(item, false)
} }
// PostNow sends a story immediately, bypassing the in-memory queue and all // PostNow sends a story immediately, bypassing the in-memory queue and all
// pacing limits. Last-mile canonical-URL dedup still applies. Used by !post // pacing limits. Last-mile canonical-URL dedup still applies. Used by !post
// to satisfy on-demand requests with stories pulled directly from storage. // to satisfy on-demand requests with stories pulled directly from storage.
func (q *Queue) PostNow(item QueueItem) { func (q *Queue) PostNow(item QueueItem) {
q.postItem(item) q.postItem(item, true)
} }
// postItem returns true when a Matrix event was actually sent, false on // postItem returns true when a Matrix event was actually sent, false on
// any non-success path (dedup-skip, transport failure with or without // any non-success path (dedup-skip, transport failure with or without
// retry, dead-letter). ForcePost uses the return value to decide whether // retry, dead-letter). ForcePost uses the return value to decide whether
// to acknowledge the user-initiated !post or fall back to a DB lookup. // to acknowledge the user-initiated !post or fall back to a DB lookup.
func (q *Queue) postItem(item QueueItem) bool { // forced=true marks the resulting post_log row so it's excluded from the
// global daily cap (manual overrides shouldn't steal the rotation budget).
func (q *Queue) postItem(item QueueItem, forced bool) bool {
// Last-mile dedup: if this canonical URL was already posted to this channel // Last-mile dedup: if this canonical URL was already posted to this channel
// within the cooldown window, drop silently. Catches "same article, different // within the cooldown window, drop silently. Catches "same article, different
// GUID across feeds" and any race where two items slipped past ingest dedup. // GUID across feeds" and any race where two items slipped past ingest dedup.
@@ -264,7 +266,7 @@ func (q *Queue) postItem(item QueueItem) bool {
} }
// Record in post log (INSERT OR IGNORE via unique index) // Record in post log (INSERT OR IGNORE via unique index)
storage.InsertPostLog(item.GUID, item.Channel, string(eventID), canonical) storage.InsertPostLog(item.GUID, item.Channel, string(eventID), canonical, forced)
slog.Info("story posted", slog.Info("story posted",
"guid", item.GUID, "guid", item.GUID,

View File

@@ -22,7 +22,7 @@ func TestHandleReaction_KnownPost(t *testing.T) {
setupTrackerTestDB(t) setupTrackerTestDB(t)
// Insert a post log entry // Insert a post log entry
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "") storage.InsertPostLog("story-1", "tech", "$post1:example.org", "", false)
// Handle a reaction to that post // Handle a reaction to that post
HandleReaction( HandleReaction(
@@ -63,7 +63,7 @@ func TestHandleReaction_UnknownPost(t *testing.T) {
func TestHandleReaction_DuplicateIgnored(t *testing.T) { func TestHandleReaction_DuplicateIgnored(t *testing.T) {
setupTrackerTestDB(t) setupTrackerTestDB(t)
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "") storage.InsertPostLog("story-1", "tech", "$post1:example.org", "", false)
// Send same reaction twice // Send same reaction twice
HandleReaction( HandleReaction(

View File

@@ -156,7 +156,7 @@ func TestTick_AlreadyPostedExcluded(t *testing.T) {
setupTestDB(t) setupTestDB(t)
insertPostable(t, "t-posted", "GuardianTech", "tech", 200) insertPostable(t, "t-posted", "GuardianTech", "tech", 200)
insertPostable(t, "t-fresh", "GuardianTech", "tech", 100) insertPostable(t, "t-fresh", "GuardianTech", "tech", 100)
storage.InsertPostLog("t-posted", "tech", "$evt1", "https://example.com/t-posted") storage.InsertPostLog("t-posted", "tech", "$evt1", "https://example.com/t-posted", false)
fq := &fakeQueue{} fq := &fakeQueue{}
rr := New([]string{"tech"}, 4, fq) rr := New([]string{"tech"}, 4, fq)

View File

@@ -83,6 +83,7 @@ func runMigrations(d *sql.DB) error {
addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0") addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
addColumnIfMissing(d, "stories", "published_at", "INTEGER") addColumnIfMissing(d, "stories", "published_at", "INTEGER")
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT") addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
addColumnIfMissing(d, "post_log", "forced", "INTEGER NOT NULL DEFAULT 0")
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT") addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")
// FTS5 virtual tables don't support IF NOT EXISTS reliably. // FTS5 virtual tables don't support IF NOT EXISTS reliably.

View File

@@ -122,10 +122,16 @@ func GetStoryByGUID(guid string) (*Story, error) {
// InsertPostLog records that a story was posted to a channel. // InsertPostLog records that a story was posted to a channel.
// Uses OR IGNORE to prevent duplicate posts for the same story+channel. // Uses OR IGNORE to prevent duplicate posts for the same story+channel.
func InsertPostLog(guid, channel, eventID, urlCanonical string) { // forced=true marks the row as a manual !post override so it doesn't count
// against the global daily cap.
func InsertPostLog(guid, channel, eventID, urlCanonical string, forced bool) {
f := 0
if forced {
f = 1
}
exec("insert post_log", exec("insert post_log",
`INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at) VALUES (?, ?, ?, ?, ?)`, `INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at, forced) VALUES (?, ?, ?, ?, ?, ?)`,
guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix()) guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix(), f)
} }
// GetLastPostTime returns the unix timestamp of the most recent post to a channel. // GetLastPostTime returns the unix timestamp of the most recent post to a channel.
@@ -139,10 +145,11 @@ func GetLastPostTime(channel string) int64 {
} }
// CountAllPostsInWindow counts posts across ALL channels within a time window. // CountAllPostsInWindow counts posts across ALL channels within a time window.
// Used for the global daily cap. // Used for the global daily cap. Excludes forced (!post) entries so manual
// overrides don't eat into the auto-rotation budget.
func CountAllPostsInWindow(windowStart int64) int { func CountAllPostsInWindow(windowStart int64) int {
var count int var count int
if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE posted_at >= ?`, windowStart).Scan(&count); err != nil { if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE posted_at >= ? AND forced = 0`, windowStart).Scan(&count); err != nil {
slog.Error("CountAllPostsInWindow query failed", "err", err) slog.Error("CountAllPostsInWindow query failed", "err", err)
return 1<<31 - 1 // fail closed: huge number prevents posting return 1<<31 - 1 // fail closed: huge number prevents posting
} }

View File

@@ -25,7 +25,8 @@ CREATE TABLE IF NOT EXISTS post_log (
channel TEXT NOT NULL, channel TEXT NOT NULL,
event_id TEXT, event_id TEXT,
url_canonical TEXT, url_canonical TEXT,
posted_at INTEGER NOT NULL posted_at INTEGER NOT NULL,
forced INTEGER NOT NULL DEFAULT 0
); );
CREATE TABLE IF NOT EXISTS round_robin_state ( CREATE TABLE IF NOT EXISTS round_robin_state (

View File

@@ -98,8 +98,8 @@ func TestMarkClassified(t *testing.T) {
func TestPostLogAndLookup(t *testing.T) { func TestPostLogAndLookup(t *testing.T) {
setupTestDB(t) setupTestDB(t)
InsertPostLog("story-1", "tech", "$event1:example.org", "") InsertPostLog("story-1", "tech", "$event1:example.org", "", false)
InsertPostLog("story-2", "politics", "$event2:example.org", "") InsertPostLog("story-2", "politics", "$event2:example.org", "", false)
guid, channel, found := LookupPostGUID("$event1:example.org") guid, channel, found := LookupPostGUID("$event1:example.org")
if !found { if !found {
@@ -193,9 +193,9 @@ func TestMarshalUnmarshalPlatforms(t *testing.T) {
func TestInsertPostLog_DuplicateIgnored(t *testing.T) { func TestInsertPostLog_DuplicateIgnored(t *testing.T) {
setupTestDB(t) setupTestDB(t)
InsertPostLog("story-1", "tech", "$event1:example.org", "") InsertPostLog("story-1", "tech", "$event1:example.org", "", false)
// Same guid+channel should be silently ignored // Same guid+channel should be silently ignored
InsertPostLog("story-1", "tech", "$event1-retry:example.org", "") InsertPostLog("story-1", "tech", "$event1-retry:example.org", "", false)
var count int var count int
Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ? AND channel = ?`, "story-1", "tech").Scan(&count) Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ? AND channel = ?`, "story-1", "tech").Scan(&count)
@@ -204,7 +204,7 @@ func TestInsertPostLog_DuplicateIgnored(t *testing.T) {
} }
// Different channel should be allowed // Different channel should be allowed
InsertPostLog("story-1", "politics", "$event2:example.org", "") InsertPostLog("story-1", "politics", "$event2:example.org", "", false)
Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ?`, "story-1").Scan(&count) Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ?`, "story-1").Scan(&count)
if count != 2 { if count != 2 {
t.Errorf("expected 2 post_log entries across channels, got %d", count) t.Errorf("expected 2 post_log entries across channels, got %d", count)