Exclude !post from the daily cap
Manual !post overrides were counted toward daily_cap_total, so a few forced posts could starve the round-robin rotation for the rest of the day. Tag forced rows in post_log and skip them in CountAllPostsInWindow so the cap only meters the auto-rotation.
This commit is contained in:
@@ -109,7 +109,7 @@ func (q *Queue) ForcePost(channel string) bool {
|
||||
|
||||
slog.Info("force-posting story on demand",
|
||||
"guid", item.GUID, "channel", channel)
|
||||
return q.postItem(item)
|
||||
return q.postItem(item, true)
|
||||
}
|
||||
|
||||
func (q *Queue) drain() {
|
||||
@@ -194,21 +194,23 @@ func (q *Queue) drainChannel(channel string) {
|
||||
q.queues[channel] = items[1:]
|
||||
q.mu.Unlock()
|
||||
|
||||
q.postItem(item)
|
||||
q.postItem(item, false)
|
||||
}
|
||||
|
||||
// PostNow sends a story immediately, bypassing the in-memory queue and all
|
||||
// pacing limits. Last-mile canonical-URL dedup still applies. Used by !post
|
||||
// to satisfy on-demand requests with stories pulled directly from storage.
|
||||
func (q *Queue) PostNow(item QueueItem) {
|
||||
q.postItem(item)
|
||||
q.postItem(item, true)
|
||||
}
|
||||
|
||||
// postItem returns true when a Matrix event was actually sent, false on
|
||||
// any non-success path (dedup-skip, transport failure with or without
|
||||
// retry, dead-letter). ForcePost uses the return value to decide whether
|
||||
// to acknowledge the user-initiated !post or fall back to a DB lookup.
|
||||
func (q *Queue) postItem(item QueueItem) bool {
|
||||
// forced=true marks the resulting post_log row so it's excluded from the
|
||||
// global daily cap (manual overrides shouldn't steal the rotation budget).
|
||||
func (q *Queue) postItem(item QueueItem, forced bool) bool {
|
||||
// Last-mile dedup: if this canonical URL was already posted to this channel
|
||||
// within the cooldown window, drop silently. Catches "same article, different
|
||||
// GUID across feeds" and any race where two items slipped past ingest dedup.
|
||||
@@ -264,7 +266,7 @@ func (q *Queue) postItem(item QueueItem) bool {
|
||||
}
|
||||
|
||||
// Record in post log (INSERT OR IGNORE via unique index)
|
||||
storage.InsertPostLog(item.GUID, item.Channel, string(eventID), canonical)
|
||||
storage.InsertPostLog(item.GUID, item.Channel, string(eventID), canonical, forced)
|
||||
|
||||
slog.Info("story posted",
|
||||
"guid", item.GUID,
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestHandleReaction_KnownPost(t *testing.T) {
|
||||
setupTrackerTestDB(t)
|
||||
|
||||
// Insert a post log entry
|
||||
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "")
|
||||
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "", false)
|
||||
|
||||
// Handle a reaction to that post
|
||||
HandleReaction(
|
||||
@@ -63,7 +63,7 @@ func TestHandleReaction_UnknownPost(t *testing.T) {
|
||||
func TestHandleReaction_DuplicateIgnored(t *testing.T) {
|
||||
setupTrackerTestDB(t)
|
||||
|
||||
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "")
|
||||
storage.InsertPostLog("story-1", "tech", "$post1:example.org", "", false)
|
||||
|
||||
// Send same reaction twice
|
||||
HandleReaction(
|
||||
|
||||
@@ -156,7 +156,7 @@ func TestTick_AlreadyPostedExcluded(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
insertPostable(t, "t-posted", "GuardianTech", "tech", 200)
|
||||
insertPostable(t, "t-fresh", "GuardianTech", "tech", 100)
|
||||
storage.InsertPostLog("t-posted", "tech", "$evt1", "https://example.com/t-posted")
|
||||
storage.InsertPostLog("t-posted", "tech", "$evt1", "https://example.com/t-posted", false)
|
||||
|
||||
fq := &fakeQueue{}
|
||||
rr := New([]string{"tech"}, 4, fq)
|
||||
|
||||
@@ -83,6 +83,7 @@ func runMigrations(d *sql.DB) error {
|
||||
addColumnIfMissing(d, "stories", "paywalled", "INTEGER NOT NULL DEFAULT 0")
|
||||
addColumnIfMissing(d, "stories", "published_at", "INTEGER")
|
||||
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
|
||||
addColumnIfMissing(d, "post_log", "forced", "INTEGER NOT NULL DEFAULT 0")
|
||||
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")
|
||||
|
||||
// FTS5 virtual tables don't support IF NOT EXISTS reliably.
|
||||
|
||||
@@ -122,10 +122,16 @@ func GetStoryByGUID(guid string) (*Story, error) {
|
||||
|
||||
// InsertPostLog records that a story was posted to a channel.
|
||||
// Uses OR IGNORE to prevent duplicate posts for the same story+channel.
|
||||
func InsertPostLog(guid, channel, eventID, urlCanonical string) {
|
||||
// forced=true marks the row as a manual !post override so it doesn't count
|
||||
// against the global daily cap.
|
||||
func InsertPostLog(guid, channel, eventID, urlCanonical string, forced bool) {
|
||||
f := 0
|
||||
if forced {
|
||||
f = 1
|
||||
}
|
||||
exec("insert post_log",
|
||||
`INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at) VALUES (?, ?, ?, ?, ?)`,
|
||||
guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix())
|
||||
`INSERT OR IGNORE INTO post_log (guid, channel, event_id, url_canonical, posted_at, forced) VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
guid, channel, eventID, nullIfEmpty(urlCanonical), nowUnix(), f)
|
||||
}
|
||||
|
||||
// GetLastPostTime returns the unix timestamp of the most recent post to a channel.
|
||||
@@ -139,10 +145,11 @@ func GetLastPostTime(channel string) int64 {
|
||||
}
|
||||
|
||||
// CountAllPostsInWindow counts posts across ALL channels within a time window.
|
||||
// Used for the global daily cap.
|
||||
// Used for the global daily cap. Excludes forced (!post) entries so manual
|
||||
// overrides don't eat into the auto-rotation budget.
|
||||
func CountAllPostsInWindow(windowStart int64) int {
|
||||
var count int
|
||||
if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE posted_at >= ?`, windowStart).Scan(&count); err != nil {
|
||||
if err := Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE posted_at >= ? AND forced = 0`, windowStart).Scan(&count); err != nil {
|
||||
slog.Error("CountAllPostsInWindow query failed", "err", err)
|
||||
return 1<<31 - 1 // fail closed: huge number prevents posting
|
||||
}
|
||||
|
||||
@@ -25,7 +25,8 @@ CREATE TABLE IF NOT EXISTS post_log (
|
||||
channel TEXT NOT NULL,
|
||||
event_id TEXT,
|
||||
url_canonical TEXT,
|
||||
posted_at INTEGER NOT NULL
|
||||
posted_at INTEGER NOT NULL,
|
||||
forced INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS round_robin_state (
|
||||
|
||||
@@ -98,8 +98,8 @@ func TestMarkClassified(t *testing.T) {
|
||||
func TestPostLogAndLookup(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
InsertPostLog("story-1", "tech", "$event1:example.org", "")
|
||||
InsertPostLog("story-2", "politics", "$event2:example.org", "")
|
||||
InsertPostLog("story-1", "tech", "$event1:example.org", "", false)
|
||||
InsertPostLog("story-2", "politics", "$event2:example.org", "", false)
|
||||
|
||||
guid, channel, found := LookupPostGUID("$event1:example.org")
|
||||
if !found {
|
||||
@@ -193,9 +193,9 @@ func TestMarshalUnmarshalPlatforms(t *testing.T) {
|
||||
func TestInsertPostLog_DuplicateIgnored(t *testing.T) {
|
||||
setupTestDB(t)
|
||||
|
||||
InsertPostLog("story-1", "tech", "$event1:example.org", "")
|
||||
InsertPostLog("story-1", "tech", "$event1:example.org", "", false)
|
||||
// Same guid+channel should be silently ignored
|
||||
InsertPostLog("story-1", "tech", "$event1-retry:example.org", "")
|
||||
InsertPostLog("story-1", "tech", "$event1-retry:example.org", "", false)
|
||||
|
||||
var count int
|
||||
Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ? AND channel = ?`, "story-1", "tech").Scan(&count)
|
||||
@@ -204,7 +204,7 @@ func TestInsertPostLog_DuplicateIgnored(t *testing.T) {
|
||||
}
|
||||
|
||||
// Different channel should be allowed
|
||||
InsertPostLog("story-1", "politics", "$event2:example.org", "")
|
||||
InsertPostLog("story-1", "politics", "$event2:example.org", "", false)
|
||||
Get().QueryRow(`SELECT COUNT(*) FROM post_log WHERE guid = ?`, "story-1").Scan(&count)
|
||||
if count != 2 {
|
||||
t.Errorf("expected 2 post_log entries across channels, got %d", count)
|
||||
|
||||
Reference in New Issue
Block a user