drwily/Pete
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feedwatch: tell the owner when a feed breaks, comes back, or goes quiet

Browse Source 
Every poll's outcome has always been written to source_health and shown on
/status, but only to someone who went and looked. Nobody did: Siliconera has
been refused on 3,769 polls in a row and Digital Foundry on 1,900, and
neither has ever fetched successfully. SendAdminWarning existed and nothing
called it.

A watcher now reads source_health every 15 minutes and sends one notice to
matrix.admin_room covering everything that changed since it last spoke:
feeds that have failed 5 polls in a row, feeds that have fetched since they
were reported down, and feeds that fetch fine but have kept nothing new for 7
days (every healthy source produces something inside about 4). What has been
said is stored on the row (down_alerted_at, quiet_alerted_at), so each
episode is reported once and a restart doesn't repeat it. A notice that
fails to send is not marked, and goes out on the next pass.

last_new_at is new, moved only when a poll keeps a story. Existing rows start
from their newest retained story, or from now if they have none, and a new
source starts at its first poll: nothing on disk says whether a source with
no stories is long dead or was added yesterday, so it gets a full window
before it can be called quiet.

With no admin room configured the watcher doesn't run at all, rather than
marking alerts as delivered to nowhere.
This commit is contained in:
prosolis
2026-09-22 23:14:12 -07:00
parent 517019d4a4
commit c6c107b936
7 changed files with 533 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/storage/db.go
+8
View File
@@ -96,6 +96,14 @@ func runMigrations(d *sql.DB) error {
addColumnIfMissing(d, "post_log", "url_canonical", "TEXT")
addColumnIfMissing(d, "post_log", "forced", "INTEGER NOT NULL DEFAULT 0")
addColumnIfMissing(d, "round_robin_state", "last_channel", "TEXT")
// Feed alerts. last_new_at is when a source last produced a story Pete kept;
// the two *_alerted_at stamps remember that the owner has already been told a
// feed is down or has gone quiet, so an alert fires once per episode and
// survives a restart. See backfillLastNewAt for what existing rows start with.
addColumnIfMissing(d, "source_health", "last_new_at", "INTEGER")
addColumnIfMissing(d, "source_health", "down_alerted_at", "INTEGER")
addColumnIfMissing(d, "source_health", "quiet_alerted_at", "INTEGER")
backfillLastNewAt(d)
// Occupancy of a shared table. Rows written before the casino went multiplayer
// are solo games and read as NULL, which is exactly what they are.
addColumnIfMissing(d, "game_live_hands", "table_id", "TEXT")
internal/storage/sourcehealth.go
+86 -7
View File
@@ -3,6 +3,7 @@ package storage
import (
"database/sql"
"fmt"
"log/slog"
)
// SourceHealth is one source's persisted poll health, as written by the poller.
@@ -30,15 +31,17 @@ type SourceContentStat struct {
// RecordPollResult upserts a source's health row after a poll attempt. On
// success it clears the error and failure counter and records the item count;
// on failure it bumps consecutive_failures and stores the message while
// preserving the last successful timestamp and item count. Fire-and-forget:
// preserving the last successful timestamp and item count. A brand-new row
// starts its last_new_at clock at the first poll, so a just-added source gets a
// full quiet window before it can be called quiet. Fire-and-forget:
// a failure here must never disrupt polling, so errors are logged and swallowed.
func RecordPollResult(source string, ok bool, itemCount int, pollErr error) {
now := nowUnix()
if ok {
exec("record poll success",
`INSERT INTO source_health
(source, last_poll_at, last_success_at, last_error, consecutive_failures, last_item_count, updated_at)
VALUES (?, ?, ?, '', 0, ?, ?)
(source, last_poll_at, last_success_at, last_error, consecutive_failures, last_item_count, updated_at, last_new_at)
VALUES (?, ?, ?, '', 0, ?, ?, ?)
ON CONFLICT(source) DO UPDATE SET
last_poll_at = excluded.last_poll_at,
last_success_at = excluded.last_success_at,
@@ -46,7 +49,7 @@ func RecordPollResult(source string, ok bool, itemCount int, pollErr error) {
consecutive_failures = 0,
last_item_count = excluded.last_item_count,
updated_at = excluded.updated_at`,
source, now, now, itemCount, now)
source, now, now, itemCount, now, now)
return
}
msg := ""
@@ -55,14 +58,14 @@ func RecordPollResult(source string, ok bool, itemCount int, pollErr error) {
}
exec("record poll failure",
`INSERT INTO source_health
(source, last_poll_at, last_success_at, last_error, consecutive_failures, last_item_count, updated_at)
VALUES (?, ?, NULL, ?, 1, 0, ?)
(source, last_poll_at, last_success_at, last_error, consecutive_failures, last_item_count, updated_at, last_new_at)
VALUES (?, ?, NULL, ?, 1, 0, ?, ?)
ON CONFLICT(source) DO UPDATE SET
last_poll_at = excluded.last_poll_at,
last_error = excluded.last_error,
consecutive_failures = source_health.consecutive_failures + 1,
updated_at = excluded.updated_at`,
source, now, msg, now)
source, now, msg, now, now)
}
// ListSourceHealth returns the poll-health row for every source that has been
@@ -147,3 +150,79 @@ func SourceContentStats() (map[string]SourceContentStat, error) {
}
return out, prows.Err()
}
// backfillLastNewAt gives rows that predate last_new_at a starting value: the
// newest story still retained for that source, or, when none is, the time of
// the migration. The second case is a grace period rather than a guess: a
// source with no retained stories might be long dead or might have been added
// yesterday, and nothing on disk says which, so it gets a full quiet window
// before it can be called quiet.
func backfillLastNewAt(d *sql.DB) {
if _, err := d.Exec(
`UPDATE source_health
SET last_new_at = COALESCE(
(SELECT MAX(seen_at) FROM stories WHERE stories.source = source_health.source),
?)
WHERE last_new_at IS NULL`, nowUnix()); err != nil {
slog.Error("backfill source_health.last_new_at failed", "err", err)
}
}
// RecordNewStories notes that a poll of source kept n genuinely new stories.
// Only a non-zero n moves last_new_at: a poll that fetched fine but found
// nothing new is exactly what "gone quiet" is made of.
func RecordNewStories(source string, n int) {
if n <= 0 {
return
}
exec("record new stories",
`UPDATE source_health SET last_new_at = ? WHERE source = ?`, nowUnix(), source)
}
// FeedAlertRow is the slice of source_health the feed watcher reads.
type FeedAlertRow struct {
Source string
ConsecutiveFailures int
LastError string
LastSuccessAt int64 // 0 = never fetched successfully
LastNewAt int64 // 0 = unknown
DownAlertedAt int64 // 0 = owner not told it's down
QuietAlertedAt int64 // 0 = owner not told it's quiet
}
// ListFeedAlertRows returns the alert-relevant health of every polled source.
func ListFeedAlertRows() ([]FeedAlertRow, error) {
rows, err := Get().Query(
`SELECT source, consecutive_failures, COALESCE(last_error, ''),
COALESCE(last_success_at, 0), COALESCE(last_new_at, 0),
COALESCE(down_alerted_at, 0), COALESCE(quiet_alerted_at, 0)
FROM source_health ORDER BY source`)
if err != nil {
return nil, fmt.Errorf("list feed alert rows: %w", err)
}
defer rows.Close()
var out []FeedAlertRow
for rows.Next() {
var r FeedAlertRow
if err := rows.Scan(&r.Source, &r.ConsecutiveFailures, &r.LastError,
&r.LastSuccessAt, &r.LastNewAt, &r.DownAlertedAt, &r.QuietAlertedAt); err != nil {
return nil, err
}
out = append(out, r)
}
return out, rows.Err()
}
// SetFeedDownAlerted records (at > 0) or clears (at == 0) that the owner has
// been told source is down.
func SetFeedDownAlerted(source string, at int64) error {
_, err := Get().Exec(`UPDATE source_health SET down_alerted_at = NULLIF(?, 0) WHERE source = ?`, at, source)
return err
}
// SetFeedQuietAlerted records (at > 0) or clears (at == 0) that the owner has
// been told source has gone quiet.
func SetFeedQuietAlerted(source string, at int64) error {
_, err := Get().Exec(`UPDATE source_health SET quiet_alerted_at = NULLIF(?, 0) WHERE source = ?`, at, source)
return err
}