feedwatch: tell the owner when a feed breaks, comes back, or goes quiet
Every poll's outcome has always been written to source_health and shown on /status, but only to someone who went and looked. Nobody did: Siliconera has been refused on 3,769 polls in a row and Digital Foundry on 1,900, and neither has ever fetched successfully. SendAdminWarning existed and nothing called it. A watcher now reads source_health every 15 minutes and sends one notice to matrix.admin_room covering everything that changed since it last spoke: feeds that have failed 5 polls in a row, feeds that have fetched since they were reported down, and feeds that fetch fine but have kept nothing new for 7 days (every healthy source produces something inside about 4). What has been said is stored on the row (down_alerted_at, quiet_alerted_at), so each episode is reported once and a restart doesn't repeat it. A notice that fails to send is not marked, and goes out on the next pass. last_new_at is new, moved only when a poll keeps a story. Existing rows start from their newest retained story, or from now if they have none, and a new source starts at its first poll: nothing on disk says whether a source with no stories is long dead or was added yesterday, so it gets a full window before it can be called quiet. With no admin room configured the watcher doesn't run at all, rather than marking alerts as delivered to nowhere.
This commit is contained in:
prosolis
@@ -0,0 +1,237 @@
|
||||
// Package feedwatch tells the owner when a feed breaks, when it comes back, and
|
||||
// when it goes quiet.
|
||||
//
|
||||
// The poller has always recorded every poll's outcome in source_health, and
|
||||
// /status shows it, but only to someone who goes and looks. Feeds died that way
|
||||
// for months: Siliconera and Digital Foundry had been refused on every poll
|
||||
// thousands of times over before anyone noticed.
|
||||
//
|
||||
// The watcher reads source_health on a timer rather than hooking the pollers, so
|
||||
// that one pass can batch every change into a single message, and so that what
|
||||
// has already been said lives in the database (down_alerted_at,
|
||||
// quiet_alerted_at) instead of in a goroutine that forgets it on restart. Each
|
||||
// episode is reported once: a feed that goes down is announced, stays silent
|
||||
// while it stays down, and is announced again when it comes back.
|
||||
package feedwatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
// DownAfter is how many failed polls in a row make a feed "down". One or
|
||||
// two failures is weather; five is a feed that has stopped working.
|
||||
DownAfter = 5
|
||||
// QuietAfter is how long a feed can fetch fine and yield nothing new before
|
||||
// it is "quiet". Every healthy source Pete carries produces something within
|
||||
// about four days.
|
||||
QuietAfter = 7 * 24 * time.Hour
|
||||
// Every is how often the watcher looks.
|
||||
Every = 15 * time.Minute
|
||||
)
|
||||
|
||||
// Report is what one pass found that the owner hasn't been told yet.
|
||||
type Report struct {
|
||||
Down []storage.FeedAlertRow
|
||||
Back []storage.FeedAlertRow
|
||||
Quiet []storage.FeedAlertRow
|
||||
// quietOver are feeds whose quiet spell has ended. They are cleared without
|
||||
// a message: a feed that simply started posting again needs no fanfare.
|
||||
quietOver []string
|
||||
}
|
||||
|
||||
// Empty reports whether there is nothing to say.
|
||||
func (r Report) Empty() bool { return len(r.Down)+len(r.Back)+len(r.Quiet) == 0 }
|
||||
|
||||
// Check reads source_health and works out what has changed, without recording
|
||||
// anything. Only sources that are enabled in the config are considered, so a
|
||||
// feed that has been switched off or removed never alerts.
|
||||
func Check(now time.Time, sources []config.SourceConfig) (Report, error) {
|
||||
enabled := make(map[string]bool, len(sources))
|
||||
for _, s := range sources {
|
||||
if s.Enabled {
|
||||
enabled[s.Name] = true
|
||||
}
|
||||
}
|
||||
rows, err := storage.ListFeedAlertRows()
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
var r Report
|
||||
quietCutoff := now.Add(-QuietAfter).Unix()
|
||||
for _, row := range rows {
|
||||
if !enabled[row.Source] {
|
||||
continue
|
||||
}
|
||||
down := row.ConsecutiveFailures >= DownAfter
|
||||
switch {
|
||||
case down && row.DownAlertedAt == 0:
|
||||
r.Down = append(r.Down, row)
|
||||
// Back means "has fetched since we said it was down", not "is fetching
|
||||
// right now": a feed that recovers and breaks again between two passes
|
||||
// still gets its all-clear, and then a fresh down alert next pass.
|
||||
case row.DownAlertedAt != 0 && row.LastSuccessAt > row.DownAlertedAt:
|
||||
r.Back = append(r.Back, row)
|
||||
}
|
||||
// A feed that isn't fetching can't be judged on whether it has news;
|
||||
// "down" already covers it.
|
||||
if row.ConsecutiveFailures > 0 {
|
||||
continue
|
||||
}
|
||||
quiet := row.LastNewAt > 0 && row.LastNewAt < quietCutoff
|
||||
switch {
|
||||
case quiet && row.QuietAlertedAt == 0:
|
||||
r.Quiet = append(r.Quiet, row)
|
||||
case !quiet && row.QuietAlertedAt != 0:
|
||||
r.quietOver = append(r.quietOver, row.Source)
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Commit records that the owner has now been told everything in r.
|
||||
func Commit(now time.Time, r Report) error {
|
||||
at := now.Unix()
|
||||
for _, row := range r.Down {
|
||||
if err := storage.SetFeedDownAlerted(row.Source, at); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, row := range r.Back {
|
||||
if err := storage.SetFeedDownAlerted(row.Source, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, row := range r.Quiet {
|
||||
if err := storage.SetFeedQuietAlerted(row.Source, at); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, src := range r.quietOver {
|
||||
if err := storage.SetFeedQuietAlerted(src, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Message renders r as the plain-text note Pete sends the owner.
|
||||
func Message(now time.Time, r Report) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("Feed check")
|
||||
if len(r.Down) > 0 {
|
||||
b.WriteString("\n\nNot fetching:")
|
||||
for _, row := range r.Down {
|
||||
since := "has never fetched successfully"
|
||||
if row.LastSuccessAt > 0 {
|
||||
since = "last worked " + ago(now, row.LastSuccessAt)
|
||||
}
|
||||
fmt.Fprintf(&b, "\n• %s: %s failed polls in a row, %s. Last error: %s",
|
||||
row.Source, count(row.ConsecutiveFailures), since, short(row.LastError))
|
||||
}
|
||||
}
|
||||
if len(r.Back) > 0 {
|
||||
b.WriteString("\n\nWorking again:")
|
||||
for _, row := range r.Back {
|
||||
fmt.Fprintf(&b, "\n• %s is fetching again.", row.Source)
|
||||
}
|
||||
}
|
||||
if len(r.Quiet) > 0 {
|
||||
b.WriteString("\n\nGone quiet:")
|
||||
for _, row := range r.Quiet {
|
||||
fmt.Fprintf(&b, "\n• %s fetches fine but hasn't had anything new in %s.",
|
||||
row.Source, span(now, row.LastNewAt))
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Send delivers a message to the owner. An error means it didn't arrive, and
|
||||
// the watcher will try again next pass rather than mark it said.
|
||||
type Send func(ctx context.Context, msg string) error
|
||||
|
||||
// Run checks once shortly after startup (so pollers have had a first go) and
|
||||
// then every Every until ctx ends.
|
||||
func Run(ctx context.Context, sources []config.SourceConfig, send Send) {
|
||||
first := time.NewTimer(2 * time.Minute)
|
||||
defer first.Stop()
|
||||
t := time.NewTicker(Every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-first.C:
|
||||
case <-t.C:
|
||||
}
|
||||
pass(ctx, time.Now(), sources, send)
|
||||
}
|
||||
}
|
||||
|
||||
func pass(ctx context.Context, now time.Time, sources []config.SourceConfig, send Send) {
|
||||
r, err := Check(now, sources)
|
||||
if err != nil {
|
||||
slog.Error("feedwatch: check failed", "err", err)
|
||||
return
|
||||
}
|
||||
if r.Empty() {
|
||||
// Still commit: a quiet spell may have ended silently.
|
||||
if err := Commit(now, r); err != nil {
|
||||
slog.Error("feedwatch: commit failed", "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
msg := Message(now, r)
|
||||
if err := send(ctx, msg); err != nil {
|
||||
slog.Error("feedwatch: alert not delivered, will retry", "err", err)
|
||||
return
|
||||
}
|
||||
slog.Info("feedwatch: alert sent", "down", len(r.Down), "back", len(r.Back), "quiet", len(r.Quiet))
|
||||
if err := Commit(now, r); err != nil {
|
||||
slog.Error("feedwatch: commit failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// span renders the time since unix as a rough human length ("9 days").
|
||||
func span(now time.Time, unix int64) string {
|
||||
d := now.Sub(time.Unix(unix, 0))
|
||||
switch {
|
||||
case d < 2*time.Hour:
|
||||
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%d hours", int(d.Hours()))
|
||||
default:
|
||||
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
||||
}
|
||||
}
|
||||
|
||||
// ago is span as a point in the past ("9 days ago").
|
||||
func ago(now time.Time, unix int64) string { return span(now, unix) + " ago" }
|
||||
|
||||
// count renders n with thousands separators.
|
||||
func count(n int) string {
|
||||
s := fmt.Sprint(n)
|
||||
for i := len(s) - 3; i > 0; i -= 3 {
|
||||
s = s[:i] + "," + s[i:]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// short trims an error to one readable line.
|
||||
func short(s string) string {
|
||||
s = strings.TrimSpace(strings.ReplaceAll(s, "\n", " "))
|
||||
if s == "" {
|
||||
return "none recorded"
|
||||
}
|
||||
if len(s) > 140 {
|
||||
s = s[:140] + "…"
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in New Issue
Block a user