feedwatch: tell the owner when a feed breaks, comes back, or goes quiet
Every poll's outcome has always been written to source_health and shown on /status, but only to someone who went and looked. Nobody did: Siliconera has been refused on 3,769 polls in a row and Digital Foundry on 1,900, and neither has ever fetched successfully. SendAdminWarning existed and nothing called it. A watcher now reads source_health every 15 minutes and sends one notice to matrix.admin_room covering everything that changed since it last spoke: feeds that have failed 5 polls in a row, feeds that have fetched since they were reported down, and feeds that fetch fine but have kept nothing new for 7 days (every healthy source produces something inside about 4). What has been said is stored on the row (down_alerted_at, quiet_alerted_at), so each episode is reported once and a restart doesn't repeat it. A notice that fails to send is not marked, and goes out on the next pass. last_new_at is new, moved only when a poll keeps a story. Existing rows start from their newest retained story, or from now if they have none, and a new source starts at its first poll: nothing on disk says whether a source with no stories is long dead or was added yesterday, so it gets a full window before it can be called quiet. With no admin room configured the watcher doesn't run at all, rather than marking alerts as delivered to nowhere.
This commit is contained in:
prosolis
@@ -0,0 +1,179 @@
|
||||
package feedwatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"pete/internal/config"
|
||||
"pete/internal/storage"
|
||||
)
|
||||
|
||||
func setup(t *testing.T) {
|
||||
t.Helper()
|
||||
storage.Close()
|
||||
if err := storage.Init(filepath.Join(t.TempDir(), "feedwatch.db")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { storage.Close() })
|
||||
}
|
||||
|
||||
func src(names ...string) []config.SourceConfig {
|
||||
var out []config.SourceConfig
|
||||
for _, n := range names {
|
||||
out = append(out, config.SourceConfig{Name: n, Enabled: true})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fail(name string, n int) {
|
||||
for range n {
|
||||
storage.RecordPollResult(name, false, 0, errors.New("feed fetch https://x/feed: status 403"))
|
||||
}
|
||||
}
|
||||
|
||||
// setLastNew backdates a source's last new story.
|
||||
func setLastNew(t *testing.T, name string, at time.Time) {
|
||||
t.Helper()
|
||||
if _, err := storage.Get().Exec(`UPDATE source_health SET last_new_at = ? WHERE source = ?`, at.Unix(), name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// capture is a Send that records what it was asked to deliver.
|
||||
type capture struct {
|
||||
msgs []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *capture) send(_ context.Context, msg string) error {
|
||||
if c.err != nil {
|
||||
return c.err
|
||||
}
|
||||
c.msgs = append(c.msgs, msg)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestDownIsReportedOnceAndBackIsReported(t *testing.T) {
|
||||
setup(t)
|
||||
sources := src("Broken", "Fine")
|
||||
now := time.Now()
|
||||
|
||||
storage.RecordPollResult("Fine", true, 10, nil)
|
||||
fail("Broken", DownAfter-1)
|
||||
|
||||
c := &capture{}
|
||||
pass(context.Background(), now, sources, c.send)
|
||||
if len(c.msgs) != 0 {
|
||||
t.Fatalf("alerted below the threshold: %q", c.msgs)
|
||||
}
|
||||
|
||||
fail("Broken", 1)
|
||||
pass(context.Background(), now, sources, c.send)
|
||||
if len(c.msgs) != 1 || !strings.Contains(c.msgs[0], "Broken: 5 failed polls in a row, has never fetched successfully") {
|
||||
t.Fatalf("down alert = %q", c.msgs)
|
||||
}
|
||||
if strings.Contains(c.msgs[0], "Fine") {
|
||||
t.Errorf("a healthy feed was mentioned: %q", c.msgs[0])
|
||||
}
|
||||
|
||||
// Still down on the next pass: nothing new to say.
|
||||
fail("Broken", 3)
|
||||
pass(context.Background(), now.Add(Every), sources, c.send)
|
||||
if len(c.msgs) != 1 {
|
||||
t.Fatalf("down alert repeated: %q", c.msgs[1:])
|
||||
}
|
||||
|
||||
// It comes back. The success has to be stamped after the alert, so step
|
||||
// past the second the alert was recorded in.
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
storage.RecordPollResult("Broken", true, 5, nil)
|
||||
pass(context.Background(), time.Now(), sources, c.send)
|
||||
if len(c.msgs) != 2 || !strings.Contains(c.msgs[1], "Broken is fetching again") {
|
||||
t.Fatalf("recovery = %q", c.msgs)
|
||||
}
|
||||
pass(context.Background(), time.Now(), sources, c.send)
|
||||
if len(c.msgs) != 2 {
|
||||
t.Fatalf("recovery repeated: %q", c.msgs[2:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestUndeliveredAlertIsRetried(t *testing.T) {
|
||||
setup(t)
|
||||
sources := src("Broken")
|
||||
fail("Broken", DownAfter)
|
||||
|
||||
c := &capture{err: errors.New("homeserver unreachable")}
|
||||
pass(context.Background(), time.Now(), sources, c.send)
|
||||
|
||||
c.err = nil
|
||||
pass(context.Background(), time.Now(), sources, c.send)
|
||||
if len(c.msgs) != 1 || !strings.Contains(c.msgs[0], "Broken") {
|
||||
t.Fatalf("alert after a failed send = %q, want it delivered on the retry", c.msgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuietFeedIsReportedOnce(t *testing.T) {
|
||||
setup(t)
|
||||
sources := src("Sleepy", "Busy")
|
||||
now := time.Now()
|
||||
storage.RecordPollResult("Sleepy", true, 10, nil)
|
||||
storage.RecordPollResult("Busy", true, 10, nil)
|
||||
setLastNew(t, "Sleepy", now.Add(-9*24*time.Hour))
|
||||
|
||||
c := &capture{}
|
||||
pass(context.Background(), now, sources, c.send)
|
||||
if len(c.msgs) != 1 || !strings.Contains(c.msgs[0], "Sleepy fetches fine but hasn't had anything new in 9 days") {
|
||||
t.Fatalf("quiet alert = %q", c.msgs)
|
||||
}
|
||||
if strings.Contains(c.msgs[0], "Busy") {
|
||||
t.Errorf("a busy feed was called quiet: %q", c.msgs[0])
|
||||
}
|
||||
pass(context.Background(), now.Add(Every), sources, c.send)
|
||||
if len(c.msgs) != 1 {
|
||||
t.Fatalf("quiet alert repeated: %q", c.msgs[1:])
|
||||
}
|
||||
|
||||
// New stories end the quiet spell without a message, and a later quiet
|
||||
// spell is a new episode that gets reported again.
|
||||
storage.RecordNewStories("Sleepy", 2)
|
||||
pass(context.Background(), now, sources, c.send)
|
||||
if len(c.msgs) != 1 {
|
||||
t.Fatalf("end of a quiet spell was announced: %q", c.msgs[1:])
|
||||
}
|
||||
setLastNew(t, "Sleepy", now.Add(-8*24*time.Hour))
|
||||
pass(context.Background(), now, sources, c.send)
|
||||
if len(c.msgs) != 2 {
|
||||
t.Fatalf("second quiet spell not reported: %q", c.msgs)
|
||||
}
|
||||
}
|
||||
|
||||
// A feed that has just been added has no history, so it must not be called
|
||||
// quiet before it has had a full window to produce something.
|
||||
func TestNewSourceGetsAGracePeriod(t *testing.T) {
|
||||
setup(t)
|
||||
storage.RecordPollResult("Fresh", true, 0, nil)
|
||||
r, err := Check(time.Now().Add(QuietAfter-time.Hour), src("Fresh"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !r.Empty() {
|
||||
t.Fatalf("new source reported inside its grace period: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisabledSourcesNeverAlert(t *testing.T) {
|
||||
setup(t)
|
||||
fail("Retired", DownAfter*2)
|
||||
sources := []config.SourceConfig{{Name: "Retired", Enabled: false}}
|
||||
r, err := Check(time.Now(), sources)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !r.Empty() {
|
||||
t.Fatalf("disabled source alerted: %+v", r)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user