diff --git a/internal/ingestion/poller.go b/internal/ingestion/poller.go index e8ac632..c99348c 100644 --- a/internal/ingestion/poller.go +++ b/internal/ingestion/poller.go @@ -118,6 +118,16 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e continue } + // An item older than the retention window is one Pete has either + // already shown and since pruned, or never needs to: maintenance would + // delete it on the next pass. Without this, a slow feed that still + // lists month-old items re-ingests them as new the day after their row + // is pruned, since that row was the only memory of them. Items with no + // date can't be judged and pass through. + if items[i].PublishedAt > 0 && time.Since(time.Unix(items[i].PublishedAt, 0)) > storage.StoryRetention { + continue + } + // Drop items in languages this source isn't configured to surface. // Items without a language tag pass through — only filter when the // feed explicitly marks an item with a non-matching language. diff --git a/internal/storage/db.go b/internal/storage/db.go index 6843c19..0190679 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -1,6 +1,7 @@ package storage import ( + "context" "database/sql" "fmt" "log/slog" @@ -8,6 +9,7 @@ import ( "path/filepath" "strings" "sync" + "time" _ "modernc.org/sqlite" ) @@ -154,12 +156,41 @@ func runMigrations(d *sql.DB) error { return nil } -// RunMaintenance prunes stale data. Called periodically. +// StoryRetention is how long a story is kept after Pete first saw it. It is also +// how long Pete remembers having seen it: the stories row is the only record of +// its GUID, canonical URL and headline, so the poller ignores feed items older +// than this rather than meet them again as strangers once their row is gone. +const StoryRetention = 30 * 24 * time.Hour + +// StartMaintenance runs RunMaintenance now and then once a day until ctx ends. +// It used to run only at startup, so on a service that went weeks between +// deploys nothing was pruned, and then everything was, all at once. +func StartMaintenance(ctx context.Context) { + RunMaintenance() + go func() { + t := time.NewTicker(24 * time.Hour) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + RunMaintenance() + } + } + }() +} + +// RunMaintenance prunes stale data. func RunMaintenance() { - // Prune old stories (30 days) and their post logs / reactions - storyCutoff := nowUnix() - int64(30*86400) + // Prune old stories and their post logs / reactions. A story someone has + // bookmarked is kept however old it is: the orphan sweep below would + // otherwise take the bookmark with it, and a saved story that quietly + // vanishes after a month is exactly what saving it was meant to prevent. + storyCutoff := nowUnix() - int64(StoryRetention/time.Second) exec("prune old stories", - `DELETE FROM stories WHERE seen_at < ? AND classified = 1`, storyCutoff) + `DELETE FROM stories WHERE seen_at < ? AND classified = 1 + AND id NOT IN (SELECT story_id FROM user_story_state WHERE bookmarked_at IS NOT NULL)`, storyCutoff) exec("prune old post_log", `DELETE FROM post_log WHERE posted_at < ?`, storyCutoff) exec("prune old reactions", diff --git a/internal/storage/userstate_test.go b/internal/storage/userstate_test.go index de7232a..af82b67 100644 --- a/internal/storage/userstate_test.go +++ b/internal/storage/userstate_test.go @@ -124,3 +124,46 @@ func TestUserStoryState_ScopedBySub(t *testing.T) { t.Fatalf("user-b count = %d, want 0", n) } } + +// Maintenance prunes stories older than the retention window, and then sweeps +// away user rows pointing at them. A bookmark is someone asking for a story to +// be kept, so a bookmarked story must survive the prune, and its bookmark with +// it. Read markers on pruned stories are fair game. +func TestMaintenanceKeepsBookmarkedStories(t *testing.T) { + setupTestDB(t) + saved := seedStory(t, "saved") // seedStory stamps 2023, far past retention + readOnly := seedStory(t, "read") // read, not saved + seedStory(t, "untouched") + const sub = "ak-sub-keep" + + if err := SetBookmark(sub, saved, true); err != nil { + t.Fatal(err) + } + if err := SetRead(sub, readOnly, true); err != nil { + t.Fatal(err) + } + + RunMaintenance() + + var n int + if err := Get().QueryRow(`SELECT COUNT(*) FROM stories`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("stories left after maintenance = %d, want 1 (the bookmarked one)", n) + } + marks, err := ListBookmarks(sub, 10, 0) + if err != nil { + t.Fatal(err) + } + if len(marks) != 1 || marks[0].GUID != "saved" { + t.Fatalf("bookmarks after maintenance = %+v, want just the saved story", marks) + } + read, _, err := UserStoryState(sub, []int64{readOnly}) + if err != nil { + t.Fatal(err) + } + if read[readOnly] { + t.Error("read marker on a pruned story survived the orphan sweep") + } +} diff --git a/main.go b/main.go index ef1714d..b0ddb2c 100644 --- a/main.go +++ b/main.go @@ -305,8 +305,9 @@ func main() { } } - // Run maintenance on startup - storage.RunMaintenance() + // Prune now and then daily. Startup-only meant a service that ran for weeks + // between deploys never pruned at all. + storage.StartMaintenance(ctx) // Wait for shutdown signal sig := <-sigCh @@ -423,7 +424,7 @@ func runLocal(cfg *config.Config) { ws.StartAdventureAlerts(ctx) slog.Info("local: web UI listening", "addr", cfg.Web.ListenAddr) - storage.RunMaintenance() + storage.StartMaintenance(ctx) sig := <-sigCh slog.Info("local: shutdown signal received", "signal", sig)