storage: keep the stories people saved, and prune more than once a deploy

Maintenance deleted every story older than 30 days and then swept up any
user_story_state row pointing at a story that no longer existed. A bookmark
is one of those rows, so every saved story quietly vanished a month after
Pete first saw it, which is exactly what saving it was meant to prevent. A
bookmarked story is now exempt from the prune; read markers on pruned
stories still go.

Maintenance also only ran at startup, on a service that can go weeks between
deploys, so nothing was pruned for weeks and then everything was at once. It
runs at startup and then daily.

Pruning daily makes one old hole steady-state: the stories row is Pete's only
memory of a GUID, so a slow feed still listing month-old items would meet
them again as new the day after their rows went. The poller now skips items
published before the retention window. Undated items can't be judged and
still pass.

internal/ingestion/poller.go

@@ -118,6 +118,16 @@ func (p *Poller) pollOnceWithErr(ctx context.Context, src config.SourceConfig) e
 			continue
 		}
 
+		// An item older than the retention window is one Pete has either
+		// already shown and since pruned, or never needs to: maintenance would
+		// delete it on the next pass. Without this, a slow feed that still
+		// lists month-old items re-ingests them as new the day after their row
+		// is pruned, since that row was the only memory of them. Items with no
+		// date can't be judged and pass through.
+		if items[i].PublishedAt > 0 && time.Since(time.Unix(items[i].PublishedAt, 0)) > storage.StoryRetention {
+			continue
+		}
+
 		// Drop items in languages this source isn't configured to surface.
 		// Items without a language tag pass through — only filter when the
 		// feed explicitly marks an item with a non-matching language.

internal/storage/db.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"log/slog"
@@ -8,6 +9,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	_ "modernc.org/sqlite"
 )
@@ -154,12 +156,41 @@ func runMigrations(d *sql.DB) error {
 	return nil
 }
 
-// RunMaintenance prunes stale data. Called periodically.
+// StoryRetention is how long a story is kept after Pete first saw it. It is also
+// how long Pete remembers having seen it: the stories row is the only record of
+// its GUID, canonical URL and headline, so the poller ignores feed items older
+// than this rather than meet them again as strangers once their row is gone.
+const StoryRetention = 30 * 24 * time.Hour
+
+// StartMaintenance runs RunMaintenance now and then once a day until ctx ends.
+// It used to run only at startup, so on a service that went weeks between
+// deploys nothing was pruned, and then everything was, all at once.
+func StartMaintenance(ctx context.Context) {
+	RunMaintenance()
+	go func() {
+		t := time.NewTicker(24 * time.Hour)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+				RunMaintenance()
+			}
+		}
+	}()
+}
+
+// RunMaintenance prunes stale data.
 func RunMaintenance() {
-	// Prune old stories (30 days) and their post logs / reactions
+	// Prune old stories and their post logs / reactions. A story someone has
-	storyCutoff := nowUnix() - int64(30*86400)
+	// bookmarked is kept however old it is: the orphan sweep below would
+	// otherwise take the bookmark with it, and a saved story that quietly
+	// vanishes after a month is exactly what saving it was meant to prevent.
+	storyCutoff := nowUnix() - int64(StoryRetention/time.Second)
 	exec("prune old stories",
-		`DELETE FROM stories WHERE seen_at < ? AND classified = 1`, storyCutoff)
+		`DELETE FROM stories WHERE seen_at < ? AND classified = 1
+		   AND id NOT IN (SELECT story_id FROM user_story_state WHERE bookmarked_at IS NOT NULL)`, storyCutoff)
 	exec("prune old post_log",
 		`DELETE FROM post_log WHERE posted_at < ?`, storyCutoff)
 	exec("prune old reactions",

internal/storage/userstate_test.go

@@ -124,3 +124,46 @@ func TestUserStoryState_ScopedBySub(t *testing.T) {
 		t.Fatalf("user-b count = %d, want 0", n)
 	}
 }
+
+// Maintenance prunes stories older than the retention window, and then sweeps
+// away user rows pointing at them. A bookmark is someone asking for a story to
+// be kept, so a bookmarked story must survive the prune, and its bookmark with
+// it. Read markers on pruned stories are fair game.
+func TestMaintenanceKeepsBookmarkedStories(t *testing.T) {
+	setupTestDB(t)
+	saved := seedStory(t, "saved")   // seedStory stamps 2023, far past retention
+	readOnly := seedStory(t, "read") // read, not saved
+	seedStory(t, "untouched")
+	const sub = "ak-sub-keep"
+
+	if err := SetBookmark(sub, saved, true); err != nil {
+		t.Fatal(err)
+	}
+	if err := SetRead(sub, readOnly, true); err != nil {
+		t.Fatal(err)
+	}
+
+	RunMaintenance()
+
+	var n int
+	if err := Get().QueryRow(`SELECT COUNT(*) FROM stories`).Scan(&n); err != nil {
+		t.Fatal(err)
+	}
+	if n != 1 {
+		t.Errorf("stories left after maintenance = %d, want 1 (the bookmarked one)", n)
+	}
+	marks, err := ListBookmarks(sub, 10, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(marks) != 1 || marks[0].GUID != "saved" {
+		t.Fatalf("bookmarks after maintenance = %+v, want just the saved story", marks)
+	}
+	read, _, err := UserStoryState(sub, []int64{readOnly})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if read[readOnly] {
+		t.Error("read marker on a pruned story survived the orphan sweep")
+	}
+}

main.go

@@ -305,8 +305,9 @@ func main() {
 		}
 	}
 
-	// Run maintenance on startup
+	// Prune now and then daily. Startup-only meant a service that ran for weeks
-	storage.RunMaintenance()
+	// between deploys never pruned at all.
+	storage.StartMaintenance(ctx)
 
 	// Wait for shutdown signal
 	sig := <-sigCh
@@ -423,7 +424,7 @@ func runLocal(cfg *config.Config) {
 	ws.StartAdventureAlerts(ctx)
 	slog.Info("local: web UI listening", "addr", cfg.Web.ListenAddr)
 
-	storage.RunMaintenance()
+	storage.StartMaintenance(ctx)
 
 	sig := <-sigCh
 	slog.Info("local: shutdown signal received", "signal", sig)