storage: keep the stories people saved, and prune more than once a deploy
Maintenance deleted every story older than 30 days and then swept up any user_story_state row pointing at a story that no longer existed. A bookmark is one of those rows, so every saved story quietly vanished a month after Pete first saw it, which is exactly what saving it was meant to prevent. A bookmarked story is now exempt from the prune; read markers on pruned stories still go. Maintenance also only ran at startup, on a service that can go weeks between deploys, so nothing was pruned for weeks and then everything was at once. It runs at startup and then daily. Pruning daily makes one old hole steady-state: the stories row is Pete's only memory of a GUID, so a slow feed still listing month-old items would meet them again as new the day after their rows went. The poller now skips items published before the retention window. Undated items can't be judged and still pass.
This commit is contained in:
+35
-4
@@ -1,6 +1,7 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -8,6 +9,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
@@ -154,12 +156,41 @@ func runMigrations(d *sql.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunMaintenance prunes stale data. Called periodically.
|
||||
// StoryRetention is how long a story is kept after Pete first saw it. It is also
|
||||
// how long Pete remembers having seen it: the stories row is the only record of
|
||||
// its GUID, canonical URL and headline, so the poller ignores feed items older
|
||||
// than this rather than meet them again as strangers once their row is gone.
|
||||
const StoryRetention = 30 * 24 * time.Hour
|
||||
|
||||
// StartMaintenance runs RunMaintenance now and then once a day until ctx ends.
|
||||
// It used to run only at startup, so on a service that went weeks between
|
||||
// deploys nothing was pruned, and then everything was, all at once.
|
||||
func StartMaintenance(ctx context.Context) {
|
||||
RunMaintenance()
|
||||
go func() {
|
||||
t := time.NewTicker(24 * time.Hour)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
RunMaintenance()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// RunMaintenance prunes stale data.
|
||||
func RunMaintenance() {
|
||||
// Prune old stories (30 days) and their post logs / reactions
|
||||
storyCutoff := nowUnix() - int64(30*86400)
|
||||
// Prune old stories and their post logs / reactions. A story someone has
|
||||
// bookmarked is kept however old it is: the orphan sweep below would
|
||||
// otherwise take the bookmark with it, and a saved story that quietly
|
||||
// vanishes after a month is exactly what saving it was meant to prevent.
|
||||
storyCutoff := nowUnix() - int64(StoryRetention/time.Second)
|
||||
exec("prune old stories",
|
||||
`DELETE FROM stories WHERE seen_at < ? AND classified = 1`, storyCutoff)
|
||||
`DELETE FROM stories WHERE seen_at < ? AND classified = 1
|
||||
AND id NOT IN (SELECT story_id FROM user_story_state WHERE bookmarked_at IS NOT NULL)`, storyCutoff)
|
||||
exec("prune old post_log",
|
||||
`DELETE FROM post_log WHERE posted_at < ?`, storyCutoff)
|
||||
exec("prune old reactions",
|
||||
|
||||
Reference in New Issue
Block a user