Co-op audit fixes: lock scheduler, idempotent payouts, deterministic ties

Audit pass addressed concurrency races, crash-recovery double-pay risks, and
non-determinism in tie tallying.

CRITICAL — concurrency
- coopResolveFloor now acquires advUserLock for every party member (sorted by
  UserID) before mutating state, so concurrent !coop fund / vote / giftvote
  commands serialize against the scheduler.

CRITICAL — crash idempotency
- Add coop_dungeon_runs.last_resolved_day and coop_dungeon_members.member_payout
  columns (with migration entries) for crash-resume tracking.
- Skip resolution only if last_resolved_day >= day AND status is terminal;
  otherwise re-enter and finish via idempotent operations.
- On resume detection (event already has outcome), reuse the saved roll
  instead of re-rolling — prevents different outcomes on retry.
- claimCoopMemberPayout / claimCoopBetPayout: atomic UPDATE...WHERE col IS NULL
  pattern. Each member/bet can only be paid once; retries are silent no-ops.
- createCoopEvent now INSERT OR IGNORE — idempotent on retry.
- last_resolved_day is set as the FINAL write per floor.

HIGH — bugs
- Lazy-create floor event in resolver if missing — covers crash window
  between lockCoopRun and the original createCoopEvent.
- coopTallyVote sorts tied winners alphabetically before fallback (was
  non-deterministic via Go map iteration).
- Removed dead day-1 wipe combat-action refund block — refund check was
  always false because midnight reset clears state before resolution.

LOW
- Log slog.Error on malformed JSON in parseCoopVotes / parseCoopFundingMap
  instead of silently dropping data.

Tests
- Tally test extended to assert determinism across 50 runs (catches map-iteration
  leaks).
- Added coverage for new tie-with-leader-vote-outside-winners case.
- Idempotency-guard sanity check on LastResolvedDay vs CurrentDay.

Race detector: clean. Full suite: green.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
prosolis
2026-04-26 09:01:29 -07:00
parent 8ad31a0009
commit e8a3b8b35d
5 changed files with 279 additions and 149 deletions

View File

@@ -4,7 +4,9 @@ import (
"fmt"
"log/slog"
"math/rand/v2"
"sort"
"strings"
"sync"
"time"
"gogobee/internal/db"
@@ -154,136 +156,183 @@ func (p *AdventurePlugin) coopProcessActiveRuns() {
// funders, rolls outcome, advances or completes the run.
func (p *AdventurePlugin) coopResolveFloor(run *CoopRun) error {
day := run.CurrentDay
// Skip only if this run is fully terminal. A "resolved" day with the run
// still in 'active' status means a prior tick crashed mid-distribution;
// we must re-enter to finish via idempotent operations (per-row claims).
if run.LastResolvedDay >= day && run.Status != "active" {
return nil
}
members, err := loadCoopMembers(run.ID)
if err != nil {
return err
}
// Auto-play any member who didn't fund.
autoPlayed := []*CoopMember{}
for i := range members {
m := &members[i]
if _, ok := m.DailyFunding[day]; !ok {
m.DailyFunding[day] = CoopFundNone
if err := saveCoopMemberFunding(m); err != nil {
slog.Error("coop: auto-play save", "run", run.ID, "user", m.UserID, "err", err)
}
autoPlayed = append(autoPlayed, m)
}
}
// Compute success modifier: base = (100 - baseDifficulty), adjusted by
// funding, party levels, pets, and the floor event vote.
totalMod := 0
choices := make(map[string]CoopFundingTier, len(members))
tierDef := coopTierTable[run.Tier]
// Lock all party members in stable UserID order so concurrent !coop fund /
// vote / giftvote commands don't race the resolver. Stable order rules
// out lock-order deadlock with other goroutines that lock multiple users.
sort.Slice(members, func(i, j int) bool { return members[i].UserID < members[j].UserID })
mutexes := make([]*sync.Mutex, 0, len(members))
for _, m := range members {
tier, mod := coopMemberModifier(&m, day)
choices[string(m.UserID)] = tier
totalMod += mod
// Level + pet bonuses pulled fresh per-floor; they reflect current
// state (a player may have leveled up mid-run).
char, err := loadAdvCharacter(m.UserID)
if err != nil {
continue
}
totalMod += coopLevelBonus(char.CombatLevel, tierDef.minLevel)
totalMod += coopPetBonus(char)
mu := p.advUserLock(m.UserID)
mu.Lock()
mutexes = append(mutexes, mu)
}
defer func() {
for _, mu := range mutexes {
mu.Unlock()
}
}()
// Tally vote on today's floor event. Ties go to the leader's vote (or
// option A if leader didn't vote either).
// Reload the run after acquiring locks. A concurrent admin action or
// another scheduler tick may have moved the state.
freshRun, err := loadCoopRun(run.ID)
if err != nil || freshRun == nil {
return err
}
run = freshRun
if run.Status != "active" {
return nil
}
// Re-check skip after lock acquisition (another tick could have already
// fully finalized — though our per-tick guard makes that unlikely).
if run.LastResolvedDay >= day && run.Status != "active" {
return nil
}
// Reload members under lock for fresh funding state.
members, err = loadCoopMembers(run.ID)
if err != nil {
return err
}
sort.Slice(members, func(i, j int) bool { return members[i].UserID < members[j].UserID })
// Resume vs. fresh detection: if the day's event already has an outcome,
// a prior tick rolled this floor before crashing. Reuse that outcome —
// re-rolling would be the worst sin (different result on retry).
event, _ := loadCoopEvent(run.ID, day)
winning := ""
eventMod := 0
if event != nil {
winning = coopTallyVote(event, run.LeaderID)
eventMod = coopEventOptionModifier(event.Category, event.EventIndex, winning)
totalMod += eventMod
}
resuming := event != nil && event.Outcome != ""
// Resolve gifts received on this day.
giftMod, giftSummary := p.coopResolveGifts(run, run.LeaderID, day)
totalMod += giftMod
autoPlayed := []*CoopMember{}
choices := make(map[string]CoopFundingTier, len(members))
var floorWon bool
var winning string
var eventMod, successPct int
var giftSummary string
successPct := 100 - run.BaseDifficulty + totalMod
if successPct < 5 {
successPct = 5
}
if successPct > 95 {
successPct = 95
}
roll := rand.IntN(100)
floorWon := roll < successPct
// Persist the event resolution before posting (so a crash mid-post still
// records what happened for TwinBee helpfulness tracking).
if event != nil {
outcomeStr := "failure"
if floorWon {
outcomeStr = "success"
if resuming {
floorWon = event.Outcome == "success"
winning = event.WinningVote
eventMod = event.ModifierApplied
// Reconstruct choices for display purposes.
for _, m := range members {
tier, _ := coopMemberModifier(&m, day)
choices[string(m.UserID)] = tier
}
successPct = 0 // unknown on resume; render hides it
} else {
// Auto-play any member who didn't fund.
for i := range members {
m := &members[i]
if _, ok := m.DailyFunding[day]; !ok {
m.DailyFunding[day] = CoopFundNone
if err := saveCoopMemberFunding(m); err != nil {
slog.Error("coop: auto-play save", "run", run.ID, "user", m.UserID, "err", err)
}
autoPlayed = append(autoPlayed, m)
}
}
totalMod := 0
tierDef := coopTierTable[run.Tier]
for _, m := range members {
tier, mod := coopMemberModifier(&m, day)
choices[string(m.UserID)] = tier
totalMod += mod
char, err := loadAdvCharacter(m.UserID)
if err != nil {
continue
}
totalMod += coopLevelBonus(char.CombatLevel, tierDef.minLevel)
totalMod += coopPetBonus(char)
}
// Lazy-create the day's event if missing — covers the crash window
// between lock and the original event creation.
if event == nil {
cat := pickCoopEventCategory(run.Tier)
idx := pickCoopEvent(cat)
_ = createCoopEvent(run.ID, day, cat, idx, coopEventRecommended(cat, idx))
event, _ = loadCoopEvent(run.ID, day)
}
if event != nil {
winning = coopTallyVote(event, run.LeaderID)
eventMod = coopEventOptionModifier(event.Category, event.EventIndex, winning)
totalMod += eventMod
}
var giftMod int
giftMod, giftSummary = p.coopResolveGifts(run, run.LeaderID, day)
totalMod += giftMod
successPct = 100 - run.BaseDifficulty + totalMod
if successPct < 5 {
successPct = 5
}
if successPct > 95 {
successPct = 95
}
roll := rand.IntN(100)
floorWon = roll < successPct
// Persist event resolution — turns this into a "resume" path on retry.
if event != nil {
outcomeStr := "failure"
if floorWon {
outcomeStr = "success"
}
_ = resolveCoopEvent(run.ID, day, winning, outcomeStr, eventMod)
}
dailyPost := renderCoopDailyResult(p, run, members, day, choices, successPct, floorWon, autoPlayed, event, winning, eventMod)
if giftSummary != "" {
dailyPost += "\n\n" + giftSummary
}
if gr := gamesRoom(); gr != "" {
_ = p.SendMessage(gr, dailyPost)
}
_ = resolveCoopEvent(run.ID, day, winning, outcomeStr, eventMod)
}
gr := gamesRoom()
dailyPost := renderCoopDailyResult(p, run, members, day, choices, successPct, floorWon, autoPlayed, event, winning, eventMod)
if giftSummary != "" {
dailyPost += "\n\n" + giftSummary
}
if gr != "" {
_ = p.SendMessage(gr, dailyPost)
}
if !floorWon {
// Wipe. Combat actions are only consumed at lock (day 1), so only a
// day-1 wipe has anything to refund.
if day == 1 {
now := time.Now().UTC().Format("2006-01-02")
for _, m := range members {
char, err := loadAdvCharacter(m.UserID)
if err != nil {
continue
}
if char.LastActionDate == now && char.CombatActionsUsed > 0 {
char.CombatActionsUsed--
_ = saveAdvCharacter(char)
}
}
// Resolve bets first (idempotent via claimCoopBetPayout), then mark
// the run wiped. Order matters: if status is set first and the bot
// crashes, the next tick won't pick this run up.
summary := p.coopResolveBets(run, false)
giftLog := renderCoopGiftLog(p, run.ID)
parts := []string{}
if summary != "" {
parts = append(parts, summary)
}
if giftLog != "" {
parts = append(parts, giftLog)
}
if len(parts) > 0 && gr != "" {
_ = p.SendMessage(gr, strings.Join(parts, "\n\n"))
}
_ = completeCoopRun(run.ID, "wiped", 0)
// Resolve betting pool — failure side wins on a wipe.
freshRun, _ := loadCoopRun(run.ID)
if freshRun != nil {
summary := p.coopResolveBets(freshRun, false)
giftLog := renderCoopGiftLog(p, run.ID)
parts := []string{}
if summary != "" {
parts = append(parts, summary)
}
if giftLog != "" {
parts = append(parts, giftLog)
}
if len(parts) > 0 {
if gr := gamesRoom(); gr != "" {
_ = p.SendMessage(gr, strings.Join(parts, "\n\n"))
}
}
}
_ = setCoopRunLastResolvedDay(run.ID, day)
return nil
}
// Floor cleared — advance or complete.
if day >= run.TotalDays {
// Run complete — split reward.
def := coopTierTable[run.Tier]
reward := def.rewardBase
p.coopDistributeReward(run, reward, members)
_ = completeCoopRun(run.ID, "complete", reward)
freshRun, _ := loadCoopRun(run.ID)
if freshRun != nil {
p.coopDistributeReward(freshRun, reward, members)
}
_ = setCoopRunLastResolvedDay(run.ID, day)
return nil
}
if err := advanceCoopDay(run.ID, day+1); err != nil {
@@ -292,16 +341,17 @@ func (p *AdventurePlugin) coopResolveFloor(run *CoopRun) error {
for _, m := range members {
_ = p.SendDM(m.UserID, renderCoopFundingPrompt(run, day+1))
}
// Generate next day's floor event.
freshRun, _ := loadCoopRun(run.ID)
freshRun, _ = loadCoopRun(run.ID)
if freshRun != nil {
p.coopGenerateAndPostEvent(freshRun, members, day+1)
}
_ = setCoopRunLastResolvedDay(run.ID, day)
return nil
}
// coopTallyVote counts votes; majority wins. Ties resolve to the leader's
// vote if any, otherwise option A.
// vote if it's among the tied options; otherwise the lexicographically first
// tied label (deterministic, since Go map iteration is randomized).
func coopTallyVote(event *CoopEvent, leader id.UserID) string {
tally := map[string]int{}
for _, v := range event.Votes {
@@ -320,6 +370,7 @@ func coopTallyVote(event *CoopEvent, leader id.UserID) string {
if len(winners) == 1 {
return winners[0]
}
sort.Strings(winners)
if leaderVote, ok := event.Votes[leader]; ok {
for _, w := range winners {
if w == leaderVote {
@@ -341,6 +392,12 @@ func (p *AdventurePlugin) coopDistributeReward(run *CoopRun, totalReward int, me
gr := gamesRoom()
var lines []string
for _, m := range members {
// Atomic claim: only credit if this is the first time we're paying
// this member for this run. Skip silently on retry.
claimed, err := claimCoopMemberPayout(run.ID, m.UserID, share)
if err != nil || !claimed {
continue
}
net, _ := communityTax(m.UserID, float64(share), coopAdventureRake)
p.euro.Credit(m.UserID, net, "coop_dungeon_reward")
lines = append(lines, fmt.Sprintf(" %s: €%.0f (after tax)", coopDisplayName(p, m.UserID), net))