Pete/internal/games/holdem/cfr.go

package holdem

import (
	_ "embed"
	"fmt"
	"log/slog"
	"math/rand/v2"
	"sync"
	"sync/atomic"

	"pete/internal/games/cards"
)

// The bots' brain.
//
// gogobee ran counterfactual regret minimisation against this game for a very
// long time, and policy.gob is what it converged on: a table from "the situation
// I am in" to "how often I fold, call, raise small, raise big, or shove". It is
// the single highest-value thing in either repository, and none of it is
// re-derived here — this file is the *runtime*, the trainer stayed behind.
//
// A situation is squeezed down to six things, and this is the whole reason the
// table fits in memory: the street, whether the bot is in position, which of
// twelve equity buckets its hand falls in, which of five stack-to-pot buckets,
// whether the board is dry, wet or paired, and the last six actions. Two hands
// that hash to the same key get the same strategy, and that is the approximation
// the whole thing is built on.
//
// The key has to be *exactly* the key the trainer wrote, character for
// character. Change the bucket edges, the position label or the history encoding
// and every lookup misses — silently, because a miss is not an error, it is a
// fall back to the pot-odds rule below. The bots would get quietly, unaccountably
// worse. So: don't touch these numbers.

//go:embed policy.gob
var policyGob []byte

// The five things the trainer let a bot consider.
const (
	actFold = iota
	actCallCheck
	actRaiseHalf
	actRaisePot
	actAllIn
	numActions
)

// policyTable maps an info-set key to how often to take each action.
type policyTable map[string][numActions]float64

var (
	policyOnce sync.Once
	policy     policyTable
)

// loadPolicy decodes the embedded table, once, on the first hand anybody plays.
//
// Not in an init(): it is megabytes of gob, and Pete is a news server that mostly
// never deals a card. Every test in the repo and every cold start would pay for
// it. The first player to sit down pays for it instead, and only they do.
func loadPolicy() policyTable {
	policyOnce.Do(func() {
		t, err := loadTrained(policyGob)
		if err != nil {
			// The bots still play — on pot odds — rather than the table 500ing.
			slog.Error("holdem: cannot decode CFR policy, bots fall back to pot odds", "err", err)
			policy = policyTable{}
			return
		}
		policy = t.Strategy
		slog.Info("holdem: CFR policy loaded", "nodes", len(policy),
			"iterations", t.Meta.Iterations, "stakes", t.Meta.Stakes, "depths", t.Meta.Depths)
	})
	return policy
}

// ---- the info-set key ------------------------------------------------------
//
// Every function below is a load-bearing copy of the trainer's. See the note at
// the top of the file before changing a number in any of them.

// equityBucket puts a hand's strength in one of twelve boxes, from trash to
// monster.
func equityBucket(eq float64) int {
	switch {
	case eq < 0.08:
		return 0
	case eq < 0.17:
		return 1
	case eq < 0.25:
		return 2
	case eq < 0.33:
		return 3
	case eq < 0.42:
		return 4
	case eq < 0.50:
		return 5
	case eq < 0.58:
		return 6
	case eq < 0.67:
		return 7
	case eq < 0.75:
		return 8
	case eq < 0.83:
		return 9
	case eq < 0.92:
		return 10
	default:
		return 11
	}
}

// sprBucket is the stack-to-pot ratio: how much room is left to play. Under 1 is
// a pot that is already committed; over 12 is a pot you can still fold out of.
func sprBucket(spr float64) int {
	switch {
	case spr < 1:
		return 0
	case spr < 3:
		return 1
	case spr < 6:
		return 2
	case spr < 12:
		return 3
	default:
		return 4
	}
}

// Board textures, which is what makes the same hand a bet or a check.
const (
	boardDry    = 0
	boardWet    = 1
	boardPaired = 2
)

// boardTexture classifies the community cards. A paired board is one somebody
// might have trips on; a wet one has a flush or straight coming.
// It is on the trainer's hot path — millions of calls — so it counts into arrays
// rather than maps. A map here cost more than the poker did.
func boardTexture(board []cards.Card) int {
	if len(board) < 3 {
		return boardDry
	}

	var ranks [14]int8
	var suits [4]int8
	var vals [5]int
	for i, c := range board {
		ranks[c.Rank]++
		suits[c.Suit]++
		vals[i] = int(c.Rank)
	}
	for _, n := range ranks {
		if n >= 2 {
			return boardPaired
		}
	}
	for _, n := range suits {
		if n >= 3 {
			return boardWet
		}
	}

	// Three cards inside a five-rank window is a straight waiting to happen.
	v := vals[:len(board)]
	for i := 1; i < len(v); i++ {
		for j := i; j > 0 && v[j] < v[j-1]; j-- {
			v[j], v[j-1] = v[j-1], v[j]
		}
	}
	for i := 0; i+2 < len(v); i++ {
		if v[i+2]-v[i] <= 4 {
			return boardWet
		}
	}
	return boardDry
}

// infoSet builds the string the policy is keyed on. The format is the trainer's.
//
// Position is IP or OOP — in position or out of it — and *nothing else*. This is
// the one thing gogobee got wrong, and it got it wrong invisibly for as long as
// the game has existed: the trainer packed a single "am I last to act" bit and
// wrote its keys as IP/OOP, while the runtime looked them up with the table
// labels a player would recognise (BTN, SB, BB, UTG…). Not one key ever matched.
// Every bot in every hand of hold'em gogobee ever dealt fell through to the
// pot-odds rule, and the five million training iterations sitting in policy.gob
// were never once read.
//
// Nothing about that looks broken from the outside. A missing key is not an
// error, it's a fallback — the bots played, they just played a heuristic. This is
// why the hit rate is now a test.
func infoSet(street Street, inPosition bool, eq, spr, texture int, history string) string {
	pos := "OOP"
	if inPosition {
		pos = "IP"
	}
	return fmt.Sprintf("%d|%s|%d|%d|%d|%s", street, pos, eq, spr, texture, history)
}

// recent keeps the last six actions, which is all the trainer's key had room for.
func recent(h string) string {
	if len(h) > 6 {
		return h[len(h)-6:]
	}
	return h
}

// ---- where a seat stands ---------------------------------------------------

// mcIters is how many runouts a bot samples to judge its hand at the table.
const mcIters = 1000

// spot is everything the policy knows about a seat's situation, and it is the
// one function that builds it.
//
// The trainer calls this too. That is the point of it: the key the policy is
// written under and the key it is read under come out of the same code, so they
// cannot quietly stop matching — which is exactly what went wrong the first time
// and went unnoticed for the life of the game.
func (s State) spot(seat, iters int, rng *rand.Rand) (string, Equity) {
	eq := s.equityFor(seat, iters, rng)
	return s.spotKey(seat, eq), eq
}

// equityFor measures how good a seat's hand is right now.
//
// It depends only on the cards — the hand, the board, how many opponents — and
// not on a single thing that happened in the betting. Which is why the trainer
// can measure it once per street and reuse it down every branch it explores, and
// why doing that is most of the difference between a run that takes half an hour
// and one that takes four.
func (s State) equityFor(seat, iters int, rng *rand.Rand) Equity {
	opponents := s.liveCount() - 1
	if opponents < 1 {
		opponents = 1
	}
	// Preflop heads-up is a lookup, not a simulation: there are only 169 hands
	// that differ, they have been measured to death, and a sampled answer would
	// only add noise to a bucket boundary.
	if s.Street == PreFlop && opponents == 1 {
		return preflopEquity(s.Seats[seat].Hole)
	}
	return equityOf(s.Seats[seat].Hole, s.Community, opponents, iters, rng)
}

// spotKey builds the key from an equity already measured.
func (s State) spotKey(seat int, eq Equity) string {
	pot := s.inPlay()
	spr := 0.0
	if pot > 0 {
		spr = float64(s.Seats[seat].Stack) / float64(pot)
	}
	return infoSet(s.Street, s.InPosition(seat), equityBucket(eq.Strength()),
		sprBucket(spr), boardTexture(s.Community), recent(s.History))
}

// ---- choosing --------------------------------------------------------------

// hits and misses count how often a bot finds itself in the trained policy.
//
// They exist because the way this can break is silently. A key the policy has
// never seen is not an error — the bot shrugs and plays pot odds — so a policy
// that has stopped matching the game looks exactly like a policy that is working.
// gogobee's never matched once, for the whole life of the game, and nobody could
// have known by watching it play. Now a test reads these and fails.
var hits, misses atomic.Int64

// botActs plays one bot's turn: work out where it stands, look up what it does
// there, throw out anything illegal, and roll for it.
func (s *State) botActs(seat int, evs *[]Event, rng *rand.Rand) {
	key, eq := s.spot(seat, mcIters, rng)

	probs, ok := loadPolicy()[key]
	if ok {
		hits.Add(1)
	} else {
		misses.Add(1)
		probs = potOdds(eq, s, seat)
	}
	probs = legal(probs, s, seat)

	move := s.moveFor(pick(probs, rng), seat)
	if err := s.act(seat, move, evs); err != nil {
		// A bot cannot be allowed to wedge the table by choosing something the rules
		// then refuse: it checks if it can and folds if it can't, and the mismatch
		// is loud, because it means legal() and the betting rules disagree.
		slog.Error("holdem: bot chose an illegal move", "seat", seat, "move", move.Kind, "err", err)
		if s.Owed(seat) > 0 {
			s.fold(seat, evs)
		} else {
			_ = s.check(seat, evs)
		}
	}
}

// potOdds is what a bot does when the trained table has never seen this spot: it
// works out whether the price it is being offered beats its chance of winning,
// and mixes in enough aggression not to be a calling station.
func potOdds(eq Equity, s *State, seat int) [numActions]float64 {
	p := &s.Seats[seat]
	strength := eq.Strength()
	owed := s.Bet - p.Bet
	pot := s.inPlay()

	price := 0.0
	if owed > 0 && pot+owed > 0 {
		price = float64(owed) / float64(pot+owed)
	}

	var probs [numActions]float64
	switch {
	case strength > 0.8:
		probs[actRaisePot], probs[actAllIn], probs[actCallCheck] = 0.6, 0.2, 0.2
	case strength > 0.6:
		probs[actRaiseHalf], probs[actCallCheck], probs[actFold] = 0.4, 0.5, 0.1
	case owed > 0 && strength > price:
		probs[actCallCheck], probs[actRaiseHalf], probs[actFold] = 0.7, 0.2, 0.1
	case owed > 0:
		probs[actFold], probs[actCallCheck] = 0.7, 0.3
	default:
		probs[actCallCheck], probs[actRaiseHalf], probs[actFold] = 0.6, 0.3, 0.1
	}
	return probs
}

// mask is what a seat may actually do here. The trainer explores exactly this
// set, so it never learns a strategy the table would turn down.
func (s State) mask(seat int) (m [numActions]bool) {
	owed := s.Bet - s.Seats[seat].Bet

	// Folding a hand you could see for free is a bug, not a strategy.
	m[actFold] = owed > 0
	m[actCallCheck] = true

	// A raise needs chips behind the call, and somebody left to bet into.
	raise := s.Seats[seat].Stack > owed && s.canBet()
	m[actRaiseHalf], m[actRaisePot], m[actAllIn] = raise, raise, raise
	return m
}

// legal zeroes out what the seat cannot do and renormalises what's left.
func legal(probs [numActions]float64, s *State, seat int) [numActions]float64 {
	m := s.mask(seat)
	var total float64
	for i := range probs {
		if !m[i] {
			probs[i] = 0
		}
		total += probs[i]
	}
	if total <= 0 {
		var only [numActions]float64
		only[actCallCheck] = 1
		return only
	}
	for i := range probs {
		probs[i] /= total
	}
	return probs
}

// pick rolls against the distribution.
func pick(probs [numActions]float64, rng *rand.Rand) int {
	r := rng.Float64()
	sum := 0.0
	for i, p := range probs {
		sum += p
		if r < sum {
			return i
		}
	}
	return actCallCheck
}

// moveFor turns an abstract action — "raise half the pot" — into a legal move at
// the actual size the table is at. A raise that would cost the bot everything it
// has is a shove, which is the same decision made honestly.
func (s *State) moveFor(action, seat int) Move {
	p := &s.Seats[seat]
	owed := s.Bet - p.Bet
	pot := s.inPlay()
	most := p.Bet + p.Stack

	sized := func(by int64) Move {
		if by < s.MinRaise {
			by = s.MinRaise
		}
		to := s.Bet + by
		if to >= most {
			return Move{Kind: Shove}
		}
		return Move{Kind: Raise, To: to}
	}

	switch action {
	case actFold:
		if owed <= 0 {
			return Move{Kind: Check} // never fold for free
		}
		return Move{Kind: Fold}
	case actCallCheck:
		if owed > 0 {
			return Move{Kind: Call}
		}
		return Move{Kind: Check}
	case actRaiseHalf:
		return sized(pot / 2)
	case actRaisePot:
		return sized(pot)
	case actAllIn:
		return Move{Kind: Shove}
	}
	return Move{Kind: Check}
}