games: the poker table opens, and the bots go back to school
Phase 4. Hold'em, and it's the only table in the casino that is a session rather than a game: you buy in, play as many hands as you like, and leave with what's in front of you. So the live row spans hands and chips cross the border exactly twice. Everything in between is inside the engine. The bots move inside ApplyMove, as UNO's do, which is what keeps poker off a socket: shove all-in and the flop, turn, river, showdown and payout all come back in one response, as a script the felt plays back. The CFR policy the plan called "the single highest-value asset in either repo" was never read. Not once, in the whole life of the game: the trainer wrote its info-set keys under IP/OOP and the runtime looked them up under BTN/SB/BB, so every lookup missed and fell silently through to a pot-odds heuristic. Nothing looked broken, because a policy miss is not an error. And it was the wrong policy anyway — ten big blinds deep, trained on a tree where a call always ends the street, which is not poker. So the trainer is rewritten to play the real engine through the real reducer, at every stack depth the table deals, and the trainer and the table now build the key with the same function so they cannot drift apart again. A test fails if the bots stop finding themselves in it. Three money bugs, and the tests earned their keep. Chip conservation across a hundred sessions caught an uncalled bet that minted chips. A var-init ordering trap meant every card was identical, every showdown tied and every bot believed it held exactly 50% equity. And the browser caught the rake being silently zero — the tier said 5 meaning percent, the casino handed it 0.05 meaning a fraction, and integer division took the house's cut down to nothing. Claude-Session: https://claude.ai/code/session_013M5nD7PgUboJXoDcYHzpuJ
This commit is contained in:
prosolis
279
internal/games/holdem/betting.go
Normal file
279
internal/games/holdem/betting.go
Normal file
@@ -0,0 +1,279 @@
|
||||
package holdem
|
||||
|
||||
import "sort"
|
||||
|
||||
// The betting rules. These are the fiddly ones — min-raise, short all-ins that
|
||||
// don't reopen the action, side pots — and they came over from gogobee, where
|
||||
// they had no tests at all. They have some now.
|
||||
|
||||
// blinds posts the small and the big. Heads-up is the exception every poker
|
||||
// implementation gets wrong once: with two players the button *is* the small
|
||||
// blind and acts first before the flop, and last after it.
|
||||
func (s *State) blinds(evs *[]Event) (bb int) {
|
||||
var sb int
|
||||
if s.dealt() == 2 {
|
||||
sb, bb = s.Button, s.nextIn(s.Button)
|
||||
} else {
|
||||
sb = s.nextIn(s.Button)
|
||||
bb = s.nextIn(sb)
|
||||
}
|
||||
|
||||
s.post(sb, s.Tier.SB, "small", evs)
|
||||
s.post(bb, s.Tier.BB, "big", evs)
|
||||
|
||||
s.Bet = s.Tier.BB
|
||||
s.MinRaise = s.Tier.BB
|
||||
s.Aggressor = bb // the big blind has the option to raise their own blind
|
||||
return bb
|
||||
}
|
||||
|
||||
// post puts a blind up. A player too short to cover it is all-in for what they
|
||||
// have, which is legal and is why the amount is clamped rather than refused.
|
||||
func (s *State) post(seat int, amount int64, which string, evs *[]Event) {
|
||||
p := &s.Seats[seat]
|
||||
if amount > p.Stack {
|
||||
amount = p.Stack
|
||||
}
|
||||
p.Stack -= amount
|
||||
p.Bet = amount
|
||||
p.Total = amount
|
||||
if p.Stack == 0 {
|
||||
p.State = AllIn
|
||||
}
|
||||
*evs = append(*evs, Event{Kind: "blind", Seat: seat, Amount: amount, Text: which})
|
||||
}
|
||||
|
||||
// firstPreFlop is under the gun: the seat after the big blind, or the button
|
||||
// itself when the table is heads-up.
|
||||
//
|
||||
// The button only gets it if the button can still act. A short stack can be
|
||||
// all-in on its own blind — post a small blind of 1 with 1 chip left and you are
|
||||
// in the hand with no chips and no say — and handing the action to a seat that
|
||||
// cannot act wedges the table.
|
||||
func (s *State) firstPreFlop(bb int) int {
|
||||
if s.dealt() == 2 && s.Seats[s.Button].State == Active {
|
||||
return s.Button
|
||||
}
|
||||
if s.dealt() == 2 {
|
||||
return s.nextCanAct(s.Button)
|
||||
}
|
||||
return s.nextCanAct(bb)
|
||||
}
|
||||
|
||||
// firstPostFlop is the first seat left of the button, on every street after the
|
||||
// flop. The button acts last from here on, which is the whole point of it.
|
||||
func (s *State) firstPostFlop() int { return s.nextCanAct(s.Button) }
|
||||
|
||||
// ---- the five things a seat can do ----------------------------------------
|
||||
|
||||
func (s *State) fold(seat int, evs *[]Event) {
|
||||
p := &s.Seats[seat]
|
||||
p.State = Folded
|
||||
p.Acted = true
|
||||
s.History += "f"
|
||||
*evs = append(*evs, Event{Kind: "action", Seat: seat, Text: "fold"})
|
||||
}
|
||||
|
||||
func (s *State) check(seat int, evs *[]Event) error {
|
||||
p := &s.Seats[seat]
|
||||
if p.Bet < s.Bet {
|
||||
return ErrCantCheck
|
||||
}
|
||||
p.Acted = true
|
||||
s.History += "c"
|
||||
*evs = append(*evs, Event{Kind: "action", Seat: seat, Text: "check"})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *State) call(seat int, evs *[]Event) error {
|
||||
p := &s.Seats[seat]
|
||||
owed := s.Bet - p.Bet
|
||||
if owed <= 0 {
|
||||
return ErrNothingToCall
|
||||
}
|
||||
if owed > p.Stack {
|
||||
owed = p.Stack // a call for less than the bet is a call all-in
|
||||
}
|
||||
|
||||
p.Stack -= owed
|
||||
p.Bet += owed
|
||||
p.Total += owed
|
||||
p.Acted = true
|
||||
|
||||
text := "call"
|
||||
if p.Stack == 0 {
|
||||
p.State = AllIn
|
||||
text = "allin"
|
||||
s.History += "a"
|
||||
} else {
|
||||
s.History += "c"
|
||||
}
|
||||
*evs = append(*evs, Event{Kind: "action", Seat: seat, Text: text, Amount: owed, Total: p.Bet})
|
||||
return nil
|
||||
}
|
||||
|
||||
// raise raises *to* a total, not *by* an amount. Every poker interface in the
|
||||
// world means the total, and a browser that means the other thing bets wrong.
|
||||
func (s *State) raise(seat int, to int64, evs *[]Event) error {
|
||||
p := &s.Seats[seat]
|
||||
most := p.Bet + p.Stack
|
||||
|
||||
if to > most {
|
||||
return ErrTooBig
|
||||
}
|
||||
if to < s.Bet+s.MinRaise && to < most {
|
||||
return ErrTooSmall // only a shove may be smaller than a legal raise
|
||||
}
|
||||
|
||||
added := to - p.Bet
|
||||
over := to - s.Bet
|
||||
|
||||
p.Stack -= added
|
||||
p.Bet = to
|
||||
p.Total += added
|
||||
p.Acted = true
|
||||
|
||||
if over > 0 {
|
||||
s.MinRaise = over
|
||||
}
|
||||
s.Bet = to
|
||||
s.Aggressor = seat
|
||||
|
||||
text := "raise"
|
||||
if p.Stack == 0 {
|
||||
p.State = AllIn
|
||||
text = "allin"
|
||||
s.History += "a"
|
||||
} else {
|
||||
// The policy was trained against a tree with two raise sizes in it, so the
|
||||
// history it reads has to say which one this was: R for a pot-sized raise
|
||||
// or bigger, r for anything smaller.
|
||||
if pot := s.inPlay(); pot > 0 && float64(over) >= float64(pot)*0.75 {
|
||||
s.History += "R"
|
||||
} else {
|
||||
s.History += "r"
|
||||
}
|
||||
}
|
||||
*evs = append(*evs, Event{Kind: "action", Seat: seat, Text: text, Amount: added, Total: to})
|
||||
return nil
|
||||
}
|
||||
|
||||
// allin pushes the lot.
|
||||
//
|
||||
// A short all-in does not reopen the betting. If a player shoves for less than
|
||||
// a full raise over the current bet, players who have already acted may call it
|
||||
// but may not raise again — otherwise a tiny stack could be used to reopen the
|
||||
// action for a partner, which is the oldest collusion trick there is.
|
||||
func (s *State) allin(seat int, evs *[]Event) error {
|
||||
p := &s.Seats[seat]
|
||||
if p.Stack <= 0 {
|
||||
return ErrNoChips
|
||||
}
|
||||
added := p.Stack
|
||||
to := p.Bet + added
|
||||
|
||||
p.Stack = 0
|
||||
p.Bet = to
|
||||
p.Total += added
|
||||
p.State = AllIn
|
||||
p.Acted = true
|
||||
|
||||
if to > s.Bet {
|
||||
if over := to - s.Bet; over >= s.MinRaise {
|
||||
s.MinRaise = over
|
||||
s.Aggressor = seat
|
||||
}
|
||||
s.Bet = to
|
||||
}
|
||||
|
||||
s.History += "a"
|
||||
*evs = append(*evs, Event{Kind: "action", Seat: seat, Text: "allin", Amount: added, Total: to})
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- when is a street over ------------------------------------------------
|
||||
|
||||
// streetDone reports whether the betting round is finished, given the seat the
|
||||
// action would pass to next.
|
||||
//
|
||||
// The "has acted" check is the load-bearing half. The big blind has money in
|
||||
// front of them without having chosen to put it there, so a round where
|
||||
// everybody merely limps in has all bets matched while the blind has never had
|
||||
// a say. Without this, they never get their option.
|
||||
func (s *State) streetDone(next int) bool {
|
||||
if s.canActCount() == 0 {
|
||||
return true
|
||||
}
|
||||
for i := range s.Seats {
|
||||
p := &s.Seats[i]
|
||||
if p.State != Active {
|
||||
continue
|
||||
}
|
||||
if p.Bet != s.Bet || !p.Acted {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// The last aggressor being all-in means the action can't get back to them:
|
||||
// everyone left has matched the bet above, so there is nothing more to do.
|
||||
if s.Seats[s.Aggressor].State == AllIn {
|
||||
return true
|
||||
}
|
||||
return next == s.Aggressor
|
||||
}
|
||||
|
||||
// ---- side pots -------------------------------------------------------------
|
||||
|
||||
// sidePots slices the pot into layers, one per distinct all-in level. A player
|
||||
// can only win the part of the pot they could have lost, so each layer is
|
||||
// contested by exactly the players who paid into it.
|
||||
//
|
||||
// Folded players' chips stay in the pot — they paid for the right to fold — but
|
||||
// they are eligible for nothing.
|
||||
func (s *State) sidePots() {
|
||||
s.collect()
|
||||
|
||||
var levels []int64
|
||||
for i := range s.Seats {
|
||||
p := &s.Seats[i]
|
||||
if p.State == Folded || p.State == Out || p.Total == 0 {
|
||||
continue
|
||||
}
|
||||
levels = append(levels, p.Total)
|
||||
}
|
||||
if len(levels) == 0 {
|
||||
return
|
||||
}
|
||||
sort.Slice(levels, func(i, j int) bool { return levels[i] < levels[j] })
|
||||
|
||||
var pots []Pot
|
||||
var prev int64
|
||||
for _, level := range levels {
|
||||
if level <= prev {
|
||||
continue
|
||||
}
|
||||
var amount int64
|
||||
var eligible []int
|
||||
for i := range s.Seats {
|
||||
p := &s.Seats[i]
|
||||
paid := p.Total - prev
|
||||
if paid > level-prev {
|
||||
paid = level - prev
|
||||
}
|
||||
if paid > 0 {
|
||||
amount += paid // folded money counts toward the pot...
|
||||
}
|
||||
if p.State != Folded && p.State != Out && p.Total >= level {
|
||||
eligible = append(eligible, i) // ...but wins no part of it
|
||||
}
|
||||
}
|
||||
if amount > 0 {
|
||||
pots = append(pots, Pot{Amount: amount, Eligible: eligible})
|
||||
}
|
||||
prev = level
|
||||
}
|
||||
|
||||
if len(pots) > 0 {
|
||||
s.Side = pots
|
||||
s.Pot = 0
|
||||
}
|
||||
}
|
||||
416
internal/games/holdem/cfr.go
Normal file
416
internal/games/holdem/cfr.go
Normal file
@@ -0,0 +1,416 @@
|
||||
package holdem
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"pete/internal/games/cards"
|
||||
)
|
||||
|
||||
// The bots' brain.
|
||||
//
|
||||
// gogobee ran counterfactual regret minimisation against this game for a very
|
||||
// long time, and policy.gob is what it converged on: a table from "the situation
|
||||
// I am in" to "how often I fold, call, raise small, raise big, or shove". It is
|
||||
// the single highest-value thing in either repository, and none of it is
|
||||
// re-derived here — this file is the *runtime*, the trainer stayed behind.
|
||||
//
|
||||
// A situation is squeezed down to six things, and this is the whole reason the
|
||||
// table fits in memory: the street, whether the bot is in position, which of
|
||||
// twelve equity buckets its hand falls in, which of five stack-to-pot buckets,
|
||||
// whether the board is dry, wet or paired, and the last six actions. Two hands
|
||||
// that hash to the same key get the same strategy, and that is the approximation
|
||||
// the whole thing is built on.
|
||||
//
|
||||
// The key has to be *exactly* the key the trainer wrote, character for
|
||||
// character. Change the bucket edges, the position label or the history encoding
|
||||
// and every lookup misses — silently, because a miss is not an error, it is a
|
||||
// fall back to the pot-odds rule below. The bots would get quietly, unaccountably
|
||||
// worse. So: don't touch these numbers.
|
||||
|
||||
//go:embed policy.gob
|
||||
var policyGob []byte
|
||||
|
||||
// The five things the trainer let a bot consider.
|
||||
const (
|
||||
actFold = iota
|
||||
actCallCheck
|
||||
actRaiseHalf
|
||||
actRaisePot
|
||||
actAllIn
|
||||
numActions
|
||||
)
|
||||
|
||||
// policyTable maps an info-set key to how often to take each action.
|
||||
type policyTable map[string][numActions]float64
|
||||
|
||||
var (
|
||||
policyOnce sync.Once
|
||||
policy policyTable
|
||||
)
|
||||
|
||||
// loadPolicy decodes the embedded table, once, on the first hand anybody plays.
|
||||
//
|
||||
// Not in an init(): it is megabytes of gob, and Pete is a news server that mostly
|
||||
// never deals a card. Every test in the repo and every cold start would pay for
|
||||
// it. The first player to sit down pays for it instead, and only they do.
|
||||
func loadPolicy() policyTable {
|
||||
policyOnce.Do(func() {
|
||||
t, err := loadTrained(policyGob)
|
||||
if err != nil {
|
||||
// The bots still play — on pot odds — rather than the table 500ing.
|
||||
slog.Error("holdem: cannot decode CFR policy, bots fall back to pot odds", "err", err)
|
||||
policy = policyTable{}
|
||||
return
|
||||
}
|
||||
policy = t.Strategy
|
||||
slog.Info("holdem: CFR policy loaded", "nodes", len(policy),
|
||||
"iterations", t.Meta.Iterations, "stakes", t.Meta.Stakes, "depths", t.Meta.Depths)
|
||||
})
|
||||
return policy
|
||||
}
|
||||
|
||||
// ---- the info-set key ------------------------------------------------------
|
||||
//
|
||||
// Every function below is a load-bearing copy of the trainer's. See the note at
|
||||
// the top of the file before changing a number in any of them.
|
||||
|
||||
// equityBucket puts a hand's strength in one of twelve boxes, from trash to
|
||||
// monster.
|
||||
func equityBucket(eq float64) int {
|
||||
switch {
|
||||
case eq < 0.08:
|
||||
return 0
|
||||
case eq < 0.17:
|
||||
return 1
|
||||
case eq < 0.25:
|
||||
return 2
|
||||
case eq < 0.33:
|
||||
return 3
|
||||
case eq < 0.42:
|
||||
return 4
|
||||
case eq < 0.50:
|
||||
return 5
|
||||
case eq < 0.58:
|
||||
return 6
|
||||
case eq < 0.67:
|
||||
return 7
|
||||
case eq < 0.75:
|
||||
return 8
|
||||
case eq < 0.83:
|
||||
return 9
|
||||
case eq < 0.92:
|
||||
return 10
|
||||
default:
|
||||
return 11
|
||||
}
|
||||
}
|
||||
|
||||
// sprBucket is the stack-to-pot ratio: how much room is left to play. Under 1 is
|
||||
// a pot that is already committed; over 12 is a pot you can still fold out of.
|
||||
func sprBucket(spr float64) int {
|
||||
switch {
|
||||
case spr < 1:
|
||||
return 0
|
||||
case spr < 3:
|
||||
return 1
|
||||
case spr < 6:
|
||||
return 2
|
||||
case spr < 12:
|
||||
return 3
|
||||
default:
|
||||
return 4
|
||||
}
|
||||
}
|
||||
|
||||
// Board textures, which is what makes the same hand a bet or a check.
|
||||
const (
|
||||
boardDry = 0
|
||||
boardWet = 1
|
||||
boardPaired = 2
|
||||
)
|
||||
|
||||
// boardTexture classifies the community cards. A paired board is one somebody
|
||||
// might have trips on; a wet one has a flush or straight coming.
|
||||
// It is on the trainer's hot path — millions of calls — so it counts into arrays
|
||||
// rather than maps. A map here cost more than the poker did.
|
||||
func boardTexture(board []cards.Card) int {
|
||||
if len(board) < 3 {
|
||||
return boardDry
|
||||
}
|
||||
|
||||
var ranks [14]int8
|
||||
var suits [4]int8
|
||||
var vals [5]int
|
||||
for i, c := range board {
|
||||
ranks[c.Rank]++
|
||||
suits[c.Suit]++
|
||||
vals[i] = int(c.Rank)
|
||||
}
|
||||
for _, n := range ranks {
|
||||
if n >= 2 {
|
||||
return boardPaired
|
||||
}
|
||||
}
|
||||
for _, n := range suits {
|
||||
if n >= 3 {
|
||||
return boardWet
|
||||
}
|
||||
}
|
||||
|
||||
// Three cards inside a five-rank window is a straight waiting to happen.
|
||||
v := vals[:len(board)]
|
||||
for i := 1; i < len(v); i++ {
|
||||
for j := i; j > 0 && v[j] < v[j-1]; j-- {
|
||||
v[j], v[j-1] = v[j-1], v[j]
|
||||
}
|
||||
}
|
||||
for i := 0; i+2 < len(v); i++ {
|
||||
if v[i+2]-v[i] <= 4 {
|
||||
return boardWet
|
||||
}
|
||||
}
|
||||
return boardDry
|
||||
}
|
||||
|
||||
// infoSet builds the string the policy is keyed on. The format is the trainer's.
|
||||
//
|
||||
// Position is IP or OOP — in position or out of it — and *nothing else*. This is
|
||||
// the one thing gogobee got wrong, and it got it wrong invisibly for as long as
|
||||
// the game has existed: the trainer packed a single "am I last to act" bit and
|
||||
// wrote its keys as IP/OOP, while the runtime looked them up with the table
|
||||
// labels a player would recognise (BTN, SB, BB, UTG…). Not one key ever matched.
|
||||
// Every bot in every hand of hold'em gogobee ever dealt fell through to the
|
||||
// pot-odds rule, and the five million training iterations sitting in policy.gob
|
||||
// were never once read.
|
||||
//
|
||||
// Nothing about that looks broken from the outside. A missing key is not an
|
||||
// error, it's a fallback — the bots played, they just played a heuristic. This is
|
||||
// why the hit rate is now a test.
|
||||
func infoSet(street Street, inPosition bool, eq, spr, texture int, history string) string {
|
||||
pos := "OOP"
|
||||
if inPosition {
|
||||
pos = "IP"
|
||||
}
|
||||
return fmt.Sprintf("%d|%s|%d|%d|%d|%s", street, pos, eq, spr, texture, history)
|
||||
}
|
||||
|
||||
// recent keeps the last six actions, which is all the trainer's key had room for.
|
||||
func recent(h string) string {
|
||||
if len(h) > 6 {
|
||||
return h[len(h)-6:]
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// ---- where a seat stands ---------------------------------------------------
|
||||
|
||||
// mcIters is how many runouts a bot samples to judge its hand at the table.
|
||||
const mcIters = 1000
|
||||
|
||||
// spot is everything the policy knows about a seat's situation, and it is the
|
||||
// one function that builds it.
|
||||
//
|
||||
// The trainer calls this too. That is the point of it: the key the policy is
|
||||
// written under and the key it is read under come out of the same code, so they
|
||||
// cannot quietly stop matching — which is exactly what went wrong the first time
|
||||
// and went unnoticed for the life of the game.
|
||||
func (s State) spot(seat, iters int, rng *rand.Rand) (string, Equity) {
|
||||
eq := s.equityFor(seat, iters, rng)
|
||||
return s.spotKey(seat, eq), eq
|
||||
}
|
||||
|
||||
// equityFor measures how good a seat's hand is right now.
|
||||
//
|
||||
// It depends only on the cards — the hand, the board, how many opponents — and
|
||||
// not on a single thing that happened in the betting. Which is why the trainer
|
||||
// can measure it once per street and reuse it down every branch it explores, and
|
||||
// why doing that is most of the difference between a run that takes half an hour
|
||||
// and one that takes four.
|
||||
func (s State) equityFor(seat, iters int, rng *rand.Rand) Equity {
|
||||
opponents := s.liveCount() - 1
|
||||
if opponents < 1 {
|
||||
opponents = 1
|
||||
}
|
||||
// Preflop heads-up is a lookup, not a simulation: there are only 169 hands
|
||||
// that differ, they have been measured to death, and a sampled answer would
|
||||
// only add noise to a bucket boundary.
|
||||
if s.Street == PreFlop && opponents == 1 {
|
||||
return preflopEquity(s.Seats[seat].Hole)
|
||||
}
|
||||
return equityOf(s.Seats[seat].Hole, s.Community, opponents, iters, rng)
|
||||
}
|
||||
|
||||
// spotKey builds the key from an equity already measured.
|
||||
func (s State) spotKey(seat int, eq Equity) string {
|
||||
pot := s.inPlay()
|
||||
spr := 0.0
|
||||
if pot > 0 {
|
||||
spr = float64(s.Seats[seat].Stack) / float64(pot)
|
||||
}
|
||||
return infoSet(s.Street, s.InPosition(seat), equityBucket(eq.Strength()),
|
||||
sprBucket(spr), boardTexture(s.Community), recent(s.History))
|
||||
}
|
||||
|
||||
// ---- choosing --------------------------------------------------------------
|
||||
|
||||
// Hits and Misses count how often a bot finds itself in the trained policy.
|
||||
//
|
||||
// They exist because the way this can break is silently. A key the policy has
|
||||
// never seen is not an error — the bot shrugs and plays pot odds — so a policy
|
||||
// that has stopped matching the game looks exactly like a policy that is working.
|
||||
// gogobee's never matched once, for the whole life of the game, and nobody could
|
||||
// have known by watching it play. Now a test reads these and fails.
|
||||
var Hits, Misses atomic.Int64
|
||||
|
||||
// botActs plays one bot's turn: work out where it stands, look up what it does
|
||||
// there, throw out anything illegal, and roll for it.
|
||||
func (s *State) botActs(seat int, evs *[]Event, rng *rand.Rand) {
|
||||
key, eq := s.spot(seat, mcIters, rng)
|
||||
|
||||
probs, ok := loadPolicy()[key]
|
||||
if ok {
|
||||
Hits.Add(1)
|
||||
} else {
|
||||
Misses.Add(1)
|
||||
probs = potOdds(eq, s, seat)
|
||||
}
|
||||
probs = legal(probs, s, seat)
|
||||
|
||||
move := s.moveFor(pick(probs, rng), seat)
|
||||
if err := s.act(seat, move, evs); err != nil {
|
||||
// A bot cannot be allowed to wedge the table by choosing something the rules
|
||||
// then refuse: it checks if it can and folds if it can't, and the mismatch
|
||||
// is loud, because it means legal() and the betting rules disagree.
|
||||
slog.Error("holdem: bot chose an illegal move", "seat", seat, "move", move.Kind, "err", err)
|
||||
if s.Owed(seat) > 0 {
|
||||
s.fold(seat, evs)
|
||||
} else {
|
||||
_ = s.check(seat, evs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// potOdds is what a bot does when the trained table has never seen this spot: it
|
||||
// works out whether the price it is being offered beats its chance of winning,
|
||||
// and mixes in enough aggression not to be a calling station.
|
||||
func potOdds(eq Equity, s *State, seat int) [numActions]float64 {
|
||||
p := &s.Seats[seat]
|
||||
strength := eq.Strength()
|
||||
owed := s.Bet - p.Bet
|
||||
pot := s.inPlay()
|
||||
|
||||
price := 0.0
|
||||
if owed > 0 && pot+owed > 0 {
|
||||
price = float64(owed) / float64(pot+owed)
|
||||
}
|
||||
|
||||
var probs [numActions]float64
|
||||
switch {
|
||||
case strength > 0.8:
|
||||
probs[actRaisePot], probs[actAllIn], probs[actCallCheck] = 0.6, 0.2, 0.2
|
||||
case strength > 0.6:
|
||||
probs[actRaiseHalf], probs[actCallCheck], probs[actFold] = 0.4, 0.5, 0.1
|
||||
case owed > 0 && strength > price:
|
||||
probs[actCallCheck], probs[actRaiseHalf], probs[actFold] = 0.7, 0.2, 0.1
|
||||
case owed > 0:
|
||||
probs[actFold], probs[actCallCheck] = 0.7, 0.3
|
||||
default:
|
||||
probs[actCallCheck], probs[actRaiseHalf], probs[actFold] = 0.6, 0.3, 0.1
|
||||
}
|
||||
return probs
|
||||
}
|
||||
|
||||
// mask is what a seat may actually do here. The trainer explores exactly this
|
||||
// set, so it never learns a strategy the table would turn down.
|
||||
func (s State) mask(seat int) (m [numActions]bool) {
|
||||
owed := s.Bet - s.Seats[seat].Bet
|
||||
|
||||
// Folding a hand you could see for free is a bug, not a strategy.
|
||||
m[actFold] = owed > 0
|
||||
m[actCallCheck] = true
|
||||
|
||||
// A raise needs chips behind the call, and somebody left to bet into.
|
||||
raise := s.Seats[seat].Stack > owed && s.canBet()
|
||||
m[actRaiseHalf], m[actRaisePot], m[actAllIn] = raise, raise, raise
|
||||
return m
|
||||
}
|
||||
|
||||
// legal zeroes out what the seat cannot do and renormalises what's left.
|
||||
func legal(probs [numActions]float64, s *State, seat int) [numActions]float64 {
|
||||
m := s.mask(seat)
|
||||
var total float64
|
||||
for i := range probs {
|
||||
if !m[i] {
|
||||
probs[i] = 0
|
||||
}
|
||||
total += probs[i]
|
||||
}
|
||||
if total <= 0 {
|
||||
var only [numActions]float64
|
||||
only[actCallCheck] = 1
|
||||
return only
|
||||
}
|
||||
for i := range probs {
|
||||
probs[i] /= total
|
||||
}
|
||||
return probs
|
||||
}
|
||||
|
||||
// pick rolls against the distribution.
|
||||
func pick(probs [numActions]float64, rng *rand.Rand) int {
|
||||
r := rng.Float64()
|
||||
sum := 0.0
|
||||
for i, p := range probs {
|
||||
sum += p
|
||||
if r < sum {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return actCallCheck
|
||||
}
|
||||
|
||||
// moveFor turns an abstract action — "raise half the pot" — into a legal move at
|
||||
// the actual size the table is at. A raise that would cost the bot everything it
|
||||
// has is a shove, which is the same decision made honestly.
|
||||
func (s *State) moveFor(action, seat int) Move {
|
||||
p := &s.Seats[seat]
|
||||
owed := s.Bet - p.Bet
|
||||
pot := s.inPlay()
|
||||
most := p.Bet + p.Stack
|
||||
|
||||
sized := func(by int64) Move {
|
||||
if by < s.MinRaise {
|
||||
by = s.MinRaise
|
||||
}
|
||||
to := s.Bet + by
|
||||
if to >= most {
|
||||
return Move{Kind: Shove}
|
||||
}
|
||||
return Move{Kind: Raise, To: to}
|
||||
}
|
||||
|
||||
switch action {
|
||||
case actFold:
|
||||
if owed <= 0 {
|
||||
return Move{Kind: Check} // never fold for free
|
||||
}
|
||||
return Move{Kind: Fold}
|
||||
case actCallCheck:
|
||||
if owed > 0 {
|
||||
return Move{Kind: Call}
|
||||
}
|
||||
return Move{Kind: Check}
|
||||
case actRaiseHalf:
|
||||
return sized(pot / 2)
|
||||
case actRaisePot:
|
||||
return sized(pot)
|
||||
case actAllIn:
|
||||
return Move{Kind: Shove}
|
||||
}
|
||||
return Move{Kind: Check}
|
||||
}
|
||||
134
internal/games/holdem/equity.go
Normal file
134
internal/games/holdem/equity.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package holdem
|
||||
|
||||
import (
|
||||
"math/rand/v2"
|
||||
|
||||
"github.com/chehsunliu/poker"
|
||||
|
||||
"pete/internal/games/cards"
|
||||
)
|
||||
|
||||
// How good is this hand, really?
|
||||
//
|
||||
// A bot's decision starts here: deal the cards it cannot see, a thousand times,
|
||||
// and count how often it wins. That number — plus the board's texture, the
|
||||
// stack-to-pot ratio and the action so far — is the key it looks its trained
|
||||
// strategy up under.
|
||||
//
|
||||
// Monte Carlo rather than exhaustive because exhaustive is 2.1 million river
|
||||
// runouts against one opponent and rather more against five, and a thousand
|
||||
// samples puts the estimate inside a percentage point or so. The bot does not
|
||||
// need the fourth decimal place; it needs to know whether it is ahead.
|
||||
|
||||
// Equity is the fraction of runouts a hand wins, ties and loses against the
|
||||
// given number of unknown opponents.
|
||||
type Equity struct {
|
||||
Win float64
|
||||
Tie float64
|
||||
Loss float64
|
||||
}
|
||||
|
||||
// Strength collapses a result into the one number the policy is keyed on: a tie
|
||||
// is worth half a win, because that is what half a pot is.
|
||||
func (e Equity) Strength() float64 { return e.Win + e.Tie*0.5 }
|
||||
|
||||
// deck52 is the evaluator's whole deck, built once.
|
||||
var deck52 = func() []poker.Card {
|
||||
d := make([]poker.Card, 0, 52)
|
||||
for s := cards.Spades; s <= cards.Clubs; s++ {
|
||||
for r := cards.Ace; r <= cards.King; r++ {
|
||||
d = append(d, pokerOf[s][r])
|
||||
}
|
||||
}
|
||||
return d
|
||||
}()
|
||||
|
||||
// equityOf runs the simulation. The RNG is threaded like everything else here,
|
||||
// so a bot's decision replays from the session's seed along with the deal.
|
||||
func equityOf(hole [2]cards.Card, board []cards.Card, opponents, iterations int, rng *rand.Rand) Equity {
|
||||
if opponents < 1 {
|
||||
opponents = 1
|
||||
}
|
||||
|
||||
h0, h1 := toPoker(hole[0]), toPoker(hole[1])
|
||||
|
||||
// Seven cards, checked by hand. A map here would be the most expensive thing
|
||||
// in the trainer, which calls this function millions of times.
|
||||
var known [7]poker.Card
|
||||
n := 2
|
||||
known[0], known[1] = h0, h1
|
||||
pb := make([]poker.Card, len(board))
|
||||
for i, c := range board {
|
||||
pb[i] = toPoker(c)
|
||||
known[n] = pb[i]
|
||||
n++
|
||||
}
|
||||
|
||||
rest := make([]poker.Card, 0, 52)
|
||||
for _, c := range deck52 {
|
||||
seen := false
|
||||
for _, k := range known[:n] {
|
||||
if k == c {
|
||||
seen = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !seen {
|
||||
rest = append(rest, c)
|
||||
}
|
||||
}
|
||||
|
||||
need := opponents*2 + (5 - len(pb))
|
||||
if need > len(rest) {
|
||||
return Equity{Tie: 1}
|
||||
}
|
||||
|
||||
hero := make([]poker.Card, 7)
|
||||
hero[0], hero[1] = h0, h1
|
||||
villain := make([]poker.Card, 7)
|
||||
full := make([]poker.Card, 5)
|
||||
|
||||
var wins, ties int
|
||||
for i := 0; i < iterations; i++ {
|
||||
// A partial Fisher-Yates: only the cards actually needed get shuffled into
|
||||
// place, which is the difference between this being cheap and being the
|
||||
// slowest thing in the request.
|
||||
for j := 0; j < need; j++ {
|
||||
k := j + rng.IntN(len(rest)-j)
|
||||
rest[j], rest[k] = rest[k], rest[j]
|
||||
}
|
||||
|
||||
copy(full, pb)
|
||||
at := opponents * 2
|
||||
for b := len(pb); b < 5; b++ {
|
||||
full[b] = rest[at]
|
||||
at++
|
||||
}
|
||||
|
||||
copy(hero[2:], full)
|
||||
mine := poker.Evaluate(hero)
|
||||
|
||||
best := int32(7463) // one worse than the worst real hand
|
||||
for o := 0; o < opponents; o++ {
|
||||
villain[0], villain[1] = rest[o*2], rest[o*2+1]
|
||||
copy(villain[2:], full)
|
||||
if r := poker.Evaluate(villain); r < best {
|
||||
best = r
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case mine < best:
|
||||
wins++
|
||||
case mine == best:
|
||||
ties++
|
||||
}
|
||||
}
|
||||
|
||||
total := float64(iterations)
|
||||
return Equity{
|
||||
Win: float64(wins) / total,
|
||||
Tie: float64(ties) / total,
|
||||
Loss: float64(iterations-wins-ties) / total,
|
||||
}
|
||||
}
|
||||
263
internal/games/holdem/eval.go
Normal file
263
internal/games/holdem/eval.go
Normal file
@@ -0,0 +1,263 @@
|
||||
package holdem
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/chehsunliu/poker"
|
||||
|
||||
"pete/internal/games/cards"
|
||||
)
|
||||
|
||||
// The bridge to the evaluator.
|
||||
//
|
||||
// The engine deals Pete's own cards.Card — the same one blackjack, solitaire and
|
||||
// the felt already speak — and converts at the door. Hand strength is the one
|
||||
// thing in this package that is genuinely hard to get right (7-card best-of-5,
|
||||
// 7,462 distinct hands), so it is not homegrown: github.com/chehsunliu/poker is
|
||||
// a lookup table and it is correct.
|
||||
//
|
||||
// The conversion is a table built once. Doing it per evaluation would matter:
|
||||
// a bot's equity estimate is a thousand seven-card evaluations, and it makes
|
||||
// several of those per hand.
|
||||
|
||||
var (
|
||||
pokerRanks = [14]string{"", "A", "2", "3", "4", "5", "6", "7", "8", "9", "T", "J", "Q", "K"}
|
||||
pokerSuits = [4]string{"s", "h", "d", "c"} // cards.Spades, Hearts, Diamonds, Clubs
|
||||
|
||||
// A var initializer, not an init(). Go builds package-level variables before
|
||||
// it runs init functions, so anything else in this package that is itself a
|
||||
// var built out of this table — equity.go's deck52 is — would otherwise be
|
||||
// built out of an empty one. It was, briefly: every card came out identical,
|
||||
// every showdown tied, and every bot believed it held exactly 50% equity.
|
||||
pokerOf = func() (t [4][14]poker.Card) {
|
||||
for s := cards.Spades; s <= cards.Clubs; s++ {
|
||||
for r := cards.Ace; r <= cards.King; r++ {
|
||||
t[s][r] = poker.NewCard(pokerRanks[r] + pokerSuits[s])
|
||||
}
|
||||
}
|
||||
return t
|
||||
}()
|
||||
)
|
||||
|
||||
// toPoker converts one card for the evaluator.
|
||||
func toPoker(c cards.Card) poker.Card { return pokerOf[c.Suit][c.Rank] }
|
||||
|
||||
func toPokerAll(cs []cards.Card) []poker.Card {
|
||||
out := make([]poker.Card, len(cs))
|
||||
for i, c := range cs {
|
||||
out[i] = toPoker(c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// rankOf evaluates a seat's best five from its hole cards and the board. Lower
|
||||
// is better — 1 is a royal flush — which is the evaluator's convention and not
|
||||
// worth inverting, since nothing outside this file ever sees the number.
|
||||
func rankOf(hole [2]cards.Card, board []cards.Card) (int32, string) {
|
||||
seven := make([]poker.Card, 0, 7)
|
||||
seven = append(seven, toPoker(hole[0]), toPoker(hole[1]))
|
||||
seven = append(seven, toPokerAll(board)...)
|
||||
r := poker.Evaluate(seven)
|
||||
return r, strings.ToLower(poker.RankString(r))
|
||||
}
|
||||
|
||||
// ---- showdown -------------------------------------------------------------
|
||||
|
||||
type ranked struct {
|
||||
seat int
|
||||
rank int32
|
||||
desc string
|
||||
}
|
||||
|
||||
// showdown turns the cards over, splits the pots, and pays. Every player still
|
||||
// in the hand shows, in the order the felt should turn them over: best hand
|
||||
// first, so the winner is the first card face the player sees.
|
||||
func (s *State) showdown(evs *[]Event) {
|
||||
s.collect()
|
||||
s.Street = Showdown
|
||||
|
||||
// Say so. The last street's bets are still sitting in front of the seats that
|
||||
// made them, as far as the felt knows, and nothing else in the script is going
|
||||
// to tell it they have been swept in.
|
||||
*evs = append(*evs, Event{Kind: "pot", Seat: -1, Amount: s.Total()})
|
||||
|
||||
var live []ranked
|
||||
for i := range s.Seats {
|
||||
p := &s.Seats[i]
|
||||
if p.State == Folded || p.State == Out {
|
||||
continue
|
||||
}
|
||||
r, desc := rankOf(p.Hole, s.Community)
|
||||
live = append(live, ranked{seat: i, rank: r, desc: desc})
|
||||
}
|
||||
sort.Slice(live, func(i, j int) bool { return live[i].rank < live[j].rank })
|
||||
|
||||
for _, e := range live {
|
||||
*evs = append(*evs, Event{
|
||||
Kind: "show", Seat: e.seat,
|
||||
Cards: []cards.Card{s.Seats[e.seat].Hole[0], s.Seats[e.seat].Hole[1]},
|
||||
Text: e.desc,
|
||||
})
|
||||
}
|
||||
|
||||
pots := s.Side
|
||||
if len(pots) == 0 {
|
||||
all := make([]int, 0, len(live))
|
||||
for _, e := range live {
|
||||
all = append(all, e.seat)
|
||||
}
|
||||
pots = []Pot{{Amount: s.Pot, Eligible: all}}
|
||||
s.Pot = 0
|
||||
}
|
||||
s.Side = nil
|
||||
|
||||
for _, pot := range pots {
|
||||
s.payPot(pot, live, evs)
|
||||
}
|
||||
s.endHand(evs)
|
||||
}
|
||||
|
||||
// payPot rakes a pot and splits it between the best eligible hands.
|
||||
//
|
||||
// The rake comes out of the pot before it is split, which is what a cardroom
|
||||
// does and is also the only thing consistent with the rest of this casino: a
|
||||
// player pays it out of a pot they *win*, never out of a bet they lose. A hand
|
||||
// that dies before the flop is not raked at all — no flop, no drop — so folding
|
||||
// your blind round after round costs you exactly the blinds and no fee.
|
||||
func (s *State) payPot(pot Pot, live []ranked, evs *[]Event) {
|
||||
if pot.Amount <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
amount := pot.Amount
|
||||
if s.Flopped {
|
||||
rake := int64(math.Floor(float64(amount) * s.Tier.RakePct))
|
||||
if cap := s.Tier.BB * rakeCapBB; rake > cap {
|
||||
rake = cap
|
||||
}
|
||||
if rake > 0 {
|
||||
amount -= rake
|
||||
s.Rake += rake
|
||||
*evs = append(*evs, Event{Kind: "rake", Seat: -1, Amount: rake})
|
||||
}
|
||||
}
|
||||
|
||||
eligible := make(map[int]bool, len(pot.Eligible))
|
||||
for _, seat := range pot.Eligible {
|
||||
eligible[seat] = true
|
||||
}
|
||||
|
||||
var winners []ranked
|
||||
best := int32(0)
|
||||
for _, e := range live {
|
||||
if !eligible[e.seat] {
|
||||
continue
|
||||
}
|
||||
if len(winners) == 0 || e.rank < best {
|
||||
best, winners = e.rank, []ranked{e}
|
||||
} else if e.rank == best {
|
||||
winners = append(winners, e)
|
||||
}
|
||||
}
|
||||
if len(winners) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
share := amount / int64(len(winners))
|
||||
odd := amount % int64(len(winners)) // the odd chip goes to the first seat left of the button
|
||||
for i, w := range winners {
|
||||
won := share
|
||||
if i == 0 {
|
||||
won += odd
|
||||
}
|
||||
s.Seats[w.seat].Stack += won
|
||||
s.Seats[w.seat].Won += won
|
||||
*evs = append(*evs, Event{Kind: "win", Seat: w.seat, Amount: won, Text: w.desc})
|
||||
}
|
||||
}
|
||||
|
||||
// takeit ends a hand nobody contested: everyone else folded, so the last player
|
||||
// standing takes the pot without showing. Their own uncalled bet comes back
|
||||
// first — it was never called, so it was never really in the pot.
|
||||
func (s *State) takeit(evs *[]Event) {
|
||||
s.uncalled(evs)
|
||||
s.collect()
|
||||
*evs = append(*evs, Event{Kind: "pot", Seat: -1, Amount: s.Total()})
|
||||
|
||||
winner := -1
|
||||
for i := range s.Seats {
|
||||
if s.Seats[i].State != Folded && s.Seats[i].State != Out {
|
||||
winner = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if winner < 0 {
|
||||
s.endHand(evs)
|
||||
return
|
||||
}
|
||||
|
||||
// There are never side pots here: they are only cut once the betting is over
|
||||
// because everybody is all-in, and a table where everybody is all-in is a table
|
||||
// where nobody is left to fold.
|
||||
pot := Pot{Amount: s.Pot, Eligible: []int{winner}}
|
||||
s.Pot = 0
|
||||
s.payPot(pot, []ranked{{seat: winner, rank: 0}}, evs)
|
||||
s.endHand(evs)
|
||||
}
|
||||
|
||||
// uncalled returns the unmatched top of a bet. If you shove 500 into a player
|
||||
// with 200 behind, 300 of that was never contested and comes straight back.
|
||||
//
|
||||
// It must run *before* the bets are swept into the pot, and the matched level it
|
||||
// measures against counts the players who folded. Their chips are in the pot —
|
||||
// they paid to see the bet and then gave up — so the money they put in is money
|
||||
// that called. Miss that and a bet folded to on the river comes back whole,
|
||||
// including the part that was called on the flop, which mints chips out of air.
|
||||
//
|
||||
// The rake is the other reason this matters at all. When everybody folds, the
|
||||
// winner takes the pot back either way and the arithmetic looks the same — but a
|
||||
// pot with an uncalled bet still in it is a pot the house rakes, and it would be
|
||||
// raking the player on their own money that nobody ever contested.
|
||||
func (s *State) uncalled(evs *[]Event) {
|
||||
top, topSeat := int64(-1), -1
|
||||
for i := range s.Seats {
|
||||
p := &s.Seats[i]
|
||||
if p.State == Folded || p.State == Out {
|
||||
continue
|
||||
}
|
||||
if p.Total > top {
|
||||
top, topSeat = p.Total, i
|
||||
}
|
||||
}
|
||||
if topSeat < 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var matched int64 // the most anybody else put in, whether or not they're still in
|
||||
for i := range s.Seats {
|
||||
if i == topSeat || s.Seats[i].State == Out {
|
||||
continue
|
||||
}
|
||||
if s.Seats[i].Total > matched {
|
||||
matched = s.Seats[i].Total
|
||||
}
|
||||
}
|
||||
|
||||
excess := top - matched
|
||||
p := &s.Seats[topSeat]
|
||||
if excess <= 0 || excess > p.Bet {
|
||||
// An uncalled bet is always part of the street it was made on, so it cannot
|
||||
// be bigger than what that seat has in front of them right now.
|
||||
return
|
||||
}
|
||||
|
||||
p.Stack += excess
|
||||
p.Total -= excess
|
||||
p.Bet -= excess
|
||||
if p.State == AllIn && p.Stack > 0 {
|
||||
p.State = Active // they were never really all-in against anybody
|
||||
}
|
||||
*evs = append(*evs, Event{Kind: "uncalled", Seat: topSeat, Amount: excess})
|
||||
}
|
||||
846
internal/games/holdem/holdem.go
Normal file
846
internal/games/holdem/holdem.go
Normal file
@@ -0,0 +1,846 @@
|
||||
// Package holdem is a pure Texas Hold'em engine, played for chips against bots.
|
||||
//
|
||||
// Same seam as every other table in the casino: ApplyMove(state, move) (state,
|
||||
// events, error), where an error means the move was illegal and nothing else.
|
||||
// No HTTP, no timers, no sockets. The state is a plain value, so a hand survives
|
||||
// a redeploy and replays from its seed.
|
||||
//
|
||||
// Three things make hold'em different from the five tables already on the felt.
|
||||
//
|
||||
// It is a cash game, not a stake. Every other game here takes a bet, plays once
|
||||
// and pays a multiple. Poker isn't that: you buy chips onto the table, you play
|
||||
// as many hands as you like, and you leave with whatever is in front of you. So
|
||||
// the session is the unit, not the hand — the live row lives across hands, the
|
||||
// buy-in is the only chip movement at the start, and the stack going home is the
|
||||
// only one at the end. In between the money is entirely inside this engine.
|
||||
//
|
||||
// The bots move inside ApplyMove, as they do in UNO. One call plays the player's
|
||||
// action and every bot action behind it, deals whatever streets that completes,
|
||||
// and hands the lot back as a script of events. Poker is where you would reach
|
||||
// for a socket, and this is what not reaching for one costs.
|
||||
//
|
||||
// The bots are the trained ones. gogobee spent a long time running CFR against
|
||||
// this game and the policy it converged on is the best asset in either repo; it
|
||||
// is embedded here whole (see cfr.go). What that means for the player: the house
|
||||
// edge at this table is not a rule, it is an opponent. There is no 3:2 and no
|
||||
// multiple. If you beat the bots you win, and the only thing the house takes is
|
||||
// the rake on the pots you win.
|
||||
package holdem
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math/rand/v2"
|
||||
|
||||
"pete/internal/games/cards"
|
||||
)
|
||||
|
||||
// Errors an illegal move can produce. An error means nothing happened.
|
||||
var (
|
||||
ErrOver = errors.New("holdem: you've left the table")
|
||||
ErrNotYourTurn = errors.New("holdem: it isn't your turn")
|
||||
ErrHandLive = errors.New("holdem: finish the hand first")
|
||||
ErrNoHand = errors.New("holdem: no hand in progress")
|
||||
ErrCantCheck = errors.New("holdem: there's a bet to you")
|
||||
ErrNothingToCall = errors.New("holdem: nothing to call")
|
||||
ErrTooSmall = errors.New("holdem: that's under the minimum raise")
|
||||
ErrTooBig = errors.New("holdem: you don't have that many chips")
|
||||
ErrNoChips = errors.New("holdem: you have no chips left")
|
||||
ErrUnknownMove = errors.New("holdem: unknown move")
|
||||
ErrUnknownTier = errors.New("holdem: no such table")
|
||||
ErrBadBuyIn = errors.New("holdem: that isn't a legal buy-in")
|
||||
ErrTableFull = errors.New("holdem: too many seats")
|
||||
)
|
||||
|
||||
// You are always seat zero. The bots are the seats after you.
|
||||
const You = 0
|
||||
|
||||
// rakeCapBB caps the rake on any one pot at three big blinds, which is what a
|
||||
// cardroom does. Without a cap, five percent of a big pot is a lot of money to
|
||||
// take off a player for winning it.
|
||||
const rakeCapBB = 3
|
||||
|
||||
// Street is how far the board has come.
|
||||
type Street uint8
|
||||
|
||||
const (
|
||||
PreFlop Street = iota
|
||||
Flop
|
||||
Turn
|
||||
River
|
||||
Showdown
|
||||
)
|
||||
|
||||
var streetNames = [5]string{"preflop", "flop", "turn", "river", "showdown"}
|
||||
|
||||
func (s Street) String() string {
|
||||
if int(s) >= len(streetNames) {
|
||||
return "?"
|
||||
}
|
||||
return streetNames[s]
|
||||
}
|
||||
|
||||
// SeatState is where a player stands in the hand being played.
|
||||
type SeatState uint8
|
||||
|
||||
const (
|
||||
Active SeatState = iota // still has chips and a say
|
||||
Folded // out of this hand
|
||||
AllIn // in the hand, but has nothing left to bet
|
||||
Out // not dealt in at all
|
||||
)
|
||||
|
||||
// Seat is one player at the table — you, or one of the bots.
|
||||
//
|
||||
// Hole is on the server and stays there. The view layer sends your two cards to
|
||||
// you and sends nobody else's to anybody, right up until a showdown turns them
|
||||
// over. A bot's cards are most of the information in this game; a browser that
|
||||
// held them would make counting cards a matter of reading the network tab.
|
||||
type Seat struct {
|
||||
Name string `json:"name"`
|
||||
Bot bool `json:"bot"`
|
||||
Stack int64 `json:"stack"`
|
||||
Hole [2]cards.Card `json:"hole"`
|
||||
Bet int64 `json:"bet"` // put in on this street
|
||||
Total int64 `json:"total"` // put in across this hand
|
||||
Won int64 `json:"won"` // taken out of the pot this hand
|
||||
State SeatState `json:"state"`
|
||||
Acted bool `json:"acted"` // has chosen to do something this street
|
||||
}
|
||||
|
||||
// Pot is a pot and the seats that may win it. A hand with no all-in has exactly
|
||||
// one; every all-in at a distinct level adds another.
|
||||
type Pot struct {
|
||||
Amount int64 `json:"amount"`
|
||||
Eligible []int `json:"eligible"`
|
||||
}
|
||||
|
||||
// Tier is a table you can sit at. The dial is the stakes, and the stakes are
|
||||
// what make a chip mean something: at 25/50 a careless call costs more than a
|
||||
// whole session at 1/2.
|
||||
//
|
||||
// The buy-in range is the standard 20 to 100 big blinds. Sitting down short is
|
||||
// a real strategy (fewer decisions, less to lose) and sitting down deep is the
|
||||
// other one, so the range is a choice and not a formality.
|
||||
type Tier struct {
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
SB int64 `json:"sb"`
|
||||
BB int64 `json:"bb"`
|
||||
MinBuy int64 `json:"min_buy"`
|
||||
MaxBuy int64 `json:"max_buy"`
|
||||
RakePct float64 `json:"rake_pct"`
|
||||
Blurb string `json:"blurb"`
|
||||
}
|
||||
|
||||
// Tiers are the three tables. The rake is the casino's five percent everywhere,
|
||||
// capped at three big blinds a pot, and taken only from a pot that saw a flop.
|
||||
//
|
||||
// RakePct is a *fraction*, 0.05, because that is what it is everywhere else in
|
||||
// the casino — blackjack's DefaultRules says 0.05 and New() takes its word for it.
|
||||
// It was 5 here for an afternoon, meaning percent, and since New overwrites the
|
||||
// tier's value with the one it is handed, every rake worked out to five percent of
|
||||
// a hundredth of the pot, which integer division rounded to nothing. The house took
|
||||
// nothing at all and no test noticed, because every test set the tier up by hand.
|
||||
var Tiers = []Tier{
|
||||
{Slug: "micro", Name: "The Kitchen Table", SB: 1, BB: 2, MinBuy: 40, MaxBuy: 200, RakePct: 0.05,
|
||||
Blurb: "1/2 blinds. Cheap enough to learn what the bots do to you."},
|
||||
{Slug: "low", Name: "The Back Room", SB: 5, BB: 10, MinBuy: 200, MaxBuy: 1000, RakePct: 0.05,
|
||||
Blurb: "5/10. A bluff here costs real chips, which is the only reason a bluff works."},
|
||||
{Slug: "high", Name: "The High Roller", SB: 25, BB: 50, MinBuy: 1000, MaxBuy: 5000, RakePct: 0.05,
|
||||
Blurb: "25/50. Three streets of this and you know whether you can play."},
|
||||
}
|
||||
|
||||
// TierBySlug finds a table by the name the browser sent.
|
||||
func TierBySlug(slug string) (Tier, error) {
|
||||
for _, t := range Tiers {
|
||||
if t.Slug == slug {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
return Tier{}, ErrUnknownTier
|
||||
}
|
||||
|
||||
// MaxBots is five, which with you makes a six-handed table — the size most
|
||||
// online poker is actually played at, and as many opponents as the felt can
|
||||
// show without the cards getting too small to read.
|
||||
const MaxBots = 5
|
||||
|
||||
// Phase is what the table is waiting for.
|
||||
type Phase string
|
||||
|
||||
const (
|
||||
PhaseBetting Phase = "betting" // a hand is live and it's your turn
|
||||
PhaseHandOver Phase = "handover" // the hand is paid; deal, top up, or leave
|
||||
PhaseDone Phase = "done" // you're up from the table; the stack goes home
|
||||
)
|
||||
|
||||
// State is the whole table. It never leaves the server: the deck is in here,
|
||||
// and so is every bot's hand.
|
||||
type State struct {
|
||||
Tier Tier `json:"tier"`
|
||||
Seats []Seat `json:"seats"`
|
||||
Button int `json:"button"`
|
||||
HandNo int `json:"hand_no"`
|
||||
|
||||
Deck cards.Deck `json:"deck"`
|
||||
Community []cards.Card `json:"community"`
|
||||
Street Street `json:"street"`
|
||||
Flopped bool `json:"flopped"` // this hand saw a flop, so its pot is rakeable
|
||||
|
||||
Pot int64 `json:"pot"`
|
||||
Side []Pot `json:"side,omitempty"`
|
||||
Bet int64 `json:"bet"` // the bet to match on this street
|
||||
MinRaise int64 `json:"min_raise"` // the smallest legal raise over it
|
||||
Aggressor int `json:"aggressor"`
|
||||
ToAct int `json:"to_act"`
|
||||
|
||||
// History is the action so far on this street, as the f/c/r/R/a characters
|
||||
// the CFR policy was trained to read. It is the bots' memory and nothing else.
|
||||
History string `json:"history"`
|
||||
|
||||
Phase Phase `json:"phase"`
|
||||
|
||||
// The money that crosses the border. BoughtIn is every chip staked onto this
|
||||
// table; Payout is the stack that goes home, and is only set once you're up.
|
||||
BoughtIn int64 `json:"bought_in"`
|
||||
Payout int64 `json:"payout"`
|
||||
Rake int64 `json:"rake"` // taken by the house across the whole session
|
||||
|
||||
// The seed rides in the state for the same reason it does in UNO: the bots
|
||||
// choose and the deck is reshuffled every hand, so the engine needs randomness
|
||||
// mid-session — and there is no generator alive between two HTTP requests to
|
||||
// hand it. Each step derives its own from the seed and the step count, so the
|
||||
// session still replays exactly as it fell.
|
||||
Seed1 uint64 `json:"seed1"`
|
||||
Seed2 uint64 `json:"seed2"`
|
||||
Step uint64 `json:"step"`
|
||||
}
|
||||
|
||||
// Event is one beat of the script the felt plays back. Seat is -1 when the beat
|
||||
// belongs to the table rather than a player.
|
||||
type Event struct {
|
||||
Kind string `json:"kind"`
|
||||
Seat int `json:"seat"`
|
||||
Cards []cards.Card `json:"cards,omitempty"`
|
||||
Amount int64 `json:"amount,omitempty"`
|
||||
Total int64 `json:"total,omitempty"`
|
||||
Text string `json:"text,omitempty"`
|
||||
}
|
||||
|
||||
// The moves a player can make. The betting five, plus the three that are about
|
||||
// the session rather than the hand.
|
||||
const (
|
||||
Fold = "fold"
|
||||
Check = "check"
|
||||
Call = "call"
|
||||
Raise = "raise"
|
||||
Shove = "allin" // the move; AllIn is the seat state it puts you in
|
||||
|
||||
Deal = "deal" // next hand
|
||||
TopUp = "topup" // put more chips on the table, between hands
|
||||
Leave = "leave" // get up; the stack goes back to your stack
|
||||
)
|
||||
|
||||
// Move is what the browser sends. To is the total a raise raises *to*, and
|
||||
// Amount is chips added in a top-up.
|
||||
type Move struct {
|
||||
Kind string `json:"move"`
|
||||
To int64 `json:"to,omitempty"`
|
||||
Amount int64 `json:"amount,omitempty"`
|
||||
}
|
||||
|
||||
// botNames are the regulars. Six of them so a full table never has two.
|
||||
var botNames = []string{"Dice", "Marjorie", "Ox", "Sunny", "Pinch", "The Reverend"}
|
||||
|
||||
// New sits you down. The buy-in is chips the caller has already taken off the
|
||||
// player's stack; this engine only ever gives them back through Leave.
|
||||
//
|
||||
// No hand is dealt yet — the table opens on PhaseHandOver, which is the state a
|
||||
// table between hands is in, and the first Deal move starts the first hand.
|
||||
func New(t Tier, bots int, buyIn int64, rakePct float64, seed1, seed2 uint64) (State, []Event, error) {
|
||||
if bots < 1 || bots > MaxBots {
|
||||
return State{}, nil, ErrTableFull
|
||||
}
|
||||
if buyIn < t.MinBuy || buyIn > t.MaxBuy {
|
||||
return State{}, nil, ErrBadBuyIn
|
||||
}
|
||||
t.RakePct = rakePct
|
||||
|
||||
s := State{
|
||||
Tier: t,
|
||||
Button: 0,
|
||||
Phase: PhaseHandOver,
|
||||
BoughtIn: buyIn,
|
||||
Seed1: seed1,
|
||||
Seed2: seed2,
|
||||
}
|
||||
s.Seats = append(s.Seats, Seat{Name: "You", Stack: buyIn})
|
||||
for i := 0; i < bots; i++ {
|
||||
s.Seats = append(s.Seats, Seat{Name: botNames[i], Bot: true, Stack: t.MaxBuy})
|
||||
}
|
||||
|
||||
// The button starts to your right, so the first hand deals you the small blind
|
||||
// heads-up and the button on a full table — either way you are in the action
|
||||
// from the first card rather than folding your way in.
|
||||
s.Button = len(s.Seats) - 1
|
||||
|
||||
evs := []Event{{Kind: "sit", Seat: You, Amount: buyIn, Text: t.Name}}
|
||||
return s, evs, nil
|
||||
}
|
||||
|
||||
// ApplyMove is the whole engine. It plays your move, then every bot behind you,
|
||||
// deals whatever streets that finishes, and stops either when it is your turn
|
||||
// again or when the hand is over.
|
||||
func ApplyMove(s State, m Move) (State, []Event, error) {
|
||||
if s.Phase == PhaseDone {
|
||||
return s, nil, ErrOver
|
||||
}
|
||||
rng := s.next()
|
||||
evs := []Event{}
|
||||
|
||||
switch m.Kind {
|
||||
case Deal:
|
||||
if s.Phase != PhaseHandOver {
|
||||
return s, nil, ErrHandLive
|
||||
}
|
||||
s.deal(&evs, rng)
|
||||
s.advance(&evs, rng, true)
|
||||
|
||||
case TopUp:
|
||||
if s.Phase != PhaseHandOver {
|
||||
return s, nil, ErrHandLive
|
||||
}
|
||||
if m.Amount <= 0 || s.Seats[You].Stack+m.Amount > s.Tier.MaxBuy {
|
||||
return s, nil, ErrBadBuyIn
|
||||
}
|
||||
s.Seats[You].Stack += m.Amount
|
||||
s.BoughtIn += m.Amount
|
||||
evs = append(evs, Event{Kind: "topup", Seat: You, Amount: m.Amount})
|
||||
|
||||
case Leave:
|
||||
if s.Phase != PhaseHandOver {
|
||||
return s, nil, ErrHandLive
|
||||
}
|
||||
s.Phase = PhaseDone
|
||||
s.Payout = s.Seats[You].Stack
|
||||
evs = append(evs, Event{Kind: "leave", Seat: You, Amount: s.Payout})
|
||||
|
||||
case Fold, Check, Call, Raise, Shove:
|
||||
if s.Phase != PhaseBetting {
|
||||
return s, nil, ErrNoHand
|
||||
}
|
||||
if s.ToAct != You {
|
||||
return s, nil, ErrNotYourTurn
|
||||
}
|
||||
if err := s.act(You, m, &evs); err != nil {
|
||||
return s, nil, err // nothing happened; the caller keeps the old state
|
||||
}
|
||||
s.ToAct = s.nextCanAct(You)
|
||||
s.advance(&evs, rng, true)
|
||||
|
||||
default:
|
||||
return s, nil, ErrUnknownMove
|
||||
}
|
||||
|
||||
return s, evs, nil
|
||||
}
|
||||
|
||||
// Step plays a move for whoever is to act — bot or player — and advances only as
|
||||
// far as the next decision, whoever's it is. Nobody's turn is taken for them.
|
||||
//
|
||||
// This is the seam the trainer plays through, and it exists so that the trainer
|
||||
// is playing *this* game: the same betting rules, the same street completion, the
|
||||
// same side pots, the same money. The alternative is a second, simplified model
|
||||
// of poker written for the trainer alone — which is what gogobee had, and it is
|
||||
// why its policy encoded a game nobody was dealing.
|
||||
func Step(s State, m Move) (State, []Event, error) {
|
||||
if s.Phase != PhaseBetting {
|
||||
return s, nil, ErrNoHand
|
||||
}
|
||||
rng := s.next()
|
||||
evs := []Event{}
|
||||
|
||||
seat := s.ToAct
|
||||
if err := s.act(seat, m, &evs); err != nil {
|
||||
return s, nil, err
|
||||
}
|
||||
s.ToAct = s.nextCanAct(seat)
|
||||
s.advance(&evs, rng, false)
|
||||
return s, evs, nil
|
||||
}
|
||||
|
||||
// Open deals one heads-up hand at the given stacks, stopping at the first
|
||||
// decision. For the trainer: a table with no history and no button rotation.
|
||||
func Open(t Tier, stack0, stack1 int64, seed1, seed2 uint64) (State, error) {
|
||||
if stack0 <= 0 || stack1 <= 0 {
|
||||
return State{}, ErrBadBuyIn
|
||||
}
|
||||
s := State{
|
||||
Tier: t,
|
||||
Phase: PhaseHandOver,
|
||||
Seats: []Seat{
|
||||
{Name: "0", Stack: stack0},
|
||||
{Name: "1", Stack: stack1, Bot: true},
|
||||
},
|
||||
Button: 1, // deal() moves it, so seat 0 takes the button and the small blind
|
||||
Seed1: seed1,
|
||||
Seed2: seed2,
|
||||
}
|
||||
rng := s.next()
|
||||
evs := []Event{}
|
||||
s.deal(&evs, rng)
|
||||
s.advance(&evs, rng, false)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Clone deep-copies the table. CFR walks a tree of what-ifs, and a shallow copy
|
||||
// would have every branch writing into the same deck.
|
||||
func (s State) Clone() State {
|
||||
out := s
|
||||
out.Seats = append([]Seat(nil), s.Seats...)
|
||||
out.Deck = append(cards.Deck(nil), s.Deck...)
|
||||
out.Community = append([]cards.Card(nil), s.Community...)
|
||||
out.Side = make([]Pot, len(s.Side))
|
||||
for i, p := range s.Side {
|
||||
out.Side[i] = Pot{Amount: p.Amount, Eligible: append([]int(nil), p.Eligible...)}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// act applies one seat's action, whoever it belongs to.
|
||||
func (s *State) act(seat int, m Move, evs *[]Event) error {
|
||||
switch m.Kind {
|
||||
case Fold:
|
||||
s.fold(seat, evs)
|
||||
return nil
|
||||
case Check:
|
||||
return s.check(seat, evs)
|
||||
case Call:
|
||||
return s.call(seat, evs)
|
||||
case Raise:
|
||||
return s.raise(seat, m.To, evs)
|
||||
case Shove:
|
||||
return s.allin(seat, evs)
|
||||
}
|
||||
return ErrUnknownMove
|
||||
}
|
||||
|
||||
// advance runs the table forward until you have a decision to make, or until
|
||||
// there is nothing left to decide.
|
||||
//
|
||||
// This is the loop the whole design turns on. Every other engine here returns
|
||||
// after one move because there is nobody else at the table; this one keeps going
|
||||
// — bot, bot, flop, bot, turn — and only stops when the answer has to come from
|
||||
// the player. Which is why one HTTP request can be a whole hand: shove all-in
|
||||
// and the board runs out and the pot is paid inside a single call.
|
||||
//
|
||||
// With bots false it stops at every decision instead of playing the bots' for
|
||||
// them. That is the trainer's way in: it wants to choose both seats' moves.
|
||||
func (s *State) advance(evs *[]Event, rng *rand.Rand, bots bool) {
|
||||
for {
|
||||
// Everyone else folded. Nobody shows; the last one standing takes it.
|
||||
if s.liveCount() <= 1 {
|
||||
s.takeit(evs)
|
||||
return
|
||||
}
|
||||
|
||||
// Nobody left with a decision to make: the rest of the board is a formality,
|
||||
// so deal it and turn the cards over.
|
||||
//
|
||||
// The subtle half is the lone player who still has chips. They only have a
|
||||
// decision if there is a bet to them — call it or fold. If there isn't, they
|
||||
// have nobody left to bet *into*, because everyone else is already all-in,
|
||||
// and poker does not let you put chips in a pot nobody can contest.
|
||||
switch s.canActCount() {
|
||||
case 0:
|
||||
s.runout(evs)
|
||||
return
|
||||
case 1:
|
||||
lone := s.onlyActor()
|
||||
if s.Owed(lone) == 0 {
|
||||
s.runout(evs)
|
||||
return
|
||||
}
|
||||
s.ToAct = lone
|
||||
}
|
||||
|
||||
if s.streetDone(s.ToAct) {
|
||||
if s.Street == River {
|
||||
s.showdown(evs)
|
||||
return
|
||||
}
|
||||
s.street(evs)
|
||||
continue
|
||||
}
|
||||
|
||||
if s.ToAct == You || !bots {
|
||||
return // a decision that isn't ours to make
|
||||
}
|
||||
|
||||
s.botActs(s.ToAct, evs, rng)
|
||||
s.ToAct = s.nextCanAct(s.ToAct)
|
||||
}
|
||||
}
|
||||
|
||||
// deal starts a hand: rebuy the broke bots, move the button, shuffle, post the
|
||||
// blinds, and put two cards in front of everybody.
|
||||
func (s *State) deal(evs *[]Event, rng *rand.Rand) {
|
||||
s.HandNo++
|
||||
|
||||
for i := range s.Seats {
|
||||
p := &s.Seats[i]
|
||||
// A bot that has been ground down to nothing reloads. It has to: a table
|
||||
// where you have taken everybody's chips is a table with no game left in it,
|
||||
// and their chips were never real anyway — the only real money at this table
|
||||
// is yours, and the only thing the house takes is the rake.
|
||||
if p.Bot && p.Stack < s.Tier.BB {
|
||||
add := s.Tier.MaxBuy - p.Stack
|
||||
p.Stack = s.Tier.MaxBuy
|
||||
*evs = append(*evs, Event{Kind: "rebuy", Seat: i, Amount: add, Total: p.Stack})
|
||||
}
|
||||
p.Bet, p.Total, p.Won, p.Acted = 0, 0, 0, false
|
||||
p.Hole = [2]cards.Card{}
|
||||
p.State = Active
|
||||
if p.Stack <= 0 {
|
||||
p.State = Out
|
||||
}
|
||||
}
|
||||
|
||||
s.Community = nil
|
||||
s.Side = nil
|
||||
s.Pot = 0
|
||||
s.Street = PreFlop
|
||||
s.Flopped = false
|
||||
s.History = ""
|
||||
s.Phase = PhaseBetting
|
||||
|
||||
s.Deck = cards.NewDeck(1)
|
||||
s.Deck.Shuffle(rng)
|
||||
|
||||
s.Button = s.nextIn(s.Button)
|
||||
*evs = append(*evs, Event{Kind: "hand", Seat: s.Button, Amount: int64(s.HandNo)})
|
||||
|
||||
bb := s.blinds(evs)
|
||||
|
||||
// Two cards each, one at a time round the table, as they are actually dealt.
|
||||
for round := 0; round < 2; round++ {
|
||||
for i := 0; i < len(s.Seats); i++ {
|
||||
seat := (s.Button + 1 + i) % len(s.Seats)
|
||||
p := &s.Seats[seat]
|
||||
if p.State == Out {
|
||||
continue
|
||||
}
|
||||
c, _ := s.Deck.Draw()
|
||||
p.Hole[round] = c
|
||||
}
|
||||
}
|
||||
// Only your cards go into the script. The bots' are in the state, on this side
|
||||
// of the wire, and the only thing that ever turns them over is a showdown.
|
||||
*evs = append(*evs, Event{Kind: "hole", Seat: You,
|
||||
Cards: []cards.Card{s.Seats[You].Hole[0], s.Seats[You].Hole[1]}})
|
||||
|
||||
s.ToAct = s.firstPreFlop(bb)
|
||||
}
|
||||
|
||||
// street burns one and deals the next board card or three.
|
||||
func (s *State) street(evs *[]Event) {
|
||||
s.collect()
|
||||
s.resetBets()
|
||||
|
||||
s.Deck.Draw() // the burn card, as printed in the rules and as dealt in a casino
|
||||
|
||||
switch s.Street {
|
||||
case PreFlop:
|
||||
s.Street = Flop
|
||||
s.Flopped = true
|
||||
for i := 0; i < 3; i++ {
|
||||
c, _ := s.Deck.Draw()
|
||||
s.Community = append(s.Community, c)
|
||||
}
|
||||
case Flop:
|
||||
s.Street = Turn
|
||||
c, _ := s.Deck.Draw()
|
||||
s.Community = append(s.Community, c)
|
||||
case Turn:
|
||||
s.Street = River
|
||||
c, _ := s.Deck.Draw()
|
||||
s.Community = append(s.Community, c)
|
||||
}
|
||||
|
||||
// Total, not Pot: by the time a board runs out behind an all-in the money has
|
||||
// already been cut into side pots, and s.Pot is zero.
|
||||
*evs = append(*evs, Event{Kind: s.Street.String(), Seat: -1,
|
||||
Cards: s.Community[len(s.Community)-cardsOn(s.Street):], Amount: s.Total()})
|
||||
|
||||
s.ToAct = s.firstPostFlop()
|
||||
s.Aggressor = s.ToAct // nobody has bet yet, so the option ends where it starts
|
||||
}
|
||||
|
||||
func cardsOn(st Street) int {
|
||||
if st == Flop {
|
||||
return 3
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// runout deals the rest of the board with no more betting, because there is no
|
||||
// longer anybody able to bet. The side pots are built first: once the chips stop
|
||||
// moving, who can win what is already decided.
|
||||
func (s *State) runout(evs *[]Event) {
|
||||
allIn := false
|
||||
for i := range s.Seats {
|
||||
if s.Seats[i].State == AllIn {
|
||||
allIn = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if allIn {
|
||||
// A shove nobody could cover comes back first — while it is still a bet in
|
||||
// front of a seat, and before the pots are cut around it.
|
||||
s.uncalled(evs)
|
||||
}
|
||||
s.collect()
|
||||
if allIn {
|
||||
s.sidePots()
|
||||
}
|
||||
|
||||
for s.Street < River {
|
||||
s.street(evs)
|
||||
}
|
||||
s.showdown(evs)
|
||||
}
|
||||
|
||||
// endHand pays out and parks the table between hands.
|
||||
func (s *State) endHand(evs *[]Event) {
|
||||
s.Pot = 0
|
||||
s.Side = nil
|
||||
s.Phase = PhaseHandOver
|
||||
*evs = append(*evs, Event{Kind: "end", Seat: -1, Amount: s.Seats[You].Stack})
|
||||
|
||||
// Busting is the end of the session, not the end of a hand. There is nothing
|
||||
// to deal you and nothing to give back, so the table closes and you sit down
|
||||
// again — which is a buy-in, and a buy-in is a decision worth making on purpose.
|
||||
if s.Seats[You].Stack <= 0 {
|
||||
s.Phase = PhaseDone
|
||||
s.Payout = 0
|
||||
*evs = append(*evs, Event{Kind: "bust", Seat: You})
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the small stuff -------------------------------------------------------
|
||||
|
||||
// next derives this step's generator and advances the step count.
|
||||
func (s *State) next() *rand.Rand {
|
||||
s.Step++
|
||||
return cards.NewRNG(s.Seed1, s.Seed2^s.Step)
|
||||
}
|
||||
|
||||
// collect sweeps the street's bets into the pot.
|
||||
func (s *State) collect() {
|
||||
for i := range s.Seats {
|
||||
s.Pot += s.Seats[i].Bet
|
||||
s.Seats[i].Bet = 0
|
||||
}
|
||||
}
|
||||
|
||||
// resetBets opens a new street: nothing to call, and nobody has spoken.
|
||||
func (s *State) resetBets() {
|
||||
for i := range s.Seats {
|
||||
s.Seats[i].Bet = 0
|
||||
s.Seats[i].Acted = false
|
||||
}
|
||||
s.Bet = 0
|
||||
s.MinRaise = s.Tier.BB
|
||||
s.History = ""
|
||||
}
|
||||
|
||||
// inPlay is the pot plus everything bet on this street — what a bot is actually
|
||||
// deciding against, and what a pot-sized raise is a size of.
|
||||
func (s State) inPlay() int64 {
|
||||
total := s.Pot
|
||||
for i := range s.Seats {
|
||||
total += s.Seats[i].Bet
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// Pot returns the money on the table, however it is currently sliced.
|
||||
func (s State) Total() int64 {
|
||||
total := s.inPlay()
|
||||
for _, p := range s.Side {
|
||||
total += p.Amount
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// liveCount is the seats still in the hand, whether or not they can bet.
|
||||
func (s State) liveCount() int {
|
||||
n := 0
|
||||
for i := range s.Seats {
|
||||
if st := s.Seats[i].State; st == Active || st == AllIn {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// canActCount is the seats that still have chips and a decision.
|
||||
func (s State) canActCount() int {
|
||||
n := 0
|
||||
for i := range s.Seats {
|
||||
if s.Seats[i].State == Active {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// dealt is the seats in this hand at all.
|
||||
func (s State) dealt() int {
|
||||
n := 0
|
||||
for i := range s.Seats {
|
||||
if s.Seats[i].State != Out {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// nextIn is the next seat that was dealt in — used to move the button, which
|
||||
// moves past a seat that is sitting out rather than landing on it.
|
||||
func (s State) nextIn(from int) int {
|
||||
n := len(s.Seats)
|
||||
for i := 1; i <= n; i++ {
|
||||
next := (from + i) % n
|
||||
if st := s.Seats[next].State; st == Active || st == AllIn {
|
||||
return next
|
||||
}
|
||||
}
|
||||
return from
|
||||
}
|
||||
|
||||
// onlyActor is the one seat that can still act. Call it when canActCount is 1.
|
||||
func (s State) onlyActor() int {
|
||||
for i := range s.Seats {
|
||||
if s.Seats[i].State == Active {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return s.ToAct
|
||||
}
|
||||
|
||||
// canBet reports whether there is anybody left to bet *into*. With one player
|
||||
// still holding chips and the rest all-in, a raise is chips nobody can call, so
|
||||
// no raise is offered — to the player or to a bot.
|
||||
func (s State) canBet() bool { return s.canActCount() > 1 }
|
||||
|
||||
// nextCanAct is the next seat with a decision to make.
|
||||
func (s State) nextCanAct(from int) int {
|
||||
n := len(s.Seats)
|
||||
for i := 1; i <= n; i++ {
|
||||
next := (from + i) % n
|
||||
if s.Seats[next].State == Active {
|
||||
return next
|
||||
}
|
||||
}
|
||||
return from
|
||||
}
|
||||
|
||||
// Owed is what the seat must put in to call.
|
||||
func (s State) Owed(seat int) int64 {
|
||||
owed := s.Bet - s.Seats[seat].Bet
|
||||
if owed < 0 {
|
||||
return 0
|
||||
}
|
||||
if owed > s.Seats[seat].Stack {
|
||||
return s.Seats[seat].Stack
|
||||
}
|
||||
return owed
|
||||
}
|
||||
|
||||
// MinRaiseTo is the smallest total a raise may raise to, clamped to a shove when
|
||||
// the seat cannot cover a full one.
|
||||
func (s State) MinRaiseTo(seat int) int64 {
|
||||
to := s.Bet + s.MinRaise
|
||||
if most := s.Seats[seat].Bet + s.Seats[seat].Stack; to > most {
|
||||
return most
|
||||
}
|
||||
return to
|
||||
}
|
||||
|
||||
// MaxRaiseTo is everything the seat has.
|
||||
func (s State) MaxRaiseTo(seat int) int64 {
|
||||
return s.Seats[seat].Bet + s.Seats[seat].Stack
|
||||
}
|
||||
|
||||
// CanRaise reports whether the seat may raise: they need chips behind the call,
|
||||
// and somebody left to bet into. The felt asks so it never offers a button the
|
||||
// table would refuse.
|
||||
func (s State) CanRaise(seat int) bool {
|
||||
return s.mask(seat)[actRaiseHalf]
|
||||
}
|
||||
|
||||
// InPosition reports whether the seat acts last after the flop, which is the
|
||||
// only thing about position the trained bots actually know.
|
||||
//
|
||||
// The postflop order runs from the seat left of the button all the way round to
|
||||
// the button itself, so the player in position is simply the last one still in
|
||||
// the hand — the button, or whoever is nearest to it once the button has folded.
|
||||
// The policy was trained heads-up, where this is exactly the button; applying it
|
||||
// to a six-handed table is an approximation, and this is where the approximation
|
||||
// lives.
|
||||
func (s State) InPosition(seat int) bool {
|
||||
last := -1
|
||||
for i := 1; i <= len(s.Seats); i++ {
|
||||
at := (s.Button + i) % len(s.Seats)
|
||||
if st := s.Seats[at].State; st == Active || st == AllIn {
|
||||
last = at
|
||||
}
|
||||
}
|
||||
return seat == last
|
||||
}
|
||||
|
||||
// Position is the seat's label at this table — BTN, SB, BB, and so on. It is for
|
||||
// the felt to print. The bots do not use it: see InPosition, and the note on
|
||||
// infoSet about what happens when you confuse the two.
|
||||
func (s State) Position(seat int) string {
|
||||
n := s.dealt()
|
||||
if n < 2 {
|
||||
return ""
|
||||
}
|
||||
if seat == s.Button {
|
||||
return "BTN"
|
||||
}
|
||||
if n == 2 {
|
||||
return "BB" // heads-up, the other seat is always the big blind
|
||||
}
|
||||
|
||||
sb := s.nextIn(s.Button)
|
||||
bb := s.nextIn(sb)
|
||||
switch seat {
|
||||
case sb:
|
||||
return "SB"
|
||||
case bb:
|
||||
return "BB"
|
||||
}
|
||||
|
||||
utg := s.nextIn(bb)
|
||||
if seat == utg {
|
||||
return "UTG"
|
||||
}
|
||||
|
||||
// Everyone between UTG and the button is somewhere in the middle; the seat
|
||||
// closest to the button is the cutoff.
|
||||
dist, cur := 0, utg
|
||||
for i := 0; i < n; i++ {
|
||||
cur = s.nextIn(cur)
|
||||
dist++
|
||||
if cur == seat {
|
||||
break
|
||||
}
|
||||
}
|
||||
if remaining := n - 4; remaining > 0 && dist >= remaining {
|
||||
return "CO"
|
||||
}
|
||||
return "MP"
|
||||
}
|
||||
641
internal/games/holdem/holdem_test.go
Normal file
641
internal/games/holdem/holdem_test.go
Normal file
@@ -0,0 +1,641 @@
|
||||
package holdem
|
||||
|
||||
import (
|
||||
"math/rand/v2"
|
||||
"testing"
|
||||
|
||||
"pete/internal/games/cards"
|
||||
)
|
||||
|
||||
// The one that matters: no chip is ever created or destroyed.
|
||||
//
|
||||
// Everything else in this package is a rule you could argue about. This is the
|
||||
// one that would lose somebody money. Every chip at the table is in exactly one
|
||||
// place — a stack, a bet in front of a seat, a pot, or the house's rake — and
|
||||
// the only thing that ever adds to the total is a bot reloading. So: play a
|
||||
// hundred sessions of real hands, with the trained bots making real decisions,
|
||||
// and count the chips after every single move.
|
||||
func TestChipsAreConserved(t *testing.T) {
|
||||
for game := 0; game < 100; game++ {
|
||||
rng := rand.New(rand.NewPCG(uint64(game), 99))
|
||||
bots := 1 + game%MaxBots
|
||||
tier := Tiers[game%len(Tiers)]
|
||||
|
||||
s, _, err := New(tier, bots, tier.MaxBuy, tier.RakePct, uint64(game), 7)
|
||||
if err != nil {
|
||||
t.Fatalf("new table: %v", err)
|
||||
}
|
||||
|
||||
want := chipsAt(s) // what the table started with
|
||||
|
||||
for hand := 0; hand < 8 && s.Phase != PhaseDone; hand++ {
|
||||
var evs []Event
|
||||
s, evs, err = ApplyMove(s, Move{Kind: Deal})
|
||||
if err != nil {
|
||||
t.Fatalf("game %d hand %d: deal: %v", game, hand, err)
|
||||
}
|
||||
want += reloaded(evs) // a bot that rebought brought new chips with it
|
||||
check(t, s, want, game, hand, "deal")
|
||||
|
||||
for step := 0; s.Phase == PhaseBetting; step++ {
|
||||
if step > 200 {
|
||||
t.Fatalf("game %d hand %d: the hand will not end", game, hand)
|
||||
}
|
||||
s, _, err = ApplyMove(s, randomMove(s, rng))
|
||||
if err != nil {
|
||||
t.Fatalf("game %d hand %d: %v", game, hand, err)
|
||||
}
|
||||
check(t, s, want, game, hand, "move")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// chipsAt totals every chip the table can see, plus every one the house has
|
||||
// already taken out of it.
|
||||
func chipsAt(s State) int64 {
|
||||
total := s.Rake + s.Pot
|
||||
for _, p := range s.Seats {
|
||||
total += p.Stack + p.Bet
|
||||
}
|
||||
for _, pot := range s.Side {
|
||||
total += pot.Amount
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// reloaded is what the bots brought back to the table on this deal. It is the
|
||||
// only thing in the game that is allowed to make chips out of nothing, which is
|
||||
// exactly why the test has to know about it and nothing else does.
|
||||
func reloaded(evs []Event) int64 {
|
||||
var n int64
|
||||
for _, e := range evs {
|
||||
if e.Kind == "rebuy" {
|
||||
n += e.Amount
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func check(t *testing.T, s State, want int64, game, hand int, when string) {
|
||||
t.Helper()
|
||||
if got := chipsAt(s); got != want {
|
||||
t.Fatalf("game %d hand %d, after %s: %d chips on the table, want %d "+
|
||||
"(pot %d, rake %d, stacks %v)", game, hand, when, got, want, s.Pot, s.Rake, stacks(s))
|
||||
}
|
||||
for i, p := range s.Seats {
|
||||
if p.Stack < 0 {
|
||||
t.Fatalf("game %d hand %d: seat %d has a negative stack (%d)", game, hand, i, p.Stack)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stacks(s State) []int64 {
|
||||
out := make([]int64, len(s.Seats))
|
||||
for i, p := range s.Seats {
|
||||
out[i] = p.Stack
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// randomMove picks something legal for the player, without any thought at all.
|
||||
// A bad player is exactly what this test wants: it gets into all-ins, folds,
|
||||
// short stacks and split pots far faster than a good one would.
|
||||
func randomMove(s State, rng *rand.Rand) Move {
|
||||
owed := s.Owed(You)
|
||||
var legal []Move
|
||||
if owed > 0 {
|
||||
legal = append(legal, Move{Kind: Fold}, Move{Kind: Call})
|
||||
} else {
|
||||
legal = append(legal, Move{Kind: Check})
|
||||
}
|
||||
if s.Seats[You].Stack > owed && s.canBet() {
|
||||
legal = append(legal, Move{Kind: Shove})
|
||||
if to := s.MinRaiseTo(You); to < s.MaxRaiseTo(You) {
|
||||
legal = append(legal, Move{Kind: Raise, To: to})
|
||||
}
|
||||
}
|
||||
return legal[rng.IntN(len(legal))]
|
||||
}
|
||||
|
||||
// ---- the rules a poker player would notice were wrong -----------------------
|
||||
|
||||
func TestHeadsUpButtonIsTheSmallBlindAndActsFirst(t *testing.T) {
|
||||
s := table(t, Tiers[0], 1, 200)
|
||||
s, evs, err := ApplyMove(s, Move{Kind: Deal})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The button posted the small blind, not the big one.
|
||||
var small, big int
|
||||
for _, e := range evs {
|
||||
if e.Kind == "blind" && e.Text == "small" {
|
||||
small = e.Seat
|
||||
}
|
||||
if e.Kind == "blind" && e.Text == "big" {
|
||||
big = e.Seat
|
||||
}
|
||||
}
|
||||
if small != s.Button {
|
||||
t.Errorf("heads-up: seat %d posted the small blind, but the button is seat %d", small, s.Button)
|
||||
}
|
||||
if big == s.Button {
|
||||
t.Error("heads-up: the button posted the big blind too")
|
||||
}
|
||||
// And it is the first to act before the flop. (If the button is a bot it has
|
||||
// already acted, so what we can check is that the player didn't get skipped.)
|
||||
if s.Phase != PhaseBetting {
|
||||
t.Fatalf("phase %q: the hand should be waiting on somebody", s.Phase)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBigBlindGetsTheirOption(t *testing.T) {
|
||||
// A table where everyone just calls: the big blind has the bet matched without
|
||||
// ever having chosen anything, and the street must not end until they speak.
|
||||
s := table(t, Tiers[0], 1, 200)
|
||||
s, _, _ = ApplyMove(s, Move{Kind: Deal})
|
||||
|
||||
// Find a hand where the player is the big blind. The button alternates, so at
|
||||
// most a couple of deals.
|
||||
for i := 0; i < 6 && s.Position(You) != "BB"; i++ {
|
||||
s = playOut(t, s)
|
||||
s, _, _ = ApplyMove(s, Move{Kind: Deal})
|
||||
}
|
||||
if s.Position(You) != "BB" {
|
||||
t.Skip("never dealt the big blind")
|
||||
}
|
||||
if s.Phase != PhaseBetting {
|
||||
return // the bot folded or raised; either way the option isn't the question
|
||||
}
|
||||
if s.ToAct == You && s.Owed(You) == 0 {
|
||||
// This is the option: nothing to call, but the hand is still ours to act on.
|
||||
if _, _, err := ApplyMove(s, Move{Kind: Check}); err != nil {
|
||||
t.Errorf("the big blind cannot check their option: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAShortAllInDoesNotReopenTheBetting(t *testing.T) {
|
||||
s := State{
|
||||
Tier: Tiers[1], // 5/10
|
||||
Seats: []Seat{{Name: "You", Stack: 1000}, {Name: "Bot", Bot: true, Stack: 1000}},
|
||||
Bet: 100,
|
||||
MinRaise: 100, // a full raise would be to 200
|
||||
Aggressor: You,
|
||||
Phase: PhaseBetting,
|
||||
}
|
||||
s.Seats[You].Bet = 100
|
||||
s.Seats[1].Bet = 0
|
||||
s.Seats[1].Stack = 150 // can only get to 150, which is a raise of 50: not a full one
|
||||
|
||||
var evs []Event
|
||||
if err := s.allin(1, &evs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.Bet != 150 {
|
||||
t.Errorf("the bet to call is %d, want 150", s.Bet)
|
||||
}
|
||||
if s.MinRaise != 100 {
|
||||
t.Errorf("min raise moved to %d — a short all-in must not change it", s.MinRaise)
|
||||
}
|
||||
if s.Aggressor != You {
|
||||
t.Errorf("the aggressor moved to seat %d — a short all-in must not reopen the action", s.Aggressor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSidePotsPayInLayers(t *testing.T) {
|
||||
// Three players all-in for different amounts. The short stack can only win
|
||||
// what everyone could have lost to them.
|
||||
s := State{
|
||||
Tier: Tiers[1],
|
||||
Seats: []Seat{{Name: "You"}, {Name: "A", Bot: true}, {Name: "B", Bot: true}},
|
||||
}
|
||||
s.Seats[0].Total, s.Seats[0].State = 100, AllIn // short
|
||||
s.Seats[1].Total, s.Seats[1].State = 500, AllIn // middle
|
||||
s.Seats[2].Total, s.Seats[2].State = 500, AllIn // covers
|
||||
|
||||
s.sidePots()
|
||||
|
||||
if len(s.Side) != 2 {
|
||||
t.Fatalf("got %d pots, want 2: %+v", len(s.Side), s.Side)
|
||||
}
|
||||
main, side := s.Side[0], s.Side[1]
|
||||
if main.Amount != 300 { // 100 from each of the three
|
||||
t.Errorf("main pot is %d, want 300", main.Amount)
|
||||
}
|
||||
if len(main.Eligible) != 3 {
|
||||
t.Errorf("main pot has %d eligible, want all 3", len(main.Eligible))
|
||||
}
|
||||
if side.Amount != 800 { // 400 more from each of the two who had it
|
||||
t.Errorf("side pot is %d, want 800", side.Amount)
|
||||
}
|
||||
if len(side.Eligible) != 2 {
|
||||
t.Errorf("side pot has %d eligible, want 2 — the short stack cannot win it", len(side.Eligible))
|
||||
}
|
||||
if total := main.Amount + side.Amount; total != 1100 {
|
||||
t.Errorf("the pots hold %d, but %d went in", total, 1100)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoldedChipsStayInThePotButWinNothing(t *testing.T) {
|
||||
s := State{
|
||||
Tier: Tiers[1],
|
||||
Seats: []Seat{{Name: "You"}, {Name: "A", Bot: true}, {Name: "B", Bot: true}},
|
||||
}
|
||||
s.Seats[0].Total, s.Seats[0].State = 200, AllIn
|
||||
s.Seats[1].Total, s.Seats[1].State = 50, Folded // called 50 and gave up
|
||||
s.Seats[2].Total, s.Seats[2].State = 200, AllIn
|
||||
|
||||
s.sidePots()
|
||||
|
||||
var total int64
|
||||
for _, p := range s.Side {
|
||||
total += p.Amount
|
||||
for _, seat := range p.Eligible {
|
||||
if seat == 1 {
|
||||
t.Error("a folded seat is eligible to win a pot")
|
||||
}
|
||||
}
|
||||
}
|
||||
if total != 450 {
|
||||
t.Errorf("the pots hold %d, want 450 — the folder's 50 has to still be in there", total)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUncalledBetComesBack(t *testing.T) {
|
||||
s := State{
|
||||
Tier: Tiers[1],
|
||||
Seats: []Seat{{Name: "You", Stack: 0}, {Name: "A", Bot: true}},
|
||||
}
|
||||
s.Seats[0].Total, s.Seats[0].Bet, s.Seats[0].State = 500, 500, AllIn
|
||||
s.Seats[1].Total, s.Seats[1].State = 200, Folded
|
||||
|
||||
var evs []Event
|
||||
s.uncalled(&evs)
|
||||
|
||||
if s.Seats[You].Stack != 300 {
|
||||
t.Errorf("got %d back, want the 300 nobody called", s.Seats[You].Stack)
|
||||
}
|
||||
if s.Seats[You].Total != 200 {
|
||||
t.Errorf("still committed for %d, want 200 — the rest was never in the pot", s.Seats[You].Total)
|
||||
}
|
||||
if s.Seats[You].State != Active {
|
||||
t.Error("still marked all-in for chips that came back")
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the rake --------------------------------------------------------------
|
||||
|
||||
func TestNoFlopNoDrop(t *testing.T) {
|
||||
s := State{Tier: Tiers[1], Flopped: false, Seats: []Seat{{Name: "You"}, {Name: "A", Bot: true}}}
|
||||
var evs []Event
|
||||
s.payPot(Pot{Amount: 1000, Eligible: []int{You}}, []ranked{{seat: You}}, &evs)
|
||||
|
||||
if s.Rake != 0 {
|
||||
t.Errorf("raked %d off a pot that never saw a flop", s.Rake)
|
||||
}
|
||||
if s.Seats[You].Stack != 1000 {
|
||||
t.Errorf("paid %d of a 1000 pot", s.Seats[You].Stack)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRakeIsCapped(t *testing.T) {
|
||||
s := State{Tier: Tiers[1], Flopped: true, Seats: []Seat{{Name: "You"}, {Name: "A", Bot: true}}}
|
||||
var evs []Event
|
||||
// 5% of 10,000 is 500, but the cap is three big blinds — 30 at 5/10.
|
||||
s.payPot(Pot{Amount: 10000, Eligible: []int{You}}, []ranked{{seat: You}}, &evs)
|
||||
|
||||
want := s.Tier.BB * rakeCapBB
|
||||
if s.Rake != want {
|
||||
t.Errorf("raked %d, want the %d cap", s.Rake, want)
|
||||
}
|
||||
if s.Seats[You].Stack != 10000-want {
|
||||
t.Errorf("paid %d, want %d", s.Seats[You].Stack, 10000-want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRakeIsFivePercentUnderTheCap(t *testing.T) {
|
||||
s := State{Tier: Tiers[0], Flopped: true, Seats: []Seat{{Name: "You"}, {Name: "A", Bot: true}}}
|
||||
var evs []Event
|
||||
s.payPot(Pot{Amount: 100, Eligible: []int{You}}, []ranked{{seat: You}}, &evs) // cap is 6 at 1/2
|
||||
|
||||
if s.Rake != 5 {
|
||||
t.Errorf("raked %d off a 100 pot, want 5", s.Rake)
|
||||
}
|
||||
if s.Seats[You].Stack != 95 {
|
||||
t.Errorf("paid %d, want 95", s.Seats[You].Stack)
|
||||
}
|
||||
}
|
||||
|
||||
// The rake has to survive the wiring, not only the arithmetic.
|
||||
//
|
||||
// This is the test that was missing, and a browser found what it would have
|
||||
// found: New() overwrites the tier's rake with the one the casino hands it, and
|
||||
// the casino hands it a *fraction* (blackjack's 0.05). The tier declared 5,
|
||||
// meaning percent. Every other rake test builds a State by hand and sets the tier
|
||||
// itself, so not one of them ever saw the number a real table runs on — and the
|
||||
// house quietly took nothing from every pot for an afternoon.
|
||||
func TestTheRakeSurvivesTheConstructor(t *testing.T) {
|
||||
tier := Tiers[1] // 5/10, so the cap is 30
|
||||
s, _, err := New(tier, 1, tier.MaxBuy, 0.05, 1, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.Flopped = true
|
||||
|
||||
var evs []Event
|
||||
s.payPot(Pot{Amount: 400, Eligible: []int{You}}, []ranked{{seat: You}}, &evs)
|
||||
|
||||
if s.Rake != 20 {
|
||||
t.Fatalf("the house took %d of a 400 pot, want 20 — five percent of it. "+
|
||||
"RakePct is a fraction (0.05), not a percentage (5): see the note on Tiers.", s.Rake)
|
||||
}
|
||||
if !has(evs, "rake") {
|
||||
t.Error("the rake was taken with no event to say so, so the felt cannot show it")
|
||||
}
|
||||
}
|
||||
|
||||
func TestASplitPotSplits(t *testing.T) {
|
||||
s := State{Tier: Tiers[1], Seats: []Seat{{Name: "You"}, {Name: "A", Bot: true}}}
|
||||
var evs []Event
|
||||
// Same rank: they chop. The odd chip goes to one of them, not into the air.
|
||||
s.payPot(Pot{Amount: 101, Eligible: []int{0, 1}},
|
||||
[]ranked{{seat: 0, rank: 500}, {seat: 1, rank: 500}}, &evs)
|
||||
|
||||
if got := s.Seats[0].Stack + s.Seats[1].Stack; got != 101 {
|
||||
t.Errorf("paid out %d of a 101 pot", got)
|
||||
}
|
||||
if s.Seats[0].Stack != 51 || s.Seats[1].Stack != 50 {
|
||||
t.Errorf("split %d/%d, want 51/50", s.Seats[0].Stack, s.Seats[1].Stack)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the session -----------------------------------------------------------
|
||||
|
||||
func TestYouCannotWalkOutOfALiveHand(t *testing.T) {
|
||||
s := table(t, Tiers[0], 2, 200)
|
||||
s, _, _ = ApplyMove(s, Move{Kind: Deal})
|
||||
if s.Phase != PhaseBetting {
|
||||
t.Skip("the hand ended before the player could act")
|
||||
}
|
||||
if _, _, err := ApplyMove(s, Move{Kind: Leave}); err != ErrHandLive {
|
||||
t.Errorf("leaving mid-hand gave %v, want ErrHandLive", err)
|
||||
}
|
||||
if _, _, err := ApplyMove(s, Move{Kind: TopUp, Amount: 10}); err != ErrHandLive {
|
||||
t.Errorf("topping up mid-hand gave %v, want ErrHandLive", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeavingTakesTheStackHome(t *testing.T) {
|
||||
s := table(t, Tiers[0], 1, 200)
|
||||
s, _, _ = ApplyMove(s, Move{Kind: Deal})
|
||||
s = playOut(t, s)
|
||||
|
||||
stack := s.Seats[You].Stack
|
||||
s, _, err := ApplyMove(s, Move{Kind: Leave})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.Phase != PhaseDone {
|
||||
t.Errorf("phase %q after leaving, want done", s.Phase)
|
||||
}
|
||||
if s.Payout != stack {
|
||||
t.Errorf("payout %d, want the %d that was in front of us", s.Payout, stack)
|
||||
}
|
||||
if _, _, err := ApplyMove(s, Move{Kind: Deal}); err != ErrOver {
|
||||
t.Errorf("dealt a hand at a table we got up from: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBustingEndsTheSession(t *testing.T) {
|
||||
s := table(t, Tiers[0], 1, 200)
|
||||
s.Seats[You].Stack = 0
|
||||
var evs []Event
|
||||
s.endHand(&evs)
|
||||
|
||||
if s.Phase != PhaseDone {
|
||||
t.Errorf("phase %q with no chips left, want done", s.Phase)
|
||||
}
|
||||
if s.Payout != 0 {
|
||||
t.Errorf("payout %d for a busted player", s.Payout)
|
||||
}
|
||||
if !has(evs, "bust") {
|
||||
t.Error("no bust event")
|
||||
}
|
||||
}
|
||||
|
||||
func TestATopUpCannotGoOverTheTableMax(t *testing.T) {
|
||||
s := table(t, Tiers[0], 1, 200) // max buy is 200, and we're at it
|
||||
if _, _, err := ApplyMove(s, Move{Kind: TopUp, Amount: 1}); err != ErrBadBuyIn {
|
||||
t.Errorf("topped up over the table maximum: %v", err)
|
||||
}
|
||||
|
||||
s = table(t, Tiers[0], 1, 100)
|
||||
s, _, err := ApplyMove(s, Move{Kind: TopUp, Amount: 50})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.Seats[You].Stack != 150 {
|
||||
t.Errorf("stack is %d, want 150", s.Seats[You].Stack)
|
||||
}
|
||||
if s.BoughtIn != 150 {
|
||||
t.Errorf("bought in for %d, want 150 — the top-up is real money too", s.BoughtIn)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuyInHasToBeInRange(t *testing.T) {
|
||||
tier := Tiers[0]
|
||||
for _, amount := range []int64{0, tier.MinBuy - 1, tier.MaxBuy + 1} {
|
||||
if _, _, err := New(tier, 1, amount, 5, 1, 2); err != ErrBadBuyIn {
|
||||
t.Errorf("buy-in of %d at a %d–%d table: %v", amount, tier.MinBuy, tier.MaxBuy, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- what the player is allowed to know ------------------------------------
|
||||
|
||||
func TestTheScriptNeverCarriesABotsCards(t *testing.T) {
|
||||
rng := rand.New(rand.NewPCG(4, 4))
|
||||
s := table(t, Tiers[0], 3, 200)
|
||||
|
||||
for hand := 0; hand < 20 && s.Phase != PhaseDone; hand++ {
|
||||
s, evs, err := ApplyMove(s, Move{Kind: Deal})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
noBotCards(t, s, evs)
|
||||
|
||||
for s.Phase == PhaseBetting {
|
||||
var next []Event
|
||||
s, next, err = ApplyMove(s, randomMove(s, rng))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
noBotCards(t, s, next)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A bot's cards may appear in exactly one kind of event: the showdown that turns
|
||||
// them face up, which is the moment they stop being secret.
|
||||
func noBotCards(t *testing.T, s State, evs []Event) {
|
||||
t.Helper()
|
||||
for _, e := range evs {
|
||||
if len(e.Cards) == 0 || e.Seat < 0 || e.Kind == "show" {
|
||||
continue
|
||||
}
|
||||
if e.Seat != You && s.Seats[e.Seat].Bot {
|
||||
t.Fatalf("a %q event carries seat %d's cards (%v) — that's a bot's hand",
|
||||
e.Kind, e.Seat, e.Cards)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- hand strength ---------------------------------------------------------
|
||||
|
||||
func TestTheEvaluatorKnowsWhichHandIsBetter(t *testing.T) {
|
||||
board := []cards.Card{
|
||||
{Rank: 10, Suit: cards.Hearts}, {Rank: cards.Jack, Suit: cards.Hearts},
|
||||
{Rank: cards.Queen, Suit: cards.Hearts}, {Rank: 2, Suit: cards.Spades},
|
||||
{Rank: 7, Suit: cards.Clubs},
|
||||
}
|
||||
flush, _ := rankOf([2]cards.Card{{Rank: 3, Suit: cards.Hearts}, {Rank: 5, Suit: cards.Hearts}}, board)
|
||||
straight, _ := rankOf([2]cards.Card{{Rank: cards.King, Suit: cards.Spades}, {Rank: cards.Ace, Suit: cards.Clubs}}, board)
|
||||
royal, _ := rankOf([2]cards.Card{{Rank: cards.King, Suit: cards.Hearts}, {Rank: cards.Ace, Suit: cards.Hearts}}, board)
|
||||
pair, _ := rankOf([2]cards.Card{{Rank: 7, Suit: cards.Spades}, {Rank: 4, Suit: cards.Diamonds}}, board)
|
||||
|
||||
if !(royal < flush && flush < straight && straight < pair) {
|
||||
t.Errorf("hands rank royal=%d flush=%d straight=%d pair=%d — lower must be better, in that order",
|
||||
royal, flush, straight, pair)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEquityKnowsAcesAreGood(t *testing.T) {
|
||||
rng := rand.New(rand.NewPCG(1, 1))
|
||||
aces := equityOf([2]cards.Card{{Rank: cards.Ace, Suit: cards.Spades}, {Rank: cards.Ace, Suit: cards.Hearts}}, nil, 1, 2000, rng)
|
||||
rags := equityOf([2]cards.Card{{Rank: 7, Suit: cards.Spades}, {Rank: 2, Suit: cards.Hearts}}, nil, 1, 2000, rng)
|
||||
|
||||
if aces.Strength() < 0.8 {
|
||||
t.Errorf("pocket aces are worth %.2f heads-up, want about 0.85", aces.Strength())
|
||||
}
|
||||
if rags.Strength() > 0.4 {
|
||||
t.Errorf("seven-deuce is worth %.2f heads-up, want about 0.35", rags.Strength())
|
||||
}
|
||||
if aces.Strength() <= rags.Strength() {
|
||||
t.Error("seven-deuce is not better than pocket aces")
|
||||
}
|
||||
}
|
||||
|
||||
// The policy loads, and every node in it is a probability distribution.
|
||||
func TestThePolicyLoads(t *testing.T) {
|
||||
p := loadPolicy()
|
||||
if len(p) < 1000 {
|
||||
t.Fatalf("the CFR policy has %d nodes in it — it did not load, or it was never trained", len(p))
|
||||
}
|
||||
for key, probs := range p {
|
||||
var sum float64
|
||||
for _, v := range probs {
|
||||
if v < 0 {
|
||||
t.Fatalf("%s: a negative probability (%v)", key, probs)
|
||||
}
|
||||
sum += v
|
||||
}
|
||||
if sum < 0.99 || sum > 1.01 {
|
||||
t.Fatalf("%s: the probabilities sum to %v, not 1", key, sum)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestTheBotsAreActuallyTrained is the test this game most needed and did not
|
||||
// have.
|
||||
//
|
||||
// A bot that cannot find itself in the policy does not fail. It shrugs, plays the
|
||||
// pot-odds rule, and looks exactly like a bot that is working — which is how
|
||||
// gogobee shipped a trained poker AI whose policy was *never read once* for the
|
||||
// entire life of the game. The trainer wrote its keys under IP/OOP and the table
|
||||
// looked them up under BTN/SB/BB, and there was nothing anywhere that would have
|
||||
// said so.
|
||||
//
|
||||
// So: deal real hands, let the bots think, and count how often the thinking lands
|
||||
// in the table. Heads-up is the number that has to hold — that is what the policy
|
||||
// was trained on. A six-handed table is a documented approximation of it and
|
||||
// drops off as seats are added, which is why this only asserts on the duel.
|
||||
func TestTheBotsAreActuallyTrained(t *testing.T) {
|
||||
Hits.Store(0)
|
||||
Misses.Store(0)
|
||||
|
||||
rng := rand.New(rand.NewPCG(11, 12))
|
||||
for game := 0; game < 40; game++ {
|
||||
tier := Tiers[1]
|
||||
s, _, err := New(tier, 1, tier.MaxBuy, tier.RakePct, uint64(game), 5)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for hand := 0; hand < 6 && s.Phase != PhaseDone; hand++ {
|
||||
s, _, err = ApplyMove(s, Move{Kind: Deal})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for s.Phase == PhaseBetting {
|
||||
s, _, err = ApplyMove(s, randomMove(s, rng))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hits, misses := Hits.Load(), Misses.Load()
|
||||
if hits+misses < 100 {
|
||||
t.Fatalf("the bots only made %d decisions — this test isn't measuring anything", hits+misses)
|
||||
}
|
||||
rate := float64(hits) / float64(hits+misses)
|
||||
if rate < 0.6 {
|
||||
t.Fatalf("heads-up, the bots found themselves in the trained policy %.0f%% of the time "+
|
||||
"(%d of %d decisions). They are playing the pot-odds fallback, which means the key the "+
|
||||
"trainer writes and the key the table reads have drifted apart. See infoSet.",
|
||||
rate*100, hits, hits+misses)
|
||||
}
|
||||
t.Logf("heads-up policy hit rate: %.0f%% (%d of %d decisions)", rate*100, hits, hits+misses)
|
||||
}
|
||||
|
||||
// ---- helpers ---------------------------------------------------------------
|
||||
|
||||
func table(t *testing.T, tier Tier, bots int, buyIn int64) State {
|
||||
t.Helper()
|
||||
s, _, err := New(tier, bots, buyIn, tier.RakePct, 1, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("new table: %v", err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// playOut folds every decision until the hand is over.
|
||||
func playOut(t *testing.T, s State) State {
|
||||
t.Helper()
|
||||
for i := 0; s.Phase == PhaseBetting; i++ {
|
||||
if i > 100 {
|
||||
t.Fatal("the hand will not end")
|
||||
}
|
||||
move := Move{Kind: Fold}
|
||||
if s.Owed(You) == 0 {
|
||||
move = Move{Kind: Check}
|
||||
}
|
||||
var err error
|
||||
s, _, err = ApplyMove(s, move)
|
||||
if err != nil {
|
||||
t.Fatalf("playing out: %v", err)
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func has(evs []Event, kind string) bool {
|
||||
for _, e := range evs {
|
||||
if e.Kind == kind {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
BIN
internal/games/holdem/policy.gob
Normal file
BIN
internal/games/holdem/policy.gob
Normal file
Binary file not shown.
428
internal/games/holdem/train.go
Normal file
428
internal/games/holdem/train.go
Normal file
@@ -0,0 +1,428 @@
|
||||
package holdem
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
"sync"
|
||||
|
||||
"pete/internal/games/cards"
|
||||
)
|
||||
|
||||
// The trainer.
|
||||
//
|
||||
// This is counterfactual regret minimisation, and what it produces is policy.gob
|
||||
// — the table the bots read at the table. It is not on any request path; it runs
|
||||
// from cmd/holdem-train, for half an hour, and then it is a file.
|
||||
//
|
||||
// The one thing worth understanding about it: **it plays the real game.** Every
|
||||
// move it explores goes through Step, which is the same reducer the felt calls,
|
||||
// so the blinds, the min-raise, the street completion and the money are the ones
|
||||
// a player will actually meet. Its info-set key comes out of State.spot, which is
|
||||
// the same function the bots look themselves up with.
|
||||
//
|
||||
// That is not tidiness, it is the whole lesson of the policy this replaces. That
|
||||
// one was trained against a hand-written model of poker sitting beside the real
|
||||
// engine — a model where a call always ended the street, the big blind had no
|
||||
// option, and the payoff was half the pot no matter who had put what in. Then it
|
||||
// was looked up under a key the trainer never wrote. The result was a 3.4MB file
|
||||
// that had never once been read, and nobody could tell, because a policy miss is
|
||||
// not an error. It just quietly isn't there.
|
||||
//
|
||||
// So: one engine, one key function, and a test that fails if the bots stop
|
||||
// finding themselves in the table.
|
||||
|
||||
// How much of the game tree to explore. Two raises a street keeps the tree small
|
||||
// enough to converge; a third barely changes how anybody plays and multiplies the
|
||||
// nodes.
|
||||
const (
|
||||
maxRaisesPerStreet = 2
|
||||
maxDepth = 40
|
||||
trainMCIters = 60 // noisy, but it is only picking a bucket
|
||||
)
|
||||
|
||||
// regrets is what CFR accumulates: how much better each action would have been.
|
||||
type regrets map[string]*[numActions]float64
|
||||
|
||||
// Trained is the file the bots read.
|
||||
type Trained struct {
|
||||
Strategy map[string][numActions]float64
|
||||
Meta TrainMeta
|
||||
}
|
||||
|
||||
// TrainMeta is what the policy can say about itself. Worth having: a policy is
|
||||
// otherwise an opaque three megabytes and there is no way to tell a good one from
|
||||
// a stale one by looking.
|
||||
type TrainMeta struct {
|
||||
Iterations int
|
||||
Stakes string
|
||||
Depths string
|
||||
Nodes int
|
||||
}
|
||||
|
||||
// ---- preflop, measured once ------------------------------------------------
|
||||
|
||||
var (
|
||||
preflopOnce sync.Once
|
||||
preflopTable [13][13]Equity // [hi][lo] offsuit, [lo][hi] suited, diagonal pairs
|
||||
)
|
||||
|
||||
// preflopEquity is the equity of a starting hand heads-up. There are only 169
|
||||
// hands that differ from each other, so they are measured properly, once, and
|
||||
// then it is a lookup — which matters twice: it takes the noise out of a bucket
|
||||
// boundary, and the trainer visits preflop on every single iteration.
|
||||
func preflopEquity(hole [2]cards.Card) Equity {
|
||||
preflopOnce.Do(func() {
|
||||
rng := cards.NewRNG(20260714, 1)
|
||||
for a := cards.Ace; a <= cards.King; a++ {
|
||||
for b := a; b <= cards.King; b++ {
|
||||
lo, hi := rankIdx(a), rankIdx(b)
|
||||
|
||||
// Suited, and the pairs (which can only be offsuit) on the diagonal.
|
||||
s1 := cards.Card{Rank: a, Suit: cards.Spades}
|
||||
s2 := cards.Card{Rank: b, Suit: cards.Spades}
|
||||
if a == b {
|
||||
s2.Suit = cards.Hearts
|
||||
}
|
||||
preflopTable[lo][hi] = equityOf([2]cards.Card{s1, s2}, nil, 1, 10000, rng)
|
||||
|
||||
if a != b {
|
||||
o2 := cards.Card{Rank: b, Suit: cards.Hearts}
|
||||
preflopTable[hi][lo] = equityOf([2]cards.Card{s1, o2}, nil, 1, 10000, rng)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
lo, hi := rankIdx(hole[0].Rank), rankIdx(hole[1].Rank)
|
||||
if lo > hi {
|
||||
lo, hi = hi, lo
|
||||
}
|
||||
if hole[0].Suit == hole[1].Suit {
|
||||
return preflopTable[lo][hi] // suited, and the pairs sit here too
|
||||
}
|
||||
if lo == hi {
|
||||
return preflopTable[lo][hi]
|
||||
}
|
||||
return preflopTable[hi][lo] // offsuit
|
||||
}
|
||||
|
||||
// rankIdx maps a rank to 0–12, with the ace high — which is what it is, before
|
||||
// the flop.
|
||||
func rankIdx(r cards.Rank) int {
|
||||
if r == cards.Ace {
|
||||
return 12
|
||||
}
|
||||
return int(r) - 2
|
||||
}
|
||||
|
||||
// ---- the traversal ---------------------------------------------------------
|
||||
|
||||
// Train runs external-sampling MCCFR for n hands and returns the average
|
||||
// strategy. Each worker keeps its own tables and they are summed at the end,
|
||||
// which is what makes this embarrassingly parallel and is the only reason it
|
||||
// finishes in half an hour.
|
||||
func Train(n, workers int, t Tier, minBB, maxBB int64, seed uint64, progress func(done int)) *Trained {
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
|
||||
type table struct {
|
||||
reg regrets
|
||||
avg regrets
|
||||
}
|
||||
out := make([]table, workers)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var done sync.Mutex
|
||||
completed := 0
|
||||
|
||||
for w := 0; w < workers; w++ {
|
||||
wg.Add(1)
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
tr := &trainer{
|
||||
reg: regrets{},
|
||||
avg: regrets{},
|
||||
tier: t,
|
||||
minBB: minBB,
|
||||
maxBB: maxBB,
|
||||
rng: cards.NewRNG(seed, uint64(w)+1),
|
||||
}
|
||||
|
||||
share := n / workers
|
||||
if w < n%workers {
|
||||
share++
|
||||
}
|
||||
for i := 0; i < share; i++ {
|
||||
tr.iterate(uint64(w)<<40 | uint64(i))
|
||||
if progress != nil && i%2000 == 0 {
|
||||
done.Lock()
|
||||
completed += 2000
|
||||
c := completed
|
||||
done.Unlock()
|
||||
progress(c)
|
||||
}
|
||||
}
|
||||
out[w] = table{tr.reg, tr.avg}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Sum the workers' average-strategy tables, then normalise each node into the
|
||||
// probabilities a bot will actually play.
|
||||
total := regrets{}
|
||||
for _, tab := range out {
|
||||
for key, v := range tab.avg {
|
||||
acc, ok := total[key]
|
||||
if !ok {
|
||||
acc = &[numActions]float64{}
|
||||
total[key] = acc
|
||||
}
|
||||
for i, x := range v {
|
||||
acc[i] += x
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
strategy := make(map[string][numActions]float64, len(total))
|
||||
for key, v := range total {
|
||||
var sum float64
|
||||
for _, x := range v {
|
||||
sum += x
|
||||
}
|
||||
var probs [numActions]float64
|
||||
if sum > 0 {
|
||||
for i, x := range v {
|
||||
probs[i] = x / sum
|
||||
}
|
||||
} else {
|
||||
for i := range probs {
|
||||
probs[i] = 1.0 / numActions
|
||||
}
|
||||
}
|
||||
strategy[key] = probs
|
||||
}
|
||||
|
||||
return &Trained{
|
||||
Strategy: strategy,
|
||||
Meta: TrainMeta{
|
||||
Iterations: n,
|
||||
Stakes: fmt.Sprintf("%d/%d", t.SB, t.BB),
|
||||
Depths: fmt.Sprintf("%d–%d BB", minBB, maxBB),
|
||||
Nodes: len(strategy),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type trainer struct {
|
||||
reg regrets
|
||||
avg regrets
|
||||
tier Tier
|
||||
minBB int64
|
||||
maxBB int64
|
||||
rng *rand.Rand
|
||||
|
||||
// A hand's equity on a given street depends on the cards and nothing else —
|
||||
// not on how the betting went to get there. The deck is fixed for the whole
|
||||
// iteration, so the flop is the same flop down every branch, and this is
|
||||
// measured once per seat per street instead of once per node.
|
||||
eq [2][4]Equity
|
||||
have [2][4]bool
|
||||
}
|
||||
|
||||
// iterate deals one hand and walks it once for each player.
|
||||
//
|
||||
// The stack depth is drawn fresh every hand, across the whole range the table
|
||||
// allows. This is the fix for the policy that came before: it was trained at ten
|
||||
// big blinds and nothing else, so four out of five spots in a real cash game fell
|
||||
// outside anything it had ever seen. A hand of poker is a different game at 20
|
||||
// big blinds than at 100 — that is most of what makes it a game — and the bots
|
||||
// have to have played both.
|
||||
func (tr *trainer) iterate(id uint64) {
|
||||
depth := tr.minBB
|
||||
if tr.maxBB > tr.minBB {
|
||||
depth += tr.rng.Int64N(tr.maxBB - tr.minBB + 1)
|
||||
}
|
||||
stack := depth * tr.tier.BB
|
||||
|
||||
// No rake while learning. The bots should learn to play poker, not to beat a
|
||||
// fee, and the fee is the house's business.
|
||||
t := tr.tier
|
||||
t.RakePct = 0
|
||||
|
||||
s, err := Open(t, stack, stack, id, tr.rng.Uint64())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
start := [2]int64{s.Seats[0].Stack + s.Seats[0].Bet, s.Seats[1].Stack + s.Seats[1].Bet}
|
||||
|
||||
tr.have = [2][4]bool{} // one deal, one set of boards, one set of equities
|
||||
|
||||
for me := 0; me < 2; me++ {
|
||||
tr.walk(s.Clone(), me, start, 0)
|
||||
}
|
||||
}
|
||||
|
||||
// equity is the cached measurement for this seat on this street.
|
||||
func (tr *trainer) equity(s State, seat int) Equity {
|
||||
st := s.Street
|
||||
if st > River {
|
||||
st = River
|
||||
}
|
||||
if !tr.have[seat][st] {
|
||||
tr.eq[seat][st] = s.equityFor(seat, trainMCIters, tr.rng)
|
||||
tr.have[seat][st] = true
|
||||
}
|
||||
return tr.eq[seat][st]
|
||||
}
|
||||
|
||||
// walk returns what the hand is worth to `me`, in chips, from here.
|
||||
func (tr *trainer) walk(s State, me int, start [2]int64, depth int) float64 {
|
||||
if s.Phase != PhaseBetting || depth > maxDepth {
|
||||
// The hand is over (or we have gone far enough to call it over). What it was
|
||||
// worth is simply what the player has now against what they sat down with —
|
||||
// the real number, out of the real engine, side pots and all.
|
||||
return float64(s.Seats[me].Stack - start[me])
|
||||
}
|
||||
|
||||
seat := s.ToAct
|
||||
key := s.spotKey(seat, tr.equity(s, seat))
|
||||
|
||||
mask := s.mask(seat)
|
||||
if raises(s.History) >= maxRaisesPerStreet {
|
||||
mask[actRaiseHalf], mask[actRaisePot] = false, false
|
||||
}
|
||||
|
||||
reg := tr.reg[key]
|
||||
if reg == nil {
|
||||
reg = &[numActions]float64{}
|
||||
tr.reg[key] = reg
|
||||
}
|
||||
strat := match(*reg, mask)
|
||||
|
||||
// The opponent's turn: sample one line and follow it. That is the "external
|
||||
// sampling" part, and it is what keeps a hand from costing 5^12 traversals.
|
||||
if seat != me {
|
||||
avg := tr.avg[key]
|
||||
if avg == nil {
|
||||
avg = &[numActions]float64{}
|
||||
tr.avg[key] = avg
|
||||
}
|
||||
for i, p := range strat {
|
||||
avg[i] += p
|
||||
}
|
||||
return tr.walk(tr.play(s, seat, sample(strat, tr.rng)), me, start, depth+1)
|
||||
}
|
||||
|
||||
// Our turn: try everything, and regret what we didn't do.
|
||||
var values [numActions]float64
|
||||
var node float64
|
||||
for a := 0; a < numActions; a++ {
|
||||
if !mask[a] {
|
||||
continue
|
||||
}
|
||||
values[a] = tr.walk(tr.play(s, seat, a), me, start, depth+1)
|
||||
node += strat[a] * values[a]
|
||||
}
|
||||
for a := 0; a < numActions; a++ {
|
||||
if mask[a] {
|
||||
reg[a] += values[a] - node
|
||||
}
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
// play applies one abstract action through the real reducer.
|
||||
func (tr *trainer) play(s State, seat, action int) State {
|
||||
next, _, err := Step(s.Clone(), s.moveFor(action, seat))
|
||||
if err != nil {
|
||||
// The mask and the rules disagreed, which is a bug in one of them. Fold and
|
||||
// carry on rather than poison the whole run.
|
||||
next, _, err = Step(s.Clone(), Move{Kind: Fold})
|
||||
if err != nil {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
// match is regret matching: play each action in proportion to how much you wish
|
||||
// you had played it. An action nobody regrets not taking gets played uniformly.
|
||||
func match(reg [numActions]float64, mask [numActions]bool) [numActions]float64 {
|
||||
var strat [numActions]float64
|
||||
var sum float64
|
||||
for i, r := range reg {
|
||||
if mask[i] && r > 0 {
|
||||
sum += r
|
||||
}
|
||||
}
|
||||
if sum > 0 {
|
||||
for i, r := range reg {
|
||||
if mask[i] && r > 0 {
|
||||
strat[i] = r / sum
|
||||
}
|
||||
}
|
||||
return strat
|
||||
}
|
||||
|
||||
n := 0
|
||||
for _, ok := range mask {
|
||||
if ok {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n == 0 {
|
||||
strat[actCallCheck] = 1
|
||||
return strat
|
||||
}
|
||||
for i, ok := range mask {
|
||||
if ok {
|
||||
strat[i] = 1 / float64(n)
|
||||
}
|
||||
}
|
||||
return strat
|
||||
}
|
||||
|
||||
func sample(strat [numActions]float64, rng *rand.Rand) int {
|
||||
r := rng.Float64()
|
||||
var sum float64
|
||||
for i, p := range strat {
|
||||
sum += p
|
||||
if r < sum {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return actCallCheck
|
||||
}
|
||||
|
||||
// raises counts the bets and raises on this street, which is what the tree is
|
||||
// capped on.
|
||||
func raises(history string) int {
|
||||
n := 0
|
||||
for _, c := range history {
|
||||
if c == 'r' || c == 'R' {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ---- the file --------------------------------------------------------------
|
||||
|
||||
// Save writes a trained policy.
|
||||
func Save(w io.Writer, t *Trained) error { return gob.NewEncoder(w).Encode(t) }
|
||||
|
||||
// Load reads one. It is only used by the tests — the bots read the embedded copy.
|
||||
func Load(r io.Reader) (*Trained, error) {
|
||||
var t Trained
|
||||
if err := gob.NewDecoder(r).Decode(&t); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
// loadTrained decodes the embedded policy in the new format.
|
||||
func loadTrained(b []byte) (*Trained, error) { return Load(bytes.NewReader(b)) }
|
||||
Reference in New Issue
Block a user