Dragon-s-Lair-X68k/tools/bench/prep_dlx.py

#!/usr/bin/env python3
"""Lay a DLX1 container out the way src/player/decode.s expects to find it.

    python3 tools/bench/prep_dlx.py <in.dlx> [--out tmp/decode]

Writes <out>_data.bin (one blob Lua pushes into emulated RAM) and <out>_meta.lua
(sizes, per-frame record offsets, and the timing anchors).

Two things happen here that the shipping player would do at load time on the
68000 itself, and are therefore NOT part of the per-frame cost being measured:

  * codebook expansion to word-per-pixel form.  CB1 -> 32 B/entry, CB4 -> 8 B,
    so the inner loop scales an index with lsl.w #5 / #3 and movems the result
    straight into GVRAM with no unpacking.  8 KB + 2 KB of the 2 MB.
  * palette packing to GGGGGRRRRRBBBBBI with the shared LSB I chosen PER ENTRY
    by minimum squared error (FINDINGS 23.3, worth 1.96 dB).

The encoder still emits 24-bit palettes and does not reserve a black entry
(known gap, docs/STATUS.md), so the letterbox here is filled with whatever
palette entry is closest to black rather than a true reserved black.  That is
cosmetic and outside the active 256x192 area the decoder is judged on.

A synthetic all-SKIP frame is appended to the stream.  No real frame is all
SKIP, but it prices the mode-header walk on its own -- the per-block cost the
"76.6% x non-SKIP fraction" model in FINDINGS 24.5 leaves out entirely.
"""
import sys, os, argparse
sys.path.insert(0, "tools/encoder")
import numpy as np
from dlx import DLX
sys.path.insert(0, "tools/bench")
import dlxload as DL
import spans as SP

# The harness loads the WHOLE container into emulated RAM at STREAM=0x30000 and
# the target is a stock 2 MB machine, so there is a hard ceiling on how much of
# a stream can be verified in one pass. The shipping player streams from disk
# into a ring buffer and has no such limit; this is a property of the test rig.
# A `scsi` window overruns it -- 2.84 MB of stream ends at 0x2E591C, 940 KB past
# the 0x200000 top of RAM -- so the frame list is truncated to what fits and the
# truncation is announced. Verifying a prefix is still a real test: SKIP blocks
# make every frame a claim about the one before it.
STREAM_BASE = 0x30000
RAM_TOP     = 0x200000
MARGIN      = 0x8000          # stack, flags, codebooks live below STREAM_BASE

ap = argparse.ArgumentParser()
ap.add_argument("container")
ap.add_argument("--out", default="tmp/decode")
ap.add_argument("--ram", type=lambda v: int(v, 0), default=RAM_TOP,
                help="top of emulated RAM (default 0x200000, a stock 2 MB machine)")
ap.add_argument("--all-frames", action="store_true",
                help="do NOT truncate to what fits in RAM (the loader will write "
                     "past the top of memory and the decoder will read garbage)")
a = ap.parse_args()

d = DLX(a.container)
if d.idx_bytes != 1:
    sys.exit("2-byte codebook indices: decode.s assumes 1 (k<=256)")
# decode.s reads a u16 span count out of every frame record (FINDINGS 41), so a
# DLX2 container is not merely span-less to it -- the first two bytes of the
# block payload would be read as a count and the frame would decode as garbage.
# Fail here rather than there.
if not d.has_spans:
    sys.exit(f"{a.container} is DLX{d.version}: src/player/decode.s expects the "
             f"DLX3 span section. Re-encode (tools/encoder/encode.py emits DLX3 "
             f"by default) or pass --spans off and use an older decoder.")

# --- codebooks and palette.  Both transforms live in tools/bench/dlxload.py so
#     that prep_stream.py's ring-buffer rig shares one copy of them rather than
#     keeping a second that could drift silently (FINDINGS 49).
cb1, cb4 = DL.expand_codebooks(d)
palb, dark, rendered = DL.pack_palette(d)

def build_synth(d):
    """The synthetic timing frames, as record bodies.

    No real frame is all one mode, but the mix is exactly what the "76.6% x
    non-SKIP fraction" model of FINDINGS 24.5 assumes away: it prices every
    non-SKIP block as one V1-style burst.  These price the modes separately,
    which is the only way to see which one is expensive.

    Every record carries the DLX3 span section, empty or not -- decode.s reads a
    u16 count out of all of them, and a synthetic frame that omitted it would
    desync the bitstream exactly where the harness is least likely to look.

    The last two are the mode the block loop cannot express: a frame that is ALL
    SPAN, its mode header entirely SKIP.  Two run lengths, because a span costs
    per-span plus per-pixel and one length cannot separate them --
      all-SPAN-64  full-row runs, the floor of the mode (154 clocks/block,
                   FINDINGS 30.4)
      all-SPAN-4   4-block runs, the break-even against V1 (FINDINGS 40.1)
    They price v7 INSIDE decode.s against the constants tools/bench/span.sh
    fitted in blit.s.  Agreement cross-checks both; disagreement means the
    decoder's span pass is not the sequence that was measured.
    """
    out = {}
    empty = SP.serialise([])
    for name, mo, per in (("all-SKIP", 0, 0), ("all-V1", 1, 1),
                          ("all-V4", 2, 4), ("all-RAW", 3, 16)):
        out[name] = (bytes([mo * 0x55] * d.mode_bytes) + empty
                     + bytes(d.nb * per))
    # MIXED-SKIP frames.  Every other synthetic frame here is a pure population,
    # which is exactly why none of them prices the commonest block in a real
    # container: a SKIP that shares its header byte with a coded block, and so
    # cannot take the all-SKIP fast path.  vq_hybrid's C_SKIP_MIXED has never
    # been measured -- it was derived -- and a spanned container is made mostly
    # of them, because a spanned block reads SKIP.  FINDINGS 41.5.
    #
    # Two mixes per coded mode, because one equation cannot separate the SKIP
    # cost from the cost of the block it shares a group with.
    #
    # THE HEADER BYTES ROTATE, and that is not decoration.  decode.s reaches a
    # block's 2 mode bits with `lsr.b #6/#4/#2` and no shift at all for the last
    # one, so a block costs 52/48/44/34 clocks of dispatch depending on WHERE in
    # its header byte it sits.  A fixed byte like 0x01 puts every SKIP at the
    # three expensive positions and every V1 at the free one, and solving two
    # such equations returns a number that describes no real frame.  Cycling the
    # byte through the four rotations puts each mode at each position equally,
    # which is what a real mode map does.
    for nm, bys, per in (("mix-3SKIP-V1",  (0x01, 0x04, 0x10, 0x40), 1),
                         ("mix-1SKIP-3V1", (0x54, 0x51, 0x45, 0x15), 1),
                         ("mix-3SKIP-RAW", (0x03, 0x0C, 0x30, 0xC0), 16),
                         ("mix-1SKIP-3RAW", (0xFC, 0xF3, 0xCF, 0x3F), 16)):
        hdr = bytes(bys[i % 4] for i in range(d.mode_bytes))
        ncoded = sum(bin(b).count("1") and
                     sum(1 for k in range(4) if (b >> (2 * k)) & 3) for b in hdr[:1])
        ncoded = sum(sum(1 for k in range(4) if (b >> (2 * k)) & 3) for b in hdr)
        out[nm] = hdr + empty + bytes(ncoded * per)
    pat = np.tile(np.arange(d.W, dtype=np.uint8), (d.H, 1))
    for name, blocks in (("all-SPAN-64", d.W // 4), ("all-SPAN-4", 4)):
        sp = [(y, x, pat[y, x:x + blocks * 4])
              for y in range(d.H) for x in range(0, d.W, blocks * 4)]
        out[name] = bytes(d.mode_bytes) + SP.serialise(sp)
    return out


# --- frame stream: [u32 len][modes][payload] per frame, each record start
#     rounded up to a 4-byte boundary.
#
#     This padding is not cosmetic.  Payload lengths are arbitrary, so laid end
#     to end the records land on odd addresses, and `move.l (a0)+` at an odd
#     address is an ADDRESS ERROR on a 68000 -- it vectors into the IPL rather
#     than reading slowly.  The container as written by encode.py is unaligned,
#     so this loader realigns it; the encoder should carry the padding itself
#     (FINDINGS 28.3).  It costs at most 3 bytes per frame -- 36 B/s at 12fps,
#     against a 110 KB/s budget.
budget = a.ram - STREAM_BASE - MARGIN
stream, rec_off, pad = bytearray(), [], 0
dropped = 0

# The synthetic timing frames are built FIRST, so their size comes out of the
# RAM budget rather than being appended past it.  It used to be appended: the
# stream ran 26 KB beyond the top of a 2 MB machine, which was survivable only
# because the modes it overran are data-independent -- their cost is in the
# mode header, and reading junk payload costs the same as reading pixels.  A
# span is not: its two jump DISPLACEMENTS come out of the stream, so an
# out-of-RAM span record jumps into open bus.  FINDINGS 41.4.
SYNTH = build_synth(d)
budget -= sum(4 + len(b) + 3 for b in SYNTH.values())

for (o, n) in d.frames:
    while len(stream) % 4:
        stream += b"\0"; pad += 1
    if not a.all_frames and len(stream) + 4 + n > budget:
        dropped = d.nframes - len(rec_off)
        break
    rec_off.append(len(stream))
    stream += n.to_bytes(4, "big") + d.raw[o:o + n]
NFRAMES = len(rec_off)
if dropped:
    print(f"  TRUNCATED: {NFRAMES}/{d.nframes} frames fit in RAM "
          f"(stream budget {budget:,} B at 0x{STREAM_BASE:X} under a "
          f"{a.ram/1024/1024:.0f} MB machine); {dropped} frames dropped.\n"
          f"  This is the TEST RIG's limit, not the player's -- the player "
          f"streams into a ring buffer.")

# Append the synthetic frames the budget above already reserved.
synth = {}
for name, body in SYNTH.items():
    while len(stream) % 4:
        stream += b"\0"; pad += 1
    synth[name] = len(stream)
    stream += len(body).to_bytes(4, "big") + body
assert STREAM_BASE + len(stream) <= a.ram, (
    f"stream ends at 0x{STREAM_BASE+len(stream):X}, past the 0x{a.ram:X} top "
    f"of RAM -- the budget arithmetic above is wrong")

# --- timing anchors: the distribution, not its mean (FINDINGS 25.6's lesson)
#
# A spanned block reads SKIP here, so this fraction is the BLOCK-LOOP workload
# and no longer the frame's whole cost: the span section is the rest of it. The
# anchors still pick out the extremes of the block loop, which is what they are
# for, but a frame's total decode time now has two terms.
ns = np.array([100 * (d.modes(i) != 0).mean() for i in range(NFRAMES)])
nsp = np.array([len(d.spans(i)[0]) for i in range(NFRAMES)])
spx = np.array([sum(len(p) for _, _, p in d.spans(i)[0]) for i in range(NFRAMES)])
order = np.argsort(ns)
pick = {
    "min non-SKIP  %.1f%%" % ns[order[0]]: int(order[0]),
    "median        %.1f%%" % np.median(ns): int(order[len(order)//2]),
    "p90           %.1f%%" % ns[order[int(.9*len(order))]]: int(order[int(.9*len(order))]),
    "max non-SKIP  %.1f%%" % ns[order[-1]]: int(order[-1]),
}
anchors = [(n, rec_off[i], float(ns[i])) for n, i in pick.items()]
for name in ("all-SKIP", "all-V1", "all-V4", "all-RAW",
             "all-SPAN-64", "all-SPAN-4", "mix-3SKIP-V1", "mix-1SKIP-3V1",
             "mix-3SKIP-RAW", "mix-1SKIP-3RAW"):
    anchors.append((f"synthetic {name}", synth[name],
                    0.0 if name.startswith(("all-SKIP", "all-SPAN")) else 100.0))

blob = cb1.tobytes() + cb4.tobytes() + palb.tobytes() + bytes(stream)
open(a.out + "_data.bin", "wb").write(blob)

with open(a.out + "_meta.lua", "w") as fh:
    fh.write("-- generated by tools/bench/prep_dlx.py -- do not edit\nreturn {\n")
    fh.write(f"  W={d.W}, H={d.H}, fps={d.fps}, nframes={NFRAMES},\n")
    fh.write(f"  k1={d.k1}, k4={d.k4}, dark={dark},\n")
    fh.write(f"  cb1_len={cb1.nbytes}, cb4_len={cb4.nbytes}, pal_len={palb.nbytes},\n")
    fh.write(f"  stream_len={len(stream)},\n")
    fh.write("  anchors={\n")
    for n, o, frac in anchors:
        fh.write(f'    {{name="{n}", off={o}, frac={frac:.1f}}},\n')
    fh.write("  },\n}\n")

print(f"{a.container}: {d.nframes} frames, {d.W}x{d.H}, k1={d.k1} k4={d.k4}")
print(f"  cb1 {cb1.nbytes} B + cb4 {cb4.nbytes} B expanded, palette {palb.nbytes} B, "
      f"stream {len(stream)} B  ->  {a.out}_data.bin ({len(blob)} B)")
print(f"  non-SKIP blocks/frame: median {np.median(ns):.1f}%  "
      f"p90 {np.percentile(ns,90):.1f}%  max {ns.max():.1f}%")
print(f"  v7 spans/frame: median {np.median(nsp):.0f}  max {nsp.max()}  "
      f"({int((nsp>0).sum())}/{NFRAMES} frames); pixels painted by one: "
      f"median {100*np.median(spx)/(d.W*d.H):.1f}%  "
      f"max {100*spx.max()/(d.W*d.H):.1f}% of the picture")
print(f"  darkest palette entry: index {dark} -> {tuple(rendered[dark])}")
# A DLX2 container already carries this padding (FINDINGS 28.3 closed, session
# 9), so the realignment above re-derives bytes that were already there and the
# loader is doing no work. On a DLX1 container it is load-bearing: 94 of 120
# record starts land on odd addresses, and each one is an address error.
src_bad = sum(1 for (o, _) in d.frames[:NFRAMES] if (o - 4) % 4)
print(f"  4-byte record alignment cost {pad} B over {NFRAMES} frames "
      f"({pad / NFRAMES:.2f} B/frame = {pad / NFRAMES * d.fps:.0f} B/s)")
print(f"  source container is DLX{d.version}: {src_bad}/{NFRAMES} record starts "
      f"unaligned" + ("  -- this loader is what makes it decodable"
                      if src_bad else "  -- the container carries its own padding"))