Dragon-s-Lair-X68k/tools/bench/prep_stream.py

#!/usr/bin/env python3
"""Lay a DLX3 container out as a DISK for the ring-buffer rig (FINDINGS 49).

    python3 tools/bench/prep_stream.py <in.dlx> [--out tmp/stream]

prep_dlx.py's output is one blob that tools/bench/decode.lua pushes into
emulated RAM in its entirety.  That is what makes its rig RAM-bound -- a `scsi`
window is 5,261,814 B of stream and needs a 6 MB machine to hold it (FINDINGS
45) -- and, much more importantly, it is nothing like the shipping player, which
never holds a window at once.

This writes three files instead:

  <out>_cb.bin    codebooks + palette.  ~10 KB, loaded into RAM once, exactly as
                  before: these are LOAD-TIME costs and not per-frame ones.
  <out>_disk.bin  the frame records, `[u32 len][body]` each padded up to 4, laid
                  end to end.  tools/bench/stream.lua reads this from the HOST
                  filesystem and feeds it into a bounded ring, so the emulated
                  machine's RAM stops bounding how much of a window can be
                  tested.  A stock 2 MB machine can now run all 120 frames.
  <out>_idx.bin   the DLX4 RECORD INDEX, nframes u16 big-endian, straight out
                  of the container's scene header.  This is what the 68000
                  producer reads (src/player/ring.i); it is not derived here,
                  because deriving it is precisely what a player cannot do.
  <out>_meta.lua  geometry, and the record index.

THE RECORD INDEX IS NOT A CONVENIENCE.  src/player/stream.s takes each frame's
base address from a descriptor the producer wrote, rather than deriving it from
where the last frame ended, because under the `aligned` wrap policy the next
record may be at the ring's base instead of just after its predecessor.  The
producer therefore has to know record boundaries before it places them -- which
is what an index is.  A branching laserdisc game needs one anyway to seek to a
branch point, so the policy that costs no clocks (tools/analysis/19_ring_stream.py)
reuses a structure the player cannot avoid.

The 4-byte record padding is the same one decode.s needs and DLX3 already
carries: `move.l (a0)+,d0` on an odd address is an ADDRESS ERROR on a 68000, not
a slow read.  FINDINGS 28.3.

NO SYNTHETIC TIMING FRAMES.  prep_dlx.py appends ten of them to price the block
modes separately; this rig measures delivery, not decode, and its per-frame cost
anchors are prep_dlx.py's job.  Mixing them in would put frames on the wire that
no encoder emits and no rate controller sized.
"""
import sys, os, argparse
sys.path.insert(0, "tools/encoder")
sys.path.insert(0, "tools/bench")
import numpy as np
from dlx import DLX
import dlxload as DL

ap = argparse.ArgumentParser()
ap.add_argument("container")
ap.add_argument("--out", default="tmp/stream")
a = ap.parse_args()

d = DLX(a.container)
if d.idx_bytes != 1:
    sys.exit("2-byte codebook indices: src/player/ assumes 1 (k<=256)")
if not d.has_spans:
    sys.exit(f"{a.container} is DLX{d.version}: src/player/stream.s expects the "
             f"DLX3 span section (see prep_dlx.py for why a DLX2 container "
             f"decodes as garbage rather than merely losing its spans).")

cb1, cb4 = DL.expand_codebooks(d)
palb, dark, rendered = DL.pack_palette(d)
open(a.out + "_cb.bin", "wb").write(cb1.tobytes() + cb4.tobytes() + palb.tobytes())

disk, index = bytearray(), []
for (o, n) in d.frames:
    start = len(disk)
    disk += n.to_bytes(4, "big") + d.raw[o:o + n]
    # The container's own alignment rule, not this script's copy of it: DLX5
    # pads to 512 so a DMA channel can read whole sectors into the ring, DLX4
    # to 4 so `move.l (a0)+` does not take an address error (28.3).
    while len(disk) % d.rec_align:
        disk += b"\0"
    index.append((start, len(disk) - start))
open(a.out + "_disk.bin", "wb").write(bytes(disk))

# THE CONTAINER'S OWN INDEX, and it is checked against this layout rather than
# regenerated from it.  src/player/ring.i walks the disk with a running sum of
# these lengths and never reads a record's length word before fetching it, so a
# container index that disagreed with the disk image by one byte would place
# every later record at the wrong address -- and the block loop reads without a
# bounds check (49.2), so the symptom would be wrong pixels, not a fault.
if d.has_index:
    want = [ln // 4 for _, ln in index]
    if d.index != want:
        bad = next(i for i in range(len(want)) if d.index[i] != want[i])
        sys.exit(f"{a.container}: the DLX4 index disagrees with this disk "
                 f"layout at record {bad}: {d.index[bad]} vs {want[bad]} "
                 f"longwords")
    open(a.out + "_idx.bin", "wb").write(
        b"".join(q.to_bytes(2, "big") for q in d.index))
else:
    if os.path.exists(a.out + "_idx.bin"):
        os.remove(a.out + "_idx.bin")   # a stale index is worse than none

rec = np.array([n for _, n in index])
with open(a.out + "_meta.lua", "w") as fh:
    fh.write("-- generated by tools/bench/prep_stream.py -- do not edit\nreturn {\n")
    fh.write(f"  W={d.W}, H={d.H}, fps={d.fps}, nframes={d.nframes}, dark={dark},\n")
    fh.write(f"  cb1_len={cb1.nbytes}, cb4_len={cb4.nbytes}, pal_len={palb.nbytes},\n")
    fh.write(f"  disk_len={len(disk)}, maxrec={int(rec.max())},\n")
    fh.write(f"  dlx_version={d.version}, "
             f"has_index={'true' if d.has_index else 'false'},\n")
    fh.write(f"  padrec={{{','.join(str(ln) for _, ln in index)}}},\n")
    fh.write("  index={\n")
    for off, ln in index:
        fh.write(f"    {{off={off}, len={ln}}},\n")
    fh.write("  },\n}\n")

print(f"{a.container}: {d.nframes} frames, {d.W}x{d.H}")
print(f"  codebooks+palette {cb1.nbytes + cb4.nbytes + palb.nbytes:,} B -> "
      f"{a.out}_cb.bin")
print(f"  disk image {len(disk):,} B -> {a.out}_disk.bin  "
      f"(records: min {rec.min():,} median {int(np.median(rec)):,} "
      f"max {rec.max():,})")
print(f"  wire rate {rec.mean()*d.fps/1024:.1f} KB/s video at {d.fps} fps")
if d.has_index:
    print(f"  DLX4 record index: {2*d.nframes:,} B of scene header, and it "
          f"agrees with the disk image on all {d.nframes} records")
print(f"  A ring must hold one whole record contiguously: >= {rec.max():,} B "
      f"({rec.max()/1024:.1f} KB) before any policy or prefill.")