drwily/Dragon-s-Lair-X68k
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Align the container to the disc, and find the decoder-free packed player fits

Browse Source 
Two sessions, unrecorded until now, committed together because their edits
share files and cannot be split cleanly after the fact.

Session 28 (FINDINGS 60): the container is DLX5 -- every record sector-aligned,
120/120 starting on a boundary where 3/120 did, +0.48% on the wire and zero
clocks -- and the ring's release rounds to RECALN so no pad is stranded.  Two
encoder levers measured and refused: `--spans all` buys +0.19 dB for +67% of
the wire, and joint span/lam selection emits byte-identical containers because
`lam` never leaves its floor on any of 120 frames.

Session 29 (FINDINGS 61): the packed full-frame blit is 27.3% of a 12 fps
frame, a channel fills GVRAM in buffer mode off the disc with the CPU halted,
and it walks the 1,024 B line stride itself through array chaining.  At the
9 clk/B dual-address floor the codec is 110.4% of a frame and a decoder-free
packed literal player is 55.2%, at +4.89 dB -- 2.75 dB past a ceiling the
codec's scene-wide palette cannot cross.  Encoder work is parked; the codec is
kept and not built on.

check.sh is ALL GREEN before and after, plus one new stage that gates the ORDER
of the measured paint costs rather than their values.

Claude-Session: https://claude.ai/code/session_01194oWYW8DQXK1SZ2DnChW6
This commit is contained in:
prosolis
2026-08-25 06:54:27 -07:00
parent 8800d8f8c0
commit 1be428c270
28 changed files with 2203 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/analysis/13_cpu_ratectl.py
+15 -3
View File
@@ -40,12 +40,24 @@ BUDGET = RC.FRAME_CYCLES
# H.build is ~55 s, nearly all k-means, and it does not depend on the profile:
# both ship k1=k4=256. One build, cached, serves every row of the table.
cache = a.cache or f"tmp/model_{os.path.basename(a.frames_dir.rstrip('/'))}.pkl"
# The build parameters are stored with the model and a mismatch rebuilds: the
# cache is keyed on the frames directory alone, and once H.build acquired an
# option (session 28's reserved black entry, 23.4) a stale pickle would quietly
# serve a model the shipping encoder no longer builds. Same guard as
# tools/analysis/16_span_roundtrip.py.
SIG = dict(k1=256, k4=256, iters=16, reserve_black=True)
m = None
if os.path.exists(cache):
m = pickle.load(open(cache, "rb"))
print(f"model from {cache}")
else:
if m.get("sig") != SIG:
print(f"{cache}: built with {m.get('sig')}, wanted {SIG} -- rebuilding")
m = None
else:
print(f"model from {cache}")
if m is None:
t = time.time()
m = H.build(a.frames_dir, k1=256, k4=256, iters=16)
m = H.build(a.frames_dir, **SIG)
m["sig"] = SIG
pickle.dump(m, open(cache, "wb"))
print(f"built model in {time.time()-t:.0f} s -> {cache}")
print(f"{a.frames_dir}: {len(m['idx'])} frames, {m['nb']} blocks, "
tools/analysis/15_bus_occupancy.py
+7 -1
View File
@@ -167,7 +167,13 @@ print("THE OTHER TWO MASTERS -- what the DMAC takes out of the same frame\n")
FPS = d.fps
CPUHZ = 10e6 # stock X68000, MAME 0.277 x68k.cpp:1133
FRAME_CLK = CPUHZ / FPS
vid_bpf = sum(n + 4 for (_, n) in d.frames[:NF]) / NF # DLX2 record padding
# WHAT THE TRANSPORT MOVES, which is the PADDED record and not the payload.
# Under DLX2/3/4 the pad was 0..3 B and the distinction was noise; under DLX5 it
# is 0..511 B, and charging the payload would price the channel for bytes it
# does not carry while the disc carries them anyway. A budget that debits only
# the bytes a frame CONTAINS is the same incomplete accounting this project has
# been caught by before -- the pad is delivered, so the pad is charged.
vid_bpf = sum(d.record_lengths()[:NF]) / NF
aud_bpf = B.ADPCM_BYTES_PER_S / FPS
a_lo = aud_bpf * B.ADPCM_CLK_BYTE_BEST
a_hi = aud_bpf * B.ADPCM_CLK_BYTE_WORST
tools/analysis/16_span_roundtrip.py
+18 -4
View File
@@ -43,12 +43,26 @@ ap.add_argument("--cache", default=None)
a = ap.parse_args()
cache = a.cache or f"tmp/model_{os.path.basename(a.frames_dir.rstrip('/'))}.pkl"
# The cache is keyed on the frames directory ALONE, which was fine while
# H.build had no options and became a trap the moment it did: session 28's
# reserved black entry (23.4) changes the palette, the codebooks and every
# index in the model, and a pickle from before it would have let this gate
# round-trip a container the shipping encoder no longer emits -- green, and
# testing the wrong artefact. So the build parameters are stored WITH the
# model and a mismatch rebuilds.
SIG = dict(k1=256, k4=256, iters=16, reserve_black=True)
m = None
if os.path.exists(cache):
m = pickle.load(open(cache, "rb"))
print(f"model from {cache}")
else:
if m.get("sig") != SIG:
print(f"{cache}: built with {m.get('sig')}, wanted {SIG} -- rebuilding")
m = None
else:
print(f"model from {cache}")
if m is None:
t = time.time()
m = H.build(a.frames_dir, k1=256, k4=256, iters=16)
m = H.build(a.frames_dir, **SIG)
m["sig"] = SIG
pickle.dump(m, open(cache, "wb"))
print(f"built model in {time.time()-t:.0f} s -> {cache}")
@@ -79,7 +93,7 @@ for span_mode in ("need", "all"):
# of the frame stream, so simply constructing it above has already gated
# that. Said out loud here because it is easy to read this as version drift.
if d.has_index:
print(f" DLX4: record index agrees with the frame stream on all "
print(f" DLX{d.version}: record index agrees with the frame stream on all "
f"{d.nframes} records ({2*d.nframes:,} B of scene header)")
# The decoder's own walk of the span section must land exactly where the
tools/analysis/19_ring_stream.py
+1 -1
View File
@@ -71,7 +71,7 @@ def records(path):
the ring must hold per frame are the padded ones, not the payload.
"""
d = DLX(path)
rec = np.array([4 + n + (-(4 + n) % 4) for _, n in d.frames], np.int64)
rec = np.array(d.record_lengths(), np.int64)
return d, rec
tools/analysis/20_seek_slack.py
+1 -1
View File
@@ -32,7 +32,7 @@ SECTOR = 512
def records(path):
d = DLX(path)
rec = np.array([4 + n + (-(4 + n) % 4) for _, n in d.frames], np.int64)
rec = np.array(d.record_lengths(), np.int64)
return d, rec
tools/analysis/22_scene_load.py
+1 -1
View File
@@ -79,7 +79,7 @@ print(f" once at boot {boot_cyc:>8,} clocks = {1000*boot_cyc/CPUHZ:6.2f}
for path in a.containers:
d = DLX(path)
hdr = int.from_bytes(d.raw[28:32], "big")
rec = np.array([4 + n + (-(4 + n) % 4) for _, n in d.frames], np.int64)
rec = np.array(d.record_lengths(), np.int64)
wire = rec.mean() * FPS / 1024 + RC.AUDIO_KBPS
print(f"\n=== {path}: header region {hdr:,} B "
f"(pal 768 + cb1 {d.k1*16:,} + cb4 {d.k4*4:,} + 32), wire {wire:.1f} KB/s")
tools/analysis/26_sector_align.py
+18 -11
View File
@@ -66,16 +66,21 @@ a = ap.parse_args()
d = DLX(a.container)
# The disc layout the 68000 walks: [u32 len][body], each record padded up to 4.
# Exactly tools/bench/prep_stream.py's, and it is rebuilt here rather than read
# from tmp/ so this tool works on any container.
# The disc layout the 68000 walks: [u32 len][body], each record padded up to the
# container's own alignment -- 4 on DLX2/3/4, 512 on DLX5. Exactly
# tools/bench/prep_stream.py's, and it comes from the reader rather than from a
# second copy of the rule here, so pointing this tool at a DLX5 container asks
# it the RIGHT question: what does the mismatch still cost once the container
# has been changed to remove it? (The answer had better be nothing.)
off, recs = 0, []
for (o, n) in d.frames:
ln = 4 + n
ln += (-ln) % 4
for ln in d.record_lengths():
recs.append((off, ln))
off += ln
payload = sum(ln for _, ln in recs)
# The DENOMINATOR is the record bytes the decoder actually reads -- [u32 len]
# plus payload -- and NOT the padded length, because on a DLX5 container the
# padding IS the cost being measured. Scoring against the padded length would
# make an already-aligned container report +0.00% and look free.
payload = sum(4 + n for _, n in d.frames)
nfr = len(recs)
budget = CPUHZ / d.fps
@@ -120,8 +125,10 @@ for _, ln in recs:
pad += SECTOR - (cur % SECTOR)
cur += SECTOR - (cur % SECTOR)
cur += ln
print("C. SECTOR-ALIGNED RECORDS (a container change; a re-encode)")
print(f" wire {cur:,} B for {payload:,} B of record = +{100*pad/payload:.2f}%")
print("C. SECTOR-ALIGNED RECORDS (a container change; a re-encode)"
+ (" -- THIS CONTAINER ALREADY IS ONE" if d.sector_aligned else ""))
print(f" wire {cur:,} B for {payload:,} B of record = "
f"+{100*(cur-payload)/payload:.2f}%")
print( " clocks ZERO: the read is a whole-sector read straight into the "
"ring, no window,")
print( " no copy, and the DMAC can do it.")
@@ -130,8 +137,8 @@ print()
ringsz = a.ring * 1024
print(f" VERDICT, in the currency this project prices delivery in. C is "
f"cheaper on the wire")
print(f" than A and B by {100*(drop_a-pad)/payload:.2f} points of the payload "
f"({drop_a-pad:,} B on this scene),")
print(f" than A and B by {100*(wire_a-cur)/payload:.2f} points of the payload "
f"({wire_a-cur:,} B on this scene),")
print(f" and it is the only one of the three a DMA channel can run without a "
f"copy. What it")
print(f" costs is a container revision and the re-measurement that comes with "
tools/analysis/29_packed_player.py
+377
View File
@@ -0,0 +1,377 @@
#!/usr/bin/env python3
"""The DECODER-FREE PACKED player, priced against the measured cost model.
python3 tools/analysis/29_packed_player.py [container.dlx] [--kbps R]
THE QUESTION, and why it is being asked again. FINDINGS 44.7 removed the codec
and asked what a player that just puts literal frames on screen would cost. It
answered "it fits the clocks and dies on the medium": 1,152 KB/s and 1.61 GB,
because 256-colour GVRAM's default write path throws away the high byte of every
word and a picture byte therefore costs two disc bytes. 46.5/47.1 then found
the off switch -- CRTC R20 bit 11 -- and 47.2 built the layout and rendered it
pixel-exactly on both emulators at 1.0 B/pixel. 47.5 re-derived the budget on
that and withdrew 44.7's conclusion CONDITIONALLY.
Everything in 47.5 is arithmetic over a cost model that has since been REPLACED.
When it was written the transport was an unmeasured `c`; sessions 25b-28 put the
transport on the 68000 and measured it (58.2: PIO is 87.28 clk/B), put it on the
DMAC and bounded it (59.2: this machine can run dual-address only, and a
dual-address byte has a 9 clk/B FLOOR), and re-derived what a frame can afford
(59.7/60.7: 6.69 clk/B on the gate container). 47.6.1 also filed the CPU paint
cost as an ASSUMPTION -- "the `movem` shape of the packed writes is an
assumption", no clock in 47.5 measured.
So this tool re-asks 44.7's question with:
* the paint MEASURED, not assumed -- tools/bench/blit.s V8 is V1 with 128
words a row instead of 256, and tools/bench/blit.lua times it next to V1,
V2 and V3 in the same run, so the packed number is quoted against a
variant whose value (53.6%) is a session-9 result that has not moved;
* the transport swept over the SAME `W` ladder 15_bus_occupancy.py uses,
every rung of it sourced or measured (buscost.py);
* the audio DMA charged, at the rate the IPL ROM's own channel-3 setup
implies (21_iplrom_dmac.py) -- 60.x's rule that a budget debits I/O;
* and the wire and the volume stated for each, because 44.7's answer was
never about clocks.
WHAT IT DOES NOT DO. It does not settle 47.4 -- whether buffer mode BLANKS the
graphics layer, which MAME asserts and px68k is silent about (48.1), and which
needs a real board. It PRICES both branches instead, and the blanking section
is where the measured paint earns its keep: the black interval is the paint, and
until now the paint was a range read off an unpacked measurement ("~27% to ~54%",
48.3) rather than a number.
"""
import sys, os, re, argparse, csv
sys.path.insert(0, "tools/encoder")
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import numpy as np
from dlx import DLX
import buscost as B
CPUHZ = 10e6 # stock X68000, MAME 0.277 x68k.cpp:1133
GAME_S = 22.8 * 60 # the full-disc survey's runtime (ROADMAP C1)
ap = argparse.ArgumentParser()
ap.add_argument("container", nargs="?", default="tmp/rc_fr_singe_scsi_span.dlx",
help="the CODEC baseline this is compared against")
ap.add_argument("--csv", default="tmp/c68k_frames.csv",
help="per-frame C68K measurement of that container")
ap.add_argument("--blit-log", default="tmp/blit_v8.log",
help="tools/bench/blit.lua's log -- where the MEASURED paint "
"costs are read from. No defaults are compiled in.")
ap.add_argument("--kbps", type=float, default=None,
help="a delivery rate to score the wire against. OPTIONAL and "
"there is no default (FINDINGS 50).")
a = ap.parse_args()
# --- the measured paint, read out of the run's own log ---------------------
# NOT transcribed into this file. A constant copied out of a log is a constant
# that stops tracking the log, and this project has been caught by a stale
# number twice (60.8). If the log is not there the tool refuses rather than
# substituting a plausible one.
if not os.path.exists(a.blit_log):
sys.exit(f"missing {a.blit_log} -- run tools/bench/blit.lua first:\n"
f" cd tmp && mame x68000 -bios ipl10 -ramsize 2M -video soft "
f"-window -sound none -nothrottle -plugins \\\n"
f" -autoboot_script ../tools/bench/blit.lua -seconds_to_run 60")
blit = {}
for line in open(a.blit_log, errors="replace"):
m = re.search(r"V(\d+)\s+(\d+) cyc", line)
if m:
blit[int(m.group(1))] = int(m.group(2))
for v in (1, 2, 3, 4, 8, 9, 10):
if v not in blit:
sys.exit(f"{a.blit_log} has no V{v} result -- the summary is incomplete, "
f"so the run did not finish and nothing here can be quoted.")
# --- the codec baseline: the container, and its MEASURED decode ------------
d = DLX(a.container)
FPS = d.fps
FRAME_CLK = CPUHZ / FPS
meas = {}
if os.path.exists(a.csv):
for r in csv.DictReader(open(a.csv)):
meas[int(r["frame"])] = int(r["cycles"])
if not meas:
sys.exit(f"missing {a.csv} -- the codec row's decode term is MEASURED and "
f"there is no derived stand-in for it.")
NF = max(meas) + 1
codec_decode = np.mean([meas[f] for f in range(NF)])
codec_bpf = sum(d.record_lengths()[:NF]) / NF # the PADDED record (60.7)
# --- geometry, which is where the decoder-free rows come from -------------
W_PX, H_PX = d.W, d.H
NPX = W_PX * H_PX
UNPACKED_BPF = NPX * 2 # one pixel per word, high byte discarded
PACKED_BPF = NPX * 1 # R20 bit 11 + page scroll (47.2, measured)
aud_bpf = B.ADPCM_BYTES_PER_S / FPS
AUD_CLK = aud_bpf * B.ADPCM_CLK_BYTE_BEST # best case, so every row is
# the optimistic end
# A device->GVRAM channel cannot walk a 1024-byte line stride inside one
# transfer: it writes a contiguous run. 192 rows therefore need 192 array-chain
# entries -- and SESSION 29 RAN THAT, off the disc, through src/player/dma.i's
# DM_BARV/DM_BTCV: eight rows at the 1024 B stride landed from ONE channel start
# with the CPU halted throughout (tools/bench/dma_run.sh, `[chain]`). So the
# MECHANISM is demonstrated and the CPU does not restart the channel per row.
# The COST is still datasheet arithmetic -- 36 clocks an entry, Fig 4-25 sheet 1,
# buscost.DMA_CHAIN_CLK -- because MAME's DMAC runs on wall-clock attotimes and
# cannot be asked what anything costs (42.5).
CHAIN_CLK = H_PX * B.DMA_CHAIN_CLK
print(f"""{a.container}: {NF} frames of {W_PX}x{H_PX} at {FPS:g} fps
frame slot on a 10 MHz 68000: {FRAME_CLK:,.0f} clocks
paint costs MEASURED by tools/bench/blit.lua, read from {a.blit_log}:
V1 unpacked movem blit {blit[1]:>9,} clk {100*blit[1]/FRAME_CLK:5.1f}% (96 KB read + 96 KB write)
V2 byte-source expansion {blit[2]:>9,} clk {100*blit[2]/FRAME_CLK:5.1f}% (48 KB read + 96 KB write)
V3 write-only floor {blit[3]:>9,} clk {100*blit[3]/FRAME_CLK:5.1f}% (no source read at all)
V8 PACKED movem blit {blit[8]:>9,} clk {100*blit[8]/FRAME_CLK:5.1f}% (48 KB read + 48 KB write)
V8 is {100*blit[8]/blit[1]:.1f}% of V1 and {100*blit[8]/blit[3]:.1f}% of V3 -- so PACKED PAINT COSTS WHAT THE
UNPACKED PATH PAYS TO WRITE ALONE, with its source read thrown in free.
It is not exactly half of V1 because the 192-row loop does not halve with
the words: per word V1 is {blit[1]/(NPX):.3f} clk and V8 is {blit[8]/(NPX//2):.3f}.""")
# --- the architectures ----------------------------------------------------
# Each is (label, bytes on the wire per frame, CPU clocks per frame that are
# NOT the transport, and whether the transport lands in GVRAM or in RAM).
ARCH = [
("CODEC, CPU-decoded (the shipping design)", codec_bpf, codec_decode, "ring"),
("free / DMAC device->GVRAM / unpacked", UNPACKED_BPF, CHAIN_CLK, "gvram"),
("free / DMAC device->GVRAM / PACKED", PACKED_BPF, CHAIN_CLK, "gvram"),
("free / CPU-painted / unpacked, 2 B/px wire", UNPACKED_BPF, blit[1], "ring"),
("free / CPU-painted / unpacked, 1 B/px wire", PACKED_BPF, blit[2], "ring"),
("free / CPU-painted / PACKED", PACKED_BPF, blit[8], "ring"),
]
LADDER = [
(5.0, "single address, held -- needs a request line (B3)"),
(9.0, "dual address, held -- the FLOOR (59.2/59.7)"),
(12.0, "single address, arbitrated"),
(16.0, "what the ROM programs for SASI, best"),
(19.0, "what the ROM programs for SASI, worst"),
(87.28, "PIO -- MEASURED, 58.2"),
]
print("\n" + "=" * 78)
print("WHAT EACH ARCHITECTURE COSTS A FRAME, over the transport ladder\n")
print(f" audio DMA is charged in every row at {AUD_CLK:,.0f} clk "
f"({100*AUD_CLK/FRAME_CLK:.2f}%), best case.\n")
hdr = f" {'architecture':<44}{'B/frame':>9}" + "".join(f"{f'W={w:g}':>9}" for w, _ in LADDER)
print(hdr)
print(" " + "-" * (len(hdr) - 2))
for label, bpf, cpu, dest in ARCH:
cells = []
for w, _ in LADDER:
tot = bpf * w + cpu + AUD_CLK
pct = 100 * tot / FRAME_CLK
cells.append(f"{pct:>8.1f}%" if pct < 1000 else f"{pct:>8.0f}%")
print(f" {label:<44}{bpf:>9,.0f}" + "".join(cells))
print(f"""
100% is the frame deadline. Every cell is CPU work plus transport plus
best-case audio; none of them overlap, because the 68000 has no cache and a
two-word prefetch queue that empties at once (buscost.DMA_OVERLAPS = False).
THE TWO ROWS THAT MATTER ARE THE FLOOR COLUMN, W=9, because 59.2 found that
the only configurations this machine can be shown to run are dual-address,
and a dual-address byte is a 4-clock read of the device plus a 5-clock write
to memory. Every column left of it is a hardware fact nobody here has.""")
# --- the wire, which is what 44.7 actually died on ------------------------
print("\n" + "=" * 78)
print("THE WIRE AND THE MEDIUM -- 44.7's real objection\n")
print(f" {'architecture':<44}{'B/frame':>9}{'KB/s':>9}{'GB for 22.8 min':>18}")
print(" " + "-" * 78)
seen = set()
for label, bpf, cpu, dest in ARCH:
kbs = bpf * FPS / 1024
gb = bpf * FPS * GAME_S / 1e9
print(f" {label:<44}{bpf:>9,.0f}{kbs:>9.1f}{gb:>18.2f}")
print(f"""
The codec row is the gate container, which is deliberately the heaviest thing
the encoder emits (59.7). The default `need` recipe is 267.9 KB/s and E7's
byte target at the 9 clk/B floor is 327 KB/s (60.7).
SO THE PACKED DECODER-FREE PLAYER ASKS FOR {PACKED_BPF*FPS/1024:.0f} KB/s -- {PACKED_BPF*FPS/1024/327:.2f}x E7's target and
{PACKED_BPF*FPS/1024/(codec_bpf*FPS/1024):.2f}x the gate container -- and it asks for it AT A FIXED RATE. A codec's
bitrate is a lever; a literal frame's is geometry, and there is no scene in
the picture that costs less than another.""")
if a.kbps:
R = a.kbps * 1024
print(f"\n against a supplied {a.kbps:g} KB/s:")
for label, bpf, cpu, dest in ARCH:
need = bpf * FPS
print(f" {label:<44}{'FITS' if need <= R else 'SHORT BY '}"
f"{'' if need <= R else f'{(need-R)/1024:.0f} KB/s'}"
f" ({need/1024:.0f} KB/s wanted)")
# --- 47.4's two branches, priced -----------------------------------------
print("\n" + "=" * 78)
print("IF BUFFER MODE BLANKS THE LAYER (47.4 / 48, MAME's reading)\n")
print(""" R20 bit 11 only has to be SET across the GVRAM writes, so the black
interval is the paint and not the frame -- and which paint depends on where
the transport lands. That asymmetry has not been stated before:\n""")
print(f" {'architecture':<44}{'black interval':>16} {'set for':<14}")
print(" " + "-" * 78)
for label, bpf, cpu, dest in ARCH[1:]:
if dest == "gvram":
# the channel writes GVRAM, so the bit is set for the whole transfer
# DMA rungs only: a PIO transport is not a channel writing GVRAM, so
# 87.28 has no meaning in a device->GVRAM row.
rows = [bpf * w + CHAIN_CLK for w, _ in LADDER if w < 20]
span = f"{100*min(rows)/FRAME_CLK:.0f}%..{100*max(rows)/FRAME_CLK:.0f}%"
note = "the whole DMA"
else:
span = f"{100*cpu/FRAME_CLK:.1f}%"
note = "the blit only"
print(f" {label:<44}{span:>16} {note:<14}")
print(f"""
THE CPU-PAINTED PACKED PATH HAS THE SMALLEST BLACK WINDOW OF ANY OF THEM --
{100*blit[8]/FRAME_CLK:.1f}% -- because its transport lands in RAM, where bit 11 is irrelevant,
and only the {blit[8]:,}-clock blit needs the bit set. The DMAC-direct path,
which is cheaper in clocks at every rung of the ladder, is the one that must
hold the bit across its whole transfer. Under MAME's reading the cheap
architecture is the dark one.
Both are a strobe at the frame rate over the whole picture, and the packed
layout has no page to flip to: both 256-colour pages carry picture, which is
the entire point of it (48.3). {100*blit[8]/FRAME_CLK:.1f}% black at 12 Hz is not a tear.
IF PX68K IS RIGHT AND IT DOES NOT BLANK, every number above stands as
written. Neither emulator is authority and 48.1 is why the prior leans
MAME's way: MAME asserts the semantic twice and deliberately, px68k's
display path never reads the bit at all. That is an assertion against a
silence, not a tie, and it is settled by a board and the two-line probe in
tools/bench/probe_bit11_blank.lua.""")
# --- 47.6.4: does the CODEC survive the packed layout? --------------------
# Open since session 16 and never touched: "under the packed layout a word spans
# two columns 128 apart. Whether the existing codec survives that is untouched."
# There are exactly two ways it could, and blit.s V9 and V10 are them.
sk_blocks = sk_tot = pair_sk = pair_tot = 0
for f in range(NF):
m = d.modes(f).reshape(d.nby, d.nbx)
L, R = m[:, :d.nbx // 2], m[:, d.nbx // 2:]
sk_blocks += int((m == 0).sum()); sk_tot += m.size
pair_sk += int(((L == 0) & (R == 0)).sum()); pair_tot += L.size
paint_now = 1 - sk_blocks / sk_tot
paint_pair = 1 - pair_sk / pair_tot
print("\n" + "=" * 78)
print("CAN THE CODEC BE PACKED TOO? -- 47.6.4, open since session 16\n")
print(f""" A 4x4 block owns four bytes at STRIDE 2 under the packed layout, because
the high bytes of its four words belong to the block 128 columns away. So a
block decoder has two options and neither is free:
{'V4 block order, UNPACKED (the shipping shape)':<52}{blit[4]:>9,} clk {100*blit[4]/FRAME_CLK:5.1f}%
{'V9 block order, PACKED, 16 move.b at stride 2':<52}{blit[9]:>9,} clk {100*blit[9]/FRAME_CLK:5.1f}%
{'V10 block order, PACKED, blocks PAIRED (movem back)':<52}{blit[10]:>9,} clk {100*blit[10]/FRAME_CLK:5.1f}%
V9 IS {100*blit[9]/blit[4]-100:.0f}% DEARER THAN V4, not cheaper. Packing buys a block decoder
nothing on the wire either -- a codeword is already one byte a pixel -- so
that route buys NOTHING and costs {blit[9]-blit[4]:,} clocks a frame to buy it.
V10 halves the paint, and pays for it in the mode map. A pair skips only if
BOTH its blocks skip, and on this container:
SKIP blocks now {100*sk_blocks/sk_tot:5.1f}% painted now {100*paint_now:5.1f}%
SKIP block PAIRS {100*pair_sk/pair_tot:5.1f}% painted paired {100*paint_pair:5.1f}%
So pairing paints {paint_pair/paint_now:.2f}x as many blocks for {blit[10]/blit[4]:.2f}x the paint per block --
{100*(paint_pair/paint_now)*(blit[10]/blit[4])-100:+.0f}% on the clock, and about {100*(paint_pair/paint_now-1):+.0f}% on the BYTES, because a coded
block is bytes in the container whether its half of the pair changed or not.
E7 needs the bytes DOWN {100*(codec_bpf*FPS/1024)/327-100:.0f}%.
SO PACKING BELONGS TO THE LITERAL PLAYER AND ONLY TO IT. 47.6.4 is closed:
the packed layout is not an upgrade the existing codec can take, it is the
thing you get INSTEAD of the codec.""")
# --- the palette, which is where the literal player stops being a compromise --
# 46.3 measured these while pricing the TEXT PLANE and the 256-colour rows were
# only there for scale. They answer a question nobody put to them: a literal
# player has no codebooks, so it is not tied to a scene-wide palette the way the
# codec is (vq.scene_palette exists BECAUSE codewords are indices into it), and
# per-frame palettes become legal. Re-run 18_text_plane_16col.py to reproduce.
PSNR_SHIPPED = 29.19 # docs/STATUS.md, --spans all, c=5, 496.7 KB/s
PSNR_SCENE_256 = 31.33 # 18_text_plane_16col.py, tmp/fr_singe, 120 frames
PSNR_FRAME_256 = 34.08 # the same window, per-frame palettes
PAL_BYTES = 512 # 256 entries x 1 word
pal_bpf = PACKED_BPF + PAL_BYTES
# The palette write, DERIVED from a MEASURED per-word constant: V8 moves a word
# into GVRAM for blit[8]/(NPX//2) clocks and the palette is 256 consecutive
# words at $E82000 in the same movem shape.
pal_clk = 256 * blit[8] / (NPX // 2)
print("\n" + "=" * 78)
print("AND THE PICTURE IS BETTER, WHICH NOBODY HAD ASKED\n")
print(f""" PSNR against the 24-bit source, 18_text_plane_16col.py over the same
120-frame window the whole tree is measured on:
{'shipping container (the codec, as it ships)':<48}{PSNR_SHIPPED:6.2f} dB
{'256 colours, SCENE palette -- the codec CEILING':<48}{PSNR_SCENE_256:6.2f} dB
{'256 colours, PER-FRAME palette':<48}{PSNR_FRAME_256:6.2f} dB
THE MIDDLE ROW IS A CEILING AND NOT A RIVAL. Every codeword the codec emits
is an index INTO the scene palette, so no amount of bitrate takes it past
{PSNR_SCENE_256:.2f} dB; it spends {codec_bpf*FPS/1024:.0f} KB/s to get within {PSNR_SCENE_256-PSNR_SHIPPED:.2f} dB of it.
A LITERAL FRAME HAS NO CODEBOOKS, so the scene palette is not forced on it,
and the bottom row is what it simply IS -- {PSNR_FRAME_256-PSNR_SHIPPED:+.2f} dB on the shipping
container and {PSNR_FRAME_256-PSNR_SCENE_256:+.2f} dB past the ceiling the codec cannot cross.
WHAT THE PER-FRAME PALETTE COSTS:
on the wire {PAL_BYTES} B a frame -> {pal_bpf:,} B, {pal_bpf*FPS/1024:.1f} KB/s (+{100*PAL_BYTES/PACKED_BPF:.1f}%)
in clocks ~{pal_clk:,.0f} ({100*pal_clk/FRAME_CLK:.2f}% of a frame) if the CPU writes it, DERIVED
from V8's measured {blit[8]/(NPX//2):.3f} clk/word in the same movem shape
in colours 254, not 256: the packed layout spends index 0 on the
transparency key and puts black at 255 (47.2,
prep_frame.py --pack-transparent), against --reserve-black's
one entry. The tree has already measured a reserved entry at
0.04 dB (60.3), so this is noise against {PSNR_FRAME_256-PSNR_SHIPPED:+.2f}.
NOT SETTLED, and it is the obvious next probe: whether a DMA CHANNEL can write
the palette registers at $E82000, which would make the palette a 193rd chain
entry and cost the CPU nothing at all. Untested; 61.2 only ever pointed a
channel at GVRAM.
AND THE PSNR FIGURES ARE PIL's MEDIANCUT, not this project's own palette
builder (vq.scene_palette / H.build). The DIRECTION is measured and the
magnitude is about right; if the packed player gets built, re-derive the
per-frame number against the builder that will actually ship it.""")
# --- the answer ----------------------------------------------------------
w9 = 9.0
free_packed_dma = PACKED_BPF * w9 + CHAIN_CLK + AUD_CLK
free_packed_cpu = PACKED_BPF * w9 + blit[8] + AUD_CLK
codec_9 = codec_bpf * w9 + codec_decode + AUD_CLK
print("\n" + "=" * 78)
print(f"""THE ANSWER, AT THE ONE RUNG THIS MACHINE CAN BE SHOWN TO RUN (W=9)
CODEC, gate container {100*codec_9/FRAME_CLK:6.1f}% of the frame -- DOES NOT FIT
free / DMAC->GVRAM / PACKED {100*free_packed_dma/FRAME_CLK:6.1f}% -- FITS, with {100-100*free_packed_dma/FRAME_CLK:.0f}% to spare
free / CPU-painted / PACKED {100*free_packed_cpu/FRAME_CLK:6.1f}% -- FITS, with {100-100*free_packed_cpu/FRAME_CLK:.0f}% to spare
THE DECODER-FREE PACKED PLAYER FITS THE CLOCK BUDGET THAT THE CODEC MISSES.
That is not a small correction to 47.5, it is the reverse of the reason the
codec exists. 44.7 said it in advance and on a different cost model: "the
codec is not there to save CPU -- it is there to save the wire." The
measured model agrees, and now says the CPU side is not merely affordable
but strictly cheaper WITHOUT the codec: at the floor, decoding {codec_bpf:,.0f} bytes
costs {100*(codec_bpf*w9+codec_decode)/FRAME_CLK:.0f}% of a frame and NOT decoding {PACKED_BPF:,} costs {100*(PACKED_BPF*w9+blit[8])/FRAME_CLK:.0f}%.
SO THE QUESTION IS ENTIRELY A MEDIUM QUESTION, and it has two halves:
1. {PACKED_BPF*FPS/1024:.0f} KB/s SUSTAINED, with no lever to pull. ROADMAP B1 is
unmeasured; the 0.7-1.7 MB/s usually quoted for BlueSCSI on an X68000
is folklore with no published benchmark behind it. {PACKED_BPF*FPS/1024:.0f} KB/s sits
inside that range, which is exactly why the range has to be measured
rather than cited. A codec at 327 KB/s survives a slower answer; a
literal frame does not degrade, it drops.
2. {PACKED_BPF*FPS*GAME_S/1e9:.2f} GB for the whole game, against the codec's {codec_bpf*FPS*GAME_S/1e9:.2f} GB at the gate
recipe and ~{327*1024*GAME_S/1e9:.2f} GB at E7's target. That is a packaging fact (C3),
not a performance one.
AND 47.4 STILL SITS OVER ALL OF IT. Everything above assumes the layer is
visible while it is written. If it is not, the packed player is a {100*blit[8]/FRAME_CLK:.0f}% duty
strobe at best and there is no version of it that is merely expensive.""")
tools/bench/blit.lua
+23 -1
View File
@@ -29,7 +29,7 @@ end
local MODE = load_mode()
local FLAG, VAR, ITER = 0x18000, 0x18004, 0x18008
local SRCW, SRCB = 0x60000, 0x80000
local SRCW, SRCB, SRCP = 0x60000, 0x80000, 0x90000
local GVRAM, GPAL = 0xC00000, 0xE82000
local CPUHZ = 10000000 -- x68k.cpp:1133, 40_MHz_XTAL/4
local FRAME12 = CPUHZ / 12 -- 833333 cycles at 12 fps
@@ -40,6 +40,16 @@ local PLAN = {
{var=2, iter= 50, name="V2 naive byte-source expansion (move.b/move.w per pixel)"},
{var=3, iter=200, name="V3 write-only floor (no source read at all)"},
{var=4, iter= 60, name="V4 same 96KB of writes, issued in 4x4 BLOCK order (decoder access pattern)"},
-- V8 is V1 with R20 bit 11's packing: 48KB read + 48KB write for the SAME
-- 49,152 pixels. It is the per-frame work of a decoder-free packed player
-- (FINDINGS 44.7 / 46.6 / 47.5), and 47.6.1 filed its `movem` shape as an
-- ASSUMPTION -- this is the measurement that assumption was standing in for.
{var=8, iter=200, name="V8 PACKED movem.l blit (48KB read + 48KB write, same 49,152 pixels as V1)"},
-- V9/V10 are the two ways a BLOCK decoder could survive the packed layout
-- (47.6.4, open since session 16): sixteen move.b at stride 2 per block, or
-- pair the blocks 128 columns apart in the encoder and get V4's movem back.
{var= 9, iter= 40, name="V9 PACKED block order, 16 move.b at stride 2 per 4x4 block"},
{var=10, iter=120, name="V10 PACKED block order, blocks PAIRED so a movem writes whole words"},
}
local code do
@@ -97,6 +107,18 @@ local function setup()
SP:write_u8 (SRCB + y*256 + x, px)
end
end
-- SRCP: the PACKED frame, interleaved the way tools/bench/show_frame256_packed.lua
-- lays it out -- word i of a row is (column i+128) << 8 | (column i), because
-- page 0 is the low byte at screen column i and page 1 the high byte at i+128.
-- Only V8 reads it, and only its SIZE (128 words a row) affects the timing;
-- the interleave is written correctly so the buffer is the real artefact and
-- not a same-sized stand-in.
for y = 0, H-1 do
local row = PIX0 + y*W
for i = 0, (W//2)-1 do
SP:write_u16(SRCP + y*(W//2)*2 + i*2, (B(row+i+W//2) << 8) | B(row+i))
end
end
for i = 1, #code do SP:write_u8(0x10000+i-1, string.byte(code,i)) end
P(string.format("loaded blit.bin=%d bytes, source frame %dx%d at yoff=%d", #code, W, H, YOFF))
end
tools/bench/blit.s
+146
View File
@@ -104,6 +104,9 @@ ITER = $18008 ; iteration count, written by Lua
SPTR = $1800C ; V5 span stream pointer, written by Lua
SRCW = $60000 ; word-expanded frame 192*512 = 96KB
SRCB = $80000 ; byte-per-pixel frame 192*256 = 48KB
SRCP = $90000 ; PACKED frame 192*256 = 48KB (V8): two picture
; bytes per word, already interleaved by the
; encoder, so the blit is a straight copy
DST0 = $C08000 ; GVRAM + 32*1024 (first picture row)
DSTE = $C38000 ; GVRAM + 224*1024 (one past last)
ROWS = 192 ; picture rows a V5 stream describes
@@ -130,6 +133,12 @@ start:
beq v6
cmp.l #7,d0
beq v7
cmp.l #8,d0
beq v8
cmp.l #9,d0
beq v9
cmp.l #10,d0
beq v10
bra v3
; ---------------------------------------------------------------- V1
@@ -372,5 +381,142 @@ v7fh:
bne v7
bra done
; ---------------------------------------------------------------- V8
; THE PACKED FULL-FRAME BLIT (FINDINGS 46.6/47.2). Identical in shape to V1 --
; a row-linear movem.l chain out of a RAM frame into GVRAM -- and different in
; exactly one thing: a row is 128 WORDS, not 256, because R20 bit 11 lets one
; word carry two picture bytes. 256 = 5*48 + 16, so five 12-register bursts
; and a 4-register tail, against V1's ten and one.
;
; TIMING ONLY, and it does not set bit 11. MAME's gvram_w carries no timing in
; either arm (blit.lua's header), so the bit cannot move a cycle here; what it
; moves is the PICTURE, and the picture is what tools/bench/show_frame256_packed.lua
; and tools/bench/gvpack already verify pixel-exactly. Setting it here would
; make this variant's snapshot right and its measurement no different, and
; would put a display-mode change inside a timing loop for no gain.
;
; The source is PRE-INTERLEAVED by the host, which is the honest half of the
; claim: the packing is an encoder-side transform (46.3's argument for the text
; plane, and the same one here), so the decoder-free player's per-frame work is
; this copy and nothing else. If the interleave had to happen at run time this
; variant would be V2, not V1.
v8: lea SRCP,a0
lea DST0,a1
lea DSTE,a6
v8row: movem.l (a0)+,d0-d7/a2-a5
movem.l d0-d7/a2-a5,(a1)
movem.l (a0)+,d0-d7/a2-a5
movem.l d0-d7/a2-a5,48(a1)
movem.l (a0)+,d0-d7/a2-a5
movem.l d0-d7/a2-a5,96(a1)
movem.l (a0)+,d0-d7/a2-a5
movem.l d0-d7/a2-a5,144(a1)
movem.l (a0)+,d0-d7/a2-a5
movem.l d0-d7/a2-a5,192(a1)
movem.l (a0)+,d0-d3
movem.l d0-d3,240(a1)
lea 1024(a1),a1
cmpa.l a6,a1
bne v8row
subq.l #1,ITER.l
bne v8
bra done
; ---------------------------------------------------------------- V9
; WHAT THE PACKED LAYOUT COSTS A BLOCK DECODER (FINDINGS 47.6.4, open).
;
; V4 is the access pattern of a decoder that writes 4x4 codewords straight into
; GVRAM: 4 rows of 8 contiguous bytes at a 1024-byte stride, so each row is one
; `movem.l` of two registers. Under the packed layout that pattern is GONE.
; A block at columns x..x+3 owns the LOW bytes of four consecutive words -- four
; bytes at STRIDE 2 -- and the high bytes of those same words belong to the
; block 128 columns away. There is no burst that writes every other byte, so
; the block is sixteen `move.b`s.
;
; V9 does the pair together, low block then high block off one base, so it
; writes every byte it touches and covers the same 49,152 pixels V1/V4/V8 do.
; It is the HONEST version of "keep the codec and pack the screen": the mode
; map is unchanged, SKIP still works per block, and the writes go byte at a
; time. V10 below is the other option, and the comparison is the point.
v9: lea SRCB,a0
lea DST0,a3
lea DSTE,a4
v9brow: move.l a3,a1
lea 256(a3),a5 ; 32 block PAIRS * 8 bytes
v9blk:
move.b (a0)+,(a1)
move.b (a0)+,2(a1)
move.b (a0)+,4(a1)
move.b (a0)+,6(a1)
move.b (a0)+,1024(a1)
move.b (a0)+,1026(a1)
move.b (a0)+,1028(a1)
move.b (a0)+,1030(a1)
move.b (a0)+,2048(a1)
move.b (a0)+,2050(a1)
move.b (a0)+,2052(a1)
move.b (a0)+,2054(a1)
move.b (a0)+,3072(a1)
move.b (a0)+,3074(a1)
move.b (a0)+,3076(a1)
move.b (a0)+,3078(a1)
move.b (a0)+,1(a1)
move.b (a0)+,3(a1)
move.b (a0)+,5(a1)
move.b (a0)+,7(a1)
move.b (a0)+,1025(a1)
move.b (a0)+,1027(a1)
move.b (a0)+,1029(a1)
move.b (a0)+,1031(a1)
move.b (a0)+,2049(a1)
move.b (a0)+,2051(a1)
move.b (a0)+,2053(a1)
move.b (a0)+,2055(a1)
move.b (a0)+,3073(a1)
move.b (a0)+,3075(a1)
move.b (a0)+,3077(a1)
move.b (a0)+,3079(a1)
addq.l #8,a1
cmpa.l a5,a1
bne v9blk
lea 4096(a3),a3
cmpa.l a4,a3
bne v9brow
subq.l #1,ITER.l
bne v9
bra done
; ---------------------------------------------------------------- V10
; THE OTHER OPTION: PAIR THE BLOCKS IN THE ENCODER. If the codec codes the
; block at x and the block at x+128 as ONE unit, the destination is whole words
; again and V4's `movem.l` shape comes straight back -- the same instructions,
; the same 32 bytes of source per unit, and TWICE the pixels, because a word now
; carries two of them. So V10 is V4's inner loop run half as many times.
;
; WHAT IT COSTS IS NOT IN THIS MEASUREMENT. A pair skips only if BOTH of its
; blocks skip, and the two are 128 columns apart with nothing in the picture
; relating them. That is a CONTAINER question -- what fraction of the mode map
; survives pairing -- and 08_mode_map.py has the data to answer it. V10 prices
; the paint; it does not price the SKIPs the pairing loses.
v10: lea SRCP,a0
lea DST0,a3
lea DSTE,a4
v10brow: move.l a3,a1
lea 256(a3),a5 ; 32 block PAIRS * 8 bytes
v10blk: movem.l (a0)+,d0-d7 ; 32 bytes = one PAIR of 4x4 blocks
movem.l d0-d1,(a1)
movem.l d2-d3,1024(a1)
movem.l d4-d5,2048(a1)
movem.l d6-d7,3072(a1)
addq.l #8,a1
cmpa.l a5,a1
bne.s v10blk
lea 4096(a3),a3
cmpa.l a4,a3
bne v10brow
subq.l #1,ITER.l
bne v10
bra done
done: move.l #$FF,FLAG.l ; timer stops here
halt: bra.s halt
tools/bench/check.sh
+91 -14
View File
@@ -353,7 +353,7 @@ echo "--- session 23: the 68000 fills its own ring (FINDINGS 55) ---"
# rather than faulting (49.2);
# * the host AUDITS every placement against its own index and its own list of
# live records, and refuses the run on the first disagreement;
# * the wrap policy still produces the SAME 18 wraps and 14.7 KB mean hole the
# * the wrap policy still produces the SAME 18 wraps the
# host producer produced in FINDINGS 49.4 -- a third independent
# implementation of `aligned` landing on the same tiling;
# * zero underruns at a two-deep request queue, which is the finding: a
@@ -445,13 +445,17 @@ echo "--- session 26: the ring is filled off a real SCSI volume (FINDINGS 58) --
# * pixel-exact, which is the only test that can see a wrong record: the
# window in scsi.i decides which of a sector's bytes reach the ring, and a
# window off by one byte desyncs the bitstream rather than faulting (49.2);
# * the SAME 18 wraps and 14.7 KB mean hole -- ring.i's placement policy must
# not be able to tell which transport answered it, and this is the assertion
# that says it could not;
# * every record accounted for: 120 READ(10)s, 4,488,588 B into the ring, and
# 4,548,608 B off the disc. The two byte counts differ by 1.34% because a
# record is not a sector, and that gap is a delivery cost (58.3) -- gating
# both numbers means neither can drift silently into the other;
# * the SAME 18 wraps -- ring.i's placement policy must not be able to tell
# which transport answered it, and this is the assertion that says it could
# not. The WRAP COUNT is gated and the mean hole is only reported: DLX5's
# records are up to 511 B longer than DLX4's, so the hole moved (14.7 KB ->
# 13.5 KB) while the tiling did not. Gating a number that the container's
# record lengths move would gate the container, not the policy;
# * every record accounted for: 120 READ(10)s, and the bytes into the ring
# EQUAL to the bytes off the disc -- both read out of the container rather
# than written here. Under DLX4 they differed by 1.34% because a record was
# not a sector (58.3); DLX5 aligns records to sectors and the covering-sector
# read disappears, so the gate is now their IDENTITY;
# * a real mid-stream SEEK with the real transport, in the second pass. This
# is the one path that could not exist before: ring_seek waits for the
# channel to go quiet, and with the transport INSIDE the machine the only
@@ -470,15 +474,34 @@ if command -v chdman > /dev/null; then
| sed "s/^ *//;s/^/ /"
grep -aq "TRANSPORT FAILED" tmp/p4b_check.log && {
echo "FAIL: a record's READ(10) reported an error."; exit 1; }
grep -aq "REAL TRANSPORT: 120 READ(10)s by the 68000, 4488588 B into the ring" \
# THE BYTE COUNTS COME OUT OF THE CONTAINER, not out of this file. They were
# two hardcoded constants fitted to the DLX4 gate container, and session 28's
# re-encode went red on both of them for the right reason -- the container had
# changed and the expectation had not. A gate whose expected value is a
# literal tests the literal.
EXPECT_B=$(python3 -c "
import sys; sys.path.insert(0, 'tools/encoder')
from dlx import DLX
print(sum(DLX('$DLX').record_lengths()))")
grep -aq "REAL TRANSPORT: 120 READ(10)s by the 68000, $EXPECT_B B into the ring" \
tmp/p4b_check.log || {
echo "FAIL: the 68000 did not fetch all 120 records, or did not fetch"
echo " 4,488,588 B of them. A short record is a desync, not a shortfall."
echo " $EXPECT_B B of them. A short record is a desync, not a shortfall."
exit 1; }
grep -aq "SECTOR OVERHEAD: 4548608 B off the disc" tmp/p4b_check.log || {
echo "FAIL: the bytes the DISC moved are no longer 4,548,608. A record is"
echo " not a sector; this is the covering-sector read, and if it moved"
echo " then either the layout or scsi.i's window did. See FINDINGS 58.3."
# DLX5 MAKES THESE THE SAME NUMBER, and that identity IS the finding (59.4,
# and 58.3 option C): a sector-aligned container has no covering-sector read,
# so the disc moves exactly the records and nothing else. Under DLX4 they
# differed by 1.34% and both were gated so neither could drift into the other;
# under DLX5 the gate is that they are EQUAL. If a windowed read ever came
# back -- a container that was not aligned, or a layout that lost the
# alignment -- the disc figure would exceed the ring figure and this goes red.
grep -aq "SECTOR OVERHEAD: $EXPECT_B B off the disc for $EXPECT_B B of record = 0.00%" \
tmp/p4b_check.log || {
echo "FAIL: the disc no longer moves EXACTLY the records. On a sector-"
echo " aligned container (DLX5) there is no covering-sector read at"
echo " all, so these two counts must be the same $EXPECT_B B. If they"
echo " differ, either the container lost its alignment or scsi.i is"
echo " windowing again -- and a DMA channel cannot window (59.4)."
exit 1; }
grep -aq "ring: 18 wraps" tmp/p4b_check.log || {
echo "FAIL: the placement policy tiled this container differently with a"
@@ -574,4 +597,58 @@ else
echo " (git clone --depth 1 https://github.com/icculus/DirkSimple)"
fi
echo "--- session 29: the packed paint, and what it does to the codec (FINDINGS 61) ---"
# tools/bench/blit.s gained V8/V9/V10 -- the packed full-frame blit, and the two
# ways a 4x4 BLOCK decoder could survive the packed layout. 47.6.1 had filed the
# packed paint's `movem` shape as an ASSUMPTION since session 16; this measures
# it, in the same run as V1/V3/V4 so it is quoted against numbers that have not
# moved since session 9.
#
# WHAT IS GATED IS STRUCTURAL, not numeric, for the reason the load stage gives:
# MAME samples these on a 1/56.69 s clock and no cost model in the tree depends
# on their exact value. What DOES depend on them is the ORDER, and the order is
# the whole of FINDINGS 61:
# V8 < V1 packing halves the full-frame literal paint
# V9 > V4 packing makes a BLOCK decoder DEARER, not cheaper
# V10 < V4 unless the blocks are paired, which costs SKIPs instead
# A tree where any of those flipped has a different answer to 44.7 and should
# say so out loud rather than let 29_packed_player.py narrate the old one.
python3 tools/bench/prep_frame.py tmp/fr_00020 tmp/frame256.bin 0 --reserve-black
rm -f tmp/blit_v8.log
( cd tmp && SDL_VIDEODRIVER=dummy stdbuf -oL timeout -k 5 300 mame x68000 \
-bios ipl10 -ramsize 2M -video soft -window -sound none -nothrottle -plugins \
-autoboot_script ../tools/bench/blit.lua -seconds_to_run 120 \
> blit_v8.log 2>&1 )
grep -aq "summary (instruction cycles only" tmp/blit_v8.log || {
echo "FAIL: the blit timing run produced no summary -- it did not finish."
tail -8 tmp/blit_v8.log; exit 1; }
python3 - <<'EOF' || exit 1
import re, sys
v = {}
for line in open("tmp/blit_v8.log", errors="replace"):
m = re.search(r"V(\d+)\s+(\d+) cyc", line)
if m: v[int(m.group(1))] = int(m.group(2))
need = (1, 2, 3, 4, 8, 9, 10)
missing = [n for n in need if n not in v]
if missing: sys.exit(f"FAIL: blit.lua reported no V{missing} -- run incomplete.")
for a, op, b, why in ((8, "<", 1, "packing did not halve the literal paint"),
(9, ">", 4, "packed BLOCK order came out CHEAPER than "
"unpacked -- 61.3's conclusion is inverted"),
(10, "<", 4, "pairing the blocks did not buy back the "
"movem shape")):
ok = v[a] < v[b] if op == "<" else v[a] > v[b]
if not ok:
sys.exit(f"FAIL: V{a} {v[a]:,} is not {op} V{b} {v[b]:,} -- {why}.")
print(f" V1 {v[1]:,} / V8 PACKED {v[8]:,} = {100*v[8]/v[1]:.0f}% -- "
f"and V3, the unpacked WRITE-ONLY floor, is {v[3]:,}")
print(f" V4 {v[4]:,} / V9 packed-block {v[9]:,} = {100*v[9]/v[4]:.0f}% -- "
f"packing costs a BLOCK decoder {100*v[9]/v[4]-100:.0f}%")
print(f" V10 paired blocks {v[10]:,} = {100*v[10]/v[4]:.0f}% of V4, and pairing "
f"is paid for in SKIPs")
EOF
python3 tools/analysis/29_packed_player.py "$DLX" > tmp/packed_player.log 2>&1 \
|| { tail -20 tmp/packed_player.log; exit 1; }
grep -aE "SKIP block PAIRS|free / DMAC->GVRAM / PACKED|^ CODEC, gate" \
tmp/packed_player.log
echo "ALL GREEN"
tools/bench/dma.lua
+87 -8
View File
@@ -25,17 +25,24 @@ local function P(s) print("[DMA] "..s) end
local function T() local t=M.time; return t.seconds + t.attoseconds/1e18 end
local DGFLAG, DGREC, DGREC_SZ = 0x18600, 0x18610, 32
local DGWIN, DGWERR = 0x18680, 0x18684
local DGWIN, DGWERR, DGR20, DGR20N, DGR20C = 0x18700, 0x18704, 0x18708, 0x1870C, 0x18710
local CHROW, CHN, CHBASE = 256, 8, 0xC10000
local R20OF -- filled in after the mailbox addresses are known
local GV = 0xC00000
local DGLBA, DGBLK = 1000, 4
local DST = {0x20000, 0x24000, 0x28000}
local DST = {0x20000, 0x24000, 0x28000, 0xC08000, 0xC0C000, 0xC10000}
local NAME = {"PIO (the path FINDINGS 58 measured)",
"DMA, BUS HELD (DCR $00 burst, OCR $81 max rate)",
"DMA, STEALING (DCR $80 cycle steal, OCR $80 limited)"}
local SHORT = {"pio", "held", "steal"}
"DMA, STEALING (DCR $80 cycle steal, OCR $80 limited)",
"DMA -> GVRAM (bus held, R20 bit 11 = BUFFER MODE) [47.6.2]",
"DMA -> GVRAM (the SAME, bit 11 CLEAR -- NEGATIVE CONTROL)",
"DMA -> GVRAM (ARRAY CHAINED, 8 rows at the 1024 B line stride)"}
local SHORT = {"pio", "held", "steal", "gvram", "masked", "chain"}
local ERRNAME = {[0]="OK", "SELECTION TIMEOUT -- no target answered",
"UNEXPECTED PHASE", "POLL TIMEOUT -- a phase never arrived",
"NON-ZERO SCSI STATUS",
"WINDOWED READ REFUSED -- a channel cannot drop bytes"}
R20OF = {[3]=DGR20, [4]=DGR20N, [5]=DGR20C}
local DISK = os.getenv("DLX_SCSI_IMG") or "dlxdisk.img"
local code do local f=io.open("dmagate.bin","rb"); code=f:read("a"); f:close() end
@@ -58,7 +65,8 @@ SUB = emu.add_machine_frame_notifier(function()
cpu.state["SP"].value = 0x8000
cpu.state["PC"].value = 0x10000
P(string.format("dmagate.bin=%d B loaded at $10000; reading LBA %d, %d B, "
.."three ways", #code, DGLBA, DGBLK*512))
.."three ways, then once more into GVRAM",
#code, DGLBA, DGBLK*512))
st = "wait"; return
end
if st == "wait" then
@@ -68,7 +76,7 @@ SUB = emu.add_machine_frame_notifier(function()
end
if not want then P("no "..DISK.." to check against"); P("done"); M:exit(); return end
local LEN = DGBLK*512
for i = 0, 2 do
for i = 0, 5 do
local b = DGREC + i*DGREC_SZ
local rc = SP:read_u32(b)
local e = SP:read_u32(b+4)
@@ -83,14 +91,85 @@ SUB = emu.add_machine_frame_notifier(function()
P(string.format(" FAILED: err=%d (%s)", e, ERRNAME[e] or "?"))
else
local bad, first = 0, nil
-- The GVRAM run is read back a WORD at a time and split by hand.
-- SP:read_u8 on $C00000 goes through gvram_r, which in buffer mode
-- returns the whole word; asking for one byte of it would hand back
-- whichever half MAME's address space happens to hand over, and the
-- question here is precisely WHICH HALF each disc byte landed in.
-- Even disc byte -> high half (page 1), odd -> low half (page 0),
-- because the 68000 is big-endian and an even address is the MS byte.
local pg1, pg0, bad_hi, bad_lo = 0, 0, 0, 0
for k = 1, LEN do
if SP:read_u8(DST[i+1]+k-1) ~= string.byte(want, k) then
bad = bad + 1; first = first or (k-1)
local got
if i == 5 then
-- The chained run's destination is not linear: byte k of the
-- transfer is byte k%256 of row k//256, and the rows are a full
-- 1024 B line stride apart. If the channel had ignored the array
-- and run contiguously, every byte past the first row would be
-- in the wrong place and this comparison would say so.
local off = (k-1) % CHROW
local a = CHBASE + ((k-1) // CHROW) * 1024 + (off & ~1)
local w = SP:read_u16(a)
if (off % 2) == 0 then got = (w >> 8) & 0xff; pg1 = pg1 + 1
else got = w & 0xff; pg0 = pg0 + 1 end
elseif i >= 3 then
local w = SP:read_u16(DST[i+1] + ((k-1) & ~1))
if ((k-1) % 2) == 0 then got = (w >> 8) & 0xff; pg1 = pg1 + 1
else got = w & 0xff; pg0 = pg0 + 1 end
else
got = SP:read_u8(DST[i+1]+k-1)
end
if got ~= string.byte(want, k) then
bad = bad + 1; first = first or (k-1)
if ((k-1) % 2) == 0 then bad_hi = bad_hi + 1
else bad_lo = bad_lo + 1 end
end
end
if i >= 3 then
P(string.format(" R20 during the run = $%04X (bit 11 %s); %d bytes "
.."read back as page 1 (high half) and %d as page 0",
SP:read_u32(R20OF[i]),
((SP:read_u32(R20OF[i]) & 0x0800) ~= 0)
and "SET" or "CLEAR",
pg1, pg0))
end
if bad == 0 then
P(string.format(" BYTES OK: %d B from LBA %d match %s byte for byte "
.."[%s]", LEN, DGLBA, DISK, SHORT[i+1]))
if i == 3 then
P(" A CHANNEL FILLS THE PACKED LAYOUT: every disc byte landed in "
.."its own half of a GVRAM word, with the CPU halted -- so a "
.."stream interleaved (right<<8)|left goes from disc to screen "
.."with no CPU in the loop (47.6.2, first half).")
end
if i == 5 then
P(string.format(" THE CHANNEL WALKED THE ARRAY ITSELF: %d rows of "
.."%d B landed at a %d B line stride from ONE start, CPU halted "
.."throughout. A frame is %d such entries; the CPU does not "
.."restart the channel per row.", CHN, CHROW, 1024, 192))
end
if i == 4 then
P(" CONTROL DID NOT FAIL: the masked write path delivered every "
.."byte too, so the run above is not evidence about R20 bit 11.")
end
elseif i == 4 then
-- THE CLAIM IS NOT "half the bytes differ". In masked 256-colour
-- mode gvram_w takes `data & 0x00ff` and ignores mem_mask, so a byte
-- written to an EVEN address is never stored and the high half keeps
-- whatever it held; some of those stale halves match the disc by
-- coincidence, and this record is full of pad, so a lot of them do.
-- The mechanism's signature is WHERE the damage is, not how much:
-- every ODD byte must survive and only EVEN ones may be lost.
P(string.format(" BYTES LOST [masked]: %d of %d differ (first at "
.."+%d) -- %d at EVEN offsets, %d at ODD.",
bad, LEN, first, bad_hi, bad_lo))
if bad_lo == 0 and bad_hi > 0 then
P(string.format(" EXACTLY THE MECHANISM: all %d survivors of the "
.."high half are stale GVRAM that happens to match "
.."(this record is mostly pad); not one of the %d "
.."ODD bytes was harmed. Bit 11 is what carried the "
.."even ones in the run above.", LEN//2 - bad_hi, LEN//2))
end
else
P(string.format(" BYTES WRONG [%s]: %d of %d differ, first at +%d",
SHORT[i+1], bad, LEN, first))
tools/bench/dma_run.sh
+32
View File
@@ -76,6 +76,38 @@ grep -aq "COC .*CER=\$00 MTC=0 .*(+2048) \[held\]" tmp/dma_run.log || \
fail "the held channel did not report a clean completion of every byte."
grep -aq "COC .*CER=\$00 MTC=0 .*(+2048) \[steal\]" tmp/dma_run.log || \
fail "the stealing channel did not report a clean completion of every byte."
# ---- the GVRAM run and its control (47.6.2). A channel that writes GVRAM in
# buffer mode is the decoder-free packed player's entire per-frame path, and a
# run with no control is 58.3's vacuous "UNDERRUNS: 0/120" again -- the IPL
# leaves R20 = $0B16, bit 11 ALREADY SET, so the first cut of this test could
# not have failed.
grep -aq "BYTES OK: 2048 B from LBA 1000 .*\[gvram\]" tmp/dma_run.log || \
fail "the channel did not fill GVRAM in buffer mode -- a device->GVRAM
transfer is the whole of the decoder-free packed player's frame."
grep -aq "R20 during the run = \$0916 (bit 11 SET)" tmp/dma_run.log || \
fail "the GVRAM run did not run in buffer mode with a KNOWN R20."
grep -aq "R20 during the run = \$0116 (bit 11 CLEAR)" tmp/dma_run.log || \
fail "the negative control did not run with bit 11 clear."
if grep -aq "BYTES OK: 2048 B from LBA 1000 .*\[masked\]" tmp/dma_run.log
then
fail "the MASKED control delivered every byte, so the run above is not a
measurement of R20 bit 11 -- it is a measurement of nothing."
fi
grep -aq "EXACTLY THE MECHANISM" tmp/dma_run.log || \
fail "the masked control lost bytes at ODD offsets too, or lost none at all.
The claim is not a COUNT -- stale GVRAM matches the disc by coincidence
wherever the record is pad -- it is a PLACE: gvram_w's 256-colour arm
drops what the channel wrote to EVEN addresses and stores what it wrote
to odd ones. Damage anywhere else is a different mechanism."
grep -aq "BYTES OK: 2048 B from LBA 1000 .*\[chain\]" tmp/dma_run.log || \
fail "the array-chained run did not put the bytes at the row bases its array
named. A picture row is 256 B of a 1024 B line stride, so a frame is 192
destinations; if the channel cannot walk them the CPU has to restart it
per row and the decoder-free path costs a per-row front end."
grep -aq "THE CHANNEL WALKED THE ARRAY ITSELF" tmp/dma_run.log || \
fail "the chained run did not report walking its own array."
grep -aq "WINDOWED DMA READ REFUSED" tmp/dma_run.log || \
fail "a WINDOWED read through the channel was not refused. 117 of 120 records
start part way into a sector (58.3), and a channel cannot drop the bytes
tools/bench/mkvol.sh
+15 -1
View File
@@ -31,7 +31,21 @@ open("tmp/dlxdisk.img", "wb").write(d + b"\0" * (n * 512 - len(d)))
print(f" disc image: {len(d)} B of records -> {n} sectors")
PY
rm -f tmp/dlxdisk.chd
chdman createhd -i tmp/dlxdisk.img -o tmp/dlxdisk.chd -ss 512 > /dev/null 2>&1
# -c none IS LOAD-BEARING, and it was found by a gate rather than by taste.
# Session 28, on the DLX5 volume: with the default (lzma/zlib/huff/flac) MAME
# 0.277 served the CHD FILE'S OWN BYTES as sector data -- the destination
# buffer after READ(10) at LBA 0 was byte-for-byte the first 4,096 bytes of
# dlxdisk.chd, starting "MComprHD" -- while `chdman verify` reported both SHA1s
# correct. Uncompressed, the identical image reads byte-exact. The trigger is
# the image's CONTENT: the same 8,768-sector length that works for the DLX4
# volume fails for the DLX5 one, a conventional 16x63 geometry fails too, and
# `-c zlib` alone fails as well. The MAME-side cause is NOT diagnosed; what is
# measured is that compression decides it and uncompressed is sound.
# Costs 4.5 MB in tmp/ against 1.6 MB. DO NOT restore compression to save the
# disc space: the failure is SILENT at the transport layer -- every READ(10)
# reports success and returns the wrong bytes -- and only the byte comparison
# in tools/bench/scsi.lua catches it.
chdman createhd -i tmp/dlxdisk.img -o tmp/dlxdisk.chd -ss 512 -c none > /dev/null 2>&1
fi
mkdir -p tmp/p4roms/x68k_cz6bs1
tools/bench/prep_stream.py
+4 -1
View File
@@ -70,7 +70,10 @@ disk, index = bytearray(), []
for (o, n) in d.frames:
start = len(disk)
disk += n.to_bytes(4, "big") + d.raw[o:o + n]
while len(disk) % 4:
# The container's own alignment rule, not this script's copy of it: DLX5
# pads to 512 so a DMA channel can read whole sectors into the ring, DLX4
# to 4 so `move.l (a0)+` does not take an address error (28.3).
while len(disk) % d.rec_align:
disk += b"\0"
index.append((start, len(disk) - start))
open(a.out + "_disk.bin", "wb").write(bytes(disk))
tools/encoder/dlx.py
+44 -10
View File
@@ -29,6 +29,13 @@ import spans as SP
MODE_SKIP, MODE_V1, MODE_V4, MODE_RAW = 0, 1, 2, 3
# A SCSI target answers in 512-byte blocks and a record is not a sector: on the
# DLX4 gate container 117 of 120 records start part way into one. DLX5 makes
# the container agree with the medium instead of making the transport reconcile
# them (tools/analysis/26_sector_align.py prices all three ways).
SECTOR = 512
class DLX:
def __init__(self, path):
self.raw = open(path, "rb").read()
@@ -38,12 +45,21 @@ class DLX:
# (FINDINGS 28.3), so the padding is part of the format, not a loader
# convenience -- but DLX1 containers stay readable, because every
# measurement in FINDINGS 28-31 was taken on one.
if b[:4] not in (b"DLX1", b"DLX2", b"DLX3", b"DLX4"):
if b[:4] not in (b"DLX1", b"DLX2", b"DLX3", b"DLX4", b"DLX5"):
raise ValueError(f"{path}: not a DLX container")
self.version = int(b[3:4])
self.aligned = self.version >= 2
self.has_spans = self.version >= 3
self.has_index = self.version >= 4
# DLX5: every record starts on a 512-BYTE SECTOR boundary, and so does
# the frame stream itself. That is not a tidier version of DLX4's
# 4-byte rule -- it is what lets a DMA channel read a record as whole
# sectors straight into the ring, with no window and no bounce copy
# (FINDINGS 58.3 option C, and 59.4 made it a precondition: sc_in_data
# REFUSES a windowed read when the data phase is the channel's).
self.sector_aligned = self.version >= 5
self.rec_align = SECTOR if self.sector_aligned else (4 if self.aligned
else 1)
(self.W, self.H, self.fps, self.nframes,
self.k1, self.k4) = struct.unpack(">HHHHHH", b[4:16])
off_pal, off_cb1, off_cb4, off_frm = struct.unpack(">IIII", b[16:32])
@@ -61,21 +77,24 @@ class DLX:
self.mode_bytes = (self.nb * 2 + 7) // 8
# frame directory: (offset of the mode header, payload length)
if self.aligned and off_frm % 4:
raise ValueError(f"{path}: DLX2 frame stream starts at {off_frm}, "
f"which is not 4-byte aligned")
if off_frm % self.rec_align:
raise ValueError(f"{path}: DLX{self.version} frame stream starts at "
f"{off_frm}, which is not {self.rec_align}-byte "
f"aligned")
self.frames = []
p = off_frm
for _ in range(self.nframes):
(n,) = struct.unpack(">I", b[p:p + 4])
self.frames.append((p + 4, n))
p += 4 + n
if self.aligned:
p += -p % 4 # skip the pad to the next record
# The writer does not pad after the LAST record -- nothing follows it --
# so `p` may have advanced past the end by up to 3 bytes there.
p += -p % self.rec_align # skip the pad to the next record
# DLX2/DLX3 do not pad after the LAST record -- nothing follows it --
# so `p` may have advanced past the end by up to 3 bytes there. DLX4
# and DLX5 DO pad it, because a producer that trusts the index fetches
# a whole padded record for the last frame like any other.
slack = len(b) - p
if not (slack == 0 or (self.aligned and -3 <= slack < 0)):
if not (slack == 0 or (self.aligned and not self.has_index
and -(self.rec_align - 1) <= slack < 0)):
raise ValueError(f"{path}: {slack} trailing bytes after "
f"{self.nframes} frames")
@@ -92,7 +111,7 @@ class DLX:
if self.has_index:
self.index = list(struct.unpack(
f">{self.nframes}H", b[off_idx:off_idx + 2 * self.nframes]))
walked = [(-(4 + n) % 4 + 4 + n) // 4 for _, n in self.frames]
walked = [self._padded(n) // 4 for _, n in self.frames]
if self.index != walked:
bad = next(i for i in range(self.nframes)
if self.index[i] != walked[i])
@@ -106,6 +125,21 @@ class DLX:
f"{off_frm + 4 * sum(self.index)} bytes and the file is "
f"{len(b)} -- a producer trusting it would run off the end")
def _padded(self, n):
"""Bytes one record of `n` payload bytes occupies, pad included."""
ln = 4 + n
return ln + (-ln % self.rec_align)
def record_lengths(self):
"""Padded record lengths in BYTES, in stream order.
The one place the container's alignment rule is applied. Every caller
that used to write `4 + n + (-(4+n) % 4)` was carrying its own copy of
that rule, which is exactly the kind of duplication that made DLX5 a
multi-file change instead of a one-line one.
"""
return [self._padded(n) for _, n in self.frames]
def modes(self, f):
o, _ = self.frames[f]
h = np.frombuffer(self.raw, np.uint8, self.mode_bytes, o)
tools/encoder/encode.py
+41 -7
View File
@@ -66,6 +66,7 @@ import argparse, struct, sys, os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import numpy as np
import vq as VQ, vq_hybrid as H, ratectl as RC, spans as SP
from dlx import SECTOR
# Measured on the emulated 68000, FINDINGS 24. Instruction cycles against
# zero-wait-state memory, so these are floors, not hardware predictions.
@@ -134,8 +135,12 @@ def build_records(m, enc, span_mode):
return out
def write_container(path, m, frames, fps, k1, k4, span_mode):
"""Write the whole container. Returns (total bytes, video bytes, pad)."""
def write_container(path, m, frames, fps, k1, k4, span_mode, sector=True):
"""Write the whole container. Returns (total bytes, video bytes, pad).
`sector` selects DLX5's 512-byte record alignment over DLX4's 4-byte one.
"""
align = SECTOR if sector else 4
palette = m["pal"][:256]
if len(palette) < 256:
palette = np.vstack([palette, np.zeros((256 - len(palette), 3), np.uint8)])
@@ -144,6 +149,9 @@ def write_container(path, m, frames, fps, k1, k4, span_mode):
cb4_b = m["cb4"].astype(np.uint8).tobytes()
off_pal = 36 if span_mode else 32
if not span_mode:
sector = False # DLX2 has no index and no sector rule
align = 4
off_cb1 = off_pal + len(pal_b)
off_cb4 = off_cb1 + len(cb1_b)
# DLX4: the record index sits with the palette and the codebooks, ahead of
@@ -154,7 +162,7 @@ def write_container(path, m, frames, fps, k1, k4, span_mode):
qlens = []
for i, rec in enumerate(frames):
n = 4 + len(rec)
n += -n % 4
n += -n % align
q = n // 4
assert q <= 0xFFFF, (f"frame {i} is {n} B: a u16 longword count "
f"caps a record at 262,140 B")
@@ -169,9 +177,15 @@ def write_container(path, m, frames, fps, k1, k4, span_mode):
# a 68000, not a slow read. It vectors into the IPL and looks exactly like
# an infinite loop (FINDINGS 28.3). tools/bench/prep_dlx.py has been
# realigning at load time; the container now carries it.
tbl_pad = -off_frm % 4
# DLX5 aligns the frame stream itself as well as the records inside it, so
# the whole container can be laid on a volume at a sector boundary and every
# record lands on one. Aligning the records to each other and not the run
# they sit in would leave 117 of 120 of them off-sector again the moment the
# scene header changed length by a byte.
tbl_pad = -off_frm % align
off_frm += tbl_pad
hdr = ((b"DLX4" if span_mode else b"DLX2")
magic = (b"DLX5" if sector else b"DLX4") if span_mode else b"DLX2"
hdr = (magic
+ struct.pack(">HHHHHH", m["W"], m["H"], fps, len(frames), k1, k4)
+ struct.pack(">IIII", off_pal, off_cb1, off_cb4, off_frm))
if span_mode:
@@ -190,7 +204,7 @@ def write_container(path, m, frames, fps, k1, k4, span_mode):
# record is padded too, and the file ends where the index says it
# does rather than up to 3 bytes short of it.
if span_mode or i + 1 < len(frames):
n = -(4 + len(rec)) % 4
n = -(4 + len(rec)) % align
fh.write(b"\0" * n); frm_pad += n
total = os.path.getsize(path)
return total, sum(len(r) + 4 for r in frames) + frm_pad, frm_pad
@@ -252,6 +266,13 @@ def main():
"Default OFF: measured, it is worth one frame of 120 "
"at --spans need and a 2%% regression at --spans all "
"(FINDINGS 44)")
ap.add_argument("--joint-spans", action="store_true",
help="re-run the lam search with the bytes the span pass "
"freed, then re-span (E3, FINDINGS 39.3 item 5). The "
"span pass removes the block payload of every block "
"it covers, so without this the frame lands under its "
"allowance and the blocks that were NOT spanned were "
"priced as if those bytes were still needed.")
ap.add_argument("--no-cpu-fit", action="store_true",
help="drop the per-frame 68000 decode ceiling (session 7 "
"behaviour: 31%% of frames on hard content do not fit)")
@@ -259,6 +280,13 @@ def main():
help="how full the player's buffer is assumed to be at "
"scene start, as a fraction of the bucket (0 = cold "
"buffer after a seek, the conservative assumption)")
ap.add_argument("--no-reserve-black", action="store_true",
help="let the scene palette spend all 256 entries on the "
"picture. The default RESERVES index 0 as true black "
"(FINDINGS 23.4), because GVRAM cleared to zero shows "
"entry 0 and the 256x192 picture sits in a 256x256 "
"mode -- so a free palette letterboxes the frame in "
"whatever colour mediancut happened to put first.")
ap.add_argument("--preview")
a = ap.parse_args()
@@ -279,6 +307,7 @@ def main():
if a.disk_clk_byte is not None:
RC.DISK_CLK_BYTE = a.disk_clk_byte
RC.JOINT_DECIDE = a.joint_decide
RC.JOINT_SPANS = a.joint_spans
RC.JOINT_BUCKET = a.joint_bucket
cyc_budget = None if a.no_cpu_fit else RC.FRAME_CYCLES
span_mode = None if (a.spans == "off" or not rc) else a.spans
@@ -301,8 +330,13 @@ def main():
else:
print(f" target {prof['kbps']} KB/s, FIXED lam={lam} (no rate control)")
print(f" k1={k1} k4={k4}, {_IDX_BYTES}-byte indices")
print(f" palette: " + ("255 picture colours, index 0 RESERVED as true "
"black for the letterbox (23.4)" if not a.no_reserve_black
else "all 256 entries to the picture (--no-reserve-black); index 0 "
"is whatever mediancut put there, and the letterbox with it"))
m = H.build(a.frames_dir, k1=k1, k4=k4, iters=a.iters)
m = H.build(a.frames_dir, k1=k1, k4=k4, iters=a.iters,
reserve_black=not a.no_reserve_black)
if rc:
enc = RC.encode_rate_controlled(m, prof["kbps"], fps=a.fps,
bucket_frames=a.bucket_frames,
tools/encoder/ratectl.py
+59
View File
@@ -157,6 +157,23 @@ JOINT_DECIDE = False
# `--joint-bucket` turns it on.
JOINT_BUCKET = False
# E3 / FINDINGS 39.3 item 5: SPAN SELECTION IS GREEDY AFTER `lam`, and this is
# the switch that makes the two joint. The lam bisection picks a mode map
# against a byte allowance, and the span pass then REMOVES the block payload of
# every block it covers -- so the frame lands under the allowance by exactly
# the bytes the spans freed, and the blocks that were NOT spanned were priced
# at a lam chosen as if those bytes were still needed. Joint mode hands the
# freed bytes back to the lam search and re-spans the result, to a fixed point
# or two rounds, whichever comes first.
#
# It is a REFINEMENT, not a different objective: lam can only fall (the
# allowance only grows), so the un-spanned blocks can only improve, and a round
# is kept only if the frame still fits both ceilings it was already fitting.
# Default OFF until measured, which is 44.3's lesson -- ask whether the lever
# is loaded before pulling it.
JOINT_SPANS = False
JOINT_SPAN_ROUNDS = 2
def _byte_clk():
"""The debit the mode decision is allowed to see (0 = the old decision)."""
@@ -287,6 +304,39 @@ def _fit_spans(m, ctx, mode, sz, room, cyc_budget, span_mode, ib):
return nmode, nsz, H.cycles(nmode) + sel["clocks"], sel
def _refit_joint(m, ctx, allow, span_allow, lam_lo, lam_hi, cyc_budget,
span_mode, ib, mode_pre, mode, sz, cyc, sel, mu=0.0):
"""Give the lam search back the bytes the span pass freed, then re-span.
`mode_pre` is the mode map BEFORE spanning and `mode` the one after, so the
difference in frame_bytes is exactly what the spans made unnecessary. The
ceiling the result is judged against is the one _fit_spans was already
working to, so a kept round is never a frame that grew past a budget it was
inside.
"""
ceiling = span_allow if span_allow is not None else allow
for _ in range(JOINT_SPAN_ROUNDS):
if sel is None:
break
freed = (H.frame_bytes(mode_pre, ctx["nb"], ib)
- H.frame_bytes(mode, ctx["nb"], ib))
if freed <= 0:
break
lam2, mode2, sz2, _ = _search_lam(ctx, allow + freed, lam_lo, lam_hi, mu=mu)
if sz2 <= H.frame_bytes(mode_pre, ctx["nb"], ib):
break # lam did not move: already at the floor
n_pre, n_mode, n_sz, n_cyc, n_sel = (
mode2, *_fit_spans(m, ctx, mode2, sz2, span_allow if span_allow
is not None else allow, cyc_budget, span_mode, ib))
if n_sel is None or n_sz > ceiling:
break
if cyc_budget is not None and n_cyc + DISK_CLK_BYTE * n_sz > cyc_budget \
and cyc + DISK_CLK_BYTE * sz <= cyc_budget:
break # round 1 made the deadline and this does not
mode_pre, mode, sz, cyc, sel = n_pre, n_mode, n_sz, n_cyc, n_sel
return mode_pre, mode, sz, cyc, sel
def encode_rate_controlled(m, target_kbps, fps=12, bucket_frames=8,
lam_lo=1.0, lam_hi=LAM_CLIFF, prefill=0.0,
steps=None, verbose=False, cycle_budget=None,
@@ -364,6 +414,10 @@ def encode_rate_controlled(m, target_kbps, fps=12, bucket_frames=8,
mode_pre = mode
mode, sz, cyc, sel = _fit_spans(m, ctx, mode, sz, span_allow,
cycle_budget, span_mode, ib)
if JOINT_SPANS:
mode_pre, mode, sz, cyc, sel = _refit_joint(
m, ctx, allow, span_budget and span_allow, lam_lo, lam_hi,
cycle_budget, span_mode, ib, mode_pre, mode, sz, cyc, sel)
if cycle_budget is not None and cyc + DISK_CLK_BYTE * sz > cycle_budget:
# The byte allowance could not buy the frame's deadline, so fall
# back to the controller that pays in picture -- and then offer
@@ -374,6 +428,11 @@ def encode_rate_controlled(m, target_kbps, fps=12, bucket_frames=8,
mode_pre = mode
mode, sz, cyc, sel = _fit_spans(m, ctx, mode, sz, span_allow,
cycle_budget, span_mode, ib)
if JOINT_SPANS:
mode_pre, mode, sz, cyc, sel = _refit_joint(
m, ctx, allow, span_budget and span_allow, lam_lo,
lam_hi, cycle_budget, span_mode, ib, mode_pre, mode,
sz, cyc, sel, mu=mu)
late = cyc + DISK_CLK_BYTE * sz > cycle_budget
# Paint from the mode map as it was BEFORE spanning. A spanned run's
# blocks read SKIP in the emitted header, but SKIP means "hold the
tools/encoder/vq.py
+33 -5
View File
@@ -28,12 +28,40 @@ def load_frames(d):
return [np.asarray(Image.open(f).convert("RGB")) for f in fs]
def scene_palette(rgb, colors=256, stride=3):
"""One shared palette for the whole scene, no dithering (cel art is flat)."""
def scene_palette(rgb, colors=256, stride=3, reserve_black=True):
"""One shared palette for the whole scene, no dithering (cel art is flat).
`reserve_black` puts TRUE BLACK at index 0 and quantises the picture into
the other 255 entries. It is not a cosmetic default (FINDINGS 23.4): the
picture is 256x192 inside a 256x256 mode, GVRAM cleared to zero displays
palette entry 0, and a free mediancut palette puts a real image colour
there -- on 00020 f0001 it was (206,192,176), used by 210 image pixels, so
the 64 blank rows of letterbox came out beige. Entry 0 also needs `I = 0`
in the X68000's GRB555 word or the bars sit at RGB (4,4,4) (23.3); that
half is `dlxload.pack_palette`'s and it needs no special case, because a
(0,0,0) entry picks I=0 by its own minimum-squared-error rule.
Black is RESERVED, not withheld: the mapper may still spend index 0 on
genuinely black pixels, which is the entry it would have wanted anyway.
What the reservation buys is that index 0 is black REGARDLESS of what the
scene contains, which is what the letterbox needs and what a free palette
cannot promise.
"""
samp = np.concatenate([r.reshape(-1, 3) for r in rgb[::stride]])
ref = Image.fromarray(samp.reshape(-1, 1, 3)).quantize(
colors=colors, method=Image.MEDIANCUT, dither=Image.NONE)
pal = np.array(ref.getpalette()[:colors * 3], dtype=np.uint8).reshape(-1, 3)
n = colors - 1 if reserve_black else colors
q = Image.fromarray(samp.reshape(-1, 1, 3)).quantize(
colors=n, method=Image.MEDIANCUT, dither=Image.NONE)
pal = np.array(q.getpalette()[:n * 3], dtype=np.uint8).reshape(-1, 3)
if not reserve_black:
return q, pal
pal = np.vstack([np.zeros((1, 3), np.uint8), pal])
# The quantiser above cannot be reused as the mapping reference: its
# palette is the 255 it chose, at the wrong indices. A P-mode image
# carrying the FINAL palette is what every frame is then mapped against,
# so the indices in the container and the entries in the container's
# palette section are the same table by construction.
ref = Image.new("P", (1, 1))
ref.putpalette(pal.tobytes().ljust(768, b"\0"))
return ref, pal
tools/encoder/vq_hybrid.py
+2 -2
View File
@@ -113,10 +113,10 @@ def blocks_of(idx, pal, bw, bh):
return VQ.blockify(idx, pal, bw, bh)
def build(frames_dir, k1=256, k4=256, iters=16, lam=0.0):
def build(frames_dir, k1=256, k4=256, iters=16, lam=0.0, reserve_black=True):
rgb = VQ.load_frames(frames_dir)
H, W = rgb[0].shape[:2]
ref, pal = VQ.scene_palette(rgb)
ref, pal = VQ.scene_palette(rgb, reserve_black=reserve_black)
idx = VQ.palettise(rgb, ref)
# --- two codebooks, trained on the whole scene ---