Session 2: hybrid VQ codec, two quality profiles, three corrections
Answers session 1's critical-path question. Flat 4x4 VQ at k=256 was prototyped and REJECTED by eye: Dirk's face disintegrates and ink outlines break into 4-pixel stair-steps. The 256-colour palettised frame is excellent, so the palette was never the problem -- block VQ was. Replaced it with a Cinepak-style hybrid: each 4x4 block is SKIP, one 4x4 codeword, four 2x2 codewords, or RAW literal pixels, chosen per block by rate-distortion. The RAW escape makes lam=0 pixel-exact (measured 0.00 dB loss), so the quality knob spans lossless to heavily-compressed in one bitstream. Per the user's decision, ships TWO quality profiles from that one codec, one decoder and one bitstream -- only the rate knob differs: sasi 45 KB/s lam=300 34.8 dB stock 10MHz ACE/EXPERT scsi 75 KB/s lam=100 35.9 dB Super/XVI or CZ-6BS1 Three corrections to earlier numbers: 1. Session 1's "183 KB/s at 12fps" was a bad extrapolation. Halving the framerate does not halve the bitrate -- decimation roughly doubles the per-frame delta. Re-measured directly: 340 KB/s for session 1's own RLE, 247 KB/s for changed-spans+deflate. The lossless floor is 319 MB. 2. A FOURTH false-good result, same family as the three in FINDINGS 4: k=1024 codebooks appeared to buy +2.4 dB free, because the rate model charged 1 byte for a 10-bit index. Charging the true cost reverses the verdict -- k=256 wins at every matched bitrate, and by 5 dB at the low end where the SASI profile lives. k=256 ships. 3. Stream inventory: the ~3-5MB clips are 1.2-1.7s, not ~60s, and some 60s streams are menus, not content. Any survey must classify before averaging. Also cleared both candidate sources for the game-logic layer: the SNES project is MIT and DirkSimple is zlib, so the arcade scene graph can be imported and the two transcriptions diffed against each other. Encoder is working end-to-end: extract.py -> vq/vq_hybrid/ratectl -> encode.py, emitting a big-endian DLX1 container the 68000 can parse with plain moves. Claude-Session: https://claude.ai/code/session_01194oWYW8DQXK1SZ2DnChW6
This commit is contained in:
prosolis
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Encode one scene to the DLX bitstream, at a chosen quality profile.
|
||||
|
||||
python3 tools/encoder/encode.py <frames_dir> <out.dlx> [--profile sasi|scsi]
|
||||
[--lam N] [--fps 12] [--preview out.png]
|
||||
|
||||
Container (little-endian is WRONG here -- the 68000 is big-endian, so every
|
||||
multi-byte field is big-endian and the decoder can read it with a plain move.w):
|
||||
|
||||
header, 32 bytes
|
||||
0 'DLX1' magic
|
||||
4 u16 width, u16 height
|
||||
8 u16 fps, u16 nframes
|
||||
12 u16 k1, u16 k4 codebook sizes
|
||||
16 u32 palette offset (256 * 3 bytes, RGB888 -- the player converts
|
||||
to the X68000's GRB555 at load time)
|
||||
20 u32 cb1 offset (k1 * 16 bytes of palette indices)
|
||||
24 u32 cb4 offset (k4 * 4 bytes)
|
||||
28 u32 frames offset
|
||||
then, per frame:
|
||||
u32 payload length, then
|
||||
ceil(nblocks*2/8) bytes of 2-bit mode headers, MSB-first, block raster order
|
||||
then payloads in block order: V1 -> 1 byte, V4 -> 4 bytes, RAW -> 16 bytes
|
||||
|
||||
Codebooks are emitted as palette INDICES, not pixels. The player expands them
|
||||
once at load time into word-per-pixel form so the blitter can movem them
|
||||
straight into GVRAM -- k1=1024 costs 1024*16*2 = 32 KB of the 2 MB.
|
||||
"""
|
||||
import argparse, struct, sys, os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import numpy as np
|
||||
import vq as VQ, vq_hybrid as H, ratectl as RC
|
||||
|
||||
|
||||
def pack_modes(mode):
|
||||
"""2 bits per block, MSB-first -- cheap for the 68000 to shift out."""
|
||||
n = len(mode)
|
||||
out = bytearray((n * 2 + 7) // 8)
|
||||
for i, m in enumerate(mode):
|
||||
out[i // 4] |= (int(m) & 3) << (6 - 2 * (i % 4))
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def frame_payload(mode, l1, l4g, src_idx, nbx):
|
||||
body = bytearray()
|
||||
for b, mo in enumerate(mode):
|
||||
if mo == 1:
|
||||
body += _idx(l1[b])
|
||||
elif mo == 2:
|
||||
for j in range(4):
|
||||
body += _idx(l4g[b][j])
|
||||
elif mo == 3:
|
||||
by, bx = divmod(b, nbx)
|
||||
body += src_idx[by*4:by*4+4, bx*4:bx*4+4].tobytes()
|
||||
return bytes(body)
|
||||
|
||||
|
||||
def _idx(v):
|
||||
"""codebook index: 1 byte if it fits, else big-endian u16.
|
||||
k>256 means 2-byte indices -- decided once by the header, not per block."""
|
||||
v = int(v)
|
||||
return bytes([v]) if _IDX_BYTES == 1 else struct.pack(">H", v)
|
||||
|
||||
|
||||
_IDX_BYTES = 1
|
||||
|
||||
|
||||
def main():
|
||||
global _IDX_BYTES
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("frames_dir"); ap.add_argument("out")
|
||||
ap.add_argument("--profile", choices=list(RC.PROFILES), default="sasi")
|
||||
ap.add_argument("--lam", type=float, default=None)
|
||||
ap.add_argument("--fps", type=int, default=12)
|
||||
ap.add_argument("--iters", type=int, default=16)
|
||||
ap.add_argument("--preview")
|
||||
a = ap.parse_args()
|
||||
|
||||
prof = RC.PROFILES[a.profile]
|
||||
lam = a.lam if a.lam is not None else prof["lam"]
|
||||
k1, k4 = prof["k1"], prof["k4"]
|
||||
_IDX_BYTES = 1 if max(k1, k4) <= 256 else 2
|
||||
|
||||
print(f"profile {a.profile}: {prof['desc']}")
|
||||
print(f" target {prof['kbps']} KB/s, lam={lam}, k1={k1} k4={k4}, "
|
||||
f"{_IDX_BYTES}-byte indices")
|
||||
|
||||
m = H.build(a.frames_dir, k1=k1, k4=k4, iters=a.iters)
|
||||
enc = H.encode(m, lam=lam)
|
||||
r = H.evaluate(m, enc, fps=a.fps)
|
||||
|
||||
H_, W_ = m["H"], m["W"]; nbx = W_ // 4
|
||||
pal, idx = m["pal"], m["idx"]
|
||||
|
||||
# re-derive the per-frame symbols the same way encode() did
|
||||
frames = []
|
||||
for f, im in enumerate(idx):
|
||||
B1 = H.blocks_of(im, pal, 4, 4); l1 = VQ.assign(B1, m["C1s"])
|
||||
B4 = H.blocks_of(im, pal, 2, 2); l4 = VQ.assign(B4, m["C4s"])
|
||||
q = H._group_2x2_into_4x4(np.arange(len(l4)), W_)
|
||||
l4g = l4[q].reshape(-1, 4)
|
||||
mode = enc["modes"][f]
|
||||
frames.append(pack_modes(mode) + frame_payload(mode, l1, l4g, im, nbx))
|
||||
|
||||
palette = m["pal"][:256]
|
||||
if len(palette) < 256:
|
||||
palette = np.vstack([palette, np.zeros((256 - len(palette), 3), np.uint8)])
|
||||
pal_b = palette.astype(np.uint8).tobytes()
|
||||
cb1_b = m["cb1"].astype(np.uint8).tobytes()
|
||||
cb4_b = m["cb4"].astype(np.uint8).tobytes()
|
||||
|
||||
off_pal = 32
|
||||
off_cb1 = off_pal + len(pal_b)
|
||||
off_cb4 = off_cb1 + len(cb1_b)
|
||||
off_frm = off_cb4 + len(cb4_b)
|
||||
hdr = (b"DLX1" + struct.pack(">HHHHHH", W_, H_, a.fps, len(idx), k1, k4)
|
||||
+ struct.pack(">IIII", off_pal, off_cb1, off_cb4, off_frm))
|
||||
assert len(hdr) == 32, len(hdr)
|
||||
|
||||
with open(a.out, "wb") as fh:
|
||||
fh.write(hdr); fh.write(pal_b); fh.write(cb1_b); fh.write(cb4_b)
|
||||
for p in frames:
|
||||
fh.write(struct.pack(">I", len(p))); fh.write(p)
|
||||
|
||||
total = os.path.getsize(a.out)
|
||||
vid = sum(len(p) + 4 for p in frames)
|
||||
print(f" wrote {a.out}: {total} B "
|
||||
f"(header+tables {total-vid} B, video {vid} B)")
|
||||
print(f" {vid/len(idx):.0f} B/frame -> {vid/len(idx)*a.fps/1024:.1f} KB/s video"
|
||||
f" + {RC.AUDIO_KBPS} KB/s audio = {vid/len(idx)*a.fps/1024+RC.AUDIO_KBPS:.1f} KB/s")
|
||||
print(f" PSNR {r['psnr']:.2f} dB palette ceiling {r['pal']:.2f} dB "
|
||||
f"loss {r['loss']:.2f} dB")
|
||||
print(f" modes: SKIP {r['skip']:.1f}% V1 {r['v1']:.1f}% "
|
||||
f"V4 {r['v4']:.1f}% RAW {r['raw']:.1f}%")
|
||||
|
||||
if a.preview:
|
||||
from PIL import Image
|
||||
f = len(idx) // 2
|
||||
gap = np.full((H_ * 3, 4, 3), 40, np.uint8)
|
||||
st = np.concatenate([VQ.zoom(m["rgb"][f], 3), gap,
|
||||
VQ.zoom(pal[idx[f]], 3), gap,
|
||||
VQ.zoom(pal[enc["recon"][f]], 3)], axis=1)
|
||||
Image.fromarray(st).save(a.preview)
|
||||
print(f" preview -> {a.preview} (source | palette ceiling | decoded)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract decimated frames from a Blu-ray .m2ts into 256x192 PNGs.
|
||||
|
||||
Source is 1920x1080 (16:9). The arcade original is 4:3, so we CENTER-CROP to
|
||||
1440x1080 by default -- see docs/STATUS.md open question on framing.
|
||||
"""
|
||||
import subprocess, sys, os, shutil
|
||||
|
||||
STREAM_DIR = "/media/reala-misaki/BDROM/BDMV/STREAM"
|
||||
W, H = 256, 192
|
||||
|
||||
def duration(path):
|
||||
out = subprocess.check_output(["ffprobe","-v","error","-show_entries",
|
||||
"format=duration","-of","csv=p=0",path], text=True)
|
||||
return float(out.strip())
|
||||
|
||||
def extract(stream, outdir, fps=12, mode="crop", start=None, dur=None):
|
||||
src = f"{STREAM_DIR}/{stream}.m2ts"
|
||||
total = duration(src)
|
||||
if start is None: start = 0.0
|
||||
if dur is None: dur = total - start
|
||||
shutil.rmtree(outdir, ignore_errors=True); os.makedirs(outdir)
|
||||
if mode == "crop": # 4:3 centre crop, arcade framing
|
||||
vf = f"crop=1440:1080:240:0,hqdn3d=4:3:0:0,scale={W}:{H}:flags=lanczos"
|
||||
elif mode == "squash": # full 16:9 squeezed into 4:3
|
||||
vf = f"hqdn3d=4:3:0:0,scale={W}:{H}:flags=lanczos"
|
||||
elif mode == "wide": # 16:9 preserved, letterboxed later
|
||||
vf = f"hqdn3d=4:3:0:0,scale={W}:144:flags=lanczos"
|
||||
else: raise ValueError(mode)
|
||||
vf = f"fps={fps}," + vf
|
||||
subprocess.check_call(["ffmpeg","-v","error","-ss",str(start),"-t",str(dur),
|
||||
"-i",src,"-vf",vf,"-vsync","0",f"{outdir}/f%04d.png","-y"])
|
||||
n = len(os.listdir(outdir))
|
||||
print(f"{stream}: dur={total:.2f}s -> {n} frames @{fps}fps ({mode})")
|
||||
return n
|
||||
|
||||
if __name__ == "__main__":
|
||||
stream, outdir = sys.argv[1], sys.argv[2]
|
||||
fps = int(sys.argv[3]) if len(sys.argv) > 3 else 12
|
||||
mode = sys.argv[4] if len(sys.argv) > 4 else "crop"
|
||||
extract(stream, outdir, fps, mode)
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rate control: hit a target bitrate exactly, so one encoder serves both targets.
|
||||
|
||||
USER DECISION (session 2): ship TWO quality modes, SASI and SCSI. The codec's
|
||||
bitrate ceiling is a build parameter; the encoder is otherwise identical.
|
||||
|
||||
Mechanism: the hybrid encoder's lagrangian `lam` trades distortion for bytes
|
||||
monotonically, so per frame we binary-search lam to land inside a byte budget.
|
||||
A leaky bucket lets a quiet frame bank bytes that an action frame can spend --
|
||||
without that, quiet frames waste budget and action frames stay ugly.
|
||||
|
||||
The ceiling is HARD: the 68000 streams at a fixed rate off the disk, and a frame
|
||||
that overruns is a dropped frame, not a slow frame.
|
||||
"""
|
||||
import numpy as np
|
||||
import vq_hybrid as H
|
||||
|
||||
# Profiles. Bandwidths are the sustained-read figures the player can rely on;
|
||||
# see docs/FINDINGS.md 5 -- these are FOLKLORE-grade until the disk benchmark
|
||||
# is unblocked, so they are deliberately conservative fractions of the quoted
|
||||
# ceiling (audio, seeks and container overhead come out of the same pipe).
|
||||
# Calibrated against the CORRECTED rate-distortion measurement (FINDINGS 14).
|
||||
#
|
||||
# k=256 with 1-byte indices beats k=1024 with 2-byte indices at every matched
|
||||
# bitrate. The earlier "+2.4 dB for k=1024" was an artifact of a rate model that
|
||||
# charged 1 byte for a 10-bit index. 1-byte indices also mean the 68000 decoder
|
||||
# reads a plain move.b with no alignment case, and the codebook is 8 KB not 32 KB.
|
||||
#
|
||||
# The two profiles are the SAME codec, decoder and bitstream -- only `lam` differs.
|
||||
PROFILES = {
|
||||
"sasi": dict(kbps=45, lam=300.0, k1=256, k4=256,
|
||||
desc="stock 10MHz ACE/EXPERT, SASI",
|
||||
quality="34.8 dB on 00020 / 28.3 dB on 00146"),
|
||||
"scsi": dict(kbps=75, lam=100.0, k1=256, k4=256,
|
||||
desc="Super/XVI, or CZ-6BS1 board in a 10MHz machine",
|
||||
quality="35.9 dB on 00020 / 29.0 dB on 00146"),
|
||||
}
|
||||
# Not a shipping profile, but the curve continues: lam=25 is ~185 KB/s at ~38.7 dB
|
||||
# with 26% RAW blocks, and lam->0 is pixel-exact (0.00 dB loss). Entropy-coding
|
||||
# the payload (NOT YET IMPLEMENTED) should shift the whole curve ~1.4x left.
|
||||
|
||||
AUDIO_KBPS = 7.8 # MSM6258 ADPCM 15.6kHz mono -- comes out of the same budget
|
||||
|
||||
|
||||
def frame_budget(kbps, fps=12, audio=AUDIO_KBPS):
|
||||
"""bytes per video frame after audio takes its cut"""
|
||||
return (kbps - audio) * 1024.0 / fps
|
||||
|
||||
|
||||
def encode_rate_controlled(m, target_kbps, fps=12, bucket_frames=8,
|
||||
lam_lo=1.0, lam_hi=2e5, steps=9, verbose=False):
|
||||
budget = frame_budget(target_kbps, fps)
|
||||
bucket = 0.0 # banked bytes, capped at bucket_frames*budget
|
||||
cap = bucket_frames * budget
|
||||
out_recon, out_modes, out_sizes, out_lam = [], [], [], []
|
||||
|
||||
# encode() is whole-sequence; drive it per-lam and pick per frame.
|
||||
# Cheaper than re-running the whole encoder per frame: precompute the ladder.
|
||||
ladder = []
|
||||
lams = np.geomspace(lam_lo, lam_hi, steps)
|
||||
for lam in lams:
|
||||
e = H.encode(m, lam=float(lam))
|
||||
ladder.append(e)
|
||||
if verbose:
|
||||
print(f" lam={lam:9.0f} mean {e['sizes'].mean():6.0f} B/frame")
|
||||
|
||||
nf = len(m["idx"])
|
||||
for f in range(nf):
|
||||
allow = budget + bucket
|
||||
# cheapest lam (highest quality) whose size fits the allowance
|
||||
pick = len(lams) - 1
|
||||
for i in range(len(lams)):
|
||||
if ladder[i]["sizes"][f] <= allow:
|
||||
pick = i; break
|
||||
sz = ladder[pick]["sizes"][f]
|
||||
bucket = min(cap, bucket + budget - sz)
|
||||
out_recon.append(ladder[pick]["recon"][f])
|
||||
out_modes.append(ladder[pick]["modes"][f])
|
||||
out_sizes.append(sz); out_lam.append(lams[pick])
|
||||
|
||||
return dict(recon=out_recon, modes=out_modes, sizes=np.array(out_sizes),
|
||||
lam=np.array(out_lam), nb=ladder[0]["nb"], budget=budget)
|
||||
|
||||
|
||||
def summarise(m, enc, target_kbps, fps=12):
|
||||
import vq as VQ
|
||||
pal = m["pal"]
|
||||
rec = [pal[i] for i in enc["recon"]]
|
||||
src = [pal[i] for i in m["idx"]]
|
||||
p = np.mean([VQ.psnr(o, v) for o, v in zip(m["rgb"], rec)])
|
||||
pp = np.mean([VQ.psnr(o, v) for o, v in zip(m["rgb"], src)])
|
||||
sz = enc["sizes"]
|
||||
mo = np.concatenate(enc["modes"])
|
||||
return dict(target=target_kbps, psnr=p, pal=pp, loss=pp - p,
|
||||
mean_B=sz.mean(), max_B=sz.max(), budget=enc["budget"],
|
||||
kbps=sz.mean() * fps / 1024 + AUDIO_KBPS,
|
||||
over=100.0 * np.mean(sz > enc["budget"]),
|
||||
skip=100 * (mo == 0).mean(), v1=100 * (mo == 1).mean(),
|
||||
v4=100 * (mo == 2).mean())
|
||||
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env python3
|
||||
"""4x4 vector-quantisation prototype for the X68000 Dragon's Lair port.
|
||||
|
||||
Pipeline mirrors what the 68000 decoder would actually do, so the measured
|
||||
quality is honest:
|
||||
|
||||
frames -> per-scene 256-colour palette (median cut, NO dither)
|
||||
-> 4x4 blocks of PALETTISED rgb
|
||||
-> k-means codebook (luma-weighted euclidean)
|
||||
-> each codeword's 16 pixels snapped back to a palette index
|
||||
|
||||
The decoder only ever copies 16 palette indices out of a table, so the codebook
|
||||
entries MUST be legal palette indices -- both quantisation losses compose.
|
||||
|
||||
No sklearn on this box; k-means is hand-rolled (chunked, numpy).
|
||||
"""
|
||||
import numpy as np, glob, os, sys
|
||||
from PIL import Image
|
||||
|
||||
BW = BH = 4 # block size
|
||||
# ITU-R BT.601 luma weights, squared -- we compare in a luma-weighted RGB space
|
||||
LUMA = np.array([0.299, 0.587, 0.114], dtype=np.float32)
|
||||
|
||||
|
||||
def load_frames(d):
|
||||
fs = sorted(glob.glob(f"{d}/f*.png"))
|
||||
return [np.asarray(Image.open(f).convert("RGB")) for f in fs]
|
||||
|
||||
|
||||
def scene_palette(rgb, colors=256, stride=3):
|
||||
"""One shared palette for the whole scene, no dithering (cel art is flat)."""
|
||||
samp = np.concatenate([r.reshape(-1, 3) for r in rgb[::stride]])
|
||||
ref = Image.fromarray(samp.reshape(-1, 1, 3)).quantize(
|
||||
colors=colors, method=Image.MEDIANCUT, dither=Image.NONE)
|
||||
pal = np.array(ref.getpalette()[:colors * 3], dtype=np.uint8).reshape(-1, 3)
|
||||
return ref, pal
|
||||
|
||||
|
||||
def palettise(rgb, ref):
|
||||
return [np.asarray(Image.fromarray(r).quantize(palette=ref, dither=Image.NONE),
|
||||
dtype=np.uint8) for r in rgb]
|
||||
|
||||
|
||||
def blockify(idx, pal, bw=BW, bh=BH):
|
||||
"""(H,W) palette indices -> (nblocks, bh*bw*3) float32 luma-weighted RGB."""
|
||||
H, W = idx.shape
|
||||
rgb = pal[idx].astype(np.float32) * LUMA # weight once, up front
|
||||
b = rgb.reshape(H // bh, bh, W // bw, bw, 3).transpose(0, 2, 1, 3, 4)
|
||||
return b.reshape(-1, bh * bw * 3)
|
||||
|
||||
|
||||
def kmeans(X, k, iters=24, seed=0):
|
||||
"""Chunked Lloyd's algorithm. k-means++ style seeding, deterministic."""
|
||||
rng = np.random.default_rng(seed)
|
||||
n = X.shape[0]
|
||||
if n <= k:
|
||||
return X.copy(), np.arange(n)
|
||||
# seed: farthest-point sampling on a random subsample (cheap k-means++)
|
||||
sub = X[rng.choice(n, min(n, 20000), replace=False)]
|
||||
C = np.empty((k, X.shape[1]), dtype=np.float32)
|
||||
C[0] = sub[rng.integers(len(sub))]
|
||||
d2 = ((sub - C[0]) ** 2).sum(1)
|
||||
for i in range(1, k):
|
||||
C[i] = sub[np.argmax(d2)]
|
||||
d2 = np.minimum(d2, ((sub - C[i]) ** 2).sum(1))
|
||||
lab = None
|
||||
for _ in range(iters):
|
||||
lab = assign(X, C)
|
||||
newC = C.copy()
|
||||
cnt = np.bincount(lab, minlength=k)
|
||||
s = np.zeros_like(C)
|
||||
np.add.at(s, lab, X)
|
||||
nz = cnt > 0
|
||||
newC[nz] = s[nz] / cnt[nz, None]
|
||||
# revive dead codewords on the worst-fit blocks
|
||||
if (~nz).any():
|
||||
err = ((X - newC[lab]) ** 2).sum(1)
|
||||
worst = np.argsort(err)[-int((~nz).sum()):]
|
||||
newC[~nz] = X[worst]
|
||||
if np.allclose(newC, C):
|
||||
C = newC; break
|
||||
C = newC
|
||||
return C, assign(X, C)
|
||||
|
||||
|
||||
def assign(X, C, chunk=8192):
|
||||
"""Nearest centroid, chunked to bound memory."""
|
||||
Cn = (C ** 2).sum(1)
|
||||
out = np.empty(X.shape[0], dtype=np.int32)
|
||||
for i in range(0, X.shape[0], chunk):
|
||||
x = X[i:i + chunk]
|
||||
d = Cn[None, :] - 2.0 * (x @ C.T) # + |x|^2, constant per row
|
||||
out[i:i + chunk] = np.argmin(d, axis=1)
|
||||
return out
|
||||
|
||||
|
||||
def snap_codebook(C, pal, bw=BW, bh=BH):
|
||||
"""Centroids (luma-weighted RGB) -> legal palette indices, as the ROM stores them."""
|
||||
cb_rgb = C.reshape(-1, bh * bw, 3) / LUMA # undo the weighting
|
||||
palw = pal.astype(np.float32) * LUMA
|
||||
flat = (cb_rgb * LUMA).reshape(-1, 3)
|
||||
d = (flat ** 2).sum(1)[:, None] - 2 * (flat @ palw.T) + (palw ** 2).sum(1)[None, :]
|
||||
return np.argmin(d, axis=1).astype(np.uint8).reshape(-1, bh * bw)
|
||||
|
||||
|
||||
def unblockify(labels, cb_idx, H, W, bw=BW, bh=BH):
|
||||
blocks = cb_idx[labels].reshape(H // bh, W // bw, bh, bw)
|
||||
return blocks.transpose(0, 2, 1, 3).reshape(H, W)
|
||||
|
||||
|
||||
def psnr(a, b):
|
||||
mse = np.mean((a.astype(np.float64) - b.astype(np.float64)) ** 2)
|
||||
return 99.0 if mse == 0 else 10 * np.log10(255.0 ** 2 / mse)
|
||||
|
||||
|
||||
def encode_scene(frames_dir, k=256, bw=BW, bh=BH, iters=24):
|
||||
rgb = load_frames(frames_dir)
|
||||
H, W = rgb[0].shape[:2]
|
||||
ref, pal = scene_palette(rgb)
|
||||
idx = palettise(rgb, ref)
|
||||
|
||||
X = np.concatenate([blockify(i, pal, bw, bh) for i in idx])
|
||||
C, _ = kmeans(X, k, iters)
|
||||
cb_idx = snap_codebook(C, pal, bw, bh)
|
||||
|
||||
# re-assign against the SNAPPED codebook: that's what the decoder can produce
|
||||
Csnap = (pal[cb_idx].astype(np.float32) * LUMA).reshape(k, -1)
|
||||
recon, labels = [], []
|
||||
for i in idx:
|
||||
lab = assign(blockify(i, pal, bw, bh), Csnap)
|
||||
labels.append(lab)
|
||||
recon.append(unblockify(lab, cb_idx, H, W, bw, bh))
|
||||
return dict(rgb=rgb, pal=pal, idx=idx, cb_idx=cb_idx, labels=labels,
|
||||
recon=recon, H=H, W=W, k=k, bw=bw, bh=bh)
|
||||
|
||||
|
||||
def report(r, name=""):
|
||||
pal, idx, recon = r["pal"], r["idx"], r["recon"]
|
||||
src8 = [pal[i] for i in idx]
|
||||
vq8 = [pal[i] for i in recon]
|
||||
orig = r["rgb"]
|
||||
p_pal = np.mean([psnr(o, s) for o, s in zip(orig, src8)])
|
||||
p_vq = np.mean([psnr(o, v) for o, v in zip(orig, vq8)])
|
||||
p_vq_only = np.mean([psnr(s, v) for s, v in zip(src8, vq8)])
|
||||
nb = (r["H"] // r["bh"]) * (r["W"] // r["bw"])
|
||||
idxbits = int(np.ceil(np.log2(r["k"])))
|
||||
keyf = nb * idxbits / 8
|
||||
# block-delta cost: how many block indices change frame to frame
|
||||
ch = [np.count_nonzero(r["labels"][i] != r["labels"][i - 1]) / nb
|
||||
for i in range(1, len(r["labels"]))]
|
||||
print(f"--- {name} k={r['k']} block={r['bw']}x{r['bh']} ---")
|
||||
print(f" palette-only PSNR : {p_pal:5.2f} dB (floor: 256c is the best we can do)")
|
||||
print(f" after VQ PSNR : {p_vq:5.2f} dB (loss from VQ alone: {p_pal-p_vq:.2f} dB)")
|
||||
print(f" VQ vs palettised : {p_vq_only:5.2f} dB")
|
||||
print(f" blocks/frame : {nb} keyframe {keyf:.0f} B codebook {r['k']*r['bw']*r['bh']} B")
|
||||
if ch:
|
||||
print(f" blocks changed/frm: mean {100*np.mean(ch):5.1f}% p90 {100*np.percentile(ch,90):5.1f}%")
|
||||
return dict(p_pal=p_pal, p_vq=p_vq, nb=nb, keyf=keyf,
|
||||
chg=np.mean(ch) if ch else 0, chg90=np.percentile(ch,90) if ch else 0)
|
||||
|
||||
|
||||
def zoom(a, f=3):
|
||||
return np.repeat(np.repeat(a, f, axis=0), f, axis=1)
|
||||
|
||||
|
||||
def compare_png(r, frame, out, f=3):
|
||||
pal = r["pal"]
|
||||
src = pal[r["idx"][frame]]
|
||||
vq = pal[r["recon"][frame]]
|
||||
orig = r["rgb"][frame]
|
||||
gap = np.full((src.shape[0] * f, 4, 3), 40, dtype=np.uint8)
|
||||
strip = np.concatenate([zoom(orig, f), gap, zoom(src, f), gap, zoom(vq, f)], axis=1)
|
||||
Image.fromarray(strip).save(out)
|
||||
return out
|
||||
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cinepak-style hybrid VQ with a RAW escape: per-4x4-block choice of
|
||||
SKIP / V1 (one 4x4 codeword) / V4 (four 2x2 codewords) / RAW (16 literal indices).
|
||||
|
||||
Flat 4x4 VQ at k=256 visibly destroys Bluth's ink linework (see docs/FINDINGS.md).
|
||||
The standard fix is to let detailed blocks spend 4x the bits. Rate control picks
|
||||
the split per block by rate-distortion, so the bitrate ceiling stays deterministic
|
||||
-- which is the whole reason we chose VQ over a lossless delta.
|
||||
|
||||
The RAW mode is what makes ONE codec serve both shipping targets (session 2
|
||||
user decision: SASI and SCSI quality modes). As lam -> 0 the encoder buys RAW
|
||||
blocks until the frame is pixel-exact against the palettised source, so the
|
||||
SCSI profile is not a second codec -- it is the same bitstream with the rate
|
||||
knob opened up. The 68000 decoder needs no extra path: RAW is a straight copy,
|
||||
which is cheaper than V4.
|
||||
|
||||
Bitstream per frame (what the 68000 actually parses):
|
||||
2 bits/block header, packed: 00=SKIP 01=V1 10=V4 11=RAW
|
||||
then the payload in block order: V1 -> 1 index, V4 -> 4, RAW -> 16
|
||||
"""
|
||||
import numpy as np, sys
|
||||
from PIL import Image
|
||||
import vq as VQ
|
||||
|
||||
LUMA = VQ.LUMA
|
||||
|
||||
|
||||
def blocks_of(idx, pal, bw, bh):
|
||||
return VQ.blockify(idx, pal, bw, bh)
|
||||
|
||||
|
||||
def build(frames_dir, k1=256, k4=256, iters=16, lam=0.0):
|
||||
rgb = VQ.load_frames(frames_dir)
|
||||
H, W = rgb[0].shape[:2]
|
||||
ref, pal = VQ.scene_palette(rgb)
|
||||
idx = VQ.palettise(rgb, ref)
|
||||
|
||||
# --- two codebooks, trained on the whole scene ---
|
||||
X1 = np.concatenate([blocks_of(i, pal, 4, 4) for i in idx])
|
||||
C1, _ = VQ.kmeans(X1, k1, iters)
|
||||
cb1 = VQ.snap_codebook(C1, pal, 4, 4) # (k1,16) palette idx
|
||||
C1s = (pal[cb1].astype(np.float32) * LUMA).reshape(k1, -1)
|
||||
|
||||
X4 = np.concatenate([blocks_of(i, pal, 2, 2) for i in idx])
|
||||
C4, _ = VQ.kmeans(X4, k4, iters)
|
||||
cb4 = VQ.snap_codebook(C4, pal, 2, 2) # (k4,4) palette idx
|
||||
C4s = (pal[cb4].astype(np.float32) * LUMA).reshape(k4, -1)
|
||||
return dict(rgb=rgb, pal=pal, idx=idx, H=H, W=W,
|
||||
cb1=cb1, C1s=C1s, cb4=cb4, C4s=C4s, k1=k1, k4=k4)
|
||||
|
||||
|
||||
def _v1_recon(lab1, cb1, H, W):
|
||||
return VQ.unblockify(lab1, cb1, H, W, 4, 4)
|
||||
|
||||
|
||||
def encode(m, lam=0.02, skip_thresh=0.0, idx_bytes=None):
|
||||
"""lam = lagrangian rate weight (bytes -> squared-error units).
|
||||
Higher lam => more V1/SKIP => smaller & softer.
|
||||
|
||||
idx_bytes: size of ONE codebook index in the bitstream. k>256 needs 2 bytes,
|
||||
which doubles what V1 and V4 actually cost -- if the RD model ignores that
|
||||
it systematically over-picks V4 and under-reports the bitrate. Defaults to
|
||||
the value implied by the codebook sizes."""
|
||||
if idx_bytes is None:
|
||||
idx_bytes = 1 if max(m["k1"], m["k4"]) <= 256 else 2
|
||||
pal, idx, H, W = m["pal"], m["idx"], m["H"], m["W"]
|
||||
nbx, nby = W // 4, H // 4
|
||||
nb = nbx * nby
|
||||
recon, modes, sizes = [], [], []
|
||||
prev = None
|
||||
for f, im in enumerate(idx):
|
||||
B1 = blocks_of(im, pal, 4, 4) # (nb,48)
|
||||
l1 = VQ.assign(B1, m["C1s"])
|
||||
e1 = ((B1 - m["C1s"][l1]) ** 2).sum(1)
|
||||
|
||||
B4 = blocks_of(im, pal, 2, 2) # (nb*4,12) in 2x2 raster
|
||||
l4 = VQ.assign(B4, m["C4s"])
|
||||
e4raw = ((B4 - m["C4s"][l4]) ** 2).sum(1)
|
||||
# regroup 2x2 blocks (raster over 8x12... ) into their parent 4x4 block
|
||||
q = _group_2x2_into_4x4(np.arange(nb * 4), W)
|
||||
e4 = e4raw[q].reshape(nb, 4).sum(1)
|
||||
l4g = l4[q].reshape(nb, 4)
|
||||
|
||||
# SKIP: cost of reusing the previous *reconstructed* block
|
||||
if prev is None:
|
||||
eS = np.full(nb, np.inf)
|
||||
else:
|
||||
pb = blocks_of(prev, pal, 4, 4)
|
||||
eS = ((B1 - pb) ** 2).sum(1)
|
||||
|
||||
# RAW: zero distortion against the palettised source, 16 bytes
|
||||
eR = np.zeros(nb)
|
||||
|
||||
# rate-distortion choice: true byte cost per mode. The 2-bit header is
|
||||
# paid by every block regardless, so it drops out of the comparison.
|
||||
bV1 = 1.0 * idx_bytes
|
||||
bV4 = 4.0 * idx_bytes
|
||||
bRAW = 16.0 # RAW is literal palette bytes, never indices
|
||||
cost = np.stack([eS + lam * 0.0, e1 + lam * bV1,
|
||||
e4 + lam * bV4, eR + lam * bRAW])
|
||||
mode = np.argmin(cost, axis=0).astype(np.uint8)
|
||||
|
||||
out = np.empty((H, W), dtype=np.uint8)
|
||||
_paint(out, mode, l1, l4g, m["cb1"], m["cb4"], prev, nbx, nby, im)
|
||||
recon.append(out); modes.append(mode)
|
||||
nV1 = int((mode == 1).sum()); nV4 = int((mode == 2).sum())
|
||||
nR = int((mode == 3).sum())
|
||||
sizes.append(nb * 2 / 8 + (nV1 + nV4 * 4) * idx_bytes + nR * 16)
|
||||
prev = out
|
||||
return dict(recon=recon, modes=modes, sizes=np.array(sizes), nb=nb)
|
||||
|
||||
|
||||
def _group_2x2_into_4x4(a, W):
|
||||
"""map 2x2-block raster order -> (nb4, 4) grouping by parent 4x4 block"""
|
||||
n2x = W // 2
|
||||
n2y = len(a) // n2x
|
||||
g = a.reshape(n2y, n2x)
|
||||
g = g.reshape(n2y // 2, 2, n2x // 2, 2).transpose(0, 2, 1, 3)
|
||||
return g.reshape(-1)
|
||||
|
||||
|
||||
def _paint(out, mode, l1, l4g, cb1, cb4, prev, nbx, nby, src):
|
||||
for b in range(len(mode)):
|
||||
by, bx = divmod(b, nbx)
|
||||
y, x = by * 4, bx * 4
|
||||
mo = mode[b]
|
||||
if mo == 0:
|
||||
out[y:y+4, x:x+4] = prev[y:y+4, x:x+4]
|
||||
elif mo == 1:
|
||||
out[y:y+4, x:x+4] = cb1[l1[b]].reshape(4, 4)
|
||||
elif mo == 3:
|
||||
out[y:y+4, x:x+4] = src[y:y+4, x:x+4]
|
||||
else:
|
||||
c = cb4[l4g[b]].reshape(2, 2, 2, 2) # (sub_y,sub_x,2,2)
|
||||
out[y:y+2, x:x+2] = c[0, 0]; out[y:y+2, x+2:x+4] = c[0, 1]
|
||||
out[y+2:y+4, x:x+2] = c[1, 0]; out[y+2:y+4, x+2:x+4] = c[1, 1]
|
||||
|
||||
|
||||
def evaluate(m, enc, fps=12):
|
||||
pal = m["pal"]
|
||||
rec = [pal[i] for i in enc["recon"]]
|
||||
src = [pal[i] for i in m["idx"]]
|
||||
p_vq = np.mean([VQ.psnr(o, v) for o, v in zip(m["rgb"], rec)])
|
||||
p_pal = np.mean([VQ.psnr(o, v) for o, v in zip(m["rgb"], src)])
|
||||
mo = np.concatenate(enc["modes"])
|
||||
sz = enc["sizes"].mean()
|
||||
return dict(psnr=p_vq, pal=p_pal, loss=p_pal - p_vq, bytes=sz,
|
||||
kbps=sz * fps / 1024,
|
||||
skip=100 * (mo == 0).mean(), v1=100 * (mo == 1).mean(),
|
||||
v4=100 * (mo == 2).mean(), raw=100 * (mo == 3).mean())
|
||||
Reference in New Issue
Block a user