Dragon-s-Lair-X68k/tools/analysis/36_branch_audio.py

#!/usr/bin/env python3
"""51.3's REFILL CLIMB WITH A SECOND CONSUMER, THROUGH A REAL BRANCH POINT.

    python3 tools/analysis/36_branch_audio.py --kbps R [R ...]
                                              [--ring KB ...] [--gate]

The oldest open item in ROADMAP P6, named by FINDINGS 65.6 and again by 67.6:

    "the slack table is here, but 51.3's refill climb with a second consumer
     through a real branch point is not."

Everything it needs already exists and none of it has ever been put in the same
room:

  * 51.3 -- slack is ACCUMULATED out of `pipe - wire`, at `pipe - wire` bytes a
    second, and a seek spends all of it.  `tools/analysis/20_seek_slack.py`.
  * 56.3 -- where the branch points ARE: 612 distinct transitions into a seek
    over the arcade's own graph, worst gap 0.000 s, median 3.473 s.
    `tools/analysis/25_scene_graph.py`, reading only DLXSCENE1.
  * 56.4 -- the climb against that distribution, for the codec container.  It
    charged audio as `ratectl.AUDIO_KBPS`, a flat 7.8 KB/s placeholder that
    predates any of the audio work.
  * 65.3/67.1 -- what a second consumer ACTUALLY costs a container: a fixed
    cadence of F frames per A sectors, because a packed record's address is
    arithmetic and cannot be an index.  `tools/analysis/32_audio_wire.py`.
  * 68 -- the player that holds both streams at once, and its buffers.

Three questions, and the tree has never asked any of them:

  1. What does the SECOND CONSUMER do to the climb?  Not to the wire -- 32
     answered that and it is 1.3% -- but to `pipe - wire`, which is a small
     difference of two large numbers and is the thing the climb is made of.

  2. What is the climb on the PACKED branch?  This is the branch the player
     runs (68) and the one B1's acceptance is written against.

  3. What does the CADENCE do at a branch point?  A group is `lump k, then F
     records`, so lump k sits at a LOWER address than every record in its group
     but the first.  A seek to record i lands inside a group whose audio is
     BEHIND it.  Nobody has ever priced entering a group off-boundary, and the
     game's own seek targets say how often it happens.

THE FRAME INDEX OF A SEEK TARGET IS A DESIGN ASSUMPTION AND IS LABELLED ONE.
DLXSCENE1 carries positions on the laserdisc timeline in ms.  This tree's design
puts ONE CONTAINER PER SCENE -- 53 and 55.1 charge a scene change 6,164 header
bytes, and 56.3 counts 203 of the 612 transitions as container changes for
exactly that reason -- so a seek target's frame index inside its container is
`(target start - the scene's own earliest start) * fps / 1000`.  If the design
ever puts one container per SEQUENCE instead, every seek lands on frame 0, the
group offset is always zero and section 3 collapses to nothing.  That is the
assumption, said out loud, in the one place the answer depends on it.
"""
import sys, os, json, argparse, importlib.util

sys.path.insert(0, "tools/encoder")
sys.path.insert(0, "tools/analysis")

TABLE = os.environ.get("DLX_SCENEGRAPH", "tmp/scenegraph.json")
FPS = 12
CHIP_HZ = 15625.0                 # MSM6258V, 8 MHz / 512 (FINDINGS 32, 65)
AU_BPS = CHIP_HZ / 2              # 4 bits a sample, two samples to a byte
AU_FRAME = AU_BPS / FPS           # 651.0416... B a slot, and the dots are 65.3
SECTOR = 512


def load(path, name):
    spec = importlib.util.spec_from_file_location(name, path)
    m = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(m)
    return m


def cadence(F):
    """Best A for this F: the fewest whole sectors that hold F frames of chip."""
    num, den = int(round(AU_BPS * 2)), 2 * SECTOR * FPS
    A = -(-(num * F) // den)                       # ceil(F * bytes/sector)
    return A, A * SECTOR


def climb_s(records, rec_b, pipe_kbps, wire_kbps):
    """Seconds of play to accumulate `records` records of lookahead (51.3)."""
    surplus = (pipe_kbps - wire_kbps) * 1024.0
    return float("inf") if surplus <= 0 else records * rec_b / surplus


# --------------------------------------------------------------- the branches

def branch_points(doc, nodes, sg):
    """The 612 transitions into a seek, plus each one's target frame index.

    Returns (gaps_s, within, changes).  `within` entries carry the frame index
    the seek lands on inside its container; `changes` are scene changes, which
    land on frame 0 of a new one.
    """
    gaps, _ = sg.worst_gap(nodes)
    play = [g for g in gaps if g[5] != "attract_mode"]
    scene_start = {}
    for scene, seqs in doc["scenes"].items():
        st = [s["start_ms"] for s in seqs.values() if s["start_ms"] >= 0]
        scene_start[scene] = min(st) if st else None

    within, changes = [], []
    for g, src, tgt, kind, ends, scene in play:
        if ends:
            changes.append((g / 1000.0, src))
            continue
        n = nodes.get(f"{scene}.{tgt}")
        if n is None or not n.seeks or scene_start[scene] is None:
            continue
        i = int(round((n.start - scene_start[scene]) / 1000.0 * FPS))
        within.append((g / 1000.0, max(0, i), f"{scene}.{tgt}"))
    return sorted(g[0] / 1000.0 for g in play), within, changes


def silences(within, F):
    """Ms of silence entering each branch's group off-boundary, at cadence F.

    A group is `lump k, then F records`.  Seek to record i, take the next lump
    that lies AHEAD of the read point -- lump k+1, which arrives at frame
    (k+1)*F -- and the frames from i to (k+1)*F-1 have no audio.  The chip's
    second is a real second (15,625 samples, 2 to a byte), so the missing
    time is exactly `(F - i mod F) mod F` frames of 1/12 s and none of
    FINDINGS 54's frame-clock remainder gets into it.
    """
    return sorted(((F - (i % F)) % F) / FPS * 1000.0 for _, i, _ in within)


def pct(xs, p):
    return xs[min(len(xs) - 1, int(p * len(xs)))] if xs else float("nan")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--kbps", type=float, nargs="+",
                    help="delivered pipe rates, KB/s. REQUIRED (FINDINGS 50) "
                         "unless --gate, which carries its own.")
    ap.add_argument("--ring", type=float, nargs="+", default=[256, 512])
    ap.add_argument("--pkbps", type=float, nargs="+",
                    default=[589.6, 600.0, 650.0, 700.0, 900.0],
                    help="pipe rates for the PACKED table.  Its own list, "
                         "because a packed record is 1.3x the codec's and "
                         "every rate that serves one starves the other.")
    ap.add_argument("--table", default=TABLE)
    ap.add_argument("--packed", default="tmp/packed_singe.dlxp")
    ap.add_argument("--codec", default="tmp/rc_fr_singe_scsi_span.dlx")
    ap.add_argument("--fsweep", type=int, default=12,
                    help="highest cadence F in the pick table")
    ap.add_argument("--gate", action="store_true",
                    help="check.sh mode: fixed rates, and assert the structural "
                         "results rather than print the essay")
    a = ap.parse_args()
    if a.gate and not a.kbps:
        a.kbps = [451.4, 488.0, 600.0]
    if not a.kbps:
        ap.error("--kbps is required and has no default (FINDINGS 50)")

    if not os.path.exists(a.table):
        print(f"no scene table at {a.table} -- run:\n"
              f"  python3 tools/import/scenegraph.py")
        return 2
    doc = json.load(open(a.table))
    if doc.get("format") != "DLXSCENE1":
        print(f"{a.table}: not a DLXSCENE1 table")
        return 2

    sg = load("tools/analysis/25_scene_graph.py", "scene_graph")
    ss = load("tools/analysis/20_seek_slack.py", "seek_slack")
    from dlxp import DLXP
    import ratectl as RC

    nodes = sg.build_graph(doc["scenes"])
    gaps, within, changes = branch_points(doc, nodes, sg)
    med = gaps[len(gaps) // 2]

    d = DLXP(a.packed)
    P_REC = d.rec_bytes
    P_VID = P_REC * FPS / 1024
    F0, A0 = d.cad_f, d.cad_a
    P_AUD = A0 * SECTOR / F0 * FPS / 1024

    print(f"""
=== WHAT EACH BRANCH ACTUALLY HOLDS ====================================
51.3's climb is a statement about an ACCUMULATOR.  The two branches of this
project do not have the same one, and one of them does not have one at all.

  codec  ({os.path.basename(a.codec)})
         a {a.ring[0]:.0f} KB ring of variable records with an index in front of it;
         lookahead is whole records and the ceiling is what 20_seek_slack.py
         simulates.
  packed ({os.path.basename(a.packed)})
         a record is {P_REC:,} B of literal picture and the channel puts it
         STRAIGHT INTO GVRAM (FINDINGS 61, 62, 64).  There is no record
         buffer, so the VIDEO lookahead is ZERO records and there is nothing
         to climb.  The only consumer on that branch with any lookahead at
         all is the AUDIO one: {d.cad_a * SECTOR:,} B a lump, PG_ANBUF={3} slots and
         PG_APRE prefilled at {2} for the run 68 measured (it is a mailbox,
         not a constant: src/player/packed.s, 68.6).
         That is {2 * F0 / FPS:.3f} s of sound held against {0.0:.3f} s of picture.

  The CPU-painted packed variant (64.2's column B) is the one that holds two
  record buffers and 99,328 B, and it is the only packed configuration the
  word "climb" applies to.  Both are priced below.

=== THE BRANCH POINTS, OUT OF THE ARCADE'S OWN GRAPH ====================
  {len(gaps)} transitions into a seek (attract mode excluded, 56.3)
  worst {gaps[0]:.3f} s   p10 {pct(gaps,.10):.3f}   median {med:.3f}   p90 {pct(gaps,.90):.3f}
  {len(changes)} of them END THE SCENE and are container changes;
  {len(within)} land INSIDE a container, at frame indices 0..{max(i for _, i, _ in within)}""")

    # ---------------------------------------------------------------- 1
    dc, rec = ss.records(a.codec)
    C_REC = float(rec.mean())
    C_VID = C_REC * FPS / 1024
    print(f"""
=== 1. THE CLIMB WITH THE SECOND CONSUMER (the codec branch) ============
Audio is {AU_BPS/1024:.3f} KB/s and this container pays no padding for it (65.4: it
already has an index and already has variable records).  Against a video wire
of {C_VID:.1f} KB/s that is {100*(AU_BPS/1024)/C_VID:+.2f}% -- and the climb is not built out of the wire,
it is built out of `pipe - wire`, so that is not the number that matters.

  {'ring':>5} {'pipe':>7} {'ceil':>5} {'climb SILENT':>13} {'SOUNDED':>9} {'x':>6}"""
          f"  {'under, silent':>13} {'under, sounded':>14}")
    tab1 = []
    for ring_kb in a.ring:
        ring = int(ring_kb * 1024)
        for kbps in a.kbps:
            fill = (kbps - AU_BPS / 1024) * 1024 / FPS
            lo, hi, ring_ref, rate_ref = ss.paced_sim(rec, ring, fill)
            ceil = int(hi.max())
            cs = climb_s(ceil, C_REC, kbps, C_VID)
            ca = climb_s(ceil, C_REC, kbps, C_VID + AU_BPS / 1024)
            us = sum(1 for g in gaps if g < cs)
            ua = sum(1 for g in gaps if g < ca)
            ratio = ca / cs if cs not in (0.0, float("inf")) else float("inf")
            tab1.append((ring_kb, kbps, ceil, cs, ca, ratio, us, ua))
            print(f"  {ring_kb:5.0f} {kbps:7.1f} {ceil:5d} {cs:12.2f}s "
                  f"{ca:8.2f}s {ratio:5.2f}x  {f'{us}/{len(gaps)}':>13} "
                  f"{f'{ua}/{len(gaps)}':>14}")

    worst = max(tab1, key=lambda r: r[5])
    print(f"""
  THE SECOND CONSUMER IS {100*(AU_BPS/1024)/C_VID:.1f}% OF THE WIRE AND UP TO {worst[5]:.2f}x OF THE CLIMB.
  At {worst[1]:.1f} KB/s in a {worst[0]:.0f} KB ring the climb goes {worst[3]:.2f} s -> {worst[4]:.2f} s and the
  branch points that arrive under it go {worst[6]}/{len(gaps)} -> {worst[7]}/{len(gaps)}
  ({100*worst[6]/len(gaps):.0f}% -> {100*worst[7]/len(gaps):.0f}%).  Nothing about audio got bigger; the
  DIFFERENCE it is subtracted from got smaller, and the climb is made of the
  difference.  This is why 51.4's rate/ring distinction matters more with a
  second consumer than without one, and why quoting audio as a share of the
  wire (32, and every budget before it) understates it at every rate close to
  the wire.

  A CORRECTION TO 56.4, and it is small: that table charged audio at
  ratectl.AUDIO_KBPS = {RC.AUDIO_KBPS} KB/s, which is {AU_BPS:.1f} B/s expressed in decimal
  kB (21_iplrom_dmac.py says so).  In binary KB the figure is {AU_BPS/1024:.4f}, i.e.
  {100*(RC.AUDIO_KBPS-AU_BPS/1024)/(AU_BPS/1024):+.2f}%.  Every column of 56.4 moves in the flattering direction by
  less than one part in six hundred of the wire.  It is recorded because a
  placeholder that turns out to be right is still a placeholder.""")

    # ---------------------------------------------------------------- 2
    print(f"""
=== 2. THE PACKED BRANCH: THERE IS NO CLIMB, AND THAT IS THE FINDING ====
  video {P_VID:.1f} + audio {P_AUD:.4f} (F={F0}, A={A0}, {A0*SECTOR:,} B a lump) = {P_VID+P_AUD:.1f} KB/s,
  which is B1's acceptance figure and is where it comes from.

  DMAC-direct (what src/player/packed.s runs, FINDINGS 64/68):
    video lookahead 0 records.  The climb does not exist, the ceiling does not
    exist, and the {len(gaps)} gaps buy it NOTHING -- there is no accumulator for
    play to fill.  Its acceptance is a PER-FRAME deadline: {P_REC:,} B must land
    inside every slot, and a rate that averages {P_VID+P_AUD:.1f} KB/s over a second is
    not the same claim.  56.4's alarming column -- most branch points arrive
    with less lookahead than the one before them -- does not apply to it,
    because every frame arrives with less lookahead than the one before it.

  CPU-painted (64.2 column B, 99,328 B, two record buffers -> 1 record of
  lookahead):
    {'pipe':>7} {'climb SILENT':>13} {'SOUNDED':>9} {'x':>6}  {'under, silent':>13} {'under, sounded':>14}""")
    tab2 = []
    def secs(x, w):
        return f"{x:{w}.2f}s" if x != float("inf") else f"{'never':>{w+1}}"
    for kbps in a.pkbps:
        cs = climb_s(1, P_REC, kbps, P_VID)
        ca = climb_s(1, P_REC, kbps, P_VID + P_AUD)
        us = sum(1 for g in gaps if g < cs)
        ua = sum(1 for g in gaps if g < ca)
        r = ca / cs if cs not in (0.0, float("inf")) else float("inf")
        tab2.append((kbps, cs, ca, r, us, ua))
        print(f"    {kbps:7.1f} {secs(cs,12)} {secs(ca,8)} "
              f"{(f'{r:5.2f}x' if r != float('inf') else '  inf ')}  "
              f"{f'{us}/{len(gaps)}':>13} {f'{ua}/{len(gaps)}':>14}")
    print(f"    -- and EVERY rate in section 1's table is below {P_VID:.1f} KB/s, so "
          f"none of\n       them serves this container at all.")
    print(f"""
  READ THE FIRST ROW. At {P_VID+P_AUD:.1f} KB/s -- the acceptance figure this project
  quotes -- the SILENT container still climbs its one record in {tab2[0][1]:.2f} s and
  the SOUNDED one NEVER DOES, because {P_VID+P_AUD:.1f} is where its surplus is exactly
  zero.  The acceptance figure is the rate at which the sounded container has
  no lookahead at any amount of play, which is a different thing from the rate
  at which it plays.

  A packed record is 1.3x the codec's mean record and the packed wire is 1.3x
  the codec's, so a rate that is generous to one is tight for the other and the
  same audio debit costs the packed climb more.  ONE RECORD of lookahead is
  1/12 s of tolerance and it takes seconds of play to earn.

  THE TWO BRANCHES DIFFER HERE ON A COLUMN THAT IS NOT CLOCKS, which is the
  third time (61.9, 64.2, and this).  The packed branch spent its ring to
  delete a decoder; what it bought with the RAM is a player with no tolerance
  for a slow record at ANY time, not merely after a branch.""")

    # ---------------------------------------------------------------- 3
    sil = silences(within, F0)
    free = sum(1 for x in sil if x == 0.0)
    P_PIPE = a.pkbps[1] if len(a.pkbps) > 1 else a.pkbps[0]
    lump_ms = A0 * SECTOR / (P_PIPE * 1024) * 1000
    behind = (F0 - 1) * P_REC + A0 * SECTOR
    print(f"""
=== 3. THE COST NOBODY HAD COUNTED: entering a group off-boundary =======
A DLXP2 group is `lump k, then F records` (dlxp.py), so lump k is at a LOWER
address than every record of its group except the first.  Reading forward from
record i, the next lump to arrive is k+1, and it carries frame (k+1)*F.  The
frames from i to (k+1)*F-1 therefore have picture and no sound.

  Measured on the {len(within)} within-container seek targets of the arcade's own
  graph, at the shipped cadence F={F0}:

    mean   {sum(sil)/len(sil):7.1f} ms of silence entering the branch
    median {pct(sil,.50):7.1f}    p90 {pct(sil,.90):7.1f}    worst {sil[-1]:7.1f}
    free   {free}/{len(within)} land on a group boundary and cost nothing

  The other {len(changes)} branch points -- the scene changes -- are FREE, and by
  construction: lump 0 sits at sector 1 and record 0 at {d.off_frm:,}, so a
  container's own first bytes are header, lump, record and a scene change reads
  them in one forward pass.  **The container's start is the one branch point
  the cadence costs nothing at, and it is the only one anybody had looked at.**

  THE FIX IS A SECOND READ AND NOBODY HAS ONE.  Lump k is {behind:,} B behind
  record i at worst, so it cannot be picked up by reading early -- it is a
  separate command at a separate LBA, of {A0*SECTOR:,} B, which at {P_PIPE:.1f} KB/s is
  {lump_ms:.1f} ms against a mean {sum(sil)/len(sil):.0f} ms of silence -- {sum(sil)/len(sil)/lump_ms:.0f}x cheaper in TIME,
  one more command per branch, and the command overhead is B1's and unmeasured.
  src/player/packed.s starts PG_AK and PG_AKF at lump 0 and has no audio seek
  path at all; the player that branches needs one.

=== 4. THE CADENCE PICK, WITH THE THIRD COLUMN IT DID NOT HAVE ==========
32_audio_wire.py chose F={F0} on two columns, padding and RAM.  Here is the same
sweep with the branch column, measured on the game's own seek targets rather
than assumed uniform:

  {'F':>3} {'A':>3} {'lump B':>8} {'pad%':>7} {'aud KB/s':>9} {'RAM x2':>8} {'mean sil':>9} {'p90':>8} {'worst':>8} {'free':>10} {'vs uniform':>11}""")
    for F in range(1, a.fsweep + 1):
        A, lump = cadence(F)
        need = F * AU_FRAME
        s = silences(within, F)
        uni = (F - 1) / 2 / FPS * 1000.0
        mean = sum(s) / len(s)
        mark = "  <- shipped" if F == F0 else ""
        print(f"  {F:3d} {A:3d} {lump:8,} {100*(lump-need)/need:6.2f}% "
              f"{lump/F*FPS/1024:8.3f} {2*lump:8,} {mean:8.1f} "
              f"{pct(s,.90):8.1f} {s[-1]:8.1f} "
              f"{f'{sum(1 for x in s if x == 0)}/{len(s)}':>10} "
              f"{(mean/uni if uni else 1.0):10.2f}x{mark}")

    ratios = []
    for F in range(2, a.fsweep + 1):
        sF_ = silences(within, F)
        ratios.append((sum(sF_) / len(sF_)) / ((F - 1) / 2 / FPS * 1000.0))
    min_r, max_r = min(ratios), max(ratios)
    A1, l1 = cadence(1)
    AF, lF = cadence(F0)
    s1, sF = silences(within, 1), silences(within, F0)
    print(f"""
  F=1 -- "one lump a record", the cadence 32 called THE WORST ONE -- has no
  group to enter off-boundary, no second read, no audio seek path and 2,048 B
  of held lump instead of {2*lF:,}.  It costs {l1/1*FPS/1024 - lF/F0*FPS/1024:+.3f} KB/s of wire, which is
  {100*(l1/1*FPS/1024 - lF/F0*FPS/1024)/(P_VID+P_AUD):+.2f}% of the packed acceptance figure, and it BUYS BACK {2*lF-2*l1:,} B
  of RAM on the branch whose whole argument is that RAM is what it has spare.

  THE PICK IS THEREFORE REOPENED, and it is a real trade rather than an error:
  padding is what F={F0} minimises and padding is not the only thing F sets.
  A player that gets its audio seek right is indifferent; a player that does
  not pays a mean {sum(sF)/len(sF):.0f} ms of silence at {len(within)} of the game's {len(gaps)} branch
  points.  Nothing here decides it -- the deciding number is the SCSI command
  overhead of the extra read, and that is B1's.

  AND THE CONTENT IS NOT UNIFORM MOD F.  A uniform assumption would put the
  mean at (F-1)/2 frames; the arcade's seek targets land where they land, and
  the ratio column above runs {min_r:.2f}x..{max_r:.2f}x over the sweep, so a design
  that assumed uniform would be out by a quarter at F=3.  At the shipped F={F0} it is
  {(sum(sF)/len(sF))/((F0-1)/2/FPS*1000):.2f}x, which is a coincidence and is reported as one.

=== 5. WHAT THIS DOES NOT ESTABLISH ====================================
  1. NO RATE HERE IS MEASURED.  Every pipe column is a sensitivity (FINDINGS
     50), and B1 -- sustained AND data-phase burst -- is still the user's.
  2. THE FRAME INDEX OF A SEEK TARGET IS A DESIGN ASSUMPTION.  One container
     per SCENE (53, 55.1, 56.3).  One container per SEQUENCE makes section 3
     zero and section 4 moot; nothing else in the file changes.
  3. NOTHING RAN ON THE MACHINE.  This is arithmetic over a scene table, two
     containers and a player's own constants.  68's player has never seeked.
  4. THE MECHANICAL SEEK IS STILL UNMODELLED (51.7.5) and is charged on top of
     every millisecond here.
  5. THE SILENCE IS A CONTAINER PROPERTY, NOT A CHIP ONE.  What the MSM6258
     does when it is not fed -- hold the last sample, or click -- is a board
     question and belongs with session 34's fifth hardware item.""")

    if a.gate:
        # Structural assertions.  Not the milliseconds -- those move with the
        # scene table -- but the ORDER and the SIGNS, which are the finding.
        ok = True

        def check(cond, msg):
            nonlocal ok
            print(f"  {'OK  ' if cond else 'FAIL'} {msg}")
            ok = ok and bool(cond)

        print("\n=== GATE ===============================================")
        check(len(gaps) == 612, f"612 transitions into a seek, got {len(gaps)}")
        check(len(within) + len(changes) == len(gaps),
              f"{len(within)} within + {len(changes)} scene changes = {len(gaps)}")
        check(all(r[5] >= 1.0 for r in tab1),
              "audio never SHORTENS the codec climb")
        check(max(r[5] for r in tab1) > 1.5,
              f"and at some rate it more than 1.5x's it "
              f"({max(r[5] for r in tab1):.2f}x)")
        check(all(r[7] >= r[6] for r in tab1),
              "and never lowers the count of branch points under the climb")
        check(free < len(within) // 2,
              f"most within-container branches enter a group off-boundary "
              f"({len(within)-free}/{len(within)})")
        check(silences(within, 1) == [0.0] * len(within),
              "F=1 has no off-boundary case at all")
        check(sum(sil) / len(sil) > 10 * lump_ms,
              f"the silence F={F0} costs is >10x the lump read that removes it "
              f"({sum(sil)/len(sil):.0f} ms vs {lump_ms:.1f} ms)")
        print("  " + ("BRANCH-AUDIO GATE GREEN" if ok else "BRANCH-AUDIO GATE RED"))
        return 0 if ok else 1
    return 0


if __name__ == "__main__":
    sys.exit(main())