"""The ONE file in this tree that knows anything about somebody else's source. Everything downstream -- `tools/analysis/25_scene_graph.py` and whatever the game-logic layer eventually becomes -- reads the neutral table this writes and knows nothing about where it came from. That is deliberate. Two outside projects are read here, neither is vendored, neither ships in this repo, and their file layouts, table names, timing formulas and magic constants are wired into THIS file and nowhere else. When one of them moves, one file breaks. python3 tools/import/scenegraph.py [-o tmp/scenegraph.json] Sources, both permissively licensed and both cloned by the reader: icculus/DirkSimple -- zlib, Ryan C. Gordon. `data/games/lair/game.lua` holds the arcade scene graph, transcribed from the arcade ROM's own data table. THE source; the measurement is made on this. DLX_DIRKSIMPLE=/path (default tmp/scenegraph/DirkSimple) astrobleem/SNES-SuperDragonsLairArcade -- MIT, Chad Doebelin. `data/events/*.xml` holds 516 chapter definitions. Read as a CROSS-CHECK only, and a weak one: their own README states the chapters are "derived from DirkSimple game data", so this is a second COPY, not a second transcription (FINDINGS 56.2). Optional; skipped when absent. DLX_SNESLAIR=/path (default tmp/scenegraph/SNES-SuperDragonsLairArcade) WHAT COMES OUT is `DLXSCENE1`, our own schema, in one JSON file: {"format": "DLXSCENE1", "sources": [{"name", "url", "licence", "holder", "role"} ...], "scenes": {scene: {sequence: { "start_ms": float, absolute position on the medium, or -1 for "keep playing from where the disc already is", "timeout_ms": float, how long the clip runs unbranched, "kills", "single_frame": bool, "timeout_next": sequence name or null, "actions": [{"input", "from_ms", "to_ms", "next"}]}}}, "rows": [[scene, scene, scene] ...], the arcade's 13x3 scene order "counts": {"scenes", "sequences", "windows"}, "crosscheck": {scene: {sequence: {"start_ms", "end_ms", "actions": [...], "chapter"}}} or null} Nothing in the schema is a copy of either project's structure: it is what this project's own analysis needs, which is a clip's length, its exits and the earliest instant each of them can fire. If a scene table is ever COMMITTED to this repo rather than regenerated into gitignored tmp/, it becomes redistribution of derived data and the attribution in "sources" has to travel with it. FINDINGS 16, 56.5. """ import sys, os, re, json, argparse import xml.etree.ElementTree as ET DIRK = os.environ.get("DLX_DIRKSIMPLE", "tmp/scenegraph/DirkSimple") SNES = os.environ.get("DLX_SNESLAIR", "tmp/scenegraph/SNES-SuperDragonsLairArcade") # DirkSimple's own constants, from data/games/lair/game.lua. Restated here # because this file evaluates its table; both are gated below against the text # of the file they came from, so they cannot go stale silently. LD_FPS = 23.976 # laserdisc_frame_to_ms ROM_OFFSET_MS = 6297.0 # time_laserdisc_frame's "magic millisecond offset" SOURCES = [ {"name": "icculus/DirkSimple", "url": "https://github.com/icculus/DirkSimple", "licence": "zlib", "holder": "Ryan C. Gordon", "role": "the scene graph itself, transcribed from the arcade ROM"}, {"name": "astrobleem/SNES-SuperDragonsLairArcade", "url": "https://github.com/astrobleem/SNES-SuperDragonsLairArcade", "licence": "MIT", "holder": "Chad Doebelin", "role": "cross-check only; itself derived from DirkSimple (FINDINGS 56.2)"}, ] # ---------------------------------------------------------------- Lua subset class LuaParser: """Just enough Lua to read game.lua's `scenes` table constructor. Not a Lua interpreter and not trying to be: the grammar is table constructors, string/number/boolean/nil literals, bare identifiers (used only for function references like `interrupt=game_over_complete`), calls to four known helpers, and + / - between them. Anything else is a parse error rather than a silent skip, and the caller asserts the whole table was consumed, so a change upstream shows up as a failure and not as a smaller scene graph. """ def __init__(self, text, helpers): self.s, self.i, self.helpers = text, 0, helpers def error(self, msg): line = self.s.count("\n", 0, self.i) + 1 raise SyntaxError(f"{msg} at line {line}: {self.s[self.i:self.i+60]!r}") def ws(self): while self.i < len(self.s): c = self.s[self.i] if c in " \t\r\n": self.i += 1 elif self.s.startswith("--", self.i): self.i = self.s.find("\n", self.i) if self.i < 0: self.i = len(self.s) else: return def take(self, lit): self.ws() if self.s.startswith(lit, self.i): self.i += len(lit) return True return False def expect(self, lit): if not self.take(lit): self.error(f"expected {lit!r}") def name(self): self.ws() m = re.compile(r"[A-Za-z_][A-Za-z0-9_]*").match(self.s, self.i) if not m: return None self.i = m.end() return m.group(0) def value(self): self.ws() if self.take("{"): return self.table() m = re.compile(r"-?\d+(\.\d+)?").match(self.s, self.i) if m: self.i = m.end() v = float(m.group(0)) return self.arith(int(v) if v.is_integer() else v) if self.s[self.i] in "\"'": q = self.s[self.i] j = self.s.index(q, self.i + 1) v = self.s[self.i + 1:j] self.i = j + 1 return v n = self.name() if n is None: self.error("expected a value") if n == "nil": return None if n in ("true", "false"): return n == "true" if self.take("("): # a call args = [] if not self.take(")"): while True: args.append(self.value()) if self.take(")"): break self.expect(",") if n not in self.helpers: self.error(f"unknown helper {n!r}") return self.arith(self.helpers[n](*args)) return Symbol(n) # a function reference def arith(self, left): """+ and - between numbers. Present in the source and load-bearing: e.g. `time_laserdisc_frame(1823) - laserdisc_frame_to_ms(2)`.""" while True: self.ws() if self.i < len(self.s) and self.s[self.i] in "+-": op = self.s[self.i] self.i += 1 right = self.value() left = left + right if op == "+" else left - right else: return left def table(self): d, arr = {}, [] while True: self.ws() if self.take("}"): break save = self.i k = self.name() if k is not None and self.take("="): d[k] = self.value() else: self.i = save arr.append(self.value()) if not (self.take(",") or self.take(";")): self.expect("}") break if d and arr: self.error("mixed array/hash table") return arr if arr or not d else d class Symbol(str): """A bare Lua identifier used as a value (a function reference).""" def load_dirksimple(path): """Parse lair/game.lua's `scenes` and `scene_manager` tables. Gates the four timing helpers against the text that defines them, so this tool cannot keep evaluating a formula upstream has changed. """ src = open(os.path.join(path, "data/games/lair/game.lua"), encoding="utf-8").read() def gate(pat, what): if not re.search(pat, src): raise AssertionError(f"DirkSimple's {what} is not what this tool " f"evaluates any more (looked for {pat!r})") gate(r"frame\s*/\s*23\.976", "laserdisc_frame_to_ms") gate(r"-\s*6297\.0", "time_laserdisc_frame's ROM offset") gate(r"time_laserdisc_noseek.*?\n\s*return -1", "time_laserdisc_noseek") gate(r"\(seconds \* 1000\) \+ ms", "time_to_ms") helpers = { "laserdisc_frame_to_ms": lambda f: (f / LD_FPS) * 1000.0, "time_laserdisc_frame": lambda f: (f / LD_FPS) * 1000.0 - ROM_OFFSET_MS, "time_laserdisc_noseek": lambda: -1, # time_to_ms takes (seconds, ms); the source calls it with a third # argument exactly once, which Lua discards. "time_to_ms": lambda s, ms=0, *_: s * 1000 + ms, } out = {} for var in ("scenes", "scene_manager"): m = re.search(rf"^{var} = \{{", src, re.M) if not m: raise AssertionError(f"{var} table not found in game.lua") p = LuaParser(src, helpers) p.i = m.end() out[var] = p.table() # The parser must have consumed the WHOLE constructor: table() returns # with the cursor just past the matching close brace, and in this file # every top-level table ends on a `}` in column 0. A parser that # stopped early would return a smaller graph, which is the direction # that flatters the measurement, so this is checked rather than assumed. if src[p.i - 1] != "}": raise AssertionError(f"{var}: parser did not end on a close brace") if src.rfind("\n", 0, p.i) != p.i - 2: raise AssertionError(f"{var}: the constructor ended mid-line at " f"{p.i}, so the parse stopped early") return out["scenes"], out["scene_manager"] # ------------------------------------------------- the SNES cross-check def load_snes(path): """Parse the SNES project's chapter XMLs into {chapter: (start_ms, end_ms, [(input, from_ms, to_ms, target)])}, plus the provenance line.""" ev = os.path.join(path, "data/events") readme = os.path.join(ev, "README.md") prov = None if os.path.exists(readme): txt = open(readme, encoding="utf-8").read() if "derived from DirkSimple" in txt: prov = "derived from DirkSimple game data (data/events/README.md)" def ms(e): return (int(e.get("min", 0)) * 60000 + int(e.get("second", 0)) * 1000 + int(e.get("ms", 0))) chapters = {} for f in sorted(os.listdir(ev)): if not f.endswith(".xml"): continue root = ET.parse(os.path.join(ev, f)).getroot() tl = root.find("timeline") s = tl.find("timestart") e = tl.find("timeend") acts = [] for evt in root.findall("./events/event"): if evt.get("type") != "direction": continue t = evt.find("timeline") p = evt.find("./params/str[@key='type']") r = evt.find("./result/playchapter") acts.append((p.get("value") if p is not None else "?", ms(t.find("timestart")), ms(t.find("timeend")) if t.find("timeend") is not None else -1, r.get("name") if r is not None else None)) chapters[f[:-4]] = (ms(s) if s is not None else -1, ms(e) if e is not None else -1, acts) return chapters, prov def map_abbrevs(chapters, scenes): """Match each SNES scene abbreviation to a DirkSimple scene from the DATA, not from a hand-written table, so a wrong match is visible rather than assumed. Three signals, in order: 1. the abbreviation's letters must be a SUBSEQUENCE of the scene name (`snkr` is snake_room, and cannot be black_knight however similar their `seqN` names are -- overlap alone got that one wrong); 2. the Jaccard overlap of the two sequence-name sets; 3. the reversed-scene rule: Dragon's Lair mirrors thirteen scenes, and where BOTH `x` and `xr` exist as abbreviations, `xr` is the `_reversed` scene and `x` is not. Their sequence names are identical, so nothing else separates them. Assignment is 1:1 and greedy on that ranking. """ def subseq(a, b): it = iter(b) return all(c in it for c in a) by_abbr = {} for ch in chapters: abbr, _, seq = ch.partition("_") by_abbr.setdefault(abbr, set()).add(seq) pairs = [] for abbr, seqs in by_abbr.items(): mirror = abbr.endswith("r") and abbr[:-1] in by_abbr for scene, s in scenes.items(): if not subseq(abbr, scene.replace("_", "")): continue union = len(seqs | set(s)) j = len(seqs & set(s)) / union if union else 0.0 rev = scene.endswith("_reversed") pairs.append((j, 1 if rev == mirror else 0, -len(scene), abbr, scene, len(seqs))) pairs.sort(reverse=True) mapping, used = {}, set() for j, _, _, abbr, scene, n in pairs: if abbr in mapping or scene in used: continue mapping[abbr] = (scene, j, n) used.add(scene) unmatched = [(a, None, 0.0, len(s)) for a, s in by_abbr.items() if a not in mapping] return mapping, unmatched # ------------------------------------------------------------------- emit def to_neutral(scenes, mgr): """DirkSimple's tables -> DLXSCENE1. The one place their shape is read.""" out = {} for scene, seqs in scenes.items(): s = out.setdefault(scene, {}) for name, seq in seqs.items(): t = seq["timeout"] s[name] = { "start_ms": float(seq["start_time"]), "timeout_ms": float(t["when"]), "timeout_next": t.get("nextsequence"), "kills": bool(seq.get("kills_player")), "single_frame": bool(seq.get("is_single_frame")), "actions": [{"input": a["input"], "from_ms": float(a["from"]), "to_ms": float(a["to"]), "next": a.get("nextsequence")} for a in seq.get("actions", [])], } rows = [list(r) for r in mgr["rows"]] return out, rows def crosscheck_neutral(chapters, scenes): """The SNES chapters, keyed the way OUR table is keyed. Their `abbr_sequence` naming is resolved to a scene here, from the data (see map_abbrevs), so nothing downstream has to know an abbreviation exists. Chapters the mapping cannot place are dropped and counted. """ mapping, unmatched = map_abbrevs(chapters, scenes) out, dropped = {}, 0 for ch, (s_ms, e_ms, acts) in chapters.items(): abbr, _, seq = ch.partition("_") if abbr not in mapping: dropped += 1 continue scene = mapping[abbr][0] pre = abbr + "_" out.setdefault(scene, {})[seq] = { "chapter": ch, "start_ms": float(s_ms), "end_ms": float(e_ms), "actions": [{"input": i, "from_ms": float(f), "to_ms": float(t2), "next": (n[len(pre):] if n and n.startswith(pre) else n)} for i, f, t2, n in acts], } weak = sorted((round(j, 3), ab, sc) for ab, (sc, j, _n) in mapping.items() if j < 1.0) return out, {"chapters": len(chapters), "dropped": dropped, "scenes_mapped": len(mapping), "weakest_mappings": weak[:6]} def main(): ap = argparse.ArgumentParser() ap.add_argument("-o", "--out", default="tmp/scenegraph.json") a = ap.parse_args() if not os.path.isdir(DIRK): sys.exit(f"no DirkSimple checkout at {DIRK} -- set DLX_DIRKSIMPLE, or\n" f" git clone --depth 1 https://github.com/icculus/DirkSimple") scenes, mgr = load_dirksimple(DIRK) neutral, rows = to_neutral(scenes, mgr) nseq = sum(len(s) for s in neutral.values()) nwin = sum(len(q["actions"]) for s in neutral.values() for q in s.values()) cross, cross_meta = None, None if os.path.isdir(SNES): chapters, prov = load_snes(SNES) cross, cross_meta = crosscheck_neutral(chapters, scenes) cross_meta["provenance"] = prov # 56.2: this is the sentence the whole "second source" claim turned on. # If upstream ever removes it, say so rather than silently promoting a # copy back to an independent transcription. if not prov: cross_meta["provenance"] = ("UNSTATED -- data/events/README.md no " "longer says where the chapters came " "from; re-check before trusting this " "as anything.") doc = {"format": "DLXSCENE1", "sources": SOURCES, "scenes": neutral, "rows": rows, "counts": {"scenes": len(neutral), "sequences": nseq, "windows": nwin}, "crosscheck": cross, "crosscheck_meta": cross_meta} os.makedirs(os.path.dirname(a.out) or ".", exist_ok=True) with open(a.out, "w") as f: json.dump(doc, f, indent=1, sort_keys=True) print(f"{a.out}: {len(neutral)} scenes, {nseq} sequences, {nwin} input " f"windows, {len(rows)} rows" + (f", cross-check {cross_meta['chapters']} chapters" if cross_meta else ", no cross-check (SNES tree absent)")) return 0 if __name__ == "__main__": sys.exit(main())