diff --git a/crates/sylpheed-formats/examples/ratc_optless_children.rs b/crates/sylpheed-formats/examples/ratc_optless_children.rs new file mode 100644 index 00000000..7f9e7755 --- /dev/null +++ b/crates/sylpheed-formats/examples/ratc_optless_children.rs @@ -0,0 +1,119 @@ +//! The 60 RATC children that carry no `opt ` block — do they lack it, or is our +//! window too small? +//! +//! [`ratc::parse`] now prefers the name a child's own `opt ` block states, and +//! falls back to the old backwards printable-run scan when there is no block +//! within 128 bytes. That fallback fires for 60 of the disc's 18 002 children +//! (0.3 %), and the reach of the finding in `docs/re/structures/ratc-child-names.md` +//! stops there: "whether they genuinely lack the block or sit past the search +//! window is not established". +//! +//! This settles that. For every child with no accepted block it reports +//! +//! * whether an `opt ` tag exists at all further back, and how far; +//! * which guard rejected a tag that WAS in the window (length, gap, charset); +//! * the child's position in its bundle and its magic, in case the opt-less +//! ones are structurally distinct (e.g. always the first child); +//! * the raw bytes before the magic, so the fallback's answer can be judged. +//! +//! cargo run -p sylpheed-formats --example ratc_optless_children -- ... +use sylpheed_formats::{pak, ratc}; + +/// Why a child has no accepted `opt ` name. Mirrors `ratc::opt_name`'s guards +/// one for one, so a rejection here is the same rejection the parser made. +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +enum Why { + /// No `opt ` tag in the 128-byte window, and none anywhere before it either. + NoTagAtAll, + /// No tag in the window, but one exists further back, this many bytes away. + TagBeyondWindow(usize), + /// Tag found, but its BE32 length is 0 or > 64. + BadLength(usize), + /// Tag found, name ends more than 8 bytes before the magic — a neighbour's. + GapTooBig(usize), + /// Tag found, the named bytes are not all printable ASCII. + NotGraphic, +} + +/// The parser's own window. +const WINDOW: usize = 128; + +fn classify(buf: &[u8], at: usize) -> Option { + let lo = at.saturating_sub(WINDOW); + let pos = match buf[lo..at].windows(4).rposition(|w| w == b"opt ") { + Some(p) => lo + p, + None => { + // Widen to the whole buffer before the child: is it merely far away? + return Some(match buf[..at].windows(4).rposition(|w| w == b"opt ") { + Some(p) => Why::TagBeyondWindow(at - p), + None => Why::NoTagAtAll, + }); + } + }; + let len = u32::from_be_bytes(buf.get(pos + 4..pos + 8)?.try_into().ok()?) as usize; + if len == 0 || len > 64 || pos + 8 + len > at { + return Some(Why::BadLength(len)); + } + let gap = at - (pos + 8 + len); + if gap > 8 { + return Some(Why::GapTooBig(gap)); + } + let s = String::from_utf8_lossy(&buf[pos + 8..pos + 8 + len]); + if s.is_empty() || !s.chars().all(|c| c.is_ascii_graphic()) { + return Some(Why::NotGraphic); + } + None // accepted — this child is not one of the 60 +} + +fn main() { + let mut total = 0usize; + let mut rows: Vec<(String, usize, usize, String, String, Why)> = Vec::new(); + let mut first_child_of_bundle = 0usize; + for path in std::env::args().skip(1) { + let Ok(ar) = pak::PakArchive::open(&path) else { continue }; + let short = path.rsplit('/').next().unwrap_or(&path).to_string(); + let entries: Vec<_> = ar.entries().to_vec(); + for (ei, e) in entries.iter().enumerate() { + let Ok(bytes) = ar.read(e) else { continue }; + let Some(kids) = ratc::parse(&bytes) else { continue }; + for (ci, c) in kids.iter().enumerate() { + total += 1; + let Some(why) = classify(&bytes, c.offset) else { continue }; + if ci == 0 { + first_child_of_bundle += 1; + } + let lo = c.offset.saturating_sub(24); + let hex = bytes[lo..c.offset] + .iter() + .map(|b| format!("{b:02x}")) + .collect::>() + .join(" "); + rows.push((short.clone(), ei, ci, c.name.clone(), hex, why)); + } + } + } + println!("RATC children scanned : {total}"); + println!(" with NO accepted `opt ` block: {}", rows.len()); + println!(" ... of which are child #0 : {first_child_of_bundle}"); + + let mut by_why: std::collections::BTreeMap = Default::default(); + for r in &rows { + let k = match &r.5 { + Why::TagBeyondWindow(_) => "TagBeyondWindow".to_string(), + Why::BadLength(_) => "BadLength".to_string(), + Why::GapTooBig(_) => "GapTooBig".to_string(), + other => format!("{other:?}"), + }; + *by_why.entry(k).or_default() += 1; + } + println!("\nwhy, by cause:"); + for (k, n) in &by_why { + println!(" {k:20} x{n}"); + } + + println!("\nevery occurrence (name is what the FALLBACK scan returned):"); + for (p, ei, ci, name, hex, why) in &rows { + println!(" {p:28} entry {ei:4} child {ci:3} {name:24} {why:?}"); + println!(" 24 bytes before the magic: {hex}"); + } +} diff --git a/crates/sylpheed-formats/src/ratc.rs b/crates/sylpheed-formats/src/ratc.rs index 67b430b4..b1a8aced 100644 --- a/crates/sylpheed-formats/src/ratc.rs +++ b/crates/sylpheed-formats/src/ratc.rs @@ -110,7 +110,12 @@ fn opt_name(bytes: &[u8], off: usize) -> Option { /// between the name and the magic, so an exact-adjacency scan isn't enough. /// /// Fallback only -- [`opt_name`] is the stated name. 60 of the disc's 18 002 -/// children have no `opt ` block and still rely on this. +/// children have no `opt ` block and still rely on this, and all 60 are +/// accounted for: they are the ten frames of the disc's only `.tan` frame +/// sequence, in six language copies of one `GP_READY_ROOM` bundle, where a +/// single `opt ` block names the whole run. See +/// `docs/re/structures/ratc-tan-frame-sequence.md` -- and note that this means +/// `parse` OVER-reports there, listing frames as children. fn name_before(bytes: &[u8], off: usize) -> String { let start = off.saturating_sub(96); let window = &bytes[start..off]; diff --git a/docs/port/HANDOFF.md b/docs/port/HANDOFF.md index 35f8468a..e8988292 100644 --- a/docs/port/HANDOFF.md +++ b/docs/port/HANDOFF.md @@ -278,6 +278,17 @@ authored version can be deleted. it): correct, but wasted fill. Draw only the full-res one, taking its *timing* from `ptbase`'s element, which carries the keyframes. [`structures/ratc-child-names.md`](../re/structures/ratc-child-names.md) + ✅ **And the fix has no remaining hole.** 60 of the disc's 18 002 RATC children + still have no `opt ` block; all 60 are now accounted for and **none is on your + screens**. They are the ten frames of the disc's only `.tan` **frame sequence** + (`pb_f15_eg_anm.tan`, six language copies of one `GP_READY_ROOM` bundle), where + a single `opt ` block names the whole run — so a name-resolution miss is not + hiding anything else the way `8AX` was. ⚠️ Two notes if you ever read outside + `GP_TITLE`: `ratc::parse` **over-reports** there, listing a `.tan`'s frames as + anonymous children; and a RATC bundle names exactly six kinds of resource — + `.t32` (14 756), `.rat` (3 311), `.prm` (367), `.tbm` (224), `.sbo` (54), + `.tan` (6). + [`structures/ratc-tan-frame-sequence.md`](../re/structures/ratc-tan-frame-sequence.md)
the original entry, kept because its reasoning still stands 🟡 **`screen render` silently drops one full-screen element per screen — and diff --git a/docs/re/INDEX.md b/docs/re/INDEX.md index 641575f5..fc5177e8 100644 --- a/docs/re/INDEX.md +++ b/docs/re/INDEX.md @@ -21,7 +21,7 @@ Promote to a prose `structures/…md` file when a format needs behavioural notes | IDXD nameless field keys | ✅/❌ | [idxd-unnamed-keys](structures/idxd-unnamed-keys.md) + [`tools/re-capture/idxd_unnamed_keys.py`](../../tools/re-capture/idxd_unnamed_keys.py) | Census of every field entry whose `name_off` is `0xFFFFFFFF`, disc-wide: **7 750 objects, 2 757 039 field entries, 0 parse failures**, `tag_hash` reproducing **1 271 462/1 271 462** named keys. **7 094 distinct keys are never named — and 7 052 of them are not hashes at all**, but author-assigned element ids (equal to the field's own index in 1 404 924 of 1 485 577 cases; `tag_hash("BGM_001")` is `0xC662435B` while the key valued `BGM_001.slb` is `0x000003E9`). ⚠️ **The "504 hash-keyed nameless fields" figure is 504 ENTRIES, not 504 names** — 42 distinct keys × 6 language copies × 2 records. All 42 are **ISL script-symbol hashes** in `\script\ID.tbl` (GP_READY_ROOM.pak), the link map built by `PrepareScript`'s "isl script prescanning"; 41 of 42 appear as little-endian call targets inside the `.isb` bytecode, forming a coherent launcher/helper call graph. The hash's own algebra pins the **trailing digits of 30 of the 42 names** (deltas of exactly `+0x01000001` across `stage01..09`, `stage10..16`, `challenge01..06`; `+0x01010000` across `tutorial0101..0601`). ❌ **No name was cracked, and the negative is quantified**: seven attacks up to a 3.5×10⁸ composition space found nothing above the noise floor; exhaustive preimage search recovers `"Stage01"` from its own hash but returns nothing for the real targets at ≤6 characters, and at 7 characters one target already has **1 176** preimages — a 24-bit modulus cannot name an 8+ character identifier uniquely | | XPR2 texture + cubemap | 🟡/✅ | `sylpheed-formats/src/texture.rs` + [colour check](xpr2-colour-check.md) | de-tile + A8R8G8B8 and DXT1. **Channel order ✅ confirmed against the running game**: the Delta Saber's decoded atlas is orange-dominant (median saturated hue 23.3°, *zero* cool pixels) and the game renders the same hull at 9.3° — a red↔blue swap would sit at ≈200°. Exact fidelity (gamma/sRGB curve, premultiplied alpha, per-channel scale) is 🟡 untested, since a hue comparison cannot see it; cubemap face ordering ❔. **2026-08-29, for the UI path only:** our composite is brighter than the emulator's frame by a gamma of **≈1.34–1.49** across three screens ([tone curve](structures/ui-render-tone-curve.md)) — 🟡 measured, not decoded, constrained only over render values ~0–60, and possibly canary's own `kernel_display_gamma_type = 2` (BT.709) output stage rather than the game's | | T8aD 2D texture | ✅ | `sylpheed-formats/src/t8ad.rs` | **100 % of the disc decodes** (19 216/19 216, measured). The "~15 % deferred variants" were a wrong model, not a variant: a surface is a list of **arbitrary sub-rectangles**, each with a 16-byte header of `dst X, dst Y, width, height`, not a 256×256 grid — `0x1c` is the **rectangle count**. Uncovered area stays transparent. **Colours ✅ CONFIRMED** ([k8888](structures/texture-color-k8888.md)) | -| RATC bundle | ✅ | `sylpheed-formats/src/ratc.rs` | **✅ 2026-08-29: a child's NAME is stated by an `opt ` block** (`"opt " | BE32 len | name | NUL | 3 bytes | magic`), not by the printable bytes before it. We had been guessing it from the last printable run, which is right 17 918 times and **wrong 24**, each time because the 3-byte tail is itself printable — `8AX` (×22) and `'OX` (×2). `8AX` is **not a name**; it hid `pteff05.t32`/`pteff04.t32`, the full-resolution background of all five menu screens, which `compose` then dropped with no diagnostic ([ratc-child-names](structures/ratc-child-names.md)). 🟡 60 of 18 002 children carry no `opt ` block and still use the scan. Child listing confirmed. **"One level deep" is not a limitation — there is nothing deeper**: 2 859 bundles hold 18 002 children at depth 1 and **0 at depth 2**, with no parse failures. Nested RATC blobs are **leaf records that reference siblings by name** (`opt `, the sprite name): 3 311 leaves, all embedding sibling names, **✅ 10 148 of 10 148 references resolve, as of 2026-08-29.** The 4 that did not were `pmbase.rat` → `pmbase.t32` in `GP_STAGE_CLEAR.pak`'s four language builds, recorded as "`pmbase.t32` is on the disc nowhere". It was on the disc all along, as the child the printable-run scan named `8AX` — the `opt ` name fix above resolves it in all four builds (3 686 767 B each). A dangling reference that closes itself when an unrelated decode lands is the corroboration that decode wanted | +| RATC bundle | ✅ | `sylpheed-formats/src/ratc.rs` | **✅ 2026-08-29: a child's NAME is stated by an `opt ` block** (`"opt " | BE32 len | name | NUL | 3 bytes | magic`), not by the printable bytes before it. We had been guessing it from the last printable run, which is right 17 918 times and **wrong 24**, each time because the 3-byte tail is itself printable — `8AX` (×22) and `'OX` (×2). `8AX` is **not a name**; it hid `pteff05.t32`/`pteff04.t32`, the full-resolution background of all five menu screens, which `compose` then dropped with no diagnostic ([ratc-child-names](structures/ratc-child-names.md)). ✅ 60 of 18 002 children carry no `opt ` block, and all 60 are explained: they are the ten frames of the disc's only `.tan` **frame sequence** (`pb_f15_eg_anm.tan`, six language copies of one `GP_READY_ROOM` bundle, 6 × 10 = 60), where one `opt ` block names the whole run — so `parse` over-reports frames as children there. A census of all **18 718** `opt ` names shows a bundle names exactly six kinds of resource: `.t32` 14 756, `.rat` 3 311, `.prm` 367, `.tbm` 224, `.sbo` 54, `.tan` 6 ([tan-frame-sequence](structures/ratc-tan-frame-sequence.md)). Child listing confirmed. **"One level deep" is not a limitation — there is nothing deeper**: 2 859 bundles hold 18 002 children at depth 1 and **0 at depth 2**, with no parse failures. Nested RATC blobs are **leaf records that reference siblings by name** (`opt `, the sprite name): 3 311 leaves, all embedding sibling names, **✅ 10 148 of 10 148 references resolve, as of 2026-08-29.** The 4 that did not were `pmbase.rat` → `pmbase.t32` in `GP_STAGE_CLEAR.pak`'s four language builds, recorded as "`pmbase.t32` is on the disc nowhere". It was on the disc all along, as the child the printable-run scan named `8AX` — the `opt ` name fix above resolves it in all four builds (3 686 767 B each). A dangling reference that closes itself when an unrelated decode lands is the corroboration that decode wanted | | LSTA sprite list | ✅ | `sylpheed-formats/src/lsta.rs` | A display list of inline elements: **T8aD sprites and `PRMD` primitives**. The `count` at `0x04` is **exact and counts both** — `count == T8aD + PRMD` for **64/64** lists on the disc, which retires the old "a few entries disagree" note (it compared sprites against a total including primitives). **All 1 281 sprite frames decode** after the T8aD rectangle-list fix | | IXUD subtitle + caption text | ✅ | `sylpheed-formats/src/ixud.rs` + `movie_subtitle.rs` ([container](structures/idxd-container.md) · [movie link](movie-subtitle-link.md)) | **The IXUD record/field table is decoded and wired in (2026-08-26)** — `IxudObject` mirrors `IdxdObject`; uniform 16-byte records, 12-byte fields, every offset in **chars**, and the word at `0x08` is record 0's hash, not a schema id. Verified disc-wide: **1104/1104** objects, **1476/1476** records, **628 165/628 165** named fields reproducing their `ixud_hash` (`tests/ixud_records_disc.rs`). **Caption text: 537 → 8800 lines, which is 8800 of 8800 distinct keys.** Two steps — generalising the key parser from `MSG_DEMO_*` to all **eight** families (`ACRO ADAN ADPL BIRD DEMO RHIN TCAF` use `MSG____`, `VOICE` alone inserts a family letter) took 537 → 8074; switching from **token adjacency to record fields** took it to 8800. ⚠️ An earlier "1.3 % of the game's text" figure of mine counted *occurrences across blocks* — the honest denominator is **8800 distinct keys**, so the real starting point was 6.1 %. The `DEMO` control shows why the field route matters: token adjacency finds 537 lines there, fields find **541** — it was dropping lines in the one family it was written for. | timed cues. **The movie↔subtitle↔voice link is solved — statically**, and as of 2026-08-25 read from the IDXD **record table** rather than scraped from the string pool: **104 cutscene slots binding 101 distinct movies**, 99 slots / 96 movies with a subtitle, 99 / 96 with a voice track, 22 / 22 with a telop. ⚠️ The previous counts (94 / 83 / 21) were the numbers of **distinct pool strings** — a repeat reference contributes no token, so 13 later `VOICE_D_450..454` references read as "no binding". **All 18 hokyu movies are bound**, not five. 93 of the 94 distinct subtitle members resolve; `SUBTITLE_S12B.tbl` resolves in none of the six languages — a dangling reference on the disc. The ~104 **script ids are no longer ❔**: they are literal positional field keys in `BASE_INFO`, each naming its record, and all 104 resolve. `movie_manifest::parse` now reads the record table; CSV regenerated by `examples/movie_map_csv.rs` | | Fonts (ttf/otf/ttc) | ✅ | `sylpheed-formats/src/font.rs` | standard OpenType, parsed via ttf-parser | diff --git a/docs/re/METHOD.md b/docs/re/METHOD.md index fc6a4b19..f95ada5f 100644 --- a/docs/re/METHOD.md +++ b/docs/re/METHOD.md @@ -857,3 +857,20 @@ agent's loop prompt, i.e. nowhere durable. See [`README.md`](README.md) for the that did not resolve: none" — the diagnostic was structurally unable to see it. When adding an early-out to a loop that already reports what it discards, make it report through the same channel, or it becomes a place findings go to die. + +* **"It has no name" can mean "it is not a thing that gets named."** Sixty RATC + children had no `opt ` name block and the open question was whether the block + was absent or merely outside our search window. It was neither: the sixty are + *frames*, ten each of six copies of one `.tan` animation, and one `opt ` block + names the whole run. The give-away was in the data before any hypothesis was — + the distances back to the nearest tag were an exact arithmetic progression + (`213 + n·60600`), i.e. ten different records finding the *same* tag. When a + negative result's measurements come out evenly spaced, the thing you are + counting is probably not the thing the format counts. + +* **This container OOM-kills `slb_leading_segment_disc` under default test + parallelism.** It dies with `signal: 9, SIGKILL` and no assertion — eight + threads each holding a slice of a ~1.1 GB bank. It is not a regression and not + a flake, and it reproduces when run alone. `-- --test-threads=1` passes 8/8 in + 20 s. Before believing a SIGKILL in this repo, re-run the suite serially; + before believing a *pass*, check nothing else heavy was sharing the box. diff --git a/docs/re/data/ratc-tan-frame-sequence.txt b/docs/re/data/ratc-tan-frame-sequence.txt new file mode 100644 index 00000000..b7518aef --- /dev/null +++ b/docs/re/data/ratc-tan-frame-sequence.txt @@ -0,0 +1,58 @@ +# The 60 RATC children with no `opt ` block — probe output + +``` +$ cargo run -p sylpheed-formats --example ratc_optless_children -- $SYLPHEED_DISC/dat/*.pak + +RATC children scanned : 18002 + with NO accepted `opt ` block: 60 + ... of which are child #0 : 0 + +why, by cause: + TagBeyondWindow x60 + +every occurrence (name is what the FALLBACK scan returned): + GP_READY_ROOM.pak entry 26 child 1 TagBeyondWindow(213) + 24 bytes before the magic: 00 00 00 07 00 00 00 28 00 00 00 08 00 00 00 28 00 00 00 09 00 00 00 28 + GP_READY_ROOM.pak entry 26 child 2 TagBeyondWindow(60813) + 24 bytes before the magic: 0a eb 61 00 08 eb 61 00 06 eb 61 00 05 eb 61 00 03 eb 61 00 01 eb 61 00 + ... (the remaining 54 rows are the same five bundles' children 1..10; + every distance is 213 + n*60600, i.e. the SAME tag) +``` + +## Reading bundle 26 directly — the ten are frames of one `.tan` + +``` +entry 26: 15 children, payload 706609 bytes + 0 @0x00000444 T8aD size 1933 opt@-40 name='pbf15_energie_generator2.t32' + 1 @0x00000bd1 T8aD size 60600 opt@-213 name='pb_f15_eg_anm.tan' + 2 @0x0000f889 T8aD size 60600 opt@-60813 name='pb_f15_eg_anm.tan' + 3 @0x0001e541 T8aD size 60600 opt@-121413 name='pb_f15_eg_anm.tan' + 4 @0x0002d1f9 T8aD size 60600 opt@-182013 name='pb_f15_eg_anm.tan' + 5 @0x0003beb1 T8aD size 60600 opt@-242613 name='pb_f15_eg_anm.tan' + 6 @0x0004ab69 T8aD size 60600 opt@-303213 name='pb_f15_eg_anm.tan' + 7 @0x00059821 T8aD size 60600 opt@-363813 name='pb_f15_eg_anm.tan' + 8 @0x000684d9 T8aD size 60600 opt@-424413 name='pb_f15_eg_anm.tan' + 9 @0x00077191 T8aD size 60600 opt@-485013 name='pb_f15_eg_anm.tan' + 10 @0x00085e49 T8aD size 60632 opt@-545613 name='pb_f15_eg_anm.tan' + 11 @0x00094b21 T8aD size 17523 opt@-32 name='pbf15_pd_inside2.t32' + 12 @0x00098f94 T8aD size 75068 opt@-35 name='pbenergie_generator.t32' + 13 @0x000ab4d0 T8aD size 4697 opt@-28 name='pbf15_eg_eff.t32' + 14 @0x000ac729 RATC size 264 opt@-25 name='pb_s15_eg.rat' +``` + +## Disc-wide: every `opt ` name, by extension + +``` +`opt ` blocks disc-wide: 18718 + +by extension: + .t32 x14756 + .rat x3311 + .prm x367 + .tbm x224 + .sbo x54 + .tan x6 + +.tan resources with >=1 child: 6 + GP_READY_ROOM.pak pb_f15_eg_anm.tan frames= 10 sizes=[60600, 60632] x6 bundles +``` diff --git a/docs/re/structures/ratc-child-names.md b/docs/re/structures/ratc-child-names.md index 3fb698f2..4ce00287 100644 --- a/docs/re/structures/ratc-child-names.md +++ b/docs/re/structures/ratc-child-names.md @@ -85,9 +85,13 @@ five menu screens. reproduces every one of them. A reading that fixed the 24 but disturbed the rest would be a different rule, not this one. -🟡 **Reach.** 60 children (0.3 %) have **no** `opt ` block within 128 bytes and -still fall back to the scan. None of them is on the five menu screens. Whether -they genuinely lack the block or sit past the search window is not established. +✅ **Reach — closed 2026-08-29.** 60 children (0.3 %) have **no** `opt ` block +within 128 bytes and fall back to the scan. **They are not children.** They are +the ten frames of the disc's only `.tan` resource, `pb_f15_eg_anm.tan`, in the +six language copies of one `GP_READY_ROOM` bundle — 6 × 10 = 60, the whole +population with nothing left over. One `opt ` block names the whole run, which is +why nine of the ten find no block of their own. None is on the five menu screens. +[`ratc-tan-frame-sequence.md`](ratc-tan-frame-sequence.md) ## What it changes in the composite diff --git a/docs/re/structures/ratc-tan-frame-sequence.md b/docs/re/structures/ratc-tan-frame-sequence.md new file mode 100644 index 00000000..2d78afa4 --- /dev/null +++ b/docs/re/structures/ratc-tan-frame-sequence.md @@ -0,0 +1,120 @@ +# `.tan` — one name over a run of frames, and the 60 "nameless" children it explains + +**Status:** ✅ `DECODED`, with a disc-wide check. This closes the 🟡 reach caveat +left open by [`ratc-child-names.md`](ratc-child-names.md): *"60 children have no +`opt ` block within 128 bytes and still fall back to the scan — whether they +genuinely lack the block or sit past the search window is not established."* + +**Neither.** They are not children. They are the **ten frames of a single `.tan` +resource**, and the one `opt ` block that names the whole run sits up to 545 KB +behind the last of them. + +## What was measured + +[`examples/ratc_optless_children.rs`](../../../crates/sylpheed-formats/examples/ratc_optless_children.rs) +re-runs `ratc::parse`'s own guards over every child on the disc and reports, for +each rejection, *which* guard fired and whether a tag exists further back +([data](../data/ratc-tan-frame-sequence.txt)): + +| | | +|---|---| +| RATC children scanned | 18 002 | +| with no accepted `opt ` block | **60** | +| of those, rejected by **length**, **gap** or **charset** | **0** | +| of those, rejected because the only tag is **beyond the 128-byte window** | **60** | +| of those, that are child #0 of their bundle | **0** | +| archives involved | **1** — `GP_READY_ROOM.pak` | +| bundles involved | **6** — entries 26, 30, 159, 160, 1029, 1050, each exactly 706 609 B | +| children involved | **1…10 of each**, never 0 and never 11+ | + +The distances are the tell. Within one bundle they are + +```text +213, 60 813, 121 413, 182 013, 242 613, 303 213, 363 813, 424 413, 485 013, 545 613 +``` + +— an exact arithmetic progression, step **60 600**. Ten different children all +find the **same** `opt ` tag, because there is only one. Nine of them do not have +a block that is merely far away; they have no block. + +## What they are + +Reading bundle 26 directly, without the Rust parser, the 15 "children" resolve: + +| # | offset | kind | size | `opt ` at | name | +|---|---|---|---|---|---| +| 0 | `0x000444` | T8aD | 1 933 | −40 | `pbf15_energie_generator2.t32` | +| **1…9** | `0x000bd1` … | T8aD | **60 600** each | −213 … −485 013 | **`pb_f15_eg_anm.tan`** | +| **10** | `0x085e49` | T8aD | **60 632** | −545 613 | **`pb_f15_eg_anm.tan`** | +| 11 | `0x094b21` | T8aD | 17 523 | −32 | `pbf15_pd_inside2.t32` | +| 12 | `0x098f94` | T8aD | 75 068 | −35 | `pbenergie_generator.t32` | +| 13 | `0x0ab4d0` | T8aD | 4 697 | −28 | `pbf15_eg_eff.t32` | +| 14 | `0x0ac729` | RATC | 264 | −25 | `pb_s15_eg.rat` | + +So the format is doing something perfectly ordinary that our scan had no concept +of: **`.tan` is a frame sequence.** One `opt ` block declares the resource, and +its payload is a run of equal-size `T8aD` blocks, one per frame. The name is on +the disc and always was. What was missing was the idea that one name can cover +more than one block. + +`anm` in `pb_f15_eg_anm` is the authors' own abbreviation, and it agrees. + +## The disc-wide check + +Every `opt ` block in every RATC bundle in all 33 `dat/*.pak`, by the extension +it names +([`tools/re-capture/ratc_opt_name_census.py`](../../../tools/re-capture/ratc_opt_name_census.py)): + +| extension | count | what it is | +|---|---|---| +| `.t32` | 14 756 | a `T8aD` sprite | +| `.rat` | 3 311 | a nested RATC leaf | +| `.prm` | 367 | a primitive | +| `.tbm` | 224 | — | +| `.sbo` | 54 | — | +| **`.tan`** | **6** | **a frame sequence** | +| | **18 718** | | + +A RATC bundle names exactly six kinds of resource, and **`.tan` occurs six times +on the whole disc** — all of them `pb_f15_eg_anm.tan`, one per language copy of +the same bundle, each holding **10 frames**. + +**6 × 10 = 60.** That is the entire population of opt-less children, with nothing +left over. The negative is closed, not narrowed. + +## ⚠️ What this says about `ratc::parse` + +The child list **over-reports**. `parse` finds children by scanning for the four +child magics, so a `.tan`'s ten frames are listed as ten anonymous children of the +bundle rather than as one named resource with ten frames. The disc's "18 002 +children" is therefore 18 002 *magic-delimited blocks*, of which 60 are frames. + +**Not changed here**, deliberately: nothing in the menu milestone reads a `.tan`, +and a rewrite of the child model is a bigger change than the one fact it would +buy. Recorded so that a later consumer of `.tan` knows the shape it needs. + +## ❔ Not established + +* **The frame timing.** Ten frames of the same size is a sequence; nothing here + shows the rate, whether it loops, or whether the frames are equal-duration. No + field was looked for. +* **The pixel layout of a 60 600-byte frame.** They decode as `T8aD` like any + other sprite as far as the magic goes; their dimensions were not read. +* **What `.tbm` and `.sbo` are.** They surfaced from the same census and are + recorded above as counts only. +* **The two `opt ` totals do not reconcile exactly** and are not forced to. + This census counts **18 718** blocks; the Rust audit in + [`ratc-child-names.md`](ratc-child-names.md) counts **17 942** children *with* a + block. They apply different guards — the Rust one additionally requires the + named thing to be one of the four child magics and to follow within 8 bytes, + which `.prm` / `.tbm` / `.sbo` (645 blocks) never satisfy. That accounts for + most of the 776 difference but not all of it, and the remainder was not chased. + Each number is reported as what its own script measured. + +## Scope + +`GP_READY_ROOM.pak` is **out of scope** for the menu milestone ([S1 is a +no-go](../ready-room-probe.md)), and `.tan` occurs in no other archive. **None of +the five menu screens contains a `.tan`**, so nothing the port draws changes. +This closes a caveat on a decode the port *does* depend on, rather than adding a +capability. diff --git a/tools/re-capture/ratc_opt_name_census.py b/tools/re-capture/ratc_opt_name_census.py new file mode 100644 index 00000000..2be41a70 --- /dev/null +++ b/tools/re-capture/ratc_opt_name_census.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Every `opt ` name in every RATC bundle on the disc, by extension -- and the +`.tan` frame sequences among them. + +Written to close the reach caveat in docs/re/structures/ratc-child-names.md: +60 of 18 002 RATC children carry no `opt ` block of their own. They are not +children. They are the ten frames of the disc's only `.tan` resource, and one +`opt ` block names the whole run. + + python3 tools/re-capture/ratc_opt_name_census.py + +Reads $SYLPHEED_DISC/dat/*.pak directly (IPFB TOC + Z1/zlib entries), so it does +not depend on the Rust parser it is checking. Takes a few minutes. +""" + +import struct, zlib, glob, os, collections, bisect +MAG = (b"T8aD", b"RATC", b"ttcf", b"\x89PNG") +ext = collections.Counter() +tan_sites = [] +opt_total = 0 +DISC = os.environ.get("SYLPHEED_DISC", "/work/sylph_extract") +for pakpath in sorted(glob.glob(f"{DISC}/dat/*.pak")): + base = pakpath[:-4] + pak = open(pakpath, "rb").read() + if pak[:4] != b"IPFB": continue + n = struct.unpack_from(">I", pak, 4)[0] + toc = [struct.unpack_from(">III", pak, 0x10 + 12*i) for i in range(n)] + segs = sorted(glob.glob(base + ".p[0-9][0-9]")) + if not segs: continue + data = b"".join(open(s, "rb").read() for s in segs) + for ei, (h, off, cs) in enumerate(toc): + raw = data[off:off+cs] + try: + b = zlib.decompress(raw[10:]) if raw[:2] == b"Z1" else raw + except Exception: + continue + if b[:4] != b"RATC": continue + names = [] + p = b.find(b"opt ") + while p >= 0: + ln = struct.unpack_from(">I", b, p+4)[0] if p+8 <= len(b) else 0 + if 0 < ln <= 64 and p+8+ln <= len(b): + nm = b[p+8:p+8+ln].decode('latin1', 'replace') + if nm and all(32 < ord(c) < 127 for c in nm): + names.append((p, nm)); opt_total += 1 + ext[os.path.splitext(nm)[1].lower()] += 1 + p = b.find(b"opt ", p+4) + offs, i = [], 4 + while i + 4 <= len(b): + if b[i:i+4] in MAG: + offs.append(i); i += 4 + else: i += 1 + if not names or not offs: continue + npos = [p for p, _ in names] + # each child -> index of the nearest preceding opt + owner = collections.defaultdict(list) + for k, o in enumerate(offs): + j = bisect.bisect_left(npos, o) - 1 + if j >= 0: owner[j].append(k) + for j, (p, nm) in enumerate(names): + if not nm.lower().endswith(".tan"): continue + ks = owner.get(j, []) + if not ks: continue + sizes = sorted({(offs[k+1] if k+1 < len(offs) else len(b)) - offs[k] for k in ks}) + tan_sites.append((os.path.basename(pakpath), ei, nm, len(ks), sizes)) +print(f"`opt ` blocks disc-wide: {opt_total}") +print("\nby extension:") +for e, c in ext.most_common(25): + print(f" {e or '(none)':10} x{c}") +print(f"\n.tan resources with >=1 child: {len(tan_sites)}") +seen = collections.Counter() +for t in tan_sites: + seen[(t[0], t[2], t[3], tuple(t[4]))] += 1 +for (pk, nm, fr, sz), c in sorted(seen.items()): + print(f" {pk:24} {nm:30} frames={fr:3} sizes={list(sz)} x{c} bundles")