From 0a64728faec4d8fb3f85dd9c35de2a48f886ee7c Mon Sep 17 00:00:00 2001 From: Sylpheed RE agent Date: Wed, 26 Aug 2026 10:06:23 +0000 Subject: [PATCH] re: retract the paint-key census and redo it over all 21 184 sprites The census filtered pak entries whose own first four bytes are T8aD. A sprite is usually a child of a RATC bundle, and a bundle entry's magic is RATC, so a top-level magic filter cannot see one: top-level T8aD entries (counted) 4 525 sprites, 45 keys T8aD inside RATC bundles (missed) 16 659 sprites, 204 keys both 21 184 sprites, 216 keys 171 of the 216 keys exist only inside bundles. The sharpest statement of the error: that census never saw GP_TITLE.pak at all -- the pak holding both of the screens this page's entire evidence comes from. Retracted: "45 values", "the keys are pak-local", "each auxiliary pak occupies its own narrow high-byte band". On the full population 68/216 keys (31%, not 9%) cross a pak family and the per-pak ranges overlap heavily -- GP_BUNK 0x8000-0xa110, GP_TITLE 0x8000-0xc150, GP_LEADERBOARD 0x8000-0xf100. The tidy banding was an artifact of seeing one or two keys per pak. So the key looks like a shared vocabulary, which is the opposite of what I published. Survives, now on the full population: the field is a u16 at +0x0A (upper half zero 21 184/21 184), and it is an enumeration (216 values for 21 184 sprites). Three wrong numbers on this page now, all the same shape -- a statistic computed over a population I had not checked was the population in question. Stated once at the end of the section rather than three times: check the sampling frame before the statistic. --- docs/re/BACKLOG.md | 23 ++--- docs/re/data/paint-key-census.txt | 48 ++++++---- docs/re/structures/ui-paint-order-key.md | 108 +++++++++++------------ tools/re-capture/paint_key_census.py | 73 +++++++++------ 4 files changed, 145 insertions(+), 107 deletions(-) diff --git a/docs/re/BACKLOG.md b/docs/re/BACKLOG.md index ca9ed97d..01a87190 100644 --- a/docs/re/BACKLOG.md +++ b/docs/re/BACKLOG.md @@ -143,17 +143,20 @@ where translucent sprites overlap. * ❔ What the field's bits mean — `0x8000`/`0x80a0`/`0xa110` look like flag words with a layer in some bits, not a plain depth. Sorting the whole word works on both measured screens; which bits carry the layer is unknown. - **Censused 2026-08-26** over all **4 525** disc sprites + **Censused 2026-08-26** over **all 21 184** disc sprites (`tools/re-capture/paint_key_census.py`): ✅ the field is a **`u16` at `+0x0A`** - (upper half zero **4 525/4 525**), ✅ it is an **enumeration — 45 values**, and - 🔴 **it is pak-local, so it is not a global layer vocabulary**: only **4/45** - keys cross a pak family and **33/45** live only in `GP_MAIN_GAME_2D`, each - other pak owning a narrow high-byte band (`0x90`–`0x94` overlays, `0xa4` - mission log, `0xb1`–`0xb2` save/load). Supports "group id in the high bits, - order in the low bits" but does **not** test it — both measured screens are - inside one pak. ⚠️ My first count said 37/45 shared; that was the six - *language* copies of `GP_MAIN_GAME_2D` (identical key sets) counted as six - paks, and it pointed at the opposite conclusion. + (upper half zero **21 184/21 184**) and ✅ an **enumeration — 216 values**. + 🔴 **"the keys are pak-local" is REFUTED** — **68/216 (31 %)** cross a pak + family and the per-pak ranges overlap heavily, so it looks like a shared + vocabulary, not a per-screen depth. ❔ Which bits carry the layer is still + unknown. + ⚠️ **A first version of this census was retracted.** It filtered pak entries on + a `T8aD` magic, but sprites are usually **`RATC` children** — it saw 4 525 of + 21 184 sprites, 45 of 216 keys, and **not `GP_TITLE.pak` at all**, the pak both + measured screens come from. It reported "45 values" and "pak-local", both wrong. + A separate earlier slip on the same page counted the six *language* copies of + `GP_MAIN_GAME_2D` as six paks. Same shape each time: a statistic computed over + an unverified sampling frame. * ❔ A third measured permutation, to promote "holds on two" to a rule. The cheapest is a screen whose object is resident at the same time as the title's. * ❔ 341 builds now composite in an order no capture has checked. diff --git a/docs/re/data/paint-key-census.txt b/docs/re/data/paint-key-census.txt index f64a8a54..2cda0040 100644 --- a/docs/re/data/paint-key-census.txt +++ b/docs/re/data/paint-key-census.txt @@ -1,17 +1,35 @@ -pak families: {'GP_CHALLENGE.pak': 2, 'GP_DEBRIEFING_PILOTLOG.pak': 2, 'GP_HANGAR_ARSENAL.pak': 3, 'GP_LEADERBOARD.pak': 3, 'GP_MAIN_GAME_2D.pak': 34, 'GP_MISSION_LOG.pak': 1, 'GP_MISSION_SELECT.pak': 1, 'GP_OPTIONS.pak': 1, 'GP_SAVE_LOAD.pak': 3, 'GP_STAGE_CLEAR.pak': 1} - +ALL T8aD sprites (top-level + RATC children): 21184 +upper half of the +0x08 word is zero : 21184/21184 +distinct u16 keys at +0x0A : 216 +keys in more than one pak family : 68/216 = 31% +keys confined to GP_MAIN_GAME_2D : 52/216 the six language 2D paks have identical key sets: True (6 paks) -keys in more than one FAMILY: 4/45 -> ['0000', '9092', '9110', '9200'] -keys confined to GP_MAIN_GAME_2D alone: 33/45 -per family, its keys: - GP_CHALLENGE.pak: ['9110', '9200'] - GP_DEBRIEFING_PILOTLOG.pak: ['9091', '9092'] - GP_HANGAR_ARSENAL.pak: ['9200', '9300', '9400'] - GP_LEADERBOARD.pak: ['9108', '9110', '9200'] - GP_MAIN_GAME_2D.pak: ['0000', '7d00', '7e00', '7e10', '7e20', '7e40', '7e50', '7e51', '7e52', '7e80', '7e90', '7f00', '7f10', '7f20', '7f30', '7f40', '7f60', '7f61', '8000', '8010', '8020', '8030', '8040', '8080', '8100', '8108', '8110', '8200', '8210', '8300', '8800', '8880', '8900', '9820'] - GP_MISSION_LOG.pak: ['a410'] - GP_MISSION_SELECT.pak: ['9110'] - GP_OPTIONS.pak: ['0000'] - GP_SAVE_LOAD.pak: ['b100', 'b110', 'b210'] - GP_STAGE_CLEAR.pak: ['9092'] +keys per family: + GP_BUNK.pak: 16 keys 8000..a110 + GP_CHALLENGE.pak: 9 keys 8000..9200 + GP_DEBRIEFING_PILOTLOG.pak: 17 keys 8200..9900 + GP_DIALOG.pak: 22 keys 8100..f102 + GP_GAMEOVER.pak: 6 keys d840..d862 + GP_HANGAR_ARSENAL.pak: 20 keys 8000..9412 + GP_LEADERBOARD.pak: 14 keys 8000..f100 + GP_MAIN_GAME_2D.pak: 72 keys 0000..9820 + GP_MISSION_LOG.pak: 5 keys a400..a420 + GP_MISSION_SELECT.pak: 13 keys 8300..9820 + GP_MOVIE_THEATER.pak: 7 keys 8300..9122 + GP_OPTIONS.pak: 21 keys 0000..b900 + GP_PAUSE_MENU.pak: 8 keys 8020..a112 + GP_READY_ROOM.pak: 45 keys 0000..9fff + GP_SAVE_LOAD.pak: 27 keys 8300..f100 + GP_STAGE_CLEAR.pak: 12 keys 8200..9112 + GP_SYSTEM.pak: 7 keys a010..a112 + GP_TITLE.pak: 28 keys 8000..c150 + GP_TUTORIAL.pak: 5 keys 8210..8312 + deu.pak: 5 keys 8000..8010 + eng.pak: 5 keys 8000..8010 + esp.pak: 5 keys 8000..8010 + fra.pak: 5 keys 8000..8010 + ita.pak: 5 keys 8000..8010 + jpn.pak: 5 keys 8000..8010 + +commonest 12 keys: [('8000', 1768), ('8100', 1524), ('9230', 1454), ('9400', 857), ('9220', 776), ('9402', 720), ('8010', 689), ('9110', 527), ('9100', 467), ('8900', 450), ('8400', 426), ('a100', 423)] diff --git a/docs/re/structures/ui-paint-order-key.md b/docs/re/structures/ui-paint-order-key.md index 224be167..729c53d3 100644 --- a/docs/re/structures/ui-paint-order-key.md +++ b/docs/re/structures/ui-paint-order-key.md @@ -326,72 +326,72 @@ X/Y, the `T8aD` header words, and `kind`. ## The corpus-wide census (2026-08-26) -Everything above rests on twelve values from two screens. This walks **4 525 -`T8aD` sprites on the disc**, so the open question can be asked of a corpus. `tools/re-capture/paint_key_census.py`, output in -[`../data/paint-key-census.txt`](../data/paint-key-census.txt). +[`../data/paint-key-census.txt`](../data/paint-key-census.txt). **All 21 184 +`T8aD` sprites on the disc** — pak entries *and* `RATC` children. -> ⚠️ **Read the next paragraph before using these numbers.** The census counts -> pak entries whose *own* first four bytes are `T8aD`. But the sprites this page -> measures paint order on are **children of a `RATC` bundle** — `ui_layout.rs` -> reaches them through `ratc::parse`, and a bundle entry's magic is `RATC`, so a -> child `T8aD` is invisible to a top-level magic filter. The two populations may -> overlap partly, wholly, or not at all, and **which one the 45 keys describe is -> being measured, not assumed.** Until that lands, treat every count below as a -> statement about top-level `T8aD` entries only. -> -> This is the same failure shape as the 37/45 note further down: a number -> computed over a population I had not checked was the population in question. +### ⚠️ The first version of this section counted 4 525 sprites, and was retracted + +It filtered pak entries whose own first four bytes are `T8aD`. But a sprite is +usually a **child of a `RATC` bundle**, and a bundle entry's magic is `RATC`, so +a top-level magic filter cannot see one: + +| | sprites | distinct keys | +|---|---|---| +| top-level `T8aD` entries (what was counted) | 4 525 | 45 | +| `T8aD` inside `RATC` bundles (what was missed) | **16 659** | **204** | +| both | **21 184** | **216** | + +**171 of the 216 keys exist only inside bundles.** The sharpest way to put it: +that census never saw **`GP_TITLE.pak` at all** — the pak holding both screens +this page's evidence comes from. Every number it produced described a fifth of +the corpus, chosen by an accident of container nesting. + +Retracted from it: "45 values", "the keys are pak-local", and "each auxiliary pak +occupies its own narrow high-byte band". What survives is below. ### ✅ The field is a `u16` at `+0x0A`, not a word at `+0x08` -The upper half of the 32-bit word this page reads is **zero in 4 525 / 4 525** -sprites. Nothing above changes — `0x00008100` sorts identically to `0x8100` — -but the port should read two bytes, and a future value with the high half set -would mean something has been misread rather than that the layer got deeper. +The upper half of the 32-bit word this page reads is **zero in 21 184 / 21 184** +sprites. Nothing above changes — `0x00008100` sorts identically to `0x8100` — but +the port should read two bytes, and a value with the high half set would mean +something has been misread rather than that the layer got deeper. -### ✅ It is an enumeration: 45 values for 4 525 sprites +### ✅ It is an enumeration: 216 values for 21 184 sprites -Not a per-sprite depth. The commonest key alone (`0x8100`) covers 1 188 sprites, -and the whole range is `0x0000`, `0x7d00`–`0x9820`, then `0xa410`, `0xb100`–`0xb210`. +Not a per-sprite depth. The commonest key (`0x8000`) covers 1 768 sprites, +`0x8100` another 1 524. -### 🔴 The keys are **pak-local** — so this is not a global layer vocabulary +### 🔴 Refuted: the keys are **not** partitioned by pak -That was the reading worth trying, and it fails. Collapsing the six language -variants of `GP_MAIN_GAME_*2D.pak` into one family: +This was the previous section's headline and the fuller population kills it. -| | | -|---|---| -| keys appearing in more than one pak family | **4 / 45** (`0x0000`, `0x9092`, `0x9110`, `0x9200`) | -| keys confined to `GP_MAIN_GAME_2D` alone | **33 / 45** | +| | 4 525-sprite version | **all 21 184** | +|---|---|---| +| keys crossing a pak family | 4 / 45 = 9 % | **68 / 216 = 31 %** | +| keys confined to `GP_MAIN_GAME_2D` | 33 / 45 | **52 / 216** | -and each auxiliary pak occupies its own narrow high-byte band: +And the per-pak *ranges overlap heavily* rather than banding: `GP_BUNK` +`0x8000`–`0xa110`, `GP_CHALLENGE` `0x8000`–`0x9200`, `GP_HANGAR_ARSENAL` +`0x8000`–`0x9412`, `GP_LEADERBOARD` `0x8000`–`0xf100`, `GP_TITLE` +`0x8000`–`0xc150`. The tidy "each pak owns a high-byte band" picture was an +artifact of seeing one or two keys per pak. Only three paks are actually narrow +(`GP_GAMEOVER` `0xd840`–`0xd862`, `GP_MISSION_LOG` `0xa400`–`0xa420`, +`GP_SYSTEM` `0xa010`–`0xa112`). -| pak | its keys | -|---|---| -| `GP_MAIN_GAME_2D` | `0x7d00`–`0x9820` (34 keys) | -| `GP_DEBRIEFING_PILOTLOG` | `0x9091`, `0x9092` | -| `GP_LEADERBOARD` | `0x9108`, `0x9110`, `0x9200` | -| `GP_CHALLENGE` | `0x9110`, `0x9200` | -| `GP_HANGAR_ARSENAL` | `0x9200`, `0x9300`, `0x9400` | -| `GP_STAGE_CLEAR` | `0x9092` | -| `GP_MISSION_SELECT` | `0x9110` | -| `GP_MISSION_LOG` | `0xa410` | -| `GP_SAVE_LOAD` | `0xb100`, `0xb110`, `0xb210` | +So the key is closer to a **shared vocabulary** than to a per-screen depth — the +opposite of what the under-sampled census said. ❔ Which bits carry the layer +remains unknown, and is now a question about 216 values rather than 45. -A screen that owns one or two keys is not using them to order itself internally — -it has nothing to order against. So the high bits look like a **group or owner -id** and the low bits like the ordering within it, which is the shape this page -already suspected. 🟡 That remains a *description of 45 numbers*: paint order has -been measured on two screens only, both inside `GP_MAIN_GAME_2D`, so there is no -ground truth anywhere else and the split between "group" and "order" bits is -still untested. +### ✅ The language paks are one screen set six times -### ⚠️ The first count I got was 37 / 45, and it was about localisation +The six `GP_MAIN_GAME_*2D.pak` have **byte-for-byte identical key sets**. This is +why counting paks rather than pak *families* first reported "37 of 45 keys appear +in more than one pak" — a statement about localisation, not about the format. -Counting paks rather than pak *families* makes every `GP_MAIN_GAME_2D` key look -like it appears in six paks. The six are the same screens in six languages — and -their key sets are **byte-for-byte identical** (verified: one distinct set across -all six). The inflated number would have supported exactly the wrong conclusion, -that the keys form a vocabulary shared across the whole UI. Recorded because the -shape recurs: a corpus with near-duplicate members will manufacture agreement. +### The recurring failure, stated once + +Three published numbers on this page were wrong the same way: each was computed +over a population I had not checked was the population in question — the six +language copies, then the top-level-only sprites, then the size of that gap. The +analysis was never the problem. **Check the sampling frame before the statistic.** diff --git a/tools/re-capture/paint_key_census.py b/tools/re-capture/paint_key_census.py index de20cd53..e507e21e 100755 --- a/tools/re-capture/paint_key_census.py +++ b/tools/re-capture/paint_key_census.py @@ -2,48 +2,65 @@ """Census the T8aD paint-order key across every sprite on the disc. `structures/ui-paint-order-key.md` established that this field sorts a screen's -paint order, from twelve values on two measured screens. This walks all 4 525 -sprites so the open question -- what the bits mean -- is asked of the whole -corpus rather than of one screen. +paint order, from twelve values on two measured screens. This walks all 21 184 +sprites so the open question -- what the bits mean -- is asked of the corpus. ./paint_key_census.py -Note the field is a **u16 at +0x0A**: the doc reads a 32-bit word at +0x08 and -its upper half is zero in every sprite. +The field is a **u16 at +0x0A**: the doc reads a 32-bit word at +0x08 and its +upper half is zero in every sprite. -The language variants of `GP_MAIN_GAME_*2D.pak` are collapsed into one family. -They hold the same screens in six languages, and counting them separately makes -every one of their keys look shared across six paks -- which is how this script -first reported "37 of 45 keys appear in more than one pak", a statement about -localisation and not about the format. +Two traps this script exists to avoid, both of which produced a wrong published +number before it did: + + * **Sprites are mostly RATC children, not pak entries.** A bundle entry's + magic is `RATC`, so filtering pak entries on a `T8aD` magic finds only + 4 525 of the 21 184 sprites and 45 of the 216 keys -- and misses GP_TITLE + entirely, which is the screen the page's own evidence comes from. + * **The language variants of GP_MAIN_GAME_*2D.pak are the same screens six + times.** Counting them separately makes every one of their keys look shared + across six paks. They are collapsed into one family here. """ import sys, glob, struct, re, collections + S="/tmp/claude-1000/-home-fabi-RE-Project-Sylpheed/b113cc12-4769-4ed8-ad66-c2b48f800773/scratchpad" sys.path.insert(0,S+"/wt-root/Syplheed-Reborn/tools/re-capture") exec(open(S+"/regn_decode.py").read().split("# ── the object")[0]) Hh=lambda b,o: struct.unpack_from(">H",b,o)[0] -U=lambda b,o: struct.unpack_from(">I",b,o)[0] -def family(n): return re.sub(r'_[DEFIJS]2D\.pak$','_2D.pak',n) -byfam=collections.defaultdict(collections.Counter); keys=collections.Counter() -langsets=collections.defaultdict(set) +def fam(n): return re.sub(r'_[DEFIJS]2D\.pak$','_2D.pak',n) +keys=collections.Counter(); kfam=collections.defaultdict(set) +hi_zero=tot=0; langsets=collections.defaultdict(set) +def take(b,o,base,lang): + global hi_zero,tot + tot+=1 + if Hh(b,o+8)==0: hi_zero+=1 + k=Hh(b,o+10); keys[k]+=1; kfam[k].add(base) + if lang: langsets[lang].add(k) ROOT = sys.argv[1] if len(sys.argv) > 1 else "/work/sylph_extract" for f in sorted(glob.glob(ROOT + "/**/*.pak", recursive=True)): try: E=pak_entries(f) except Exception: continue - base=f.split('/')[-1] + nm=f.split('/')[-1]; base=fam(nm); lang=nm if nm.endswith('2D.pak') else None for h,b in (E.items() if isinstance(E,dict) else E): - if b[:4]!=b'T8aD': continue - k=Hh(b,10); keys[k]+=1; byfam[family(base)][k]+=1 - if base.endswith('2D.pak'): langsets[base].add(k) -fams=sorted(byfam) -print("pak families:",{f:len(c) for f,c in byfam.items()}) -print("\nthe six language 2D paks have identical key sets:", - len({frozenset(v) for v in langsets.values()})==1, f"({len(langsets)} paks)") -multi=[k for k in keys if sum(1 for f in fams if k in byfam[f])>1] -print(f"keys in more than one FAMILY: {len(multi)}/{len(keys)} -> {[('%04x'%k) for k in sorted(multi)]}") -only2d=[k for k in keys if list(f for f in fams if k in byfam[f])==['GP_MAIN_GAME_2D.pak']] -print(f"keys confined to GP_MAIN_GAME_2D alone: {len(only2d)}/{len(keys)}") -print("\nper family, its keys:") + m=bytes(b[:4]) + if m==b'T8aD': take(b,0,base,lang) + elif m==b'RATC': + i=b.find(b'T8aD',4) + while i!=-1: + if i+12<=len(b): take(b,i,base,lang) + i=b.find(b'T8aD',i+4) +print(f"ALL T8aD sprites (top-level + RATC children): {tot}") +print(f"upper half of the +0x08 word is zero : {hi_zero}/{tot}") +print(f"distinct u16 keys at +0x0A : {len(keys)}") +fams=sorted({f for s in kfam.values() for f in s}) +multi=[k for k in keys if len(kfam[k])>1] +print(f"keys in more than one pak family : {len(multi)}/{len(keys)} = {100*len(multi)/len(keys):.0f}%") +only=[k for k in keys if kfam[k]=={'GP_MAIN_GAME_2D.pak'}] +print(f"keys confined to GP_MAIN_GAME_2D : {len(only)}/{len(keys)}") +print(f"the six language 2D paks have identical key sets: {len({frozenset(v) for v in langsets.values()})==1} ({len(langsets)} paks)") +print(f"\nkeys per family:") for f in fams: - print(f" {f:>28}: {sorted('%04x'%k for k in byfam[f])}") + ks=sorted(k for k in keys if f in kfam[k]) + print(f" {f:>28}: {len(ks):3d} keys {ks[0]:04x}..{ks[-1]:04x}") +print(f"\ncommonest 12 keys: {[('%04x'%k,n) for k,n in keys.most_common(12)]}")