From 39b3a4de7d2afea526c5fb863ebfa20173fe3828 Mon Sep 17 00:00:00 2001 From: Sylpheed RE agent Date: Thu, 27 Aug 2026 12:29:30 +0000 Subject: [PATCH] re: correction -- Generic is a per-FILE header record, not a table Yesterday's page said "Generic (394 per pak) is the unit datasheet". Only 114 of the 394 are. Every IDXD file carries exactly one Generic record and its schema is set by what kind of file it is. Partitioned by field set, identically in all six GP_MAIN_GAME_*.pak: 114 has HP -- a unit datasheet 204 {Count} only -- a dialogue file 64 {ID, Name, SideID, Unique} -- a character (36 TCAF + 28 ADAN) 10 {EnumAsteroidGroup} -- an asteroid group 2 degenerate 204+114+64+10+2 = 394, and 178 distinct Generic.ID = 114 unit + 64 character, the only two shapes carrying an ID. That settles the "178 IDs vs 394 records" question the previous entry left open. Positive control in the same loop: for the 204 dialogue headers Count equals the number of Message_NNN siblings, 204/204, zero mismatches. Cross-check from the other side: Maneuver = 114, Effect = 114, the carrying-entry sets are identical, every unit Generic has a Maneuver sibling, and Generic.Type splits 43 Craft + 71 Vessel -- the same 43/71/114 unit-struct-runtime.md reached from live guest memory. New tool generic_partition.py + artefact; ISL artefacts byte-identical. --- docs/re/BACKLOG.md | 28 +++++ docs/re/data/generic-record-partition.txt | 124 ++++++++++++++++++++ docs/re/structures/unit-datasheet-static.md | 40 ++++++- tools/re-capture/generic_partition.py | 43 +++++++ 4 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 docs/re/data/generic-record-partition.txt create mode 100644 tools/re-capture/generic_partition.py diff --git a/docs/re/BACKLOG.md b/docs/re/BACKLOG.md index 22bbb93b..393c592f 100644 --- a/docs/re/BACKLOG.md +++ b/docs/re/BACKLOG.md @@ -827,6 +827,34 @@ unknown, what evidence exists, and what the first step would be. Move an item in field; the stat keys in records not opened here (`MegaTons`, `BulletLimit`, `CrewCount`, `HP_CLASS`); why the roster lists 59 of 131. +* โœ… **(2026-08-27) CORRECTION โ€” `Generic` is a per-FILE HEADER record, not a + table. Only **114** of the 394 are unit datasheets.** + [structures/unit-datasheet-static](structures/unit-datasheet-static.md), + artefact `data/generic-record-partition.txt`, tool + `tools/re-capture/generic_partition.py`. + Answers "178 distinct `Generic` IDs vs 394 records per pak โ€” what are the + extra records?" from the entry below. **Every IDXD file carries exactly one + `Generic` record**, and its schema is set by what kind of file it is. + Partitioned by field set, identically in all six `GP_MAIN_GAME_*.pak`: + **114** have `HP` = unit datasheets; **204** are `{Count}` only = dialogue + files; **64** are `{ID, Name, SideID, Unique}` = characters (36 `TCAF` + + 28 `ADAN`, each with a `Faces` sibling); **10** are `{EnumAsteroidGroup}`; + 2 are degenerate (one empty, one naming `eff_n0071`). 204+114+64+10+2 = 394, + and **178 distinct `Generic.ID` = 114 unit + 64 character** โ€” the only two + shapes that carry an `ID`. + ๐Ÿงช **Positive control in the same loop**: for the 204 dialogue headers, + `Count` equals the number of `Message_NNN` siblings **204 / 204**, zero + mismatches โ€” so "header record describing its own file" is not a story fitted + to the counts. + ๐Ÿ”‘ Cross-check from the other side: `Maneuver` = 114, `Effect` = 114, the + carrying-entry sets are **identical**, every unit `Generic` has a `Maneuver` + sibling, and `Generic.Type` splits **43 `Craft` + 71 `Vessel`** โ€” the same + 43/71/114 `unit-struct-runtime.md` reached by reading live guest memory. + ๐ŸŸก Still open from the entry below: no field-by-field reconciliation of the + static sheet against the runtime capture (much of it is already done in + `unit-struct-runtime.md`); why the arsenal roster lists 59 of 131 weapons; + what `Maneuver`'s `AA_`/`AV_` prefixes select. + * โœ…โœ… **(2026-08-27) THE STATIC UNIT DATASHEET **AND** THE AI FLIGHT MODEL โ€” `Generic` (394), `Maneuver` (114), `Effect` (114). [structures/unit-datasheet-static](structures/unit-datasheet-static.md), diff --git a/docs/re/data/generic-record-partition.txt b/docs/re/data/generic-record-partition.txt new file mode 100644 index 00000000..556f08e5 --- /dev/null +++ b/docs/re/data/generic-record-partition.txt @@ -0,0 +1,124 @@ +# `Generic` is the per-FILE header record, not a table +# +# One `Generic` per IDXD pak entry; its schema is set by the file's TYPE. +# Regenerate: see docs/re/structures/unit-datasheet-static.md + +== GP_MAIN_GAME_D.pak (1004 IDXD entries) + message-file header 204 + unit datasheet 114 + character 64 + asteroid group 10 + other (0 fields) 1 + other (1 fields) 1 + TOTAL Generic 394 + distinct Generic.ID 178 (= unit 114 + character 64) + unit Generic.Type {'Craft': 43, 'Vessel': 71} + entries w/ Maneuver 114 w/ Effect 114 + CONTROL message-header Count == #Message_NNN siblings: 204 ok / 0 bad + +== GP_MAIN_GAME_D2D.pak (6 IDXD entries) + TOTAL Generic 0 + distinct Generic.ID 0 (= unit 0 + character 0) + unit Generic.Type {} + entries w/ Maneuver 0 w/ Effect 0 + CONTROL message-header Count == #Message_NNN siblings: 0 ok / 0 bad + +== GP_MAIN_GAME_E.pak (1004 IDXD entries) + message-file header 204 + unit datasheet 114 + character 64 + asteroid group 10 + other (0 fields) 1 + other (1 fields) 1 + TOTAL Generic 394 + distinct Generic.ID 178 (= unit 114 + character 64) + unit Generic.Type {'Craft': 43, 'Vessel': 71} + entries w/ Maneuver 114 w/ Effect 114 + CONTROL message-header Count == #Message_NNN siblings: 204 ok / 0 bad + +== GP_MAIN_GAME_E2D.pak (6 IDXD entries) + TOTAL Generic 0 + distinct Generic.ID 0 (= unit 0 + character 0) + unit Generic.Type {} + entries w/ Maneuver 0 w/ Effect 0 + CONTROL message-header Count == #Message_NNN siblings: 0 ok / 0 bad + +== GP_MAIN_GAME_F.pak (1004 IDXD entries) + message-file header 204 + unit datasheet 114 + character 64 + asteroid group 10 + other (0 fields) 1 + other (1 fields) 1 + TOTAL Generic 394 + distinct Generic.ID 178 (= unit 114 + character 64) + unit Generic.Type {'Craft': 43, 'Vessel': 71} + entries w/ Maneuver 114 w/ Effect 114 + CONTROL message-header Count == #Message_NNN siblings: 204 ok / 0 bad + +== GP_MAIN_GAME_F2D.pak (6 IDXD entries) + TOTAL Generic 0 + distinct Generic.ID 0 (= unit 0 + character 0) + unit Generic.Type {} + entries w/ Maneuver 0 w/ Effect 0 + CONTROL message-header Count == #Message_NNN siblings: 0 ok / 0 bad + +== GP_MAIN_GAME_I.pak (1004 IDXD entries) + message-file header 204 + unit datasheet 114 + character 64 + asteroid group 10 + other (0 fields) 1 + other (1 fields) 1 + TOTAL Generic 394 + distinct Generic.ID 178 (= unit 114 + character 64) + unit Generic.Type {'Craft': 43, 'Vessel': 71} + entries w/ Maneuver 114 w/ Effect 114 + CONTROL message-header Count == #Message_NNN siblings: 204 ok / 0 bad + +== GP_MAIN_GAME_I2D.pak (6 IDXD entries) + TOTAL Generic 0 + distinct Generic.ID 0 (= unit 0 + character 0) + unit Generic.Type {} + entries w/ Maneuver 0 w/ Effect 0 + CONTROL message-header Count == #Message_NNN siblings: 0 ok / 0 bad + +== GP_MAIN_GAME_J.pak (1004 IDXD entries) + message-file header 204 + unit datasheet 114 + character 64 + asteroid group 10 + other (0 fields) 1 + other (1 fields) 1 + TOTAL Generic 394 + distinct Generic.ID 178 (= unit 114 + character 64) + unit Generic.Type {'Craft': 43, 'Vessel': 71} + entries w/ Maneuver 114 w/ Effect 114 + CONTROL message-header Count == #Message_NNN siblings: 204 ok / 0 bad + +== GP_MAIN_GAME_J2D.pak (6 IDXD entries) + TOTAL Generic 0 + distinct Generic.ID 0 (= unit 0 + character 0) + unit Generic.Type {} + entries w/ Maneuver 0 w/ Effect 0 + CONTROL message-header Count == #Message_NNN siblings: 0 ok / 0 bad + +== GP_MAIN_GAME_S.pak (1004 IDXD entries) + message-file header 204 + unit datasheet 114 + character 64 + asteroid group 10 + other (0 fields) 1 + other (1 fields) 1 + TOTAL Generic 394 + distinct Generic.ID 178 (= unit 114 + character 64) + unit Generic.Type {'Craft': 43, 'Vessel': 71} + entries w/ Maneuver 114 w/ Effect 114 + CONTROL message-header Count == #Message_NNN siblings: 204 ok / 0 bad + +== GP_MAIN_GAME_S2D.pak (6 IDXD entries) + TOTAL Generic 0 + distinct Generic.ID 0 (= unit 0 + character 0) + unit Generic.Type {} + entries w/ Maneuver 0 w/ Effect 0 + CONTROL message-header Count == #Message_NNN siblings: 0 ok / 0 bad diff --git a/docs/re/structures/unit-datasheet-static.md b/docs/re/structures/unit-datasheet-static.md index c7c84f6a..9ec901cf 100644 --- a/docs/re/structures/unit-datasheet-static.md +++ b/docs/re/structures/unit-datasheet-static.md @@ -7,14 +7,44 @@ only measured at runtime โ€” [flight-speed-law](../flight-speed-law.md), [flight-controls-runtime](../flight-controls-runtime.md), [unit-struct-runtime](unit-struct-runtime.md). +> ## โŒ CORRECTION (2026-08-27) โ€” `Generic` is a per-FILE header, not a table +> +> This page first read "`Generic` (394 per pak) is the unit datasheet". Wrong: +> **only 114 of the 394 are.** `Generic` is the record name every IDXD *file* +> gives its own header record โ€” exactly one per pak entry โ€” and its schema is +> set by what kind of file it is. Partitioned by field set, in each of the six +> `GP_MAIN_GAME_*.pak` identically: +> +> | `Generic` shape | per pak | what the file is | +> |---|---:|---| +> | has `HP` | **114** | **a unit datasheet** (`Maneuver`, `Effect`, `Shield`, `Mass`, `SE`, `Explosion`, `Turret_00N` siblings) | +> | `{Count}` only | 204 | a **dialogue file** โ€” `Count` is the number of `Message_NNN` siblings | +> | `{ID, Name, SideID, Unique}` | 64 | a **character** (`Faces` sibling) โ€” 36 `TCAF` + 28 `ADAN` | +> | `{EnumAsteroidGroup}` | 10 | an asteroid-group file | +> | 0 and 1 field | 2 | one empty, one naming `eff_n0071` | +> +> 204 + 114 + 64 + 10 + 2 = 394, and **178 distinct `Generic.ID` = 114 unit + +> 64 character** โ€” the only two shapes that carry an `ID`. Positive control in +> the same loop: for the 204 dialogue headers, `Count` equals the number of +> `Message_NNN` siblings **204 / 204**, no mismatches. +> +> ๐Ÿ”‘ The 114 is not a coincidence: `Maneuver` = 114, `Effect` = 114, the entry +> sets are **identical**, every unit `Generic` has a `Maneuver` sibling, and +> `Generic.Type` splits **43 `Craft` + 71 `Vessel`** โ€” reproducing the count +> [unit-struct-runtime](unit-struct-runtime.md) reached from the other side. +> Artefact `../data/generic-record-partition.txt`, tool +> `tools/re-capture/generic_partition.py`. + | record | per pak | field names | distinct `ID`s | |---|---|---|---| -| `Generic` | **394** | **110** | 178 | +| `Generic` (unit shape) | **114** | 67 max on one record | 114 | +| `Generic` (all shapes) | 394 | 110 pooled | 178 | | `Maneuver` | **114** | 101 | โ€” | | `Effect` | **114** | 9 | โ€” | Repeated identically in every per-language `GP_MAIN_GAME_*.pak`. A unit's -`Generic`, `Maneuver` and `Effect` share a pak entry. +`Generic`, `Maneuver` and `Effect` share a pak entry โ€” one entry is one unit's +`.tbl`, which is why the runtime flattens all of them into a single object. โš ๏ธ `Generic` is the game's own record name, not a placeholder โ€” it is also one of the 65 section names in `data/config-keys.txt`. @@ -96,5 +126,7 @@ adopted**, nothing here shows the selection. `Carrier_TCAF1`, `NP_Facility`, โ€ฆ), 120โ€“144 instances each, alongside `HPGauge` / `HPGaugeSub` / `RadarCursorType` on the `ObjectiveMarker_*` and `TutorialMarker_*` records โ€” a **HUD gauge binding table**, not a datasheet. -* 178 distinct `Generic` IDs against 394 records per pak โ€” the surplus is - unexplained here. +* ~~178 distinct `Generic` IDs against 394 records per pak โ€” the surplus is + unexplained here.~~ **Settled 2026-08-27 โ€” see the correction at the top.** +* The `Generic` field-name count of 110 is a POOL across all four shapes; no + single record carries more than 67. diff --git a/tools/re-capture/generic_partition.py b/tools/re-capture/generic_partition.py new file mode 100644 index 00000000..4f9f0af1 --- /dev/null +++ b/tools/re-capture/generic_partition.py @@ -0,0 +1,43 @@ +import sys, glob, collections +S="/tmp/claude-1000/-home-fabi-RE-Project-Sylpheed/b113cc12-4769-4ed8-ad66-c2b48f800773/scratchpad" +sys.path.insert(0,S+"/wt-root/Syplheed-Reborn/tools/re-capture") +exec(open(S+"/regn_decode.py").read().split("# โ”€โ”€ the object")[0]) +import unitgroup as U +print("# `Generic` is the per-FILE header record, not a table") +print("#") +print("# One `Generic` per IDXD pak entry; its schema is set by the file's TYPE.") +print("# Regenerate: see docs/re/structures/unit-datasheet-static.md") +for pk in sorted(glob.glob("/work/sylph_extract/**/GP_MAIN_GAME_*.pak",recursive=True)): + E=pak_entries(pk); items=E.items() if isinstance(E,dict) else E + kinds=collections.Counter(); ids=set(); types=collections.Counter() + nidxd=0; man=eff=0; ctl_ok=ctl_bad=0; one_per=True + for h,b in items: + if bytes(b[:4])!=b'IDXD': continue + try: recs=U.parse(bytes(b)) + except Exception: continue + nidxd+=1 + names=[r['squadron'] for r in recs] + if names.count('Generic')>1: one_per=False + man+= 'Maneuver' in names; eff+= 'Effect' in names + for r in recs: + if r['squadron']!='Generic': continue + n=U.named(r) + if 'HP' in n: + kinds['unit datasheet']+=1; types[n.get('Type','')]+=1 + elif set(n)=={'Count'}: + kinds['message-file header']+=1 + nmsg=sum(1 for x in names if x.startswith('Message_')) + ctl_ok+= str(nmsg)==n['Count'].strip(); ctl_bad+= str(nmsg)!=n['Count'].strip() + elif 'SideID' in n: kinds['character']+=1 + elif 'EnumAsteroidGroup' in n: kinds['asteroid group']+=1 + else: kinds['other (%d fields)'%len(n)]+=1 + if 'ID' in n: ids.add(n['ID']) + print() + print("== %s (%d IDXD entries)" % (pk.split('/')[-1], nidxd)) + for k,v in kinds.most_common(): print(" %-22s %4d" % (k,v)) + print(" %-22s %4d" % ("TOTAL Generic", sum(kinds.values()))) + print(" distinct Generic.ID %4d (= unit %d + character %d)" + % (len(ids), kinds['unit datasheet'], kinds['character'])) + print(" unit Generic.Type %s" % dict(types)) + print(" entries w/ Maneuver %d w/ Effect %d" % (man,eff)) + print(" CONTROL message-header Count == #Message_NNN siblings: %d ok / %d bad" % (ctl_ok,ctl_bad))