re: decode the UnitGroup squadron roster, and correct yesterday's container reading
stage\UnitGroup_S<NN>.tbl is now fully readable. A squadron record is Count
member tuples -- (unit model, message set, n, identity/nameplate) -- followed by
five named fields: Count, SideID, AIID, FormationID, DisableInterval. The
property entries carry their own field names inline, so the tag hash never has
to be inverted.
Two independent self-checks validate it corpus-wide, 1160/1160 each across all
28 stage tables on the disc:
- the length identity len(fields) == Count * 4 + 5, which is what pins the
member-tuple width at 4 and the named-field count at 5;
- agreement with the file's own Enumerate_Squadrons roster, which maps record
key to squadron id independently of the per-record string offset.
Adds tools/re-capture/unitgroup.py (pure static, runs no emulator) with a
--all --check self-check mode, and commits the Stage 02 dump as evidence.
Corrections to the container layout written yesterday, all three wrong:
- the 20-byte "(tag, 0, 0, count, size) section header" does not exist. It
was the file's last 16-byte record followed by a plain npool word. The
corrected layout is uniform across all 28 files; the old one failed on 9.
- squadron ids do not use a separate string base. Every offset in the file is
relative to the one string pool. The earlier "109 of 111" score was an
artefact of the uniform 7-byte id stride and had silently shifted every
name by three entries, which is why 17 TC*-named squadrons came out as
SideID=ADAN. The roster record refuted it outright.
- the roster is not always the last record; 9 stages put it elsewhere, so it
is found by its missing Count.
Refuted and kept: the 4-byte record key is not the squadron id's name hash
(0 of 112).
Not settled: what the key encodes, the member tuple's third field n, and where
the arrival interval values live. DisableInterval is only a per-squadron flag
(Yes for 31 of 1160); the durations, triggers and arrival positions are not in
this file. Formation_*.tbl and EnumSquadron_Test.tbl are next.
This commit is contained in:
Sylpheed RE agent
141
tools/re-capture/unitgroup.py
Normal file
141
tools/re-capture/unitgroup.py
Normal file
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Decode a stage's squadron roster: hidden/dat `stage\\UnitGroup_S<NN>.tbl`.
|
||||
|
||||
Pure static work — reads the extracted disc, runs no emulator.
|
||||
|
||||
python3 tools/re-capture/unitgroup.py S02
|
||||
python3 tools/re-capture/unitgroup.py --all --check
|
||||
|
||||
Format (see docs/re/structures/unit-group-table.md):
|
||||
|
||||
"IDXD" magic
|
||||
u32 nrec record count; exactly one record is the
|
||||
Enumerate_Squadrons roster, not a squadron
|
||||
nrec x 16 (key, squadron_string_off, field_lo, field_hi)
|
||||
u32 npool property-entry count
|
||||
npool x 12 (tag, name_off | 0xffffffff, value_off)
|
||||
u32 strsize string-pool byte size; STR+strsize == len(file)
|
||||
strsize bytes string pool; [0] is "Enumerate_Squadrons",
|
||||
[0x14] is the empty string. Every offset in
|
||||
the file -- squadron ids included -- is relative
|
||||
to STR.
|
||||
|
||||
A squadron's field list is `Count` member tuples of four positional entries
|
||||
(unit model, message set, a number, pilot character) followed by five named
|
||||
entries: Count, SideID, AIID, FormationID, DisableInterval. So
|
||||
|
||||
len(fields) == Count * 4 + 5
|
||||
|
||||
which holds for 1160 of 1160 squadrons across the 28 stage tables present in
|
||||
GP_MAIN_GAME_E.pak.
|
||||
"""
|
||||
import argparse, glob, os, struct, sys, zlib
|
||||
|
||||
MODULUS, RECIP = 0x00FFF9D7, 0x80031493
|
||||
|
||||
def _rotl(v, n): return ((v << n) | (v >> (32 - n))) & 0xFFFFFFFF
|
||||
|
||||
def name_hash(s):
|
||||
bs = bytearray(s.encode('latin-1', 'replace'))
|
||||
for i, x in enumerate(bs):
|
||||
if 65 <= x <= 90: bs[i] = x + 0x20
|
||||
a = b = 0
|
||||
for byte in bs:
|
||||
c = ((byte - 256) if byte > 127 else byte) & 0xFFFFFFFF
|
||||
a = ((_rotl(a, 8) & 0xFFFFFF00) + c) & 0xFFFFFFFF
|
||||
b = (b + c) & 0xFFFFFFFF
|
||||
q = _rotl(((a * RECIP) >> 32) & 0xFFFFFFFF, 9) & 0x1FF
|
||||
a = (a - (q * MODULUS)) & 0xFFFFFFFF
|
||||
return (((b << 24) & 0xFF000000) | (a & 0x00FFFFFF)) & 0xFFFFFFFF
|
||||
|
||||
def read_entry(pak, h):
|
||||
idx = open(pak, 'rb').read()
|
||||
base = pak[:-4]
|
||||
data = b''.join(open(p, 'rb').read() for p in sorted(glob.glob(base + '.p[0-9][0-9]')))
|
||||
for i in range(struct.unpack_from('>I', idx, 4)[0]):
|
||||
k, off, csize = struct.unpack_from('>III', idx, 16 + i * 12)
|
||||
if k == h:
|
||||
s = data[off:off + csize]
|
||||
return zlib.decompress(s[10:]) if s[:2] == b'Z1' else s
|
||||
raise KeyError('%#010x not in %s' % (h, os.path.basename(pak)))
|
||||
|
||||
def parse(b):
|
||||
u = lambda o: struct.unpack_from('>I', b, o)[0]
|
||||
if b[:4] != b'IDXD': raise ValueError('bad magic')
|
||||
nrec = u(0x04)
|
||||
npool_off = 0x08 + nrec * 16
|
||||
npool = u(npool_off)
|
||||
pool = npool_off + 4
|
||||
strsize_off = pool + npool * 12
|
||||
STR = strsize_off + 4
|
||||
if STR + u(strsize_off) != len(b):
|
||||
raise ValueError('string-pool size trailer mismatch')
|
||||
def s(o):
|
||||
return b[o:b.index(b'\x00', o)].decode('latin-1')
|
||||
out = []
|
||||
for i in range(nrec):
|
||||
key, sq, lo, hi = (u(0x08 + i * 16 + j * 4) for j in range(4))
|
||||
fields = []
|
||||
for j in range(hi - lo):
|
||||
o = pool + (lo + j) * 12
|
||||
tag, name, val = u(o), u(o + 4), u(o + 8)
|
||||
fields.append((tag, s(STR + name) if name != 0xFFFFFFFF else None, s(STR + val)))
|
||||
out.append({'key': key, 'squadron': s(STR + sq), 'fields': fields})
|
||||
return out
|
||||
|
||||
def named(rec): return {n: v for _, n, v in rec['fields'] if n is not None}
|
||||
def members(rec):
|
||||
pos = [v for _, n, v in rec['fields'] if n is None]
|
||||
return [tuple(pos[i:i + 4]) for i in range(0, len(pos), 4)]
|
||||
def is_roster(rec): return 'Count' not in named(rec)
|
||||
|
||||
def roster(recs):
|
||||
"""One record enumerates key -> squadron id; use it to cross-check the rest.
|
||||
|
||||
It is usually the last record but not always (S05, S28, ... put it earlier),
|
||||
so find it by its lack of a Count field rather than by position.
|
||||
"""
|
||||
for r in recs:
|
||||
if is_roster(r):
|
||||
return {tag: n for tag, n, _ in r['fields']}
|
||||
return {}
|
||||
|
||||
def load_stage(pak, stage):
|
||||
return parse(read_entry(pak, name_hash('stage\\UnitGroup_%s.tbl' % stage)))
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('stage', nargs='?', default='S02')
|
||||
ap.add_argument('--pak', default='/work/sylph_extract/dat/GP_MAIN_GAME_E.pak')
|
||||
ap.add_argument('--all', action='store_true', help='every stage S01..S29')
|
||||
ap.add_argument('--check', action='store_true', help='only report the Count*4+5 identity')
|
||||
a = ap.parse_args()
|
||||
stages = ['S%02d' % n for n in range(1, 30)] if a.all else [a.stage]
|
||||
held = total = 0
|
||||
for st in stages:
|
||||
try:
|
||||
recs = load_stage(a.pak, st)
|
||||
except KeyError as e:
|
||||
print('%s: absent (%s)' % (st, e)); continue
|
||||
squads = [r for r in recs if not is_roster(r)]
|
||||
ok = sum(1 for r in squads if len(r['fields']) == int(named(r)['Count']) * 4 + 5)
|
||||
rst = roster(recs)
|
||||
agree = sum(1 for r in squads if rst.get(r['key']) == r['squadron'])
|
||||
held += ok; total += len(squads)
|
||||
print('%s: %3d squadrons, %4d members, Count*4+5 %d/%d, roster agrees %d/%d'
|
||||
% (st, len(squads), sum(len(members(r)) for r in squads), ok, len(squads),
|
||||
agree, len(squads)))
|
||||
if a.check: continue
|
||||
for r in squads:
|
||||
d = named(r)
|
||||
print(' %-10s %-8s Count=%-3s DisableInterval=%-4s %-30s %s'
|
||||
% (r['squadron'], d['SideID'], d['Count'], d['DisableInterval'],
|
||||
d['FormationID'], d['AIID']))
|
||||
for m in members(r):
|
||||
print(' unit=%-34s msg=%-24s n=%-4s pilot=%s' % m)
|
||||
if len(stages) > 1:
|
||||
print('\nTOTAL %d squadrons, identity held %d/%d' % (total, held, total))
|
||||
return 0 if held == total else 1
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user