b39aabf5c3
stage\UnitGroup_S<NN>.tbl is now fully readable. A squadron record is Count
member tuples -- (unit model, message set, n, identity/nameplate) -- followed by
five named fields: Count, SideID, AIID, FormationID, DisableInterval. The
property entries carry their own field names inline, so the tag hash never has
to be inverted.
Two independent self-checks validate it corpus-wide, 1160/1160 each across all
28 stage tables on the disc:
- the length identity len(fields) == Count * 4 + 5, which is what pins the
member-tuple width at 4 and the named-field count at 5;
- agreement with the file's own Enumerate_Squadrons roster, which maps record
key to squadron id independently of the per-record string offset.
Adds tools/re-capture/unitgroup.py (pure static, runs no emulator) with a
--all --check self-check mode, and commits the Stage 02 dump as evidence.
Corrections to the container layout written yesterday, all three wrong:
- the 20-byte "(tag, 0, 0, count, size) section header" does not exist. It
was the file's last 16-byte record followed by a plain npool word. The
corrected layout is uniform across all 28 files; the old one failed on 9.
- squadron ids do not use a separate string base. Every offset in the file is
relative to the one string pool. The earlier "109 of 111" score was an
artefact of the uniform 7-byte id stride and had silently shifted every
name by three entries, which is why 17 TC*-named squadrons came out as
SideID=ADAN. The roster record refuted it outright.
- the roster is not always the last record; 9 stages put it elsewhere, so it
is found by its missing Count.
Refuted and kept: the 4-byte record key is not the squadron id's name hash
(0 of 112).
Not settled: what the key encodes, the member tuple's third field n, and where
the arrival interval values live. DisableInterval is only a per-squadron flag
(Yes for 31 of 1160); the durations, triggers and arrival positions are not in
this file. Formation_*.tbl and EnumSquadron_Test.tbl are next.
142 lines
5.8 KiB
Python
142 lines
5.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Decode a stage's squadron roster: hidden/dat `stage\\UnitGroup_S<NN>.tbl`.
|
|
|
|
Pure static work — reads the extracted disc, runs no emulator.
|
|
|
|
python3 tools/re-capture/unitgroup.py S02
|
|
python3 tools/re-capture/unitgroup.py --all --check
|
|
|
|
Format (see docs/re/structures/unit-group-table.md):
|
|
|
|
"IDXD" magic
|
|
u32 nrec record count; exactly one record is the
|
|
Enumerate_Squadrons roster, not a squadron
|
|
nrec x 16 (key, squadron_string_off, field_lo, field_hi)
|
|
u32 npool property-entry count
|
|
npool x 12 (tag, name_off | 0xffffffff, value_off)
|
|
u32 strsize string-pool byte size; STR+strsize == len(file)
|
|
strsize bytes string pool; [0] is "Enumerate_Squadrons",
|
|
[0x14] is the empty string. Every offset in
|
|
the file -- squadron ids included -- is relative
|
|
to STR.
|
|
|
|
A squadron's field list is `Count` member tuples of four positional entries
|
|
(unit model, message set, a number, pilot character) followed by five named
|
|
entries: Count, SideID, AIID, FormationID, DisableInterval. So
|
|
|
|
len(fields) == Count * 4 + 5
|
|
|
|
which holds for 1160 of 1160 squadrons across the 28 stage tables present in
|
|
GP_MAIN_GAME_E.pak.
|
|
"""
|
|
import argparse, glob, os, struct, sys, zlib
|
|
|
|
MODULUS, RECIP = 0x00FFF9D7, 0x80031493
|
|
|
|
def _rotl(v, n): return ((v << n) | (v >> (32 - n))) & 0xFFFFFFFF
|
|
|
|
def name_hash(s):
|
|
bs = bytearray(s.encode('latin-1', 'replace'))
|
|
for i, x in enumerate(bs):
|
|
if 65 <= x <= 90: bs[i] = x + 0x20
|
|
a = b = 0
|
|
for byte in bs:
|
|
c = ((byte - 256) if byte > 127 else byte) & 0xFFFFFFFF
|
|
a = ((_rotl(a, 8) & 0xFFFFFF00) + c) & 0xFFFFFFFF
|
|
b = (b + c) & 0xFFFFFFFF
|
|
q = _rotl(((a * RECIP) >> 32) & 0xFFFFFFFF, 9) & 0x1FF
|
|
a = (a - (q * MODULUS)) & 0xFFFFFFFF
|
|
return (((b << 24) & 0xFF000000) | (a & 0x00FFFFFF)) & 0xFFFFFFFF
|
|
|
|
def read_entry(pak, h):
|
|
idx = open(pak, 'rb').read()
|
|
base = pak[:-4]
|
|
data = b''.join(open(p, 'rb').read() for p in sorted(glob.glob(base + '.p[0-9][0-9]')))
|
|
for i in range(struct.unpack_from('>I', idx, 4)[0]):
|
|
k, off, csize = struct.unpack_from('>III', idx, 16 + i * 12)
|
|
if k == h:
|
|
s = data[off:off + csize]
|
|
return zlib.decompress(s[10:]) if s[:2] == b'Z1' else s
|
|
raise KeyError('%#010x not in %s' % (h, os.path.basename(pak)))
|
|
|
|
def parse(b):
|
|
u = lambda o: struct.unpack_from('>I', b, o)[0]
|
|
if b[:4] != b'IDXD': raise ValueError('bad magic')
|
|
nrec = u(0x04)
|
|
npool_off = 0x08 + nrec * 16
|
|
npool = u(npool_off)
|
|
pool = npool_off + 4
|
|
strsize_off = pool + npool * 12
|
|
STR = strsize_off + 4
|
|
if STR + u(strsize_off) != len(b):
|
|
raise ValueError('string-pool size trailer mismatch')
|
|
def s(o):
|
|
return b[o:b.index(b'\x00', o)].decode('latin-1')
|
|
out = []
|
|
for i in range(nrec):
|
|
key, sq, lo, hi = (u(0x08 + i * 16 + j * 4) for j in range(4))
|
|
fields = []
|
|
for j in range(hi - lo):
|
|
o = pool + (lo + j) * 12
|
|
tag, name, val = u(o), u(o + 4), u(o + 8)
|
|
fields.append((tag, s(STR + name) if name != 0xFFFFFFFF else None, s(STR + val)))
|
|
out.append({'key': key, 'squadron': s(STR + sq), 'fields': fields})
|
|
return out
|
|
|
|
def named(rec): return {n: v for _, n, v in rec['fields'] if n is not None}
|
|
def members(rec):
|
|
pos = [v for _, n, v in rec['fields'] if n is None]
|
|
return [tuple(pos[i:i + 4]) for i in range(0, len(pos), 4)]
|
|
def is_roster(rec): return 'Count' not in named(rec)
|
|
|
|
def roster(recs):
|
|
"""One record enumerates key -> squadron id; use it to cross-check the rest.
|
|
|
|
It is usually the last record but not always (S05, S28, ... put it earlier),
|
|
so find it by its lack of a Count field rather than by position.
|
|
"""
|
|
for r in recs:
|
|
if is_roster(r):
|
|
return {tag: n for tag, n, _ in r['fields']}
|
|
return {}
|
|
|
|
def load_stage(pak, stage):
|
|
return parse(read_entry(pak, name_hash('stage\\UnitGroup_%s.tbl' % stage)))
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('stage', nargs='?', default='S02')
|
|
ap.add_argument('--pak', default='/work/sylph_extract/dat/GP_MAIN_GAME_E.pak')
|
|
ap.add_argument('--all', action='store_true', help='every stage S01..S29')
|
|
ap.add_argument('--check', action='store_true', help='only report the Count*4+5 identity')
|
|
a = ap.parse_args()
|
|
stages = ['S%02d' % n for n in range(1, 30)] if a.all else [a.stage]
|
|
held = total = 0
|
|
for st in stages:
|
|
try:
|
|
recs = load_stage(a.pak, st)
|
|
except KeyError as e:
|
|
print('%s: absent (%s)' % (st, e)); continue
|
|
squads = [r for r in recs if not is_roster(r)]
|
|
ok = sum(1 for r in squads if len(r['fields']) == int(named(r)['Count']) * 4 + 5)
|
|
rst = roster(recs)
|
|
agree = sum(1 for r in squads if rst.get(r['key']) == r['squadron'])
|
|
held += ok; total += len(squads)
|
|
print('%s: %3d squadrons, %4d members, Count*4+5 %d/%d, roster agrees %d/%d'
|
|
% (st, len(squads), sum(len(members(r)) for r in squads), ok, len(squads),
|
|
agree, len(squads)))
|
|
if a.check: continue
|
|
for r in squads:
|
|
d = named(r)
|
|
print(' %-10s %-8s Count=%-3s DisableInterval=%-4s %-30s %s'
|
|
% (r['squadron'], d['SideID'], d['Count'], d['DisableInterval'],
|
|
d['FormationID'], d['AIID']))
|
|
for m in members(r):
|
|
print(' unit=%-34s msg=%-24s n=%-4s pilot=%s' % m)
|
|
if len(stages) > 1:
|
|
print('\nTOTAL %d squadrons, identity held %d/%d' % (total, held, total))
|
|
return 0 if held == total else 1
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|