re: decode CollisionSet_*.bin -- the per-object collision-mesh library

All 18 blobs are byte-identical: the per-stage naming is nominal, and every
stage points at one shared 1675148-byte library stored eighteen times.  That
identical size was the reason to open the item, and it turned out to be the
answer to it.

Record layout: {u32 size, u32 name_len, char name[name_len], u32 nv, u32 nt,
f32[3] x nv, u32[3] x nt}, next record at off + 8 + size.  The indices are u32
here where MCOL uses u16 -- two different serialisers in one archive.

What makes this a decode rather than a plausible reading: the walk consumes the
file to the byte over 158 variable-length records, with the size word predicted
from the two counts 158/158.  A wrong field would desynchronise within a few
records and could not land exactly on the end.  All indices in range 158/158;
98.24% of edges shared by exactly two triangles; 147/158 fully manifold.

158 meshes, 90 836 triangles: per-part ship proxies (_bdy/_brg/_eng/_wep/_sld,
the XBG7 sub-part vocabulary) plus 46 stage asteroid meshes whose prefixes are
exactly the stages that have an _AsteroidVolume_wp MCOL.

Two things this file makes me walk back:

  * The "1 unit = 1 metre" reading from mapmesh_box_500km is downgraded to
    amber.  The 500000 arithmetic stands, but it implies that a craft the game's
    own tables call "small" is 133 m and that rob_f002 is 447 km -- 89% of the
    arena width.  The format check survives; the interpretation has no
    independent support.
  * The _cmesh <-> render-model name link is recorded as UNTESTED, not
    confirmed: only one .xbg build manifest survives on the disc, so matching
    stems against object names covers 4 of 158, which is no coverage at all.

tools/re-capture/collisionset.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Decode `CollisionSet_*.bin` from `hidden/MiscBin.pak`.
+
+All 18 of them are byte-identical, so the per-stage naming is nominal: there is
+one shared library of 158 named per-object collision meshes (ships, stations,
+asteroids), separate from the per-map `MCOL` hull.
+
+    ./collisionset.py list   <MiscBin.pak>          # the 158 meshes
+    ./collisionset.py verify <MiscBin.pak>          # the checks in the doc
+    ./collisionset.py obj    <MiscBin.pak> <name> <out.obj>
+
+A record is
+
+    u32 size          bytes that follow this word's own 8-byte prefix
+    u32 name_len      16, 24 or 32 -- the name field, padded to a multiple of 8
+    char name[name_len]
+    u32 vertex_count
+    u32 triangle_count
+    f32 vertex[3] x vertex_count
+    u32 index[3]  x triangle_count      -- note u32, not the u16 MCOL uses
+
+and the next record begins at `offset + 8 + size`.
+"""
+
+import collections
+import math
+import os
+import struct
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from regn_decode import pak_entries
+from unitgroup import name_hash
+
+
+def meshes(pak_path, which="CollisionSet_S01.bin"):
+    """[(name, vertices, triangles)] for one CollisionSet blob."""
+    blob = dict(pak_entries(pak_path))[name_hash(which)]
+    u32 = lambda o: struct.unpack_from(">I", blob, o)[0]
+    out, p = [], 0
+    while p < len(blob):
+        size, nlen = u32(p), u32(p + 4)
+        name = blob[p + 8:p + 8 + nlen].split(b"\0")[0].decode("latin-1")
+        nv, nt = u32(p + 8 + nlen), u32(p + 12 + nlen)
+        vb = p + 16 + nlen
+        tb = vb + 12 * nv
+        verts = [struct.unpack_from(">fff", blob, vb + 12 * i) for i in range(nv)]
+        tris = [struct.unpack_from(">III", blob, tb + 12 * i) for i in range(nt)]
+        expect = 16 + nlen + 12 * nv + 12 * nt - 8
+        out.append((name, verts, tris, size == expect))
+        p += 8 + size
+    return out, len(blob), p
+
+
+def cmd_list(pak):
+    ms, _, _ = meshes(pak)
+    print(f"{'name':>28} {'verts':>6} {'tris':>6} {'bbox diagonal':>14}")
+    for name, v, t, _ in ms:
+        lo = [min(q[k] for q in v) for k in range(3)]
+        hi = [max(q[k] for q in v) for k in range(3)]
+        print(f"{name:>28} {len(v):6d} {len(t):6d} {math.dist(lo, hi):14.1f}")
+    print(f"\n{len(ms)} meshes, {sum(len(t) for _, _, t, _ in ms)} triangles")
+    return True
+
+
+def cmd_verify(pak):
+    blobs = dict(pak_entries(pak))
+    tags = ["test", "Tutorial"] + ["S%02d" % i for i in range(1, 17)]
+    distinct = {blobs[name_hash(f"CollisionSet_{t}.bin")] for t in tags}
+    print(f"CollisionSet blobs: {len(tags)}   distinct contents: {len(distinct)}")
+
+    ms, total, consumed = meshes(pak)
+    print(f"records: {len(ms)}   bytes consumed: {consumed}/{total}"
+          f"  ({'EXACT' if consumed == total else 'MISMATCH'})")
+    print(f"size word == 16 + name_len + 12*nv + 12*nt - 8 : "
+          f"{sum(ok for _, _, _, ok in ms)}/{len(ms)}")
+
+    in_range = sum(all(max(t) < len(v) for t in tris) if tris else 1
+                   for _, v, tris, _ in ms)
+    print(f"every triangle index < vertex count            : {in_range}/{len(ms)}")
+
+    closed = e2 = etot = 0
+    for _, v, tris, _ in ms:
+        ec = collections.Counter()
+        for t in tris:
+            for k in range(3):
+                a, b = t[k], t[(k + 1) % 3]
+                ec[(min(a, b), max(a, b))] += 1
+        etot += len(ec)
+        e2 += sum(1 for n in ec.values() if n == 2)
+        closed += all(n == 2 for n in ec.values())
+    print(f"edges shared by exactly two triangles          : "
+          f"{e2}/{etot} = {100*e2/etot:.2f}%")
+    print(f"fully manifold meshes                          : {closed}/{len(ms)}")
+    return (len(distinct) == 1 and consumed == total
+            and all(ok for _, _, _, ok in ms) and in_range == len(ms))
+
+
+def cmd_obj(pak, want, out):
+    for name, v, tris, _ in meshes(pak)[0]:
+        if name != want:
+            continue
+        with open(out, "w") as fh:
+            fh.write(f"# CollisionSet {name} -- {len(v)} vertices, {len(tris)} triangles\n")
+            for q in v:
+                fh.write("v %.4f %.4f %.4f\n" % q)
+            for t in tris:
+                fh.write(f"f {t[0]+1} {t[1]+1} {t[2]+1}\n")
+        print(f"{out}: {len(v)} vertices, {len(tris)} triangles")
+        return True
+    raise SystemExit(f"no mesh named {want!r}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        raise SystemExit(__doc__)
+    cmd, pak = sys.argv[1], sys.argv[2]
+    fn = {"list": cmd_list, "verify": cmd_verify}.get(cmd)
+    ok = cmd_obj(pak, sys.argv[3], sys.argv[4]) if cmd == "obj" else fn(pak)
+    print("\nPASS" if ok else "\nFAIL")
+    sys.exit(0 if ok else 1)