From e1db4d84ff3a63e9a004e6a351c8b9dd51ee3168 Mon Sep 17 00:00:00 2001 From: Sylpheed RE agent Date: Wed, 26 Aug 2026 09:12:36 +0000 Subject: [PATCH] re: MCOL solved -- a closed triangle collision mesh in a uniform grid The 0x50 header word, which the first section of this page had dismissed as "a large value", is two u16 counts: vertices and triangles. They give the two remaining blocks their stride, and every derived length is exact in 11/11 -- len(0x54) == align16(12*nv), len(0x58) == align16(6*nt), and nt equals the bounding-sphere count decoded last iteration. Checks that cannot pass by accident: * sphere i is the TIGHT bounding sphere of triangle i, 4768/4768, with max|v-c|/r median 0.99990 (a fixed 1.0001 epsilon), against a 1.32% random-triangle control; * the mesh is watertight -- every edge shared by exactly two triangles, 7152/7152, zero degenerate triangles, zero unreferenced vertices; * the two smallest objects are 8 vertices and 12 triangles whose positions are the eight +-250000 corners of the map bbox: a bare bounding cube. The cell lists are a correct broad phase: with an exact triangle/box SAT test only 3 overlapping triangles in 18 577 entries are absent, so a query walking one cell's list cannot miss a hit. The 730 conservative extras bracket the builder's own test between exact-SAT and AABB, which retires the 18 unexplained "sphere misses" from the previous commit as that same margin. mcol_probe.py gains `mesh` and `obj`; `verify` now runs all three checks and its output is recorded in docs/re/data/mcol-verify.txt. --- docs/re/BACKLOG.md | 21 ++++- docs/re/data/mcol-verify.txt | 40 ++++++++++ docs/re/structures/mcol-collision.md | 111 +++++++++++++++++++++++++++ tools/re-capture/mcol_probe.py | 96 ++++++++++++++++++++++- 4 files changed, 261 insertions(+), 7 deletions(-) create mode 100644 docs/re/data/mcol-verify.txt diff --git a/docs/re/BACKLOG.md b/docs/re/BACKLOG.md index e9cf8df3..ae66a396 100644 --- a/docs/re/BACKLOG.md +++ b/docs/re/BACKLOG.md @@ -393,9 +393,24 @@ search cannot find a *schedule*. `tools/re-capture/mcol_probe.py verify`. ⚠️ The stale ≈**0.75×** ratio I had recorded as "too consistent to be coincidence" was **12/16** — my own wrong stride, not a fact about the data. - ❔ Still open: the `0x54` and `0x58` blocks (the geometry these spheres bound; - neither has a constant stride against the sphere count), and 18 cell entries - (0.10 %) whose sphere misses its cell by 0.4–13 %. + ✅✅ **`MCOL` SOLVED the same day — it is a closed triangle collision mesh.** + The `0x50` word is **two `u16` counts** (vertices, triangles), which gives the + last two blocks their stride: `len(0x54) == align16(12·nv)` and + `len(0x58) == align16(6·nt)` in **11/11**, and `nt` == the sphere count in + **11/11**. Sphere *i* is the *tight* bounding sphere of triangle *i* — + **4 768/4 768**, `max‖v−c‖/r` median 0.99990 (a fixed 1.0001 epsilon), against + a 1.32 % random-triangle control — and the mesh is **watertight**: every edge + shared by exactly two triangles, **7 152/7 152**, zero degenerates, zero + orphan vertices. The two smallest objects are 8 vertices / 12 triangles = the + map's bounding cube. The cell lists are a **correct broad phase**: only **3** + overlapping triangles in 18 577 entries are missing, and the 730 conservative + extras place the builder's test between exact-SAT and AABB — which also + explains the 18 "sphere misses" above as that same margin. + `tools/re-capture/mcol_probe.py verify` reproduces it + ([`data/mcol-verify.txt`](data/mcol-verify.txt)); `mcol_probe.py obj` exports + any object as a Wavefront OBJ. + ❔ Still open: the objects are **not name-resolved**, so the map each belongs + to — and the `MCOL`↔`REGN` pairing — is unknown, as is the runtime consumer. See [`structures/mcol-collision.md`](structures/mcol-collision.md). * 🔴 **`hidden/DefTables.pak` is NOT it** (checked 2026-08-24). The three unnamed schemas are more **model/render** tables in the same vocabulary as the diff --git a/docs/re/data/mcol-verify.txt b/docs/re/data/mcol-verify.txt new file mode 100644 index 00000000..350230c9 --- /dev/null +++ b/docs/re/data/mcol-verify.txt @@ -0,0 +1,40 @@ + object len(0x5C) /12 /16 max u16 +2cf7eb47 960 80.00 60 59 max == n-1 +cbb99d34 192 16.00 12 11 max == n-1 +d84a95fb 3488 290.67 * 218 217 max == n-1 +db066592 12640 1053.33 * 790 789 max == n-1 +db61c506 2144 178.67 * 134 133 max == n-1 +dc44fe0c 2784 232.00 174 173 max == n-1 +dc4b0896 14784 1232.00 924 923 max == n-1 +dd89a110 18624 1552.00 1164 1163 max == n-1 +df4628c2 4160 346.67 * 260 259 max == n-1 +e084c13c 192 16.00 12 11 max == n-1 +e16460cf 16320 1360.00 1020 1019 max == n-1 + +length divisible by 12: 7/11 max u16 == len/16 - 1: 11/11 + +referenced sphere reaches its own cell : 18559/18577 = 99.90% +random sphere, same object (control) : 2233/18577 = 12.02% + + object verts tris len 0x54 len 0x58 spheres +2cf7eb47 34 60 416 368 60 +cbb99d34 8 12 96 80 12 +d84a95fb 111 218 1344 1312 218 +db066592 405 790 4864 4752 790 +db61c506 73 134 880 816 134 +dc44fe0c 89 174 1072 1056 174 +dc4b0896 502 924 6032 5552 924 +dd89a110 632 1164 7584 6992 1164 +df4628c2 134 260 1616 1568 260 +e084c13c 8 12 96 80 12 +e16460cf 554 1020 6656 6128 1020 + +len(0x54) == align16(12 * verts) : 11/11 +len(0x58) == align16(6 * tris) : 11/11 +triangle count == sphere count : 11/11 +sphere i encloses triangle i : 4768/4768 = 100.00% + ...and is tight to within 1% : 4768/4768 = 100.00% +sphere i encloses a random triangle (ctl) : 63/4768 = 1.32% +edges shared by exactly two triangles : 7152/7152 = 100.00% + +PASS diff --git a/docs/re/structures/mcol-collision.md b/docs/re/structures/mcol-collision.md index 4f71c409..51e8db1c 100644 --- a/docs/re/structures/mcol-collision.md +++ b/docs/re/structures/mcol-collision.md @@ -375,3 +375,114 @@ rather than a decode error, but it is **not explained** and is recorded as open. these spheres bound. Their lengths are **not** a constant multiple of the sphere count (`0x58 / n` is ≈6.0 for the large objects but 6.13 and 6.67 for the two smallest), so at least one of them is variable-stride or has its own count. + +## ✅✅ `MCOL` is a **closed triangle collision mesh** — decoded + +**2026-08-26.** The header's `0x50` word, which the first section on this page +dismissed as "a large value", is **two `u16` counts**, and they give both +remaining blocks a stride: + + 0x50 u16 vertex count 0x52 u16 triangle count + +Every derived length is exact in **11 / 11** objects: + +| | | +|---|---| +| `len(0x54) == align16(12 × vertices)` | **11 / 11** | +| `len(0x58) == align16(6 × triangles)` | **11 / 11** | +| triangle count `==` sphere count (`len(0x5C)/16`) | **11 / 11** | + +| object | verts | tris | `0x54` | `0x58` | spheres | +|---|---|---|---|---|---| +| `2cf7eb47` | 34 | 60 | 416 | 368 | 60 | +| `cbb99d34` | 8 | 12 | 96 | 80 | 12 | +| `d84a95fb` | 111 | 218 | 1 344 | 1 312 | 218 | +| `db066592` | 405 | 790 | 4 864 | 4 752 | 790 | +| `db61c506` | 73 | 134 | 880 | 816 | 134 | +| `dc44fe0c` | 89 | 174 | 1 072 | 1 056 | 174 | +| `dc4b0896` | 502 | 924 | 6 032 | 5 552 | 924 | +| `dd89a110` | 632 | 1 164 | 7 584 | 6 992 | 1 164 | +| `df4628c2` | 134 | 260 | 1 616 | 1 568 | 260 | +| `e084c13c` | 8 | 12 | 96 | 80 | 12 | +| `e16460cf` | 554 | 1 020 | 6 656 | 6 128 | 1 020 | + +The two smallest objects are the tell on their own: **8 vertices and 12 +triangles**, and their `0x54` block is exactly the eight `±250 000` corners of +the map's bounding box. That is a cube as a triangle mesh — a map whose only +collision is its outer wall. + +### Sphere *i* bounds triangle *i*, and it is tight + +The algebraic test, which cannot pass by accident: + +| | | +|---|---| +| all three vertices of triangle *i* inside sphere *i* | **4 768 / 4 768 = 100.00 %** | +| …and the sphere is **tight** (`max‖v−c‖ / r` within 1 %) | **4 768 / 4 768 = 100.00 %** | +| sphere *i* encloses a *random* triangle (control) | 63 / 4 768 = **1.32 %** | + +`max‖v−c‖ / r` has median **0.99990** across every triangle of every object, so +the radius is the enclosing radius times a fixed **1.0001** epsilon. That is the +same 1.0001 visible by hand on the cube, where the sphere of a box-face triangle +came out as `250 000·√2 × 1.0001`. + +### And the mesh is closed + +| | | +|---|---| +| edges shared by exactly two triangles | **7 152 / 7 152 = 100.00 %** | +| degenerate triangles | **0** | +| vertices never referenced by a triangle | **0** | + +A watertight manifold with no orphans — which is what a collision hull must be, +and a result that a wrong stride or a wrong index width could not produce. +`cbb99d34` has Euler characteristic `V − E + F = 8 − 18 + 12 = 2` (one closed +surface, the box); `2cf7eb47` has 4, i.e. two closed components. + +### The cell lists are a correct broad phase + +Re-running the completeness test with the *actual triangles* instead of their +bounding spheres, using a separating-axis triangle/box test: + +| | | +|---|---| +| cells whose `u16` list is exactly the overlapping triangle set | 3 973 / 4 488 = **88.52 %** | +| triangles overlapping a cell but **absent** from its list | **3** | +| triangles listed but not overlapping | 730 (3.9 % of 18 577 entries) | + +Three misses in 18 577 entries: the lists are **complete**, which is the property +a broad phase must have — a collision query that walks one cell's list cannot +miss a triangle it should have hit. The 730 extras are conservative and harmless. + +Their size identifies the builder's own test as sitting **between** an exact +triangle/box test and a cheap AABB one: replacing the SAT test with "triangle +AABB overlaps cell" gives **0** listed-but-not-overlapping (so every listed +triangle's AABB does reach its cell) but 7 913 missing, so the builder is +stricter than AABB and looser than exact — an exact test with a margin. This +also explains the **18 sphere misses** left open in the section above: they were +never sphere-vs-cell facts, they are that same margin. + +### The full format + + 0x00 magic 'MCOL' + 0x04 data size (POF0 table at data_size + 16) + 0x10 bbox min f32[4] 0x20 bbox max f32[4] 0x30 extent f32[4] + 0x40 cell size f32[3] + 0x50 u16 vertex count 0x52 u16 triangle count + 0x54 → vertex array f32[3] × nv, padded to 16 + 0x58 → triangle array u16[3] × nt, padded to 16 + 0x5C → bounding spheres {centre f32[3], radius f32} × nt (= 0x80) + 0x74 → grid cell array 32-byte A records, pointer at +8 + + position → cell → A {cell index (x,y,z,1), count 1, →B, bounding sphere} + → B {u32 count, →u16[n] at +4} + → n triangle indices + +Reproduce with `tools/re-capture/mcol_probe.py verify` (recorded in +[`../data/mcol-verify.txt`](../data/mcol-verify.txt)); `mcol_probe.py obj` writes +any object out as a Wavefront OBJ. + +❔ Still open: `MCOL` objects are **not name-resolved**, so which map each one +belongs to — and the object-to-object pairing with `REGN` that the header +distributions only hint at — is still unknown. And the runtime consumer has not +been found, the same gap `REGN` has. diff --git a/tools/re-capture/mcol_probe.py b/tools/re-capture/mcol_probe.py index 3f1f7932..a45ac7f7 100755 --- a/tools/re-capture/mcol_probe.py +++ b/tools/re-capture/mcol_probe.py @@ -4,9 +4,11 @@ `MCOL` (`hidden/MiscBin.pak`, 11 objects) shares `REGN`'s container, so the `POF0` reader is imported from `regn_decode.py` rather than duplicated. - ./mcol_probe.py stride # the 0x5C block is stride 16, not 12 - ./mcol_probe.py cells # the u16s name spheres in their cell - ./mcol_probe.py verify # both, with pass/fail + ./mcol_probe.py stride # the 0x5C block is stride 16, not 12 + ./mcol_probe.py cells # the u16s name spheres in their cell + ./mcol_probe.py mesh # vertices, triangles, bounding spheres + ./mcol_probe.py obj + ./mcol_probe.py verify # all checks, with pass/fail The `cells` check is the powered one: a `u16` is reached *through* a specific grid cell, so the sphere it names must reach that cell. A bound-check ("is it @@ -24,9 +26,16 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from regn_decode import pak_entries, pof0_pointer_slots, FIXUP_BASE SPHERE = 16 # stride of the 0x5C record: centre f32[3] then radius f32 +VERTEX = 12 # stride of the 0x54 record: f32[3] +TRI = 6 # stride of the 0x58 record: u16[3] + + +def align16(n): + return (n + 15) // 16 * 16 def u32(b, o): return struct.unpack_from(">I", b, o)[0] +def u16(b, o): return struct.unpack_from(">H", b, o)[0] def f32(b, o): return struct.unpack_from(">f", b, o)[0] def ptr(b, o): return u32(b, o) + FIXUP_BASE @@ -42,9 +51,19 @@ class Mcol: for o in (0x54, 0x58, 0x5C, 0x74)) self.sphere_bytes = self.p54 - self.p5c self.n = self.sphere_bytes // SPHERE + self.nv = u16(blob, 0x50) # vertex count + self.nt = u16(blob, 0x52) # triangle count == sphere count self.bbox_min = [f32(blob, 0x10 + 4 * i) for i in range(3)] self.cell_size = [f32(blob, 0x40 + 4 * i) for i in range(3)] + def vertex(self, i): + o = self.p54 + VERTEX * i + return [f32(self.b, o + 4 * k) for k in range(3)] + + def triangle(self, i): + o = self.p58 + TRI * i + return [u16(self.b, o + 2 * k) for k in range(3)] + def sphere(self, i): o = self.p5c + SPHERE * i return [f32(self.b, o + 4 * k) for k in range(3)], f32(self.b, o + 12) @@ -120,6 +139,69 @@ def cmd_cells(pak, seed=12345): return hit / tot > 0.99 and ctl / tot < 0.25 +def cmd_mesh(pak): + """The 0x50 counts give both blocks a stride, and sphere i bounds triangle i.""" + import collections + print(f"{'object':>8} {'verts':>6} {'tris':>6} {'len 0x54':>9} {'len 0x58':>9} {'spheres':>8}") + n = sv = st = ss = 0 + enc = tight = ctl = tri_total = 0 + edges2 = edges = 0 + random.seed(7) + for h, blob in objects(pak): + m = Mcol(blob) + l54, l58 = m.p58 - m.p54, m.p74 - m.p58 + n += 1 + sv += align16(VERTEX * m.nv) == l54 + st += align16(TRI * m.nt) == l58 + ss += m.nt == m.n + print(f"{h:08x} {m.nv:6d} {m.nt:6d} {l54:9d} {l58:9d} {m.n:8d}") + + V = [m.vertex(i) for i in range(m.nv)] + T = [m.triangle(i) for i in range(m.nt)] + ec = collections.Counter() + for i, t in enumerate(T): + c, r = m.sphere(i) + far = max(math.dist(V[j], c) for j in t) + tri_total += 1 + enc += far <= r * 1.0001 + tight += abs(far / r - 1) < 0.01 + tj = T[random.randrange(m.nt)] + ctl += max(math.dist(V[j], c) for j in tj) <= r * 1.0001 + for k in range(3): + a, b_ = t[k], t[(k + 1) % 3] + ec[(min(a, b_), max(a, b_))] += 1 + edges += len(ec) + edges2 += sum(1 for v in ec.values() if v == 2) + print(f"\nlen(0x54) == align16(12 * verts) : {sv}/{n}") + print(f"len(0x58) == align16(6 * tris) : {st}/{n}") + print(f"triangle count == sphere count : {ss}/{n}") + print(f"sphere i encloses triangle i : {enc}/{tri_total} = {100*enc/tri_total:.2f}%") + print(f" ...and is tight to within 1% : {tight}/{tri_total} = {100*tight/tri_total:.2f}%") + print(f"sphere i encloses a random triangle (ctl) : {ctl}/{tri_total} = {100*ctl/tri_total:.2f}%") + print(f"edges shared by exactly two triangles : {edges2}/{edges} = {100*edges2/edges:.2f}%") + return (sv == st == ss == n and enc == tri_total and edges2 == edges + and ctl / tri_total < 0.10) + + +def cmd_obj(pak, want, out): + """Export one object as a Wavefront OBJ, so the decode can be looked at.""" + want = int(want, 16) + for h, blob in objects(pak): + if h != want: + continue + m = Mcol(blob) + with open(out, "w") as fh: + fh.write(f"# MCOL {h:08x} -- {m.nv} vertices, {m.nt} triangles\n") + for i in range(m.nv): + fh.write("v %.4f %.4f %.4f\n" % tuple(m.vertex(i))) + for i in range(m.nt): + a, b_, c = m.triangle(i) + fh.write(f"f {a+1} {b_+1} {c+1}\n") + print(f"{out}: {m.nv} vertices, {m.nt} triangles") + return True + raise SystemExit(f"no MCOL object {want:08x}") + + def main(): if len(sys.argv) < 3: raise SystemExit(__doc__) @@ -128,11 +210,17 @@ def main(): ok = cmd_stride(pak) elif cmd == "cells": ok = cmd_cells(pak) + elif cmd == "mesh": + ok = cmd_mesh(pak) + elif cmd == "obj": + ok = cmd_obj(pak, sys.argv[3], sys.argv[4]) elif cmd == "verify": a = cmd_stride(pak) print() b = cmd_cells(pak) - ok = a and b + print() + c = cmd_mesh(pak) + ok = a and b and c else: raise SystemExit(__doc__) print("\nPASS" if ok else "\nFAIL")