Compare commits
13 Commits
auto/re-we
...
auto/re-au
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8ee3bfbcef | ||
|
|
91fb022caf | ||
|
|
93d6534efe | ||
| 41c34dc3ca | |||
| 2e9903d0fc | |||
| 4623387f6c | |||
| 72365b217a | |||
| 22d2518847 | |||
| 1ee2d5e406 | |||
| 2196ac112d | |||
| 2fa4c33d1a | |||
| 6a41525c41 | |||
| 9daac6f1f0 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -20,3 +20,4 @@ Thumbs.db
|
||||
|
||||
# Trunk build output
|
||||
dist/
|
||||
__pycache__/
|
||||
|
||||
108
crates/sylpheed-formats/examples/idxd_tokens.rs
Normal file
108
crates/sylpheed-formats/examples/idxd_tokens.rs
Normal file
@@ -0,0 +1,108 @@
|
||||
//! RE diagnostic: emit every `weapon\Weapon_*.tbl` in a pak as machine-readable
|
||||
//! records, split at the sub-object boundaries the schema declares.
|
||||
//!
|
||||
//! An IDXD `.tbl` for a weapon holds `count` sub-records — a `Weapon` and its
|
||||
//! `Shell` — flattened into one string pool. `sylpheed-cli pak dump` shows them
|
||||
//! merged, which is ambiguous (both declare `ID`/`Name`). The title's schema
|
||||
//! name pool gives the split point: the pool contains the literal type names
|
||||
//! (`Weapon`, `Shell`), and each sub-record's fields follow its type name.
|
||||
//!
|
||||
//! Here we split on a type-name token appearing as a *key* position, so each
|
||||
//! record is emitted with its own ID and field set:
|
||||
//!
|
||||
//! ```text
|
||||
//! REC <tbl-hash> <index> <TypeName> <ID>
|
||||
//! F <key> <value>
|
||||
//! ```
|
||||
//!
|
||||
//! Run: cargo run --release -p sylpheed-formats --example idxd_tokens -- <pak> [types...]
|
||||
|
||||
use sylpheed_formats::idxd::IdxdObject;
|
||||
use sylpheed_formats::pak::PakArchive;
|
||||
|
||||
fn is_number(s: &str) -> bool {
|
||||
let s = s.strip_suffix(['f', 'F']).unwrap_or(s);
|
||||
let t = s.strip_prefix(['-', '+']).unwrap_or(s);
|
||||
!t.is_empty() && t.chars().all(|c| c.is_ascii_digit() || c == '.')
|
||||
}
|
||||
|
||||
/// `Yes`/`No`-style scalar literals are values, not field names, even though
|
||||
/// they are identifier-shaped.
|
||||
fn is_enum_value(s: &str) -> bool {
|
||||
matches!(s, "Yes" | "No" | "YES" | "NO" | "On" | "Off" | "ON" | "OFF")
|
||||
}
|
||||
|
||||
fn is_key_like(s: &str) -> bool {
|
||||
!s.is_empty()
|
||||
&& s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
|
||||
&& s.chars().next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_')
|
||||
&& !is_number(s)
|
||||
&& !is_enum_value(s)
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let mut args = std::env::args().skip(1);
|
||||
let pak_path = args.next().expect("usage: idxd_tokens <pak> [TypeName...]");
|
||||
let types: Vec<String> = {
|
||||
let v: Vec<String> = args.collect();
|
||||
if v.is_empty() {
|
||||
vec!["Weapon".into(), "Shell".into()]
|
||||
} else {
|
||||
v
|
||||
}
|
||||
};
|
||||
|
||||
let pak = PakArchive::open(&pak_path).expect("open pak");
|
||||
for entry in pak.entries() {
|
||||
let Ok(bytes) = pak.read(entry) else { continue };
|
||||
let Ok(obj) = IdxdObject::parse(&bytes) else { continue };
|
||||
let toks = obj.tokens();
|
||||
// Only entries that actually declare one of the requested sub-record
|
||||
// types (the pool-start heuristic can prefix one stray byte, so match a
|
||||
// short suffix rather than equality -- same rule as the splitter below).
|
||||
if !toks
|
||||
.iter()
|
||||
.any(|t| types.iter().any(|ty| t == ty || (t.ends_with(ty.as_str()) && t.len() <= ty.len() + 2)))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if std::env::var("SYLPH_RAW").is_ok() {
|
||||
println!("RAW {:08x}", entry.name_hash);
|
||||
for t in toks {
|
||||
println!("T {t}");
|
||||
}
|
||||
}
|
||||
|
||||
// Sub-record boundaries: a bare type-name token. The first one is the
|
||||
// schema's own declaration (`EnumWeapon`-style header lives in title
|
||||
// code, not here), so we take every occurrence in order.
|
||||
let mut bounds: Vec<(usize, &str)> = Vec::new();
|
||||
for (i, t) in toks.iter().enumerate() {
|
||||
// The pool-start heuristic can glue one stray binary byte onto the
|
||||
// first token ("YWeapon"), so match a short suffix, not equality.
|
||||
if let Some(ty) = types
|
||||
.iter()
|
||||
.find(|ty| t == *ty || (t.ends_with(ty.as_str()) && t.len() <= ty.len() + 2))
|
||||
{
|
||||
bounds.push((i, ty.as_str()));
|
||||
}
|
||||
}
|
||||
for (n, &(start, ty)) in bounds.iter().enumerate() {
|
||||
let end = bounds.get(n + 1).map(|b| b.0).unwrap_or(toks.len());
|
||||
let slice = &toks[start..end];
|
||||
// The record's own ID value is the token straight after its type
|
||||
// name (`Shell` -> `Shell_DSaber_P_wep_01_Beam`). The `ID`/`Name`
|
||||
// *keys* are interned once in the pool, so only the first record
|
||||
// shows them -- position-of-key lookup would miss the rest.
|
||||
let id = slice.get(1).map(|s| s.as_str()).unwrap_or("?");
|
||||
println!("REC {:08x} {n} {ty} {id}", entry.name_hash);
|
||||
for i in 1..slice.len() {
|
||||
let (k, v) = (slice[i].as_str(), slice[i - 1].as_str());
|
||||
if is_key_like(k) && (!is_key_like(v) || is_number(v)) {
|
||||
println!("F {k} {v}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
46
docs/re/BACKLOG.md
Normal file
46
docs/re/BACKLOG.md
Normal file
@@ -0,0 +1,46 @@
|
||||
# RE backlog
|
||||
|
||||
Open items that are *not* being worked right now. Each entry says what is wrong or
|
||||
unknown, what evidence exists, and what the first step would be. Move an item into
|
||||
`INDEX.md` (with a `structures/…md` or a parser + test) once it is actually settled.
|
||||
|
||||
---
|
||||
|
||||
## Capital ships assemble wrong in the viewer
|
||||
|
||||
**Reported:** 2026-07-30, by the user. **Status:** ❔ open, not investigated.
|
||||
|
||||
The reborn viewer builds capital ships from the split XBG7 parts via
|
||||
`sylpheed-formats::ship::assemble_ship`, and they come out **wrong** — parts in the
|
||||
wrong place / wrong orientation.
|
||||
|
||||
**Why this is a real finding and not a known limitation:** the RE write-up
|
||||
[`ship-placement-runtime-capture.md`](ship-placement-runtime-capture.md) declares
|
||||
static assembly ✅ **exact** as of 2026-07-26 — 9-channel joint tables
|
||||
`[TX TY TZ RY RX RZ SX SY SZ]`, Euler `Ry·Rx·Rz`, with
|
||||
`ship::tests::static_assembly_matches_runtime_capture` asserting static == runtime
|
||||
capture (T < 1.0, R < 0.02). So either the viewer is not using that path, or the
|
||||
claim generalises worse than the test suggests.
|
||||
|
||||
**The likely gap:** that test is **one ship** — the `e106` destroyer, 8 parts plus
|
||||
two nacelles, two turrets and the hull mirror. Nothing pins the other classes.
|
||||
Rules that were derived from `e106` and could easily be `e106`-specific:
|
||||
|
||||
- the engine cluster rig mounted at `GN_Engine_01` (two mirrored nacelles + centre);
|
||||
- "X-reflect the shared-geometry twin whose lateral offset opposes the geometry's
|
||||
dominant side" — a heuristic, not a decoded flag;
|
||||
- cross-id turret instancing (×2).
|
||||
|
||||
**First step (the oracle already exists):** re-run the runtime capture on a *different*
|
||||
capital ship and diff static vs captured, exactly as `e106` was done — F10 in the
|
||||
`capture-ship-placement` build of `xenia-canary-native` dumps the ship shader's
|
||||
`c0..c2` WorldViewProjection rows per part; `WV_ref⁻¹ · WV_p` is the ship-space rigid
|
||||
transform, which is ground truth. Pick a class whose rig differs from `e106`
|
||||
(different engine count, a ship with no `sld`, a carrier). Then extend
|
||||
`static_assembly_matches_runtime_capture` into a per-ship table so a regression in one
|
||||
class cannot hide behind `e106` passing.
|
||||
|
||||
**Also worth ruling out first, cheaply:** that the viewer's own transform stack (scale,
|
||||
handedness, node-instance recursion) is not re-breaking a correct assembly — compare
|
||||
the viewer's placement against `assemble_ship`'s output directly before blaming the
|
||||
format layer.
|
||||
@@ -21,9 +21,19 @@ Promote to a prose `structures/…md` file when a format needs behavioural notes
|
||||
| Fonts (ttf/otf/ttc) | ✅ | `sylpheed-formats/src/font.rs` | standard OpenType, parsed via ttf-parser |
|
||||
| XBG7 mesh | 🟡/❔ | `sylpheed-formats/src/mesh.rs` + `tests/mesh_disc.rs` ([xbg7](structures/xbg7-mesh.md)) | weapons/props: declaration-driven variable stride (36 models), GPU-confirmed. **Stage containers: 5662 sub-models across 22 stages** via content-anchored grouped pools (`stage_models`). Quantized hero bodies (DeltaSaber `f004`) still declined |
|
||||
| Capital-ship part placement | 🟡 | `sylpheed-formats/src/ship.rs` (static) + [runtime capture](ship-placement-runtime-capture.md) | hull placement static-exact; external parts approximate statically. **Runtime capture** (Canary F10 → VS-constant WorldView) gives ground truth — validated on `e106` destroyer; not yet baked into the viewer |
|
||||
| Weapon fields defaulted on disc | ✅/🟡 | [runtime DATA SHEET](weapon-datasheet-runtime.md) | The Arsenal Gallery-Mode panel reads the live record: `Ammo Capacity` = `LoadingCount` ✅, `Max. Lock Ons` = `TriggerShotCount` ✅. Recovers `TriggerShotCount` for `wep_05`/`wep_60` (both **4**); brackets defaulted `Power`/`MaximumRange` via the letter classes. 9 of ~61 weapons reachable at 5 % save progress |
|
||||
| Weapon fields defaulted on disc | ✅ | [runtime struct](structures/weapon-struct-runtime.md) · [DATA SHEET route](weapon-datasheet-runtime.md) | **Solved.** Canary maps guest RAM into `/dev/shm`, so the parsed `Weapon`/`Shell` objects are readable live; their layout is solved against disc ground truth (zero contradictions over 100+ records). All 126 weapons, exact numbers, no story progress needed — [4 393 values](captures/weapon-runtime-fields.csv) the disc does not carry. Supersedes the letter-bucket limit of the DATA SHEET route, which now serves as the independent cross-check |
|
||||
| Unit (craft/vessel) fields defaulted on disc | ✅/🟡 | [runtime struct](structures/unit-struct-runtime.md) | The parsed `unit\UN_*.tbl` definition object, vtable `0x820af844`, ≥`0x380` bytes, one per unit — **discovered, not assumed** (`unit_discover.py`), and distinguished from the spawned-entity class `0x820af030` by being one-per-ID and byte-constant within a run. Across runs only pointer words move — `--crosscheck` proves **no reported field offset is run-dependent** (two words, `+0x2c8`/`+0x2d0`, are stage-dependent and remain unidentified). 27 fields ✅ (21 units, 7 runs); the `Maneuver` block is **schema declaration order, 4 bytes/field, base `0x9c` with a two-slot gap after `AA_Roll_Min`** (29 anchors, 0 conflicts), which also pins 5 fields *no* disc record ever values. Angles are **radians at runtime, degrees on disc**. Unlike weapons, unit definitions are instantiated **per stage**, so coverage (21/110) grows by visiting missions — [values](captures/unit-runtime-fields.csv) |
|
||||
| UI screen layout (`.rat`) | ✅/🟡 | [ui-rat-layout](structures/ui-rat-layout.md) | One pak per UI screen; each RATC = one (context × language) build; every `<name>.t32` sprite has a `<name>.rat` **layout record** (BE u32; 1280×720 design space; scale/tint/X/Y, keyframes for animated elements, `opt ` link to the focused state). **The tutorial PAUSE menu and the title main menu both rebuild pixel-accurately from the disc.** `loop1.rat` (screen-level draw order) not yet decoded |
|
||||
|
||||
## Runtime / dynamic-capture technique
|
||||
|
||||
| Technique | Conf. | Spec | Notes |
|
||||
|-----------|-------|------|-------|
|
||||
| Live guest-memory read | ✅ | [`tools/re-capture/gmem.py`](../../tools/re-capture/gmem.py) | Canary backs the guest address space with `/dev/shm/xenia_memory_*`; guest VAs map in through Xenia's fixed table. Full-RAM search ~0.2 s (sparse, `SEEK_DATA`). No debugger, no emulator patch, game keeps running |
|
||||
| IDXD object layout solver | ✅ | [`tools/re-capture/weapon_runtime.py`](../../tools/re-capture/weapon_runtime.py) | Scan RAM for a class's vtable → enumerate its objects → brute-force `(field, offset, encoding)` against the disc records. Accepts a binding only on **zero** contradictions. Generalizes to any IDXD-backed definition |
|
||||
| Live entity state, anchored on the definition | ✅ | [`tools/re-capture/own_state.py`](../../tools/re-capture/own_state.py) · [autopilot](autopilot-memory-driven.md) | An undamaged craft holds its definition's own numbers, so a *solved definition field* locates the matching live field without a value scan: definition `HP` (1500) → **hull at `position+0x154`**, confirmed by a trace across a death (30/60/90 per hit, negative at 0). Reusable for any live counter whose maximum the definition carries |
|
||||
| Input → dynamics calibration | ✅ | [`tools/re-capture/ctrl_probe.py`](../../tools/re-capture/ctrl_probe.py) · [`binq.py`](../../tools/re-capture/binq.py) | Hold each pad input in turn and measure the craft's speed as displacement/s of its own position triple — no speed field needed first. Settled the throttle: **`RT` accelerates, `LT` brakes, and the setting persists** (488 → 1510 → 174 units/s), overturning an earlier field-scan conclusion |
|
||||
|
||||
## Functions / code paths
|
||||
|
||||
_None documented yet — populated during the dynamic-RE phase._
|
||||
|
||||
299
docs/re/autopilot-memory-driven.md
Normal file
299
docs/re/autopilot-memory-driven.md
Normal file
@@ -0,0 +1,299 @@
|
||||
# Memory-driven autopilot — build log and current state
|
||||
|
||||
**Status: 🟢 IT FLIES, KILLS AND SURVIVES — but it loses the mission anyway.**
|
||||
Updated 2026-07-30. `pilot.py` flew Stage 02 for **300 s with the hull untouched
|
||||
at 1500/1500** and scored the first confirmed autopilot kill (`YOU KILLED
|
||||
WARPLANES 0001` on the HUD, screenshots `shots/pilot1-*.png`); the scene's
|
||||
hostile count fell from 134 to 111 over the run. The day before, every run was
|
||||
dead inside 35 s. What is still missing is the *end* of a mission: the objective
|
||||
counter (`REMAINING OB`) rises as new waves spawn, and nothing yet tracks which
|
||||
targets actually close it out — and the second run proved the point the hard
|
||||
way: `GAME OVER` with the hull at 1500/1500, because Stage 02 is an **escort**
|
||||
and the ACROPOLIS was sunk while the pilot chased fighters two kilometres away.
|
||||
|
||||
## 2026-07-30 — the numbers survival needs
|
||||
|
||||
Three things the loop was missing were measured this session, each by
|
||||
consequence rather than by reading a field and hoping.
|
||||
|
||||
### Hull is `position + 0x154` ✅ CONFIRMED
|
||||
|
||||
The unit definition already had `HP` solved at `+0x054`
|
||||
([unit-struct-runtime](structures/unit-struct-runtime.md)); the Delta Saber's is
|
||||
**1500**. A craft that has taken no damage must therefore *contain that number*,
|
||||
which turns "find the HP field" into a two-float lookup rather than a value scan
|
||||
(`own_state.py`). It appears once in the entity object, at `pos+0x154`, and the
|
||||
trace across a death settles it (`ctrl_probe.py` capture, `binq.py trace`):
|
||||
|
||||
```
|
||||
t phase hull
|
||||
0.00 base 1500.00 <- == definition HP
|
||||
23.25 rest_A 1380.00 <- first hit, -120
|
||||
26.68 … 27.18 B 1320 … 930 <- seven hits in 0.5 s
|
||||
31.93 X 150.00
|
||||
35.21 Y -30.00 <- goes negative
|
||||
35.26 Y -180.00 -> GAME OVER on screen
|
||||
```
|
||||
|
||||
Damage arrives in 30/60/90-point steps and the field goes *negative* at death,
|
||||
so it is the raw hull counter, not a clamped display value. **1500 hull lost in
|
||||
12 s** of sitting in a turret's line of fire is the whole reason every earlier
|
||||
run died.
|
||||
|
||||
### Shield is `position + 0x430` 🟡 PROBABLE — not yet confirmed live
|
||||
|
||||
Same anchor trick: the definition's shield `MaxValue` is **400** and
|
||||
`ChargeSpeed` **25**, and the entity object holds `400.0` at `+0x430`, `+0x434`
|
||||
and `+0x438`, with `25.0` at `+0x448`. Which of the three is the *current* value
|
||||
is unproven — the capture that spanned the death used a ±0x400 window and
|
||||
cropped them out. `ctrl_probe.py` now samples ±0x800.
|
||||
|
||||
### `RT` accelerates, `LT` brakes, and the throttle is a *setting* ✅ CONFIRMED
|
||||
|
||||
`ctrl_probe.py` holds each input in turn and measures the craft's own speed as
|
||||
displacement per second from the position triple, so no speed field is needed.
|
||||
Distance flown / phase duration, one 3 s hold each, sticks neutral:
|
||||
|
||||
| phase | speed (units/s) | | phase | speed (units/s) |
|
||||
|---|---|---|---|---|
|
||||
| base (no input) | 488 | | A | 287 |
|
||||
| **RT** | **1510** | | B | 139 |
|
||||
| rest after RT | 1056 | | X | 125 |
|
||||
| **LT** | **174** | | Y | (dying) |
|
||||
| rest after LT | 428 | | LB / LS / RS / RY / RX / dpad | no effect |
|
||||
|
||||
RT triples the speed, LT cuts it to a third, and **the braked state persists**:
|
||||
after the LT phase the craft sat at 125–140 units/s with the sticks and triggers
|
||||
neutral for the remaining 40 s, and nothing but RT brought it back. So these are
|
||||
a throttle setting, not a momentary boost — which also means a control loop must
|
||||
send only the *changes*.
|
||||
|
||||
This **corrects** the earlier note in this file ("`RT` is *not* the throttle,
|
||||
and no button tested is"). That conclusion came from `findspeed.py`, which
|
||||
assumed the control and went looking for a *field* that rose; measuring the
|
||||
speed directly reverses it.
|
||||
|
||||
### Two method corrections
|
||||
|
||||
* **Pick the attitude block by the flight path, not by address order.** The
|
||||
player object contains **20** orthonormal 3×3 blocks (identity frames, bone
|
||||
or camera frames), and `pos-0x70` and `pos-0x30` hold the *same* matrix.
|
||||
Taking `found[0]` wrote a config with `rot_delta = -0x764` and a nonsense
|
||||
forward axis; `entities2.py self` now scores every block against the measured
|
||||
direction of travel and picks the best (`cos = +1.000`, row 2, sign +1).
|
||||
* **The entity-heap scan has to be numpy.** A per-word Python loop over the
|
||||
16 MB entity region costs seconds per scan, which is the whole budget of a
|
||||
10 Hz control loop; `np.isin` over a `>u4` view is milliseconds.
|
||||
|
||||
### Stage 02 as an autopilot testbed (from the in-flight HUD)
|
||||
|
||||
`OBJECTIVE: shoot down all invading enemy fighters while watching out for
|
||||
attacks on the ACROPOLIS` · `DEFEAT: your fighter is shot down, or the ACROPOLIS
|
||||
is sunk` · `HINT: you can resupply at the ACROPOLIS`. The HUD shows
|
||||
**`REMAINING OB 004`** — only four objective targets — so this mission is
|
||||
winnable by an autopilot that survives. It also shows separate **SHIELD** and
|
||||
**ARMOR** bars (matching a 400-point shield over 1500 hull), `A/B 7,635`
|
||||
afterburner, and `NOSE BM 06000` / `MAIN MPM 00300` ammo.
|
||||
|
||||
## What the loop did before that, observed
|
||||
|
||||
```
|
||||
[ 82.5] tgt=e007_ADAN_Turret d=3384 yaw= -7.3 pit=+14.6 stick=(-0.13,-0.34) fire=0
|
||||
[ 85.7] tgt=e007_ADAN_Turret d=2797 yaw= +1.9 pit=+27.2 stick=(+0.20,-0.59) fire=0
|
||||
[117.3] tgt=e007_ADAN_Turret d=4211 yaw= +5.7 pit= +9.0 stick=(+0.25,-0.40) fire=1
|
||||
```
|
||||
|
||||
Distance closes monotonically, yaw error is driven from −8° to ~0, and once both
|
||||
errors are inside the firing cone it holds RB and the ammo counter falls. A
|
||||
rescan reports the scene as e.g. `136 entities {'TCAF': 16, 'ADAN': 120}`.
|
||||
|
||||
**The chain that made it work** — each link checked, not assumed:
|
||||
|
||||
1. **Entity typing.** A live entity's definition pointer sits at
|
||||
**position + 0x130**. One heap scan then yields every craft *with its unit
|
||||
type*, which is what separates 20-odd real combatants from ~30 000 moving
|
||||
particles. Verified by the result being coherent: wingmen, enemy turrets and
|
||||
attackers, friendly capital ships, and exactly one `…_Player`.
|
||||
2. **Orientation.** A 3×3 rotation at **position − 0x70**, stored with a
|
||||
**16-byte row stride** (a 4×4 transform whose translation row *is* the
|
||||
position). An earlier search for nine *contiguous* floats structurally could
|
||||
not find this, which is why the first pass concluded "no transform". The
|
||||
binding is confirmed independently: its row 2 matches the craft's measured
|
||||
direction of travel with **cos = +1.000**.
|
||||
3. **The fire button is RB** — established by consequence, not by guessing:
|
||||
of RB/LB/A/B/X/Y/RT/LT, pressing RB is the only one that makes the nose-ammo
|
||||
counter in RAM fall (5958 → 5940). (This entry also claimed `RT` is *not* the
|
||||
throttle — **wrong**, see the 2026-07-30 measurement above.)
|
||||
4. **Control.** PD on the aiming error with the derivative taken from the
|
||||
craft's own body angular velocity (from two consecutive rotation matrices),
|
||||
and target selection weighted by off-boresight angle
|
||||
(`score = d·(1 + 3·(θ/π)²)`) rather than pure nearest — closing on a target
|
||||
90° off the nose only raises the bearing rate, which is what held the first
|
||||
run outside its firing cone at a steady ~27° pitch error.
|
||||
|
||||
## Goal
|
||||
|
||||
Fly and fight a mission by reading the game's own world state out of guest RAM
|
||||
and driving the pad from it — the RE payoff being an oracle for the
|
||||
reimplementation's flight model and AI, and a way to reach missions the save
|
||||
cannot otherwise reach (unit coverage for
|
||||
[unit-struct-runtime](structures/unit-struct-runtime.md) is capped at 21/110
|
||||
because definitions load **per stage**).
|
||||
|
||||
## What works (verified)
|
||||
|
||||
| Piece | Tool | Evidence |
|
||||
|---|---|---|
|
||||
| Live guest-RAM reads at loop rate | `gworld.py` | `/dev/shm` file opened once, `pread` per tick; a whole-RAM scan is ~6 s, a targeted read is microseconds |
|
||||
| Whole-RAM float scanning | numpy over `SEEK_DATA` extents | 1 270 orthonormal 3×3 blocks located in 6.2 s |
|
||||
| Entity enumeration by unit type | `gworld.py entities` | 116 live instances in Stage 02, typed by unit ID, incl. exactly one `…_Player` |
|
||||
| Pad control at loop rate | `flight_probe.Pad` | writes command lines straight into the vgamepad FIFO; the `vgamepad` CLI spawns a process per command and its `tap`/`hold` sleep *inside* the server, so neither is usable in a control loop |
|
||||
| Unattended mission entry | `launch_mission.sh` | title → LOAD GAME → slot 01 → READY ROOM → TAKE OFF → in flight, repeatable |
|
||||
| Hangar loadout | `launch_mission.sh --hangar` | the "Recommended" control is **AUTO SELECT — "Mount most suitable weapons"**, already the default cursor position. At 5 % progress it is a no-op: only two weapons are developed, and they are already mounted |
|
||||
| Finding moving objects | `findplayer.py` | position triples recovered from motion alone — straight-line, constant-speed filter over K whole-RAM samples |
|
||||
|
||||
## What is NOT solved
|
||||
|
||||
**Survival, and therefore mission completion.** The loop has no evasion, no
|
||||
shield/armour awareness and no throttle control, so it flies a straight pursuit
|
||||
into defended space and is eventually shot down — every long run so far has
|
||||
ended in GAME OVER. Completing a mission needs, at least: reading own
|
||||
shield/armour, breaking off when hit, and prioritising the mission's actual
|
||||
objective targets over the nearest turret.
|
||||
|
||||
### Superseded (kept because the reasoning still matters)
|
||||
|
||||
The notes below were written before the chain above worked. They remain true as
|
||||
statements about `0x820af030`, which is *not* the live entity —
|
||||
|
||||
1. **The `0x820af030` class is not the live entity.** It has one object per
|
||||
spawned thing and carries the unit-ID string, so it looked like the entity
|
||||
list — but over a 29 s in-flight capture **all 384 words of it are
|
||||
constant** (`whatchanges.py`). It is a static spawn record. The earlier
|
||||
claim in `unit-struct-runtime.md` that this class is "the spawned entity
|
||||
instance — live state" is **wrong in the second half**: it is per-spawn, but
|
||||
it is not live state. The parts of that document that depend on the
|
||||
*definition* class `0x820af844` are unaffected.
|
||||
2. **No transform in or one hop from that object**: no orthonormal 3×3, and no
|
||||
unit quaternion, within `0x2000` of it or behind any of its 121 pointers.
|
||||
3. **Input correlation finds *a* self-object, but not obviously the craft.**
|
||||
Holding hard-left then hard-right yaw and looking for a position whose turn
|
||||
axis reverses (`findself.py`, `selfstate.py`) gives clean hits
|
||||
(`cos ≈ −0.99`), but they cluster at `0x40009xxx` in what looks like an
|
||||
8-corner box with ±45 000 coordinates — a camera/skybox volume that follows
|
||||
the player, not the craft. Its speed (≈359/s) is suspiciously close to the
|
||||
HUD's 350, which supports "follows the player" but is not proof of identity.
|
||||
4. **Entity typing is unavailable**: no definition pointer within ±0x800 of the
|
||||
self-position, so the trick of learning one object's layout and applying it
|
||||
to all the others has nothing to anchor on. Without typing, the 33 418
|
||||
moving triples in a firefight cannot be separated into enemies, friendlies
|
||||
and bullets, so there is nothing to aim at.
|
||||
|
||||
## Dead ends, recorded so they are not re-run
|
||||
|
||||
* **Speed-scan for the player object** (`findspeed.py`, the classic two-state
|
||||
value scan: coast → boost → coast). Sound method, but **`RT` is not the
|
||||
throttle** — 5 864 floats matched the cruise speed and none rose. The control
|
||||
actually bound to acceleration was never established, and the run that would
|
||||
have established it ended in GAME OVER.
|
||||
* **Comparing orientation matrices 2 s apart.** At a real turn rate that is far
|
||||
outside the small-angle regime, so the skew part of `A·Bᵀ` is not the rotation
|
||||
vector and the "angular velocity" comes out as ~30 000. Sample incrementally
|
||||
(6 Hz) and re-check orthonormality on every read — blocks found by a scan get
|
||||
overwritten between the scan and the read.
|
||||
|
||||
## The second run lost the mission **without being hit** (2026-07-30)
|
||||
|
||||
A second 240 s flight, with the two fixes above, ended on the `GAME OVER`
|
||||
screen — while the hull read **1500/1500 on the last live tick**. Nothing shot
|
||||
us down. The other defeat condition fired: *the ACROPOLIS is sunk*. The HUD had
|
||||
been showing a red `WARNING` banner for a while, and the pilot spent the whole
|
||||
run pursuing an `e010_ADAN_Attacker_S` two kilometres away.
|
||||
|
||||
So surviving is necessary and not sufficient, and "nearest hostile fighter" is
|
||||
the wrong objective function for this stage. **The mission is an escort.** What
|
||||
follows:
|
||||
|
||||
* **Prioritise hostiles by their distance to the protected asset, not to us.**
|
||||
The attackers worth killing are the ones closing on the ACROPOLIS.
|
||||
* **The protected asset's health is readable with the same anchor as ours** —
|
||||
hull at `position + 0x154`, its maximum being its own definition's `HP`. That
|
||||
gives a live "are we winning" signal for the escort, and it should drive the
|
||||
target choice directly.
|
||||
* A frozen tail in the log (identical position, speed and target for the last
|
||||
five seconds) is what mission-end looks like from the outside, **not** an
|
||||
emulator wedge. Worth knowing before diagnosing the wrong thing.
|
||||
* Practical: do **not** pipe a long run's log through `tail` — that discards
|
||||
everything but the end, and the interesting part of this run is gone.
|
||||
|
||||
## After survival, the blocker is lethality (2026-07-30)
|
||||
|
||||
The 300 s run took **no damage at all** and killed **one** warplane, spending
|
||||
~800 rounds of nose ammo (`06000` → `05193`) to do it, while `REMAINING OB` rose
|
||||
from `004` to `011` as fresh waves spawned. So attrition at this rate never
|
||||
finishes the mission, and the ranking of open problems has changed:
|
||||
|
||||
1. **Hit rate.** It opens fire at 2–5 km with a 9° cone and a crude lead
|
||||
(`p + v·d/speed`, no projectile speed). The `Shell` records in
|
||||
[weapon-struct-runtime](structures/weapon-struct-runtime.md) carry the real
|
||||
projectile speed and `MaximumRange` per weapon — the lead and the firing
|
||||
range should come from *those*, not from constants.
|
||||
2. **Which targets count.** `REMAINING OB` is the mission's own objective
|
||||
counter and it is on screen, so it is in RAM; finding it turns "shoot
|
||||
whatever is nearest" into "shoot what closes the mission". Objective-marked
|
||||
entities also draw an `OB` badge in the HUD, so the flag is likely a word in
|
||||
the entity object.
|
||||
3. **Confirming the shield word** — needs a run that actually takes damage; the
|
||||
pilot is now good enough at avoiding that to make it awkward, so drive
|
||||
straight at a turret on purpose with `--dry` steering disabled.
|
||||
4. **Does the ACROPOLIS repair?** RETIRE mode has never triggered (the hull
|
||||
never fell), so the resupply hint is still untested.
|
||||
|
||||
## The next step that unblocks the most (superseded — kept for the reasoning)
|
||||
|
||||
**Update 2026-07-30: this is no longer the blocker.** Entity typing via the
|
||||
definition pointer already solved target selection, so the game's own target
|
||||
pointer is now a convenience rather than a prerequisite. It would still be the
|
||||
cheapest route to problem 2 above (objective targets), because whatever the HUD
|
||||
locks on to is what the game itself considers a target.
|
||||
|
||||
**Find the game's own target pointer instead of typing entities ourselves.**
|
||||
The HUD has a lock-on system (a `TARGET` marker and a target-cycle button), so
|
||||
a global almost certainly holds a pointer to the currently-targeted entity.
|
||||
Reading that gives an enemy's live object address directly — which yields both
|
||||
target selection *and* the entity layout (position offset within it), i.e. it
|
||||
collapses problems 3 and 4 into one. It is also cheap to find: cycle the target
|
||||
with the pad and watch which pointer-shaped global changes in step.
|
||||
|
||||
## Operational notes
|
||||
|
||||
* An unattended craft **dies** — the ship flies straight into a firefight, and
|
||||
two long scans were invalidated by a GAME OVER mid-run. Any scan longer than
|
||||
~30 s needs either a survivable holding pattern or a fresh mission.
|
||||
* `Xvfb` and the emulator die on their own every few minutes here, cleanly
|
||||
(exit 0), cause unidentified. Everything that must not be interrupted is run
|
||||
as **one background task** that starts the display, the emulator, the
|
||||
navigation and the measurement together, so nothing has to survive between
|
||||
tool calls.
|
||||
* `pgrep` cannot be used for liveness in this container: PID 1 is
|
||||
`sleep infinity` and never reaps, so dead processes linger as `<defunct>` and
|
||||
still match by name. Use `ps -o stat=` and skip `Z`, or `xdpyinfo` for X.
|
||||
* numpy is not installed system-wide; `pip install --break-system-packages
|
||||
numpy` puts it in `/sylph-home/.local`, which is only on `sys.path` when
|
||||
`HOME=/sylph-home`. Scripts run with `HOME=/sylph-home/re` need
|
||||
`PYTHONPATH=/sylph-home/.local/lib/python3.12/site-packages`.
|
||||
|
||||
## Files
|
||||
|
||||
`pilot.py` (**the survival loop**) · `ctrl_probe.py` (input → speed calibration,
|
||||
plus a per-tick window of the player object) · `binq.py` (query that capture) ·
|
||||
`own_state.py` (definition-anchored hull/shield lookup) · `fly_session.sh`
|
||||
(boot → mission → bind → fly, one task) · `wait_flight.sh` (wait for the real
|
||||
HUD instead of a fixed sleep) · `navigator.py` (drift-aware steering + CPA
|
||||
avoidance, reused by the pilot) ·
|
||||
`gworld.py` (live reader + entity list) · `flight_probe.py` (scripted inputs +
|
||||
sampling, and the `Pad` FIFO client) · `flight_analyze.py` · `whatchanges.py`
|
||||
(encoding-agnostic "which words are live") · `findplayer.py` · `findself.py` ·
|
||||
`findrot_global.py` · `findspeed.py` · `liveents.py` · `selfstate.py` (the
|
||||
whole chain → JSON) · `autopilot2.py` (PD controller; **untested — it has never
|
||||
had a valid config to run against**) · `launch_mission.sh`.
|
||||
BIN
docs/re/captures/autopilot-300s-undamaged-stage02.png
Normal file
BIN
docs/re/captures/autopilot-300s-undamaged-stage02.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.7 MiB |
BIN
docs/re/captures/autopilot-escort-warning-stage02.png
Normal file
BIN
docs/re/captures/autopilot-escort-warning-stage02.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.6 MiB |
BIN
docs/re/captures/autopilot-first-kill-stage02.png
Normal file
BIN
docs/re/captures/autopilot-first-kill-stage02.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.3 MiB |
1781
docs/re/captures/ctrl-probe-stage02.csv
Normal file
1781
docs/re/captures/ctrl-probe-stage02.csv
Normal file
File diff suppressed because it is too large
Load Diff
1828
docs/re/captures/unit-runtime-fields.csv
Normal file
1828
docs/re/captures/unit-runtime-fields.csv
Normal file
File diff suppressed because it is too large
Load Diff
7183
docs/re/captures/weapon-runtime-fields.csv
Normal file
7183
docs/re/captures/weapon-runtime-fields.csv
Normal file
File diff suppressed because it is too large
Load Diff
274
docs/re/structures/unit-struct-runtime.md
Normal file
274
docs/re/structures/unit-struct-runtime.md
Normal file
@@ -0,0 +1,274 @@
|
||||
# Runtime `Unit` struct (craft / vessel definitions) — read from live guest memory
|
||||
|
||||
**Confidence: ✅ CONFIRMED** for the 27 fields marked ✅ below (each binding is
|
||||
reproduced by 10–21 independent disc records, on ≥3 distinct values, with
|
||||
**zero** contradictions), plus the Maneuver block's declaration-order layout
|
||||
(29 anchors, two bases, no conflicts). 🟡 PROBABLE for the fields interpolated
|
||||
between confirmed anchors. 🟡/❔ for the thin single-value bindings, which are
|
||||
listed but must not be trusted yet.
|
||||
|
||||
Captured 2026-07-29 from Xenia Canary running the retail disc in the sylph-re
|
||||
container: **all six tutorials** and **Stage 02 "Declaration of War"** loaded
|
||||
from save slot 01. 21 of the disc's 110 units, over 7 snapshots from 7 separate
|
||||
emulator runs.
|
||||
|
||||
## Why this exists
|
||||
|
||||
Craft stats were the one Route-B target the previous two passes could not
|
||||
reach. The Hangar exposes only `Gross Weight` (a class, not a number), and
|
||||
[the menu route](../weapon-datasheet-runtime.md) has no surface at all for the
|
||||
flight model. From the memory side, menus were equally useless: **only the
|
||||
player craft's name string is resident there — the definition objects do not
|
||||
exist until a mission loads.**
|
||||
|
||||
They do exist in-mission. This documents their layout and the values the disc
|
||||
leaves defaulted.
|
||||
|
||||
## Finding the class (discovered, not assumed)
|
||||
|
||||
[`tools/re-capture/unit_discover.py`](../../../tools/re-capture/unit_discover.py)
|
||||
takes no vtable as input. It locates every disc unit-ID string in a memory
|
||||
snapshot, finds every aligned word pointing at `string_va - d` for a range of
|
||||
`d`, and tallies the word at `pointer_site - k` across **distinct** unit IDs.
|
||||
One `(d, k, word)` combination wins by a wide margin:
|
||||
|
||||
| δ (name-record → string) | ID pointer at | word | distinct units |
|
||||
|---|---|---|---|
|
||||
| `0x10` | object `+0x04` | `0x820af030` | 4 |
|
||||
| `0x10` | object `+0x04` | `0x820af844` | 4 |
|
||||
|
||||
— i.e. exactly the `Weapon` shape: the object holds a name-record pointer at
|
||||
`+0x04`, and the ID string sits at `name_record + 0x10`.
|
||||
|
||||
The two vtables are **two different things**, and telling them apart matters:
|
||||
|
||||
| vtable | what it is | evidence |
|
||||
|---|---|---|
|
||||
| `0x820af030` | **spawned entity record** — one per spawned thing, but **not live state** (see below) | 12 objects for 4 IDs; the same ID appears many times (one per box in the scene); irregular spacing |
|
||||
| `0x820af844` | **parsed definition** — the `.tbl` | exactly one object per distinct unit ID; minimum spacing `0x380` |
|
||||
|
||||
**Correction (2026-07-29, from the autopilot work):** `0x820af030` was
|
||||
described here as holding live state. It does not — across a 29 s in-flight
|
||||
capture **all 384 words of it are constant**. It is one record per spawned
|
||||
thing, but the flying entity's transform is somewhere else entirely. See
|
||||
[autopilot-memory-driven](../autopilot-memory-driven.md). Nothing below depends
|
||||
on it; the definition class `0x820af844` is unaffected.
|
||||
|
||||
Only `0x820af844` is used. It is the runtime image of the `.tbl`:
|
||||
|
||||
* **Within one run** it is byte-identical across two snapshots taken ~12 minutes
|
||||
apart with combat in between (14/14 objects, 0 differing bytes) — definition
|
||||
data, not live state.
|
||||
* **Across runs** the same unit is *not* byte-identical, and that had to be
|
||||
explained rather than waved away. `unit_runtime.py --crosscheck` compares
|
||||
every unit that appears in more than one snapshot (5 of them, over 7 runs):
|
||||
exactly **15 words differ**, and 13 of them hold guest pointers
|
||||
(`0x8xxxxxxx`/`0xbxxxxxxx` — heap addresses, which move per process).
|
||||
**No solved or interpolated field offset is among the 15** — every value
|
||||
reported here is run-invariant.
|
||||
* The two non-pointer stragglers, `+0x2c8` and `+0x2d0`, are **stage-dependent**:
|
||||
for one and the same unit (`UN_f001_TCAF_DeltaSaber_T_Ttrl`) `+0x2c8` reads
|
||||
`8000.0` in two tutorials and `10000.0` in a third, with `+0x2d0` a 0/1 flag
|
||||
beside it. So the object is *mostly* but not *entirely* the parsed table —
|
||||
a couple of words are set per stage. Unidentified; **NEEDS-HUMAN**.
|
||||
|
||||
One `.tbl` → **one** object. A unit table is several sub-records (`Generic`,
|
||||
`Maneuver`, `Shield`, `Explosion`, `Mass`, `Effect`, `SE`, `Turret_00N`), and
|
||||
they are all flattened into that single ≥`0x380`-byte object — unlike weapons,
|
||||
where `Weapon` and `Shell` are separate arrays.
|
||||
|
||||
## Solving the layout
|
||||
|
||||
Same discipline as
|
||||
[`weapon_runtime.py`](../../../tools/re-capture/weapon_runtime.py): score every
|
||||
`(field, byte-offset, encoding)` triple against the disc records and accept a
|
||||
binding only on **zero contradictions**, requiring ≥3 distinct values so a
|
||||
field whose samples are all one number cannot match any offset holding that
|
||||
constant.
|
||||
|
||||
```bash
|
||||
python3 tools/re-capture/unit_runtime.py unit_tokens.txt snap_a.bin snap_b.bin --csv
|
||||
```
|
||||
|
||||
Snapshots are **unioned** — each mission instantiates only the units in its own
|
||||
stage, so coverage grows by visiting stages. `cp --sparse=always` a copy of
|
||||
`/dev/shm/xenia_memory_*` first (~2 s); the running emulator pegs every core
|
||||
under lavapipe and makes repeated live reads flaky.
|
||||
|
||||
### Encoding note — angles are radians at runtime
|
||||
|
||||
Every `AV_*` / `AA_*` / `*Bank*` / `Turn_AngularVelocity` field is stored as
|
||||
**float32 radians**, while the disc writes **degrees**. The solver needed a
|
||||
`rad` encoding (`degrees(f32)`) to bind them at all; 18 units agree on
|
||||
`AV_PitchPlus_Max` alone. A reimplementation reading the `.tbl` must convert.
|
||||
|
||||
## Confirmed layout
|
||||
|
||||
✅ = ≥10 disc records agree on ≥3 distinct values, zero contradict.
|
||||
|
||||
| offset | enc | field | agree | distinct |
|
||||
|---|---|---|---:|---:|
|
||||
| `+0x030` | f32 | `Size_X` | 21 | 13 |
|
||||
| `+0x034` | f32 | `Size_Y` | 12 | 7 |
|
||||
| `+0x038` | f32 | `Size_Z` | 19 | 13 |
|
||||
| `+0x040` | f32 | `Color_R` | 21 | 5 |
|
||||
| `+0x044` | f32 | `Color_G` | 19 | 6 |
|
||||
| `+0x048` | f32 | `Color_B` | 17 | 5 |
|
||||
| `+0x050` | f32 | `Size_Radius` | 12 | 10 |
|
||||
| `+0x054` | f32 | `HP` | 19 | 10 |
|
||||
| `+0x074` | f32 | `ResistanceToOptics` | 11 | 3 |
|
||||
| `+0x08c` | f32 | `ScorePoint` | 21 | 9 |
|
||||
| `+0x094` | f32 | `MassScore` | 10 | 7 |
|
||||
| `+0x09c` | f32 | `MinimumVelocity` | 11 | 3 |
|
||||
| `+0x0a0` | f32 | `MaximumVelocity` | 19 | 8 |
|
||||
| `+0x0a4` | f32 | `CruisingVelocity` | 17 | 7 |
|
||||
| `+0x0a8` | f32 | `Acceleration` | 18 | 5 |
|
||||
| `+0x0ac` | f32 | `Deceleration` | 17 | 5 |
|
||||
| `+0x0b0` | rad | `AV_PitchPlus_Max` | 18 | 7 |
|
||||
| `+0x0b4` | rad | `AV_PitchPlus_Min` | 10 | 6 |
|
||||
| `+0x0c4` | rad | `AV_PitchMinus_Min` | 10 | 5 |
|
||||
| `+0x0f8` | f32 | `SideThrustVelocity_Max` | 14 | 3 |
|
||||
| `+0x238` | f32 | `MaxValue` (Shield) | 13 | 6 |
|
||||
| `+0x244` | f32 | `ChargeSpeed` (Shield) | 13 | 6 |
|
||||
| `+0x270` | f32 | `DestroyMotionTime` | 19 | 7 |
|
||||
| `+0x2a0` | f32 | `RadarRange` | 18 | 9 |
|
||||
| `+0x2a4` | f32 | `FCSRange` | 12 | 8 |
|
||||
| `+0x2b4` | f32 | `AttackVesselPoint` | 14 | 8 |
|
||||
| `+0x2bc` | f32 | `DefencePoint` | 12 | 7 |
|
||||
|
||||
`HQRatio +0x058`, `ShieldRatio +0x05c`,
|
||||
`ThrusterRatio +0x060`, `ResistanceToShell +0x078`,
|
||||
`ResistanceToExplosion +0x07c`, `ResistanceToPlayer +0x080`,
|
||||
`ResistanceParalyze +0x084`, `BridgeCount +0x070` (i32),
|
||||
`DryMass +0x274`, `GrossMass +0x278`,
|
||||
`LowerHPThresholdRatio +0x298`, `AttackCraftPoint +0x2b8` bind with zero
|
||||
contradictions on fewer records or fewer distinct values — 🟡 PROBABLE. The
|
||||
full solver output is in
|
||||
[`captures/unit-runtime-fields.csv`](../captures/unit-runtime-fields.csv)
|
||||
(1 827 values, `conf` column).
|
||||
|
||||
## The Maneuver block is laid out in schema declaration order
|
||||
|
||||
This is the strongest structural result and it is independent of any single
|
||||
field's agreement count.
|
||||
|
||||
[`schema_order.py`](../../../tools/re-capture/schema_order.py) merges the
|
||||
`Maneuver` field-name order from **all 113 unit tables** by topological sort
|
||||
over their pairwise "k[i] precedes k[i+1]" constraints. The merge is acyclic and
|
||||
**every one of the 113 tables is a subsequence of the merged 102-field order** —
|
||||
so that order is the schema's.
|
||||
|
||||
Against it, the solved offsets fall into two exact runs:
|
||||
|
||||
| declaration indices | offset rule | anchors that fit |
|
||||
|---|---|---|
|
||||
| 0 … 20 (`MinimumVelocity` … `AA_Roll_Min`) | `0x09c + 4·i` | 21 / 21 |
|
||||
| 21 … 32 (`SideThrustVelocity_Max` … `AccPitchFactor`) | `0x0a4 + 4·i` | 8 / 8 |
|
||||
|
||||
One 4-byte slot per field, with a **two-slot gap after `AA_Roll_Min`**
|
||||
(`0x0f0`–`0x0f4`, purpose unknown). 29 independently-derived anchors, zero
|
||||
conflicts, across a 0x9c–0x125 span.
|
||||
|
||||
### Fields NO disc record ever values — 🟡 PROBABLE
|
||||
|
||||
Five `Maneuver` fields are declared by the schema but left at their default by
|
||||
*every* one of the 110 unit tables, so no amount of disc analysis can ever
|
||||
reach them. The declaration-order rule pins them between confirmed anchors
|
||||
(`YawDragFactor +0x104` … `ArterBurner_Vc +0x114`, and
|
||||
`ReverseThrust_Vc +0x118` … `ReverseThrust_Acc +0x120`):
|
||||
|
||||
| offset | field | craft (`f00*`, `e0*`) | capital ships (`*1**`, `e2*`) | inert (`SchlosBase`, `Box`) |
|
||||
|---|---|---:|---:|---:|
|
||||
| `+0x108` | `PitchDragFactor` | 3 | 2 | 0 |
|
||||
| `+0x10c` | `RollDragFactor` | 3 | 2 | 0 |
|
||||
| `+0x110` | `DragFactorThreshold` | 0.5 | 0.5 | 0 |
|
||||
| `+0x11c` | `ArterBurner_Acc` | 2 | 2 | 0 |
|
||||
| `+0x128` | `DecPitchFactor` | 30 | 30 | 0 |
|
||||
|
||||
Corroboration beyond the interpolation, checked over all 18 units:
|
||||
|
||||
* `PitchDragFactor == RollDragFactor == YawDragFactor` holds **18/18** — and
|
||||
`YawDragFactor` is *disc-supplied* (3.0 for craft, 2.0 for warships), so two
|
||||
interpolated offsets reproduce a known number, per unit, every time.
|
||||
* `DecPitchFactor == AccPitchFactor` holds **17/18**. The exception is
|
||||
`UN_e015_ADAN_Puppy` (`AccPitchFactor` 1, `DecPitchFactor` 0.5) — which
|
||||
defaults *both* on disc, so it is two independent fields that happen to be
|
||||
set equal elsewhere, not a broken binding.
|
||||
|
||||
`UN_e015_ADAN_Puppy` also breaks the craft/warship bucketing above for
|
||||
`DecPitchFactor` (0.5, not 30); the per-unit values are in the CSV.
|
||||
|
||||
The tail of the `Maneuver` block (the AI-behaviour fields — `SideRoll_*`,
|
||||
`BarrelRoll_*`, `TurnAttack_*`, `HoldPosition_*`, `Slalom_*`, `Through_*`,
|
||||
`SolidCutoff_*`, and the `AxisMode` / `AB_*` sub-block) is **NOT resolved**.
|
||||
Those fields are declared by only a handful of tables and almost always with a
|
||||
single distinct value, so the solver's bindings there are coincidences: it
|
||||
placed `Slalom_CutoffRatio` at `+0x00c` and `TurnAttack_DoubleTimeMin` at
|
||||
`+0x110`, both of which the declaration-order rule contradicts. They are marked
|
||||
`tentative` in the CSV. **NEEDS-HUMAN / needs more coverage** — more stages
|
||||
would give those fields distinct values and settle it.
|
||||
|
||||
## The player craft, `UN_f001_TCAF_DeltaSaber_T_Player`
|
||||
|
||||
30 of its fields are defaulted on disc. Notable recovered values:
|
||||
|
||||
| field | value | note |
|
||||
|---|---|---|
|
||||
| `Size_Radius` | 10 | ✅ |
|
||||
| `FCSRange` | 500000 | ✅ — same as `RadarRange` |
|
||||
| `ChargeSpeed` (shield) | 25 | ✅ |
|
||||
| `ResistanceToOptics` | 1 | ✅ |
|
||||
| `HQRatio` / `ShieldRatio` / `ThrusterRatio` | 1 / 1 / 1 | 🟡 |
|
||||
| `ResistanceToShell` / `ResistanceToExplosion` | 1 / 1 | 🟡 |
|
||||
| `DryMass` | 100 | 🟡 (`GrossMass` 250 is on disc) |
|
||||
| `LowerHPThresholdRatio` | 0.3 | 🟡 |
|
||||
| `ChargeDelay_Break` | 10 | 🟡 |
|
||||
| `MassScore` | 0 | 🟡 |
|
||||
|
||||
Every AI-behaviour field the solver bound reads **0** for the player craft,
|
||||
which is the expected shape (the player is not AI-driven) — but see the caveat
|
||||
above: those offsets are not settled, so treat the zeros as consistent, not
|
||||
proven.
|
||||
|
||||
The `…Ratio` family that the Route-B target list parked is **1.0 for almost
|
||||
every unit**, with real exceptions that only the runtime shows:
|
||||
`UN_e105_ADAN_Cruiser` `HQRatio` = 0.2, `UN_bf001_TCAF_SchlosBase` and
|
||||
`UN_e106_ADAN_Destroyer` `ThrusterRatio` = 0.2,
|
||||
`UN_n001_TTRL_Box` `ShieldRatio` = 0.3.
|
||||
|
||||
## Coverage and how to extend it
|
||||
|
||||
21 of 110 units. Unlike weapons — where one snapshot held all 126 — **unit
|
||||
definitions are instantiated per stage**, so coverage is bounded by the stages
|
||||
reachable from the save (slot 01 is at 5 %, Stage 02). `unit_runtime.py` unions
|
||||
any number of snapshots and re-solves, and more units directly promote the 🟡
|
||||
bindings to ✅ by adding distinct values — the six tutorials took the confirmed
|
||||
set from 22 fields to 27.
|
||||
|
||||
The tutorials are nearly exhausted as a source: all six together contribute only
|
||||
3 units the missions do not already have (`UN_f001_TCAF_DeltaSaber_T_Ttrl`,
|
||||
`UN_e015_ADAN_Puppy_2`, `UN_f001_TCAF_DeltaSaber_T_Player_Ttrl2`) — they reuse
|
||||
one training box, one target drone and the player craft. **Real coverage now
|
||||
needs real missions**, i.e. story progress on the save.
|
||||
|
||||
`tools/re-capture/grab_tutorial.sh` captures one tutorial per invocation
|
||||
(cold boot → menu → Nth entry → snapshot, ~2.5 min). It cold-boots for each
|
||||
because backing out of a loaded mission via PAUSE → BACK TO MENU wedges the
|
||||
emulator. Two timing facts it encodes, both learned the hard way: the main menu
|
||||
is **not input-ready for ~10 s** after the title tap, and d-pad presses before
|
||||
that are silently dropped — which sends the A to `NEW GAME` instead of
|
||||
`TUTORIAL`. And `NEW GAME` is not a cheap way to reach Stage 01: it gates on a
|
||||
DIFFICULTY menu and then plays the prologue movie.
|
||||
|
||||
A NEW GAME excursion as far as the READY ROOM leaves
|
||||
`535107D4/00000001/game01/savedata` byte-identical — only the profile `.gpd`
|
||||
achievement files change — so it does not endanger the 5 % save. Verified by
|
||||
diff against a backup, not assumed.
|
||||
|
||||
**A stage's whole unit set is parsed at load, not as waves spawn** — checked by
|
||||
counting the definition objects at three points in Stage 02: immediately after
|
||||
take-off, ~12 minutes in mid-combat, and after GAME OVER. 14 objects, the same
|
||||
14 IDs, every time. So capturing a stage costs one load and one snapshot; there
|
||||
is no need to play it, and no need to survive it.
|
||||
|
||||
Stages captured so far: `Ttrl` (BASIC CONTROLS), Stage 02.
|
||||
294
docs/re/structures/weapon-struct-runtime.md
Normal file
294
docs/re/structures/weapon-struct-runtime.md
Normal file
@@ -0,0 +1,294 @@
|
||||
# Runtime `Weapon` / `Shell` structs — read from live guest memory
|
||||
|
||||
**Confidence: ✅ CONFIRMED** for the fields marked ✅ below (each binding is
|
||||
reproduced by 10–125 independent disc records with **zero** contradictions);
|
||||
🟡 for the thin ones. Captured 2026-07-29 from Xenia Canary running the retail
|
||||
disc, save slot 01 (Stage 02, 5 % progress), READY ROOM and ARSENAL.
|
||||
|
||||
## Why this exists
|
||||
|
||||
`weapon\Weapon_*.tbl` is an [IDXD](../INDEX.md) record whose string pool **omits
|
||||
every field left at its default**. That parked a long list of stats as
|
||||
unreadable — the `…Ratio` / `…Count` family, `Power`, `MaximumRange`. The
|
||||
[Arsenal DATA SHEET route](../weapon-datasheet-runtime.md) recovered a few of
|
||||
them but only as letter buckets (Range/Damage as `A`…`E`), and only for the 9
|
||||
weapons unlocked at 5 % progress.
|
||||
|
||||
This reads the values **directly out of the running game's memory** instead.
|
||||
All 126 weapons, all fields, exact numbers, in one pass — and it needs no story
|
||||
progress, because the definitions are parsed at load time whether or not the
|
||||
player has unlocked the weapon.
|
||||
|
||||
## The lever: Canary maps guest RAM into `/dev/shm`
|
||||
|
||||
Xenia Canary backs the entire guest address space with one shared-memory file,
|
||||
`/dev/shm/xenia_memory_<id>`. It is a plain file: **the guest's RAM is readable
|
||||
from the host with `open`/`seek`/`read`, live, no debugger and no emulator
|
||||
patch.** Guest VAs map into it through Xenia's fixed table (`memory.cc`), which
|
||||
[`tools/re-capture/gmem.py`](../../../tools/re-capture/gmem.py) implements:
|
||||
|
||||
```bash
|
||||
python3 tools/re-capture/gmem.py find "Weapon_DSaber_P_wep_01" # search all of RAM
|
||||
python3 tools/re-capture/gmem.py words 0xbccce500 48 # dump as BE u32/f32
|
||||
```
|
||||
|
||||
The file is sparse (~212 MB resident of 4.5 GB), and the scan uses
|
||||
`SEEK_DATA`/`SEEK_HOLE`, so a full-RAM search costs ~0.2 s.
|
||||
|
||||
## Finding the objects
|
||||
|
||||
1. The title's **schema field-name pool** is in the XEX at `0x82086a30`, in
|
||||
declaration order: `EnumWeapon`, type `Weapon` (`ID`, `Name`, `TargetType`,
|
||||
… `CartridgeModelName`), then type `Shell` (`ID`, `Name`, `MovementType`, …
|
||||
`Explosion_MaxDamageRadius`). This is exactly the on-disc field order. No
|
||||
pointer to these strings exists anywhere in RAM — PPC builds the addresses
|
||||
with `lis`/`ori` immediate pairs — so the schema descriptor cannot be found
|
||||
by pointer-chasing; the layout has to be solved instead (below).
|
||||
2. Each parsed record becomes **two C++ objects**, a `Weapon` and its `Shell`,
|
||||
each in its own contiguous array, each identified by its **vtable pointer**:
|
||||
|
||||
| class | vtable VA | stride | count |
|
||||
|-------|-----------|--------|-------|
|
||||
| `Weapon` | `0x820af548` | `0xc0` | 126 |
|
||||
| `Shell` | `0x820af58c` | `0x200` | 126 |
|
||||
|
||||
The vtable VAs are static (XEX `.data`); the array base addresses are heap
|
||||
and are **not** assumed — the tool locates every object by scanning RAM for
|
||||
the vtable word.
|
||||
3. Object `+0x04` points at a 0x40-byte **name record**; the ID string sits at
|
||||
`+0x10` inside it. That is what keys each object back to its disc record.
|
||||
|
||||
`Weapon` ↔ `Shell` pairing is by ID (`Weapon_X` ↔ `Shell_X`) — the two arrays
|
||||
are in **different orders**, so index-pairing would be wrong.
|
||||
|
||||
## Solving the layout (the part that makes this evidence, not guesswork)
|
||||
|
||||
[`tools/re-capture/weapon_runtime.py`](../../../tools/re-capture/weapon_runtime.py)
|
||||
brute-forces every `(field, byte offset, encoding)` triple and scores it against
|
||||
the disc: how many records does this offset *reproduce*, and how many does it
|
||||
*contradict*? A binding is accepted only with **zero contradictions**, and is
|
||||
marked ✅ only when ≥10 records agree on ≥3 distinct values.
|
||||
|
||||
That threshold matters: a field whose disc samples are all the same number
|
||||
matches any offset holding that constant, so agreement count alone is not
|
||||
evidence — the number of **distinct** values pinned down is. Two fields landing
|
||||
on one offset is impossible in a real struct, so collisions are resolved to the
|
||||
better-evidenced field and the loser is reported as unsolved. Bindings that
|
||||
contradict more than 20 % of their samples are discarded outright rather than
|
||||
reported on their agreeing subset.
|
||||
|
||||
Encodings found: `f32`, `u32`, `deg` (**angles are stored in radians**;
|
||||
`SprayAngle`, `AngularVelocity` and `SplitCone` are degrees on disc), and `cnt`
|
||||
— see the next section.
|
||||
|
||||
### `IsCharging` switches the counter encoding ✅
|
||||
|
||||
`LoadingCount` and `TriggerShotCount` at `+0x28` / `+0x44` are **int32 on 118
|
||||
records and float32 on 8**. The 8 are exactly the records with
|
||||
`IsCharging = Yes` — an exact partition, found by testing every disc field/value
|
||||
pair against the float-encoded set. A charging weapon drains its magazine
|
||||
continuously, so its counter needs a fraction.
|
||||
|
||||
Any reimplementation must branch on `IsCharging` when reading these two fields;
|
||||
reading them as int unconditionally yields `1125515264` instead of `150`.
|
||||
|
||||
## Struct maps
|
||||
|
||||
See [`docs/re/captures/weapon-runtime-fields.csv`](../captures/weapon-runtime-fields.csv)
|
||||
for the full machine-readable table — 7 182 rows, every confirmed field × every
|
||||
one of the 126 records, tagged `disc` or `defaulted-on-disc`. **4 393 of those
|
||||
values were not readable from the disc at all.**
|
||||
|
||||
### `Weapon` (`0xc0` bytes)
|
||||
|
||||
| offset | enc | field | agree | distinct | conf |
|
||||
|--------|-----|-------|------:|---------:|------|
|
||||
| `+0x010` | u32 | `ReticleType` | 62 | 13 | ✅ |
|
||||
| `+0x01c` | u32 | `SpecialWeaponType` | 73 | 6 | ✅ |
|
||||
| `+0x028` | cnt | `LoadingCount` | 121 | 33 | ✅ |
|
||||
| `+0x02c` | f32 | `Interval` | 125 | 28 | ✅ |
|
||||
| `+0x030` | f32 | `ReadyInterval` | 120 | 10 | ✅ |
|
||||
| `+0x034` | f32 | `Heating` | 114 | 30 | ✅ |
|
||||
| `+0x038` | f32 | `Cooling` | 106 | 21 | ✅ |
|
||||
| `+0x03c` | f32 | `Mass` | 122 | 42 | ✅ |
|
||||
| `+0x040` | f32 | `HitRatio` | 102 | 5 | ✅ |
|
||||
| `+0x044` | cnt | `TriggerShotCount` | 114 | 17 | ✅ |
|
||||
| `+0x048` | f32 | `TriggerShotInterval` | 99 | 11 | ✅ |
|
||||
| `+0x050` | deg | `SprayAngle` | 96 | 17 | ✅ |
|
||||
| `+0x06c` | f32 | `LockIntervalSingle` | 61 | 9 | ✅ |
|
||||
| `+0x070` | f32 | `LockIntervalMulti` | 28 | 9 | ✅ |
|
||||
| `+0x09c` | f32 | `MaximumCharging` | 3 | 3 | 🟡 |
|
||||
|
||||
Also identified structurally, not by the solver: `+0x00` vtable, `+0x04` name
|
||||
record, `+0x0c` `TargetType` as a bitmask (`Vessel|Craft|Structure` = 7),
|
||||
`+0x14`/`+0x18` name hashes, `+0x7c`/`+0x80` muzzle-flash FX name record + hash.
|
||||
|
||||
Unsolved (no consistent offset): `Cracker_ShotInterval`, `Cracker_SubShotCount`,
|
||||
`MinimumCharging`, `MultiTargetCount`.
|
||||
|
||||
### `Shell` (`0x200` bytes)
|
||||
|
||||
| offset | enc | field | agree | distinct | conf |
|
||||
|--------|-----|-------|------:|---------:|------|
|
||||
| `+0x00c` | cnt | `DeleteFadeSpeed` | 39 | 2 | 🟡 |
|
||||
| `+0x010` | cnt | `GuidanceType` | 9 | 4 | 🟡 |
|
||||
| `+0x014` | cnt | `SpiralType` | 3 | 1 | 🟡 |
|
||||
| `+0x020` | f32 | `Length` | 107 | 3 | ✅ |
|
||||
| `+0x024` | f32 | `Volume` | 19 | 3 | ✅ |
|
||||
| `+0x028` | f32 | `ShellMass` | 110 | 34 | ✅ |
|
||||
| `+0x03c` | f32 | `HP` | 28 | 2 | 🟡 |
|
||||
| `+0x040` | f32 | `LifeTime` | 94 | 43 | ✅ |
|
||||
| `+0x044` | f32 | `FadeInTime` | 13 | 2 | 🟡 |
|
||||
| `+0x048` | f32 | `FadeOutTime` | 7 | 1 | 🟡 |
|
||||
| `+0x04c` | f32 | `Velocity` | 106 | 15 | ✅ |
|
||||
| `+0x050` | f32 | `MinimumVelocity` | 11 | 2 | 🟡 |
|
||||
| `+0x054` | f32 | `MaximumVelocity` | 29 | 7 | ✅ |
|
||||
| `+0x058` | deg | `AngularVelocity` | 49 | 19 | ✅ |
|
||||
| `+0x05c` | f32 | `Acceleration` | 9 | 4 | 🟡 |
|
||||
| `+0x064` | f32 | `BeginGuidanceTimeAdjust` | 24 | 2 | 🟡 |
|
||||
| `+0x068` | f32 | `EndGuidanceTime` | 19 | 4 | ✅ |
|
||||
| `+0x070` | f32 | `Spiral_BeginTime` | 2 | 2 | 🟡 |
|
||||
| `+0x074` | f32 | `Spiral_BeginTimeAdjust` | 25 | 2 | 🟡 |
|
||||
| `+0x08c` | f32 | `MinimumRange` | 43 | 7 | ✅ |
|
||||
| `+0x090` | f32 | `MaximumRange` | 117 | 21 | ✅ |
|
||||
| `+0x0a0` | f32 | `Color_R` | 18 | 4 | ✅ |
|
||||
| `+0x0a4` | f32 | `Color_G` | 121 | 3 | ✅ |
|
||||
| `+0x0a8` | f32 | `Color_B` | 100 | 2 | 🟡 |
|
||||
| `+0x0b4` | f32 | `Radius` | 89 | 10 | ✅ |
|
||||
| `+0x0b8` | f32 | `Power` | 101 | 37 | ✅ |
|
||||
| `+0x0bc` | f32 | `FailedDamageRatio` | 2 | 2 | 🟡 |
|
||||
| `+0x0c0` | f32 | `PlayerLaserPower` | 11 | 6 | ✅ |
|
||||
| `+0x0fc` | f32 | `ChaffResistRatio` | 24 | 1 | 🟡 |
|
||||
| `+0x104` | f32 | `ChargingSizeRatio` | 3 | 1 | 🟡 |
|
||||
| `+0x10c` | f32 | `SphereRadiusBegin` | 9 | 6 | 🟡 |
|
||||
| `+0x110` | f32 | `SphereRadiusTurn` | 9 | 8 | 🟡 |
|
||||
| `+0x114` | f32 | `SphereRadiusEnd` | 10 | 8 | ✅ |
|
||||
| `+0x118` | f32 | `ExplosionTurnTime` | 3 | 2 | 🟡 |
|
||||
| `+0x11c` | f32 | `ExplosionLifeTime` | 7 | 6 | 🟡 |
|
||||
| `+0x120` | f32 | `ExplosionDamage_Maximum` | 4 | 4 | 🟡 |
|
||||
| `+0x124` | f32 | `ExplosionDamage_OuterEdge` | 8 | 6 | 🟡 |
|
||||
| `+0x128` | f32 | `Explosion_MaxDamageRadius` | 8 | 3 | 🟡 |
|
||||
| `+0x130` | f32 | `SplitTime_Maximum` | 2 | 2 | 🟡 |
|
||||
| `+0x134` | deg | `SplitCone` | 2 | 2 | 🟡 |
|
||||
| `+0x148` | cnt | `NodeCount` | 39 | 4 | ✅ |
|
||||
| `+0x164` | f32 | `Deceleration` | 9 | 2 | 🟡 |
|
||||
|
||||
Unsolved: `BeginGuidanceTime`, `EndGuidanceTimeAdjust`, `Spiral_EndTime`,
|
||||
`SplitTime_Minimum`, `StartingVelocity`, `Straight1Type`, `st1_df_pitch`,
|
||||
`pitch0`, and the `sp_qu_*` / `sp_sp_*` / `sp_zg_*` spiral-motion family. Those
|
||||
are declared by too few records (or by none that the solver could separate) —
|
||||
they need a state where the shells are actually in flight.
|
||||
|
||||
## The answer to the parked Route-B question
|
||||
|
||||
Player-weapon fields that are **defaulted on disc**, now read exactly (`·` = the
|
||||
disc carries the value already):
|
||||
|
||||
| wep | LoadingCount | TriggerShotCount | MaximumRange | Power | Heating | Cooling | ReadyInterval | HitRatio | SprayAngle |
|
||||
|---|---|---|---|---|---|---|---|---|---|
|
||||
| 01 | · | **1** | · | · | · | · | · | · | · |
|
||||
| 02 | · | · | · | **100** | · | · | · | **1** | · |
|
||||
| 03 | · | · | · | · | · | · | · | · | **1** |
|
||||
| 05 | · | **4** | · | · | · | · | · | · | · |
|
||||
| 08 | · | · | · | · | · | · | · | **1** | · |
|
||||
| 09 | · | · | · | · | **0.02** | · | · | · | **1** |
|
||||
| 11 | **6** | · | · | · | · | · | · | · | · |
|
||||
| 12 | · | · | · | · | **0** | · | · | **1** | · |
|
||||
| 13 | · | · | · | **300** | · | · | · | · | · |
|
||||
| 14 | · | · | · | · | · | · | · | · | **1** |
|
||||
| 19 | · | · | · | · | · | **0.2** | · | · | **1** |
|
||||
| 24 | · | **1** | · | · | · | · | · | · | **1** |
|
||||
| 25 | · | · | **4000** | · | · | · | · | · | · |
|
||||
| 26 | · | · | · | **500** | · | · | · | · | · |
|
||||
| 27 | · | **1** | · | · | · | · | · | · | · |
|
||||
| 28 | **5** | · | · | · | · | · | · | · | · |
|
||||
| 29 | · | · | · | **100** | · | · | · | · | · |
|
||||
| 30 | · | **4** | · | · | · | · | · | · | · |
|
||||
| 36 | **5** | · | · | · | · | · | · | · | · |
|
||||
| 37 | · | **1** | · | · | · | · | · | · | **1** |
|
||||
| 38 | · | · | · | · | · | · | · | **1** | · |
|
||||
| 39 | · | **1** | · | · | · | · | · | · | **1** |
|
||||
| 40 | · | · | · | · | · | · | **0.1** | · | · |
|
||||
| 48 | · | · | · | · | · | · | · | · | **1** |
|
||||
| 50 | · | · | · | · | · | · | · | · | **1** |
|
||||
| 52 | · | · | · | · | · | · | · | **1** | · |
|
||||
| 53 | · | **1** | · | · | · | · | · | · | · |
|
||||
| 54 | · | · | · | · | · | · | · | · | **1** |
|
||||
| 55 | · | · | · | · | **0** | · | · | **1** | · |
|
||||
| 56 | · | · | · | · | · | · | · | **1** | · |
|
||||
| 57 | · | · | · | · | **0** | · | · | **1** | · |
|
||||
| 58 | · | · | **10000** | · | · | · | · | · | **1** |
|
||||
| 59 | · | · | · | · | · | · | · | **1** | **1** |
|
||||
| 60 | · | **4** | · | **1000** | · | · | · | · | · |
|
||||
| 62 | · | **1** | · | · | **0** | · | · | · | · |
|
||||
| 66 | · | · | · | · | · | · | · | **1** | · |
|
||||
| 67 | · | · | · | · | · | · | · | **1** | · |
|
||||
| 68 | · | · | · | · | · | · | · | **1** | · |
|
||||
| 69 | · | · | · | · | · | · | · | **1** | · |
|
||||
| 70 | **0** | · | · | · | · | · | · | **1** | · |
|
||||
| 71 | · | · | · | **10** | · | · | · | **1** | · |
|
||||
| 81 | · | · | · | **300** | · | · | · | · | · |
|
||||
| 82 | · | · | **10000** | · | · | · | · | · | **1** |
|
||||
| 84 | · | · | · | · | · | · | · | · | **0.1** |
|
||||
| 85 | · | **1** | · | · | **0** | · | · | · | · |
|
||||
|
||||
`HitRatio` is **1.0 for every one of the 24 records that default it** — the
|
||||
whole `…Ratio` family that blocked Route B is simply "no penalty".
|
||||
`wep_82`'s defaulted field is `PlayerLaserPower` = **12000** (its `Power`,
|
||||
12000, is on disc).
|
||||
|
||||
## Cross-checks
|
||||
|
||||
- **Against the independent screenshot route.** The Arsenal DATA SHEET
|
||||
([weapon-datasheet-runtime.md](../weapon-datasheet-runtime.md)) established
|
||||
`Max. Lock Ons == TriggerShotCount`, and read **4** for `wep_05` and `wep_60`
|
||||
off the panel. The memory read gives **4** for both — two unrelated methods,
|
||||
same numbers.
|
||||
- **Against the in-flight HUD.** `NOSE BM 06000` / `MAIN MPM 00300` matches
|
||||
`wep_01` `LoadingCount = 6000` and `wep_02` `= 300` at `+0x28`.
|
||||
- **Stability.** All 252 objects are **byte-identical** between the READY ROOM
|
||||
and the ARSENAL, so these are load-time definition data, not transient state.
|
||||
- **Self-consistency.** 121 records agree on `LoadingCount`'s offset across 33
|
||||
distinct values with zero contradictions; `Power`, 101 records / 37 distinct
|
||||
values. A wrong offset cannot do that.
|
||||
|
||||
## Incidental findings
|
||||
|
||||
- **A duplicate, conflicting disc record.** Two `.tbl` entries declare
|
||||
`Shell_TCAF_Ship_AAGun`; one sets `Power = 60.0`, the other omits it. The
|
||||
runtime object holds **5**, i.e. the *omitting* declaration won. Any
|
||||
reimplementation loading both will silently pick one — this says which.
|
||||
- **Not every disc record is instantiated.** 131 disc records produced 126
|
||||
objects; the 5 with no runtime object are context variants that this save
|
||||
never loads — `Weapon_TCAF_DeltaSaber_NoseGun_Ttrl` / `_Laser_Ttrl`
|
||||
(tutorial), `_NoseGun_None`, `Weapon_TCAF_Ship_AAGun_EX5`,
|
||||
`Weapon_ADAN_Attacker_S_GunTurret`. Running the tutorial should instantiate
|
||||
the `_Ttrl` pair. No runtime object lacked a disc record.
|
||||
- Defaulted `Power` values vary per weapon (1, 10, 100, 300, 500, 1000), so they
|
||||
are **not** one constructor constant. Where they come from — the IDXD's
|
||||
undecoded binary node/index region, or per-type code defaults — is **not
|
||||
determined here** (`NEEDS-HUMAN` / follow-up). Either way the runtime values
|
||||
above are ground truth, and they are now a decoding oracle for that region.
|
||||
|
||||
## Reproduce
|
||||
|
||||
```bash
|
||||
export HOME=/sylph-home/re SDL_AUDIODRIVER=dummy
|
||||
run-canary --audio --apu=sdl --log_mask=13 --logged_profile_slot_0_xuid=E0300000EFBEA3D4 &
|
||||
tools/re-capture/skip_intro.sh # -> title
|
||||
# LOAD GAME -> slot 01 -> YES -> READY ROOM (weapon tables load with the save)
|
||||
|
||||
cargo run --release -p sylpheed-formats --example idxd_tokens -- \
|
||||
<disc>/dat/GP_MAIN_GAME_E.pak > /tmp/wep_tokens.txt
|
||||
python3 tools/re-capture/weapon_runtime.py /tmp/wep_tokens.txt # report
|
||||
python3 tools/re-capture/weapon_runtime.py /tmp/wep_tokens.txt --csv # full table
|
||||
```
|
||||
|
||||
## What this unlocks
|
||||
|
||||
`gmem.py` + "scan for the vtable, solve the layout against disc ground truth" is
|
||||
**not weapon-specific**. The same three steps apply to any IDXD-backed
|
||||
definition whose fields the disc defaults — craft/`UNIT` stats, which the Hangar
|
||||
UI exposes only as a `Gross Weight` class and which
|
||||
[the menu route could not reach at all](../weapon-datasheet-runtime.md), are the
|
||||
obvious next target.
|
||||
@@ -18,3 +18,18 @@ Canary + lavapipe, headless. They assume the container helpers `screenshot`,
|
||||
auto-repeat (two rows per press).
|
||||
|
||||
Findings produced with these: [`docs/re/weapon-datasheet-runtime.md`](../../docs/re/weapon-datasheet-runtime.md).
|
||||
|
||||
## Guest-memory tools (no screenshots)
|
||||
|
||||
| Script | What it does |
|
||||
|---|---|
|
||||
| `gmem.py` | Read the live guest address space out of `/dev/shm/xenia_memory_*` (Xenia's backing file), addressed by guest VA. `find` / `read` / `words`. |
|
||||
| `weapon_runtime.py` | Solve the `Weapon`/`Shell` struct layouts against the disc records and read the fields the disc defaults. |
|
||||
| `unit_discover.py` | Find *which* runtime class carries a set of ID strings, assuming no vtable: tallies the word at `pointer_site - k` across distinct IDs. |
|
||||
| `unit_runtime.py` | Same solver for the `unit\UN_*.tbl` definition objects (vtable `0x820af844`). Unions several snapshots — unit definitions are per-stage. |
|
||||
| `schema_order.py` | Merge a sub-record's field-declaration order across all tables (topological sort); the layout check that pins fields no table ever values. |
|
||||
| `order_check.py` | Test `offset = base + 4*index` for one table's sub-record against solver output. |
|
||||
| `grab_tutorial.sh` | Cold-boot Canary, walk to the Nth TUTORIAL entry, wait for the stage load, snapshot guest RAM. One emulator per capture — backing out of a loaded mission wedges it. |
|
||||
|
||||
Snapshot first — `cp --sparse=always /dev/shm/xenia_memory_* snap.bin` (~2 s) — and
|
||||
point `$GMEM_FILE` at the copy; the running emulator pegs every core under lavapipe.
|
||||
|
||||
212
tools/re-capture/autopilot2.py
Normal file
212
tools/re-capture/autopilot2.py
Normal file
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Memory-driven autopilot: fly and fight from the game's own world state.
|
||||
|
||||
The earlier screen-scraping autopilot oscillated because it only ever saw a
|
||||
2-D arrow on the HUD and had no rate feedback. This one reads the actual
|
||||
transform of every entity out of guest RAM, so it can do proper PD control:
|
||||
the derivative term uses the craft's real body angular velocity, recovered from
|
||||
two consecutive rotation matrices, not a differenced pixel position.
|
||||
|
||||
world state <- /dev/shm/xenia_memory_* (see gworld.py)
|
||||
control -> the vgamepad FIFO, written directly at loop rate
|
||||
|
||||
Offsets come from flight_analyze.py and are passed in / stored in offsets.json;
|
||||
nothing here hard-codes a value that was not derived from a capture.
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gworld # noqa: E402
|
||||
from flight_probe import Pad # noqa: E402
|
||||
|
||||
|
||||
# --------------------------------------------------------------- 3-D helpers
|
||||
|
||||
def dot(a, b):
|
||||
return a[0] * b[0] + a[1] * b[1] + a[2] * b[2]
|
||||
|
||||
|
||||
def sub(a, b):
|
||||
return (a[0] - b[0], a[1] - b[1], a[2] - b[2])
|
||||
|
||||
|
||||
def norm(a):
|
||||
return math.sqrt(dot(a, a))
|
||||
|
||||
|
||||
def clamp(v, lo=-1.0, hi=1.0):
|
||||
return max(lo, min(hi, v))
|
||||
|
||||
|
||||
def body_rates(R_prev, R, dt):
|
||||
"""Angular velocity in body axes from two rotation matrices.
|
||||
|
||||
R rows are the craft's axes in world space, so R_prev @ R^T is the
|
||||
incremental rotation; its skew part is the rotation vector.
|
||||
"""
|
||||
if dt <= 0:
|
||||
return (0.0, 0.0, 0.0)
|
||||
# M = R_prev * R^T (3x3, row-major tuples)
|
||||
M = [[sum(R_prev[i * 3 + k] * R[j * 3 + k] for k in range(3)) for j in range(3)]
|
||||
for i in range(3)]
|
||||
wx = (M[2][1] - M[1][2]) / 2.0
|
||||
wy = (M[0][2] - M[2][0]) / 2.0
|
||||
wz = (M[1][0] - M[0][1]) / 2.0
|
||||
return (wx / dt, wy / dt, wz / dt)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- reader
|
||||
|
||||
class Flight:
|
||||
def __init__(self, cfg):
|
||||
self.cfg = cfg
|
||||
self.w = gworld.World()
|
||||
self.pos_off = cfg["pos"]
|
||||
self.rot_off = cfg["rot"]
|
||||
self.hp_off = cfg.get("hp")
|
||||
self.player_name = cfg.get("player", "Player")
|
||||
self.entities = []
|
||||
self.last_scan = 0.0
|
||||
|
||||
def rescan(self):
|
||||
self.entities = self.w.refresh()
|
||||
self.last_scan = time.time()
|
||||
|
||||
def state(self, off):
|
||||
buf = self.w.read_off(off, gworld.WINDOW)
|
||||
if len(buf) < gworld.WINDOW:
|
||||
return None
|
||||
pos = struct.unpack_from(">3f", buf, self.pos_off)
|
||||
rot = struct.unpack_from(">9f", buf, self.rot_off)
|
||||
if not all(map(math.isfinite, pos + rot)):
|
||||
return None
|
||||
hp = struct.unpack_from(">f", buf, self.hp_off)[0] if self.hp_off else None
|
||||
return {"pos": pos, "rot": rot, "hp": hp}
|
||||
|
||||
def player(self):
|
||||
for va, off, nm in self.entities:
|
||||
if self.player_name in nm:
|
||||
s = self.state(off)
|
||||
if s:
|
||||
s["name"] = nm
|
||||
return s
|
||||
return None
|
||||
|
||||
def hostiles(self):
|
||||
"""ADAN units are the enemy; the disc IDs encode the faction.
|
||||
|
||||
`UN_e*` / `UN_be*` = ADAN, `UN_f*` / `UN_bf*` = TCAF (ours).
|
||||
"""
|
||||
out = []
|
||||
for va, off, nm in self.entities:
|
||||
base = nm[3:]
|
||||
if not (base.startswith("e") or base.startswith("be")):
|
||||
continue
|
||||
s = self.state(off)
|
||||
if s:
|
||||
s["name"] = nm
|
||||
s["off"] = off
|
||||
out.append(s)
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- controller
|
||||
|
||||
class Autopilot:
|
||||
KP_YAW, KD_YAW = 1.6, 0.35
|
||||
KP_PITCH, KD_PITCH = 1.6, 0.35
|
||||
FIRE_CONE = math.radians(6)
|
||||
FIRE_RANGE = 4000.0
|
||||
|
||||
def __init__(self, flight, pad, log=sys.stdout):
|
||||
self.f = flight
|
||||
self.pad = pad
|
||||
self.log = log
|
||||
self.prev_rot = None
|
||||
self.prev_t = None
|
||||
self.target = None
|
||||
self.firing = False
|
||||
|
||||
def pick_target(self, me):
|
||||
hs = self.f.hostiles()
|
||||
if not hs:
|
||||
return None
|
||||
# nearest, mildly preferring what is already ahead of us
|
||||
def score(h):
|
||||
v = sub(h["pos"], me["pos"])
|
||||
d = norm(v) or 1.0
|
||||
fwd = me["rot"][6:9]
|
||||
ahead = dot(v, fwd) / d
|
||||
return d * (1.0 if ahead > 0 else 2.0)
|
||||
return min(hs, key=score)
|
||||
|
||||
def step(self, dt):
|
||||
me = self.f.player()
|
||||
if not me:
|
||||
return "no-player"
|
||||
R = me["rot"]
|
||||
w = body_rates(self.prev_rot, R, dt) if self.prev_rot else (0, 0, 0)
|
||||
self.prev_rot = R
|
||||
|
||||
tgt = self.pick_target(me)
|
||||
if not tgt:
|
||||
self.pad.axis("LX", 0.0)
|
||||
self.pad.axis("LY", 0.0)
|
||||
self.pad.trig("RT", 0.6)
|
||||
return "no-target"
|
||||
|
||||
v = sub(tgt["pos"], me["pos"])
|
||||
dist = norm(v) or 1.0
|
||||
# into body axes: rows are right / up / forward
|
||||
lx = dot(R[0:3], v)
|
||||
ly = dot(R[3:6], v)
|
||||
lz = dot(R[6:9], v)
|
||||
yaw_err = math.atan2(lx, lz if lz > 1e-3 else 1e-3)
|
||||
pitch_err = math.atan2(ly, lz if lz > 1e-3 else 1e-3)
|
||||
if lz < 0: # behind us: turn the short way, hard
|
||||
yaw_err = math.copysign(math.pi / 2, lx if lx else 1.0)
|
||||
|
||||
stick_x = clamp(self.KP_YAW * yaw_err - self.KD_YAW * w[1])
|
||||
stick_y = clamp(-(self.KP_PITCH * pitch_err - self.KD_PITCH * w[0]))
|
||||
self.pad.axis("LX", stick_x)
|
||||
self.pad.axis("LY", stick_y)
|
||||
self.pad.trig("RT", 1.0 if dist > 1500 else 0.3)
|
||||
|
||||
aligned = abs(yaw_err) < self.FIRE_CONE and abs(pitch_err) < self.FIRE_CONE
|
||||
want_fire = aligned and dist < self.FIRE_RANGE
|
||||
if want_fire != self.firing:
|
||||
(self.pad.press if want_fire else self.pad.release)(self.f.cfg["fire_btn"])
|
||||
self.firing = want_fire
|
||||
return (f"tgt={tgt['name'][3:20]:<18} d={dist:8.0f} yaw={math.degrees(yaw_err):+6.1f} "
|
||||
f"pit={math.degrees(pitch_err):+6.1f} stick=({stick_x:+.2f},{stick_y:+.2f}) "
|
||||
f"fire={int(self.firing)} hp={me['hp']}")
|
||||
|
||||
def run(self, seconds, hz=15.0):
|
||||
self.f.rescan()
|
||||
t0 = time.time()
|
||||
last = t0
|
||||
while time.time() - t0 < seconds:
|
||||
t = time.time()
|
||||
if t - self.f.last_scan > 3.0:
|
||||
self.f.rescan()
|
||||
msg = self.step(t - last)
|
||||
last = t
|
||||
print(f"[{t-t0:6.1f}] {msg}", file=self.log, flush=True)
|
||||
time.sleep(max(0, 1.0 / hz - (time.time() - t)))
|
||||
self.pad.reset()
|
||||
|
||||
|
||||
def main():
|
||||
cfg = json.load(open(sys.argv[1]))
|
||||
secs = float(sys.argv[2]) if len(sys.argv) > 2 else 60.0
|
||||
ap = Autopilot(Flight(cfg), Pad())
|
||||
ap.run(secs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
207
tools/re-capture/autopilot3.py
Normal file
207
tools/re-capture/autopilot3.py
Normal file
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fly and fight from guest memory.
|
||||
|
||||
World state comes from entities2.py's typing rule: a live entity's position
|
||||
triple sits at a fixed offset before its definition pointer, so one scan of the
|
||||
entity heap yields every craft in the scene *with its unit type* — which is what
|
||||
separates enemies from wingmen, capital ships and the thousands of moving
|
||||
particles.
|
||||
|
||||
Control is PD on the aiming error, with the derivative taken from the craft's
|
||||
own body angular velocity (recovered from two consecutive orientation matrices)
|
||||
rather than from the differenced error — that is what the old screen-scraping
|
||||
autopilot lacked, and why it oscillated.
|
||||
|
||||
Usage: autopilot3.py <config.json> [seconds] [--dry]
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
import gworld # noqa: E402
|
||||
import entities2 # noqa: E402
|
||||
from flight_probe import Pad # noqa: E402
|
||||
|
||||
HOSTILE = ("e", "be") # UN_e* / UN_be* are ADAN; UN_f*/UN_bf* are ours
|
||||
|
||||
|
||||
def unit_faction(nm):
|
||||
base = nm[3:]
|
||||
return "ADAN" if base.startswith("be") or base.startswith("e") else "TCAF"
|
||||
|
||||
|
||||
class World:
|
||||
def __init__(self, cfg):
|
||||
self.cfg = cfg
|
||||
self.w = gworld.World()
|
||||
self.fd, self.size = self.w.fd, self.w.size
|
||||
self.defs = entities2.definitions(self.w)
|
||||
self.delta = cfg["def_delta"]
|
||||
self.rot_delta = cfg["rot_delta"]
|
||||
self.rot_stride = cfg.get("rot_stride", 12)
|
||||
self.fwd_row = cfg["fwd_row"]
|
||||
self.fwd_sign = cfg["fwd_sign"]
|
||||
self.ents = []
|
||||
|
||||
def rescan(self):
|
||||
movers = entities2.moving(self.fd, self.size, dt=0.35,
|
||||
va_range=(self.cfg["va_lo"], self.cfg["va_hi"]))
|
||||
ents = entities2.typed(self.fd, self.defs, movers, self.delta)
|
||||
uniq = {}
|
||||
for off, nm, pos, sp in ents:
|
||||
uniq.setdefault(off, (off, nm, pos, sp))
|
||||
self.ents = list(uniq.values())
|
||||
return self.ents
|
||||
|
||||
def pos(self, off):
|
||||
b = os.pread(self.fd, 12, off)
|
||||
return np.array(struct.unpack(">3f", b)) if len(b) == 12 else None
|
||||
|
||||
def rot(self, off):
|
||||
n = self.rot_stride * 2 + 12
|
||||
b = os.pread(self.fd, n, off + self.rot_delta)
|
||||
if len(b) < n:
|
||||
return None
|
||||
M = np.array([struct.unpack_from(">3f", b, self.rot_stride * r) for r in range(3)])
|
||||
if not np.all(np.isfinite(M)):
|
||||
return None
|
||||
if np.max(np.abs(M @ M.T - np.eye(3))) > 5e-3:
|
||||
return None
|
||||
return M
|
||||
|
||||
|
||||
def clamp(v, lo=-1.0, hi=1.0):
|
||||
return max(lo, min(hi, v))
|
||||
|
||||
|
||||
def body_rate(Mprev, M, dt):
|
||||
if Mprev is None or M is None or dt <= 0:
|
||||
return np.zeros(3)
|
||||
D = Mprev @ M.T
|
||||
w = np.array([D[2, 1] - D[1, 2], D[0, 2] - D[2, 0], D[1, 0] - D[0, 1]]) / 2.0
|
||||
return w / dt
|
||||
|
||||
|
||||
class Autopilot:
|
||||
KP, KD = 2.2, 0.45
|
||||
FIRE_CONE = math.radians(10)
|
||||
FIRE_RANGE = 6000.0
|
||||
|
||||
def __init__(self, world, pad, dry=False):
|
||||
self.W = world
|
||||
self.pad = pad
|
||||
self.dry = dry
|
||||
self.prevM = None
|
||||
self.firing = False
|
||||
|
||||
def me(self):
|
||||
for off, nm, pos, sp in self.W.ents:
|
||||
if "Player" in nm:
|
||||
return off, nm
|
||||
return None, None
|
||||
|
||||
def step(self, dt):
|
||||
off, nm = self.me()
|
||||
if off is None:
|
||||
return "no-player"
|
||||
p = self.W.pos(off)
|
||||
M = self.W.rot(off)
|
||||
if p is None or M is None:
|
||||
return "no-state"
|
||||
fwd = M[self.W.fwd_row] * self.W.fwd_sign
|
||||
rows = [M[i] for i in range(3)]
|
||||
right = rows[(self.W.fwd_row + 1) % 3]
|
||||
up = np.cross(fwd, right)
|
||||
|
||||
w = body_rate(self.prevM, M, dt)
|
||||
self.prevM = M
|
||||
|
||||
# nearest hostile, preferring what is already in front
|
||||
best, bestscore = None, 1e18
|
||||
for eoff, enm, epos, esp in self.W.ents:
|
||||
if unit_faction(enm) != "ADAN":
|
||||
continue
|
||||
q = self.W.pos(eoff)
|
||||
if q is None:
|
||||
continue
|
||||
v = q - p
|
||||
d = float(np.linalg.norm(v))
|
||||
if d < 1e-3:
|
||||
continue
|
||||
# Prefer targets we can actually bring the nose onto. Nearest-first
|
||||
# picks whatever is closest even at 90 deg off the nose, and closing
|
||||
# on an off-boresight target only raises the bearing rate -- which is
|
||||
# exactly the lag that kept the first run outside its firing cone.
|
||||
ahead = float(v @ fwd) / d
|
||||
ang = math.acos(max(-1.0, min(1.0, ahead)))
|
||||
score = d * (1.0 + 3.0 * (ang / math.pi) ** 2)
|
||||
if score < bestscore:
|
||||
best, bestscore = (eoff, enm, q, v, d), score
|
||||
if best is None:
|
||||
if not self.dry:
|
||||
self.pad.axis("LX", 0.0)
|
||||
self.pad.axis("LY", 0.0)
|
||||
return "no-hostiles"
|
||||
|
||||
eoff, enm, q, v, d = best
|
||||
lx = float(v @ right)
|
||||
ly = float(v @ up)
|
||||
lz = float(v @ fwd)
|
||||
yaw = math.atan2(lx, lz if abs(lz) > 1e-3 else 1e-3)
|
||||
pitch = math.atan2(ly, lz if abs(lz) > 1e-3 else 1e-3)
|
||||
if lz < 0:
|
||||
yaw = math.copysign(math.pi / 2, lx if lx else 1.0)
|
||||
|
||||
sx = clamp(self.KP * yaw - self.KD * float(w @ up))
|
||||
sy = clamp(-(self.KP * pitch - self.KD * float(w @ right)))
|
||||
aligned = abs(yaw) < self.FIRE_CONE and abs(pitch) < self.FIRE_CONE
|
||||
fire = aligned and d < self.FIRE_RANGE
|
||||
|
||||
if not self.dry:
|
||||
self.pad.axis("LX", sx)
|
||||
self.pad.axis("LY", sy)
|
||||
if fire != self.firing:
|
||||
(self.pad.press if fire else self.pad.release)("RB")
|
||||
self.firing = fire
|
||||
self.shots = getattr(self, "shots", 0) + (1 if fire else 0)
|
||||
return (f"tgt={enm[3:24]:<22} d={d:8.0f} yaw={math.degrees(yaw):+6.1f} "
|
||||
f"pit={math.degrees(pitch):+6.1f} stick=({sx:+.2f},{sy:+.2f}) fire={int(fire)}")
|
||||
|
||||
def run(self, secs, hz=10.0):
|
||||
t0 = time.time()
|
||||
last = t0
|
||||
last_scan = 0.0
|
||||
while time.time() - t0 < secs:
|
||||
t = time.time()
|
||||
if t - last_scan > 2.5:
|
||||
ents = self.W.rescan()
|
||||
last_scan = t
|
||||
c = Counter(unit_faction(e[1]) for e in ents)
|
||||
print(f"[{t-t0:6.1f}] rescan: {len(ents)} entities {dict(c)}", flush=True)
|
||||
print(f"[{t-t0:6.1f}] {self.step(t - last)}", flush=True)
|
||||
last = t
|
||||
time.sleep(max(0, 1.0 / hz - (time.time() - t)))
|
||||
if not self.dry:
|
||||
self.pad.reset()
|
||||
|
||||
|
||||
def main():
|
||||
cfg = json.load(open(sys.argv[1]))
|
||||
secs = float(sys.argv[2]) if len(sys.argv) > 2 else 60.0
|
||||
dry = "--dry" in sys.argv
|
||||
W = World(cfg)
|
||||
W.rescan()
|
||||
ap = Autopilot(W, Pad(), dry=dry)
|
||||
ap.run(secs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
144
tools/re-capture/binq.py
Normal file
144
tools/re-capture/binq.py
Normal file
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Ask questions of a ctrl_probe.py capture: which words answer to which input?
|
||||
|
||||
The capture is a window of the player entity object sampled every tick, tagged
|
||||
with the pad state that produced it. That makes the interesting question
|
||||
mechanical: for each 4-byte offset, does its value during phase X differ from
|
||||
its value during the rest phases either side? A word that only moves while `RT`
|
||||
is held is that input's state — a throttle setting, an afterburner tank, a heat
|
||||
gauge — and one that moves in *every* phase is just live physics.
|
||||
|
||||
Sub-commands
|
||||
phases per-phase mean of every offset that moves at all
|
||||
respond <phase> offsets that move during <phase> and not at rest
|
||||
near <value> [tol] offsets whose first sample is ~= value (HUD anchor)
|
||||
trace <off> [off...] full time series of specific offsets (pos-relative hex)
|
||||
"""
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
HDR = b"SYLPHCTR"
|
||||
REC = struct.Struct("<d32sff3f")
|
||||
|
||||
|
||||
def load(path):
|
||||
with open(path, "rb") as f:
|
||||
blob = f.read()
|
||||
assert blob[:8] == HDR, "not a ctrl_probe capture"
|
||||
n, win, back = struct.unpack_from("<III", blob, 8)
|
||||
stride = REC.size + win
|
||||
base = 8 + 12
|
||||
ts, phase, sp, dodge, pos, wins = [], [], [], [], [], []
|
||||
for i in range(n):
|
||||
o = base + i * stride
|
||||
t, ph, s, dg, x, y, z = REC.unpack_from(blob, o)
|
||||
ts.append(t)
|
||||
phase.append(ph.split(b"\0")[0].decode())
|
||||
sp.append(s)
|
||||
dodge.append(dg)
|
||||
pos.append((x, y, z))
|
||||
wins.append(blob[o + REC.size:o + REC.size + win])
|
||||
# A run that ended in GAME OVER keeps sampling a dead object, and those
|
||||
# frames dominate every statistic. $BINQ_TMAX truncates the capture to the
|
||||
# part that was still flying.
|
||||
tmax = float(os.environ.get("BINQ_TMAX", "inf"))
|
||||
if tmax < float("inf"):
|
||||
keep = [i for i, t in enumerate(ts) if t <= tmax]
|
||||
n = len(keep)
|
||||
ts = [ts[i] for i in keep]
|
||||
phase = [phase[i] for i in keep]
|
||||
sp = [sp[i] for i in keep]
|
||||
dodge = [dodge[i] for i in keep]
|
||||
pos = [pos[i] for i in keep]
|
||||
wins = [wins[i] for i in keep]
|
||||
A = np.frombuffer(b"".join(wins), dtype=">f4").reshape(n, win // 4).astype(np.float64)
|
||||
U = np.frombuffer(b"".join(wins), dtype=">u4").reshape(n, win // 4)
|
||||
return dict(n=n, win=win, back=back, t=np.array(ts), phase=phase,
|
||||
speed=np.array(sp), dodge=np.array(dodge),
|
||||
pos=np.array(pos), F=A, U=U)
|
||||
|
||||
|
||||
def label(d, i):
|
||||
return f"pos{i * 4 - d['back']:+#07x}"
|
||||
|
||||
|
||||
def finite(d):
|
||||
F = d["F"]
|
||||
return np.all(np.isfinite(F), axis=0) & (np.max(np.abs(F), axis=0) < 1e12)
|
||||
|
||||
|
||||
def cmd_phases(d, args):
|
||||
ok = finite(d)
|
||||
phases = []
|
||||
for p in d["phase"]:
|
||||
if p not in phases:
|
||||
phases.append(p)
|
||||
F = d["F"]
|
||||
mv = np.zeros(F.shape[1])
|
||||
means = {}
|
||||
for p in phases:
|
||||
m = np.array([x == p for x in d["phase"]])
|
||||
means[p] = F[m].mean(axis=0)
|
||||
for p in phases:
|
||||
mv = np.maximum(mv, np.abs(means[p] - means[phases[0]]))
|
||||
idx = np.flatnonzero(ok & (mv > 1e-3))
|
||||
order = idx[np.argsort(-mv[idx])][:int(args[0]) if args else 25]
|
||||
print("offset " + "".join(f"{p[:8]:>10}" for p in phases))
|
||||
for i in order:
|
||||
print(f"{label(d, i):<10}" + "".join(f"{means[p][i]:10.2f}" for p in phases))
|
||||
|
||||
|
||||
def cmd_respond(d, args):
|
||||
want = args[0]
|
||||
F, ok = d["F"], finite(d)
|
||||
inp = np.array([p == want for p in d["phase"]])
|
||||
rest = np.array([p.startswith("rest") or p == "base" for p in d["phase"]])
|
||||
if not inp.any():
|
||||
sys.exit(f"no phase {want!r}")
|
||||
# A word answering to this input must move *while it is held* and be quiet
|
||||
# at rest; a word that also moves at rest is live physics, not the input.
|
||||
a = F[inp]
|
||||
r = F[rest]
|
||||
d_in = a.max(axis=0) - a.min(axis=0)
|
||||
d_rest = r.max(axis=0) - r.min(axis=0)
|
||||
score = d_in - 2.0 * d_rest
|
||||
idx = np.flatnonzero(ok & (d_in > 1e-3) & (score > 0))
|
||||
for i in idx[np.argsort(-score[idx])][:20]:
|
||||
print(f"{label(d, i):<10} in-phase {a[:, i].min():12.3f}..{a[:, i].max():12.3f}"
|
||||
f" at-rest {r[:, i].min():12.3f}..{r[:, i].max():12.3f}")
|
||||
if not len(idx):
|
||||
print("(nothing moves under this input that is quiet at rest)")
|
||||
|
||||
|
||||
def cmd_near(d, args):
|
||||
v = float(args[0])
|
||||
tol = float(args[1]) if len(args) > 1 else max(1e-3, abs(v) * 1e-3)
|
||||
F, U = d["F"], d["U"]
|
||||
for i in np.flatnonzero(np.abs(F[0] - v) <= tol):
|
||||
print(f"{label(d, i):<10} f32 {F[0, i]:.4f} -> {F[-1, i]:.4f}")
|
||||
for i in np.flatnonzero(np.abs(U[0].astype(np.float64) - v) <= tol):
|
||||
print(f"{label(d, i):<10} u32 {U[0, i]} -> {U[-1, i]}")
|
||||
|
||||
|
||||
def cmd_trace(d, args):
|
||||
offs = [int(a, 0) for a in args]
|
||||
idx = [(o + d["back"]) // 4 for o in offs]
|
||||
print("t phase speed " + " ".join(f"{o:+#07x}" for o in offs))
|
||||
for k in range(d["n"]):
|
||||
print(f"{d['t'][k]:6.2f} {d['phase'][k]:<12} {d['speed'][k]:6.0f} "
|
||||
+ " ".join(f"{d['F'][k, i]:8.2f}" for i in idx))
|
||||
|
||||
|
||||
def main():
|
||||
d = load(sys.argv[1])
|
||||
print(f"# {d['n']} ticks, window {d['win']:#x} bytes, back {d['back']:#x}")
|
||||
cmd = sys.argv[2] if len(sys.argv) > 2 else "phases"
|
||||
{"phases": cmd_phases, "respond": cmd_respond, "near": cmd_near,
|
||||
"trace": cmd_trace}[cmd](d, sys.argv[3:])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
165
tools/re-capture/calibrate.py
Normal file
165
tools/re-capture/calibrate.py
Normal file
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Learn which body axis each stick drives, and which button fires.
|
||||
|
||||
The first autopilot run nulled yaw but held a steady ~27 deg pitch error, which
|
||||
is the signature of a wrong pitch axis or sign — guessing `right = row0` and
|
||||
`up = fwd x right` assumes a handedness the game need not share. So measure it:
|
||||
hold each stick axis, read the craft's body angular velocity from consecutive
|
||||
orientation matrices, and see which body axis actually responds and in which
|
||||
direction.
|
||||
|
||||
The fire button is found the same way — by consequence, not assumption: the
|
||||
nose ammo counter in the player's object must go down.
|
||||
|
||||
Usage: calibrate.py <config.json> [out.json]
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
import gworld # noqa: E402
|
||||
import entities2 # noqa: E402
|
||||
from flight_probe import Pad # noqa: E402
|
||||
|
||||
|
||||
def player_off(w, fd, size, delta):
|
||||
defs = entities2.definitions(w)
|
||||
for _ in range(6):
|
||||
mv = entities2.moving(fd, size, dt=0.5)
|
||||
ents = entities2.typed(fd, defs, mv, delta)
|
||||
for off, nm, pos, sp in ents:
|
||||
if "Player" in nm:
|
||||
return off
|
||||
time.sleep(0.5)
|
||||
return None
|
||||
|
||||
|
||||
def read_rot(fd, off, rot_delta, stride):
|
||||
n = stride * 2 + 12
|
||||
b = os.pread(fd, n, off + rot_delta)
|
||||
if len(b) < n:
|
||||
return None
|
||||
M = np.array([struct.unpack_from(">3f", b, stride * r) for r in range(3)])
|
||||
if not np.all(np.isfinite(M)) or np.max(np.abs(M @ M.T - np.eye(3))) > 5e-3:
|
||||
return None
|
||||
return M
|
||||
|
||||
|
||||
def mean_body_rate(fd, off, cfg, secs=2.0, hz=8.0):
|
||||
"""Mean angular velocity expressed in the craft's own axes."""
|
||||
prev, t_prev = None, None
|
||||
acc, n = np.zeros(3), 0
|
||||
end = time.time() + secs
|
||||
while time.time() < end:
|
||||
t = time.time()
|
||||
M = read_rot(fd, off, cfg["rot_delta"], cfg.get("rot_stride", 12))
|
||||
if M is not None and prev is not None:
|
||||
dt = t - t_prev
|
||||
D = prev @ M.T
|
||||
w = np.array([D[2, 1] - D[1, 2], D[0, 2] - D[2, 0], D[1, 0] - D[0, 1]]) / 2.0
|
||||
if dt > 0 and np.linalg.norm(w) < 0.5:
|
||||
# into body axes
|
||||
acc += M @ (w / dt)
|
||||
n += 1
|
||||
if M is not None:
|
||||
prev, t_prev = M, t
|
||||
time.sleep(max(0, 1.0 / hz - (time.time() - t)))
|
||||
return acc / max(n, 1)
|
||||
|
||||
|
||||
def main():
|
||||
cfg = json.load(open(sys.argv[1]))
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else sys.argv[1]
|
||||
w = gworld.World()
|
||||
fd, size = w.fd, w.size
|
||||
off = player_off(w, fd, size, cfg["def_delta"])
|
||||
if off is None:
|
||||
sys.exit("player not found")
|
||||
print(f"# player pos va {gmem.primary_va(off):#010x}")
|
||||
pad = Pad()
|
||||
|
||||
res = {}
|
||||
for axis, name in (("LX", "yaw"), ("LY", "pitch")):
|
||||
pad.reset()
|
||||
time.sleep(1.0)
|
||||
base = mean_body_rate(fd, off, cfg, 1.2)
|
||||
pad.axis(axis, 0.9)
|
||||
time.sleep(0.4)
|
||||
wpos = mean_body_rate(fd, off, cfg, 2.0)
|
||||
pad.axis(axis, 0.0)
|
||||
time.sleep(1.2)
|
||||
d = wpos - base
|
||||
k = int(np.argmax(np.abs(d)))
|
||||
res[name] = {"axis": k, "sign": 1 if d[k] > 0 else -1,
|
||||
"mag": float(abs(d[k]))}
|
||||
print(f"# {axis} (+0.9) -> body rate {d.round(3)} => {name} axis {k} "
|
||||
f"sign {res[name]['sign']:+d}")
|
||||
pad.reset()
|
||||
|
||||
# ---- fire button: the nose ammo counter must fall -----------------------
|
||||
blob = os.pread(fd, 0x1000, max(0, off - 0x800))
|
||||
ammo_offs = []
|
||||
for i in range(0, len(blob) - 3, 4):
|
||||
(u,) = struct.unpack_from(">I", blob, i)
|
||||
(f,) = struct.unpack_from(">f", blob, i)
|
||||
if u == 6000 or (math.isfinite(f) and abs(f - 6000.0) < 0.5):
|
||||
ammo_offs.append((max(0, off - 0x800) + i) - off)
|
||||
print(f"# ammo-like words (==6000) near the player: "
|
||||
f"{[hex(o) for o in ammo_offs] or 'none'}")
|
||||
|
||||
def ammo():
|
||||
vals = []
|
||||
for d in ammo_offs:
|
||||
b = os.pread(fd, 4, off + d)
|
||||
if len(b) == 4:
|
||||
vals.append(struct.unpack(">I", b)[0])
|
||||
return vals
|
||||
|
||||
fire_btn = None
|
||||
if ammo_offs:
|
||||
for btn in ("RB", "LB", "A", "B", "X", "Y"):
|
||||
before = ammo()
|
||||
pad.press(btn)
|
||||
time.sleep(1.2)
|
||||
pad.release(btn)
|
||||
time.sleep(0.5)
|
||||
after = ammo()
|
||||
drop = [a - b for a, b in zip(before, after)]
|
||||
print(f"# {btn}: ammo delta {drop}")
|
||||
if any(x > 0 for x in drop):
|
||||
fire_btn = btn
|
||||
break
|
||||
for trig in ("RT", "LT"):
|
||||
if fire_btn:
|
||||
break
|
||||
before = ammo()
|
||||
pad.trig(trig, 1.0)
|
||||
time.sleep(1.2)
|
||||
pad.trig(trig, 0.0)
|
||||
time.sleep(0.5)
|
||||
after = ammo()
|
||||
drop = [a - b for a, b in zip(before, after)]
|
||||
print(f"# {trig}: ammo delta {drop}")
|
||||
if any(x > 0 for x in drop):
|
||||
fire_btn = trig
|
||||
pad.reset()
|
||||
|
||||
cfg["yaw_axis"] = res["yaw"]["axis"]
|
||||
cfg["yaw_sign"] = res["yaw"]["sign"]
|
||||
cfg["pitch_axis"] = res["pitch"]["axis"]
|
||||
cfg["pitch_sign"] = res["pitch"]["sign"]
|
||||
cfg["fire"] = fire_btn
|
||||
cfg["ammo_offs"] = ammo_offs
|
||||
json.dump(cfg, open(out, "w"), indent=1)
|
||||
print("\n" + json.dumps(cfg, indent=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
272
tools/re-capture/ctrl_probe.py
Normal file
272
tools/re-capture/ctrl_probe.py
Normal file
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Which control is the throttle, and where does the craft keep its own state?
|
||||
|
||||
Two questions, one flight. Both are answered by *consequence* rather than by
|
||||
reading a field we hope is the right one:
|
||||
|
||||
* **Throttle.** The craft's speed is measured from its own position — a finite
|
||||
difference on the position triple in guest RAM — so no speed field has to be
|
||||
found first. The probe then holds each candidate input in turn and asks which
|
||||
one changes that measured speed. (`findspeed.py` failed the other way round:
|
||||
it assumed `RT` was the throttle and went looking for a field that rose.)
|
||||
|
||||
* **Own state.** Every tick also copies a window of the player entity object.
|
||||
Afterwards, offsets whose float value tracks the measured speed are candidate
|
||||
speed/throttle fields, and offsets that only ever *fall* are candidate
|
||||
hull/shield/ammo — the numbers survival needs.
|
||||
|
||||
Flying straight into a firefight for 90 s is how earlier runs died, so the loop
|
||||
keeps the collision avoidance from navigator.py armed the whole time and marks
|
||||
any sample where it had to intervene: a phase that had to dodge is not a clean
|
||||
speed measurement, and says so rather than being quietly averaged in.
|
||||
|
||||
Usage: ctrl_probe.py <config.json> <out-prefix> [hold_s] [settle_s]
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
import navigator # noqa: E402
|
||||
from flight_probe import Pad # noqa: E402
|
||||
|
||||
# 0x800 each way: the shield lives at pos+0x430 (own_state.py), which a
|
||||
# 0x400 window silently cropped out of the first capture.
|
||||
WIN_BACK = 0x800 # bytes of the player object kept before the position
|
||||
WIN_FWD = 0x800 # ...and after
|
||||
WIN = WIN_BACK + WIN_FWD
|
||||
HZ = 20.0
|
||||
|
||||
# (label, [pad commands]) — everything the pad can do that might be a throttle.
|
||||
# RB is left out: it is the fire button (autopilot-memory-driven.md) and firing
|
||||
# during a speed measurement only invites return fire.
|
||||
CANDIDATES = [
|
||||
("RT", [("trig", "RT", 1.0)]),
|
||||
("LT", [("trig", "LT", 1.0)]),
|
||||
("RT+LT", [("trig", "RT", 1.0), ("trig", "LT", 1.0)]),
|
||||
("A", [("press", "A")]),
|
||||
("B", [("press", "B")]),
|
||||
("X", [("press", "X")]),
|
||||
("Y", [("press", "Y")]),
|
||||
("LB", [("press", "LB")]),
|
||||
("LS", [("press", "LS")]),
|
||||
("RS", [("press", "RS")]),
|
||||
("RY_up", [("axis", "RY", -1.0)]),
|
||||
("RY_down", [("axis", "RY", 1.0)]),
|
||||
("RX_right", [("axis", "RX", 1.0)]),
|
||||
("dpad_up", [("dpad", "up")]),
|
||||
("dpad_down", [("dpad", "down")]),
|
||||
("dpad_left", [("dpad", "left")]),
|
||||
("dpad_right", [("dpad", "right")]),
|
||||
]
|
||||
|
||||
|
||||
def apply(pad, cmds):
|
||||
for c in cmds:
|
||||
if c[0] == "trig":
|
||||
pad.trig(c[1], c[2])
|
||||
elif c[0] == "axis":
|
||||
pad.axis(c[1], c[2])
|
||||
elif c[0] == "press":
|
||||
pad.press(c[1])
|
||||
elif c[0] == "dpad":
|
||||
pad.f.write(f"dpad {c[1]}\n")
|
||||
|
||||
|
||||
def clear(pad):
|
||||
pad.reset()
|
||||
pad.f.write("dpad center\n")
|
||||
|
||||
|
||||
class Run:
|
||||
def __init__(self, cfg, prefix, hold, settle):
|
||||
self.W = navigator.World(cfg)
|
||||
self.nav = navigator.Navigator(self.W, None, dry=True)
|
||||
self.prefix = prefix
|
||||
self.hold = hold
|
||||
self.settle = settle
|
||||
self.pad = Pad()
|
||||
self.rows = [] # (t, phase, pos, speed, dodged)
|
||||
self.win = [] # raw window bytes per tick
|
||||
ents = self.W.scan()
|
||||
me = [(off, va) for off, va in ents if "Player" in self.W.defs[va]]
|
||||
if not me:
|
||||
sys.exit("player entity not in the scan — not in flight?")
|
||||
self.me_off = me[0][0]
|
||||
self.me_name = self.W.defs[me[0][1]]
|
||||
print(f"# player {self.me_name} pos off {self.me_off:#x} "
|
||||
f"va {gmem.primary_va(self.me_off):#x} | {len(ents)} entities",
|
||||
flush=True)
|
||||
|
||||
# ------------------------------------------------------------- helpers
|
||||
def pos(self):
|
||||
return self.W.pos(self.me_off)
|
||||
|
||||
def sticks_for(self, vec):
|
||||
"""Stick deflections that point the nose along `vec` (escape steering)."""
|
||||
M = self.W.rot(self.me_off)
|
||||
if M is None:
|
||||
return 0.0, 0.0
|
||||
fwd = M[self.W.fwd_row] * self.W.fwd_sign
|
||||
right = M[(self.W.fwd_row + 1) % 3]
|
||||
up = np.cross(fwd, right)
|
||||
ez = float(np.dot(vec, fwd))
|
||||
yaw = math.atan2(float(np.dot(vec, right)), ez if abs(ez) > 1e-3 else 1e-3)
|
||||
pitch = math.atan2(float(np.dot(vec, up)), ez if abs(ez) > 1e-3 else 1e-3)
|
||||
return (max(-1.0, min(1.0, 2.0 * yaw)), max(-1.0, min(1.0, -2.0 * pitch)))
|
||||
|
||||
# ---------------------------------------------------------------- phase
|
||||
def phase(self, label, cmds, secs):
|
||||
clear(self.pad)
|
||||
apply(self.pad, cmds)
|
||||
t_end = time.time() + secs
|
||||
dodged = False
|
||||
while time.time() < t_end:
|
||||
t = time.time()
|
||||
p = self.pos()
|
||||
if p is None:
|
||||
break
|
||||
self.rows.append([t, label, p, 0.0, 0])
|
||||
self.win.append(os.pread(self.W.fd, WIN, self.me_off - WIN_BACK))
|
||||
|
||||
# avoidance runs at 5 Hz; a real threat overrides the phase and the
|
||||
# samples from here on are flagged
|
||||
if len(self.rows) % 4 == 0:
|
||||
ents = self.W.sample(t)
|
||||
me = [e for e in ents if e[0] == self.me_off]
|
||||
if me:
|
||||
_, _, mp, mv, mr = me[0]
|
||||
push, worst = self.nav.avoidance(mp, mv, mr, ents, self.me_off)
|
||||
if float(np.linalg.norm(push)) > 0.6:
|
||||
sx, sy = self.sticks_for(navigator.norm(push))
|
||||
self.pad.axis("LX", sx)
|
||||
self.pad.axis("LY", sy)
|
||||
dodged = True
|
||||
self.rows[-1][4] = 1
|
||||
elif dodged:
|
||||
self.pad.axis("LX", 0.0)
|
||||
self.pad.axis("LY", 0.0)
|
||||
time.sleep(max(0.0, 1.0 / HZ - (time.time() - t)))
|
||||
return dodged
|
||||
|
||||
def run(self):
|
||||
t0 = time.time()
|
||||
self.phase("base", [], self.settle * 2)
|
||||
for label, cmds in CANDIDATES:
|
||||
d = self.phase(label, cmds, self.hold)
|
||||
self.phase(f"rest_{label}", [], self.settle)
|
||||
sp = self.phase_speed(label)
|
||||
print(f"[{time.time()-t0:6.1f}] {label:<10} "
|
||||
f"v0={sp[0]:7.1f} v1={sp[1]:7.1f} d={sp[1]-sp[0]:+7.1f}"
|
||||
f"{' (dodged)' if d else ''}", flush=True)
|
||||
clear(self.pad)
|
||||
self.speeds()
|
||||
self.dump()
|
||||
|
||||
# ------------------------------------------------------------ analysis
|
||||
def speeds(self):
|
||||
"""Fill in per-tick speed by central difference on position."""
|
||||
for i, r in enumerate(self.rows):
|
||||
j, k = max(0, i - 2), min(len(self.rows) - 1, i + 2)
|
||||
dt = self.rows[k][0] - self.rows[j][0]
|
||||
if dt > 1e-3:
|
||||
r[3] = float(np.linalg.norm(self.rows[k][2] - self.rows[j][2])) / dt
|
||||
|
||||
def phase_speed(self, label):
|
||||
"""(speed early, speed late) within a phase — needs speeds() first."""
|
||||
self.speeds()
|
||||
v = [r[3] for r in self.rows if r[1] == label]
|
||||
if len(v) < 6:
|
||||
return (0.0, 0.0)
|
||||
n = max(2, len(v) // 4)
|
||||
return (float(np.mean(v[:n])), float(np.mean(v[-n:])))
|
||||
|
||||
def dump(self):
|
||||
with open(self.prefix + ".csv", "w") as f:
|
||||
f.write("t,phase,x,y,z,speed,dodged\n")
|
||||
t0 = self.rows[0][0]
|
||||
for t, ph, p, sp, dg in self.rows:
|
||||
f.write(f"{t-t0:.3f},{ph},{p[0]:.3f},{p[1]:.3f},{p[2]:.3f},"
|
||||
f"{sp:.3f},{dg}\n")
|
||||
with open(self.prefix + ".bin", "wb") as f:
|
||||
f.write(b"SYLPHCTR")
|
||||
f.write(struct.pack("<III", len(self.rows), WIN, WIN_BACK))
|
||||
t0 = self.rows[0][0]
|
||||
for (t, ph, p, sp, dg), w in zip(self.rows, self.win):
|
||||
f.write(struct.pack("<d32sff3f", t - t0, ph.encode()[:32], sp,
|
||||
float(dg), *[float(c) for c in p]))
|
||||
f.write(w.ljust(WIN, b"\0"))
|
||||
print(f"# wrote {self.prefix}.csv and {self.prefix}.bin "
|
||||
f"({len(self.rows)} ticks)", flush=True)
|
||||
|
||||
# ---- per-phase summary
|
||||
print("\n# phase n v_early v_late delta dodged")
|
||||
order, seen = [], set()
|
||||
for r in self.rows:
|
||||
if r[1] not in seen:
|
||||
seen.add(r[1])
|
||||
order.append(r[1])
|
||||
for ph in order:
|
||||
v = [r[3] for r in self.rows if r[1] == ph]
|
||||
dg = sum(r[4] for r in self.rows if r[1] == ph)
|
||||
if len(v) < 6:
|
||||
continue
|
||||
n = max(2, len(v) // 4)
|
||||
a, b = float(np.mean(v[:n])), float(np.mean(v[-n:]))
|
||||
print(f" {ph:<12} {len(v):4d} {a:9.1f} {b:9.1f} {b-a:+9.1f} {dg:5d}")
|
||||
|
||||
# ---- which words in the object track speed, and which only fall
|
||||
W_ = np.frombuffer(b"".join(x.ljust(WIN, b"\0") for x in self.win),
|
||||
dtype=">f4").reshape(len(self.win), WIN // 4)
|
||||
sp = np.array([r[3] for r in self.rows], dtype=np.float64)
|
||||
with np.errstate(invalid="ignore", over="ignore"):
|
||||
A = W_.astype(np.float64)
|
||||
ok = np.all(np.isfinite(A), axis=0) & (np.max(np.abs(A), axis=0) < 1e9)
|
||||
var = np.std(A, axis=0)
|
||||
cand = np.flatnonzero(ok & (var > 1e-6))
|
||||
cor = []
|
||||
for i in cand:
|
||||
c = np.corrcoef(A[:, i], sp)[0, 1]
|
||||
if np.isfinite(c):
|
||||
cor.append((abs(c), c, i))
|
||||
cor.sort(reverse=True)
|
||||
print("\n# object words correlating with measured speed "
|
||||
"(offset relative to the position triple)")
|
||||
for ac, c, i in cor[:12]:
|
||||
off = i * 4 - WIN_BACK
|
||||
print(f" pos{off:+#07x} r={c:+.3f} "
|
||||
f"range {A[:, i].min():.3f} .. {A[:, i].max():.3f}")
|
||||
|
||||
print("\n# words that never rise (candidate hull / shield / ammo)")
|
||||
shown = 0
|
||||
for i in cand:
|
||||
col = A[:, i]
|
||||
if col[-1] >= col[0] - 1e-6:
|
||||
continue
|
||||
if np.max(np.diff(col)) > 1e-6:
|
||||
continue
|
||||
off = i * 4 - WIN_BACK
|
||||
print(f" pos{off:+#07x} {col[0]:.3f} -> {col[-1]:.3f}")
|
||||
shown += 1
|
||||
if shown >= 12:
|
||||
break
|
||||
if not shown:
|
||||
print(" (none — nothing in the window decreased monotonically)")
|
||||
|
||||
|
||||
def main():
|
||||
cfg = json.load(open(sys.argv[1]))
|
||||
prefix = sys.argv[2]
|
||||
hold = float(sys.argv[3]) if len(sys.argv) > 3 else 4.0
|
||||
settle = float(sys.argv[4]) if len(sys.argv) > 4 else 2.5
|
||||
Run(cfg, prefix, hold, settle).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
31
tools/re-capture/ctrl_session.sh
Executable file
31
tools/re-capture/ctrl_session.sh
Executable file
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# One background task: bind the player's transform, then run the control probe.
|
||||
#
|
||||
# Everything that must not be interrupted lives in a single task here, because
|
||||
# the display and the emulator both die on their own every few minutes in this
|
||||
# container and nothing may depend on surviving between tool calls.
|
||||
#
|
||||
# Assumes the game is ALREADY in flight (launch_mission.sh). Pass --boot to
|
||||
# have it get there itself.
|
||||
set -u
|
||||
export HOME=/sylph-home/re SDL_AUDIODRIVER=dummy DISPLAY=:98
|
||||
export PYTHONPATH=/sylph-home/.local/lib/python3.12/site-packages
|
||||
SD="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
CFG=/tmp/nav-live.json
|
||||
PRE=/tmp/ctrl
|
||||
HOLD=3.0
|
||||
SETTLE=2.0
|
||||
if [ "${1:-}" = "--boot" ]; then
|
||||
shift
|
||||
"$SD/launch_mission.sh" fly || { echo "BOOT FAILED"; exit 1; }
|
||||
fi
|
||||
[ $# -ge 1 ] && CFG="$1"
|
||||
[ $# -ge 2 ] && PRE="$2"
|
||||
[ $# -ge 3 ] && HOLD="$3"
|
||||
[ $# -ge 4 ] && SETTLE="$4"
|
||||
|
||||
python3 "$SD/entities2.py" self 0x130 "$CFG" || { echo "TRANSFORM BIND FAILED"; exit 1; }
|
||||
echo "--- config: $(cat "$CFG")"
|
||||
python3 "$SD/ctrl_probe.py" "$CFG" "$PRE" "$HOLD" "$SETTLE"
|
||||
echo "PROBE DONE"
|
||||
219
tools/re-capture/entities2.py
Normal file
219
tools/re-capture/entities2.py
Normal file
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Type every live entity, and locate the player, via the definition pointer.
|
||||
|
||||
A live entity object keeps a pointer to its parsed `.tbl` definition (vtable
|
||||
0x820af844, addresses known) at a **fixed offset from its position triple**.
|
||||
Finding that offset once types every moving object in the scene at a stroke:
|
||||
enemies, friendlies and us, separated from the thousands of moving particles.
|
||||
|
||||
The offset is *derived*, not assumed: for every moving triple, every nearby word
|
||||
is checked against the set of definition addresses, and the winning delta is the
|
||||
one that repeats across many independent entities.
|
||||
|
||||
Sub-commands:
|
||||
delta find the (position -> definition pointer) offset
|
||||
list [delta] type every live entity and print it
|
||||
self [delta] the player's entity, its object window, and its orientation
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
import gworld # noqa: E402
|
||||
|
||||
|
||||
def definitions(w):
|
||||
out = {}
|
||||
for off in w.scan_vtable(gworld.DEF_VTABLE):
|
||||
nm = w.name_of(off)
|
||||
if nm and nm.startswith("UN_"):
|
||||
va = gmem.primary_va(off)
|
||||
if va is not None:
|
||||
out[struct.pack(">I", va)] = nm
|
||||
return out
|
||||
|
||||
|
||||
# Entity objects live in one heap region; scanning only it turns a 250 MB
|
||||
# double-read into a few MB. That matters for more than speed: the full scan
|
||||
# competes with the emulator for every core under lavapipe, and a game that
|
||||
# does not advance a frame between the two samples has nothing "moving" in it.
|
||||
ENT_VA_LO, ENT_VA_HI = 0xBD000000, 0xBE000000
|
||||
|
||||
|
||||
def moving(fd, size, dt=0.5, lo=1.0, hi=4000.0, va_range=(ENT_VA_LO, ENT_VA_HI)):
|
||||
def arrays():
|
||||
if va_range:
|
||||
f0, f1 = gmem.va_to_off(va_range[0]), gmem.va_to_off(va_range[1])
|
||||
else:
|
||||
f0, f1 = 0, size
|
||||
for a, b in gmem.extents(fd, size):
|
||||
a, b = max(a, f0), min(b, f1)
|
||||
n = (b - a) // 4 * 4
|
||||
if n >= 64:
|
||||
yield a, np.frombuffer(os.pread(fd, n, a), dtype=">f4")
|
||||
|
||||
s0 = dict(arrays())
|
||||
t0 = time.time()
|
||||
time.sleep(dt)
|
||||
s1 = dict(arrays())
|
||||
t1 = time.time()
|
||||
out = []
|
||||
# Match extents by their file offset. Zipping the two lists positionally is
|
||||
# wrong: the game allocates between the samples, the sparse extent layout
|
||||
# shifts, and every subsequent pair misaligns -- which silently reports
|
||||
# almost nothing as moving.
|
||||
for o, a in s0.items():
|
||||
b = s1.get(o)
|
||||
if b is None or len(a) != len(b):
|
||||
continue
|
||||
with np.errstate(invalid="ignore"):
|
||||
af = np.nan_to_num(a.astype(np.float64), nan=0, posinf=0, neginf=0)
|
||||
bf = np.nan_to_num(b.astype(np.float64), nan=0, posinf=0, neginf=0)
|
||||
d = bf - af
|
||||
idx = np.flatnonzero(np.abs(d) > 1e-4)
|
||||
cand = idx[np.isin(idx + 1, idx) & np.isin(idx + 2, idx)]
|
||||
for i in cand:
|
||||
v = np.array([d[i], d[i + 1], d[i + 2]])
|
||||
sp = float(np.linalg.norm(v)) / (t1 - t0)
|
||||
if lo < sp < hi:
|
||||
out.append((o + int(i) * 4, tuple(af[i:i + 3]), sp))
|
||||
return out
|
||||
|
||||
|
||||
def find_delta(fd, defs, movers, radius=0x400):
|
||||
votes = Counter()
|
||||
for off, pos, sp in movers[:4000]:
|
||||
lo = max(0, off - radius)
|
||||
blob = os.pread(fd, radius * 2, lo)
|
||||
for k in range(0, len(blob) - 3, 4):
|
||||
if blob[k:k + 4] in defs:
|
||||
votes[(lo + k) - off] += 1
|
||||
return votes
|
||||
|
||||
|
||||
def typed(fd, defs, movers, delta):
|
||||
out = []
|
||||
for off, pos, sp in movers:
|
||||
try:
|
||||
b = os.pread(fd, 4, off + delta)
|
||||
except OSError:
|
||||
continue
|
||||
nm = defs.get(b)
|
||||
if nm:
|
||||
out.append((off, nm, pos, sp))
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
cmd = sys.argv[1] if len(sys.argv) > 1 else "delta"
|
||||
w = gworld.World()
|
||||
fd, size = w.fd, w.size
|
||||
defs = definitions(w)
|
||||
print(f"# {len(defs)} unit definitions", flush=True)
|
||||
movers = moving(fd, size)
|
||||
print(f"# {len(movers)} moving triples", flush=True)
|
||||
|
||||
if cmd == "delta":
|
||||
votes = find_delta(fd, defs, movers)
|
||||
print("# candidate (position -> definition pointer) deltas:")
|
||||
for d, n in votes.most_common(8):
|
||||
print(f" {d:+#08x} seen {n}")
|
||||
return
|
||||
|
||||
delta = int(sys.argv[2], 0) if len(sys.argv) > 2 else 0x130
|
||||
ents = typed(fd, defs, movers, delta)
|
||||
uniq = {}
|
||||
for off, nm, pos, sp in ents:
|
||||
uniq.setdefault((nm, tuple(round(c, 1) for c in pos)), (off, nm, pos, sp))
|
||||
ents = list(uniq.values())
|
||||
print(f"# {len(ents)} typed live entities (delta {delta:+#x})")
|
||||
|
||||
if cmd == "list":
|
||||
c = Counter(nm for _, nm, _, _ in ents)
|
||||
for nm, k in c.most_common():
|
||||
print(f" {k:4d} {nm}")
|
||||
for off, nm, pos, sp in sorted(ents, key=lambda e: e[1])[:60]:
|
||||
print(f" {gmem.primary_va(off):#010x} {nm:<40} "
|
||||
f"({pos[0]:+9.1f},{pos[1]:+9.1f},{pos[2]:+9.1f}) {sp:7.1f}/s")
|
||||
return
|
||||
|
||||
if cmd == "self":
|
||||
me = [e for e in ents if "Player" in e[1]]
|
||||
if not me:
|
||||
sys.exit("player entity not found")
|
||||
off, nm, pos, sp = me[0]
|
||||
print(f"# player entity: pos va {gmem.primary_va(off):#010x} {nm}")
|
||||
print(f"# pos ({pos[0]:+.1f},{pos[1]:+.1f},{pos[2]:+.1f}) speed {sp:.1f}/s")
|
||||
# orientation inside the same object
|
||||
# The rotation is stored with a 16-byte row stride (a 4x4 transform
|
||||
# whose translation row is the position we already have), so a test for
|
||||
# nine *contiguous* floats structurally cannot find it. Test both.
|
||||
lo = max(0, off - 0x800)
|
||||
blob = os.pread(fd, 0x1000, lo)
|
||||
found = []
|
||||
for k in range(0, len(blob) - 48, 4):
|
||||
for stride, tag in ((12, "3x3"), (16, "4x4")):
|
||||
try:
|
||||
rows = [np.array(struct.unpack_from(">3f", blob, k + stride * r))
|
||||
for r in range(3)]
|
||||
except struct.error:
|
||||
continue
|
||||
M = np.array(rows)
|
||||
if not np.all(np.isfinite(M)) or np.max(np.abs(M)) > 1.001:
|
||||
continue
|
||||
if np.max(np.abs(M @ M.T - np.eye(3))) > 3e-3:
|
||||
continue
|
||||
if abs(np.linalg.det(M) - 1.0) > 1e-2:
|
||||
continue
|
||||
found.append(((lo + k) - off, M, stride))
|
||||
break
|
||||
print(f"# orthonormal 3x3 blocks inside the player object: {len(found)}")
|
||||
for d, M, st in found[:6]:
|
||||
print(f" pos{d:+#07x} stride {st}: " + " ".join(
|
||||
"(" + ",".join(f"{v:+.3f}" for v in row) + ")" for row in M))
|
||||
if len(sys.argv) > 3 and found:
|
||||
import json
|
||||
# Which of the orthonormal blocks is the CRAFT's attitude? The one
|
||||
# with a row along the direction of travel. Taking found[0] is what
|
||||
# produced a config with rot_delta -0x764 and a nonsense forward
|
||||
# axis: several blocks inside the object are orthonormal (bone or
|
||||
# camera frames), and only the craft's own has a row that tracks
|
||||
# where the craft is going.
|
||||
p0 = np.array(pos)
|
||||
time.sleep(0.35)
|
||||
p1 = np.array(struct.unpack(">3f", os.pread(fd, 12, off)))
|
||||
step = p1 - p0
|
||||
if np.linalg.norm(step) < 1e-3:
|
||||
sys.exit("craft is not moving — cannot bind the forward axis")
|
||||
vdir = step / np.linalg.norm(step)
|
||||
best = None
|
||||
for d, M, st in found:
|
||||
cs = [float(M[r] @ vdir) for r in range(3)]
|
||||
r = int(np.argmax([abs(c) for c in cs]))
|
||||
if best is None or abs(cs[r]) > abs(best[3]):
|
||||
best = (d, M, st, cs[r], r)
|
||||
rot_delta, M, rot_stride, cos, row = best
|
||||
sign = 1 if cos > 0 else -1
|
||||
print(f"# attitude block pos{rot_delta:+#07x} stride {rot_stride}: "
|
||||
f"forward = row {row} (sign {sign:+d}), cos={cos:+.3f}")
|
||||
if abs(cos) < 0.9:
|
||||
print("# WARNING: no block tracks the flight path (|cos| < 0.9)"
|
||||
" — the craft may be drifting hard; re-run while flying straight")
|
||||
cfg = {"def_delta": delta, "rot_delta": rot_delta,
|
||||
"rot_stride": rot_stride,
|
||||
"fwd_row": row, "fwd_sign": sign, "fwd_cos": round(cos, 4),
|
||||
"va_lo": ENT_VA_LO, "va_hi": ENT_VA_HI}
|
||||
json.dump(cfg, open(sys.argv[3], "w"), indent=1)
|
||||
print("# wrote " + sys.argv[3] + ": " + json.dumps(cfg))
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
44
tools/re-capture/escort_session.sh
Executable file
44
tools/re-capture/escort_session.sh
Executable file
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# One background task: boot -> Stage 02 -> fly the pilot while a second reader
|
||||
# records EVERY entity's hull, so the escort question ("who is losing, and how
|
||||
# fast") is answered from memory instead of from the HUD.
|
||||
#
|
||||
# Same one-task rule as fly_session.sh: the display and the emulator both die on
|
||||
# their own in this container, so nothing may depend on surviving between tool
|
||||
# calls.
|
||||
set -u
|
||||
export HOME=/sylph-home/re SDL_AUDIODRIVER=dummy DISPLAY=:98
|
||||
export PYTHONPATH=/sylph-home/.local/lib/python3.12/site-packages
|
||||
SD="$(cd "$(dirname "$0")" && pwd)"
|
||||
SECS="${1:-300}"
|
||||
TAG="${2:-escort}"
|
||||
SHOTS=/sylph-home/re/shots
|
||||
CFG=/tmp/nav-live.json
|
||||
|
||||
"$SD/launch_mission.sh" fly || { echo "BOOT FAILED"; exit 1; }
|
||||
python3 "$SD/entities2.py" self 0x130 "$CFG" || { echo "BIND FAILED"; exit 1; }
|
||||
echo "--- config: $(cat "$CFG")"
|
||||
|
||||
echo "=== initial entity table ==="
|
||||
python3 "$SD/mission_state.py" scan "$CFG"
|
||||
|
||||
# The HUD's REMAINING OB counter has no known address yet, so the correlation
|
||||
# material is a screenshot every 20 s stamped against the same clock as the
|
||||
# memory samples.
|
||||
( for i in $(seq 1 20); do
|
||||
printf '%s SHOT %02d\n' "$(date +%s.%N)" "$i" >> "/tmp/$TAG-shots.log"
|
||||
screenshot "$SHOTS/$TAG-$i.png" >/dev/null 2>&1
|
||||
sleep 20
|
||||
done ) &
|
||||
SHOTTER=$!
|
||||
|
||||
date +%s.%N > "/tmp/$TAG-t0"
|
||||
python3 "$SD/mission_state.py" watch "$CFG" "$SECS" 1 "/tmp/$TAG-mission.jsonl" \
|
||||
> "/tmp/$TAG-mission.log" 2>&1 &
|
||||
WATCHER=$!
|
||||
|
||||
python3 "$SD/pilot.py" "$CFG" "$SECS" > "/tmp/$TAG-pilot.log" 2>&1
|
||||
wait $WATCHER 2>/dev/null
|
||||
kill $SHOTTER 2>/dev/null
|
||||
screenshot "$SHOTS/$TAG-end.png" >/dev/null 2>&1
|
||||
echo "SESSION DONE"
|
||||
120
tools/re-capture/findplayer.py
Normal file
120
tools/re-capture/findplayer.py
Normal file
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Find the player craft's live position in guest RAM, from motion alone.
|
||||
|
||||
The spawn records at vtable 0x820af030 turned out to hold no live state (every
|
||||
word is constant across a 29 s capture), so the flying entity is some other
|
||||
object. Rather than guess which class, this finds it by what it must *do*:
|
||||
|
||||
a position triple moves in a straight line at constant speed while coasting.
|
||||
|
||||
Method: sample all of RAM K times ~dt apart, keep word indices whose float
|
||||
value changes every time, then require three consecutive such words to satisfy
|
||||
* equal step length each interval (constant speed), and
|
||||
* a constant direction (straight flight),
|
||||
* with speed in a sane range for a craft.
|
||||
|
||||
Only the wholly-mechanical properties of motion are used; no offset, class or
|
||||
encoding is assumed.
|
||||
|
||||
Usage: findplayer.py [samples] [interval_s]
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
|
||||
|
||||
def snapshot(fd, size):
|
||||
"""[(start_off, np.ndarray big-endian f32)] over allocated extents."""
|
||||
out = []
|
||||
for a, b in gmem.extents(fd, size):
|
||||
n = (b - a) // 4 * 4
|
||||
if n < 64:
|
||||
continue
|
||||
buf = os.pread(fd, n, a)
|
||||
out.append((a, np.frombuffer(buf, dtype=">f4")))
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
K = int(sys.argv[1]) if len(sys.argv) > 1 else 5
|
||||
dt = float(sys.argv[2]) if len(sys.argv) > 2 else 0.35
|
||||
path = gmem.mem_path()
|
||||
fd = os.open(path, os.O_RDONLY)
|
||||
size = os.path.getsize(path)
|
||||
|
||||
print(f"# sampling {K}x every {dt}s", flush=True)
|
||||
snaps, times = [], []
|
||||
for k in range(K):
|
||||
t = time.time()
|
||||
snaps.append(snapshot(fd, size))
|
||||
times.append(t)
|
||||
print(f"# sample {k} ({len(snaps[-1])} extents, "
|
||||
f"{sum(len(a) for _, a in snaps[-1])*4/1e6:.0f} MB) "
|
||||
f"in {time.time()-t:.2f}s", flush=True)
|
||||
time.sleep(max(0, dt - (time.time() - t)))
|
||||
|
||||
# extents must line up across samples for a word-wise comparison
|
||||
shapes = [tuple((o, len(a)) for o, a in s) for s in snaps]
|
||||
if len(set(shapes)) != 1:
|
||||
common = set(shapes[0])
|
||||
for sh in shapes[1:]:
|
||||
common &= set(sh)
|
||||
print(f"# extents shifted between samples; using {len(common)} common ones")
|
||||
keep = lambda s: [(o, a) for o, a in s if (o, len(a)) in common]
|
||||
snaps = [keep(s) for s in snaps]
|
||||
|
||||
hits = []
|
||||
for e in range(len(snaps[0])):
|
||||
base = snaps[0][e][0]
|
||||
arrs = [np.nan_to_num(s[e][1].astype(np.float64), nan=0, posinf=0, neginf=0)
|
||||
for s in snaps]
|
||||
# per-interval deltas
|
||||
d = [arrs[k + 1] - arrs[k] for k in range(K - 1)]
|
||||
moving = np.ones(len(arrs[0]), dtype=bool)
|
||||
for dk in d:
|
||||
moving &= np.abs(dk) > 1e-3
|
||||
idx = np.flatnonzero(moving)
|
||||
if len(idx) == 0:
|
||||
continue
|
||||
# candidate triples: i, i+1, i+2 all moving
|
||||
cand = idx[np.isin(idx + 1, idx) & np.isin(idx + 2, idx)]
|
||||
for i in cand:
|
||||
steps = []
|
||||
dirs = []
|
||||
ok = True
|
||||
for k in range(K - 1):
|
||||
v = np.array([d[k][i], d[k][i + 1], d[k][i + 2]])
|
||||
n = float(np.linalg.norm(v))
|
||||
ddt = times[k + 1] - times[k]
|
||||
sp = n / ddt
|
||||
if not (20.0 < sp < 3000.0):
|
||||
ok = False
|
||||
break
|
||||
steps.append(sp)
|
||||
dirs.append(v / n)
|
||||
if not ok or len(steps) < 3:
|
||||
continue
|
||||
if max(steps) - min(steps) > 0.25 * (sum(steps) / len(steps)):
|
||||
continue
|
||||
if min(float(np.dot(dirs[0], dd)) for dd in dirs) < 0.985:
|
||||
continue
|
||||
va = gmem.primary_va(base + int(i) * 4)
|
||||
pos = tuple(float(arrs[0][i + j]) for j in range(3))
|
||||
hits.append((va, base + int(i) * 4, sum(steps) / len(steps), pos))
|
||||
|
||||
print(f"\n# {len(hits)} position-like triples (straight, constant speed)")
|
||||
hits.sort(key=lambda h: -h[2])
|
||||
for va, off, sp, pos in hits[:40]:
|
||||
print(f" va {va:#010x} speed {sp:8.1f}/s pos "
|
||||
f"({pos[0]:+11.1f},{pos[1]:+11.1f},{pos[2]:+11.1f})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
147
tools/re-capture/findrot_global.py
Normal file
147
tools/re-capture/findrot_global.py
Normal file
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scan ALL of guest RAM for 3x3 orthonormal float blocks (rotation matrices).
|
||||
|
||||
Vectorised with numpy: nine shifted views of each extent give every candidate
|
||||
9-float window at once, so the whole 250 MB working set tests in seconds.
|
||||
|
||||
A rotation matrix is a very strong signature — unit rows, mutually orthogonal —
|
||||
so the hits are almost entirely real orientations. Two passes taken a moment
|
||||
apart, with a known stick input in between, then say which of them is *ours*.
|
||||
|
||||
Usage: findrot_global.py [--track seconds]
|
||||
"""
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
|
||||
FIFO = "/tmp/sylph-vgamepad.fifo"
|
||||
TOL = 2e-3
|
||||
|
||||
|
||||
def pad(line):
|
||||
with open(FIFO, "w") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def scan(fd, size):
|
||||
"""[(file_offset, 9 floats)] for every orthonormal 3x3 block."""
|
||||
hits = []
|
||||
for a, b in gmem.extents(fd, size):
|
||||
n = (b - a) // 4 * 4
|
||||
if n < 64:
|
||||
continue
|
||||
arr = np.frombuffer(os.pread(fd, n, a), dtype=">f4").astype(np.float32)
|
||||
if arr.size < 16:
|
||||
continue
|
||||
m = arr.size - 8
|
||||
cols = [arr[i:i + m] for i in range(9)]
|
||||
with np.errstate(invalid="ignore", over="ignore"):
|
||||
finite = np.ones(m, dtype=bool)
|
||||
for c in cols:
|
||||
finite &= np.isfinite(c) & (np.abs(c) <= 1.001)
|
||||
# row norms
|
||||
n0 = cols[0] ** 2 + cols[1] ** 2 + cols[2] ** 2
|
||||
n1 = cols[3] ** 2 + cols[4] ** 2 + cols[5] ** 2
|
||||
n2 = cols[6] ** 2 + cols[7] ** 2 + cols[8] ** 2
|
||||
ok = finite
|
||||
ok &= np.abs(n0 - 1) < TOL
|
||||
ok &= np.abs(n1 - 1) < TOL
|
||||
ok &= np.abs(n2 - 1) < TOL
|
||||
d01 = cols[0] * cols[3] + cols[1] * cols[4] + cols[2] * cols[5]
|
||||
d02 = cols[0] * cols[6] + cols[1] * cols[7] + cols[2] * cols[8]
|
||||
d12 = cols[3] * cols[6] + cols[4] * cols[7] + cols[5] * cols[8]
|
||||
ok &= np.abs(d01) < TOL
|
||||
ok &= np.abs(d02) < TOL
|
||||
ok &= np.abs(d12) < TOL
|
||||
for i in np.flatnonzero(ok):
|
||||
hits.append(a + int(i) * 4)
|
||||
return hits
|
||||
|
||||
|
||||
def read_mat(fd, off):
|
||||
b = os.pread(fd, 36, off)
|
||||
if len(b) < 36:
|
||||
return None
|
||||
return np.array(struct.unpack(">9f", b))
|
||||
|
||||
|
||||
def main():
|
||||
fd = os.open(gmem.mem_path(), os.O_RDONLY)
|
||||
size = os.path.getsize(gmem.mem_path())
|
||||
pad("reset")
|
||||
time.sleep(0.6)
|
||||
t0 = time.time()
|
||||
hits = scan(fd, size)
|
||||
print(f"# {len(hits)} orthonormal 3x3 blocks in RAM ({time.time()-t0:.1f}s)", flush=True)
|
||||
if not hits:
|
||||
return
|
||||
|
||||
# which of them rotate when WE yaw? measure change under left vs right
|
||||
def deltas(lx, secs=2.5, hz=6.0):
|
||||
"""Mean per-step rotation vector while the stick is held.
|
||||
|
||||
Sampled incrementally: at a few degrees per step the skew part of
|
||||
A·Bᵀ is the rotation vector, which it is not over a 2 s turn. Any block
|
||||
that stops being orthonormal mid-phase is memory that got reused, not an
|
||||
orientation, and is dropped.
|
||||
"""
|
||||
pad(f"axis LX {lx}")
|
||||
time.sleep(0.5)
|
||||
seq = []
|
||||
for _ in range(int(secs * hz)):
|
||||
t = time.time()
|
||||
seq.append({o: read_mat(fd, o) for o in hits})
|
||||
time.sleep(max(0, 1.0 / hz - (time.time() - t)))
|
||||
pad("axis LX 0")
|
||||
time.sleep(1.0)
|
||||
|
||||
def ortho(m):
|
||||
if m is None or not np.all(np.isfinite(m)):
|
||||
return False
|
||||
M = m.reshape(3, 3)
|
||||
return np.max(np.abs(M @ M.T - np.eye(3))) < 5e-3
|
||||
|
||||
out = {}
|
||||
for o in hits:
|
||||
ms = [s[o] for s in seq]
|
||||
if not all(ortho(m) for m in ms):
|
||||
continue
|
||||
ws = []
|
||||
for a, b in zip(ms, ms[1:]):
|
||||
M = a.reshape(3, 3) @ b.reshape(3, 3).T
|
||||
w = np.array([M[2, 1] - M[1, 2], M[0, 2] - M[2, 0], M[1, 0] - M[0, 1]]) / 2
|
||||
if np.linalg.norm(w) < 0.5: # small-angle regime only
|
||||
ws.append(w)
|
||||
if len(ws) >= 4:
|
||||
out[o] = np.mean(ws, axis=0)
|
||||
return out
|
||||
|
||||
dl = deltas(-0.9)
|
||||
dr = deltas(+0.9)
|
||||
rows = []
|
||||
for o in hits:
|
||||
if o not in dl or o not in dr:
|
||||
continue
|
||||
a, b = dl[o], dr[o]
|
||||
na, nb = np.linalg.norm(a), np.linalg.norm(b)
|
||||
if na < 0.01 or nb < 0.01:
|
||||
continue
|
||||
cos = float(a @ b / (na * nb))
|
||||
rows.append((cos, o, na, nb))
|
||||
rows.sort()
|
||||
print("\n# orientation blocks that rotate OPPOSITE ways for left vs right stick")
|
||||
for cos, o, na, nb in rows[:12]:
|
||||
va = gmem.primary_va(o)
|
||||
print(f" va {va:#010x} cos={cos:+.3f} |wL|={na:.3f} |wR|={nb:.3f}")
|
||||
if not rows:
|
||||
print(" (none)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
146
tools/re-capture/findself.py
Normal file
146
tools/re-capture/findself.py
Normal file
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Identify which moving object in RAM is the *player* craft, by input correlation.
|
||||
|
||||
Position triples are easy to find (findplayer.py); saying which one is us is the
|
||||
hard part, and no static property answers it. This does: hold hard-left yaw for
|
||||
a few seconds, then hard-right, and look for the object whose turn axis reverses
|
||||
in phase with the stick. Every other craft is AI-flown and uncorrelated with our
|
||||
input, so the discriminator is causal rather than circumstantial.
|
||||
|
||||
Phase 1 finds candidate triples from two whole-RAM samples; after that only the
|
||||
candidates' 12 bytes are re-read, so the sampling loop is cheap.
|
||||
|
||||
Usage: findself.py [phase_seconds] [hz]
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
|
||||
FIFO = "/tmp/sylph-vgamepad.fifo"
|
||||
|
||||
|
||||
def pad(line):
|
||||
with open(FIFO, "w") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def snapshot(fd, size):
|
||||
return [(a, np.frombuffer(os.pread(fd, (b - a) // 4 * 4, a), dtype=">f4"))
|
||||
for a, b in gmem.extents(fd, size) if (b - a) >= 64]
|
||||
|
||||
|
||||
def candidates(fd, size, dt=0.35):
|
||||
s0 = snapshot(fd, size)
|
||||
t0 = time.time()
|
||||
time.sleep(dt)
|
||||
s1 = snapshot(fd, size)
|
||||
t1 = time.time()
|
||||
out = []
|
||||
for (o, a), (o2, b) in zip(s0, s1):
|
||||
if o != o2 or len(a) != len(b):
|
||||
continue
|
||||
with np.errstate(invalid="ignore"):
|
||||
af = np.nan_to_num(a.astype(np.float64), nan=0, posinf=0, neginf=0)
|
||||
bf = np.nan_to_num(b.astype(np.float64), nan=0, posinf=0, neginf=0)
|
||||
d = bf - af
|
||||
idx = np.flatnonzero(np.abs(d) > 1e-3)
|
||||
cand = idx[np.isin(idx + 1, idx) & np.isin(idx + 2, idx)]
|
||||
for i in cand:
|
||||
v = np.array([d[i], d[i + 1], d[i + 2]])
|
||||
sp = float(np.linalg.norm(v)) / (t1 - t0)
|
||||
if 80.0 < sp < 2000.0:
|
||||
out.append(o + int(i) * 4)
|
||||
return out
|
||||
|
||||
|
||||
def sample(fd, offs):
|
||||
t = time.time()
|
||||
pos = np.empty((len(offs), 3))
|
||||
for k, o in enumerate(offs):
|
||||
b = os.pread(fd, 12, o)
|
||||
if len(b) < 12:
|
||||
pos[k] = np.nan
|
||||
continue
|
||||
pos[k] = struct.unpack(">3f", b)
|
||||
return t, pos
|
||||
|
||||
|
||||
def turn_axis(series):
|
||||
"""Mean normalised cross(v_k, v_k+1) over a phase, per candidate."""
|
||||
ts = [t for t, _ in series]
|
||||
ps = [p for _, p in series]
|
||||
vs = []
|
||||
for k in range(len(ps) - 1):
|
||||
dt = ts[k + 1] - ts[k]
|
||||
vs.append((ps[k + 1] - ps[k]) / max(dt, 1e-3))
|
||||
ax = np.zeros((ps[0].shape[0], 3))
|
||||
n = 0
|
||||
for k in range(len(vs) - 1):
|
||||
c = np.cross(vs[k], vs[k + 1])
|
||||
nrm = np.linalg.norm(c, axis=1, keepdims=True)
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
ax += np.where(nrm > 1e-6, c / nrm, 0.0)
|
||||
n += 1
|
||||
return ax / max(n, 1)
|
||||
|
||||
|
||||
def main():
|
||||
secs = float(sys.argv[1]) if len(sys.argv) > 1 else 3.0
|
||||
hz = float(sys.argv[2]) if len(sys.argv) > 2 else 6.0
|
||||
path = gmem.mem_path()
|
||||
fd = os.open(path, os.O_RDONLY)
|
||||
size = os.path.getsize(path)
|
||||
|
||||
pad("reset")
|
||||
time.sleep(0.5)
|
||||
offs = candidates(fd, size)
|
||||
print(f"# {len(offs)} moving-triple candidates", flush=True)
|
||||
if not offs:
|
||||
sys.exit("nothing is moving — in flight?")
|
||||
|
||||
phases = {}
|
||||
for name, lx in (("left", -0.9), ("right", 0.9)):
|
||||
pad(f"axis LX {lx}")
|
||||
time.sleep(0.6) # let the turn establish
|
||||
series = []
|
||||
n = int(secs * hz)
|
||||
for _ in range(n):
|
||||
t0 = time.time()
|
||||
series.append(sample(fd, offs))
|
||||
time.sleep(max(0, 1.0 / hz - (time.time() - t0)))
|
||||
phases[name] = turn_axis(series)
|
||||
pad("axis LX 0")
|
||||
time.sleep(1.2)
|
||||
print(f"# phase {name} done", flush=True)
|
||||
pad("reset")
|
||||
|
||||
aL, aR = phases["left"], phases["right"]
|
||||
nL = np.linalg.norm(aL, axis=1)
|
||||
nR = np.linalg.norm(aR, axis=1)
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
cos = np.sum(aL * aR, axis=1) / (nL * nR)
|
||||
good = np.isfinite(cos) & (nL > 0.5) & (nR > 0.5)
|
||||
order = np.argsort(np.where(good, cos, 9e9))
|
||||
print("\n# objects whose turn axis REVERSES with the stick (most negative first)")
|
||||
shown = 0
|
||||
for k in order:
|
||||
if not good[k]:
|
||||
break
|
||||
print(f" va {gmem.primary_va(offs[k]):#010x} cos(axisL,axisR) = {cos[k]:+.3f}"
|
||||
f" |L|={nL[k]:.2f} |R|={nR[k]:.2f}")
|
||||
shown += 1
|
||||
if shown >= 12:
|
||||
break
|
||||
if not shown:
|
||||
print(" (none — try longer phases)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
115
tools/re-capture/findspeed.py
Normal file
115
tools/re-capture/findspeed.py
Normal file
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Locate the player's flight object via its speed scalar, then its position.
|
||||
|
||||
The HUD prints the craft's speed, so it is a known quantity we can *change on
|
||||
demand* — the classic two-state value scan, which is far more reliable than
|
||||
hunting for a structure by shape:
|
||||
|
||||
1. coast -> RAM holds the cruise speed somewhere; collect every float ≈ it;
|
||||
2. boost -> the real one rises; everything coincidental is filtered out;
|
||||
3. coast -> it must come back down.
|
||||
|
||||
Whatever survives all three is the player's speed. Its object then contains the
|
||||
position, which is confirmed independently: a position triple near that scalar
|
||||
must move at exactly the speed the scalar reports.
|
||||
|
||||
Usage: findspeed.py <cruise> [boost_wait]
|
||||
"""
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
|
||||
FIFO = "/tmp/sylph-vgamepad.fifo"
|
||||
|
||||
|
||||
def pad(line):
|
||||
with open(FIFO, "w") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def extents_arrays(fd, size):
|
||||
for a, b in gmem.extents(fd, size):
|
||||
n = (b - a) // 4 * 4
|
||||
if n >= 64:
|
||||
yield a, np.frombuffer(os.pread(fd, n, a), dtype=">f4")
|
||||
|
||||
|
||||
def scan_equal(fd, size, val, tol):
|
||||
hits = []
|
||||
for a, arr in extents_arrays(fd, size):
|
||||
with np.errstate(invalid="ignore"):
|
||||
ok = np.isfinite(arr) & (np.abs(arr.astype(np.float64) - val) <= tol)
|
||||
for i in np.flatnonzero(ok):
|
||||
hits.append(a + int(i) * 4)
|
||||
return hits
|
||||
|
||||
|
||||
def read_f(fd, off):
|
||||
b = os.pread(fd, 4, off)
|
||||
return struct.unpack(">f", b)[0] if len(b) == 4 else float("nan")
|
||||
|
||||
|
||||
def main():
|
||||
cruise = float(sys.argv[1])
|
||||
wait = float(sys.argv[2]) if len(sys.argv) > 2 else 4.0
|
||||
fd = os.open(gmem.mem_path(), os.O_RDONLY)
|
||||
size = os.path.getsize(gmem.mem_path())
|
||||
|
||||
pad("reset")
|
||||
time.sleep(2.0)
|
||||
c0 = scan_equal(fd, size, cruise, 2.0)
|
||||
print(f"# pass 1 (coast {cruise}): {len(c0)} candidates", flush=True)
|
||||
|
||||
pad("trig RT 1.0")
|
||||
time.sleep(wait)
|
||||
c1 = [o for o in c0 if read_f(fd, o) > cruise + 40]
|
||||
print(f"# pass 2 (boost): {len(c1)} rose above {cruise+40:.0f}", flush=True)
|
||||
vals = {o: read_f(fd, o) for o in c1}
|
||||
|
||||
pad("reset")
|
||||
time.sleep(wait + 2.0)
|
||||
c2 = [o for o in c1 if abs(read_f(fd, o) - cruise) <= 6.0]
|
||||
print(f"# pass 3 (coast again): {len(c2)} returned to ≈{cruise}", flush=True)
|
||||
for o in c2[:20]:
|
||||
print(f" va {gmem.primary_va(o):#010x} boost value was {vals[o]:.1f}")
|
||||
|
||||
if not c2:
|
||||
print("# no survivors — is the craft actually accelerating?")
|
||||
return
|
||||
|
||||
# position triple in the same object: must move at exactly that speed
|
||||
print("\n# looking for a position triple near each survivor", flush=True)
|
||||
RAD = 0x400
|
||||
for o in c2[:8]:
|
||||
lo = max(0, o - RAD)
|
||||
n = RAD * 2
|
||||
t0 = time.time()
|
||||
a0 = np.frombuffer(os.pread(fd, n, lo), dtype=">f4").astype(np.float64)
|
||||
time.sleep(0.4)
|
||||
t1 = time.time()
|
||||
a1 = np.frombuffer(os.pread(fd, n, lo), dtype=">f4").astype(np.float64)
|
||||
sp_now = read_f(fd, o)
|
||||
dt = t1 - t0
|
||||
best = []
|
||||
for i in range(len(a0) - 2):
|
||||
d = a1[i:i + 3] - a0[i:i + 3]
|
||||
if not np.all(np.isfinite(d)):
|
||||
continue
|
||||
v = float(np.linalg.norm(d)) / dt
|
||||
if abs(v - sp_now) <= max(8.0, 0.06 * sp_now):
|
||||
best.append((lo + i * 4, v, tuple(a0[i:i + 3])))
|
||||
print(f" survivor va {gmem.primary_va(o):#010x} (speed {sp_now:.1f}): "
|
||||
f"{len(best)} matching triples")
|
||||
for off, v, p in best[:4]:
|
||||
print(f" pos va {gmem.primary_va(off):#010x} delta-off "
|
||||
f"{off - o:+#07x} |v|={v:7.1f} ({p[0]:+9.1f},{p[1]:+9.1f},{p[2]:+9.1f})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
157
tools/re-capture/flight_analyze.py
Normal file
157
tools/re-capture/flight_analyze.py
Normal file
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Derive the player craft's live transform offsets from a flight_probe capture.
|
||||
|
||||
Nothing here is guessed from plausible-looking numbers. Two independent
|
||||
structural facts do the work:
|
||||
|
||||
1. a rotation matrix is 9 floats that are orthonormal with det +1 — a
|
||||
property essentially no other data has;
|
||||
2. a position triple's frame-to-frame delta must point along the craft's
|
||||
own forward axis when it is flying straight, and its magnitude must be the
|
||||
same for every frame at constant speed.
|
||||
|
||||
Test 2 validates the position candidate *and* the rotation candidate against
|
||||
each other, so a coincidence has to satisfy both at once.
|
||||
|
||||
Usage: flight_analyze.py <probe.bin> [name-substring]
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gworld # noqa: E402
|
||||
|
||||
|
||||
def load(path):
|
||||
f = open(path, "rb")
|
||||
assert f.read(8) == b"SYLPHPRB"
|
||||
n_inst, window, n_in = struct.unpack("<III", f.read(12))
|
||||
insts = []
|
||||
for _ in range(n_inst):
|
||||
va, off = struct.unpack("<IQ", f.read(12))
|
||||
nm = f.read(64).split(b"\0")[0].decode()
|
||||
insts.append((va, off, nm))
|
||||
frames = []
|
||||
rec = 8 + 4 * n_in + 32 + n_inst * window
|
||||
while True:
|
||||
b = f.read(rec)
|
||||
if len(b) < rec:
|
||||
break
|
||||
t = struct.unpack_from("<d", b, 0)[0]
|
||||
inp = struct.unpack_from("<%df" % n_in, b, 8)
|
||||
label = struct.unpack_from("<32s", b, 8 + 4 * n_in)[0].split(b"\0")[0].decode()
|
||||
base = 8 + 4 * n_in + 32
|
||||
bufs = [b[base + i * window: base + (i + 1) * window] for i in range(n_inst)]
|
||||
frames.append((t, inp, label, bufs))
|
||||
return insts, window, frames
|
||||
|
||||
|
||||
def f32(buf, o):
|
||||
return struct.unpack_from(">f", buf, o)[0]
|
||||
|
||||
|
||||
def vec(buf, o):
|
||||
return (f32(buf, o), f32(buf, o + 4), f32(buf, o + 8))
|
||||
|
||||
|
||||
def sub(a, b):
|
||||
return (a[0] - b[0], a[1] - b[1], a[2] - b[2])
|
||||
|
||||
|
||||
def norm(a):
|
||||
return math.sqrt(sum(c * c for c in a))
|
||||
|
||||
|
||||
def dot(a, b):
|
||||
return sum(x * y for x, y in zip(a, b))
|
||||
|
||||
|
||||
def analyse(insts, window, frames, want):
|
||||
idx = [i for i, (_, _, nm) in enumerate(insts) if want in nm]
|
||||
if not idx:
|
||||
sys.exit(f"no instance matching {want!r}; have: "
|
||||
+ ", ".join(sorted({nm for _, _, nm in insts})))
|
||||
i = idx[0]
|
||||
va, off, nm = insts[i]
|
||||
print(f"# player instance {va:#010x} {nm} ({len(frames)} frames)")
|
||||
|
||||
# ---- rotation blocks that hold up across every frame ------------------
|
||||
per_frame = []
|
||||
for t, inp, lab, bufs in frames:
|
||||
per_frame.append({o for o, kind, m in gworld.find_rotations(bufs[i]) if kind == "3x3"})
|
||||
stable = set.intersection(*per_frame) if per_frame else set()
|
||||
print(f"# 3x3 orthonormal blocks valid in ALL frames: "
|
||||
+ (", ".join(f"{o:#05x}" for o in sorted(stable)) or "none"))
|
||||
|
||||
# ---- position candidates ---------------------------------------------
|
||||
# constant-speed straight flight: |delta| equal every frame, direction fixed
|
||||
idle = [k for k, (t, inp, lab, b) in enumerate(frames) if lab.startswith("idle")]
|
||||
if len(idle) < 6:
|
||||
idle = list(range(min(20, len(frames))))
|
||||
seg = idle[:20]
|
||||
cands = []
|
||||
for o in range(0, window - 12, 4):
|
||||
ds, ok = [], True
|
||||
for a, b in zip(seg, seg[1:]):
|
||||
if b != a + 1:
|
||||
continue
|
||||
dt = frames[b][0] - frames[a][0]
|
||||
p0, p1 = vec(frames[a][3][i], o), vec(frames[b][3][i], o)
|
||||
if not all(map(math.isfinite, p0 + p1)):
|
||||
ok = False
|
||||
break
|
||||
d = sub(p1, p0)
|
||||
if dt <= 0:
|
||||
ok = False
|
||||
break
|
||||
ds.append((norm(d) / dt, d))
|
||||
if not ok or len(ds) < 4:
|
||||
continue
|
||||
sp = [s for s, _ in ds]
|
||||
mean = sum(sp) / len(sp)
|
||||
if mean < 1.0 or mean > 5000.0: # a craft, not a counter
|
||||
continue
|
||||
spread = max(sp) - min(sp)
|
||||
if spread > 0.15 * mean: # constant speed while coasting
|
||||
continue
|
||||
# direction must be steady too
|
||||
dirs = [(d[0] / (norm(d) or 1), d[1] / (norm(d) or 1), d[2] / (norm(d) or 1))
|
||||
for _, d in ds]
|
||||
if min(dot(dirs[0], d) for d in dirs) < 0.99:
|
||||
continue
|
||||
cands.append((o, mean, dirs[0]))
|
||||
|
||||
print(f"# position candidates (steady speed + steady heading while coasting): "
|
||||
f"{len(cands)}")
|
||||
for o, sp, d in cands[:12]:
|
||||
print(f" +{o:#05x} speed {sp:8.2f}/s dir ({d[0]:+.3f},{d[1]:+.3f},{d[2]:+.3f})")
|
||||
|
||||
# ---- cross-validate: velocity must lie along a rotation row -----------
|
||||
print("\n# cross-check: does the motion direction match a rotation-matrix axis?")
|
||||
best = []
|
||||
for o, sp, d in cands:
|
||||
for ro in sorted(stable):
|
||||
m = struct.unpack_from(">9f", frames[seg[0]][3][i], ro)
|
||||
for r, label in ((m[0:3], "row0/right"), (m[3:6], "row1/up"), (m[6:9], "row2/fwd")):
|
||||
c = abs(dot(d, r))
|
||||
if c > 0.98:
|
||||
best.append((c, o, ro, label, sp))
|
||||
best.sort(reverse=True)
|
||||
if not best:
|
||||
print(" none — position and orientation candidates do not corroborate")
|
||||
for c, o, ro, label, sp in best[:10]:
|
||||
print(f" pos +{o:#05x} ∥ rot +{ro:#05x} {label} |cos|={c:.5f} speed {sp:.1f}")
|
||||
return stable, cands
|
||||
|
||||
|
||||
def main():
|
||||
path = sys.argv[1]
|
||||
want = sys.argv[2] if len(sys.argv) > 2 else "Player"
|
||||
insts, window, frames = load(path)
|
||||
analyse(insts, window, frames, want)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
107
tools/re-capture/flight_probe.py
Executable file
107
tools/re-capture/flight_probe.py
Executable file
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Drive a scripted input sequence in flight while sampling guest RAM, so the
|
||||
player craft's live transform can be *derived* rather than guessed.
|
||||
|
||||
The point is correlation: each sample is tagged with the stick/trigger state
|
||||
that produced it, so afterwards we can ask "which floats move only while the
|
||||
yaw axis is deflected?" and "which triple integrates to the velocity?".
|
||||
|
||||
Writes a .npz-ish flat binary: a header of (va, offset, name) per instance,
|
||||
then per frame a timestamp, the input vector, and every instance's raw window.
|
||||
|
||||
Usage: flight_probe.py <out.bin> [seconds]
|
||||
"""
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gworld # noqa: E402
|
||||
|
||||
FIFO = "/tmp/sylph-vgamepad.fifo"
|
||||
|
||||
|
||||
class Pad:
|
||||
"""Talk to the vgamepad server directly over its FIFO.
|
||||
|
||||
The `vgamepad` CLI spawns a process per command (~20 ms); a control loop
|
||||
cannot afford that, and `tap`/`hold` additionally sleep *inside* the server.
|
||||
Writing lines to the FIFO ourselves keeps a tick under a millisecond.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.f = open(FIFO, "w", buffering=1)
|
||||
self.state = {"LX": 0.0, "LY": 0.0, "RX": 0.0, "RY": 0.0, "LT": 0.0, "RT": 0.0}
|
||||
|
||||
def axis(self, name, v):
|
||||
self.state[name] = v
|
||||
self.f.write(f"axis {name} {v:.3f}\n")
|
||||
|
||||
def trig(self, name, v):
|
||||
self.state[name] = v
|
||||
self.f.write(f"trig {name} {v:.3f}\n")
|
||||
|
||||
def press(self, b):
|
||||
self.f.write(f"press {b}\n")
|
||||
|
||||
def release(self, b):
|
||||
self.f.write(f"release {b}\n")
|
||||
|
||||
def reset(self):
|
||||
self.f.write("reset\n")
|
||||
for k in self.state:
|
||||
self.state[k] = 0.0
|
||||
|
||||
def vector(self):
|
||||
return [self.state[k] for k in ("LX", "LY", "RX", "RY", "LT", "RT")]
|
||||
|
||||
|
||||
# (duration_s, description, action)
|
||||
SCRIPT = [
|
||||
(3.0, "idle", lambda p: p.reset()),
|
||||
(4.0, "pitch-up", lambda p: p.axis("LY", -0.9)),
|
||||
(2.0, "idle2", lambda p: p.reset()),
|
||||
(4.0, "yaw-right", lambda p: p.axis("LX", 0.9)),
|
||||
(2.0, "idle3", lambda p: p.reset()),
|
||||
(4.0, "yaw-left", lambda p: p.axis("LX", -0.9)),
|
||||
(2.0, "idle4", lambda p: p.reset()),
|
||||
(5.0, "boost", lambda p: p.trig("RT", 1.0)),
|
||||
(3.0, "idle5", lambda p: p.reset()),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
out = sys.argv[1]
|
||||
w = gworld.World()
|
||||
inst = w.refresh()
|
||||
print(f"# {len(inst)} instances", flush=True)
|
||||
if not inst:
|
||||
sys.exit("no instances — not in a mission?")
|
||||
|
||||
pad = Pad()
|
||||
hz = 10.0
|
||||
with open(out, "wb") as f:
|
||||
f.write(b"SYLPHPRB")
|
||||
f.write(struct.pack("<III", len(inst), gworld.WINDOW, 6))
|
||||
for va, off, nm in inst:
|
||||
f.write(struct.pack("<IQ", va, off) + nm.encode()[:63].ljust(64, b"\0"))
|
||||
t_start = time.time()
|
||||
for dur, label, act in SCRIPT:
|
||||
act(pad)
|
||||
print(f"# {label} ({dur}s)", flush=True)
|
||||
n = int(dur * hz)
|
||||
for _ in range(n):
|
||||
t = time.time()
|
||||
f.write(struct.pack("<d", t - t_start))
|
||||
f.write(struct.pack("<6f", *pad.vector()))
|
||||
f.write(struct.pack("<32s", label.encode()[:32]))
|
||||
for va, off, nm in inst:
|
||||
f.write(w.read_off(off, gworld.WINDOW).ljust(gworld.WINDOW, b"\0"))
|
||||
time.sleep(max(0, 1.0 / hz - (time.time() - t)))
|
||||
pad.reset()
|
||||
print(f"# wrote {out} ({os.path.getsize(out)/1e6:.1f} MB)", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
27
tools/re-capture/fly_session.sh
Executable file
27
tools/re-capture/fly_session.sh
Executable file
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
# One background task: boot -> mission -> bind the transform -> fly the pilot,
|
||||
# with periodic screenshots so the outcome has visual evidence and not just a log.
|
||||
#
|
||||
# It is one task on purpose: the display and the emulator both die on their own
|
||||
# every few minutes in this container, so nothing may depend on surviving
|
||||
# between tool calls.
|
||||
set -u
|
||||
export HOME=/sylph-home/re SDL_AUDIODRIVER=dummy DISPLAY=:98
|
||||
export PYTHONPATH=/sylph-home/.local/lib/python3.12/site-packages
|
||||
SD="$(cd "$(dirname "$0")" && pwd)"
|
||||
SECS="${1:-240}"
|
||||
TAG="${2:-pilot}"
|
||||
SHOTS=/sylph-home/re/shots
|
||||
CFG=/tmp/nav-live.json
|
||||
|
||||
"$SD/launch_mission.sh" fly || { echo "BOOT FAILED"; exit 1; }
|
||||
python3 "$SD/entities2.py" self 0x130 "$CFG" || { echo "BIND FAILED"; exit 1; }
|
||||
echo "--- config: $(cat "$CFG")"
|
||||
python3 "$SD/own_state.py" "$CFG" 3 "/tmp/$TAG-own.json"
|
||||
|
||||
( for i in $(seq 1 12); do sleep 25; screenshot "$SHOTS/$TAG-$i.png" >/dev/null 2>&1; done ) &
|
||||
SHOTTER=$!
|
||||
python3 "$SD/pilot.py" "$CFG" "$SECS"
|
||||
kill $SHOTTER 2>/dev/null
|
||||
screenshot "$SHOTS/$TAG-end.png" >/dev/null 2>&1
|
||||
echo "SESSION DONE"
|
||||
168
tools/re-capture/gmem.py
Normal file
168
tools/re-capture/gmem.py
Normal file
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read the *live* guest memory of a running Xenia Canary.
|
||||
|
||||
Canary backs the whole guest address space with a single shared-memory file,
|
||||
`/dev/shm/xenia_memory_<pid-ish>`, so the guest's RAM is readable from the host
|
||||
with no debugger, no emulator patch and no pause: open the file, seek, read.
|
||||
|
||||
The file is one flat image of Xenia's *physical* backing store; guest virtual
|
||||
addresses map into it through Xenia's fixed table (memory.cc `map_info`).
|
||||
`va_to_off()` implements that table, so callers work in guest VAs.
|
||||
|
||||
Sub-commands
|
||||
find <pattern> search every allocated extent, print guest VAs
|
||||
read <va> [len] hexdump guest memory
|
||||
words <va> [n] dump n big-endian u32 / f32 pairs
|
||||
|
||||
`<pattern>` is a python literal-ish string: plain text, or `hex:0011aabb`.
|
||||
Numbers accept 0x form. The scan uses SEEK_DATA so the ~4.6 GB of sparse holes
|
||||
cost nothing.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import struct
|
||||
|
||||
# (guest_va_lo, guest_va_hi_inclusive, file_offset_of_lo) — Xenia memory.cc.
|
||||
MAP = [
|
||||
(0x00000000, 0x3FFFFFFF, 0x00000000),
|
||||
(0x40000000, 0x7EFFFFFF, 0x40000000),
|
||||
(0x7F000000, 0x7F0FFFFF, 0x00000000),
|
||||
(0x7F100000, 0x7FFFFFFF, 0x00100000),
|
||||
(0x80000000, 0x8FFFFFFF, 0x80000000),
|
||||
(0x90000000, 0x9FFFFFFF, 0x80000000),
|
||||
(0xA0000000, 0xBFFFFFFF, 0x100000000),
|
||||
(0xC0000000, 0xDFFFFFFF, 0x100000000),
|
||||
(0xE0000000, 0xFFFFFFFF, 0x100000000),
|
||||
]
|
||||
|
||||
|
||||
def va_to_off(va):
|
||||
for lo, hi, base in MAP:
|
||||
if lo <= va <= hi:
|
||||
return base + (va - lo)
|
||||
raise ValueError(f"va {va:#x} outside the guest map")
|
||||
|
||||
|
||||
def off_to_vas(off):
|
||||
"""All guest VAs that alias this file offset (the map is many-to-one)."""
|
||||
out = []
|
||||
for lo, hi, base in MAP:
|
||||
if base <= off <= base + (hi - lo):
|
||||
out.append(lo + (off - base))
|
||||
return out
|
||||
|
||||
|
||||
def primary_va(off):
|
||||
"""The most useful VA for an offset: physical 0xA0000000+ / xex 0x80000000+."""
|
||||
vas = off_to_vas(off)
|
||||
return vas[0] if vas else None
|
||||
|
||||
|
||||
def mem_path():
|
||||
# A snapshot (`cp --sparse=always /dev/shm/xenia_memory_* snap.bin`, ~2 s)
|
||||
# reads identically and does not contend with the running emulator, which
|
||||
# pegs every core under lavapipe. Point $GMEM_FILE at one to work offline.
|
||||
env = os.environ.get("GMEM_FILE")
|
||||
if env:
|
||||
return env
|
||||
if len(sys.argv) > 1 and sys.argv[1].startswith("/dev/shm/"):
|
||||
return sys.argv.pop(1)
|
||||
cands = [f"/dev/shm/{n}" for n in os.listdir("/dev/shm") if n.startswith("xenia_memory_")]
|
||||
if not cands:
|
||||
sys.exit("no /dev/shm/xenia_memory_* — is Canary running?")
|
||||
if len(cands) > 1:
|
||||
sys.exit(f"several memory files, pass one explicitly: {cands}")
|
||||
return cands[0]
|
||||
|
||||
|
||||
def extents(fd, size):
|
||||
"""Yield (start, end) of the file's allocated (non-hole) ranges."""
|
||||
pos = 0
|
||||
while pos < size:
|
||||
try:
|
||||
data = os.lseek(fd, pos, os.SEEK_DATA)
|
||||
except OSError:
|
||||
return
|
||||
try:
|
||||
hole = os.lseek(fd, data, os.SEEK_HOLE)
|
||||
except OSError:
|
||||
hole = size
|
||||
yield (data, hole)
|
||||
pos = hole
|
||||
|
||||
|
||||
def parse_pattern(s):
|
||||
if s.startswith("hex:"):
|
||||
return bytes.fromhex(s[4:])
|
||||
return s.encode("latin-1")
|
||||
|
||||
|
||||
def cmd_find(f, size, args):
|
||||
pat = parse_pattern(args[0])
|
||||
limit = int(args[1]) if len(args) > 1 else 64
|
||||
hits = 0
|
||||
CHUNK = 1 << 24
|
||||
for start, end in extents(f.fileno(), size):
|
||||
pos = start
|
||||
carry = b""
|
||||
carry_at = start
|
||||
while pos < end:
|
||||
f.seek(pos)
|
||||
buf = f.read(min(CHUNK, end - pos))
|
||||
if not buf:
|
||||
break
|
||||
blob = carry + buf
|
||||
base = carry_at
|
||||
for m in re.finditer(re.escape(pat), blob):
|
||||
off = base + m.start()
|
||||
va = primary_va(off)
|
||||
print(f"{off:#013x} va {va:#010x}" if va is not None else f"{off:#013x} va ?")
|
||||
hits += 1
|
||||
if hits >= limit:
|
||||
return
|
||||
keep = len(pat) - 1
|
||||
carry = blob[-keep:] if keep else b""
|
||||
carry_at = base + len(blob) - len(carry)
|
||||
pos += len(buf)
|
||||
if hits == 0:
|
||||
print("(no hits)", file=sys.stderr)
|
||||
|
||||
|
||||
def cmd_read(f, size, args):
|
||||
va = int(args[0], 0)
|
||||
n = int(args[1], 0) if len(args) > 1 else 256
|
||||
f.seek(va_to_off(va))
|
||||
data = f.read(n)
|
||||
for i in range(0, len(data), 16):
|
||||
row = data[i : i + 16]
|
||||
hexs = " ".join(f"{b:02x}" for b in row)
|
||||
txt = "".join(chr(b) if 0x20 <= b < 0x7F else "." for b in row)
|
||||
print(f"{va + i:08x} {hexs:<47} |{txt}|")
|
||||
|
||||
|
||||
def cmd_words(f, size, args):
|
||||
va = int(args[0], 0)
|
||||
n = int(args[1], 0) if len(args) > 1 else 32
|
||||
f.seek(va_to_off(va))
|
||||
data = f.read(n * 4)
|
||||
for i in range(0, len(data) - 3, 4):
|
||||
(u,) = struct.unpack_from(">I", data, i)
|
||||
(fl,) = struct.unpack_from(">f", data, i)
|
||||
fs = f"{fl:.6g}" if -1e30 < fl < 1e30 else ""
|
||||
print(f"{va + i:08x} +{i:04x} {u:#010x} {u:>12} {fs}")
|
||||
|
||||
|
||||
def main():
|
||||
path = mem_path()
|
||||
if len(sys.argv) < 2:
|
||||
sys.exit(__doc__)
|
||||
cmd, args = sys.argv[1], sys.argv[2:]
|
||||
size = os.path.getsize(path)
|
||||
with open(path, "rb") as f:
|
||||
{"find": cmd_find, "read": cmd_read, "words": cmd_words}[cmd](f, size, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
70
tools/re-capture/grab_tutorial.sh
Executable file
70
tools/re-capture/grab_tutorial.sh
Executable file
@@ -0,0 +1,70 @@
|
||||
#!/usr/bin/env bash
|
||||
# Capture one tutorial's unit definitions: cold-boot Canary, walk
|
||||
# title -> TUTORIAL -> the Nth entry, wait for the stage to load, snapshot RAM.
|
||||
#
|
||||
# One emulator per tutorial on purpose: backing out of a loaded mission via
|
||||
# PAUSE -> BACK TO MENU wedged the emulator (log stops, CPU spins), while a
|
||||
# cold boot here is ~15 s. The unit definitions are all parsed at stage load,
|
||||
# so a single snapshot right after the load is everything this needs.
|
||||
set -u
|
||||
N="${1:?usage: grab_tutorial.sh <index 0..5> <outfile>}"
|
||||
OUT="${2:?}"
|
||||
export HOME=/sylph-home/re SDL_AUDIODRIVER=dummy DISPLAY=:98
|
||||
SD="$(cd "$(dirname "$0")" && pwd)"
|
||||
RC="/home/fabi/RE - Project Sylpheed/sylpheed-reborn/tools/re-capture"
|
||||
|
||||
# The container entrypoint's Xvfb/openbox do NOT restart on their own, and a
|
||||
# dead Xvfb leaves a <defunct> entry that still matches `pgrep` -- so test the
|
||||
# display with xdpyinfo, never by process name.
|
||||
ensure_display(){
|
||||
if ! xdpyinfo -display "$DISPLAY" >/dev/null 2>&1; then
|
||||
echo "[grab] $DISPLAY is down -> restarting Xvfb + openbox"
|
||||
setsid nohup Xvfb "$DISPLAY" -screen 0 1280x720x24 -ac -nolisten tcp \
|
||||
+extension GLX +extension RANDR </dev/null >/tmp/xvfb98.log 2>&1 &
|
||||
for _ in $(seq 1 50); do xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 && break; sleep 0.2; done
|
||||
setsid nohup env DISPLAY="$DISPLAY" HOME=/sylph-home openbox </dev/null >/tmp/openbox98.log 2>&1 &
|
||||
sleep 1
|
||||
fi
|
||||
xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 || { echo "DISPLAY UNAVAILABLE"; exit 1; }
|
||||
}
|
||||
|
||||
step(){ vgamepad dpad "$1"; sleep 0.2; vgamepad dpad center; sleep 0.6; }
|
||||
shot(){ screenshot "$SD/$1" >/dev/null 2>&1; }
|
||||
|
||||
# Container PID 1 is `sleep infinity`, which never reaps, so every killed
|
||||
# emulator stays as a <defunct> entry that `pgrep` still matches. Ask ps for the
|
||||
# state and ignore anything zombie, or this always thinks one is running.
|
||||
alive(){ ps -o pid=,stat= -C xenia_canary 2>/dev/null | awk '$2 !~ /^Z/ {print $1}'; }
|
||||
pkill -x xenia_canary 2>/dev/null; sleep 2
|
||||
[ -n "$(alive)" ] && { kill -9 $(alive) 2>/dev/null; sleep 2; }
|
||||
rm -f /dev/shm/xenia_memory_* /dev/shm/xenia_code_cache_* 2>/dev/null
|
||||
|
||||
ensure_display
|
||||
|
||||
cd /sylph-home/re
|
||||
setsid nohup run-canary --audio --apu=sdl --log_mask=13 \
|
||||
--logged_profile_slot_0_xuid=E0300000EFBEA3D4 </dev/null >/dev/null 2>&1 &
|
||||
sleep 5
|
||||
"$RC/skip_intro.sh" 600 || { echo "BOOT FAILED"; exit 1; }
|
||||
# The main menu is not input-ready for ~10 s after the title tap under lavapipe;
|
||||
# d-pad presses before that are silently dropped, and the A then lands on NEW
|
||||
# GAME instead of TUTORIAL. This wait is the whole fix.
|
||||
sleep 14
|
||||
shot "nav-$N-menu.png"
|
||||
|
||||
step down; step down # NEW GAME -> TUTORIAL
|
||||
vgamepad tap A 250; sleep 8
|
||||
shot "nav-$N-list.png"
|
||||
|
||||
i=0; while [ "$i" -lt "$N" ]; do step down; i=$((i+1)); done
|
||||
shot "nav-$N-pick.png"
|
||||
vgamepad tap A 250
|
||||
|
||||
# the stage load takes ~25-45 s under lavapipe; wait for the screen to settle
|
||||
sleep 50
|
||||
shot "nav-$N-loaded.png"
|
||||
|
||||
MEM=$(ls /dev/shm/xenia_memory_* 2>/dev/null | head -1)
|
||||
[ -n "$MEM" ] || { echo "NO SHM"; exit 1; }
|
||||
cp --sparse=always "$MEM" "$SD/$OUT" || exit 1
|
||||
echo "SNAPSHOT $OUT ($(du -h "$SD/$OUT" | cut -f1))"
|
||||
203
tools/re-capture/gworld.py
Executable file
203
tools/re-capture/gworld.py
Executable file
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live world-state reader: the running game's entities, straight out of guest RAM.
|
||||
|
||||
Builds on gmem.py (Canary backs the guest address space with
|
||||
/dev/shm/xenia_memory_*, so it is an ordinary file). The difference here is that
|
||||
this is meant to run *in a loop* while the game plays, not against a snapshot:
|
||||
the memory file is opened once and re-read with pread, and the expensive
|
||||
whole-RAM vtable scan is done rarely and cached.
|
||||
|
||||
Two classes matter (see docs/re/structures/unit-struct-runtime.md):
|
||||
0x820af844 the parsed `.tbl` definition — one per unit type
|
||||
0x820af030 the spawned entity instance — one per thing in the scene
|
||||
|
||||
This module finds instances and reads their live transform. The transform
|
||||
offsets are NOT guessed: `find_transform` scans an instance for a 3x3 block of
|
||||
floats that is orthonormal to 1e-3 with det = +1, which a rotation matrix is and
|
||||
essentially nothing else is.
|
||||
|
||||
Sub-commands:
|
||||
entities list live instances (name, address)
|
||||
probe <seconds> [hz] [out] record every instance's raw bytes over time
|
||||
orient report orthonormal 3x3 blocks per instance
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
|
||||
INST_VTABLE = 0x820AF030
|
||||
DEF_VTABLE = 0x820AF844
|
||||
WINDOW = 0x600 # bytes of an instance we read; its real size is unknown
|
||||
NAME_STR_OFF = 0x10
|
||||
|
||||
|
||||
class World:
|
||||
def __init__(self, path=None):
|
||||
self.path = path or gmem.mem_path()
|
||||
self.fd = os.open(self.path, os.O_RDONLY)
|
||||
self.size = os.path.getsize(self.path)
|
||||
self.instances = [] # [(va, file_off, name)]
|
||||
|
||||
# ---------------------------------------------------------------- io
|
||||
def read(self, va, n):
|
||||
return os.pread(self.fd, n, gmem.va_to_off(va))
|
||||
|
||||
def read_off(self, off, n):
|
||||
return os.pread(self.fd, n, off)
|
||||
|
||||
def u32(self, va):
|
||||
return struct.unpack(">I", self.read(va, 4))[0]
|
||||
|
||||
def cstr(self, va, n=96):
|
||||
b = os.pread(self.fd, n, gmem.va_to_off(va)).split(b"\0")[0]
|
||||
try:
|
||||
return b.decode("ascii")
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------- scan
|
||||
def scan_vtable(self, vt):
|
||||
pat = struct.pack(">I", vt)
|
||||
hits = []
|
||||
for a, b in gmem.extents(self.fd, self.size):
|
||||
blob = os.pread(self.fd, b - a, a)
|
||||
start = 0
|
||||
while True:
|
||||
i = blob.find(pat, start)
|
||||
if i < 0:
|
||||
break
|
||||
off = a + i
|
||||
if off % 4 == 0:
|
||||
hits.append(off)
|
||||
start = i + 4
|
||||
return sorted(set(hits))
|
||||
|
||||
def name_of(self, off):
|
||||
"""The unit ID an object at `off` names, via its name-record pointer."""
|
||||
raw = self.read_off(off + 4, 4)
|
||||
if len(raw) < 4:
|
||||
return None
|
||||
(p,) = struct.unpack(">I", raw)
|
||||
try:
|
||||
return self.cstr(p + NAME_STR_OFF)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def refresh(self, vt=INST_VTABLE):
|
||||
"""Re-scan for live instances. Costs ~1 s; call it rarely."""
|
||||
out = []
|
||||
for off in self.scan_vtable(vt):
|
||||
nm = self.name_of(off)
|
||||
if nm and nm.startswith("UN_"):
|
||||
out.append((gmem.primary_va(off), off, nm))
|
||||
self.instances = out
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------- geometry
|
||||
|
||||
|
||||
def floats(buf, off, n):
|
||||
return struct.unpack_from(">" + "f" * n, buf, off)
|
||||
|
||||
|
||||
def is_rotation(m, tol=2e-3):
|
||||
"""m = 9 floats, row-major. Orthonormal rows + det +1?"""
|
||||
r = [m[0:3], m[3:6], m[6:9]]
|
||||
for row in r:
|
||||
n2 = sum(c * c for c in row)
|
||||
if abs(n2 - 1.0) > tol:
|
||||
return False
|
||||
for i in range(3):
|
||||
for j in range(i + 1, 3):
|
||||
if abs(sum(r[i][k] * r[j][k] for k in range(3))) > tol:
|
||||
return False
|
||||
det = (r[0][0] * (r[1][1] * r[2][2] - r[1][2] * r[2][1])
|
||||
- r[0][1] * (r[1][0] * r[2][2] - r[1][2] * r[2][0])
|
||||
+ r[0][2] * (r[1][0] * r[2][1] - r[1][1] * r[2][0]))
|
||||
return abs(det - 1.0) < 1e-2
|
||||
|
||||
|
||||
def find_rotations(buf, stride=4):
|
||||
"""Offsets where 9 consecutive floats form a rotation matrix.
|
||||
|
||||
Also tries the 4-float-per-row (3x4 / 4x4 matrix) stride, which is how a
|
||||
transform with a translation column is normally stored.
|
||||
"""
|
||||
out = []
|
||||
for off in range(0, len(buf) - 36, stride):
|
||||
m = floats(buf, off, 9)
|
||||
if all(abs(c) <= 1.001 for c in m) and is_rotation(m):
|
||||
out.append((off, "3x3", m))
|
||||
for off in range(0, len(buf) - 48, stride):
|
||||
rows = [floats(buf, off + 16 * i, 3) for i in range(3)]
|
||||
m = rows[0] + rows[1] + rows[2]
|
||||
if all(abs(c) <= 1.001 for c in m) and is_rotation(m):
|
||||
out.append((off, "4x4", m))
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ cmds
|
||||
|
||||
|
||||
def cmd_entities(w, args):
|
||||
t0 = time.time()
|
||||
inst = w.refresh()
|
||||
print(f"# {len(inst)} live instances (scan {time.time()-t0:.2f}s)")
|
||||
from collections import Counter
|
||||
c = Counter(n for _, _, n in inst)
|
||||
for nm, k in c.most_common():
|
||||
print(f" {k:4d} {nm}")
|
||||
|
||||
|
||||
def cmd_orient(w, args):
|
||||
inst = w.refresh()
|
||||
want = args[0] if args else None
|
||||
for va, off, nm in inst:
|
||||
if want and want not in nm:
|
||||
continue
|
||||
buf = w.read_off(off, WINDOW)
|
||||
rots = find_rotations(buf)
|
||||
print(f"\n{va:#010x} {nm} -> {len(rots)} rotation-like block(s)")
|
||||
for o, kind, m in rots[:6]:
|
||||
print(f" +{o:#05x} {kind} "
|
||||
+ " ".join(f"{v:+.3f}" for v in m))
|
||||
|
||||
|
||||
def cmd_probe(w, args):
|
||||
secs = float(args[0]) if args else 6.0
|
||||
hz = float(args[1]) if len(args) > 1 else 5.0
|
||||
out = args[2] if len(args) > 2 else "probe.bin"
|
||||
inst = w.refresh()
|
||||
print(f"# probing {len(inst)} instances for {secs}s at {hz}Hz -> {out}")
|
||||
with open(out, "wb") as f:
|
||||
f.write(struct.pack("<II", len(inst), WINDOW))
|
||||
for va, off, nm in inst:
|
||||
nb = nm.encode()[:63].ljust(64, b"\0")
|
||||
f.write(struct.pack("<II", va, off) + nb)
|
||||
n = int(secs * hz)
|
||||
for i in range(n):
|
||||
t = time.time()
|
||||
f.write(struct.pack("<d", t))
|
||||
for va, off, nm in inst:
|
||||
f.write(w.read_off(off, WINDOW).ljust(WINDOW, b"\0"))
|
||||
time.sleep(max(0, 1.0 / hz - (time.time() - t)))
|
||||
print(f"# wrote {n} frames")
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
sys.exit(__doc__)
|
||||
w = World()
|
||||
cmd, args = sys.argv[1], sys.argv[2:]
|
||||
{"entities": cmd_entities, "orient": cmd_orient, "probe": cmd_probe}[cmd](w, args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
75
tools/re-capture/launch_mission.sh
Executable file
75
tools/re-capture/launch_mission.sh
Executable file
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
# Boot Canary and drive it into the Stage 02 mission from save slot 01, then
|
||||
# LEAVE IT RUNNING (unlike grab_tutorial.sh, which snapshots and exits) so a
|
||||
# live control loop can attach to /dev/shm.
|
||||
#
|
||||
# Nav is the verified route: title -> LOAD GAME -> slot 01 -> YES -> READY ROOM
|
||||
# -> TAKE OFF. With `--hangar` it stops in the HANGAR instead so a loadout can
|
||||
# be picked first.
|
||||
set -u
|
||||
MODE="${1:-fly}"
|
||||
export HOME=/sylph-home/re SDL_AUDIODRIVER=dummy DISPLAY=:98
|
||||
SD="$(cd "$(dirname "$0")" && pwd)"
|
||||
SHOTS=/sylph-home/re/shots
|
||||
|
||||
alive(){ ps -o pid=,stat= -C xenia_canary 2>/dev/null | awk '$2 !~ /^Z/ {print $1}'; }
|
||||
|
||||
# DO NOT `setsid` THE DISPLAY OR THE EMULATOR. They used to be detached into
|
||||
# their own sessions so they would outlive the shell that started them. What
|
||||
# that actually bought was the opposite: a process nothing owns is a process
|
||||
# nothing keeps alive, and both were being reaped a couple of minutes in — the
|
||||
# long-standing "Xvfb and the emulator die on their own every few minutes" note.
|
||||
# Run this whole script as ONE tracked background task and leave Xvfb, openbox
|
||||
# and xenia as its children: they then live exactly as long as the session does.
|
||||
# `nohup` still shields them from a stray HUP; the exit-status wrapper means a
|
||||
# death is reported with the server's own account instead of being inferred.
|
||||
ensure_display(){
|
||||
if ! xdpyinfo -display "$DISPLAY" >/dev/null 2>&1; then
|
||||
rm -f "/tmp/.X${DISPLAY#:}-lock" 2>/dev/null || true
|
||||
nohup bash -c 'Xvfb "$0" -screen 0 1280x720x24 -ac -nolisten tcp \
|
||||
+extension GLX +extension RANDR >/tmp/xvfb98.log 2>&1
|
||||
echo "$(date +%T) XVFB EXIT $? (128+N means signal N)" >>/tmp/xvfb-exit.log' \
|
||||
"$DISPLAY" </dev/null >/dev/null 2>&1 &
|
||||
for _ in $(seq 1 50); do xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 && break; sleep 0.2; done
|
||||
nohup env DISPLAY="$DISPLAY" HOME=/sylph-home openbox </dev/null >/tmp/openbox98.log 2>&1 &
|
||||
sleep 1
|
||||
fi
|
||||
xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 || { echo "DISPLAY UNAVAILABLE"; exit 1; }
|
||||
}
|
||||
step(){ vgamepad dpad "$1"; sleep 0.25; vgamepad dpad center; sleep 0.7; }
|
||||
shot(){ screenshot "$SHOTS/$1" >/dev/null 2>&1; }
|
||||
|
||||
pkill -x xenia_canary 2>/dev/null; sleep 2
|
||||
[ -n "$(alive)" ] && { kill -9 $(alive) 2>/dev/null; sleep 2; }
|
||||
rm -f /dev/shm/xenia_memory_* /dev/shm/xenia_code_cache_* 2>/dev/null
|
||||
ensure_display
|
||||
|
||||
cd /sylph-home/re
|
||||
nohup run-canary --audio --apu=sdl --log_mask=13 \
|
||||
--logged_profile_slot_0_xuid=E0300000EFBEA3D4 </dev/null >/dev/null 2>&1 &
|
||||
sleep 5
|
||||
"$SD/skip_intro.sh" 600 || { echo "BOOT FAILED (skip_intro exit $?)"; exit 1; }
|
||||
sleep 14 # main menu is not input-ready before this
|
||||
|
||||
step down # NEW GAME -> LOAD GAME
|
||||
vgamepad tap A 250; sleep 8 # save list, slot 01 preselected
|
||||
vgamepad tap A 250; sleep 4 # "Load game?" -- cursor starts on NO
|
||||
step up
|
||||
vgamepad tap A 250
|
||||
sleep 28 # -> READY ROOM
|
||||
shot "lm-readyroom.png"
|
||||
|
||||
if [ "$MODE" = "--hangar" ]; then
|
||||
step down; step down # BRIEFINGS -> HANGAR
|
||||
vgamepad tap A 250; sleep 12
|
||||
shot "lm-hangar.png"
|
||||
echo "STOPPED IN HANGAR"; exit 0
|
||||
fi
|
||||
|
||||
step up # BRIEFINGS -> TAKE OFF
|
||||
vgamepad tap A 250
|
||||
# The launch cinematic + stage load + objective card is NOT a fixed 75 s under
|
||||
# lavapipe; wait for the flight HUD itself.
|
||||
"$SD/wait_flight.sh" 300 || { echo "NEVER REACHED FLIGHT"; exit 1; }
|
||||
shot "lm-flight.png"
|
||||
echo "IN FLIGHT (emulator left running)"
|
||||
105
tools/re-capture/liveents.py
Normal file
105
tools/re-capture/liveents.py
Normal file
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Type the live entities: link each moving position back to a unit definition.
|
||||
|
||||
findplayer.py locates positions by motion alone, but a bare (x,y,z) says nothing
|
||||
about *what* is moving. Every live entity should reference its parsed `.tbl`
|
||||
definition (vtable 0x820af844, one object per unit type, addresses known), so
|
||||
scanning the neighbourhood of each position for a word equal to a definition
|
||||
address both types the entity and reveals the live object's own layout — the
|
||||
delta from that pointer to the position triple is the position offset.
|
||||
|
||||
Usage: liveents.py [radius_hex]
|
||||
"""
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
import gworld # noqa: E402
|
||||
|
||||
|
||||
def definitions(w):
|
||||
"""{definition_va: unit_id}"""
|
||||
out = {}
|
||||
for off in w.scan_vtable(gworld.DEF_VTABLE):
|
||||
nm = w.name_of(off)
|
||||
if nm and nm.startswith("UN_"):
|
||||
va = gmem.primary_va(off)
|
||||
if va is not None:
|
||||
out[va] = nm
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
radius = int(sys.argv[1], 16) if len(sys.argv) > 1 else 0x600
|
||||
w = gworld.World()
|
||||
defs = definitions(w)
|
||||
print(f"# {len(defs)} unit definitions")
|
||||
by_word = {struct.pack(">I", va): nm for va, nm in defs.items()}
|
||||
|
||||
# sample twice to locate movers, exactly as findplayer.py does
|
||||
import time
|
||||
fd, size = w.fd, w.size
|
||||
|
||||
def snap():
|
||||
return [(a, np.frombuffer(os.pread(fd, (b - a) // 4 * 4, a), dtype=">f4"))
|
||||
for a, b in gmem.extents(fd, size) if (b - a) >= 64]
|
||||
|
||||
s0 = snap(); t0 = time.time()
|
||||
time.sleep(0.4)
|
||||
s1 = snap(); t1 = time.time()
|
||||
shapes0 = {(o, len(a)) for o, a in s0}
|
||||
pairs = [(o, a, b) for (o, a), (o2, b) in zip(s0, s1)
|
||||
if o == o2 and len(a) == len(b) and (o, len(a)) in shapes0]
|
||||
|
||||
movers = []
|
||||
for base, a, b in pairs:
|
||||
af = np.nan_to_num(a.astype(np.float64), nan=0, posinf=0, neginf=0)
|
||||
bf = np.nan_to_num(b.astype(np.float64), nan=0, posinf=0, neginf=0)
|
||||
d = bf - af
|
||||
m = np.abs(d) > 1e-3
|
||||
idx = np.flatnonzero(m)
|
||||
cand = idx[np.isin(idx + 1, idx) & np.isin(idx + 2, idx)]
|
||||
for i in cand:
|
||||
v = np.array([d[i], d[i + 1], d[i + 2]])
|
||||
sp = float(np.linalg.norm(v)) / (t1 - t0)
|
||||
if 20.0 < sp < 3000.0:
|
||||
movers.append((base + int(i) * 4, tuple(float(af[i + j]) for j in range(3)), sp))
|
||||
|
||||
print(f"# {len(movers)} moving triples; typing them...")
|
||||
typed, untyped = [], 0
|
||||
seen = set()
|
||||
for off, pos, sp in movers:
|
||||
lo = max(0, off - radius)
|
||||
blob = os.pread(fd, radius * 2, lo)
|
||||
hit = None
|
||||
for k in range(0, len(blob) - 3, 4):
|
||||
nm = by_word.get(blob[k:k + 4])
|
||||
if nm:
|
||||
hit = (lo + k, nm)
|
||||
break
|
||||
if hit:
|
||||
ptr_off, nm = hit
|
||||
key = (ptr_off, nm)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
typed.append((ptr_off, off - ptr_off, nm, pos, sp))
|
||||
else:
|
||||
untyped += 1
|
||||
|
||||
print(f"# typed {len(typed)}, untyped {untyped}")
|
||||
deltas = Counter(d for _, d, _, _, _ in typed)
|
||||
print(f"# most common (def-pointer -> position) deltas: {deltas.most_common(6)}")
|
||||
for ptr_off, d, nm, pos, sp in sorted(typed, key=lambda x: x[2])[:40]:
|
||||
va = gmem.primary_va(ptr_off)
|
||||
print(f" obj~{va:#010x} defptr+{d:#06x} {nm:<40} "
|
||||
f"({pos[0]:+10.1f},{pos[1]:+10.1f},{pos[2]:+10.1f}) {sp:7.1f}/s")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
167
tools/re-capture/mission_state.py
Executable file
167
tools/re-capture/mission_state.py
Executable file
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Mission state from guest RAM: every entity's hull, not just the player's.
|
||||
|
||||
`own_state.py` found the player's hull by anchoring on a solved definition
|
||||
field: an undamaged craft carries its definition's own `HP` (+0x054), so the
|
||||
live counter is the copy of that number that *falls*. The result was
|
||||
`hull = position + 0x154`.
|
||||
|
||||
The escort question needs the same number for **someone else's** ship. Stage 02
|
||||
is lost when the ACROPOLIS sinks, not when the player dies, and the 240 s run in
|
||||
autopilot-memory-driven.md hit GAME OVER with our own hull untouched. So the
|
||||
claim to test here is that `+0x154` is a property of the *entity class*, not of
|
||||
the player object: every entity, hostile or friendly, fighter or capital ship,
|
||||
should hold its own definition's `HP` there at spawn and lose it when hit.
|
||||
|
||||
That is falsifiable in one run: read `pos+0x154` and the definition `HP` for
|
||||
every entity in the scene and compare. If the anchor is class-wide, the ratio
|
||||
is 1.0 for everything undamaged and nothing else lines up by accident; if it is
|
||||
player-specific, most entities hold something unrelated.
|
||||
|
||||
Sub-commands:
|
||||
scan [cfg] one table of every entity: hull vs definition HP
|
||||
watch <cfg> <secs> [hz] [out] sample over time; report what took damage
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
import navigator # noqa: E402
|
||||
|
||||
HULL_OFF = 0x154 # own_state.py, player craft
|
||||
DEF_HP = 0x054 # unit-struct-runtime.md, ✅ CONFIRMED
|
||||
DEF_SIZE_R = 0x50
|
||||
|
||||
|
||||
def f32(fd, off):
|
||||
b = os.pread(fd, 4, off)
|
||||
if len(b) < 4:
|
||||
return float("nan")
|
||||
(v,) = struct.unpack(">f", b)
|
||||
return v
|
||||
|
||||
|
||||
class Mission:
|
||||
def __init__(self, cfg):
|
||||
self.W = navigator.World(cfg)
|
||||
self.def_hp = {}
|
||||
for va in self.W.defs:
|
||||
self.def_hp[va] = f32(self.W.fd, gmem.va_to_off(va) + DEF_HP)
|
||||
|
||||
def snapshot(self):
|
||||
"""[(off, name, faction, radius, pos, hull, hp_max)] for every entity."""
|
||||
out = []
|
||||
for off, va in self.W.ents:
|
||||
p = self.W.pos(off)
|
||||
if p is None:
|
||||
continue
|
||||
hull = f32(self.W.fd, off + HULL_OFF)
|
||||
nm = self.W.defs[va]
|
||||
out.append((off, nm, navigator.faction(nm), self.W.radius[va],
|
||||
p, hull, self.def_hp.get(va, float("nan"))))
|
||||
return out
|
||||
|
||||
|
||||
def fmt(e):
|
||||
off, nm, fac, r, p, hull, hp0 = e
|
||||
frac = hull / hp0 if hp0 and math.isfinite(hp0) and hp0 > 0 else float("nan")
|
||||
return (f"{gmem.primary_va(off):#010x} {nm[:34]:<34} {fac:<4} r={r:6.0f} "
|
||||
f"({p[0]:+8.0f},{p[1]:+8.0f},{p[2]:+8.0f}) hull={hull:9.1f} "
|
||||
f"HP={hp0:9.1f} frac={frac:6.3f}")
|
||||
|
||||
|
||||
def cmd_scan(cfg):
|
||||
m = Mission(cfg)
|
||||
m.W.scan()
|
||||
ents = m.snapshot()
|
||||
print(f"# {len(ents)} entities")
|
||||
ok = sum(1 for e in ents
|
||||
if math.isfinite(e[6]) and e[6] > 0 and abs(e[5] / e[6] - 1.0) < 1e-3)
|
||||
fin = sum(1 for e in ents if math.isfinite(e[6]) and e[6] > 0)
|
||||
print(f"# hull(+{HULL_OFF:#x}) == definition HP for {ok}/{fin} entities "
|
||||
f"whose definition has an HP")
|
||||
for e in sorted(ents, key=lambda e: -e[3]):
|
||||
print(" " + fmt(e))
|
||||
|
||||
|
||||
def cmd_watch(cfg, secs, hz, out):
|
||||
m = Mission(cfg)
|
||||
m.W.scan()
|
||||
base = {e[0]: e for e in m.snapshot()}
|
||||
print(f"# watching {len(base)} entities for {secs:g}s at {hz:g}Hz", flush=True)
|
||||
fh = open(out, "w") if out else None
|
||||
t0 = time.time()
|
||||
last_scan = t0
|
||||
last_seen = {}
|
||||
while time.time() - t0 < secs:
|
||||
t = time.time()
|
||||
if t - last_scan > 15.0: # new waves spawn; re-enumerate rarely
|
||||
m.W.scan()
|
||||
last_scan = t
|
||||
for e in m.snapshot():
|
||||
base.setdefault(e[0], e)
|
||||
ents = m.snapshot()
|
||||
rec = {"t": round(t - t0, 2),
|
||||
"n": len(ents),
|
||||
"adan": sum(1 for e in ents if e[2] == "ADAN"),
|
||||
"tcaf": sum(1 for e in ents if e[2] == "TCAF"),
|
||||
"ents": []}
|
||||
for e in ents:
|
||||
off, nm, fac, r, p, hull, hp0 = e
|
||||
b = base.get(off)
|
||||
drop = (b[5] - hull) if b and math.isfinite(b[5]) else 0.0
|
||||
big = r >= navigator.Navigator.BIG_RADIUS
|
||||
if big or drop > 0.5:
|
||||
rec["ents"].append({"va": gmem.primary_va(off), "nm": nm,
|
||||
"fac": fac, "r": round(r, 1),
|
||||
"hull": round(hull, 1), "hp0": round(hp0, 1),
|
||||
"drop": round(drop, 1),
|
||||
"pos": [round(float(c), 1) for c in p]})
|
||||
if drop > 0.5 and abs(last_seen.get(off, 0.0) - hull) > 0.5:
|
||||
last_seen[off] = hull
|
||||
print(f"[{t-t0:6.1f}] HIT {nm[:30]:<30} {fac} "
|
||||
f"hull {hull:9.1f}/{hp0:9.1f} (-{drop:.0f})", flush=True)
|
||||
if fh:
|
||||
fh.write(json.dumps(rec) + "\n")
|
||||
fh.flush()
|
||||
time.sleep(max(0.0, 1.0 / hz - (time.time() - t)))
|
||||
if fh:
|
||||
fh.close()
|
||||
|
||||
print("\n# damage summary")
|
||||
ents = {e[0]: e for e in m.snapshot()}
|
||||
for off, b in sorted(base.items(), key=lambda kv: -kv[1][3]):
|
||||
cur = ents.get(off)
|
||||
gone = cur is None
|
||||
hull = cur[5] if cur else float("nan")
|
||||
if not gone and abs(hull - b[5]) < 0.5 and b[3] < navigator.Navigator.BIG_RADIUS:
|
||||
continue
|
||||
print(f" {b[1][:34]:<34} {b[2]:<4} r={b[3]:6.0f} "
|
||||
f"{b[5]:9.1f} -> {hull:9.1f}" + (" GONE" if gone else ""))
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3:
|
||||
sys.exit(__doc__)
|
||||
cmd = sys.argv[1]
|
||||
cfg = json.load(open(sys.argv[2]))
|
||||
if cmd == "scan":
|
||||
cmd_scan(cfg)
|
||||
elif cmd == "watch":
|
||||
cmd_watch(cfg,
|
||||
float(sys.argv[3]) if len(sys.argv) > 3 else 120.0,
|
||||
float(sys.argv[4]) if len(sys.argv) > 4 else 1.0,
|
||||
sys.argv[5] if len(sys.argv) > 5 else None)
|
||||
else:
|
||||
sys.exit(__doc__)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
397
tools/re-capture/navigator.py
Normal file
397
tools/re-capture/navigator.py
Normal file
@@ -0,0 +1,397 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Drift-aware navigation with collision avoidance, driven from guest memory.
|
||||
|
||||
Three things the pursuit loop in autopilot3.py did not do:
|
||||
|
||||
* **See everything.** It enumerated entities by looking for things that *move*,
|
||||
so stations, hulls and parked structures were invisible — exactly the objects
|
||||
you crash into. This scans the entity heap for words that equal a known unit
|
||||
definition address and takes `position = hit - 0x130`, which finds every
|
||||
entity whether it is moving or not.
|
||||
|
||||
* **Know how big they are.** Each entity's definition carries `Size_Radius`
|
||||
(+0x50) and `Size_X/Y/Z` (+0x30/34/38) — fields already solved in
|
||||
docs/re/structures/unit-struct-runtime.md — so the avoidance radius is the
|
||||
game's own number, not a guess.
|
||||
|
||||
* **Account for drift.** The craft does not turn where it points: velocity lags
|
||||
the nose like an aircraft with sideslip. Steering the *nose* at a target
|
||||
therefore steers the *flight path* somewhere else, wide and late. This
|
||||
measures the lag online (the angle between nose and velocity, and how fast
|
||||
the velocity vector is actually swinging) and commands the nose *ahead* of
|
||||
where the flight path should go, by that lag.
|
||||
|
||||
Avoidance is closest-point-of-approach, not distance: what matters is whether
|
||||
the two paths will intersect within a horizon, which is why a fast crossing
|
||||
target is dangerous at 2 km and a station drifting away is not at 300 m.
|
||||
|
||||
Usage: navigator.py <config.json> [seconds] [--dry]
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
import gworld # noqa: E402
|
||||
import entities2 # noqa: E402
|
||||
from flight_probe import Pad # noqa: E402
|
||||
|
||||
# confirmed fields of the parsed unit definition (unit-struct-runtime.md)
|
||||
DEF_SIZE_X, DEF_SIZE_Y, DEF_SIZE_Z, DEF_SIZE_R = 0x30, 0x34, 0x38, 0x50
|
||||
|
||||
|
||||
def norm(v):
|
||||
n = float(np.linalg.norm(v))
|
||||
return v / n if n > 1e-9 else v * 0.0
|
||||
|
||||
|
||||
def ang(a, b):
|
||||
return math.acos(max(-1.0, min(1.0, float(np.dot(norm(a), norm(b))))))
|
||||
|
||||
|
||||
class World:
|
||||
def __init__(self, cfg):
|
||||
self.cfg = cfg
|
||||
self.w = gworld.World()
|
||||
self.fd, self.size = self.w.fd, self.w.size
|
||||
self.delta = cfg["def_delta"]
|
||||
self.rot_delta = cfg["rot_delta"]
|
||||
self.rot_stride = cfg.get("rot_stride", 12)
|
||||
self.fwd_row = cfg["fwd_row"]
|
||||
self.fwd_sign = cfg["fwd_sign"]
|
||||
self.defs = {} # def_va -> name
|
||||
self.def_word = {} # 4-byte BE -> def_va
|
||||
self.radius = {} # def_va -> collision radius
|
||||
self.prev = {} # pos_off -> (t, pos) for velocity
|
||||
self.vel = {} # pos_off -> smoothed velocity
|
||||
self.ents = []
|
||||
self._load_defs()
|
||||
|
||||
def _load_defs(self):
|
||||
for off in self.w.scan_vtable(gworld.DEF_VTABLE):
|
||||
nm = self.w.name_of(off)
|
||||
if not (nm and nm.startswith("UN_")):
|
||||
continue
|
||||
va = gmem.primary_va(off)
|
||||
if va is None:
|
||||
continue
|
||||
self.defs[va] = nm
|
||||
self.def_word[struct.pack(">I", va)] = va
|
||||
b = os.pread(self.fd, 0x60, off)
|
||||
try:
|
||||
sx, sy, sz = (struct.unpack_from(">f", b, o)[0]
|
||||
for o in (DEF_SIZE_X, DEF_SIZE_Y, DEF_SIZE_Z))
|
||||
sr = struct.unpack_from(">f", b, DEF_SIZE_R)[0]
|
||||
except struct.error:
|
||||
sx = sy = sz = sr = 0.0
|
||||
vals = [v for v in (sr, sx, sy, sz) if math.isfinite(v) and 0 < v < 1e5]
|
||||
self.radius[va] = max(vals) if vals else 50.0
|
||||
|
||||
# ------------------------------------------------------------- entities
|
||||
def scan(self):
|
||||
"""Every entity in the heap — moving or not — by its definition pointer."""
|
||||
lo = gmem.va_to_off(self.cfg["va_lo"])
|
||||
hi = gmem.va_to_off(self.cfg["va_hi"])
|
||||
found = []
|
||||
# numpy, not a per-word python loop: the entity heap is 16 MB, so
|
||||
# stepping it 4 bytes at a time costs seconds per scan and the control
|
||||
# loop spends its whole budget scanning instead of flying.
|
||||
want = np.array(sorted(self.defs.keys()), dtype=np.uint32)
|
||||
for a, b in gmem.extents(self.fd, self.size):
|
||||
a, b = max(a, lo), min(b, hi)
|
||||
n = (b - a) // 4 * 4
|
||||
if n < 64:
|
||||
continue
|
||||
arr = np.frombuffer(os.pread(self.fd, n, a), dtype=">u4").astype(np.uint32)
|
||||
for k4 in np.flatnonzero(np.isin(arr, want)):
|
||||
k = int(k4) * 4
|
||||
va = int(arr[k4])
|
||||
poff = a + k - self.delta
|
||||
if poff < 0:
|
||||
continue
|
||||
pb = os.pread(self.fd, 12, poff)
|
||||
if len(pb) < 12:
|
||||
continue
|
||||
p = np.array(struct.unpack(">3f", pb))
|
||||
if not np.all(np.isfinite(p)) or np.max(np.abs(p)) > 1e7:
|
||||
continue
|
||||
found.append((poff, va))
|
||||
# De-duplicate by POSITION: one entity is mirrored at several
|
||||
# addresses, so keying on the address keeps every copy and the scene
|
||||
# looks several times more crowded than it is.
|
||||
seen, uniq = {}, {}
|
||||
for poff, va in found:
|
||||
p = self.pos(poff)
|
||||
if p is None:
|
||||
continue
|
||||
key = (va, tuple(np.round(p, 0)))
|
||||
if key in seen:
|
||||
continue
|
||||
seen[key] = poff
|
||||
uniq[poff] = va
|
||||
self.ents = list(uniq.items())
|
||||
return self.ents
|
||||
|
||||
def pos(self, off):
|
||||
b = os.pread(self.fd, 12, off)
|
||||
if len(b) < 12:
|
||||
return None
|
||||
p = np.array(struct.unpack(">3f", b))
|
||||
return p if np.all(np.isfinite(p)) else None
|
||||
|
||||
def rot(self, off):
|
||||
n = self.rot_stride * 2 + 12
|
||||
b = os.pread(self.fd, n, off + self.rot_delta)
|
||||
if len(b) < n:
|
||||
return None
|
||||
M = np.array([struct.unpack_from(">3f", b, self.rot_stride * r) for r in range(3)])
|
||||
if not np.all(np.isfinite(M)) or np.max(np.abs(M @ M.T - np.eye(3))) > 5e-3:
|
||||
return None
|
||||
return M
|
||||
|
||||
def sample(self, t):
|
||||
"""[(off, name, pos, vel, radius)] with velocity by finite difference."""
|
||||
out = []
|
||||
for off, va in self.ents:
|
||||
p = self.pos(off)
|
||||
if p is None:
|
||||
continue
|
||||
# A frame the emulator did not advance gives an identical position
|
||||
# and a bogus zero velocity, which then reads as "stopped" and
|
||||
# wrecks both the drift estimate and every closing-rate. Keep the
|
||||
# last good velocity instead, and smooth it.
|
||||
prev = self.prev.get(off)
|
||||
v = self.vel.get(off, np.zeros(3))
|
||||
if prev is not None:
|
||||
dt = t - prev[0]
|
||||
moved = float(np.linalg.norm(p - prev[1]))
|
||||
if dt > 0.02 and moved > 1e-4:
|
||||
inst = (p - prev[1]) / dt
|
||||
if float(np.linalg.norm(inst)) < 5000.0:
|
||||
v = 0.5 * v + 0.5 * inst if np.any(v) else inst
|
||||
self.prev[off] = (t, p)
|
||||
else:
|
||||
self.prev[off] = (t, p)
|
||||
self.vel[off] = v
|
||||
out.append((off, self.defs[va], p, v, self.radius[va]))
|
||||
return out
|
||||
|
||||
|
||||
def faction(nm):
|
||||
b = nm[3:]
|
||||
return "ADAN" if b.startswith("be") or b.startswith("e") else "TCAF"
|
||||
|
||||
|
||||
class Navigator:
|
||||
KP, KD = 2.2, 0.45
|
||||
FIRE_CONE = math.radians(9)
|
||||
FIRE_RANGE = 6000.0
|
||||
HORIZON = 6.0 # s of look-ahead for collision checks
|
||||
MARGIN_BIG = 220.0 # clearance around hulls and structures
|
||||
MARGIN_SMALL = 45.0 # ...around fighters, which manoeuvre themselves
|
||||
BIG_RADIUS = 120.0 # above this an entity counts as a structure
|
||||
SELF_MIRROR = 25.0 # the same object is mirrored at several addresses
|
||||
|
||||
def __init__(self, W, pad, dry=False, log=sys.stdout):
|
||||
self.W = W
|
||||
self.pad = pad
|
||||
self.dry = dry
|
||||
self.log = log
|
||||
self.prevM = None
|
||||
self.firing = False
|
||||
self.tau = 0.8 # velocity-lag time constant, refined online
|
||||
|
||||
# ------------------------------------------------------------ drift
|
||||
def update_tau(self, fwd, vel, prev_vhat, dt):
|
||||
"""How long the flight path takes to catch the nose.
|
||||
|
||||
The velocity vector swings toward the nose; the angle between them
|
||||
divided by the rate the velocity is actually swinging is that lag, and
|
||||
it is what the nose has to be commanded ahead by.
|
||||
"""
|
||||
if prev_vhat is None or dt <= 1e-3:
|
||||
return
|
||||
vh = norm(vel)
|
||||
if np.linalg.norm(vh) < 1e-6:
|
||||
return
|
||||
swing = ang(prev_vhat, vh) / dt # rad/s the path is turning
|
||||
lag = ang(fwd, vh) # rad the path is behind
|
||||
if swing > 0.02 and lag > 0.02:
|
||||
tau = lag / swing
|
||||
if 0.05 < tau < 5.0:
|
||||
self.tau = 0.9 * self.tau + 0.1 * tau
|
||||
|
||||
# ------------------------------------------------- collision avoidance
|
||||
def avoidance(self, me_p, me_v, me_r, ents, me_off):
|
||||
"""Sum of escape directions, weighted by how soon and how close."""
|
||||
push = np.zeros(3)
|
||||
worst = None
|
||||
for off, nm, p, v, r in ents:
|
||||
if off == me_off:
|
||||
continue
|
||||
rel_p = p - me_p
|
||||
rel_v = v - me_v
|
||||
d = float(np.linalg.norm(rel_p))
|
||||
# The same entity exists at several mirrored addresses, so our own
|
||||
# copy shows up as an obstacle at zero distance and pins the sticks
|
||||
# at full deflection forever. Anything this close is us.
|
||||
if d < self.SELF_MIRROR:
|
||||
continue
|
||||
# A wingman flying formation is metres away by design and steers
|
||||
# itself; giving it a hull-sized margin makes the loop thrash.
|
||||
big = r >= self.BIG_RADIUS
|
||||
margin = self.MARGIN_BIG if big else self.MARGIN_SMALL
|
||||
if not big and faction(nm) == "TCAF":
|
||||
margin *= 0.5
|
||||
safe = me_r + r + margin
|
||||
if d > 1e4:
|
||||
continue
|
||||
vv = float(np.dot(rel_v, rel_v))
|
||||
t_cpa = 0.0 if vv < 1e-6 else -float(np.dot(rel_p, rel_v)) / vv
|
||||
t_cpa = max(0.0, min(self.HORIZON, t_cpa))
|
||||
cpa = rel_p + rel_v * t_cpa
|
||||
miss = float(np.linalg.norm(cpa))
|
||||
if miss >= safe:
|
||||
continue
|
||||
urgency = (1.0 - miss / safe) * (1.0 - t_cpa / self.HORIZON)
|
||||
if urgency <= 0:
|
||||
continue
|
||||
esc = -norm(cpa) if miss > 1e-3 else norm(np.cross(rel_p, me_v))
|
||||
if np.linalg.norm(esc) < 1e-6:
|
||||
esc = norm(np.cross(rel_p, np.array([0.0, 1.0, 0.0])))
|
||||
push += esc * urgency
|
||||
if worst is None or urgency > worst[0]:
|
||||
worst = (urgency, nm, d, miss, t_cpa)
|
||||
return push, worst
|
||||
|
||||
# -------------------------------------------------------------- target
|
||||
def pick(self, me_p, me_v, fwd, ents, me_off):
|
||||
speed = max(float(np.linalg.norm(me_v)), 1.0)
|
||||
best, bestscore = None, 1e18
|
||||
for off, nm, p, v, r in ents:
|
||||
if off == me_off or faction(nm) != "ADAN":
|
||||
continue
|
||||
rel = p - me_p
|
||||
d = float(np.linalg.norm(rel))
|
||||
if d < 1e-3:
|
||||
continue
|
||||
# lead: where it will be when a shot gets there
|
||||
lead = p + v * (d / max(speed, 200.0))
|
||||
rel_l = lead - me_p
|
||||
theta = ang(rel_l, fwd)
|
||||
score = d * (1.0 + 3.0 * (theta / math.pi) ** 2)
|
||||
if score < bestscore:
|
||||
best, bestscore = (off, nm, lead, rel_l, d), score
|
||||
return best
|
||||
|
||||
# ---------------------------------------------------------------- step
|
||||
def step(self, t, dt, prev_vhat):
|
||||
ents = self.W.sample(t)
|
||||
me = None
|
||||
for e in ents:
|
||||
if "Player" in e[1]:
|
||||
me = e
|
||||
break
|
||||
if me is None:
|
||||
return "no-player", prev_vhat
|
||||
me_off, me_nm, me_p, me_v, me_r = me
|
||||
M = self.W.rot(me_off)
|
||||
if M is None:
|
||||
return "no-orientation", prev_vhat
|
||||
fwd = M[self.W.fwd_row] * self.W.fwd_sign
|
||||
right = M[(self.W.fwd_row + 1) % 3]
|
||||
up = np.cross(fwd, right)
|
||||
|
||||
speed = float(np.linalg.norm(me_v))
|
||||
vhat = norm(me_v) if speed > 1.0 else fwd
|
||||
self.update_tau(fwd, me_v, prev_vhat, dt)
|
||||
|
||||
# body angular velocity for the damping term
|
||||
w = np.zeros(3)
|
||||
if self.prevM is not None and dt > 1e-3:
|
||||
D = self.prevM @ M.T
|
||||
w = np.array([D[2, 1] - D[1, 2], D[0, 2] - D[2, 0], D[1, 0] - D[0, 1]]) / (2 * dt)
|
||||
self.prevM = M
|
||||
|
||||
tgt = self.pick(me_p, me_v, fwd, ents, me_off)
|
||||
goal = norm(tgt[3]) if tgt else fwd
|
||||
|
||||
push, worst = self.avoidance(me_p, me_v, me_r, ents, me_off)
|
||||
pn = float(np.linalg.norm(push))
|
||||
# avoidance outranks the target when it is urgent
|
||||
want = norm(goal + push * (3.0 if pn > 0.6 else 1.5)) if pn > 1e-6 else goal
|
||||
|
||||
# Command the NOSE ahead of where the flight path must go, by the
|
||||
# measured lag -- steering the nose straight at the target makes the
|
||||
# path arrive wide and late.
|
||||
nose_cmd = norm(want + (want - vhat) * min(self.tau * 1.6, 2.5))
|
||||
|
||||
ex = float(np.dot(nose_cmd, right))
|
||||
ey = float(np.dot(nose_cmd, up))
|
||||
ez = float(np.dot(nose_cmd, fwd))
|
||||
yaw = math.atan2(ex, ez if abs(ez) > 1e-3 else 1e-3)
|
||||
pitch = math.atan2(ey, ez if abs(ez) > 1e-3 else 1e-3)
|
||||
if ez < 0:
|
||||
yaw = math.copysign(math.pi / 2, ex if ex else 1.0)
|
||||
|
||||
sx = max(-1.0, min(1.0, self.KP * yaw - self.KD * float(np.dot(w, up))))
|
||||
sy = max(-1.0, min(1.0, -(self.KP * pitch - self.KD * float(np.dot(w, right)))))
|
||||
|
||||
# fire only when the *flight path* is clear and the nose is on target
|
||||
aim_ok = tgt and abs(yaw) < self.FIRE_CONE and abs(pitch) < self.FIRE_CONE
|
||||
fire = bool(aim_ok and tgt[4] < self.FIRE_RANGE and pn < 1.2)
|
||||
|
||||
if not self.dry:
|
||||
self.pad.axis("LX", sx)
|
||||
self.pad.axis("LY", sy)
|
||||
if fire != self.firing:
|
||||
(self.pad.press if fire else self.pad.release)("RB")
|
||||
self.firing = fire
|
||||
|
||||
drift = math.degrees(ang(fwd, vhat))
|
||||
msg = (f"spd={speed:6.0f} drift={drift:5.1f}d tau={self.tau:4.2f} "
|
||||
f"yaw={math.degrees(yaw):+6.1f} pit={math.degrees(pitch):+6.1f} "
|
||||
f"stick=({sx:+.2f},{sy:+.2f}) fire={int(fire)}")
|
||||
if tgt:
|
||||
msg += f" tgt={tgt[1][3:22]:<20} d={tgt[4]:7.0f}"
|
||||
if worst:
|
||||
msg += (f" | AVOID {worst[1][3:20]} miss={worst[3]:6.0f} "
|
||||
f"t={worst[4]:4.1f}s u={worst[0]:.2f}")
|
||||
return msg, vhat
|
||||
|
||||
def run(self, secs, hz=10.0):
|
||||
self.W.scan()
|
||||
t0 = time.time()
|
||||
last, last_scan, prev_vhat = t0, 0.0, None
|
||||
while time.time() - t0 < secs:
|
||||
t = time.time()
|
||||
if t - last_scan > 4.0:
|
||||
ents = self.W.scan()
|
||||
last_scan = t
|
||||
c = Counter(faction(self.W.defs[va]) for _, va in ents)
|
||||
print(f"[{t-t0:6.1f}] scan: {len(ents)} entities {dict(c)}",
|
||||
file=self.log, flush=True)
|
||||
msg, prev_vhat = self.step(t, t - last, prev_vhat)
|
||||
last = t
|
||||
print(f"[{t-t0:6.1f}] {msg}", file=self.log, flush=True)
|
||||
time.sleep(max(0, 1.0 / hz - (time.time() - t)))
|
||||
if not self.dry:
|
||||
self.pad.reset()
|
||||
|
||||
|
||||
def main():
|
||||
cfg = json.load(open(sys.argv[1]))
|
||||
secs = float(sys.argv[2]) if len(sys.argv) > 2 else 90.0
|
||||
W = World(cfg)
|
||||
Navigator(W, Pad(), dry="--dry" in sys.argv).run(secs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
95
tools/re-capture/order_check.py
Normal file
95
tools/re-capture/order_check.py
Normal file
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test the hypothesis that a unit sub-record's runtime fields are laid out in
|
||||
*schema declaration order*, one 4-byte slot each.
|
||||
|
||||
The IDXD string pool lists a sub-record's field names in declaration order,
|
||||
including the ones this .tbl leaves at their default (they appear as a bare key
|
||||
with no preceding value). If the runtime struct is `base + 4*index` over that
|
||||
list, then the offsets the solver found independently must all satisfy it.
|
||||
|
||||
That is a much stronger check than any single field's agreement count: one base
|
||||
offset has to explain dozens of independently-derived anchors at once.
|
||||
|
||||
Usage: order_check.py <raw-token-dump> <tbl-hash> <SubRecord> <solved.txt>
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
NUMERIC = re.compile(r"^[-+]?[0-9]*\.?[0-9]+[fF]?$|^0x[0-9a-fA-F]+$")
|
||||
VALUE_WORDS = {"Yes", "No", "YES", "NO", "On", "Off", "ON", "OFF"}
|
||||
|
||||
|
||||
def section(dump, tbl, sub, subs):
|
||||
"""Ordered token list of one sub-record of one .tbl."""
|
||||
toks, on = [], False
|
||||
for line in open(dump):
|
||||
if line.startswith("RAW "):
|
||||
on = line.split()[1] == tbl
|
||||
continue
|
||||
if on and line.startswith("T "):
|
||||
toks.append(line[2:].rstrip("\n"))
|
||||
starts = [i for i, t in enumerate(toks) if t in subs or t.lstrip("(") in subs]
|
||||
for n, i in enumerate(starts):
|
||||
name = toks[i].lstrip("(")
|
||||
if name != sub:
|
||||
continue
|
||||
end = starts[n + 1] if n + 1 < len(starts) else len(toks)
|
||||
return toks[i + 1 : end]
|
||||
return []
|
||||
|
||||
|
||||
def keys_in_order(toks):
|
||||
"""Field names, in declaration order: every token that is not a value.
|
||||
|
||||
In the numeric sub-records every value is a number literal, so a token that
|
||||
is not numeric and not a Yes/No word is a key.
|
||||
"""
|
||||
out = []
|
||||
for t in toks:
|
||||
if NUMERIC.match(t) or t in VALUE_WORDS:
|
||||
continue
|
||||
if t not in out:
|
||||
out.append(t)
|
||||
return out
|
||||
|
||||
|
||||
def solved_offsets(path):
|
||||
out = {}
|
||||
for line in open(path):
|
||||
m = re.match(r"\s*(0x[0-9a-f]+)\s+(\w+)\s+(\w+)\s+(\d+)\s+(\d+)\s+(\d+)\s*$", line)
|
||||
if m:
|
||||
off, enc, field, agree, dist, bad = m.groups()
|
||||
out[field] = (int(off, 16), enc, int(agree), int(dist), int(bad))
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
dump, tbl, sub, solvedf = sys.argv[1:5]
|
||||
subs = {"Generic", "Maneuver", "Shield", "Explosion", "Mass", "Effect", "SE"}
|
||||
keys = keys_in_order(section(dump, tbl, sub, subs))
|
||||
solved = solved_offsets(solvedf)
|
||||
anchors = [(i, k, solved[k]) for i, k in enumerate(keys) if k in solved]
|
||||
if not anchors:
|
||||
sys.exit(f"no solved field lands in {sub}")
|
||||
|
||||
# base that the most anchors agree on
|
||||
votes = {}
|
||||
for i, k, (off, *_) in anchors:
|
||||
votes.setdefault(off - 4 * i, []).append(k)
|
||||
base, winners = max(votes.items(), key=lambda kv: len(kv[1]))
|
||||
print(f"# {sub}: {len(keys)} declared fields, {len(anchors)} solved anchors")
|
||||
print(f"# best base = {base:#x} -> {len(winners)}/{len(anchors)} anchors fit "
|
||||
f"offset = base + 4*index\n")
|
||||
for i, k in enumerate(keys):
|
||||
off = base + 4 * i
|
||||
if k in solved:
|
||||
so, enc, agree, dist, bad = solved[k]
|
||||
mark = "fits" if so == off else f"CONFLICT solver={so:#x}"
|
||||
print(f" +{off:#05x} [{i:3d}] {k:<34} {enc} agree={agree:<3} "
|
||||
f"dist={dist:<3} {mark}")
|
||||
else:
|
||||
print(f" +{off:#05x} [{i:3d}] {k:<34} — (never valued on disc)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
117
tools/re-capture/own_state.py
Normal file
117
tools/re-capture/own_state.py
Normal file
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Where does the craft keep its hull and shield? Anchor on the definition.
|
||||
|
||||
The parsed unit definition already has solved fields (unit-struct-runtime.md):
|
||||
`HP` at `+0x054`, shield `MaxValue` at `+0x238`, shield `ChargeSpeed` at
|
||||
`+0x244`. A craft that has taken no damage is at full hull and full shield, so
|
||||
its live entity object must *contain those very numbers*. That turns "find the
|
||||
HP field" from a value-scan over 4 GB into: read two floats from the definition,
|
||||
then look for them inside the entity object.
|
||||
|
||||
Then watch the candidates while the craft is under fire. The live field is the
|
||||
one that falls; a copy of the definition value that never moves is not it.
|
||||
|
||||
Usage: own_state.py <config.json> [watch_seconds] [out.json]
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
import navigator # noqa: E402
|
||||
|
||||
DEF_HP, DEF_SHIELD_MAX, DEF_SHIELD_CHG = 0x054, 0x238, 0x244
|
||||
BACK, FWD = 0x800, 0x800
|
||||
|
||||
|
||||
def near(a, v):
|
||||
return np.abs(a - v) <= max(1e-3, abs(v) * 1e-4)
|
||||
|
||||
|
||||
def main():
|
||||
cfg = json.load(open(sys.argv[1]))
|
||||
secs = float(sys.argv[2]) if len(sys.argv) > 2 else 40.0
|
||||
out = sys.argv[3] if len(sys.argv) > 3 else None
|
||||
|
||||
W = navigator.World(cfg)
|
||||
ents = W.scan()
|
||||
me = [(off, va) for off, va in ents if "Player" in W.defs[va]]
|
||||
if not me:
|
||||
sys.exit("player entity not found — not in flight?")
|
||||
me_off, def_va = me[0]
|
||||
print(f"# player {W.defs[def_va]} pos off {me_off:#x} def {def_va:#010x}")
|
||||
|
||||
d = os.pread(W.fd, 0x300, gmem.va_to_off(def_va))
|
||||
want = {}
|
||||
for nm, o in (("HP", DEF_HP), ("Shield_MaxValue", DEF_SHIELD_MAX),
|
||||
("Shield_ChargeSpeed", DEF_SHIELD_CHG)):
|
||||
(v,) = struct.unpack_from(">f", d, o)
|
||||
want[nm] = v
|
||||
print(f"# definition {nm:<18} = {v:g}")
|
||||
|
||||
blob = os.pread(W.fd, BACK + FWD, me_off - BACK)
|
||||
arr = np.frombuffer(blob, dtype=">f4").astype(np.float64)
|
||||
cands = [] # (label, delta_from_position)
|
||||
for nm, v in want.items():
|
||||
if not np.isfinite(v) or v == 0.0:
|
||||
continue
|
||||
for i in np.flatnonzero(near(arr, v)):
|
||||
cands.append((nm, int(i) * 4 - BACK))
|
||||
print(f"# {len(cands)} candidate offsets inside the entity object:")
|
||||
for nm, dlt in cands:
|
||||
print(f" pos{dlt:+#07x} == definition {nm}")
|
||||
if not cands:
|
||||
print("# none — the object does not carry the definition's own numbers"
|
||||
" at this offset window; widen BACK/FWD or the craft is damaged")
|
||||
|
||||
# ---- watch them; the live field is the one that moves
|
||||
hist = {c: [] for c in cands}
|
||||
t0 = time.time()
|
||||
last_print = 0.0
|
||||
while time.time() - t0 < secs:
|
||||
p = W.pos(me_off)
|
||||
if p is None:
|
||||
print("# player object gone (death / stage change?)")
|
||||
break
|
||||
w = os.pread(W.fd, BACK + FWD, me_off - BACK)
|
||||
a = np.frombuffer(w, dtype=">f4").astype(np.float64)
|
||||
for c in cands:
|
||||
i = (c[1] + BACK) // 4
|
||||
hist[c].append(float(a[i]))
|
||||
t = time.time() - t0
|
||||
if t - last_print > 4.0:
|
||||
last_print = t
|
||||
cur = " ".join(f"{c[0][:4]}{c[1]:+#x}={hist[c][-1]:.1f}" for c in cands[:6])
|
||||
print(f"[{t:6.1f}] {cur}", flush=True)
|
||||
time.sleep(0.2)
|
||||
|
||||
print("\n# offset anchor first last min moved")
|
||||
moving = []
|
||||
for c in cands:
|
||||
h = np.array(hist[c])
|
||||
if not len(h):
|
||||
continue
|
||||
mv = float(h.max() - h.min())
|
||||
print(f" pos{c[1]:+#07x} {c[0]:<18} {h[0]:8.1f} {h[-1]:8.1f} "
|
||||
f"{h.min():8.1f} {mv:7.3f}")
|
||||
if mv > 1e-3:
|
||||
moving.append({"anchor": c[0], "delta": c[1],
|
||||
"first": h[0], "last": float(h[-1]),
|
||||
"min": float(h.min())})
|
||||
if not moving:
|
||||
print("# nothing moved — the craft took no damage during the window")
|
||||
if out:
|
||||
json.dump({"player": W.defs[def_va], "def_va": def_va,
|
||||
"definition": want,
|
||||
"candidates": [{"anchor": a, "delta": b} for a, b in cands],
|
||||
"moved": moving}, open(out, "w"), indent=1)
|
||||
print(f"# wrote {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
319
tools/re-capture/pilot.py
Normal file
319
tools/re-capture/pilot.py
Normal file
@@ -0,0 +1,319 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A pilot that tries to stay alive, not just to shoot.
|
||||
|
||||
Every earlier loop flew a straight pursuit and was shot down; the trace from
|
||||
ctrl_probe.py shows why — 1500 hull points gone in twelve seconds while sitting
|
||||
in a turret's line of fire, with no reaction of any kind. Three measured facts
|
||||
make a reaction possible:
|
||||
|
||||
* **Hull is `position + 0x154`** — at spawn it equals the unit definition's own
|
||||
`HP` (1500 for the Delta Saber), it steps down 30/60/90 per hit, and it goes
|
||||
negative at death. So damage is observable *as it happens*, not inferred.
|
||||
* **`RT` accelerates and `LT` brakes**, and the setting persists: measured
|
||||
ground speed went 488 → 1510 under RT, 488 → 174 under LT and then *stayed*
|
||||
near 130 with the sticks neutral. (The older note "RT is not the throttle" was
|
||||
drawn from a value-scan for a speed field, not from measuring the speed.)
|
||||
* **`RB` fires** (autopilot-memory-driven.md).
|
||||
|
||||
So the loop is a state machine on damage rather than a pure pursuit:
|
||||
|
||||
ENGAGE chase and shoot the nearest hostile fighter
|
||||
EVADE entered the moment the hull drops — turn away from the threats,
|
||||
full throttle, jink; leave only after several quiet seconds
|
||||
RETIRE hull below a floor: break for the friendly capital ship, which the
|
||||
mission's own hint says is where you resupply
|
||||
|
||||
Turrets are treated as threats to be *kept at a distance*, not as targets: the
|
||||
objective is the invading fighters, and the turret is what killed every previous
|
||||
run.
|
||||
|
||||
Usage: pilot.py <config.json> [seconds] [--dry]
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from collections import deque
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import navigator # noqa: E402
|
||||
from navigator import ang, norm # noqa: E402
|
||||
from flight_probe import Pad # noqa: E402
|
||||
|
||||
HULL_OFF = 0x154 # confirmed: == definition HP at spawn, falls when hit
|
||||
SHIELD_OFF = 0x430 # candidate: == definition Shield MaxValue at spawn
|
||||
|
||||
|
||||
class Pilot:
|
||||
KP, KD = 2.2, 0.45
|
||||
FIRE_CONE = math.radians(9)
|
||||
FIRE_RANGE = 5000.0
|
||||
TURRET_KEEPOUT = 2500.0 # ...and stay this far from things that shoot back
|
||||
EVADE_QUIET = 5.0 # seconds without damage before re-engaging
|
||||
RETIRE_FRAC = 0.30 # hull fraction that sends us home
|
||||
HZ = 8.0
|
||||
|
||||
def __init__(self, W, pad, dry=False, log=sys.stdout):
|
||||
self.W = W
|
||||
self.pad = pad
|
||||
self.dry = dry
|
||||
self.log = log
|
||||
# closest-point-of-approach avoidance is navigator.py's, reused as-is
|
||||
self.av = navigator.Navigator(W, pad, dry=True, log=log)
|
||||
self.prevM = None
|
||||
self.firing = False
|
||||
self.throttle = 0 # -1 brake, 0 coast, +1 accelerate
|
||||
self.mode = "ENGAGE"
|
||||
self.hp_hist = deque(maxlen=64)
|
||||
self.hp0 = None
|
||||
self.last_hit = -1e9
|
||||
self.threat_dir = None
|
||||
|
||||
# ------------------------------------------------------------ own state
|
||||
def own(self, off):
|
||||
b = os.pread(self.W.fd, 8, off + HULL_OFF)
|
||||
hull = struct.unpack_from(">f", b, 0)[0] if len(b) >= 4 else float("nan")
|
||||
b2 = os.pread(self.W.fd, 4, off + SHIELD_OFF)
|
||||
shield = struct.unpack(">f", b2)[0] if len(b2) == 4 else float("nan")
|
||||
return hull, shield
|
||||
|
||||
def set_throttle(self, want):
|
||||
"""RT / LT are a persistent setting, so only send the change."""
|
||||
if want == self.throttle or self.dry:
|
||||
return
|
||||
self.pad.trig("RT", 1.0 if want > 0 else 0.0)
|
||||
self.pad.trig("LT", 1.0 if want < 0 else 0.0)
|
||||
self.throttle = want
|
||||
|
||||
# -------------------------------------------------------------- targets
|
||||
def hostiles(self, ents, me_off):
|
||||
out = []
|
||||
for off, nm, p, v, r in ents:
|
||||
if off == me_off or navigator.faction(nm) != "ADAN":
|
||||
continue
|
||||
out.append((off, nm, p, v, r, "Turret" in nm or r >= navigator.Navigator.BIG_RADIUS))
|
||||
return out
|
||||
|
||||
def pick(self, me_p, me_v, fwd, hos):
|
||||
"""Nearest *fighter*, weighted by how far off the nose it is."""
|
||||
speed = max(float(np.linalg.norm(me_v)), 1.0)
|
||||
best, bestscore = None, 1e18
|
||||
for off, nm, p, v, r, hard in hos:
|
||||
if hard:
|
||||
continue # turrets and hulls are not the objective
|
||||
rel = p - me_p
|
||||
d = float(np.linalg.norm(rel))
|
||||
if d < 1e-3:
|
||||
continue
|
||||
lead = p + v * (d / max(speed, 300.0))
|
||||
theta = ang(lead - me_p, fwd)
|
||||
score = d * (1.0 + 3.0 * (theta / math.pi) ** 2)
|
||||
if score < bestscore:
|
||||
best, bestscore = (off, nm, lead, lead - me_p, d), score
|
||||
return best
|
||||
|
||||
def threat_vector(self, me_p, hos):
|
||||
"""Where the danger is: inverse-square weighted direction to shooters."""
|
||||
acc = np.zeros(3)
|
||||
for off, nm, p, v, r, hard in hos:
|
||||
rel = p - me_p
|
||||
d = float(np.linalg.norm(rel))
|
||||
if d < 1.0 or d > 8000.0:
|
||||
continue
|
||||
w = (1500.0 / d) ** 2 * (3.0 if hard else 1.0)
|
||||
acc += norm(rel) * w
|
||||
return norm(acc) if np.linalg.norm(acc) > 1e-6 else None
|
||||
|
||||
def friendly_base(self, ents, me_off):
|
||||
"""The biggest friendly — the carrier the briefing says to resupply at."""
|
||||
best = None
|
||||
for off, nm, p, v, r in ents:
|
||||
if off == me_off or navigator.faction(nm) != "TCAF":
|
||||
continue
|
||||
if best is None or r > best[4]:
|
||||
best = (off, nm, p, v, r)
|
||||
return best
|
||||
|
||||
# ------------------------------------------------------------ steering
|
||||
def sticks(self, want, M, w):
|
||||
fwd = M[self.W.fwd_row] * self.W.fwd_sign
|
||||
right = M[(self.W.fwd_row + 1) % 3]
|
||||
up = np.cross(fwd, right)
|
||||
ex, ey, ez = (float(np.dot(want, right)), float(np.dot(want, up)),
|
||||
float(np.dot(want, fwd)))
|
||||
yaw = math.atan2(ex, ez if abs(ez) > 1e-3 else 1e-3)
|
||||
pitch = math.atan2(ey, ez if abs(ez) > 1e-3 else 1e-3)
|
||||
if ez < 0: # target behind: commit to a full turn
|
||||
yaw = math.copysign(math.pi / 2, ex if ex else 1.0)
|
||||
sx = max(-1.0, min(1.0, self.KP * yaw - self.KD * float(np.dot(w, up))))
|
||||
sy = max(-1.0, min(1.0, -(self.KP * pitch - self.KD * float(np.dot(w, right)))))
|
||||
return sx, sy, yaw, pitch
|
||||
|
||||
# ---------------------------------------------------------------- step
|
||||
def step(self, t, dt):
|
||||
ents = self.W.sample(t)
|
||||
me = next((e for e in ents if "Player" in e[1]), None)
|
||||
if me is None:
|
||||
return None
|
||||
me_off, me_nm, me_p, me_v, me_r = me
|
||||
M = self.W.rot(me_off)
|
||||
if M is None:
|
||||
return "no-orientation"
|
||||
fwd = M[self.W.fwd_row] * self.W.fwd_sign
|
||||
right = M[(self.W.fwd_row + 1) % 3]
|
||||
up = np.cross(fwd, right)
|
||||
speed = float(np.linalg.norm(me_v))
|
||||
|
||||
hull, shield = self.own(me_off)
|
||||
if self.hp0 is None and math.isfinite(hull) and hull > 0:
|
||||
self.hp0 = hull
|
||||
self.hp_hist.append((t, hull))
|
||||
# damage over the last ~2 s; the hull only ever falls, so any drop is a hit
|
||||
recent = [h for (ts, h) in self.hp_hist if t - ts <= 2.0]
|
||||
dmg = (max(recent) - hull) if recent else 0.0
|
||||
if dmg > 0.5:
|
||||
self.last_hit = t
|
||||
if hull <= 0:
|
||||
return "DEAD"
|
||||
|
||||
# body angular velocity, for the damping term
|
||||
w = np.zeros(3)
|
||||
if self.prevM is not None and dt > 1e-3:
|
||||
D = self.prevM @ M.T
|
||||
w = np.array([D[2, 1] - D[1, 2], D[0, 2] - D[2, 0], D[1, 0] - D[0, 1]]) / (2 * dt)
|
||||
self.prevM = M
|
||||
|
||||
hos = self.hostiles(ents, me_off)
|
||||
self.threat_dir = self.threat_vector(me_p, hos)
|
||||
frac = hull / self.hp0 if self.hp0 else 1.0
|
||||
|
||||
# ---- mode
|
||||
if frac <= self.RETIRE_FRAC:
|
||||
self.mode = "RETIRE"
|
||||
elif t - self.last_hit < self.EVADE_QUIET:
|
||||
self.mode = "EVADE"
|
||||
else:
|
||||
self.mode = "ENGAGE"
|
||||
|
||||
tgt = self.pick(me_p, me_v, fwd, hos)
|
||||
push, worst = self.av.avoidance(me_p, me_v, me_r, ents, me_off)
|
||||
|
||||
if self.mode == "EVADE":
|
||||
# Away from the guns, plus a jink so a straight escape line is not
|
||||
# itself an easy solution for whatever is shooting.
|
||||
away = -self.threat_dir if self.threat_dir is not None else fwd
|
||||
jink = right * math.sin(t * 1.7) * 0.5 + up * math.cos(t * 2.3) * 0.35
|
||||
want = norm(away + jink)
|
||||
self.set_throttle(+1)
|
||||
fire = False
|
||||
elif self.mode == "RETIRE":
|
||||
base = self.friendly_base(ents, me_off)
|
||||
if base is not None:
|
||||
rel = base[2] - me_p
|
||||
d = float(np.linalg.norm(rel))
|
||||
want = norm(rel) if d > base[4] + 400.0 else norm(np.cross(rel, up))
|
||||
else:
|
||||
want = -self.threat_dir if self.threat_dir is not None else fwd
|
||||
self.set_throttle(+1)
|
||||
fire = False
|
||||
else:
|
||||
# Straight lead pursuit and fly *through*. An earlier version orbited
|
||||
# once inside a standoff radius and braked while doing it: it then
|
||||
# circled one attacker for 40 s at ~700 m, at 60-100 units/s, never
|
||||
# inside the firing cone. Overshooting and re-acquiring is better
|
||||
# than a stall in the middle of a battle; collision avoidance already
|
||||
# keeps a fighter-sized margin.
|
||||
want = norm(tgt[3]) if tgt else fwd
|
||||
if tgt and tgt[4] > 2500.0:
|
||||
self.set_throttle(+1)
|
||||
elif tgt and tgt[4] < 500.0 and speed > 900.0:
|
||||
self.set_throttle(-1)
|
||||
else:
|
||||
self.set_throttle(0)
|
||||
fire = True
|
||||
|
||||
# a turret inside its keep-out radius outranks the target
|
||||
for off, nm, p, v, r, hard in hos:
|
||||
if not hard:
|
||||
continue
|
||||
d = float(np.linalg.norm(p - me_p))
|
||||
if d < self.TURRET_KEEPOUT:
|
||||
want = norm(want + norm(me_p - p) * (2.0 * (1.0 - d / self.TURRET_KEEPOUT)))
|
||||
break
|
||||
|
||||
pn = float(np.linalg.norm(push))
|
||||
if pn > 1e-6:
|
||||
want = norm(want + push * (3.0 if pn > 0.6 else 1.5))
|
||||
|
||||
sx, sy, yaw, pitch = self.sticks(want, M, w)
|
||||
# The firing gate has to be measured against the TARGET, not against the
|
||||
# commanded direction: `want` carries the avoidance and keep-out terms,
|
||||
# so gating on it means the guns stay cold exactly when the loop is
|
||||
# manoeuvring — which is most of a dogfight.
|
||||
aim = self.sticks(norm(tgt[3]), M, w)[2:] if tgt else (math.pi, math.pi)
|
||||
aim_ok = abs(aim[0]) < self.FIRE_CONE and abs(aim[1]) < self.FIRE_CONE
|
||||
fire = bool(fire and tgt and aim_ok and tgt[4] < self.FIRE_RANGE and pn < 1.2)
|
||||
|
||||
if not self.dry:
|
||||
self.pad.axis("LX", sx)
|
||||
self.pad.axis("LY", sy)
|
||||
if fire != self.firing:
|
||||
(self.pad.press if fire else self.pad.release)("RB")
|
||||
self.firing = fire
|
||||
|
||||
msg = (f"{self.mode:<7} hull={hull:6.0f} shd={shield:6.0f} spd={speed:6.0f} "
|
||||
f"thr={self.throttle:+d} yaw={math.degrees(yaw):+6.1f} "
|
||||
f"pit={math.degrees(pitch):+6.1f} aim={math.degrees(aim[0]):+6.1f}"
|
||||
f"/{math.degrees(aim[1]):+6.1f} fire={int(fire)}")
|
||||
if dmg > 0.5:
|
||||
msg += f" HIT -{dmg:.0f}"
|
||||
if tgt:
|
||||
msg += f" tgt={tgt[1][3:24]:<21} d={tgt[4]:6.0f}"
|
||||
if worst:
|
||||
msg += f" | AVOID {worst[1][3:18]} miss={worst[3]:5.0f} t={worst[4]:4.1f}"
|
||||
return msg
|
||||
|
||||
def run(self, secs):
|
||||
self.W.scan()
|
||||
t0 = time.time()
|
||||
last, last_scan = t0, 0.0
|
||||
hostiles0 = None
|
||||
while time.time() - t0 < secs:
|
||||
t = time.time()
|
||||
if t - last_scan > 5.0:
|
||||
ents = self.W.scan()
|
||||
last_scan = t
|
||||
n_ad = sum(1 for _, va in ents
|
||||
if navigator.faction(self.W.defs[va]) == "ADAN")
|
||||
if hostiles0 is None:
|
||||
hostiles0 = n_ad
|
||||
print(f"[{t-t0:6.1f}] scan: {len(ents)} entities, {n_ad} ADAN "
|
||||
f"(start {hostiles0})", file=self.log, flush=True)
|
||||
msg = self.step(t, t - last)
|
||||
last = t
|
||||
if msg is None:
|
||||
print(f"[{t-t0:6.1f}] player object gone — stopping",
|
||||
file=self.log, flush=True)
|
||||
break
|
||||
print(f"[{t-t0:6.1f}] {msg}", file=self.log, flush=True)
|
||||
if msg == "DEAD":
|
||||
break
|
||||
time.sleep(max(0.0, 1.0 / self.HZ - (time.time() - t)))
|
||||
if not self.dry:
|
||||
self.pad.reset()
|
||||
print(f"# flew {time.time()-t0:.0f}s", file=self.log, flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
cfg = json.load(open(sys.argv[1]))
|
||||
secs = float(sys.argv[2]) if len(sys.argv) > 2 else 180.0
|
||||
W = navigator.World(cfg)
|
||||
Pilot(W, Pad(), dry="--dry" in sys.argv).run(secs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
80
tools/re-capture/schema_order.py
Normal file
80
tools/re-capture/schema_order.py
Normal file
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Recover a sub-record's FULL schema field order by merging every unit .tbl.
|
||||
|
||||
Each .tbl's string pool lists only the fields that .tbl declares, in schema
|
||||
order. Merging them is a topological sort over the pairwise "k[i] precedes
|
||||
k[i+1]" constraints of all 110 tables: any single table is a subsequence of the
|
||||
schema, so the union order is the schema order — provided the constraints are
|
||||
acyclic, which is itself the check that the premise holds.
|
||||
|
||||
Usage: schema_order.py <raw-token-dump> <SubRecord> [solved.txt]
|
||||
"""
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
sys.path.insert(0, ".")
|
||||
from order_check import section, keys_in_order, solved_offsets # noqa: E402
|
||||
|
||||
SUBS = {"Generic", "Maneuver", "Shield", "Explosion", "Mass", "Effect", "SE"}
|
||||
|
||||
|
||||
def merged_order(dump, sub):
|
||||
tbls = [l.split()[1] for l in open(dump) if l.startswith("RAW ")]
|
||||
succ, indeg, nodes = defaultdict(set), defaultdict(int), []
|
||||
seqs = []
|
||||
for t in tbls:
|
||||
ks = keys_in_order(section(dump, t, sub, SUBS))
|
||||
if ks:
|
||||
seqs.append(ks)
|
||||
for k in ks:
|
||||
if k not in nodes:
|
||||
nodes.append(k)
|
||||
for a, b in zip(ks, ks[1:]):
|
||||
if b not in succ[a]:
|
||||
succ[a].add(b)
|
||||
indeg[b] += 1
|
||||
# Kahn, breaking ties by first-seen order so the result is deterministic
|
||||
ready = [n for n in nodes if indeg[n] == 0]
|
||||
out, cycles = [], False
|
||||
while ready:
|
||||
ready.sort(key=nodes.index)
|
||||
n = ready.pop(0)
|
||||
out.append(n)
|
||||
for m in sorted(succ[n], key=nodes.index):
|
||||
indeg[m] -= 1
|
||||
if indeg[m] == 0:
|
||||
ready.append(m)
|
||||
if len(out) != len(nodes):
|
||||
cycles = True
|
||||
out += [n for n in nodes if n not in out]
|
||||
return out, seqs, cycles
|
||||
|
||||
|
||||
def main():
|
||||
dump, sub = sys.argv[1], sys.argv[2]
|
||||
order, seqs, cycles = merged_order(dump, sub)
|
||||
print(f"# {sub}: {len(order)} fields merged from {len(seqs)} tables"
|
||||
f"{' !! CYCLE — order is not consistent' if cycles else ''}")
|
||||
# every table must be a subsequence of the merged order — the real check
|
||||
pos = {k: i for i, k in enumerate(order)}
|
||||
bad = [s for s in seqs if [pos[k] for k in s] != sorted(pos[k] for k in s)]
|
||||
print(f"# tables that are NOT a subsequence of the merged order: {len(bad)}")
|
||||
|
||||
if len(sys.argv) > 3:
|
||||
solved = solved_offsets(sys.argv[3])
|
||||
anchors = [(i, k, solved[k]) for i, k in enumerate(order) if k in solved]
|
||||
votes = defaultdict(list)
|
||||
for i, k, (off, *_) in anchors:
|
||||
votes[off - 4 * i].append((i, k))
|
||||
print(f"# {len(anchors)} solved anchors; base votes:")
|
||||
for b, ks in sorted(votes.items(), key=lambda kv: -len(kv[1])):
|
||||
idx = sorted(i for i, _ in ks)
|
||||
print(f"# base {b:#07x}: {len(ks):3d} anchors, indices {idx[0]}..{idx[-1]}")
|
||||
for i, k in enumerate(order):
|
||||
s = solved.get(k)
|
||||
note = f"solver={s[0]:#x} {s[1]} agree={s[2]} dist={s[3]}" if s else ""
|
||||
print(f" [{i:3d}] {k:<34} {note}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
208
tools/re-capture/selfstate.py
Normal file
208
tools/re-capture/selfstate.py
Normal file
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Derive everything the autopilot needs about the live world, and write it to
|
||||
a JSON config. Each step is checked against an independent property, so a wrong
|
||||
answer has to survive several unrelated tests.
|
||||
|
||||
1. moving position triples (motion, whole-RAM diff)
|
||||
2. which one is US (turn axis reverses with our stick)
|
||||
3. our orientation matrix (a row must track our velocity)
|
||||
4. how to type any entity (fixed offset to its definition
|
||||
pointer, learned from ours)
|
||||
|
||||
Output: {"pos_va":…, "rot_va":…, "fwd_row":…, "def_delta":…}
|
||||
|
||||
Usage: selfstate.py <out.json>
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
import gworld # noqa: E402
|
||||
from findrot_global import scan as scan_rot # noqa: E402
|
||||
|
||||
FIFO = "/tmp/sylph-vgamepad.fifo"
|
||||
|
||||
|
||||
def pad(line):
|
||||
with open(FIFO, "w") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
|
||||
def arrays(fd, size):
|
||||
for a, b in gmem.extents(fd, size):
|
||||
n = (b - a) // 4 * 4
|
||||
if n >= 64:
|
||||
yield a, np.frombuffer(os.pread(fd, n, a), dtype=">f4")
|
||||
|
||||
|
||||
def moving_triples(fd, size, lo=120.0, hi=1600.0, dt=0.4):
|
||||
s0 = list(arrays(fd, size))
|
||||
t0 = time.time()
|
||||
time.sleep(dt)
|
||||
s1 = list(arrays(fd, size))
|
||||
t1 = time.time()
|
||||
out = []
|
||||
for (o, a), (o2, b) in zip(s0, s1):
|
||||
if o != o2 or len(a) != len(b):
|
||||
continue
|
||||
with np.errstate(invalid="ignore"):
|
||||
af = np.nan_to_num(a.astype(np.float64), nan=0, posinf=0, neginf=0)
|
||||
bf = np.nan_to_num(b.astype(np.float64), nan=0, posinf=0, neginf=0)
|
||||
d = bf - af
|
||||
idx = np.flatnonzero(np.abs(d) > 1e-3)
|
||||
cand = idx[np.isin(idx + 1, idx) & np.isin(idx + 2, idx)]
|
||||
for i in cand:
|
||||
v = np.array([d[i], d[i + 1], d[i + 2]])
|
||||
sp = float(np.linalg.norm(v)) / (t1 - t0)
|
||||
if lo < sp < hi:
|
||||
out.append(o + int(i) * 4)
|
||||
return out
|
||||
|
||||
|
||||
def read3(fd, off):
|
||||
b = os.pread(fd, 12, off)
|
||||
return np.array(struct.unpack(">3f", b)) if len(b) == 12 else np.full(3, np.nan)
|
||||
|
||||
|
||||
def sample_positions(fd, offs, secs, hz):
|
||||
seq = []
|
||||
n = int(secs * hz)
|
||||
for _ in range(n):
|
||||
t = time.time()
|
||||
seq.append((t, np.array([read3(fd, o) for o in offs])))
|
||||
time.sleep(max(0, 1.0 / hz - (time.time() - t)))
|
||||
return seq
|
||||
|
||||
|
||||
def mean_turn_axis(seq):
|
||||
ts = [t for t, _ in seq]
|
||||
ps = [p for _, p in seq]
|
||||
vs = [(ps[k + 1] - ps[k]) / max(ts[k + 1] - ts[k], 1e-3) for k in range(len(ps) - 1)]
|
||||
acc = np.zeros((ps[0].shape[0], 3))
|
||||
n = 0
|
||||
for k in range(len(vs) - 1):
|
||||
c = np.cross(vs[k], vs[k + 1])
|
||||
nr = np.linalg.norm(c, axis=1, keepdims=True)
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
acc += np.where(nr > 1e-9, c / nr, 0.0)
|
||||
n += 1
|
||||
return acc / max(n, 1)
|
||||
|
||||
|
||||
def main():
|
||||
out_path = sys.argv[1]
|
||||
fd = os.open(gmem.mem_path(), os.O_RDONLY)
|
||||
size = os.path.getsize(gmem.mem_path())
|
||||
|
||||
pad("reset")
|
||||
time.sleep(1.5)
|
||||
offs = moving_triples(fd, size)
|
||||
print(f"# {len(offs)} moving triples", flush=True)
|
||||
if not offs:
|
||||
sys.exit("nothing moving")
|
||||
|
||||
# ---- 2. which one is us: turn axis must reverse with the stick ----------
|
||||
axes = {}
|
||||
for name, lx in (("L", -0.95), ("R", 0.95)):
|
||||
pad(f"axis LX {lx}")
|
||||
time.sleep(0.6)
|
||||
seq = sample_positions(fd, offs, 2.5, 6.0)
|
||||
axes[name] = mean_turn_axis(seq)
|
||||
pad("axis LX 0")
|
||||
time.sleep(1.5)
|
||||
print(f"# phase {name} sampled", flush=True)
|
||||
pad("reset")
|
||||
aL, aR = axes["L"], axes["R"]
|
||||
nL, nR = np.linalg.norm(aL, axis=1), np.linalg.norm(aR, axis=1)
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
cos = np.sum(aL * aR, axis=1) / (nL * nR)
|
||||
good = np.isfinite(cos) & (nL > 0.6) & (nR > 0.6)
|
||||
if not good.any():
|
||||
sys.exit("no object reversed with the stick")
|
||||
k = int(np.argmin(np.where(good, cos, 9e9)))
|
||||
pos_off = offs[k]
|
||||
print(f"# self position: va {gmem.primary_va(pos_off):#010x} cos={cos[k]:+.3f}")
|
||||
|
||||
# ---- 3. orientation: a row must track our velocity ----------------------
|
||||
time.sleep(1.0)
|
||||
rots = scan_rot(fd, size)
|
||||
print(f"# {len(rots)} orthonormal blocks; matching one to our velocity", flush=True)
|
||||
seq = sample_positions(fd, [pos_off], 2.0, 8.0)
|
||||
ps = [p[0] for _, p in seq]
|
||||
ts = [t for t, _ in seq]
|
||||
vdirs = []
|
||||
for a, b, ta, tb in zip(ps, ps[1:], ts, ts[1:]):
|
||||
v = (b - a) / max(tb - ta, 1e-3)
|
||||
n = np.linalg.norm(v)
|
||||
if n > 1e-3:
|
||||
vdirs.append(v / n)
|
||||
vdir = np.mean(vdirs, axis=0)
|
||||
vdir /= np.linalg.norm(vdir)
|
||||
speed = float(np.mean([np.linalg.norm((b - a) / max(tb - ta, 1e-3))
|
||||
for a, b, ta, tb in zip(ps, ps[1:], ts, ts[1:])]))
|
||||
print(f"# our heading {vdir.round(3)} speed {speed:.1f}/s")
|
||||
|
||||
# A block found by the earlier scan may have been overwritten by the time we
|
||||
# read it, so re-verify orthonormality here -- otherwise a garbage window
|
||||
# yields a meaningless (and unbounded) "cosine".
|
||||
best = None
|
||||
for ro in rots:
|
||||
b = os.pread(fd, 36, ro)
|
||||
if len(b) < 36:
|
||||
continue
|
||||
m = np.array(struct.unpack(">9f", b))
|
||||
if not np.all(np.isfinite(m)):
|
||||
continue
|
||||
M = m.reshape(3, 3)
|
||||
if np.max(np.abs(M @ M.T - np.eye(3))) > 5e-3:
|
||||
continue
|
||||
for r in range(3):
|
||||
c = float(M[r] @ vdir)
|
||||
if best is None or abs(c) > abs(best[0]):
|
||||
best = (c, ro, r)
|
||||
if best is None:
|
||||
sys.exit("no valid orientation block")
|
||||
print(f"# best orientation match: va {gmem.primary_va(best[1]):#010x} "
|
||||
f"row {best[2]} cos={best[0]:+.4f}")
|
||||
|
||||
# ---- 4. how to type an entity ------------------------------------------
|
||||
w = gworld.World()
|
||||
defs = {}
|
||||
for doff in w.scan_vtable(gworld.DEF_VTABLE):
|
||||
nm = w.name_of(doff)
|
||||
if nm and nm.startswith("UN_"):
|
||||
va = gmem.primary_va(doff)
|
||||
if va is not None:
|
||||
defs[struct.pack(">I", va)] = nm
|
||||
delta = None
|
||||
blob = os.pread(fd, 0x1000, max(0, pos_off - 0x800))
|
||||
for i in range(0, len(blob) - 3, 4):
|
||||
nm = defs.get(blob[i:i + 4])
|
||||
if nm:
|
||||
delta = (pos_off - 0x800) + i - pos_off
|
||||
print(f"# definition pointer at pos{delta:+#07x} -> {nm}")
|
||||
break
|
||||
if delta is None:
|
||||
print("# no definition pointer near our position (entity typing unavailable)")
|
||||
|
||||
cfg = {
|
||||
"pos_va": gmem.primary_va(pos_off),
|
||||
"rot_va": gmem.primary_va(best[1]),
|
||||
"fwd_row": best[2],
|
||||
"fwd_sign": 1 if best[0] > 0 else -1,
|
||||
"def_delta": delta,
|
||||
"cruise_speed": speed,
|
||||
}
|
||||
json.dump(cfg, open(out_path, "w"), indent=1)
|
||||
print("\n" + json.dumps(cfg, indent=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -3,16 +3,37 @@
|
||||
# - movie playing (two grabs 0.6s apart differ a lot) -> tap A to skip
|
||||
# - static screen -> if the green "PRESS (A) BUTTON" glyph is there, tap A and stop
|
||||
# Static logo screens are left alone, so a stray tap can never land on NEW GAME.
|
||||
#
|
||||
# LIVENESS IS CHECKED EVERY ITERATION, and that is not a nicety. When the
|
||||
# display died 3 minutes into a run, `screenshot` started failing silently and
|
||||
# left /tmp/f1.png and /tmp/f2.png at their last contents — two *stale* files,
|
||||
# which compare to a constant non-zero RMSE, which reads exactly like "the
|
||||
# screen is changing a lot". So the loop reported "movie -> skip A" every 5 s
|
||||
# for the full 600 s timeout with no emulator and no X server running. A dead
|
||||
# display and a playing movie must never be able to look the same.
|
||||
set -u
|
||||
export HOME=/sylph-home/re
|
||||
DISP="${DISPLAY:-:98}"
|
||||
alive(){ ps -o pid=,stat= -C xenia_canary 2>/dev/null | awk '$2 !~ /^Z/ {print $1}'; }
|
||||
# The X root keeps the DEAD session's last frame, so a fresh launch would be
|
||||
# detected as "already at the title". Blank it, and wait for the new window.
|
||||
xsetroot -solid black 2>/dev/null || true
|
||||
until xdotool search --name "Xenia-canary" >/dev/null 2>&1; do sleep 1; done
|
||||
until xdotool search --name "Xenia-canary" >/dev/null 2>&1; do
|
||||
xdpyinfo -display "$DISP" >/dev/null 2>&1 || { echo "DISPLAY LOST at ${SECONDS}s (before the window appeared)"; exit 3; }
|
||||
[ -n "$(alive)" ] || { echo "EMULATOR GONE at ${SECONDS}s (before the window appeared)"; exit 4; }
|
||||
sleep 1
|
||||
done
|
||||
deadline=$(( SECONDS + ${1:-900} ))
|
||||
while [ $SECONDS -lt $deadline ]; do
|
||||
rm -f /tmp/f1.png /tmp/f2.png
|
||||
screenshot /tmp/f1.png >/dev/null 2>&1; sleep 0.6
|
||||
screenshot /tmp/f2.png >/dev/null 2>&1
|
||||
if [ ! -s /tmp/f1.png ] || [ ! -s /tmp/f2.png ]; then
|
||||
xdpyinfo -display "$DISP" >/dev/null 2>&1 \
|
||||
|| { echo "DISPLAY LOST at ${SECONDS}s"; exit 3; }
|
||||
echo "SCREENSHOT FAILED at ${SECONDS}s with the display up"; exit 5
|
||||
fi
|
||||
[ -n "$(alive)" ] || { echo "EMULATOR GONE at ${SECONDS}s"; exit 4; }
|
||||
d=$(compare -metric RMSE /tmp/f1.png /tmp/f2.png null: 2>&1 | sed 's/ .*//' | cut -d. -f1)
|
||||
d=${d:-0}
|
||||
read -r r g b < <(convert /tmp/f2.png -format "%[fx:int(255*p{625,618}.r)] %[fx:int(255*p{625,618}.g)] %[fx:int(255*p{625,618}.b)]" info:)
|
||||
|
||||
123
tools/re-capture/unit_discover.py
Normal file
123
tools/re-capture/unit_discover.py
Normal file
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Discover the runtime class that carries the parsed `UN_*` unit definitions.
|
||||
|
||||
No vtable VA is assumed. Procedure:
|
||||
1. locate every disc unit-ID string in the snapshot;
|
||||
2. find every 4-byte-aligned word in RAM that points at (string_va - d) for a
|
||||
few plausible name-record deltas;
|
||||
3. for each such pointer site P and each small back-offset k, read the word at
|
||||
P-k and tally it across *distinct* unit IDs.
|
||||
A word that appears at the same (d, k) for many different units is that class's
|
||||
vtable pointer, and k is the ID-pointer offset inside the object.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
sys.path.insert(0, "/home/fabi/RE - Project Sylpheed/sylpheed-reborn/tools/re-capture")
|
||||
import gmem # noqa: E402
|
||||
|
||||
DELTAS = [0x00, 0x10, 0x08, 0x04, 0x0C, 0x14, 0x18, 0x20]
|
||||
BACK = list(range(0, 0x60, 4))
|
||||
|
||||
|
||||
def load_ids(path):
|
||||
ids = []
|
||||
for line in open(path):
|
||||
if line.startswith("REC "):
|
||||
p = line.split()
|
||||
if p[3] == "Generic" and p[4].startswith("UN_"):
|
||||
ids.append(p[4])
|
||||
return sorted(set(ids))
|
||||
|
||||
|
||||
def scan_strings(f, size, ids):
|
||||
"""{id: [va,...]} — every NUL-terminated occurrence of each id string."""
|
||||
pats = {i: (i.encode() + b"\0") for i in ids}
|
||||
rx = re.compile(b"|".join(re.escape(p) for p in pats.values()))
|
||||
out = defaultdict(list)
|
||||
for a, b in gmem.extents(f.fileno(), size):
|
||||
f.seek(a)
|
||||
blob = f.read(b - a)
|
||||
for m in rx.finditer(blob):
|
||||
off = a + m.start()
|
||||
va = gmem.primary_va(off)
|
||||
if va is not None:
|
||||
out[m.group(0)[:-1].decode()].append(va)
|
||||
return out
|
||||
|
||||
|
||||
def scan_pointers(f, size, targets):
|
||||
"""{target_va: [site_va,...]} for every aligned word equal to a target."""
|
||||
want = {struct.pack(">I", t): t for t in targets}
|
||||
rx = re.compile(b"|".join(re.escape(k) for k in want))
|
||||
out = defaultdict(list)
|
||||
for a, b in gmem.extents(f.fileno(), size):
|
||||
f.seek(a)
|
||||
blob = f.read(b - a)
|
||||
for m in rx.finditer(blob):
|
||||
off = a + m.start()
|
||||
if off % 4:
|
||||
continue
|
||||
va = gmem.primary_va(off)
|
||||
if va is not None:
|
||||
out[want[m.group(0)]].append(va)
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ids = load_ids(sys.argv[1])
|
||||
path = gmem.mem_path()
|
||||
size = os.path.getsize(path)
|
||||
with open(path, "rb") as f:
|
||||
strs = scan_strings(f, size, ids)
|
||||
print(f"# {len(ids)} disc unit IDs, {len(strs)} found in RAM "
|
||||
f"({sum(len(v) for v in strs.values())} occurrences)", file=sys.stderr)
|
||||
|
||||
# every (id, string_va - delta) we want pointers to
|
||||
targets, owner = set(), defaultdict(set)
|
||||
for uid, vas in strs.items():
|
||||
for va in vas:
|
||||
for d in DELTAS:
|
||||
t = va - d
|
||||
targets.add(t)
|
||||
owner[t].add((uid, d))
|
||||
ptrs = scan_pointers(f, size, targets)
|
||||
print(f"# {len(ptrs)} of {len(targets)} candidate targets are pointed at",
|
||||
file=sys.stderr)
|
||||
|
||||
# tally (delta, back-offset, word) over distinct unit IDs
|
||||
tally = defaultdict(set)
|
||||
sites = defaultdict(list)
|
||||
for tgt, plist in ptrs.items():
|
||||
for uid, d in owner[tgt]:
|
||||
for p in plist:
|
||||
for k in BACK:
|
||||
base = p - k
|
||||
try:
|
||||
f.seek(gmem.va_to_off(base))
|
||||
except ValueError:
|
||||
continue
|
||||
w = f.read(4)
|
||||
if len(w) < 4:
|
||||
continue
|
||||
(vt,) = struct.unpack(">I", w)
|
||||
if 0x82000000 <= vt < 0x83000000:
|
||||
tally[(d, k, vt)].add(uid)
|
||||
sites[(d, k, vt)].append((base, uid))
|
||||
|
||||
rank = sorted(tally.items(), key=lambda kv: -len(kv[1]))
|
||||
print(f"{'delta':>6} {'idoff':>6} {'word':>10} {'#ids':>5}")
|
||||
for (d, k, vt), s in rank[:25]:
|
||||
print(f"{d:#6x} {k:#6x} {vt:#010x} {len(s):5d} e.g. {sorted(s)[:2]}")
|
||||
if rank:
|
||||
best = rank[0][0]
|
||||
print("\n# object bases for the top candidate (first 20):", file=sys.stderr)
|
||||
for base, uid in sorted(set(sites[best]))[:20]:
|
||||
print(f"# {base:#010x} {uid}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
304
tools/re-capture/unit_runtime.py
Normal file
304
tools/re-capture/unit_runtime.py
Normal file
@@ -0,0 +1,304 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Solve the runtime `Unit` (craft/vessel definition) struct layout against the
|
||||
disc's `unit\\UN_*.tbl` records, then read the fields the disc leaves defaulted.
|
||||
|
||||
Same method as weapon_runtime.py, with two differences forced by the data:
|
||||
* the runtime class is *discovered*, not assumed — see unit_discover.py; the
|
||||
definition objects carry vtable 0x820af844 and hold a pointer to their name
|
||||
record at +0x04 (string at name+0x10), exactly like `Weapon`;
|
||||
* a unit `.tbl` is one object made of several sub-records (Generic, Maneuver,
|
||||
Shield, Explosion, Mass, Effect, SE, Turret_*), all flattened into ONE
|
||||
runtime object, so the disc side is merged per unit ID.
|
||||
|
||||
Only the *definition* objects are used. The other class found in RAM
|
||||
(0x820af030) is the spawned-entity instance — same ID string, live state, not
|
||||
definition data — and is deliberately ignored.
|
||||
|
||||
Usage: unit_runtime.py <tokens.txt> <snapshot...> (snapshots are unioned)
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
sys.path.insert(0, "/home/fabi/RE - Project Sylpheed/sylpheed-reborn/tools/re-capture")
|
||||
import gmem # noqa: E402
|
||||
|
||||
DEF_VTABLE = 0x820AF844
|
||||
STRIDE = 0x380
|
||||
SUBRECS = ("Generic", "Maneuver", "Shield", "Explosion", "Mass", "Effect", "SE")
|
||||
|
||||
# `Maneuver` fields that EVERY unit table leaves at its default, so the solver
|
||||
# can never bind them: there is no disc value anywhere to score against. Their
|
||||
# offsets come from the declaration-order rule (see schema_order.py), which 29
|
||||
# solved anchors confirm, and each of these sits *between* two of those anchors
|
||||
# -- `YawDragFactor +0x104` .. `ArterBurner_Vc +0x114` and
|
||||
# `ReverseThrust_Vc +0x118` .. `ReverseThrust_Acc +0x120`. Reported separately
|
||||
# and marked `interpolated`, never mixed into the solved set.
|
||||
INTERPOLATED = [
|
||||
(0x108, "PitchDragFactor", "f32"),
|
||||
(0x10C, "RollDragFactor", "f32"),
|
||||
(0x110, "DragFactorThreshold", "f32"),
|
||||
(0x11C, "ArterBurner_Acc", "f32"),
|
||||
(0x128, "DecPitchFactor", "f32"),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- disc side
|
||||
|
||||
|
||||
def read_tokens(path):
|
||||
"""{unit_id: {field: value}} merged over the tbl's sub-records.
|
||||
|
||||
A field name that two sub-records of the same unit define with different
|
||||
values is ambiguous *within* the object, so it is dropped rather than
|
||||
silently resolved.
|
||||
"""
|
||||
per_hash = defaultdict(lambda: {"id": None, "f": defaultdict(set)})
|
||||
cur = None
|
||||
for line in open(path):
|
||||
if line.startswith("REC "):
|
||||
_, h, _, ty, rid = line.split()
|
||||
cur = per_hash[h]
|
||||
if ty == "Generic" and rid.startswith("UN_"):
|
||||
cur["id"] = rid
|
||||
elif line.startswith("F ") and cur is not None:
|
||||
_, k, v = line.rstrip("\n").split(" ", 2)
|
||||
cur["f"][k].add(v)
|
||||
out, ambiguous = {}, {}
|
||||
for rec in per_hash.values():
|
||||
if not rec["id"]:
|
||||
continue
|
||||
fields, bad = {}, []
|
||||
for k, vs in rec["f"].items():
|
||||
if len(vs) == 1:
|
||||
fields[k] = next(iter(vs))
|
||||
else:
|
||||
bad.append(k)
|
||||
out[rec["id"]] = fields
|
||||
if bad:
|
||||
ambiguous[rec["id"]] = sorted(bad)
|
||||
return out, ambiguous
|
||||
|
||||
|
||||
# ------------------------------------------------------------- runtime side
|
||||
|
||||
|
||||
def scan_vtable(f, size, vt):
|
||||
pat = struct.pack(">I", vt)
|
||||
hits = []
|
||||
for a, b in gmem.extents(f.fileno(), size):
|
||||
f.seek(a)
|
||||
blob = f.read(b - a)
|
||||
for m in re.finditer(re.escape(pat), blob):
|
||||
off = a + m.start()
|
||||
if off % 4 == 0:
|
||||
hits.append(off)
|
||||
return sorted(set(hits))
|
||||
|
||||
|
||||
def runtime_objects(paths):
|
||||
"""{id: raw} unioned over snapshots; identical IDs must agree byte-for-byte."""
|
||||
objs, conflicts = {}, []
|
||||
for path in paths:
|
||||
size = os.path.getsize(path)
|
||||
with open(path, "rb") as f:
|
||||
for off in scan_vtable(f, size, DEF_VTABLE):
|
||||
f.seek(off)
|
||||
raw = f.read(STRIDE)
|
||||
(p,) = struct.unpack_from(">I", raw, 4)
|
||||
try:
|
||||
f.seek(gmem.va_to_off(p + 0x10))
|
||||
except ValueError:
|
||||
continue
|
||||
nm = f.read(96).split(b"\0")[0]
|
||||
try:
|
||||
nm = nm.decode("ascii")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if not nm.startswith("UN_"):
|
||||
continue
|
||||
if nm in objs and objs[nm] != raw:
|
||||
conflicts.append((nm, os.path.basename(path)))
|
||||
objs[nm] = raw
|
||||
return objs, conflicts
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- solver
|
||||
|
||||
ENC = {
|
||||
"f32": lambda raw, o: struct.unpack_from(">f", raw, o)[0],
|
||||
"u32": lambda raw, o: float(struct.unpack_from(">I", raw, o)[0]),
|
||||
"i32": lambda raw, o: float(struct.unpack_from(">i", raw, o)[0]),
|
||||
"rad": lambda raw, o: math.degrees(struct.unpack_from(">f", raw, o)[0]),
|
||||
}
|
||||
|
||||
|
||||
def near(a, b):
|
||||
if a == b:
|
||||
return True
|
||||
return abs(a - b) <= 2e-4 * max(abs(a), abs(b), 1e-6)
|
||||
|
||||
|
||||
def solve(disc, run):
|
||||
numeric = {}
|
||||
for rid, fields in disc.items():
|
||||
if rid not in run:
|
||||
continue
|
||||
for k, v in fields.items():
|
||||
try:
|
||||
numeric.setdefault(k, {})[rid] = float(v.rstrip("fF"))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
solved = {}
|
||||
for field, samples in numeric.items():
|
||||
if len(samples) < 2:
|
||||
continue
|
||||
cands = []
|
||||
# +0x00 is the vtable and +0x04 the name-record pointer; no field lives
|
||||
# there. They are excluded explicitly because a huge-exponent pointer
|
||||
# word read as f32 is a denormal, which the tolerance test happily
|
||||
# "matches" against any disc value of 0.0.
|
||||
for off in range(8, STRIDE - 3, 4):
|
||||
for enc, fn in ENC.items():
|
||||
agree, bad = 0, []
|
||||
for rid, want in samples.items():
|
||||
got = fn(run[rid], off)
|
||||
if math.isfinite(got) and near(got, want):
|
||||
agree += 1
|
||||
else:
|
||||
bad.append((rid, want, got))
|
||||
if agree >= 2:
|
||||
cands.append((len(bad), -agree, off, enc, agree, bad))
|
||||
if not cands:
|
||||
continue
|
||||
cands.sort()
|
||||
nbad, _, off, enc, agree, bad = cands[0]
|
||||
if len(bad) > 0.2 * len(samples):
|
||||
continue
|
||||
good = {rid for rid in samples} - {b[0] for b in bad}
|
||||
distinct = len({round(samples[r], 6) for r in good})
|
||||
solved[field] = (off, enc, agree, bad, distinct)
|
||||
|
||||
# two fields cannot share one byte offset -- and the offset alone is the
|
||||
# identity, not (offset, encoding): the same word read as f32 and as i32 is
|
||||
# still one field.
|
||||
best_at = {}
|
||||
for field, (off, enc, agree, bad, distinct) in solved.items():
|
||||
score = (distinct, agree, -len(bad))
|
||||
if off not in best_at or score > best_at[off][1]:
|
||||
best_at[off] = (field, score)
|
||||
winners = {f for f, _ in best_at.values()}
|
||||
return {f: v for f, v in solved.items() if f in winners}, numeric
|
||||
|
||||
|
||||
def crosscheck(paths, solved):
|
||||
"""Which words of a unit's object vary between snapshots, and are any of
|
||||
them fields we claim to have solved?
|
||||
|
||||
Within one run the objects are byte-identical, but across runs the same
|
||||
unit differs -- pointer words hold heap addresses, which move. This
|
||||
separates "the emulator relocated it" from "the value is not definition
|
||||
data", and it is the check that keeps the solved set honest.
|
||||
"""
|
||||
per = {}
|
||||
for p in paths:
|
||||
for k, v in runtime_objects([p])[0].items():
|
||||
per.setdefault(k, {})[os.path.basename(p)] = v
|
||||
diff = set()
|
||||
shared = 0
|
||||
for k, d in per.items():
|
||||
if len(d) < 2:
|
||||
continue
|
||||
shared += 1
|
||||
vs = list(d.values())
|
||||
for v in vs[1:]:
|
||||
diff |= {i // 4 * 4 for i in range(min(len(v), len(vs[0]))) if v[i] != vs[0][i]}
|
||||
|
||||
def looks_ptr(off):
|
||||
vals = [struct.unpack_from(">I", v, off)[0]
|
||||
for d in per.values() for v in d.values() if len(v) > off + 3]
|
||||
return sum(1 for x in vals if 0x80000000 <= x < 0xC0000000), len(vals)
|
||||
|
||||
print(f"\n# cross-run check: {shared} unit(s) appear in more than one snapshot")
|
||||
print(f"# words that differ between runs: {len(diff)}")
|
||||
for off in sorted(diff):
|
||||
p, n = looks_ptr(off)
|
||||
kind = "guest pointer" if p == n else ("mixed" if p else "non-pointer")
|
||||
print(f"# +{off:#05x} {kind} ({p}/{n} look like pointers)")
|
||||
offs = {o for o, _, _, _, _ in solved.values()} | {o for o, _, _ in INTERPOLATED}
|
||||
clash = sorted(offs & diff)
|
||||
print("# solved/interpolated offsets that vary between runs: "
|
||||
+ (", ".join(f"{o:#05x}" for o in clash) if clash
|
||||
else "NONE — every reported field is run-invariant"))
|
||||
|
||||
|
||||
def main():
|
||||
tokens = sys.argv[1]
|
||||
snaps = [a for a in sys.argv[2:] if not a.startswith("--")]
|
||||
disc, ambiguous = read_tokens(tokens)
|
||||
run, conflicts = runtime_objects(snaps)
|
||||
matched = sorted(set(disc) & set(run))
|
||||
print(f"# disc units: {len(disc)} runtime objects: {len(run)} matched: {len(matched)}")
|
||||
if conflicts:
|
||||
print(f"# !! objects that differ between snapshots: {conflicts}")
|
||||
unknown = sorted(set(run) - set(disc))
|
||||
if unknown:
|
||||
print(f"# runtime IDs with no disc record: {unknown}")
|
||||
|
||||
solved, numeric = solve(disc, run)
|
||||
print(f"# numeric disc fields on matched units: {len(numeric)} solved: {len(solved)}\n")
|
||||
print(f"{'offset':>8} {'enc':>4} {'field':<32} {'agree':>5} {'dist':>4} {'bad':>3}")
|
||||
for field, (off, enc, agree, bad, distinct) in sorted(solved.items(), key=lambda kv: kv[1][0]):
|
||||
print(f"{off:#8x} {enc:>4} {field:<32} {agree:5d} {distinct:4d} {len(bad):3d}")
|
||||
|
||||
if "--crosscheck" in sys.argv:
|
||||
crosscheck(snaps, solved)
|
||||
|
||||
if "--csv" in sys.argv:
|
||||
import csv
|
||||
|
||||
w = csv.writer(open("unit-runtime-fields.csv", "w", newline=""))
|
||||
w.writerow(["unit", "field", "offset", "enc", "value", "source", "conf"])
|
||||
for rid in matched:
|
||||
for field, (off, enc, agree, bad, distinct) in sorted(
|
||||
solved.items(), key=lambda kv: kv[1][0]
|
||||
):
|
||||
conf = "confirmed" if (not bad and agree >= 10 and distinct >= 3) else "tentative"
|
||||
src = "disc" if field in disc[rid] else "defaulted-on-disc"
|
||||
w.writerow([rid, field, f"{off:#05x}", enc,
|
||||
f"{ENC[enc](run[rid], off):g}", src, conf])
|
||||
for off, field, enc in INTERPOLATED:
|
||||
w.writerow([rid, field, f"{off:#05x}", enc,
|
||||
f"{ENC[enc](run[rid], off):g}",
|
||||
"never-valued-on-disc", "interpolated"])
|
||||
|
||||
for arg in sys.argv:
|
||||
if arg.startswith("--unit="):
|
||||
rid = arg.split("=", 1)[1]
|
||||
print(f"\n# every solved field of {rid}")
|
||||
for field, (off, enc, agree, bad, distinct) in sorted(
|
||||
solved.items(), key=lambda kv: kv[1][0]
|
||||
):
|
||||
conf = "OK " if (not bad and agree >= 10 and distinct >= 3) else "thin"
|
||||
src = "disc" if field in disc.get(rid, {}) else "DEFAULTED"
|
||||
print(f" +{off:#05x} {enc} {conf} {field:<32} "
|
||||
f"{ENC[enc](run[rid], off):>14g} {src}")
|
||||
|
||||
if "--values" in sys.argv:
|
||||
print("\n# defaulted-on-disc values read from the runtime objects")
|
||||
for field, (off, enc, agree, bad, distinct) in sorted(solved.items(), key=lambda kv: kv[1][0]):
|
||||
if bad:
|
||||
continue
|
||||
miss = [(r, ENC[enc](run[r], off)) for r in matched if field not in disc[r]]
|
||||
if miss:
|
||||
print(f"\n{field} (+{off:#x}, {enc}):")
|
||||
for r, v in miss:
|
||||
print(f" {r:<48} {v:g}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
45
tools/re-capture/wait_flight.sh
Executable file
45
tools/re-capture/wait_flight.sh
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# Wait until the game is actually FLYING, then return — do not guess with sleeps.
|
||||
#
|
||||
# `launch_mission.sh` used to allow a fixed 75 s for the launch cinematic, the
|
||||
# stage load and the objective card. Under lavapipe that is not a constant: one
|
||||
# run needed 75 s, the next was still in Raymond's dialogue at 140 s, so the A
|
||||
# meant for the OBJECTIVE card was swallowed by the cutscene and the card sat
|
||||
# there forever.
|
||||
#
|
||||
# The in-flight HUD is unmistakable: the SHIELD bar is a solid bright green
|
||||
# block at the bottom of the screen, and no cutscene or menu has anything green
|
||||
# there. So poll that pixel, and tap A every few seconds until it appears (which
|
||||
# dismisses the objective card whenever it happens to be up).
|
||||
#
|
||||
# Same liveness rule as skip_intro.sh, for the same reason: a failed screenshot
|
||||
# leaves r/g/b unset, they default to 0, the green test fails, and waiting for
|
||||
# the HUD becomes indistinguishable from waiting for a dead emulator.
|
||||
set -u
|
||||
export HOME=/sylph-home/re
|
||||
DISP="${DISPLAY:-:98}"
|
||||
alive(){ ps -o pid=,stat= -C xenia_canary 2>/dev/null | awk '$2 !~ /^Z/ {print $1}'; }
|
||||
DEADLINE=$(( SECONDS + ${1:-240} ))
|
||||
PX=450; PY=640 # inside the SHIELD bar of the flight HUD
|
||||
last_tap=0
|
||||
while [ $SECONDS -lt $DEADLINE ]; do
|
||||
rm -f /tmp/wf.png
|
||||
if ! screenshot /tmp/wf.png >/dev/null 2>&1 || [ ! -s /tmp/wf.png ]; then
|
||||
xdpyinfo -display "$DISP" >/dev/null 2>&1 \
|
||||
|| { echo "DISPLAY LOST at ${SECONDS}s"; exit 3; }
|
||||
sleep 2; continue
|
||||
fi
|
||||
[ -n "$(alive)" ] || { echo "EMULATOR GONE at ${SECONDS}s"; exit 4; }
|
||||
read -r r g b < <(convert /tmp/wf.png -format \
|
||||
"%[fx:int(255*p{$PX,$PY}.r)] %[fx:int(255*p{$PX,$PY}.g)] %[fx:int(255*p{$PX,$PY}.b)]" info:)
|
||||
if [ "${g:-0}" -gt 140 ] && [ $(( g - r )) -gt 60 ] && [ $(( g - b )) -gt 60 ]; then
|
||||
echo "IN FLIGHT at ${SECONDS}s (HUD shield bar visible)"
|
||||
exit 0
|
||||
fi
|
||||
if [ $(( SECONDS - last_tap )) -ge 6 ]; then
|
||||
vgamepad tap A 250
|
||||
last_tap=$SECONDS
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "NO FLIGHT HUD within ${1:-240}s"; exit 1
|
||||
303
tools/re-capture/weapon_runtime.py
Normal file
303
tools/re-capture/weapon_runtime.py
Normal file
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Solve the runtime `Weapon`/`Shell` struct layouts against disc ground truth,
|
||||
then read the fields the disc leaves defaulted.
|
||||
|
||||
The game parses every `weapon\\Weapon_*.tbl` IDXD record into two C++ objects, a
|
||||
`Weapon` and its `Shell`. Each class lives in its own contiguous array in guest
|
||||
RAM and is identifiable by its vtable pointer. Fields the IDXD omits (the
|
||||
Route-B "defaulted" list) still hold their real values in those objects — put
|
||||
there by the constructor before the parser overwrites what the disc supplies.
|
||||
|
||||
Method (no guessing):
|
||||
1. read every disc record's explicitly-valued fields, split per sub-record
|
||||
(`sylpheed-formats --example idxd_tokens`);
|
||||
2. read every runtime object out of the live emulator (`gmem`), keyed by the
|
||||
object's own ID string;
|
||||
3. for each (field, byte-offset, encoding) triple, count how many weapons the
|
||||
offset reproduces and how many it contradicts. An offset that agrees on
|
||||
many and contradicts none is a confirmed binding;
|
||||
4. print the map, then the values at those offsets for the weapons whose disc
|
||||
record omits the field.
|
||||
|
||||
Step 3 is the whole safeguard: a wrong offset cannot silently agree with ~100
|
||||
independent records.
|
||||
|
||||
Usage: weapon_runtime.py <tokens.txt> [--md]
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import gmem # noqa: E402
|
||||
|
||||
# vtable VA -> (class name, object stride)
|
||||
CLASSES = {
|
||||
0x820AF548: ("Weapon", 0xC0),
|
||||
0x820AF58C: ("Shell", 0x200),
|
||||
}
|
||||
NAME_STR_OFF = 0x10 # the name string sits +0x10 into a name record
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- disc side
|
||||
|
||||
|
||||
def read_tokens(path):
|
||||
"""({class: {ID: {field: value}}}, {(class, ID): [field...]}) from the dump.
|
||||
|
||||
A handful of IDs are declared by more than one `.tbl`. Where two such
|
||||
declarations disagree on a field, that field is *ambiguous on disc* — the
|
||||
runtime holds whichever definition won, so it must not be scored as a
|
||||
contradiction. Those are collected separately, not silently merged.
|
||||
"""
|
||||
seen, ambiguous = {}, {}
|
||||
cur = None
|
||||
for line in open(path):
|
||||
if line.startswith("REC "):
|
||||
_, _, _, cls, rid = line.split()
|
||||
cur = (cls, rid)
|
||||
seen.setdefault(cur, [])
|
||||
seen[cur].append({})
|
||||
elif line.startswith("F ") and cur is not None:
|
||||
_, k, v = line.rstrip("\n").split(" ", 2)
|
||||
seen[cur][-1][k] = v
|
||||
|
||||
out = {}
|
||||
for (cls, rid), defs in seen.items():
|
||||
merged = {}
|
||||
for d in defs:
|
||||
merged.update(d)
|
||||
if len(defs) > 1:
|
||||
bad = {
|
||||
k
|
||||
for k in {k for d in defs for k in d}
|
||||
if len({d.get(k) for d in defs}) > 1
|
||||
}
|
||||
if bad:
|
||||
ambiguous[(cls, rid)] = sorted(bad)
|
||||
for k in bad:
|
||||
merged.pop(k, None)
|
||||
out.setdefault(cls, {})[rid] = merged
|
||||
return out, ambiguous
|
||||
|
||||
|
||||
# ------------------------------------------------------------- runtime side
|
||||
|
||||
|
||||
def scan_vtable(f, size, vt):
|
||||
pat = struct.pack(">I", vt)
|
||||
hits = []
|
||||
for a, b in gmem.extents(f.fileno(), size):
|
||||
f.seek(a)
|
||||
blob = f.read(b - a)
|
||||
for m in re.finditer(re.escape(pat), blob):
|
||||
off = a + m.start()
|
||||
if off % 4 == 0:
|
||||
hits.append(off)
|
||||
return sorted(set(hits))
|
||||
|
||||
|
||||
def runtime_objects(f, size):
|
||||
"""{class: {ID: raw_bytes}} plus {class: [(va, ID)]} in array order."""
|
||||
objs, order = {}, {}
|
||||
for vt, (cls, stride) in CLASSES.items():
|
||||
objs[cls], order[cls] = {}, []
|
||||
for off in scan_vtable(f, size, vt):
|
||||
f.seek(off)
|
||||
raw = f.read(stride)
|
||||
(nameptr,) = struct.unpack_from(">I", raw, 4)
|
||||
try:
|
||||
f.seek(gmem.va_to_off(nameptr + NAME_STR_OFF))
|
||||
name = f.read(64).split(b"\0")[0].decode("latin-1")
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
continue
|
||||
objs[cls][name] = raw
|
||||
order[cls].append((gmem.primary_va(off), name))
|
||||
return objs, order
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- solver
|
||||
|
||||
ENC = {
|
||||
"f32": lambda raw, o, ch: struct.unpack_from(">f", raw, o)[0],
|
||||
"u32": lambda raw, o, ch: float(struct.unpack_from(">I", raw, o)[0]),
|
||||
"deg": lambda raw, o, ch: math.degrees(struct.unpack_from(">f", raw, o)[0]),
|
||||
# A charging weapon drains its magazine continuously, so the two counter
|
||||
# fields are float32 on `IsCharging = Yes` records and int32 on the rest.
|
||||
# Discovered from the runtime: `IsCharging` partitions the 8 float-encoded
|
||||
# records exactly (see docs/re/weapon-struct-runtime.md).
|
||||
"cnt": lambda raw, o, ch: (
|
||||
struct.unpack_from(">f", raw, o)[0] if ch else float(struct.unpack_from(">I", raw, o)[0])
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def near(a, b):
|
||||
if a == b:
|
||||
return True
|
||||
return abs(a - b) <= 2e-4 * max(abs(a), abs(b), 1e-6)
|
||||
|
||||
|
||||
def solve(disc_cls, run_cls, stride, charging):
|
||||
"""{field: (off, enc, n_agree, [mismatch...])}"""
|
||||
numeric = {}
|
||||
for rid, fields in disc_cls.items():
|
||||
if rid not in run_cls:
|
||||
continue
|
||||
for k, v in fields.items():
|
||||
try:
|
||||
numeric.setdefault(k, {})[rid] = float(v.rstrip("fF"))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
solved = {}
|
||||
for field, samples in numeric.items():
|
||||
if len(samples) < 2:
|
||||
continue
|
||||
cands = []
|
||||
for off in range(0, stride - 3, 4):
|
||||
for enc, fn in ENC.items():
|
||||
agree, bad = 0, []
|
||||
for rid, want in samples.items():
|
||||
got = fn(run_cls[rid], off, rid in charging)
|
||||
if math.isfinite(got) and near(got, want):
|
||||
agree += 1
|
||||
else:
|
||||
bad.append((rid, want, got))
|
||||
if agree >= 2:
|
||||
cands.append((len(bad), -agree, off, enc, agree, bad))
|
||||
if not cands:
|
||||
continue
|
||||
cands.sort()
|
||||
nbad, _, off, enc, agree, bad = cands[0]
|
||||
# A binding that contradicts a large slice of the evidence is not a
|
||||
# binding at all -- it is a coincidence on the agreeing subset.
|
||||
if len(bad) > 0.2 * len(samples):
|
||||
continue
|
||||
# Discriminating power: a field whose on-disc samples are all the same
|
||||
# value matches any offset holding that constant, so such a binding is
|
||||
# a coincidence waiting to happen. Count the distinct values that the
|
||||
# *agreeing* records pin down.
|
||||
distinct = len({round(v, 6) for rid, v in samples.items() if rid not in {b[0] for b in bad}})
|
||||
solved[field] = (off, enc, agree, bad, distinct)
|
||||
|
||||
# Two fields cannot share one byte offset. Where the solver lands two on the
|
||||
# same (offset, enc), keep the better-evidenced one and drop the other.
|
||||
best_at = {}
|
||||
for field, (off, enc, agree, bad, distinct) in solved.items():
|
||||
key = (off, enc)
|
||||
score = (distinct, agree, -len(bad))
|
||||
if key not in best_at or score > best_at[key][1]:
|
||||
best_at[key] = (field, score)
|
||||
winners = {f for f, _ in best_at.values()}
|
||||
return {f: v for f, v in solved.items() if f in winners}, numeric
|
||||
|
||||
|
||||
def report(cls, disc_cls, run_cls, stride, out, charging):
|
||||
solved, numeric = solve(disc_cls, run_cls, stride, charging)
|
||||
p = out.append
|
||||
p(f"\n### `{cls}` — runtime struct ({stride:#x} bytes/object)\n")
|
||||
p("Confidence: ✅ = ≥10 disc records agree on ≥3 distinct values, none contradict.")
|
||||
p("🟡 = consistent but thin evidence. ⚠️ = the runtime contradicts the disc "
|
||||
"(listed below the table).\n")
|
||||
p("| offset | enc | field | agree | distinct values | contradict | conf |")
|
||||
p("|--------|-----|-------|------:|----------------:|-----------:|------|")
|
||||
contradictions = []
|
||||
for field, (off, enc, agree, bad, distinct) in sorted(solved.items(), key=lambda kv: kv[1][0]):
|
||||
if bad:
|
||||
conf = "⚠️"
|
||||
contradictions.append((field, off, enc, bad))
|
||||
elif agree >= 10 and distinct >= 3:
|
||||
conf = "✅"
|
||||
else:
|
||||
conf = "🟡"
|
||||
p(f"| `+{off:#05x}` | {enc} | `{field}` | {agree} | {distinct} | {len(bad)} | {conf} |")
|
||||
|
||||
unsolved = sorted(set(numeric) - set(solved))
|
||||
if unsolved:
|
||||
p(f"\nNumeric fields with no consistent offset (unsolved): "
|
||||
f"{', '.join('`'+u+'`' for u in unsolved)}")
|
||||
|
||||
for field, off, enc, bad in contradictions:
|
||||
p(f"\n**`{field}` (`+{off:#05x}`) — runtime disagrees with disc on "
|
||||
f"{len(bad)} record(s):**\n")
|
||||
for rid, want, got in sorted(bad)[:24]:
|
||||
p(f"- `{rid}`: disc `{want:g}` → runtime `{got:g}`")
|
||||
|
||||
p(f"\n#### `{cls}` values the disc defaults, read from the live objects\n")
|
||||
for field, (off, enc, agree, bad, distinct) in sorted(solved.items(), key=lambda kv: kv[1][0]):
|
||||
if bad or agree < 10 or distinct < 3:
|
||||
continue # only report from ✅ bindings
|
||||
missing = [
|
||||
(rid, ENC[enc](run_cls[rid], off, rid in charging))
|
||||
for rid in sorted(disc_cls)
|
||||
if rid in run_cls and field not in disc_cls[rid]
|
||||
]
|
||||
if not missing:
|
||||
continue
|
||||
vals = sorted({round(v, 6) for _, v in missing})
|
||||
p(f"\n**`{field}`** (`+{off:#05x}`, {enc}) — defaulted by {len(missing)} of "
|
||||
f"{len(run_cls)} records")
|
||||
if len(vals) == 1:
|
||||
p(f"\n> all = **{vals[0]:g}**")
|
||||
else:
|
||||
p("")
|
||||
for rid, v in missing:
|
||||
p(f"- `{rid}` = **{v:g}**")
|
||||
return solved
|
||||
|
||||
|
||||
def main():
|
||||
tokens = sys.argv[1] if len(sys.argv) > 1 else "/tmp/wep_tokens.txt"
|
||||
disc, ambiguous = read_tokens(tokens)
|
||||
path = gmem.mem_path()
|
||||
size = os.path.getsize(path)
|
||||
with open(path, "rb") as f:
|
||||
objs, order = runtime_objects(f, size)
|
||||
|
||||
# `IsCharging = Yes` switches the two counter fields to float32; the Shell
|
||||
# inherits the flag from its Weapon (same ID suffix).
|
||||
charging = {
|
||||
rid for rid, f in disc.get("Weapon", {}).items() if f.get("IsCharging") == "Yes"
|
||||
}
|
||||
charging |= {"Shell" + rid[len("Weapon"):] for rid in set(charging)}
|
||||
|
||||
out = []
|
||||
for cls, stride in [(c, s) for c, s in CLASSES.values()]:
|
||||
d, r = disc.get(cls, {}), objs.get(cls, {})
|
||||
matched = set(d) & set(r)
|
||||
out.append(f"\n<!-- {cls}: {len(r)} runtime objects, {len(d)} disc records, "
|
||||
f"{len(matched)} matched by ID -->")
|
||||
report(cls, d, r, stride, out, charging)
|
||||
if "--csv" in sys.argv:
|
||||
import csv
|
||||
|
||||
w = csv.writer(sys.stdout)
|
||||
w.writerow(["class", "id", "field", "offset", "enc", "value", "source", "conf"])
|
||||
for cls, stride in [(c, st) for c, st in CLASSES.values()]:
|
||||
d, r = disc.get(cls, {}), objs.get(cls, {})
|
||||
solved, _ = solve(d, r, stride, charging)
|
||||
for field, (off, enc, agree, bad, distinct) in sorted(
|
||||
solved.items(), key=lambda kv: kv[1][0]
|
||||
):
|
||||
conf = "confirmed" if (not bad and agree >= 10 and distinct >= 3) else "tentative"
|
||||
for rid in sorted(r):
|
||||
val = ENC[enc](r[rid], off, rid in charging)
|
||||
src = "disc" if field in d.get(rid, {}) else "defaulted-on-disc"
|
||||
w.writerow([cls, rid, field, f"{off:#05x}", enc, f"{val:g}", src, conf])
|
||||
return
|
||||
|
||||
if ambiguous:
|
||||
out.append("\n### Disc records declared twice, with conflicting values\n")
|
||||
out.append("These fields are ambiguous *on disc*; the runtime shows which "
|
||||
"declaration won. They are excluded from the scoring above.\n")
|
||||
for (cls, rid), fields in sorted(ambiguous.items()):
|
||||
out.append(f"- `{cls}` `{rid}`: {', '.join('`'+f+'`' for f in fields)}")
|
||||
print("\n".join(out))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
76
tools/re-capture/whatchanges.py
Normal file
76
tools/re-capture/whatchanges.py
Normal file
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Encoding-agnostic: which words of an object actually carry live state?
|
||||
|
||||
Rather than assume the orientation is a matrix (it is not) or a quaternion,
|
||||
classify every 4-byte word of the captured window by how it behaves over the
|
||||
capture: constant, smoothly varying (a continuous quantity), or jumpy.
|
||||
|
||||
Usage: whatchanges.py <probe.bin> [name-substring] [max-report]
|
||||
"""
|
||||
import math
|
||||
import struct
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/home/fabi/RE - Project Sylpheed/sylpheed-reborn/tools/re-capture")
|
||||
from flight_analyze import load # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
path = sys.argv[1]
|
||||
want = sys.argv[2] if len(sys.argv) > 2 else "Player"
|
||||
top = int(sys.argv[3]) if len(sys.argv) > 3 else 60
|
||||
insts, window, frames = load(path)
|
||||
i = next(k for k, (_, _, nm) in enumerate(insts) if want in nm)
|
||||
print(f"# {insts[i][2]} @ {insts[i][0]:#010x}, {len(frames)} frames, window {window:#x}")
|
||||
|
||||
nconst = nsmooth = njump = 0
|
||||
smooth = []
|
||||
for o in range(0, window - 3, 4):
|
||||
vals = []
|
||||
ok = True
|
||||
for t, inp, lab, bufs in frames:
|
||||
(v,) = struct.unpack_from(">f", bufs[i], o)
|
||||
if not math.isfinite(v):
|
||||
ok = False
|
||||
break
|
||||
vals.append(v)
|
||||
if not ok:
|
||||
continue
|
||||
lo, hi = min(vals), max(vals)
|
||||
if lo == hi:
|
||||
nconst += 1
|
||||
continue
|
||||
rng = hi - lo
|
||||
steps = [abs(b - a) for a, b in zip(vals, vals[1:])]
|
||||
mx = max(steps)
|
||||
# smooth = no single step is a large fraction of the whole excursion
|
||||
if mx < 0.25 * rng:
|
||||
nsmooth += 1
|
||||
smooth.append((o, lo, hi, vals))
|
||||
else:
|
||||
njump += 1
|
||||
print(f"# constant {nconst} smooth {nsmooth} jumpy {njump}")
|
||||
print(f"\n# smoothly varying words (candidate continuous state):")
|
||||
for o, lo, hi, vals in smooth[:top]:
|
||||
print(f" +{o:#05x} [{lo:12.4g} .. {hi:12.4g}] "
|
||||
f"start {vals[0]:12.4g} end {vals[-1]:12.4g}")
|
||||
|
||||
# consecutive runs of smooth words -> vectors
|
||||
offs = [o for o, _, _, _ in smooth]
|
||||
runs, cur = [], []
|
||||
for o in offs:
|
||||
if cur and o == cur[-1] + 4:
|
||||
cur.append(o)
|
||||
else:
|
||||
if len(cur) >= 3:
|
||||
runs.append(cur)
|
||||
cur = [o]
|
||||
if len(cur) >= 3:
|
||||
runs.append(cur)
|
||||
print(f"\n# runs of >=3 consecutive smooth words (vector-shaped):")
|
||||
for r in runs:
|
||||
print(f" +{r[0]:#05x} .. +{r[-1]:#05x} ({len(r)} words)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user