From f0638dc50bc5c9cbb5e60ad7dff14f3448fce4ba Mon Sep 17 00:00:00 2001 From: Fabian Hamm Date: Mon, 14 Sep 2026 20:43:16 +0200 Subject: [PATCH 1/4] feat(xexdb): port import-thunk naming + schema-wide foreign keys Carries `xenia-rs` `harvest/import-thunk-naming` (b4f19f1) across into the lifted crate. That work was found UNCOMMITTED in the retired repo's working tree on 2026-09-14 and exists nowhere else: absent from `iterate-4A`, absent from this crate as lifted. CONSOLIDATION.md Phase 5 ends with "delete the local clone" and Phase 7 drops the emulator, so it had a deletion scheduled against it. Not a cherry-pick. The lift's base is exactly the harvest's parent (8401d4d), so each file was merged three-way -- lifted vs base vs harvest -- which is what made the port reviewable: 10 conflicts, 9 of them pure rustfmt reflow from the lift's formatting pass, and 1 semantic. The semantic one is insertion ORDER. `xdbf_achievements.image_id` is now a foreign key onto `xdbf_images(id)`, so the image rows must be inserted before the achievement rows. The merge moved that block; the conflict was the stale copy left at the old position. What arrives: * **Import-thunk recognition** (`imports.rs`, 338 lines). An XEX import is not a PLT jump: the linker emits a four-word thunk whose first two words are import RECORDS that the loader rewrites at module load. On disc they are still records, so a PowerPC-only decoder prints two meaningless `.long`s in front of an indirect branch. This maps every word of every thunk, and every direct branch into one, back to its `imports` row. Shape-validated rather than trusted: an entry is indexed only when the four words it points at actually have the thunk shape. Adds `instructions.import_address` (FK onto `imports.address`) and `import_role` (`'record'` | `'thunk'` | `'call'`, NULL iff import_address is NULL), plus an index. `tools/zq.py` gains `imp` and `impcalls`, and `dis` now names imports instead of printing `.long 0x01010194`. * **Sixteen schema-wide foreign keys**, declared wherever a column is derived from another table. CREATE TABLE and insertion order become load-bearing. `functions` is deliberately NOT an FK parent and the golden test now asserts zero inbound FKs onto it: DuckDB implements UPDATE as delete+insert, so one inbound FK would make `functions.name` un-updatable and break `apply_re_symbols.sql`, which re-applies RE symbol names after every regeneration. The database path needed no change in the binary: `DbWriter` builds its own index inside `ingest_instructions` from `info.import_libraries`. Only the two output paths (`enrich_section` for JSONL, `write_asm`) take it as an argument. Verified: `cargo test -p sylpheed-xexdb` = 10 passed / 0 failed, including `db_schema_golden` (41s, builds a real DuckDB) which locks the 16-FK set and the no-FK-onto-functions rule. `cargo clippy -p sylpheed-xexdb --all-targets -- -D warnings` clean; `cargo fmt --all --check` clean. Stacked on #32 -- it ports into a crate that only exists there. Co-Authored-By: Claude Opus 5 --- crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs | 12 + crates/sylpheed-xexdb/src/db.rs | 317 +++++++++++----- crates/sylpheed-xexdb/src/disasm.rs | 22 +- crates/sylpheed-xexdb/src/formatter.rs | 21 +- crates/sylpheed-xexdb/src/imports.rs | 354 ++++++++++++++++++ crates/sylpheed-xexdb/src/lib.rs | 2 + crates/sylpheed-xexdb/src/sinks/duckdb.rs | 2 + crates/sylpheed-xexdb/src/sql_views.rs | 34 +- .../sylpheed-xexdb/tests/db_schema_golden.rs | 60 +++ tools/zq.py | 117 +++++- 10 files changed, 835 insertions(+), 106 deletions(-) create mode 100644 crates/sylpheed-xexdb/src/imports.rs diff --git a/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs b/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs index af6b5db4..48fda0de 100644 --- a/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs +++ b/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs @@ -919,6 +919,16 @@ fn cmd_dis( info!(db = %db, "database written"); } + // Word-level index of the import thunks, used to annotate the instruction + // stream: which words are loader-patched import records, and which + // branches are really kernel/xam calls. The database path builds its own + // inside `ingest_instructions`; these two output paths need it here. + let import_sites = sylpheed_xexdb::ImportSites::build( + disasm_info.import_libraries, + &pe_image, + disasm_info.image_base, + ); + // JSON Lines output: one row per instruction, structured columns. if let Some(json) = json_path { info!(json = %json, "writing JSON Lines"); @@ -939,6 +949,7 @@ fn cmd_dis( &func_analysis, &xref_result.labels, &jt_data_words, + &import_sites, ); total += sylpheed_xexdb::sinks::json::write_jsonl(&mut out, items)?; } @@ -962,6 +973,7 @@ fn cmd_dis( &xref_result.xrefs, &xref_result.data_annotations, &jt_data_words, + &import_sites, )?; if let Some(path) = output { diff --git a/crates/sylpheed-xexdb/src/db.rs b/crates/sylpheed-xexdb/src/db.rs index 76ea3bd8..6f3ba5f7 100644 --- a/crates/sylpheed-xexdb/src/db.rs +++ b/crates/sylpheed-xexdb/src/db.rs @@ -15,6 +15,33 @@ //! - `"jump"` : bcctrx without LK //! - `"branch"` : bx/bcx without LK //! +//! # Referential integrity +//! +//! The schema declares foreign keys wherever a column is *derived from* the +//! row it points at, so a build that would produce a dangling reference fails +//! loudly instead of shipping a broken database. Two consequences: +//! +//! - **Insert order is load-bearing.** DuckDB checks each row as it is +//! appended, so a parent must be populated before its children, and +//! `CREATE TABLE` statements must be ordered so a table exists before +//! anything references it. `write_analysis_results` documents its order. +//! - **`functions` is deliberately NOT a foreign-key parent.** DuckDB +//! implements `UPDATE` as delete-then-insert, so *any* FK onto +//! `functions(address)` makes the table un-updatable — even an update that +//! does not touch `address`. That would break `apply_re_symbols.sql`, which +//! re-stamps reverse-engineered names onto `functions.name` after every +//! regeneration. `xrefs.source_func`, `vptr_writes.writer_function` and +//! `jump_tables.function` are therefore plain columns. The same restriction +//! applies to any table a human post-edits. +//! +//! Some references are intentionally left undeclared because the relationship +//! genuinely does not hold: see `rtti_locators.vtable_address` (41 RTTI-named +//! vftables the M3 scanner misses), `methods.function_address` and +//! `indirect_dispatch_candidates.method_address` (vtable slots pointing at +//! code no `.pdata` record or `bl` ever declared), and `xrefs.target` (which +//! is frequently data). Those orphan sets are findings, not noise — querying +//! them is how the gaps stay visible. +//! //! # Schema //! //! ## `metadata` @@ -48,8 +75,9 @@ //! - `library` — Module name (e.g. `xboxkrnl.exe`, `xam.xex`) //! - `ordinal` — Numeric ordinal identifying the export within the library //! - `name` — Resolved human-readable symbol name; `NULL` if not in symbol table -//! - `record_type` — XEX import record type: `0` = function thunk, `1` = variable -//! - `address` — Absolute VA of the import thunk or variable in the binary +//! - `record_type` — XEX import record type: `0` = variable, `1` = function thunk +//! - `address` — Absolute VA of the import thunk or variable in the binary. +//! UNIQUE, because `instructions.import_address` is a foreign key onto it. //! //! ## `functions` //! One row per detected function. Candidates are `bl` targets ∪ `.pdata` @@ -94,9 +122,18 @@ //! - `function` — VA of the enclosing function; `NULL` if not inside a detected function //! - `label` — Label name at this address; `NULL` if none //! - `is_data` — `true` when this word is data embedded in a code section (a -//! recovered jump table or index map). The decoded `mnemonic` / -//! `disasm` columns are meaningless on such rows; filter them out -//! (`WHERE NOT is_data`) for any instruction-level analysis. +//! recovered jump table or index map, or an XEX import record). The decoded +//! `mnemonic` / `disasm` columns are meaningless on such rows; filter them +//! out (`WHERE NOT is_data`) for any instruction-level analysis. +//! - `import_address` — FK onto `imports.address` (a thunk head) when this word is +//! part of an XEX import thunk, or is a direct branch into one; `NULL` +//! otherwise. Join it to name the callee. +//! - `import_role` — Which of the three this row is: `'record'` (one of the two +//! loader-patched words at the thunk head — also `is_data`), `'thunk'` (the +//! `mtctr r11` / `bctr` body), or `'call'` (a `bl`/`b` elsewhere in the binary +//! whose target is that thunk). `NULL` iff `import_address` is `NULL`. Without +//! these columns an import call reads as a branch to a bare address and the +//! thunk head as two meaningless `.long`s. //! //! ## `jump_tables` / `jump_table_entries` / `data_in_code` //! Recovered `switch` dispatches (see [`crate::jumptables`]). `jump_tables` has @@ -153,7 +190,7 @@ //! - `kind` — `call`, `return`, `jump`, or `branch` (see top-level doc) //! - `lr` — Link register value at time of branch -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::Path; use duckdb::{Connection, params}; @@ -220,6 +257,11 @@ pub struct DbWriter { trace_instructions: bool, trace_imports: bool, trace_branches: bool, + /// Import-thunk index, built during `ingest_instructions` and reused when + /// `write_analysis_results` records the thunk records in `data_in_code`. + /// Empty until then — with no instruction rows there is nothing to + /// cross-reference it against. + import_sites: crate::imports::ImportSites, } impl DbWriter { @@ -241,6 +283,7 @@ impl DbWriter { trace_instructions: false, trace_imports: false, trace_branches: false, + import_sites: crate::imports::ImportSites::default(), }) } @@ -270,8 +313,14 @@ impl DbWriter { library VARCHAR NOT NULL, -- module name (e.g. xboxkrnl.exe, xam.xex) ordinal BIGINT NOT NULL, -- ordinal identifying the export within the library name VARCHAR, -- resolved symbol name; NULL if not in symbol table - record_type BIGINT NOT NULL, -- 0 = function thunk, 1 = variable - address BIGINT NOT NULL -- absolute VA of the thunk or variable + -- 0 = variable (a pointer slot in .rdata the loader fills in), + -- 1 = function thunk (four words of code in .text). This was + -- documented the wrong way round; `ImportEntry::record_type` + -- in xenia-xex has always had it right. + record_type BIGINT NOT NULL, + -- Absolute VA of the thunk or variable. UNIQUE because + -- `instructions.import_address` is a foreign key onto it. + address BIGINT NOT NULL UNIQUE ); ", )?; @@ -322,11 +371,31 @@ impl DbWriter { section VARCHAR NOT NULL, -- PE section name containing this instruction function BIGINT, -- VA of the enclosing function; NULL if unknown label VARCHAR, -- label at this address; NULL if none - is_data BOOLEAN NOT NULL -- M12: word is data embedded in code (jump table / index map), NOT an instruction + is_data BOOLEAN NOT NULL, -- M12: word is data embedded in code (jump table / index map / import record), NOT an instruction + -- XEX import this word references, as a foreign key into `imports` + -- (the `record_type = 1` row, whose address is the thunk head). + -- NULL for everything that is not part of, or a branch into, an + -- import thunk. See `import_role` for which of the three it is. + import_address BIGINT REFERENCES imports(address), + import_role VARCHAR -- 'record' | 'thunk' | 'call'; NULL iff import_address IS NULL ); ")?; - insert_instructions_streaming(&self.conn, pe, info, func_analysis, labels, data_words)?; + // Built here rather than passed in: everything it needs (the parsed + // import table and the loaded image) is already on `info`/`pe`, and + // the index is only ever consumed by this one streaming pass. + self.import_sites = + crate::imports::ImportSites::build(info.import_libraries, pe, info.image_base); + + insert_instructions_streaming( + &self.conn, + pe, + info, + func_analysis, + labels, + data_words, + &self.import_sites, + )?; let indices = [ ( @@ -357,6 +426,10 @@ impl DbWriter { "idx_instructions_is_data", "CREATE INDEX idx_instructions_is_data ON instructions(is_data)", ), + ( + "idx_instructions_import", + "CREATE INDEX idx_instructions_import ON instructions(import_address)", + ), ]; for (name, sql) in indices { tracing::debug!(index = name, "creating instructions index"); @@ -422,17 +495,63 @@ impl DbWriter { kind VARCHAR NOT NULL -- function | import | saverestore | local | data | other ); + -- M13 — MSVC RTTI. `rtti_type_descriptors` is the authoritative + -- source of C++ class identity: the linker wrote these names, they + -- are not inferred. + CREATE TABLE rtti_type_descriptors ( + address BIGINT PRIMARY KEY, -- VA of the TypeDescriptor + mangled_name VARCHAR NOT NULL, -- decorated name, e.g. .?AVSilph@silph@@ + demangled_name VARCHAR NOT NULL -- readable form, e.g. silph::Silph + ); + + -- M13 — RTTICompleteObjectLocator. One per emitted vftable; the + -- `subobject_offset` column separates a class's primary vftable (0) + -- from the extra vftables it emits for secondary base subobjects. + CREATE TABLE rtti_locators ( + address BIGINT PRIMARY KEY, + subobject_offset BIGINT NOT NULL, -- this-offset of the subobject this vftable serves + cd_offset BIGINT NOT NULL, -- constructor-displacement offset + type_descriptor BIGINT NOT NULL REFERENCES rtti_type_descriptors(address), + class_hierarchy BIGINT NOT NULL, -- VA of the RTTIClassHierarchyDescriptor + -- VA of vftable[0]; NULL if no referencing word was found. + -- Deliberately NOT a foreign key: on the reference title 41 + -- locators name a vftable the M3 scanner did not detect (see + -- `crate::vtables`), so declaring one would fail the build on + -- a real gap rather than reporting it. Query the orphans with + -- SELECT * FROM rtti_locators l WHERE NOT EXISTS + -- (SELECT 1 FROM vtables v WHERE v.address = l.vtable_address) + vtable_address BIGINT + ); + + -- M13 — linearised RTTIBaseClassArray. Index 0 is the class itself; + -- the remaining rows are its bases in MSVC's depth-first order, + -- each with the PMD displacement triple needed to locate the base + -- subobject inside an instance. + CREATE TABLE rtti_base_classes ( + class_hierarchy BIGINT NOT NULL, -- VA of the deriving class's hierarchy descriptor + base_index BIGINT NOT NULL, -- position in the base-class array + type_descriptor BIGINT NOT NULL REFERENCES rtti_type_descriptors(address), + name VARCHAR NOT NULL, + num_contained_bases BIGINT NOT NULL, + mdisp BIGINT NOT NULL, -- member displacement + pdisp BIGINT NOT NULL, -- vbtable displacement (-1 = non-virtual base) + vdisp BIGINT NOT NULL, -- displacement inside the vbtable + attributes BIGINT NOT NULL, + PRIMARY KEY (class_hierarchy, base_index) + ); + CREATE TABLE vtables ( address BIGINT PRIMARY KEY, -- absolute VA of vtable[0] length BIGINT NOT NULL, -- number of method slots - col_address BIGINT, -- VA of CompleteObjectLocator (NULL when no RTTI) + -- VA of the CompleteObjectLocator; NULL when the class has no RTTI. + col_address BIGINT REFERENCES rtti_locators(address), class_name VARCHAR NOT NULL, -- demangled class name OR ANON_Class_ when stripped rtti_present BOOLEAN NOT NULL, -- true when COL → TypeDescriptor walk succeeded base_classes_json VARCHAR -- JSON array of base class names (NULL if none / parse failure) ); CREATE TABLE methods ( - vtable_address BIGINT NOT NULL, -- vtable this slot belongs to + vtable_address BIGINT NOT NULL REFERENCES vtables(address), -- vtable this slot belongs to slot BIGINT NOT NULL, -- 0-based slot index function_address BIGINT NOT NULL, -- VA of the function this slot points at mangled_name VARCHAR, -- raw label name when mangled (?...) @@ -442,7 +561,7 @@ impl DbWriter { CREATE TABLE classes ( name VARCHAR PRIMARY KEY, -- class name (demangled or ANON_*) - vtable_address BIGINT NOT NULL, -- representative vtable (first detected) + vtable_address BIGINT NOT NULL REFERENCES vtables(address), -- representative vtable (first detected) rtti_present BOOLEAN NOT NULL, base_classes_json VARCHAR -- JSON of base class names (NULL when stripped) ); @@ -476,7 +595,7 @@ impl DbWriter { ); CREATE TABLE function_pointer_array_entries ( - array_address BIGINT NOT NULL, -- FK to function_pointer_arrays.address + array_address BIGINT NOT NULL REFERENCES function_pointer_arrays(address), slot BIGINT NOT NULL, -- 0-based slot index function_address BIGINT NOT NULL, -- VA of the function this slot points at PRIMARY KEY (array_address, slot) @@ -503,8 +622,10 @@ impl DbWriter { -- ind_call xref edges in the `xrefs` table are derived from -- this; this view lets you join back to vtable / method info. CREATE TABLE indirect_dispatch_candidates ( - dispatch_pc BIGINT NOT NULL, - vtable_address BIGINT NOT NULL, + dispatch_pc BIGINT NOT NULL REFERENCES indirect_dispatch_sites(dispatch_pc), + vtable_address BIGINT NOT NULL REFERENCES vtables(address), + -- No FK: a vtable slot may point at code `.pdata` never declared + -- and no `bl` ever targets, so this is not always in `functions`. method_address BIGINT NOT NULL, PRIMARY KEY (dispatch_pc, vtable_address) ); @@ -514,7 +635,7 @@ impl DbWriter { -- has (or does not have) coverage in the dispatch resolver. CREATE TABLE vptr_writes ( writer_pc BIGINT NOT NULL, - vtable_address BIGINT NOT NULL, + vtable_address BIGINT NOT NULL REFERENCES vtables(address), vptr_offset BIGINT NOT NULL, writer_function BIGINT NOT NULL, PRIMARY KEY (writer_pc, vtable_address, vptr_offset) @@ -536,7 +657,7 @@ impl DbWriter { ); CREATE TABLE eh_unwind_map ( - funcinfo_address BIGINT NOT NULL, -- FK to eh_funcinfo.address + funcinfo_address BIGINT NOT NULL REFERENCES eh_funcinfo(address), state_index BIGINT NOT NULL, to_state BIGINT NOT NULL, action_pc BIGINT NOT NULL, @@ -544,7 +665,7 @@ impl DbWriter { ); CREATE TABLE eh_try_blocks ( - funcinfo_address BIGINT NOT NULL, -- FK to eh_funcinfo.address + funcinfo_address BIGINT NOT NULL REFERENCES eh_funcinfo(address), try_index BIGINT NOT NULL, try_low BIGINT NOT NULL, try_high BIGINT NOT NULL, @@ -566,6 +687,14 @@ impl DbWriter { PRIMARY KEY (namespace, id) ); + CREATE TABLE xdbf_images ( + id BIGINT PRIMARY KEY, -- image id referenced by achievements + is_title_icon BOOLEAN NOT NULL, -- id 0x8000 — the title's own icon + body_offset BIGINT NOT NULL, -- offset within the image buffer + size BIGINT NOT NULL, + format VARCHAR NOT NULL -- 'png' when the body carries the PNG signature + ); + CREATE TABLE xdbf_achievements ( id BIGINT PRIMARY KEY, -- 1-based achievement id name VARCHAR, -- resolved via the default language's string table @@ -574,7 +703,7 @@ impl DbWriter { label_id BIGINT NOT NULL, -- string ids, for joining other languages description_id BIGINT NOT NULL, unachieved_id BIGINT NOT NULL, - image_id BIGINT NOT NULL, -- FK to xdbf_images.id + image_id BIGINT NOT NULL REFERENCES xdbf_images(id), gamerscore BIGINT NOT NULL, flags BIGINT NOT NULL ); @@ -589,14 +718,6 @@ impl DbWriter { PRIMARY KEY (language, string_id) ); - CREATE TABLE xdbf_images ( - id BIGINT PRIMARY KEY, -- image id referenced by achievements - is_title_icon BOOLEAN NOT NULL, -- id 0x8000 — the title's own icon - body_offset BIGINT NOT NULL, -- offset within the image buffer - size BIGINT NOT NULL, - format VARCHAR NOT NULL -- 'png' when the body carries the PNG signature - ); - CREATE TABLE demangled_names ( address BIGINT, -- VA the mangled name is associated with; NULL when from a non-address source (e.g. RTTI-only string) mangled VARCHAR NOT NULL, -- original mangled symbol (e.g. ?Foo@Bar@@QEAAXXZ) @@ -624,7 +745,7 @@ impl DbWriter { -- M12 — one row per case value, in case order. `target_address` -- repeats when several case values share a body. CREATE TABLE jump_table_entries ( - bctr_pc BIGINT NOT NULL, -- FK to jump_tables.bctr_pc + bctr_pc BIGINT NOT NULL REFERENCES jump_tables(bctr_pc), case_index BIGINT NOT NULL, -- 0-based case value target_address BIGINT NOT NULL, -- VA of the case body PRIMARY KEY (bctr_pc, case_index) @@ -634,52 +755,16 @@ impl DbWriter { -- instructions. Anything listed here is a decoding hazard: linear -- disassembly of these words produces garbage rows. CREATE TABLE data_in_code ( - address BIGINT PRIMARY KEY, -- VA of the first byte + address BIGINT PRIMARY KEY REFERENCES instructions(address), -- VA of the first byte length BIGINT NOT NULL, -- byte length - kind VARCHAR NOT NULL -- 'jump_table' | 'jump_index_map' - ); - - -- M13 — MSVC RTTI. `rtti_type_descriptors` is the authoritative - -- source of C++ class identity: the linker wrote these names, they - -- are not inferred. - CREATE TABLE rtti_type_descriptors ( - address BIGINT PRIMARY KEY, -- VA of the TypeDescriptor - mangled_name VARCHAR NOT NULL, -- decorated name, e.g. .?AVSilph@silph@@ - demangled_name VARCHAR NOT NULL -- readable form, e.g. silph::Silph - ); - - -- M13 — RTTICompleteObjectLocator. One per emitted vftable; the - -- `subobject_offset` column separates a class's primary vftable (0) - -- from the extra vftables it emits for secondary base subobjects. - CREATE TABLE rtti_locators ( - address BIGINT PRIMARY KEY, - subobject_offset BIGINT NOT NULL, -- this-offset of the subobject this vftable serves - cd_offset BIGINT NOT NULL, -- constructor-displacement offset - type_descriptor BIGINT NOT NULL, -- FK to rtti_type_descriptors.address - class_hierarchy BIGINT NOT NULL, -- VA of the RTTIClassHierarchyDescriptor - vtable_address BIGINT -- VA of vftable[0]; NULL if no referencing word was found - ); - - -- M13 — linearised RTTIBaseClassArray. Index 0 is the class itself; - -- the remaining rows are its bases in MSVC's depth-first order, - -- each with the PMD displacement triple needed to locate the base - -- subobject inside an instance. - CREATE TABLE rtti_base_classes ( - class_hierarchy BIGINT NOT NULL, -- VA of the deriving class's hierarchy descriptor - base_index BIGINT NOT NULL, -- position in the base-class array - type_descriptor BIGINT NOT NULL, - name VARCHAR NOT NULL, - num_contained_bases BIGINT NOT NULL, - mdisp BIGINT NOT NULL, -- member displacement - pdisp BIGINT NOT NULL, -- vbtable displacement (-1 = non-virtual base) - vdisp BIGINT NOT NULL, -- displacement inside the vbtable - attributes BIGINT NOT NULL, - PRIMARY KEY (class_hierarchy, base_index) + kind VARCHAR NOT NULL -- 'jump_table' | 'jump_index_map' | 'import_record' ); CREATE TABLE xrefs ( - source BIGINT NOT NULL, -- VA of the referencing instruction - target BIGINT NOT NULL, -- VA of the referenced destination + source BIGINT NOT NULL REFERENCES instructions(address), + -- No FK: a target may be data (.rdata/.data) or an address with + -- no decoded word, so it is not always an `instructions` row. + target BIGINT NOT NULL, kind VARCHAR NOT NULL, -- call | ind_call | j | br | read | write | ref addr_mode VARCHAR, -- M6 sub-classification of how source computes target (NULL for control-flow) instruction VARCHAR, -- mnemonic of source instruction; NULL if not in binary @@ -703,17 +788,24 @@ impl DbWriter { // which is both fast and flat in memory. It bypasses the SQL layer, // so `ON CONFLICT DO NOTHING` is unavailable and each converted sink // documents why its key cannot collide (or dedupes explicitly). + // Insertion order is load-bearing: the schema declares foreign keys, + // and DuckDB checks them per row, so a parent table must be populated + // before any child that references it. `insert_rtti` runs ahead of + // `insert_vtables` because `vtables.col_address` points into + // `rtti_locators`; everything else follows the natural dependency + // order (functions -> vtables -> methods/classes -> dispatch). insert_functions(&self.conn, func_analysis, labels)?; insert_pdata_entries(&self.conn, &func_analysis.pdata_entries)?; insert_labels(&self.conn, labels)?; insert_demangled_from_labels(&self.conn, labels, info.import_libraries)?; + insert_rtti(&self.conn, rtti)?; insert_vtables(&self.conn, vtables, pe, info.image_base)?; insert_methods_and_classes(&self.conn, vtables, labels)?; insert_strings(&self.conn, strings)?; insert_funcptr_arrays(&self.conn, funcptr_arrays)?; insert_eh_records(&self.conn, eh_records)?; insert_jump_tables(&self.conn, jump_tables)?; - insert_rtti(&self.conn, rtti)?; + insert_import_record_extents(&self.conn, &self.import_sites)?; insert_xdbf(&self.conn, xdbf)?; if let Some(t) = typed_ind { insert_typed_ind_dispatch(&self.conn, t)?; @@ -1363,8 +1455,24 @@ fn insert_imports(conn: &Connection, info: &DisasmInfo) -> anyhow::Result<()> { "INSERT INTO imports (library, ordinal, name, record_type, address) VALUES (?, ?, ?, ?, ?)", )?; + // `address` is UNIQUE so that `instructions.import_address` can key off + // it. Two entries sharing an address would be a malformed import table — + // one slot cannot hold two imports — but a duplicate must not abort the + // whole database build, so keep the first and say what was dropped. + let mut seen: HashSet = HashSet::new(); + let mut duplicates = 0usize; for lib in info.import_libraries { for imp in &lib.imports { + if !seen.insert(imp.address) { + duplicates += 1; + tracing::warn!( + library = %lib.name, + ordinal = imp.ordinal, + address = format_args!("{:#010x}", imp.address), + "duplicate import address; keeping the first entry", + ); + continue; + } let resolved = crate::resolve_ordinal(&lib.name, imp.ordinal); stmt.execute(params![ lib.name, @@ -1375,6 +1483,9 @@ fn insert_imports(conn: &Connection, info: &DisasmInfo) -> anyhow::Result<()> { ])?; } } + if duplicates > 0 { + tracing::warn!(duplicates, "import table has entries sharing an address"); + } Ok(()) } @@ -1756,6 +1867,23 @@ fn insert_xdbf(conn: &Connection, xdbf: Option<&crate::xdbf::Xdbf>) -> anyhow::R .map(|(_, s)| s.clone()) }; + // Images before achievements: `xdbf_achievements.image_id` is a foreign + // key onto `xdbf_images.id`, so the icon rows must already exist. + let mut stmt = conn.prepare( + "INSERT INTO xdbf_images (id, is_title_icon, body_offset, size, format) + VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING", + )?; + for i in &x.images { + stmt.execute(params![ + i.id as i64, + i.id == crate::xdbf::ID_TITLE, + i.offset as i64, + i.size as i64, + i.format, + ])?; + } + drop(stmt); + let mut stmt = conn.prepare( "INSERT INTO xdbf_achievements (id, name, unlocked_desc, locked_desc, label_id, description_id, @@ -1778,21 +1906,6 @@ fn insert_xdbf(conn: &Connection, xdbf: Option<&crate::xdbf::Xdbf>) -> anyhow::R } drop(stmt); - let mut stmt = conn.prepare( - "INSERT INTO xdbf_images (id, is_title_icon, body_offset, size, format) - VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING", - )?; - for i in &x.images { - stmt.execute(params![ - i.id as i64, - i.id == crate::xdbf::ID_TITLE, - i.offset as i64, - i.size as i64, - i.format, - ])?; - } - drop(stmt); - let mut meta = conn.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)")?; meta.execute(params!["xdbf.entry_count", x.entries.len().to_string()])?; if let Some(l) = x.default_language { @@ -1991,6 +2104,7 @@ fn insert_instructions_streaming( func_analysis: &FuncAnalysis, labels: &HashMap, data_words: &std::collections::BTreeSet, + import_sites: &crate::imports::ImportSites, ) -> anyhow::Result<()> { let mut appender = conn.appender("instructions")?; let mut total: u64 = 0; @@ -2010,6 +2124,7 @@ fn insert_instructions_streaming( func_analysis, labels, data_words, + import_sites, ); total += crate::sinks::duckdb::append_instructions(&mut appender, items)?; } @@ -2020,6 +2135,34 @@ fn insert_instructions_streaming( Ok(()) } +/// Record each import thunk's two loader-rewritten words in `data_in_code`. +/// +/// `data_in_code` already lists recovered jump tables; import records are the +/// other kind of non-instruction word sitting in a code section, so a consumer +/// asking "which bytes of `.text` are not code?" gets a complete answer from +/// one table instead of having to know about thunk layout. +fn insert_import_record_extents( + conn: &Connection, + import_sites: &crate::imports::ImportSites, +) -> anyhow::Result<()> { + let mut heads: Vec = import_sites.thunk_heads().collect(); + heads.sort_unstable(); + let mut stmt = + conn.prepare("INSERT INTO data_in_code (address, length, kind) VALUES (?, ?, ?)")?; + for head in &heads { + // The two records are contiguous at the head of the thunk; the two + // instructions that follow them are real code. + stmt.execute(params![*head as i64, 8i64, "import_record"])?; + } + metrics::counter!("db.rows", "table" => "data_in_code").increment(heads.len() as u64); + tracing::info!( + rows = heads.len(), + kind = "import_record", + "data_in_code write complete" + ); + Ok(()) +} + /// Write the M12 jump-table tables plus the data-in-code extent list. fn insert_jump_tables( conn: &Connection, diff --git a/crates/sylpheed-xexdb/src/disasm.rs b/crates/sylpheed-xexdb/src/disasm.rs index adcb9436..0a2a1e39 100644 --- a/crates/sylpheed-xexdb/src/disasm.rs +++ b/crates/sylpheed-xexdb/src/disasm.rs @@ -10,6 +10,7 @@ use std::collections::{BTreeSet, HashMap}; use sylpheed_ppc::disasm::DisasmItem; use crate::func::FuncAnalysis; +use crate::imports::{ImportRole, ImportSites}; /// `DisasmItem` plus the analysis context (section/function/label). #[derive(Debug, Clone)] @@ -19,8 +20,13 @@ pub struct RichDisasmItem<'a> { pub function: Option, pub label: Option<&'a str>, /// True when this word is data embedded in a code section (a recovered - /// jump table or its index map), so its decoded text is meaningless. + /// jump table or its index map, or an XEX import record), so its decoded + /// text is meaningless. pub is_data: bool, + /// The XEX import this word belongs to, as `(thunk head VA, role)`. The + /// head VA is `imports.address` for that import, so it is written out as + /// a foreign key. `None` for the overwhelming majority of instructions. + pub import: Option<(u32, ImportRole)>, } /// Walk one code section, yielding rich items annotated with section name, @@ -42,6 +48,12 @@ pub struct RichDisasmItem<'a> { /// Rows at those addresses are still emitted — their `raw` value is the table /// entry a consumer wants — but flagged so nothing mistakes the decoded text /// for a real instruction. +/// +/// `import_sites` resolves XEX import thunks. It contributes two things: the +/// two import-record words at each thunk head are also flagged `is_data` +/// (they are loader-rewritten data, not code, and decode to nonsense), and +/// every word of a thunk plus every direct branch into one carries the +/// import it belongs to. pub fn enrich_section<'a>( image: &'a [u8], image_base: u32, @@ -51,6 +63,7 @@ pub fn enrich_section<'a>( func_analysis: &'a FuncAnalysis, labels: &'a HashMap, data_words: &'a BTreeSet, + import_sites: &'a ImportSites, ) -> impl Iterator> + 'a { // (start, end) of the function currently being walked. let mut current: Option<(u32, u32)> = None; @@ -67,13 +80,16 @@ pub fn enrich_section<'a>( } let current_func = current.map(|(start, _)| start); let label = labels.get(&item.addr).map(|s| s.as_str()); - let is_data = data_words.contains(&item.addr); + let import = import_sites.lookup(item.addr, item.text.branch_target); + let is_data = + data_words.contains(&item.addr) || matches!(import, Some((_, ImportRole::Record))); RichDisasmItem { item, section: section_name, function: current_func, label, is_data, + import, } }) } @@ -127,6 +143,7 @@ mod tests { &fa, &labels, &data_words, + &ImportSites::default(), ) .map(|r| (r.item.addr, r.function)) .collect(); @@ -170,6 +187,7 @@ mod tests { &fa, &labels, &data_words, + &ImportSites::default(), ) .map(|r| r.function) .collect(); diff --git a/crates/sylpheed-xexdb/src/formatter.rs b/crates/sylpheed-xexdb/src/formatter.rs index 5e573976..fe6f3dcc 100644 --- a/crates/sylpheed-xexdb/src/formatter.rs +++ b/crates/sylpheed-xexdb/src/formatter.rs @@ -8,6 +8,7 @@ use sylpheed_xex::pe::PeSection; use crate::disasm::enrich_section; use crate::func::FuncAnalysis; +use crate::imports::{ImportRole, ImportSites}; use crate::sinks::text::write_instr_line; use crate::xref::{Xref, XrefKind, XrefMap, resolve_source_label}; @@ -38,6 +39,7 @@ pub fn write_asm( xrefs: &XrefMap, data_annotations: &HashMap, data_words: &BTreeSet, + import_sites: &ImportSites, ) -> anyhow::Result<()> { // Header writeln!( @@ -136,6 +138,7 @@ pub fn write_asm( func_analysis, labels, data_words, + import_sites, ); for ri in items { let abs_addr = ri.item.addr; @@ -208,10 +211,26 @@ pub fn write_asm( } } - // Import thunk annotation + // Import annotations. The thunk head keeps its banner; call + // sites get a trailing note so a `bl` into a thunk reads as + // the kernel call it actually is rather than a bare address. if let Some(imp_name) = import_map.get(&abs_addr) { writeln!(out, " ; IMPORT: {imp_name}")?; } + if let Some((head, role)) = ri.import { + match role { + ImportRole::Record => writeln!( + out, + " ; import record (loader-patched; not an instruction)", + )?, + ImportRole::Call => { + if let Some(name) = import_map.get(&head) { + writeln!(out, " ; -> IMPORT {name}")?; + } + } + ImportRole::Thunk => {} + } + } let data_annot = data_annotations.get(&abs_addr).copied(); write_instr_line(out, &ri, labels, info.sections, info.image_base, data_annot)?; diff --git a/crates/sylpheed-xexdb/src/imports.rs b/crates/sylpheed-xexdb/src/imports.rs new file mode 100644 index 00000000..eaf27940 --- /dev/null +++ b/crates/sylpheed-xexdb/src/imports.rs @@ -0,0 +1,354 @@ +//! Import-thunk recognition — makes XEX imports visible in the code stream. +//! +//! An XEX import is not a PLT-style jump. The linker emits a four-word thunk +//! whose first two words are *import records*, which the loader rewrites at +//! module load into `lis r11, hi(target)` / `ori r11, r11, lo(target)`. On +//! disc they are still records, so a decoder that only knows PowerPC renders +//! them as two meaningless `.long`s in front of an indirect branch: +//! +//! ```text +//! 8284DA7C .long 0x0100028C <- record: type 1, library 0, ordinal 652 +//! 8284DA80 .long 0x0200028C <- record: type 2, library 0, ordinal 652 +//! 8284DA84 mtctr r11 +//! 8284DA88 bctr +//! ``` +//! +//! Record layout, matching xenia-canary `XexModule::SetupLibraryImports`: +//! +//! | bits | meaning | +//! |---------|------------------------------------------------------------| +//! | 31..24 | record type — `1` and `2` mark the two halves of a thunk | +//! | 23..16 | index of the owning import library | +//! | 15..0 | ordinal within that library | +//! +//! This module maps every word of every thunk — and every direct branch that +//! targets one — back to the `imports` row it belongs to, so the disassembly +//! can say `xboxkrnl.exe::RtlEnterCriticalSection` where it used to say +//! `.long 0x01010194`. +//! +//! The mapping is deliberately *shape-validated* rather than trusted: an +//! import entry is only indexed when the four words it points at actually +//! look like a thunk for that exact ordinal. A malformed or already-patched +//! image annotates nothing rather than mislabelling real code. + +use std::collections::HashMap; + +use sylpheed_xex::header::ImportLibrary; + +/// `mtspr CTR, r11` — third word of every import thunk. +const MTCTR_R11: u32 = 0x7D69_03A6; +/// `bcctr 20, lt` (`bctr`) — fourth word of every import thunk. +const BCTR: u32 = 0x4E80_0420; +/// Record type in the first thunk word. +const RECORD_TYPE_FIRST: u32 = 1; +/// Record type in the second thunk word. +const RECORD_TYPE_SECOND: u32 = 2; +/// A thunk is four 4-byte words. +pub const THUNK_SIZE: u32 = 16; + +/// How an instruction relates to the import it references. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ImportRole { + /// One of the two import-record words at the head of a thunk. The word is + /// data the loader rewrites, not an instruction — its decoded text is + /// meaningless and it is flagged `is_data` alongside jump-table words. + Record, + /// A real instruction in the thunk body (`mtctr r11` / `bctr`). + Thunk, + /// A direct branch (`bl` or a tail-call `b`) whose target is this + /// import's thunk head. + Call, +} + +impl ImportRole { + /// Stable lowercase tag, as stored in `instructions.import_role`. + pub fn as_str(self) -> &'static str { + match self { + Self::Record => "record", + Self::Thunk => "thunk", + Self::Call => "call", + } + } +} + +/// Word-level index from code addresses to the import they belong to. +/// +/// Values are the thunk head VA, which is exactly `imports.address` for the +/// `record_type = 1` row — so it doubles as the foreign key written into +/// `instructions.import_address`. +#[derive(Debug, Default)] +pub struct ImportSites { + /// Every word inside a validated thunk -> (thunk head VA, role). + words: HashMap, +} + +impl ImportSites { + /// Index every import thunk in `libraries` that `image` actually backs. + /// + /// `record_type = 0` entries are plain pointer slots in `.rdata`: the + /// loader writes the resolved address there and no code is emitted, so + /// there is nothing in the instruction stream to annotate. + pub fn build(libraries: &[ImportLibrary], image: &[u8], image_base: u32) -> Self { + let read = |va: u32| -> Option { + let off = va.wrapping_sub(image_base) as usize; + if off.checked_add(4)? > image.len() { + return None; + } + Some(u32::from_be_bytes([ + image[off], + image[off + 1], + image[off + 2], + image[off + 3], + ])) + }; + + let mut words = HashMap::new(); + let mut indexed = 0usize; + let mut rejected = 0usize; + + for lib in libraries { + for imp in &lib.imports { + if imp.record_type != 1 { + continue; + } + let head = imp.address; + let Some(w0) = read(head) else { + rejected += 1; + continue; + }; + let (Some(w1), Some(w2), Some(w3)) = + (read(head + 4), read(head + 8), read(head + 12)) + else { + rejected += 1; + continue; + }; + if !is_thunk([w0, w1, w2, w3], imp.ordinal) { + rejected += 1; + continue; + } + + words.insert(head, (head, ImportRole::Record)); + words.insert(head + 4, (head, ImportRole::Record)); + words.insert(head + 8, (head, ImportRole::Thunk)); + words.insert(head + 12, (head, ImportRole::Thunk)); + indexed += 1; + } + } + + tracing::info!( + thunks = indexed, + rejected, + words = words.len(), + "import thunk index built", + ); + Self { words } + } + + /// The import this instruction references, and in what capacity. + /// + /// `branch_target` is the resolved target of a direct branch. A `bl` into + /// a thunk head yields [`ImportRole::Call`], which is what lets a query + /// find *call sites* of a kernel function rather than just the thunk. + /// Landing mid-thunk is not a call, so only the head matches. + pub fn lookup(&self, addr: u32, branch_target: Option) -> Option<(u32, ImportRole)> { + if let Some(&hit) = self.words.get(&addr) { + return Some(hit); + } + let target = branch_target?; + match self.words.get(&target) { + // `head == target` rejects a branch into the second record word. + Some(&(head, _)) if head == target => Some((head, ImportRole::Call)), + _ => None, + } + } + + /// Addresses of the import-record words — data embedded in a code section. + pub fn record_words(&self) -> impl Iterator + '_ { + self.words + .iter() + .filter(|(_, (_, role))| *role == ImportRole::Record) + .map(|(&addr, _)| addr) + } + + /// Thunk head VAs, one per indexed import. + pub fn thunk_heads(&self) -> impl Iterator + '_ { + self.words + .iter() + .filter(|(addr, (head, _))| *addr == head) + .map(|(&addr, _)| addr) + } + + /// Number of thunks indexed. + pub fn len(&self) -> usize { + self.words.len() / 4 + } + + pub fn is_empty(&self) -> bool { + self.words.is_empty() + } +} + +/// Does this four-word run look like an import thunk for `ordinal`? +/// +/// The library-index byte is checked for *consistency between the two +/// records* rather than against the library's position in the header: the +/// ordinal match plus the fixed `mtctr`/`bctr` tail is already conclusive, +/// and not depending on header ordering keeps this robust for titles whose +/// import libraries are listed in a different order. +fn is_thunk(w: [u32; 4], ordinal: u16) -> bool { + w[0] >> 24 == RECORD_TYPE_FIRST + && w[1] >> 24 == RECORD_TYPE_SECOND + && (w[0] & 0xFFFF) as u16 == ordinal + && (w[1] & 0xFFFF) as u16 == ordinal + && (w[0] >> 16) & 0xFF == (w[1] >> 16) & 0xFF + && w[2] == MTCTR_R11 + && w[3] == BCTR +} + +#[cfg(test)] +mod tests { + use super::*; + use sylpheed_xex::header::ImportEntry; + + const BASE: u32 = 0x8200_0000; + + /// Build a one-library header plus a backing image containing `thunks` + /// laid out back to back from `BASE`. + fn fixture(thunks: &[(u16, u8)]) -> (Vec, Vec) { + let mut image = Vec::new(); + let mut imports = Vec::new(); + for (i, &(ordinal, lib_index)) in thunks.iter().enumerate() { + let head = BASE + i as u32 * THUNK_SIZE; + let li = (lib_index as u32) << 16; + for w in [ + (RECORD_TYPE_FIRST << 24) | li | ordinal as u32, + (RECORD_TYPE_SECOND << 24) | li | ordinal as u32, + MTCTR_R11, + BCTR, + ] { + image.extend_from_slice(&w.to_be_bytes()); + } + imports.push(ImportEntry { + ordinal, + record_type: 1, + address: head, + }); + } + let lib = ImportLibrary { + name: "xboxkrnl.exe".into(), + id: 0, + version_min: 0, + version_cur: 0, + imports, + }; + (vec![lib], image) + } + + #[test] + fn indexes_every_word_of_a_thunk() { + let (libs, image) = fixture(&[(0x28C, 0)]); + let sites = ImportSites::build(&libs, &image, BASE); + assert_eq!(sites.len(), 1); + assert_eq!(sites.lookup(BASE, None), Some((BASE, ImportRole::Record))); + assert_eq!( + sites.lookup(BASE + 4, None), + Some((BASE, ImportRole::Record)) + ); + assert_eq!( + sites.lookup(BASE + 8, None), + Some((BASE, ImportRole::Thunk)) + ); + assert_eq!( + sites.lookup(BASE + 12, None), + Some((BASE, ImportRole::Thunk)) + ); + assert_eq!(sites.lookup(BASE + 16, None), None); + } + + /// The point of the whole exercise: a `bl` somewhere else in the binary + /// must resolve to the import it ends up calling. + #[test] + fn direct_branch_to_a_thunk_head_is_a_call() { + let (libs, image) = fixture(&[(0x28C, 0)]); + let sites = ImportSites::build(&libs, &image, BASE); + assert_eq!( + sites.lookup(0x8215_0000, Some(BASE)), + Some((BASE, ImportRole::Call)), + ); + } + + /// A branch into the *second* record word is not a call to the import; + /// treating it as one would invent a call edge that does not exist. + #[test] + fn branch_into_thunk_interior_is_not_a_call() { + let (libs, image) = fixture(&[(0x28C, 0)]); + let sites = ImportSites::build(&libs, &image, BASE); + assert_eq!(sites.lookup(0x8215_0000, Some(BASE + 4)), None); + assert_eq!(sites.lookup(0x8215_0000, Some(BASE + 8)), None); + } + + /// A word inside a thunk keeps its own role even when it also carries a + /// branch target, so the thunk body is never relabelled as a call. + #[test] + fn own_role_wins_over_branch_target() { + let (libs, image) = fixture(&[(0x28C, 0), (0x194, 1)]); + let sites = ImportSites::build(&libs, &image, BASE); + let second = BASE + THUNK_SIZE; + assert_eq!( + sites.lookup(BASE + 8, Some(second)), + Some((BASE, ImportRole::Thunk)) + ); + } + + /// Shape validation must reject an entry whose ordinal does not match the + /// record it points at — otherwise a stale import table would silently + /// stamp the wrong kernel name onto real code. + #[test] + fn ordinal_mismatch_is_rejected() { + let (mut libs, image) = fixture(&[(0x28C, 0)]); + libs[0].imports[0].ordinal = 0x999; + let sites = ImportSites::build(&libs, &image, BASE); + assert!(sites.is_empty()); + } + + /// An already-patched image (records rewritten to `lis`/`ori`) has no + /// records left to recognise, and must annotate nothing rather than + /// guessing. + #[test] + fn patched_thunk_is_rejected() { + let (libs, mut image) = fixture(&[(0x28C, 0)]); + image[0..4].copy_from_slice(&0x3D60_8200u32.to_be_bytes()); // lis r11, 0x8200 + let sites = ImportSites::build(&libs, &image, BASE); + assert!(sites.is_empty()); + } + + /// A `record_type = 0` entry is a data slot in `.rdata`, not code. + #[test] + fn variable_records_are_not_indexed() { + let (mut libs, image) = fixture(&[(0x28C, 0)]); + libs[0].imports[0].record_type = 0; + let sites = ImportSites::build(&libs, &image, BASE); + assert!(sites.is_empty()); + } + + /// An import pointing outside the image must not panic or index garbage. + #[test] + fn out_of_bounds_thunk_is_rejected() { + let (mut libs, image) = fixture(&[(0x28C, 0)]); + libs[0].imports[0].address = BASE + 0x10_0000; + let sites = ImportSites::build(&libs, &image, BASE); + assert!(sites.is_empty()); + } + + #[test] + fn record_words_lists_both_halves() { + let (libs, image) = fixture(&[(0x28C, 0), (0x194, 1)]); + let sites = ImportSites::build(&libs, &image, BASE); + let mut got: Vec = sites.record_words().collect(); + got.sort_unstable(); + assert_eq!(got, vec![BASE, BASE + 4, BASE + 16, BASE + 20]); + let mut heads: Vec = sites.thunk_heads().collect(); + heads.sort_unstable(); + assert_eq!(heads, vec![BASE, BASE + 16]); + } +} diff --git a/crates/sylpheed-xexdb/src/lib.rs b/crates/sylpheed-xexdb/src/lib.rs index a0a3025b..ed426568 100644 --- a/crates/sylpheed-xexdb/src/lib.rs +++ b/crates/sylpheed-xexdb/src/lib.rs @@ -25,6 +25,7 @@ pub mod eh_scope; pub mod formatter; pub mod func; pub mod funcptr_arrays; +pub mod imports; pub mod ind_dispatch_typed; pub mod indirect; pub mod jumptables; @@ -42,5 +43,6 @@ pub mod xref; mod ordinals; pub use db::{BranchTraceEntry, DbWriter, ExecTraceEntry, ImportCallEntry}; pub use disasm::{RichDisasmItem, enrich_section}; +pub use imports::{ImportRole, ImportSites}; pub use ordinals::resolve_ordinal; pub use xref::{Xref, XrefKind, XrefMap, resolve_source_label}; diff --git a/crates/sylpheed-xexdb/src/sinks/duckdb.rs b/crates/sylpheed-xexdb/src/sinks/duckdb.rs index 40db1117..23306937 100644 --- a/crates/sylpheed-xexdb/src/sinks/duckdb.rs +++ b/crates/sylpheed-xexdb/src/sinks/duckdb.rs @@ -32,6 +32,8 @@ pub fn append_instructions<'a>( ri.function.map(|f| f as i64), ri.label, ri.is_data, + ri.import.map(|(head, _)| head as i64), + ri.import.map(|(_, role)| role.as_str()), ])?; count += 1; } diff --git a/crates/sylpheed-xexdb/src/sql_views.rs b/crates/sylpheed-xexdb/src/sql_views.rs index ec3ede27..083c0269 100644 --- a/crates/sylpheed-xexdb/src/sql_views.rs +++ b/crates/sylpheed-xexdb/src/sql_views.rs @@ -192,20 +192,34 @@ FROM functions f JOIN instructions i ON i.address = f.address; "; -/// Per-function summary of which kernel/library imports it calls. Joins -/// xrefs (call edges) against the labels table to surface import names. +/// Every call site of a kernel/library import, one row per branch. +/// +/// Driven by `instructions.import_address` — the foreign key onto `imports` +/// laid down by [`crate::imports::ImportSites`] — rather than by matching +/// `xrefs` against import-kind labels. That change fixes two things: +/// +/// - a tail-call `b` into a thunk is a call to that import, but `xrefs` tags +/// it `'j'`, so the label-matching version silently dropped all of them +/// (64 sites on the reference title, against 3,217 `bl`); +/// - the library and ordinal are now available without re-parsing the name, +/// so `WHERE library = 'xboxkrnl.exe'` works directly. +/// +/// `call_site` is the branch instruction; `import_addr` is the thunk head. const V_IMPORTS_CALLED: &str = " CREATE OR REPLACE VIEW v_imports_called AS SELECT - x.source_func AS function_addr, + i.address AS call_site, + i.function AS function_addr, f.name AS function_name, - x.target AS import_addr, - l.name AS import_name -FROM xrefs x -JOIN labels l ON l.address = x.target -LEFT JOIN functions f ON f.address = x.source_func -WHERE x.kind = 'call' - AND l.kind = 'import'; + i.mnemonic AS instruction, + i.import_address AS import_addr, + im.library AS library, + im.name AS import_name, + im.ordinal AS ordinal +FROM instructions i +JOIN imports im ON im.address = i.import_address +LEFT JOIN functions f ON f.address = i.function +WHERE i.import_role = 'call'; "; /// Every recovered `switch` case, joined to the dispatching function and to diff --git a/crates/sylpheed-xexdb/tests/db_schema_golden.rs b/crates/sylpheed-xexdb/tests/db_schema_golden.rs index 303cf12f..12611695 100644 --- a/crates/sylpheed-xexdb/tests/db_schema_golden.rs +++ b/crates/sylpheed-xexdb/tests/db_schema_golden.rs @@ -176,6 +176,8 @@ fn db_schema_matches_expected_columns() { ("function", "BIGINT"), ("label", "VARCHAR"), ("is_data", "BOOLEAN"), + ("import_address", "BIGINT"), + ("import_role", "VARCHAR"), ], ), ( @@ -564,5 +566,63 @@ fn db_schema_matches_expected_columns() { assert_eq!(exists, 1, "missing SQL view: {v}"); } + // ── Foreign keys ──────────────────────────────────────────────────── + // + // Lock the declared FK set. These are correctness assertions baked into + // the schema: a build that would produce a dangling reference fails at + // insert time rather than shipping a broken database. + let expected_fks = [ + ("classes", "vtables"), + ("data_in_code", "instructions"), + ("eh_try_blocks", "eh_funcinfo"), + ("eh_unwind_map", "eh_funcinfo"), + ("function_pointer_array_entries", "function_pointer_arrays"), + ("indirect_dispatch_candidates", "indirect_dispatch_sites"), + ("indirect_dispatch_candidates", "vtables"), + ("instructions", "imports"), + ("jump_table_entries", "jump_tables"), + ("methods", "vtables"), + ("rtti_base_classes", "rtti_type_descriptors"), + ("rtti_locators", "rtti_type_descriptors"), + ("vptr_writes", "vtables"), + ("vtables", "rtti_locators"), + ("xdbf_achievements", "xdbf_images"), + ("xrefs", "instructions"), + ]; + for (child, parent) in expected_fks { + let n: i64 = conn + .query_row( + "SELECT COUNT(*) FROM duckdb_constraints() \ + WHERE constraint_type = 'FOREIGN KEY' AND table_name = ? \ + AND referenced_table = ?", + [child, parent], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(n, 1, "missing foreign key: {child} -> {parent}"); + } + + // `functions` must never become a foreign-key parent. DuckDB implements + // UPDATE as delete-then-insert, so a single inbound FK makes the table + // un-updatable — even for an update that does not touch `address` — and + // that breaks `apply_re_symbols.sql`, which re-stamps reverse-engineered + // names onto `functions.name` after every regeneration. Adding one back + // looks harmless in review and only fails at symbol-stamping time, hence + // this guard. + let inbound_to_functions: i64 = conn + .query_row( + "SELECT COUNT(*) FROM duckdb_constraints() \ + WHERE constraint_type = 'FOREIGN KEY' \ + AND referenced_table = 'functions'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + inbound_to_functions, 0, + "no table may declare a foreign key onto `functions`: it would make \ + functions.name un-updatable and break apply_re_symbols.sql", + ); + let _ = std::fs::remove_file(&tmp); } diff --git a/tools/zq.py b/tools/zq.py index 7b4b0c8b..0dc6dbc2 100755 --- a/tools/zq.py +++ b/tools/zq.py @@ -15,7 +15,7 @@ both static, neither needing anything to run: alongside a metadata JSON. Usage: - zq.py dis # disassemble [lo,hi) (jump-table words shown as .long) + zq.py dis # disassemble [lo,hi) (data words as .long; imports named) zq.py fn # function containing pc (address,name,end) zq.py xref # xrefs whose target == addr (callers) zq.py callers # call-sites of vtable slot at byte offset N @@ -29,13 +29,21 @@ Usage: zq.py class # one class: bases, vtable, virtual methods zq.py str # string literals matching, with referencing functions + zq.py gaps # dangling refs = known detector gaps (see db.rs schema docs) + zq.py imp [substr] # imports + how many call sites each has + zq.py impcalls # every call site of the matching imports + zq.py xdbf [substr] # XDBF title text (all locales); substr filters zq.py ach # XDBF achievements (id, gamerscore, name, descriptions) A command that needs a table the current DB predates prints what to regenerate rather than a SQL error. """ -import duckdb, sys +import duckdb, sys, signal + +# Output is routinely piped into `head`; without this Python prints a +# BrokenPipeError traceback when the reader closes early. +signal.signal(signal.SIGPIPE, signal.SIG_DFL) DB = '/home/fabi/RE - Project Sylpheed/xenia-rs/sylpheed.db' c = duckdb.connect(DB, read_only=True) @@ -65,17 +73,108 @@ def _fn(pc): return f'{r[0][1]}({H(r[0][0])})' if r else '?' +def _import_names(): + """thunk head VA -> 'library::name' for every import, or {} on an old db.""" + if not _has_col('instructions', 'import_address'): + return {} + return {a: (f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}') + for a, lib, nm, o in c.execute( + 'SELECT address,library,name,ordinal FROM imports').fetchall()} + + def cmd_dis(lo, hi): data_col = 'is_data' if _has_col('instructions', 'is_data') else 'false' - rows = c.execute(f'SELECT address,mnemonic,operands,raw,{data_col} FROM instructions ' - 'WHERE address>=? AND address=? AND address {who}' if imp_role == 'record' else 'jump-table data' + print(H(a), '.long', H(raw & 0xffffffff), f' ; {why}') + elif imp_role == 'call': + print(H(a), m, o, f' ; -> IMPORT {who}') + elif imp_role == 'thunk': + print(H(a), m, o, f' ; import thunk {who}') else: print(H(a), m, o) +def cmd_gaps(): + """Dangling references the schema deliberately does NOT declare as FKs. + + Each of these is a real gap in a detector, kept queryable rather than + hidden: declaring the FK would abort the build instead of reporting it. + """ + checks = [ + ("vtables RTTI names but M3 missed", + "SELECT count(*) FROM rtti_locators l WHERE l.vtable_address IS NOT NULL " + "AND NOT EXISTS (SELECT 1 FROM vtables v WHERE v.address=l.vtable_address)"), + ("vtable slots -> undetected function", + "SELECT count(DISTINCT function_address) FROM methods m WHERE NOT EXISTS " + "(SELECT 1 FROM functions f WHERE f.address=m.function_address)"), + ("funcptr-array slots -> undetected function", + "SELECT count(DISTINCT function_address) FROM function_pointer_array_entries e " + "WHERE NOT EXISTS (SELECT 1 FROM functions f WHERE f.address=e.function_address)"), + ("dispatch candidates -> undetected function", + "SELECT count(DISTINCT method_address) FROM indirect_dispatch_candidates x " + "WHERE NOT EXISTS (SELECT 1 FROM functions f WHERE f.address=x.method_address)"), + ("virtual sites with no candidates (truncated)", + "SELECT count(*) FROM indirect_dispatch_sites WHERE truncated"), + ] + for label, q in checks: + try: + print(f"{c.execute(q).fetchone()[0]:>7} {label}") + except Exception as e: + print(f" ? {label} ({str(e).splitlines()[0][:50]})") + print("\n -- RTTI-named vftables the vtable scanner missed --") + try: + rows = c.execute( + "SELECT l.vtable_address, t.demangled_name FROM rtti_locators l " + "JOIN rtti_type_descriptors t ON t.address=l.type_descriptor " + "WHERE l.vtable_address IS NOT NULL AND NOT EXISTS " + "(SELECT 1 FROM vtables v WHERE v.address=l.vtable_address) " + "ORDER BY 2 LIMIT 15").fetchall() + for a, nm in rows: + print(f" {H(a)} {nm}") + except Exception as e: + print(" ", str(e).splitlines()[0][:70]) + + +def cmd_imp(pat=None): + """Import call sites, grouped by import. `pat` filters on library::name.""" + _need('imports') + if not _has_col('instructions', 'import_address'): + sys.exit(f'this db predates instructions.import_address — regenerate with:\n {REGEN}') + q = ("SELECT im.library, im.name, im.ordinal, im.address, count(i.address) " + "FROM imports im LEFT JOIN instructions i " + " ON i.import_address = im.address AND i.import_role = 'call' " + "WHERE im.record_type = 1 GROUP BY 1,2,3,4 ORDER BY 5 DESC, 1, 2") + for lib, nm, o, addr, n in c.execute(q).fetchall(): + label = f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}' + if pat and pat.lower() not in label.lower(): + continue + print(f'{n:>5} calls {H(addr)} {label}') + + +def cmd_impcalls(pat): + """Every call site of the imports matching `pat`.""" + if not _has_col('instructions', 'import_address'): + sys.exit(f'this db predates instructions.import_address — regenerate with:\n {REGEN}') + q = ("SELECT i.address, i.mnemonic, im.library, im.name, im.ordinal " + "FROM instructions i JOIN imports im ON im.address = i.import_address " + "WHERE i.import_role = 'call' ORDER BY i.address") + for a, m, lib, nm, o in c.execute(q).fetchall(): + label = f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}' + if pat.lower() not in label.lower(): + continue + print(H(a), m, 'in', _fn(a), '->', label) + + def cmd_switch(pc): _need('jump_tables', 'jump_table_entries') r = c.execute('SELECT bctr_pc,function,table_address,kind,entry_count FROM jump_tables ' @@ -194,6 +293,12 @@ def main(): for (a,) in c.execute("SELECT address FROM instructions WHERE mnemonic='lwz' AND operands=? " 'ORDER BY address', [pat]).fetchall(): print(H(a), 'in', _fn(a)) + elif cmd == 'gaps': + cmd_gaps() + elif cmd == 'imp': + cmd_imp(args[0] if args else None) + elif cmd == 'impcalls': + cmd_impcalls(args[0]) elif cmd == 'grep': for a, m, o in c.execute("SELECT address,mnemonic,operands FROM instructions " "WHERE operands LIKE ? ORDER BY address", [f'%{args[0]}%']).fetchall(): -- 2.49.1 From 38cc170491c2798ff3e57f4173e2f09247aef238 Mon Sep 17 00:00:00 2001 From: Fabian Hamm Date: Mon, 14 Sep 2026 20:49:49 +0200 Subject: [PATCH 2/4] fix(zq): stop defaulting the database path into the repo being deleted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `zq.py` hardcoded one absolute path, and it pointed inside `xenia-rs` -- a repository CONSOLIDATION.md archives in Phase 5 and drops in Phase 7. It resolved on exactly one machine, and on that machine it was days from becoming a `duckdb` exception about a missing file, with nothing saying why. Resolution order is now `$SYLPH_XEXDB`, else `/sylpheed.db`, else a refusal that names both. The repo-root path is taken relative to this script, not the caller's cwd, because zq.py is run from wherever the investigation is. No fallback beyond that, on purpose. The database is a build artefact of a few hundred MB and is not tracked, so there is nothing to fall back *to*; a default that silently resolves to the wrong database is worse than no default. That is issue #16's lesson, which was about exactly this shape in the test suite. Both failure paths say what to do and exit 1: $ zq.py fn 82000000 # nothing set, nothing at the repo root no database found. looked for: /…/Sylpheed/sylpheed.db set $SYLPH_XEXDB to an existing one, or build it with: sylph-xexdb dis --db sylpheed.db --analyze sql $ SYLPH_XEXDB=/nope/missing.db zq.py fn 82000000 $SYLPH_XEXDB is set but is not a file: /nope/missing.db Also fixes the regeneration hint, which still named `xenia-rs dis` -- the retired binary. It is `sylph-xexdb dis`. Connecting is skipped when there is no subcommand, so `zq.py` still prints its usage on a machine that has no database yet. Verified: usage with no db (rc 0); missing db (rc 1); `$SYLPH_XEXDB` set to a non-file (rc 1); and `$SYLPH_XEXDB` pointed at a real 336 MB database, where `imp Rtl` returns its 1,280 `RtlLeaveCriticalSection` call sites. That database turns out to have been generated from the uncommitted tree this branch ports: it carries `import_address`, `import_role` and all 16 foreign keys -- an independent confirmation of the schema the golden test now locks. Co-Authored-By: Claude Opus 5 --- tools/zq.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/tools/zq.py b/tools/zq.py index 0dc6dbc2..81757f24 100755 --- a/tools/zq.py +++ b/tools/zq.py @@ -38,18 +38,59 @@ Usage: A command that needs a table the current DB predates prints what to regenerate rather than a SQL error. + +The database is found via `$SYLPH_XEXDB`, else `/sylpheed.db`. It is a +build artefact, not a tracked file; if neither exists, zq.py says so and prints +the command that builds one. """ -import duckdb, sys, signal +import duckdb, sys, os, pathlib, signal # Output is routinely piped into `head`; without this Python prints a # BrokenPipeError traceback when the reader closes early. signal.signal(signal.SIGPIPE, signal.SIG_DFL) -DB = '/home/fabi/RE - Project Sylpheed/xenia-rs/sylpheed.db' -c = duckdb.connect(DB, read_only=True) -H = lambda x: '0x%08x' % x +REGEN = "sylph-xexdb dis --db sylpheed.db --analyze sql" -REGEN = ("xenia-rs dis --db sylpheed.db --analyze sql") + +def _resolve_db(): + """Locate the database: `$SYLPH_XEXDB`, else `/sylpheed.db`. + + 🔴 This used to be one hardcoded absolute path, and it pointed *inside* + `xenia-rs` -- a repository `docs/agents/CONSOLIDATION.md` archives in Phase 5 + and drops in Phase 7. It resolved on exactly one machine and would have + started failing there too, with `duckdb` raising about a missing file rather + than anything saying why. + + The database is a build artefact of several hundred MB and is not in the + repository, so there is nothing to fall back *to*: if neither location has + one, say so and print how to build it. A default that silently resolves to + the wrong database is worse than no default -- see issue #16. + """ + env = os.environ.get('SYLPH_XEXDB') + if env: + p = pathlib.Path(env).expanduser() + if not p.is_file(): + sys.exit(f'$SYLPH_XEXDB is set but is not a file:\n {p}') + return p + # Relative to this script, not to the caller's cwd: `zq.py` is run from + # wherever the investigation happens to be. + p = pathlib.Path(__file__).resolve().parent.parent / 'sylpheed.db' + if p.is_file(): + return p + sys.exit( + f'no database found.\n' + f' looked for: {p}\n' + f' set $SYLPH_XEXDB to an existing one, or build it with:\n' + f' {REGEN}' + ) + + +# Resolved lazily enough that `zq.py` with no command still prints its usage on +# a machine that has no database yet. +_WANTS_DB = len(sys.argv) > 1 and sys.argv[1] not in ('-h', '--help', 'help') +DB = _resolve_db() if _WANTS_DB else None +c = duckdb.connect(str(DB), read_only=True) if DB is not None else None +H = lambda x: '0x%08x' % x def _need(*tables): -- 2.49.1 From 10150f719d071789a014401bc715c328fca7f7d7 Mon Sep 17 00:00:00 2001 From: sim Date: Tue, 15 Sep 2026 19:20:38 +0200 Subject: [PATCH 3/4] =?UTF-8?q?chore:=20ignore=20/sylpheed.db=20=E2=80=94?= =?UTF-8?q?=20zq.py=20now=20defaults=20to=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit on this branch made `tools/zq.py` look for its database at `/sylpheed.db`, and its refusal message tells people to put one there. Nothing ignored that path. The database is a build artefact of several hundred MB, so a single `git add -A` would have committed it into a public repository's history — the class of blob CONSOLIDATION.md's history-rewrite fork exists to argue about. Anchored (`/sylpheed.db`, plus DuckDB's `.wal`) so it covers only the default location, not any file of that name elsewhere. Verified by creating the files: both ignored, `git status` silent, and a `sylpheed.db` in a subdirectory is not swallowed by the rule. Found while writing the Phase 7 steps: moving the 336 MB database out of `xenia-rs` before that clone is deleted is exactly how someone would put it here. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 6ef6a2e3..3cfd3d05 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,13 @@ Thumbs.db # Local dev overrides .env +# The static-analysis database `tools/zq.py` reads by default. A build artefact +# of several hundred MB (`sylph-xexdb dis ... --db sylpheed.db`), and zq.py +# now tells people to put it exactly here -- so it must never be committable. +# One `git add -A` would otherwise put it in public history for good. +/sylpheed.db +/sylpheed.db.wal + # Trunk build output dist/ __pycache__/ -- 2.49.1 From b57733aadef577872e818d490d670123dca582ee Mon Sep 17 00:00:00 2001 From: sim Date: Tue, 15 Sep 2026 19:23:07 +0200 Subject: [PATCH 4/4] harvest: RE_SYMBOLS.md and apply_re_symbols.sql, which Phase 3 left behind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 lifted the DuckDB tool out of `xenia-rs` but not the two files that go with it, and this branch's own code already depends on one of them: the FK rule in `db.rs` and `db_schema_golden.rs` exists *because* `apply_re_symbols.sql` re-stamps reverse-engineered names onto `functions` after every regeneration. Sylpheed referenced that file three times and contained it nowhere — on no branch. CONSOLIDATION.md's end state is "Sylpheed holds everything", and the `xenia-rs` clone is scheduled for deletion. docs/re/RE_SYMBOLS.md byte-identical to xenia-rs/RE_SYMBOLS.md (cmp) tools/apply_re_symbols.sql every statement identical; the two comment lines that name paths now name this repo's paths Verified against the real 336 MB database, which carries all 16 foreign keys, inside a transaction that was rolled back (the database is unchanged): every statement runs, and 208 functions carry reverse-engineered names. Run from both the old location and this one. Not harvested: `xenia-rs/zq_dis.py`, an untracked six-line subset of `zq.py dis` with a hardcoded path — superseded, nothing to keep. Co-Authored-By: Claude Opus 5 (1M context) --- docs/re/RE_SYMBOLS.md | 121 +++++++++++++++++++++++++++++++++++++ tools/apply_re_symbols.sql | 21 +++++++ 2 files changed, 142 insertions(+) create mode 100644 docs/re/RE_SYMBOLS.md create mode 100644 tools/apply_re_symbols.sql diff --git a/docs/re/RE_SYMBOLS.md b/docs/re/RE_SYMBOLS.md new file mode 100644 index 00000000..a69dcb85 --- /dev/null +++ b/docs/re/RE_SYMBOLS.md @@ -0,0 +1,121 @@ +# Project Sylpheed — reverse-engineered symbol map + +Durable record of guest functions/data in the retail title (`default.xex`, +base `0x82000000`) that have been positively identified by RE. Addresses are +guest virtual addresses. This is the source of truth for names; the DuckDB +`sylpheed.db` (this dir) `functions.name` column can be re-stamped from here +with `apply_re_symbols.sql` (names are wiped on a full DB regen). + +Confidence: **H** = behaviour proven (reproduced/verified against data), **M** = +strongly inferred from disassembly, **L** = tentative. + +## Archive (IPFB `.pak`) subsystem + +| Address | Name | Conf | Description | +|---|---|---|---| +| `0x824609C8` | `Pak_FindEntryByName` | H | Look up a TOC entry by path. Args `(r3 = pak object, r4 = char* path)`. Calls `Pak_HashPathName(path)`, then binary-searches the sorted 12-byte TOC (base `pak+4`, count `pak+8`, stride 12; compares the 32-bit `name_hash`). Returns the matching record pointer or 0. | +| `0x82460928` | `Pak_HashPathName` | H | Duplicate `r3 = char* path`, lowercase it in place (`Str_ToLowerAscii`), hash with `Sylph_NameHash`, free the copy, return the 32-bit hash. This is the path→key bridge. | +| `0x82455C78` | `Sylph_NameHash` | H | The IPFB TOC / cache-path name-hash. See [Name-hash](#name-hash-sylph_namehash). Input must already be lowercased. `r3 = char*` → `r3 = u32 hash`. | +| `0x825F4F90` | `Str_ToLowerAscii` | H | In-place ASCII lowercase (`A`–`Z` → `+0x20`); other bytes pass through. `r3 = char*`. (When `r3 == 0` it takes an unrelated init branch.) | +| `0x824607B0` | `Pak_IsIPFBHeader` | H | Validate an IPFB header. Loads the header's first word and checks it equals `"IPFB"` (`0x49504642`) via a delta trick: `first_word - 0x49504620 == 0x22` (`'B' - ' '`). `r5 = header ptr`. Returns bool-ish in `r3`. | +| `0x82460AD0`… | *(pak header parse, unregistered container)* | M | The pak-open/validate path around `0x82460DC0`–`0x82460EC4` builds an expected-header template (`"IPFB"`, `0x10000000` flags) on the stack and calls `Pak_IsIPFBHeader`. Function-boundary detection missed the enclosing frame (entry ≈ `0x82460E34`). | +| `0x82458508` | `Archive_StreamReadCrc32` | M | Streaming block reader that validates content with a reflected CRC-32 (init `0xFFFFFFFF`, final `~`, table at `g_Crc32Table`). Compares the computed CRC to a stored expected value (`state+144`). This is a **content-integrity** CRC, *not* the name-hash. | +| `0x828992F0` | `g_Crc32Table` (data) | H | 256-entry reflected CRC-32 lookup table. Referenced only by `Archive_StreamReadCrc32` and its sibling loop at `0x82457AF8`. | + +## IDXD reflective serialization framework + +Generic tagged-object (de)serializer shared by ~15 object families (craft, +weapon, message/character, effects, …). The `.pak` IDXD payloads are read +through this. + +| Address | Name | Conf | Description | +|---|---|---|---| +| `0x824486C0` | `IdxdLoad_Dispatch` | M | Entry: reads the magic and dispatches across variants `IDXD` (`0x49445844`), `IDX2`, `IDX3`, `IDXC`, `IXUD` → the matching parser. **15 callers** (one per object family). The target object is passed in `r3` — already constructed (defaults set) by the caller. | +| `0x82448D00` | `IdxdLoad_Variant2` | M | Sibling variant loader (also materializes the `IDXD` magic). | +| `0x82449640` | `Idxd_Parse` | M | The IDXD body parser: skips a BOM, then loops the string pool — per field, reads the name token, `Reflect_FindFieldIndex`, then dispatches on the **value's first char** (jump table at `0x824497F8`, index `firstchar - 0x22`) to type-specific writers. Only fields **present** in the pool are written. | +| `0x8244A2F0` | `Reflect_FindFieldIndex` | M | Look up a field-name string in a global name-vector at `0x820B4F18` (std::vector-like: count `+20`, data `+24`/SSO), via a `strncmp`-style compare (`0x825EDCE0`). Returns index or `-1`. | +| `0x8244A4B0` | `Reflect_SetField` | L | Apply a value to `object` using its per-instance property registry at `object+64` and a config string at `0x820B4F28`. | +| `0x820B4F18` | `g_FieldNameRegistry` (data) | L | Global reflection name-vector (runtime-built) used by `Reflect_FindFieldIndex`. | + +**Defaulted fields:** a field is written to the IDXD pool only when it differs +from its default, so `Idxd_Parse` never sets defaulted fields — their values are +established by each **type's constructor** (in the 15 `IdxdLoad_Dispatch` +callers) before load, via this same registry. Recovering them statically means +reversing those constructors and joining `field-name → struct-offset → +default-store`; a runtime dump of a loaded object's registry gives +`name → value` directly. (Open — see below.) + +## 3D resource subsystem + +| Address | Name | Conf | Description | +|---|---|---|---| +| `0x82640290` | `Res3D_LoadMeshChunk` | L | 3D-resource loader: walks 12-byte records, checks chunk tags (`0x1A22AA26`, `0x2DA2AA24`), and copies `float` triples (x/y/z vertices) via `lfs`/`stfs`. Part of the DefTables/`machines\\…` mesh path. | + +## Name-hash (`Sylph_NameHash`) + +Per-byte Barrett-reduced polynomial hash over the **lowercased** name: + +``` +A = 0 ; B = 0 +for each (sign-extended) byte c of the lowercased name: + A = (A << 8) + c # 32-bit + A = A - (((A * 0x8003_1493) >> 32) rol 9 & 0x1FF) * 0x00FF_F9D7 # A mod 0x00FF_F9D7 (no final fixup) + B = B + c +hash = ((B & 0xFF) << 24) | (A & 0x00FF_FFFF) +``` + +- Modulus `0x00FF_F9D7`, reciprocal `0x8003_1493` (the `mulhwu`/`mullw` Barrett step). +- Low 24 bits = modular polynomial hash; top byte = 8-bit additive checksum of the bytes. +- No trailing conditional subtract — the value is defined by the exact op sequence. +- Faithful Rust reproduction: `sylpheed-formats/src/hash.rs` (`name_hash`). + +**Verified** against retail TOCs: `name_hash("files.tbl") == 0x8342_1153`, +`name_hash("eng\\weapon.tbl") == 0x900C_8DCD`, `name_hash("eng\\strings.tbl") == 0x10C8_0B87`, +`name_hash("jpn\\weapon.tbl") == 0x9E85_FEFF`. + +## TOC key path conventions (preimages) + +The 32-bit TOC key is `name_hash` of a **backslash** path with the entry's +internal identity string. Confirmed schemes: + +| Scheme | Example | Where | +|---|---|---| +| `unit\.tbl` | `unit\UN_f001_TCAF_DeltaSaber_T_EX5.tbl` → `0x7C96296C` | craft/ship entries in `GP_MAIN_GAME_*.pak` | +| `weapon\.tbl` | `weapon\Weapon_DSaber_P_wep_26_Missile.tbl` | weapon entries | +| `message\.tbl` | `message\CharacterCARL.tbl` | character/dialog entries | +| `effect\.tbl` | — | effect definitions | +| `\.tbl` | `eng\weapon.tbl`, `jpn\strings.tbl` | localized loadout/text tables | +| `.tbl` | `files.tbl` | root manifest / DefTables resource entries | + +`` = the entry's internal `ID` field (IDXD string pool). Craft/weapon +entries also self-describe by content, so stats are readable without the key. + +Coverage with these schemes (IDXD entries, whole-pool ID extraction): +GP_MAIN_GAME_E 308/1004 (≈31%), DefTables 804/1425 (≈56%). Shipped in +`sylpheed-formats` (`hash::recover_toc_name`, `pak list` prints resolved +paths). The unresolved remainder use deeper cross-referenced directory +paths (e.g. per-mission dialog `MSG_*` entries) — a later hunt, not an +extraction limit. + +## Open threads + +- **Defaulted IDXD fields.** A field is omitted from the pool exactly when it + equals the schema default, so present values are all *overrides* and the + defaults cannot be inferred from them. Gap is large: across the 534 craft + (`schema 0x43FAA517`) entries, explicit-presence is `Acceleration` 42%, + `MaximumVelocity` 51%, `Turn_AngularVelocity` 31%, `FCSRange` 34%, + `ShieldRatio` 49% (`Size_X` 100%, `HP` 85%). So 40–70% of craft rely on + unknown defaults for core flight/defense stats. + - Ruled out: the big 16-byte-record binary node table inside a craft IDXD is + **spatial/mesh data**, not defaults (not keyed by `name_hash(field_key)`). + - `schema_hash` is **not** a code immediate and **not** `name_hash(typename)`. + - Defaults are set by each type's **constructor** (one per `IdxdLoad_Dispatch` + caller) via the reflection registry, not by `Idxd_Parse`. + - **Two finish routes.** (A, static) reverse the craft constructor: read its + field registrations (`name → offset`) and default-init stores + (`offset → value`), join them — heavy, and offset↔name correlation is + error-prone. (B, runtime) dump a loaded craft object's registry + (`obj+64`) / struct from xenia-rs or Canary — a fully-loaded craft already + holds every default; one dump yields `name → value`. **B is the efficient + finish** given the framework is generic; A's framework map above is the + prerequisite either way. diff --git a/tools/apply_re_symbols.sql b/tools/apply_re_symbols.sql new file mode 100644 index 00000000..8045a4d0 --- /dev/null +++ b/tools/apply_re_symbols.sql @@ -0,0 +1,21 @@ +-- Re-stamp reverse-engineered function names onto sylpheed.db. +-- Source of truth: docs/re/RE_SYMBOLS.md. Re-run after any full DB regen +-- (regen wipes functions.name back to sub_XXXX). +-- python3 -c "import duckdb; duckdb.connect('sylpheed.db').execute(open('tools/apply_re_symbols.sql').read())" +-- (from the repo root; `sylpheed.db` is where `tools/zq.py` looks by default) +-- Addresses are decimal (DuckDB rejects 0x literals in some contexts); hex in comments. + +UPDATE functions SET name = 'Pak_FindEntryByName' WHERE address = 2185628104; -- 0x824609C8 +UPDATE functions SET name = 'Pak_HashPathName' WHERE address = 2185627944; -- 0x82460928 +UPDATE functions SET name = 'Sylph_NameHash' WHERE address = 2185583736; -- 0x82455C78 +UPDATE functions SET name = 'Str_ToLowerAscii' WHERE address = 2187284368; -- 0x825F4F90 +UPDATE functions SET name = 'Pak_IsIPFBHeader' WHERE address = 2185627568; -- 0x824607B0 +UPDATE functions SET name = 'Archive_StreamReadCrc32' WHERE address = 2185594120; -- 0x82458508 +UPDATE functions SET name = 'Res3D_LoadMeshChunk' WHERE address = 2187592336; -- 0x82640290 + +-- IDXD reflective serialization framework (defaulted-field hunt, 2026-07-09) +UPDATE functions SET name = 'IdxdLoad_Dispatch' WHERE address = 2185529024; -- 0x824486C0 (magic dispatch IDXD/IDX2/IDX3/IDXC/IXUD; 15 callers) +UPDATE functions SET name = 'IdxdLoad_Variant2' WHERE address = 2185530624; -- 0x82448D00 (sibling variant loader) +UPDATE functions SET name = 'Idxd_Parse' WHERE address = 2185532992; -- 0x82449640 (tokenize pool; per-field lookup+set) +UPDATE functions SET name = 'Reflect_FindFieldIndex' WHERE address = 2185536240; -- 0x8244A2F0 (field-name -> index in name-vector @0x820B4F18) +UPDATE functions SET name = 'Reflect_SetField' WHERE address = 2185536688; -- 0x8244A4B0 (apply value via object registry @obj+64) -- 2.49.1