From ac20afb1a972273342cf6ce703121f068da825e4 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Thu, 10 Sep 2026 19:57:14 +0200 Subject: [PATCH] analysis: complete, correct and faster XEX static analysis Reworks the `dis --db` static-analysis pipeline. Everything here is derived from the XEX alone; no run traces. Completeness - `.pdata` is now authoritative for function boundaries rather than a hint: all 23,073 linker-declared RUNTIME_FUNCTIONs are emitted and `end_address` is the declared end, not a prologue-walk guess. Adds tail-call targets (with a preceding-terminator fallback for the ~450 KB of .text `.pdata` leaves unclaimed): 12,156 -> 25,676 functions. - New `jumptables.rs`: 394 switch tables / 6,356 case edges recovered from 941 `bctr` sites. Table words are flagged `instructions.is_data` and kept out of the xref pass, and render as `.long` with case labels in `-o`. - New `rtti.rs`: RTTI is not stripped in this title. 611 type descriptors, 653 locators, 1,491 base-class records, giving a real inheritance graph with mdisp/pdisp/vdisp. Verified against the STL: std::out_of_range : std::logic_error : std::exception. - New `xdbf.rs` + `xenia-xex/resources.rs`: the XDBF/SPA package the XEX names via its resource table -- 24 achievements (1000G, matching the project's independently derived figure), 888 localized strings across 6 locales, 25 PNGs, title name/id/version. Located through the entry table, not a magic scan; a scan finds a phantom 7th string table where the entry table declares 6, which shifts every scan-derived language index. - Strings scan extended to `.data`; metadata grows from 5 keys to 41. Correctness - `instructions.function` was a rolling window -- set at each function start and never cleared -- so every word in a `.pdata` gap was attributed to the preceding function. 55,227 rows were wrong and the bogus 100% attribution hid the gap. Now cleared at `end_address`; 0 wrong, honest 97.0%. - Shift-JIS scanning accepted half-width katakana, which turned IEEE-754 float tables into "Japanese" (`3f 66 66 66` = 0.9f reads as "fff"), and emitted escaped bytes rather than text. 837 mostly-noise rows -> 115 real strings, decoded via encoding_rs, with resync so a run starting one byte early reports the true address instead of mangling the first character. - `indirect_dispatch_candidates` was a cross product, not a resolution: at vptr_offset 0 the (offset, slot) match hits nearly every class, so 6,556 of 6,983 sites produced 1.80M of 1.81M rows (one site claiming 764 callees) and the derived ind_call edges were 84% of the xrefs table. Adds `--max-indirect-candidates` (default 16); over-ceiling sites keep their row with a truthful count and a new `truncated` flag but emit no candidate rows and no xrefs. 1.81M -> 4,199 candidates, xrefs 2.16M -> 357k. - xenia-xex: `TLS_INFO` and `DEFAULT_STACK_SIZE` header keys were swapped. `get_stack_size()` would have returned the TLS descriptor's file offset; it has no callers today, so nothing regressed. Performance - DuckDB autocommits per statement, so row-at-a-time INSERT paid a transaction + WAL flush per row: 221k rows took 20 minutes and the 1.8M candidate rows ~59 more, for an 81-minute build that never finished. One big transaction fixes the time but not the cause (uncommitted state grew to 16.6 GB RSS). Every table above a few thousand rows now uses the Appender; each converted sink documents why its key cannot collide or dedupes first. 2m20s wall, 486 MB peak, DB 635 MB -> 318 MB. Also: 6 new SQL views, `zq.py` gains switch/switches/classes/class/str/ xdbf/ach, and an `analysis_report` example that runs the passes without building a database. 80 tests pass (29 new). Co-Authored-By: Claude Opus 5 --- .gitignore | 4 + Cargo.lock | 138 +-- README.md | 37 +- crates/xenia-analysis/Cargo.toml | 1 + .../examples/analysis_report.rs | 90 ++ crates/xenia-analysis/src/db.rs | 784 +++++++++++++++--- crates/xenia-analysis/src/demangle.rs | 99 +++ crates/xenia-analysis/src/disasm.rs | 119 ++- crates/xenia-analysis/src/formatter.rs | 9 +- crates/xenia-analysis/src/func.rs | 178 +++- .../xenia-analysis/src/ind_dispatch_typed.rs | 103 ++- crates/xenia-analysis/src/indirect.rs | 3 + crates/xenia-analysis/src/jumptables.rs | 758 +++++++++++++++++ crates/xenia-analysis/src/lib.rs | 3 + crates/xenia-analysis/src/rtti.rs | 453 ++++++++++ crates/xenia-analysis/src/sinks/duckdb.rs | 4 +- crates/xenia-analysis/src/sinks/json.rs | 2 + crates/xenia-analysis/src/sinks/text.rs | 13 + crates/xenia-analysis/src/sql_views.rs | 124 ++- crates/xenia-analysis/src/static_init.rs | 4 +- crates/xenia-analysis/src/strings.rs | 201 +++-- crates/xenia-analysis/src/vtables.rs | 72 +- crates/xenia-analysis/src/xdbf.rs | 450 ++++++++++ crates/xenia-analysis/src/xref.rs | 40 +- .../xenia-analysis/tests/db_schema_golden.rs | 94 ++- crates/xenia-app/src/main.rs | 139 +++- crates/xenia-xex/src/header.rs | 14 +- crates/xenia-xex/src/lib.rs | 1 + crates/xenia-xex/src/pdata.rs | 5 +- crates/xenia-xex/src/resources.rs | 127 +++ zq.py | 180 +++- 31 files changed, 3950 insertions(+), 299 deletions(-) create mode 100644 crates/xenia-analysis/examples/analysis_report.rs create mode 100644 crates/xenia-analysis/src/jumptables.rs create mode 100644 crates/xenia-analysis/src/rtti.rs create mode 100644 crates/xenia-analysis/src/xdbf.rs create mode 100644 crates/xenia-xex/src/resources.rs diff --git a/.gitignore b/.gitignore index 952ac1e..c541a84 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,7 @@ vkd3d-proton.cache* # local analysis-DB backups (regenerable; too large to track) *.db.bak* sylpheed.db.bak-* +sylpheed.db.pre-*.bak + +# full-disassembly listings from `dis -o` (133 MB on a retail title, regenerable) +*.asm diff --git a/Cargo.lock b/Cargo.lock index 8f28cd9..f7991d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1226,6 +1226,15 @@ dependencies = [ "strum", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -1521,7 +1530,7 @@ dependencies = [ "vec_map", "wasm-bindgen", "web-sys", - "windows 0.62.2", + "windows 0.58.0", ] [[package]] @@ -2100,13 +2109,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -2244,7 +2252,7 @@ dependencies = [ "bitflags 2.11.0", "libc", "plain", - "redox_syscall 0.7.4", + "redox_syscall 0.7.5", ] [[package]] @@ -3360,9 +3368,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" +checksum = "4666a1a60d8412eab19d94f6d13dcc9cea0a5ef4fdf6a5db306537413c661b1b" dependencies = [ "bitflags 2.11.0", ] @@ -4540,9 +4548,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -4554,9 +4562,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.68" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ "js-sys", "wasm-bindgen", @@ -4564,9 +4572,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4574,9 +4582,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", @@ -4587,9 +4595,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] @@ -4726,9 +4734,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.95" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -4908,23 +4916,12 @@ dependencies = [ [[package]] name = "windows" -version = "0.62.2" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" dependencies = [ - "windows-collections", - "windows-core 0.62.2", - "windows-future", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" -dependencies = [ - "windows-core 0.62.2", + "windows-core 0.58.0", + "windows-targets", ] [[package]] @@ -4936,28 +4933,41 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement 0.58.0", + "windows-interface 0.58.0", + "windows-result 0.2.0", + "windows-strings 0.1.0", + "windows-targets", +] + [[package]] name = "windows-core" version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "windows-implement", - "windows-interface", + "windows-implement 0.60.2", + "windows-interface 0.59.3", "windows-link", - "windows-result", - "windows-strings", + "windows-result 0.4.1", + "windows-strings 0.5.1", ] [[package]] -name = "windows-future" -version = "0.3.2" +name = "windows-implement" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ - "windows-core 0.62.2", - "windows-link", - "windows-threading", + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] @@ -4971,6 +4981,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "windows-interface" version = "0.59.3" @@ -4989,13 +5010,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "windows-numerics" -version = "0.3.1" +name = "windows-result" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" dependencies = [ - "windows-core 0.62.2", - "windows-link", + "windows-targets", ] [[package]] @@ -5007,6 +5027,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result 0.2.0", + "windows-targets", +] + [[package]] name = "windows-strings" version = "0.5.1" @@ -5059,15 +5089,6 @@ dependencies = [ "windows_x86_64_msvc", ] -[[package]] -name = "windows-threading" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" -dependencies = [ - "windows-link", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -5252,6 +5273,7 @@ version = "0.1.0" dependencies = [ "anyhow", "duckdb", + "encoding_rs", "metrics", "msvc-demangler", "serde", diff --git a/README.md b/README.md index 7ecd809..1c21f43 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,10 @@ for CPU context setup, kernel export behavior, and XEX loading semantics. - **XEX loader** — XEX2 header parsing, LZX decompression, AES decryption, PE section parsing. - **VFS / XISO** — XGD2 dual-layer disc images (with the 0x0FD90000 partition offset). - **PPC interpreter** — 200+ opcodes, PowerPC 32/64-bit GPR/FPR, VMX128 decoding. -- **Static analyzer** — function discovery (prolog/epilog heuristics), cross-references, labels, - save/restore helper detection, assembly text + SQLite database output. +- **Static analyzer** — function discovery (`.pdata` ground truth + prolog/epilog heuristics), + cross-references, labels, save/restore helper detection, `switch`/jump-table recovery, + MSVC RTTI class + vtable + inheritance recovery, string scan, EH scope tables, + assembly text + DuckDB database output. - **Kernel HLE** — minimal subset driving Project Sylpheed: ~170 xboxkrnl + xam exports (critical sections, events, TLS, virtual memory, Vd stubs, XAM input/user/content). - **Debugger** — in-memory step/break, SQLite execution + import-call + branch tracing. @@ -70,10 +72,35 @@ Writes `.pe` (decompressed/decrypted PE image) and `.xex.json` xenia-rs dis [-o ] [--db ] [--quiet] ``` -Runs function + cross-reference analysis and produces: +Runs the full static-analysis pipeline and produces: - assembly text to stdout or `-o ` (unless `--quiet`) -- optional SQLite DB with the **base tables + disasm tables**: - `functions`, `labels`, `instructions`, `xrefs` +- optional DB with the **base tables + analysis tables**: + +| group | tables | +|---|---| +| disasm | `instructions` (`is_data` flags jump-table words), `functions`, `labels`, `xrefs`, `pdata_entries` | +| switches | `jump_tables`, `jump_table_entries`, `data_in_code` | +| C++ / RTTI | `rtti_type_descriptors`, `rtti_locators`, `rtti_base_classes`, `vtables`, `methods`, `classes`, `demangled_names` | +| other | `strings`, `eh_funcinfo`/`eh_unwind_map`/`eh_try_blocks`, `function_pointer_arrays`, `indirect_dispatch_*`, `vptr_writes`, `tls_info`/`tls_callbacks` | + +`--analyze sql` additionally creates query views: `v_switch_cases`, +`v_class_hierarchy`, `v_class_methods`, `v_function_strings`, `v_call_graph`, +`v_reachability_from_entry`, `v_indirect_reachability_from_entry`, +`v_branch_xrefs`, `v_function_first_instruction`, `v_imports_called`. + +Function boundaries come from the linker's `.pdata` `RUNTIME_FUNCTION` table +where it covers the function (exact), and from prologue/epilogue walking +elsewhere (`functions.pdata_validated` says which). **Rows in `instructions` +with `is_data = true` are jump-table words, not code** — filter them out of any +instruction-level statistic. + +To iterate on an analysis pass without paying for a full DB build: + +```sh +cargo run --release -p xenia-analysis --example analysis_report -- +``` + +which runs the passes and prints their headline counts in seconds. ### `exec` — interpret with tracing diff --git a/crates/xenia-analysis/Cargo.toml b/crates/xenia-analysis/Cargo.toml index ae9a234..5a4c08b 100644 --- a/crates/xenia-analysis/Cargo.toml +++ b/crates/xenia-analysis/Cargo.toml @@ -15,3 +15,4 @@ tracing = { workspace = true } metrics = { workspace = true } duckdb = { workspace = true } msvc-demangler = "0.11" +encoding_rs = "0.8" diff --git a/crates/xenia-analysis/examples/analysis_report.rs b/crates/xenia-analysis/examples/analysis_report.rs new file mode 100644 index 0000000..a5bffd8 --- /dev/null +++ b/crates/xenia-analysis/examples/analysis_report.rs @@ -0,0 +1,90 @@ +//! Fast static-analysis report — runs the passes that need only the XEX and +//! prints their headline counts, without building a database. +//! +//! `dis --db` builds and indexes ~2.4M rows, which is a couple of minutes even +//! after the Appender rework. When you are iterating on an analysis pass itself +//! that round-trip is still dead time, so this example runs just the passes — +//! all of which finish in well under a second — and prints what they found: +//! +//! ```text +//! cargo run --release -p xenia-analysis --example analysis_report -- +//! ``` +use std::env; + +fn main() -> anyhow::Result<()> { + let path = env::args().nth(1).ok_or_else(|| anyhow::anyhow!("usage: analysis_report "))?; + let data = std::fs::read(&path)?; + let mut header = xenia_xex::loader::parse_xex2_header(&data)?; + let entry = xenia_xex::loader::get_entry_point(&header) + .ok_or_else(|| anyhow::anyhow!("no entry point"))?; + let base = xenia_xex::loader::get_image_base(&header) + .ok_or_else(|| anyhow::anyhow!("no image base"))?; + let pe = xenia_xex::loader::load_image(&data, &header)?; + xenia_xex::loader::resolve_imports(&mut header, &pe); + let sections = xenia_xex::pe::parse_sections(&pe)?; + + println!("image_base {base:#010x} entry {entry:#010x} sections {}", sections.len()); + + let code_sections: Vec<(u32, u32, u32)> = sections + .iter() + .filter(|s| s.is_code()) + .map(|s| (s.virtual_address, s.virtual_size, s.flags)) + .collect(); + let pdata = xenia_xex::pdata::parse_pdata(&pe, base, §ions); + let fa = xenia_analysis::func::analyze_with_pdata(&pe, base, entry, &code_sections, &pdata); + + let validated = fa.functions.values().filter(|f| f.pdata_validated).count(); + let exact = fa + .functions + .values() + .filter(|f| f.pdata_length.is_some_and(|n| f.end - f.start == n)) + .count(); + println!( + "pdata entries {} functions {} pdata-validated {} boundary==pdata {} has_eh {}", + pdata.len(), + fa.functions.len(), + validated, + exact, + fa.functions.values().filter(|f| f.has_eh).count(), + ); + + if std::env::var("DUMP_FNS").is_ok() { + for (a, f) in &fa.functions { + println!("FN {a:08X} {:08X} {}", f.end, f.pdata_validated); + } + } + let (jts, rejects) = xenia_analysis::jumptables::analyze_with_stats(&pe, base, §ions, &fa); + let cases: usize = jts.iter().map(|t| t.targets.len()).sum(); + let words = xenia_analysis::jumptables::data_word_addresses(&jts); + println!( + "jump tables {} ({} indexed) case entries {} data-in-code words {}", + jts.len(), + jts.iter().filter(|t| t.kind == "indexed").count(), + cases, + words.len(), + ); + println!(" rejected: {rejects:?}"); + if std::env::var("DUMP_JT").is_ok() { + for t in &jts { + println!("JT {:#010x} {:#010x} {} {}", t.bctr_pc, t.table_address, t.kind, t.entry_count); + } + } + + let rtti = xenia_analysis::rtti::analyze(&pe, base, §ions); + println!( + "RTTI: type descriptors {} locators {} vtables {} base-class records {}", + rtti.type_descriptors.len(), + rtti.locators.len(), + rtti.vtable_to_locator.len(), + rtti.base_classes.len(), + ); + for t in rtti.type_descriptors.iter().take(8) { + println!(" {:#010x} {}", t.address, t.demangled_name); + } + + let strings = xenia_analysis::strings::analyze(&pe, base, §ions); + let in_data = strings.iter().filter(|s| s.section == ".data").count(); + println!("strings {} ({} in .data)", strings.len(), in_data); + + Ok(()) +} diff --git a/crates/xenia-analysis/src/db.rs b/crates/xenia-analysis/src/db.rs index 155c75a..152ac36 100644 --- a/crates/xenia-analysis/src/db.rs +++ b/crates/xenia-analysis/src/db.rs @@ -18,7 +18,12 @@ //! # Schema //! //! ## `metadata` -//! Key-value table. One row per XEX header field. Values are strings. +//! Key-value table, one row per XEX header field; values are strings. Beyond +//! the five columns tabulated below it also carries the module/image flag words +//! (raw + decoded), image size and load address, encryption + compression type, +//! disc number/count, per-import-library SDK versions +//! (`import_lib..version_cur`), and one `xex_optional_header.0x…` row for +//! every optional header present, so nothing in the XEX is silently dropped. //! //! | key | value format | meaning | //! |--------------------|------------------|----------------------------------------------------| @@ -47,14 +52,26 @@ //! - `address` — Absolute VA of the import thunk or variable in the binary //! //! ## `functions` -//! One row per detected function (from prologue analysis). -//! - `address` — Absolute VA of the function entry point (PK) -//! - `name` — Symbol name, or `sub_XXXXXXXX` if unresolved -//! - `end_address` — Absolute VA of last instruction + 4 (exclusive end) -//! - `frame_size` — Stack frame size in bytes (from prologue) -//! - `saved_gprs` — Bitmask of GPRs saved in prologue (bit N set ⇒ rN is saved) -//! - `is_leaf` — `true` if the function has no outgoing calls (no `bl`/`blr`) -//! - `is_saverestore` — `true` if this is a `__savegprlr_*`/`__restgprlr_*` compiler stub +//! One row per detected function. Candidates are `bl` targets ∪ `.pdata` +//! `BeginAddress`es ∪ tail-call targets ∪ the entry point. `.pdata` is the +//! linker's own function table and is treated as authoritative: where it +//! covers a function, `end_address` is its declared end rather than a +//! prologue-walk guess. +//! - `address` — Absolute VA of the function entry point (PK) +//! - `name` — Symbol name, or `sub_XXXXXXXX` if unresolved +//! - `end_address` — Absolute VA of last instruction + 4 (exclusive end) +//! - `frame_size` — Stack frame size in bytes (from prologue) +//! - `saved_gprs` — Bitmask of GPRs saved in prologue (bit N set ⇒ rN is saved) +//! - `is_leaf` — `true` if the function has no outgoing calls (no `bl`/`blr`) +//! - `is_saverestore` — `true` if this is a `__savegprlr_*`/`__restgprlr_*` compiler stub +//! - `pdata_validated` — `true` when `.pdata` declares a function at this VA +//! - `pdata_length` — Declared size in bytes; `NULL` when prologue-only +//! - `prolog_length` — Declared prolog size in bytes; `NULL` when prologue-only +//! - `has_eh` — `.pdata` exception-handler bit; function has C++ EH/SEH +//! +//! ## `pdata_entries` +//! The raw `.pdata` `RUNTIME_FUNCTION` table, one row per entry, so a query +//! can distinguish linker ground truth from this crate's inferences. //! //! ## `labels` //! One row per named address; superset of functions. @@ -76,13 +93,34 @@ //! - `section` — Name of the PE section containing this instruction //! - `function` — VA of the enclosing function; `NULL` if not inside a detected function //! - `label` — Label name at this address; `NULL` if none +//! - `is_data` — `true` when this word is data embedded in a code section (a +//! recovered jump table or index map). The decoded `mnemonic` / +//! `disasm` columns are meaningless on such rows; filter them out +//! (`WHERE NOT is_data`) for any instruction-level analysis. +//! +//! ## `jump_tables` / `jump_table_entries` / `data_in_code` +//! Recovered `switch` dispatches (see [`crate::jumptables`]). `jump_tables` has +//! one row per resolved `bctr`; `jump_table_entries` one row per case value in +//! case order. `data_in_code` lists the byte ranges those tables occupy inside +//! code sections — every word listed there is also flagged `instructions.is_data`. +//! +//! ## `rtti_type_descriptors` / `rtti_locators` / `rtti_base_classes` +//! The MSVC RTTI walk (see [`crate::rtti`]). These are the authoritative source +//! of C++ class identity: `rtti_type_descriptors.demangled_name` is the name +//! the linker wrote, not a heuristic guess. `rtti_locators.vtable_address` +//! binds a class to its vftable (`subobject_offset` separates the primary +//! vftable from the extra ones a multiply-inheriting class emits), and +//! `rtti_base_classes` is the linearised inheritance list with the PMD +//! displacement triple for each base. //! //! ## `xrefs` //! One row per cross-reference edge (call, jump, data access). //! - `source` — Absolute VA of the instruction making the reference //! - `target` — Absolute VA of the referenced destination //! - `kind` — Reference type as the short tag from [`crate::xref::XrefKind::tag`]: -//! `call`, `j` (jump), `br` (branch), `read` (data_read), +//! `call`, `ind_call` (resolved vtable `bcctrl`), +//! `jt` (recovered `switch` case), `j` (jump), +//! `br` (branch), `read` (data_read), //! `write` (data_write), `ref` (data_ref). //! Note: this is a different convention from `branch_trace.kind`, //! which uses the long names (`call` / `return` / `jump` / `branch`). @@ -264,6 +302,7 @@ impl DbWriter { info: &DisasmInfo, func_analysis: &FuncAnalysis, labels: &HashMap, + data_words: &std::collections::BTreeSet, ) -> anyhow::Result<()> { self.conn.execute_batch(" CREATE TABLE instructions ( @@ -278,11 +317,12 @@ impl DbWriter { target_hex BIGINT, -- resolved absolute target for direct branches; NULL for indirect/non-branch section VARCHAR NOT NULL, -- PE section name containing this instruction function BIGINT, -- VA of the enclosing function; NULL if unknown - label VARCHAR -- label at this address; NULL if none + label VARCHAR, -- label at this address; NULL if none + is_data BOOLEAN NOT NULL -- M12: word is data embedded in code (jump table / index map), NOT an instruction ); ")?; - insert_instructions_streaming(&self.conn, pe, info, func_analysis, labels)?; + insert_instructions_streaming(&self.conn, pe, info, func_analysis, labels, data_words)?; let indices = [ ("idx_instructions_function", "CREATE INDEX idx_instructions_function ON instructions(function)"), @@ -291,6 +331,7 @@ impl DbWriter { ("idx_instructions_section", "CREATE INDEX idx_instructions_section ON instructions(section)"), ("idx_instructions_label", "CREATE INDEX idx_instructions_label ON instructions(label)"), ("idx_instructions_target_hex", "CREATE INDEX idx_instructions_target_hex ON instructions(target_hex)"), + ("idx_instructions_is_data", "CREATE INDEX idx_instructions_is_data ON instructions(is_data)"), ]; for (name, sql) in indices { tracing::debug!(index = name, "creating instructions index"); @@ -308,7 +349,8 @@ impl DbWriter { /// not run the vtable scan (the tables are still created, just empty). /// `strings` is the M7 result; same convention. `funcptr_arrays` is the /// M8/M11 result. `typed_ind` is the M5.5 result. `eh_records` is the - /// M9.5 result. + /// M9.5 result. `xdbf` is the embedded title package, `None` when the XEX + /// declares no resource. #[tracing::instrument(skip_all, name = "db.write_analysis_results")] pub fn write_analysis_results( &mut self, @@ -322,6 +364,9 @@ impl DbWriter { funcptr_arrays: &[crate::funcptr_arrays::FuncPtrArray], typed_ind: Option<&crate::ind_dispatch_typed::TypedIndirectResult>, eh_records: &[crate::eh_scope::EhFuncInfo], + jump_tables: &[crate::jumptables::JumpTable], + rtti: &crate::rtti::RttiResult, + xdbf: Option<&crate::xdbf::Xdbf>, ) -> anyhow::Result<()> { self.conn.execute_batch(" CREATE TABLE functions ( @@ -334,6 +379,7 @@ impl DbWriter { is_saverestore BOOLEAN NOT NULL, -- true if __savegprlr_* / __restgprlr_* stub pdata_validated BOOLEAN NOT NULL, -- true if .pdata RUNTIME_FUNCTION exists at this VA pdata_length BIGINT, -- length in bytes per .pdata; NULL if no pdata entry + prolog_length BIGINT, -- prolog size in bytes per .pdata; NULL if no pdata entry has_eh BOOLEAN NOT NULL -- M9: pdata exception-flag bit set; function has C++ EH/SEH ); @@ -378,9 +424,10 @@ impl DbWriter { CREATE TABLE strings ( address BIGINT PRIMARY KEY, -- absolute VA of first byte - encoding VARCHAR NOT NULL, -- 'ascii' or 'utf16le' + encoding VARCHAR NOT NULL, -- 'ascii' | 'utf16le' | 'shift_jis' | 'utf8' length BIGINT NOT NULL, -- length in bytes (excluding NUL terminator) - content VARCHAR NOT NULL -- UTF-8 representation of the string + content VARCHAR NOT NULL, -- UTF-8 representation of the string + section VARCHAR NOT NULL -- PE section the string lives in (.rdata / .data) ); CREATE TABLE tls_info ( @@ -414,12 +461,17 @@ impl DbWriter { -- bcctrl site that matched the canonical lwz vt, off(this); -- lwz fn, slot(vt); mtctr; bcctrl pattern. candidate_count > 1 -- means the analysis could not pick a single class; downstream - -- queries should treat such rows as reachability-only. + -- queries should treat such rows as reachability-only. When + -- `truncated` is set the site had more candidates than the + -- ceiling and none were materialised — the call is virtual and + -- unresolved, and `candidate_count` says how unresolved. CREATE TABLE indirect_dispatch_sites ( dispatch_pc BIGINT PRIMARY KEY, vptr_offset BIGINT NOT NULL, slot BIGINT NOT NULL, - candidate_count BIGINT NOT NULL + candidate_count BIGINT NOT NULL, -- candidates that matched, materialised or not + truncated BOOLEAN NOT NULL -- true => candidate_count exceeded the ceiling, + -- so no rows in indirect_dispatch_candidates ); -- M5.5 — one row per (dispatch site × candidate vtable). The @@ -477,6 +529,49 @@ impl DbWriter { PRIMARY KEY (funcinfo_address, try_index) ); + -- XDBF/SPA package embedded in the XEX (see `crate::xdbf`). + -- One row per entry of the container's entry table. + CREATE TABLE xdbf_entries ( + namespace BIGINT NOT NULL, -- 1=metadata, 2=image, 3=string table + namespace_name VARCHAR NOT NULL, + id BIGINT NOT NULL, -- fourcc / language / image id per namespace + body_offset BIGINT NOT NULL, -- offset of the body within the image buffer + size BIGINT NOT NULL, + magic VARCHAR, -- leading fourcc of the body, when printable + PRIMARY KEY (namespace, id) + ); + + CREATE TABLE xdbf_achievements ( + id BIGINT PRIMARY KEY, -- 1-based achievement id + name VARCHAR, -- resolved via the default language's string table + unlocked_desc VARCHAR, + locked_desc VARCHAR, + label_id BIGINT NOT NULL, -- string ids, for joining other languages + description_id BIGINT NOT NULL, + unachieved_id BIGINT NOT NULL, + image_id BIGINT NOT NULL, -- FK to xdbf_images.id + gamerscore BIGINT NOT NULL, + flags BIGINT NOT NULL + ); + + -- Every localized string in the package. This is where the title + -- name, mission titles, game-phase labels and leaderboard names live. + CREATE TABLE xdbf_strings ( + language BIGINT NOT NULL, -- XLanguage value + language_name VARCHAR NOT NULL, + string_id BIGINT NOT NULL, + value VARCHAR NOT NULL, + PRIMARY KEY (language, string_id) + ); + + CREATE TABLE xdbf_images ( + id BIGINT PRIMARY KEY, -- image id referenced by achievements + is_title_icon BOOLEAN NOT NULL, -- id 0x8000 — the title's own icon + body_offset BIGINT NOT NULL, -- offset within the image buffer + size BIGINT NOT NULL, + format VARCHAR NOT NULL -- 'png' when the body carries the PNG signature + ); + CREATE TABLE demangled_names ( address BIGINT, -- VA the mangled name is associated with; NULL when from a non-address source (e.g. RTTI-only string) mangled VARCHAR NOT NULL, -- original mangled symbol (e.g. ?Foo@Bar@@QEAAXXZ) @@ -487,6 +582,76 @@ impl DbWriter { params_signature VARCHAR -- contents of the outermost (...) (NULL = not a function) ); + -- M12 — recovered `switch` dispatches. One row per `bctr` whose + -- jump table the analyzer could resolve and validate. + CREATE TABLE jump_tables ( + bctr_pc BIGINT PRIMARY KEY, -- VA of the dispatching bctr + function BIGINT, -- VA of the enclosing function + table_address BIGINT NOT NULL, -- VA of the absolute-target table + entry_count BIGINT NOT NULL, -- number of case values (after index-map expansion) + table_slots BIGINT NOT NULL, -- 4-byte slots occupied by the target table itself + index_map_address BIGINT, -- VA of the byte-wide index map (sparse switch only) + index_map_count BIGINT, -- bytes read from the index map + case_bound BIGINT, -- largest valid case index per the cmplwi bound check + kind VARCHAR NOT NULL -- 'direct' (table[idx]) | 'indexed' (table[map[idx]]) + ); + + -- M12 — one row per case value, in case order. `target_address` + -- repeats when several case values share a body. + CREATE TABLE jump_table_entries ( + bctr_pc BIGINT NOT NULL, -- FK to jump_tables.bctr_pc + case_index BIGINT NOT NULL, -- 0-based case value + target_address BIGINT NOT NULL, -- VA of the case body + PRIMARY KEY (bctr_pc, case_index) + ); + + -- M12 — byte ranges inside code sections that hold data, not + -- instructions. Anything listed here is a decoding hazard: linear + -- disassembly of these words produces garbage rows. + CREATE TABLE data_in_code ( + address BIGINT PRIMARY KEY, -- VA of the first byte + length BIGINT NOT NULL, -- byte length + kind VARCHAR NOT NULL -- 'jump_table' | 'jump_index_map' + ); + + -- M13 — MSVC RTTI. `rtti_type_descriptors` is the authoritative + -- source of C++ class identity: the linker wrote these names, they + -- are not inferred. + CREATE TABLE rtti_type_descriptors ( + address BIGINT PRIMARY KEY, -- VA of the TypeDescriptor + mangled_name VARCHAR NOT NULL, -- decorated name, e.g. .?AVSilph@silph@@ + demangled_name VARCHAR NOT NULL -- readable form, e.g. silph::Silph + ); + + -- M13 — RTTICompleteObjectLocator. One per emitted vftable; the + -- `subobject_offset` column separates a class's primary vftable (0) + -- from the extra vftables it emits for secondary base subobjects. + CREATE TABLE rtti_locators ( + address BIGINT PRIMARY KEY, + subobject_offset BIGINT NOT NULL, -- this-offset of the subobject this vftable serves + cd_offset BIGINT NOT NULL, -- constructor-displacement offset + type_descriptor BIGINT NOT NULL, -- FK to rtti_type_descriptors.address + class_hierarchy BIGINT NOT NULL, -- VA of the RTTIClassHierarchyDescriptor + vtable_address BIGINT -- VA of vftable[0]; NULL if no referencing word was found + ); + + -- M13 — linearised RTTIBaseClassArray. Index 0 is the class itself; + -- the remaining rows are its bases in MSVC's depth-first order, + -- each with the PMD displacement triple needed to locate the base + -- subobject inside an instance. + CREATE TABLE rtti_base_classes ( + class_hierarchy BIGINT NOT NULL, -- VA of the deriving class's hierarchy descriptor + base_index BIGINT NOT NULL, -- position in the base-class array + type_descriptor BIGINT NOT NULL, + name VARCHAR NOT NULL, + num_contained_bases BIGINT NOT NULL, + mdisp BIGINT NOT NULL, -- member displacement + pdisp BIGINT NOT NULL, -- vbtable displacement (-1 = non-virtual base) + vdisp BIGINT NOT NULL, -- displacement inside the vbtable + attributes BIGINT NOT NULL, + PRIMARY KEY (class_hierarchy, base_index) + ); + CREATE TABLE xrefs ( source BIGINT NOT NULL, -- VA of the referencing instruction target BIGINT NOT NULL, -- VA of the referenced destination @@ -499,6 +664,20 @@ impl DbWriter { ); ")?; + // Every table above a few thousand rows goes through the DuckDB + // Appender rather than a row-at-a-time `INSERT`. + // + // This is not a micro-optimisation. DuckDB autocommits each statement, + // so a per-row `INSERT` loop pays a transaction + WAL flush per row: + // the 221k rows across `functions` / `labels` / `pdata_entries` alone + // took 20 minutes, and the 1.8M `indirect_dispatch_candidates` rows + // took ~59 more — 81 minutes for one database. Wrapping the lot in a + // single explicit transaction fixes the time but not the cause: DuckDB + // buffers per-statement, so the uncommitted set grew to ~16 GB RSS. + // The Appender writes directly to columnar storage in bounded chunks, + // which is both fast and flat in memory. It bypasses the SQL layer, + // so `ON CONFLICT DO NOTHING` is unavailable and each converted sink + // documents why its key cannot collide (or dedupes explicitly). insert_functions(&self.conn, func_analysis, labels)?; insert_pdata_entries(&self.conn, &func_analysis.pdata_entries)?; insert_labels(&self.conn, labels)?; @@ -507,10 +686,13 @@ impl DbWriter { insert_methods_and_classes(&self.conn, vtables, labels)?; insert_strings(&self.conn, strings)?; insert_funcptr_arrays(&self.conn, funcptr_arrays)?; + insert_eh_records(&self.conn, eh_records)?; + insert_jump_tables(&self.conn, jump_tables)?; + insert_rtti(&self.conn, rtti)?; + insert_xdbf(&self.conn, xdbf)?; if let Some(t) = typed_ind { insert_typed_ind_dispatch(&self.conn, t)?; } - insert_eh_records(&self.conn, eh_records)?; insert_xrefs_streaming(&self.conn, xrefs, pe, info.image_base, func_analysis, labels)?; let indices = [ @@ -560,8 +742,12 @@ impl DbWriter { labels: &HashMap, xrefs: &XrefMap, ) -> anyhow::Result<()> { - self.ingest_instructions(pe, info, func_analysis, labels)?; - self.write_analysis_results(pe, info, func_analysis, labels, xrefs, &[], &[], &[], None, &[])?; + let empty = std::collections::BTreeSet::new(); + self.ingest_instructions(pe, info, func_analysis, labels, &empty)?; + self.write_analysis_results( + pe, info, func_analysis, labels, xrefs, + &[], &[], &[], None, &[], &[], &crate::rtti::RttiResult::default(), None, + )?; Ok(()) } @@ -838,21 +1024,115 @@ pub fn write_db( // ── Helpers ──────────────────────────────────────────────────────────────── fn insert_metadata(conn: &Connection, info: &DisasmInfo) -> anyhow::Result<()> { - let mut stmt = conn.prepare("INSERT INTO metadata (key, value) VALUES (?, ?)")?; - stmt.execute(params!["image_base", format!("0x{:08X}", info.image_base)])?; - stmt.execute(params!["entry_point", format!("0x{:08X}", info.entry_point)])?; + let mut stmt = conn.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)")?; + let mut put = |k: &str, v: String| -> anyhow::Result<()> { + stmt.execute(params![k, v])?; + Ok(()) + }; + + put("image_base", format!("0x{:08X}", info.image_base))?; + put("entry_point", format!("0x{:08X}", info.entry_point))?; if let Some(name) = info.original_pe_name { - stmt.execute(params!["original_pe_name", name])?; + put("original_pe_name", name.to_string())?; } if let Some(title_id) = info.title_id { - stmt.execute(params!["title_id", format!("0x{:08X}", title_id)])?; + put("title_id", format!("0x{:08X}", title_id))?; } if let Some(media_id) = info.media_id { - stmt.execute(params!["media_id", format!("0x{:08X}", media_id)])?; + put("media_id", format!("0x{:08X}", media_id))?; } + + // Section geometry is useful enough on its own to be worth denormalising: + // a query that just wants "how big is the code" should not have to join. + let code_bytes: u64 = info.sections.iter().filter(|s| s.is_code()) + .map(|s| s.virtual_size as u64).sum(); + put("section_count", info.sections.len().to_string())?; + put("code_bytes", code_bytes.to_string())?; + + let Some(header) = info.xex_header else { return Ok(()) }; + + put("xex_module_flags", format!("0x{:08X}", header.module_flags))?; + put("xex_module_flags_decoded", decode_module_flags(header.module_flags))?; + put("xex_header_count", header.header_count.to_string())?; + + if let Some(sec) = &header.security_info { + put("image_size", format!("0x{:08X}", sec.image_size))?; + put("load_address", format!("0x{:08X}", sec.load_address))?; + put("image_flags", format!("0x{:08X}", sec.image_flags))?; + put("page_descriptor_count", sec.page_descriptors.len().to_string())?; + if sec.export_table_address != 0 { + put("export_table_address", format!("0x{:08X}", sec.export_table_address))?; + } + } + + if let Some(ff) = &header.file_format_info { + put("encryption_type", match ff.encryption_type { + 0 => "none".into(), + 1 => "normal (AES-128-CBC)".into(), + n => format!("unknown ({n})"), + })?; + put("compression_type", match ff.compression_type { + 0 => "none".into(), + 1 => "basic (raw + zero-fill blocks)".into(), + 2 => "normal (LZX)".into(), + n => format!("unknown ({n})"), + })?; + if ff.compression_type == 1 { + put("basic_block_count", ff.basic_blocks.len().to_string())?; + } + if ff.compression_type == 2 { + put("lzx_window_size", format!("0x{:08X}", ff.normal_window_size))?; + } + } + + if let Some(exec) = &header.execution_info { + put("disc_number", exec.disc_number.to_string())?; + put("disc_count", exec.disc_count.to_string())?; + } + + // Import libraries carry the SDK version each module was linked against — + // the single most useful "what toolchain built this" signal in the header. + put("import_library_count", header.import_libraries.len().to_string())?; + for lib in &header.import_libraries { + put(&format!("import_lib.{}.version_min", lib.name), format_xex_version(lib.version_min))?; + put(&format!("import_lib.{}.version_cur", lib.name), format_xex_version(lib.version_cur))?; + put(&format!("import_lib.{}.imports", lib.name), lib.imports.len().to_string())?; + } + + // Any optional header we do not model explicitly is still recorded by key, + // so nothing in the XEX is silently dropped. + for oh in &header.optional_headers { + put(&format!("xex_optional_header.0x{:08X}", oh.key), format!("0x{:08X}", oh.value))?; + } + Ok(()) } +/// Render a XEX version word (`major.minor.build.qfe`, 4/4/16/8 bits). +fn format_xex_version(v: u32) -> String { + let major = (v >> 28) & 0xF; + let minor = (v >> 24) & 0xF; + let build = (v >> 8) & 0xFFFF; + let qfe = v & 0xFF; + format!("{major}.{minor}.{build}.{qfe}") +} + +/// Human-readable form of the XEX2 module flags bit field. +fn decode_module_flags(flags: u32) -> String { + const NAMES: &[(u32, &str)] = &[ + (0x0000_0001, "title_module"), + (0x0000_0002, "exports_to_title"), + (0x0000_0004, "system_debugger"), + (0x0000_0008, "dll_module"), + (0x0000_0010, "module_patch"), + (0x0000_0020, "patch_full"), + (0x0000_0040, "patch_delta"), + (0x0000_0080, "user_mode"), + ]; + let set: Vec<&str> = NAMES.iter().filter(|&&(b, _)| flags & b != 0).map(|&(_, n)| n).collect(); + if set.is_empty() { "none".to_string() } else { set.join("|") } +} + fn insert_sections(conn: &Connection, sections: &[xenia_xex::pe::PeSection]) -> anyhow::Result<()> { let mut stmt = conn.prepare( "INSERT INTO sections (name, virtual_address, virtual_size, raw_offset, raw_size, flags, is_code) @@ -897,17 +1177,12 @@ fn insert_functions( func_analysis: &FuncAnalysis, labels: &HashMap, ) -> anyhow::Result<()> { - let mut stmt = conn.prepare( - "INSERT INTO functions - (address, name, end_address, frame_size, saved_gprs, is_leaf, is_saverestore, - pdata_validated, pdata_length, has_eh) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" - )?; + let mut appender = conn.appender("functions")?; for (&addr, fi) in &func_analysis.functions { let name = labels.get(&addr) .cloned() .unwrap_or_else(|| format!("sub_{addr:08X}")); - stmt.execute(params![ + appender.append_row(params![ addr as i64, name, fi.end as i64, @@ -917,9 +1192,11 @@ fn insert_functions( fi.is_saverestore, fi.pdata_validated, fi.pdata_length.map(|n| n as i64), + fi.pdata_prolog_length.map(|n| n as i64), fi.has_eh, ])?; } + appender.flush()?; Ok(()) } @@ -960,17 +1237,13 @@ fn insert_methods_and_classes( ) -> anyhow::Result<()> { if vtables.is_empty() { return Ok(()); } - // methods rows + // methods rows — keyed by (vtable_address, slot), which `methods_table` + // emits at most once each. let methods = crate::vtables::methods_table(vtables, labels); if !methods.is_empty() { - let mut stmt = conn.prepare( - "INSERT INTO methods - (vtable_address, slot, function_address, mangled_name, demangled_name) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT DO NOTHING" - )?; + let mut appender = conn.appender("methods")?; for (vt_addr, slot, fn_addr, mangled, demangled) in &methods { - stmt.execute(params![ + appender.append_row(params![ *vt_addr as i64, *slot as i64, *fn_addr as i64, @@ -978,6 +1251,7 @@ fn insert_methods_and_classes( demangled.as_deref(), ])?; } + appender.flush()?; metrics::counter!("db.rows", "table" => "methods").increment(methods.len() as u64); tracing::info!(rows = methods.len(), table = "methods", "bulk insert complete"); } @@ -1011,20 +1285,25 @@ fn insert_strings( strings: &[crate::strings::DetectedString], ) -> anyhow::Result<()> { if strings.is_empty() { return Ok(()); } - let mut stmt = conn.prepare( - "INSERT INTO strings (address, encoding, length, content) VALUES (?, ?, ?, ?) - ON CONFLICT DO NOTHING" - )?; + // The ascii / shift_jis / utf8 scans all run over the same bytes, so two + // of them can report a string at the same address. `address` is the + // primary key and the Appender cannot absorb that the way + // `ON CONFLICT DO NOTHING` did — keep the first detection per address. + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let mut appender = conn.appender("strings")?; let mut count = 0u64; for s in strings { - stmt.execute(params![ + if !seen.insert(s.address) { continue; } + appender.append_row(params![ s.address as i64, s.encoding, s.length as i64, s.content.as_str(), + s.section.as_str(), ])?; count += 1; } + appender.flush()?; metrics::counter!("db.rows", "table" => "strings").increment(count); tracing::info!(rows = count, table = "strings", "bulk insert complete"); Ok(()) @@ -1043,23 +1322,10 @@ fn insert_eh_records( VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING" )?; - let mut stmt_unwind = conn.prepare( - "INSERT INTO eh_unwind_map - (funcinfo_address, state_index, to_state, action_pc) - VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING" - )?; - let mut stmt_try = conn.prepare( - "INSERT INTO eh_try_blocks - (funcinfo_address, try_index, try_low, try_high, catch_high, - n_catches, p_handler_array) - VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT DO NOTHING" - )?; let mut n_fi = 0u64; - let mut n_unwind = 0u64; - let mut n_try = 0u64; + let mut kept: Vec<&crate::eh_scope::EhFuncInfo> = Vec::with_capacity(records.len()); for r in records { - stmt_fi.execute(params![ + let inserted = stmt_fi.execute(params![ r.address as i64, r.magic as i64, r.max_state as i64, r.p_unwind_map as i64, r.n_try_blocks as i64, r.p_try_block_map as i64, r.n_ip_map_entries as i64, @@ -1067,22 +1333,45 @@ fn insert_eh_records( r.p_es_type_list.map(|p| p as i64), r.eh_flags.map(|f| f as i64), ])?; - n_fi += 1; - for (i, e) in r.unwind_map.iter().enumerate() { - stmt_unwind.execute(params![ - r.address as i64, i as i64, e.to_state as i64, e.action_pc as i64, - ])?; - n_unwind += 1; - } - for (i, t) in r.try_blocks.iter().enumerate() { - stmt_try.execute(params![ - r.address as i64, i as i64, - t.try_low as i64, t.try_high as i64, t.catch_high as i64, - t.n_catches as i64, t.p_handler_array as i64, - ])?; - n_try += 1; + if inserted > 0 { + n_fi += 1; + kept.push(r); } } + drop(stmt_fi); + + // The child rows are keyed by (funcinfo_address, index) and each parent + // survived the ON CONFLICT above, so these cannot collide. + let mut n_unwind = 0u64; + { + let mut appender = conn.appender("eh_unwind_map")?; + for r in &kept { + for (i, e) in r.unwind_map.iter().enumerate() { + appender.append_row(params![ + r.address as i64, i as i64, e.to_state as i64, e.action_pc as i64, + ])?; + n_unwind += 1; + } + } + appender.flush()?; + } + + let mut n_try = 0u64; + { + let mut appender = conn.appender("eh_try_blocks")?; + for r in &kept { + for (i, t) in r.try_blocks.iter().enumerate() { + appender.append_row(params![ + r.address as i64, i as i64, + t.try_low as i64, t.try_high as i64, t.catch_high as i64, + t.n_catches as i64, t.p_handler_array as i64, + ])?; + n_try += 1; + } + } + appender.flush()?; + } + metrics::counter!("db.rows", "table" => "eh_funcinfo").increment(n_fi); metrics::counter!("db.rows", "table" => "eh_unwind_map").increment(n_unwind); metrics::counter!("db.rows", "table" => "eh_try_blocks").increment(n_try); @@ -1100,31 +1389,48 @@ fn insert_typed_ind_dispatch( if !t.dispatches.is_empty() { let mut stmt_site = conn.prepare( "INSERT INTO indirect_dispatch_sites - (dispatch_pc, vptr_offset, slot, candidate_count) - VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING" - )?; - let mut stmt_cand = conn.prepare( - "INSERT INTO indirect_dispatch_candidates - (dispatch_pc, vtable_address, method_address) - VALUES (?, ?, ?) ON CONFLICT DO NOTHING" + (dispatch_pc, vptr_offset, slot, candidate_count, truncated) + VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING" )?; let mut n_sites = 0u64; - let mut n_cand = 0u64; for d in &t.dispatches { stmt_site.execute(params![ d.dispatch_pc as i64, d.vptr_offset as i64, d.slot as i64, - d.candidate_vtables.len() as i64, + d.total_candidates as i64, + d.truncated, ])?; n_sites += 1; + } + drop(stmt_site); + + // `indirect_dispatch_candidates` used to be by far the largest table + // this writer produced — 1.8M rows before unresolved sites stopped + // materialising their cross product (see + // `ind_dispatch_typed::analyze`). It still goes through the Appender: + // the ceiling is configurable and a caller that raises it gets the + // volume back. + // + // The Appender bypasses the SQL layer, which means `ON CONFLICT DO + // NOTHING` is not available to absorb duplicates and a repeated + // `(dispatch_pc, vtable_address)` would violate the primary key at + // flush. Dedupe up front instead. + let mut appender = conn.appender("indirect_dispatch_candidates")?; + let mut seen: std::collections::HashSet<(u32, u32)> = std::collections::HashSet::new(); + let mut n_cand = 0u64; + for d in &t.dispatches { for (vt, m) in d.candidate_vtables.iter().zip(d.method_pcs.iter()) { - stmt_cand.execute(params![ - d.dispatch_pc as i64, *vt as i64, *m as i64, - ])?; - n_cand += 1; + if seen.insert((d.dispatch_pc, *vt)) { + appender.append_row(params![ + d.dispatch_pc as i64, *vt as i64, *m as i64, + ])?; + n_cand += 1; + } } } + appender.flush()?; + metrics::counter!("db.rows", "table" => "indirect_dispatch_sites").increment(n_sites); metrics::counter!("db.rows", "table" => "indirect_dispatch_candidates").increment(n_cand); tracing::info!(sites = n_sites, candidates = n_cand, "typed indirect-dispatch insert complete"); @@ -1151,31 +1457,158 @@ fn insert_typed_ind_dispatch( Ok(()) } +/// Write the XDBF package tables. Achievement names are resolved through the +/// package's own default-language string table (`XSTC`), falling back to +/// English and then to whatever table exists, so the text columns are populated +/// even for a title that ships no `XSTC`. +fn insert_xdbf(conn: &Connection, xdbf: Option<&crate::xdbf::Xdbf>) -> anyhow::Result<()> { + let Some(x) = xdbf else { return Ok(()) }; + + let mut stmt = conn.prepare( + "INSERT INTO xdbf_entries (namespace, namespace_name, id, body_offset, size, magic) + VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING" + )?; + for e in &x.entries { + let ns_name = match e.namespace { + 1 => "metadata", + 2 => "image", + 3 => "string_table", + _ => "unknown", + }; + stmt.execute(params![ + e.namespace as i64, ns_name, e.id as i64, + e.offset as i64, e.size as i64, e.magic.as_deref(), + ])?; + } + drop(stmt); + + let mut stmt = conn.prepare( + "INSERT INTO xdbf_strings (language, language_name, string_id, value) + VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING" + )?; + let mut n_strings = 0u64; + for t in &x.string_tables { + let name = crate::xdbf::language_name(t.language); + for (id, v) in &t.strings { + stmt.execute(params![t.language as i64, name, *id as i64, v.as_str()])?; + n_strings += 1; + } + } + drop(stmt); + + // Pick the table used to resolve achievement text. + let preferred = x.default_language.unwrap_or(1); + let lookup = x + .string_tables + .iter() + .find(|t| t.language == preferred) + .or_else(|| x.string_tables.iter().find(|t| t.language == 1)) + .or_else(|| x.string_tables.first()); + let text = |id: u16| -> Option { + lookup? + .strings + .iter() + .find(|(sid, _)| *sid == id) + .map(|(_, s)| s.clone()) + }; + + let mut stmt = conn.prepare( + "INSERT INTO xdbf_achievements + (id, name, unlocked_desc, locked_desc, label_id, description_id, + unachieved_id, image_id, gamerscore, flags) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING" + )?; + for a in &x.achievements { + stmt.execute(params![ + a.id as i64, + text(a.label_id), + text(a.description_id), + text(a.unachieved_id), + a.label_id as i64, a.description_id as i64, a.unachieved_id as i64, + a.image_id as i64, a.gamerscore as i64, a.flags as i64, + ])?; + } + drop(stmt); + + let mut stmt = conn.prepare( + "INSERT INTO xdbf_images (id, is_title_icon, body_offset, size, format) + VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING" + )?; + for i in &x.images { + stmt.execute(params![ + i.id as i64, + i.id == crate::xdbf::ID_TITLE, + i.offset as i64, + i.size as i64, + i.format, + ])?; + } + drop(stmt); + + let mut meta = conn.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)")?; + meta.execute(params!["xdbf.entry_count", x.entries.len().to_string()])?; + if let Some(l) = x.default_language { + meta.execute(params!["xdbf.default_language", crate::xdbf::language_name(l)])?; + } + if let Some(t) = x.title { + meta.execute(params!["xdbf.title_id", format!("{:#010X}", t.title_id)])?; + meta.execute(params![ + "xdbf.title_version", + format!("{}.{}.{}.{}", t.major, t.minor, t.build, t.revision) + ])?; + } + // The title's own name lives at the well-known string id 0x8000, in the + // package's default language. + if let Some(name) = text(crate::xdbf::ID_TITLE as u16) { + meta.execute(params!["xdbf.title_name", name])?; + } + + metrics::counter!("db.rows", "table" => "xdbf_strings").increment(n_strings); + tracing::info!( + entries = x.entries.len(), + achievements = x.achievements.len(), + strings = n_strings, + images = x.images.len(), + "XDBF tables written", + ); + Ok(()) +} + fn insert_funcptr_arrays( conn: &Connection, arrays: &[crate::funcptr_arrays::FuncPtrArray], ) -> anyhow::Result<()> { if arrays.is_empty() { return Ok(()); } + // Parents first (few, and `ON CONFLICT` decides which survive), then the + // entry rows in one Appender pass — only entries of a parent that was + // actually inserted. let mut stmt_arr = conn.prepare( "INSERT INTO function_pointer_arrays (address, length, kind) VALUES (?, ?, ?) ON CONFLICT DO NOTHING" )?; - let mut stmt_ent = conn.prepare( - "INSERT INTO function_pointer_array_entries (array_address, slot, function_address) - VALUES (?, ?, ?) ON CONFLICT DO NOTHING" - )?; let mut n_arr = 0u64; - let mut n_ent = 0u64; + let mut kept: Vec<&crate::funcptr_arrays::FuncPtrArray> = Vec::with_capacity(arrays.len()); for a in arrays { let inserted = stmt_arr.execute(params![ a.address as i64, a.length as i64, a.kind, ])?; - if inserted > 0 { n_arr += 1; } + if inserted > 0 { + n_arr += 1; + kept.push(a); + } + } + drop(stmt_arr); + + let mut appender = conn.appender("function_pointer_array_entries")?; + let mut n_ent = 0u64; + for a in kept { for (i, &fn_va) in a.entries.iter().enumerate() { - stmt_ent.execute(params![a.address as i64, i as i64, fn_va as i64])?; + appender.append_row(params![a.address as i64, i as i64, fn_va as i64])?; n_ent += 1; } } + appender.flush()?; + metrics::counter!("db.rows", "table" => "function_pointer_arrays").increment(n_arr); metrics::counter!("db.rows", "table" => "function_pointer_array_entries").increment(n_ent); tracing::info!(arrays = n_arr, entries = n_ent, "function-pointer arrays insert complete"); @@ -1252,14 +1685,11 @@ fn insert_pdata_entries( if entries.is_empty() { return Ok(()); } - let mut stmt = conn.prepare( - "INSERT INTO pdata_entries - (begin_address, end_address, function_length, prolog_length, flags) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT DO NOTHING" - )?; + // `parse_pdata` already guarantees strictly ascending, unique + // `begin_address` values, so the primary key cannot collide. + let mut appender = conn.appender("pdata_entries")?; for e in entries { - stmt.execute(params![ + appender.append_row(params![ e.begin_address as i64, e.end_address() as i64, e.function_length as i64, @@ -1267,6 +1697,7 @@ fn insert_pdata_entries( e.flags as i64, ])?; } + appender.flush()?; Ok(()) } @@ -1274,9 +1705,9 @@ fn insert_labels( conn: &Connection, labels: &HashMap, ) -> anyhow::Result<()> { - let mut stmt = conn.prepare( - "INSERT INTO labels (address, name, kind) VALUES (?, ?, ?) ON CONFLICT DO NOTHING" - )?; + // `labels` is keyed by address, so it cannot contain a duplicate primary + // key — the Appender is safe here without a dedupe pass. + let mut appender = conn.appender("labels")?; for (&addr, name) in labels { let kind = if name.starts_with("sub_") || name == "entry_point" { "function" @@ -1291,8 +1722,9 @@ fn insert_labels( } else { "other" }; - stmt.execute(params![addr as i64, name, kind])?; + appender.append_row(params![addr as i64, name, kind])?; } + appender.flush()?; Ok(()) } @@ -1302,6 +1734,7 @@ fn insert_instructions_streaming( info: &DisasmInfo, func_analysis: &FuncAnalysis, labels: &HashMap, + data_words: &std::collections::BTreeSet, ) -> anyhow::Result<()> { let mut appender = conn.appender("instructions")?; let mut total: u64 = 0; @@ -1312,6 +1745,7 @@ fn insert_instructions_streaming( let va_end = info.image_base + section.virtual_address + section.virtual_size; let items = crate::disasm::enrich_section( pe, info.image_base, §ion.name, va_start, va_end, func_analysis, labels, + data_words, ); total += crate::sinks::duckdb::append_instructions(&mut appender, items)?; } @@ -1322,6 +1756,144 @@ fn insert_instructions_streaming( Ok(()) } +/// Write the M12 jump-table tables plus the data-in-code extent list. +fn insert_jump_tables( + conn: &Connection, + tables: &[crate::jumptables::JumpTable], +) -> anyhow::Result<()> { + let mut t = conn.prepare( + "INSERT INTO jump_tables + (bctr_pc, function, table_address, entry_count, table_slots, + index_map_address, index_map_count, case_bound, kind) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" + )?; + for jt in tables { + t.execute(params![ + jt.bctr_pc as i64, + jt.function.map(|f| f as i64), + jt.table_address as i64, + jt.entry_count as i64, + jt.table_slots as i64, + jt.index_map_address.map(|a| a as i64), + jt.index_map_count.map(|n| n as i64), + jt.bound.map(|n| n as i64), + jt.kind, + ])?; + } + drop(t); + + // Keyed by (bctr_pc, case_index); each bctr_pc yields one table. + let mut e = conn.appender("jump_table_entries")?; + for jt in tables { + for (i, &target) in jt.targets.iter().enumerate() { + e.append_row(params![jt.bctr_pc as i64, i as i64, target as i64])?; + } + } + e.flush()?; + drop(e); + + // `data_in_code` is keyed by address, so the per-table extents are merged + // first — two switches in the same function can share one table. + let mut kinds: HashMap = HashMap::new(); + for jt in tables { + kinds.insert(jt.table_address, "jump_table"); + if let Some(a) = jt.index_map_address { + kinds.insert(a, "jump_index_map"); + } + } + let mut d = conn.prepare("INSERT INTO data_in_code (address, length, kind) VALUES (?, ?, ?)")?; + for (addr, len) in crate::jumptables::data_regions(tables) { + let kind = kinds.get(&addr).copied().unwrap_or("jump_table"); + d.execute(params![addr as i64, len as i64, kind])?; + } + + metrics::counter!("db.rows", "table" => "jump_tables").increment(tables.len() as u64); + tracing::info!( + rows = tables.len(), + entries = tables.iter().map(|t| t.targets.len()).sum::(), + table = "jump_tables", + "insert complete", + ); + Ok(()) +} + +/// Write the M13 RTTI tables. +fn insert_rtti(conn: &Connection, rtti: &crate::rtti::RttiResult) -> anyhow::Result<()> { + let mut td = conn.prepare( + "INSERT INTO rtti_type_descriptors (address, mangled_name, demangled_name) + VALUES (?, ?, ?)" + )?; + // RTTI descriptors are the only mangled names a stripped retail binary + // still carries, so they are also the only thing `demangled_names` can be + // populated from — without this it stays empty on every shipped title. + let mut dn = conn.prepare( + "INSERT INTO demangled_names + (address, mangled, raw_demangled, namespace_path, class_name, + method_name, params_signature) + VALUES (?, ?, ?, ?, ?, ?, ?)" + )?; + for t in &rtti.type_descriptors { + td.execute(params![t.address as i64, t.mangled_name, t.demangled_name])?; + let (ns, cls) = match t.demangled_name.rfind("::") { + Some(i) => (Some(&t.demangled_name[..i]), &t.demangled_name[i + 2..]), + None => (None, t.demangled_name.as_str()), + }; + dn.execute(params![ + t.address as i64, + t.mangled_name, + t.demangled_name, + ns, + cls, + Option::<&str>::None, + Option::<&str>::None, + ])?; + } + + let mut col = conn.prepare( + "INSERT INTO rtti_locators + (address, subobject_offset, cd_offset, type_descriptor, class_hierarchy, vtable_address) + VALUES (?, ?, ?, ?, ?, ?)" + )?; + for c in &rtti.locators { + col.execute(params![ + c.address as i64, + c.offset as i64, + c.cd_offset as i64, + c.type_descriptor as i64, + c.class_hierarchy as i64, + c.vtable_address.map(|v| v as i64), + ])?; + } + + let mut bc = conn.prepare( + "INSERT INTO rtti_base_classes + (class_hierarchy, base_index, type_descriptor, name, num_contained_bases, + mdisp, pdisp, vdisp, attributes) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" + )?; + for b in &rtti.base_classes { + bc.execute(params![ + b.class_hierarchy as i64, + b.index as i64, + b.type_descriptor as i64, + b.name, + b.num_contained_bases as i64, + b.mdisp as i64, + b.pdisp as i64, + b.vdisp as i64, + b.attributes as i64, + ])?; + } + + tracing::info!( + type_descriptors = rtti.type_descriptors.len(), + locators = rtti.locators.len(), + base_classes = rtti.base_classes.len(), + "RTTI tables written", + ); + Ok(()) +} + fn insert_xrefs_streaming( conn: &Connection, xrefs: &XrefMap, diff --git a/crates/xenia-analysis/src/demangle.rs b/crates/xenia-analysis/src/demangle.rs index a9b955c..e2c332a 100644 --- a/crates/xenia-analysis/src/demangle.rs +++ b/crates/xenia-analysis/src/demangle.rs @@ -275,3 +275,102 @@ mod tests { assert_eq!(parts, vec!["a", "b", "e"]); } } + +// ── RTTI type-descriptor names ───────────────────────────────────────────── + +/// Demangle an RTTI `TypeDescriptor` decorated name into a readable class path. +/// +/// These are not ordinary symbols: they are *type* encodings prefixed with a +/// literal `.`, e.g. `.?AVSilph@silph@@` → `silph::Silph`, +/// `.?AUGAME_PART_PARAM@silph@@` → `silph::GAME_PART_PARAM`. +/// +/// A bare descriptor name is not a symbol the demangler accepts, and feeding it +/// one anyway silently mis-parses (`?AVSilph@silph@@` demangles to +/// `silph::AVSilph`, keeping the `AV` type tag as part of the class name). The +/// correct move is to rebuild the symbol MSVC would have emitted for this +/// descriptor — `??_R0@8` — demangle *that*, and strip the +/// ``::`RTTI Type Descriptor' `` suffix and the leading type keyword. That path +/// is the only one that renders template arguments properly +/// (`.?AV?$vector@H@std@@` → `std::vector`). +/// +/// If the demangler still declines, the decorated name is decoded directly: +/// strip the `.?A[VU]` tag, split the remainder on `@`, and re-join the +/// components in reverse (MSVC emits innermost scope first). The +/// anonymous-namespace component `?A0x` becomes `(anonymous namespace)`. +/// +/// Returns `None` only when the input is not a type descriptor at all. +pub fn demangle_type_descriptor(decorated: &str) -> Option { + let body = decorated.strip_prefix('.')?; + if !(body.starts_with("?AV") || body.starts_with("?AU") || body.starts_with("?AW")) { + return None; + } + + const RTTI_SUFFIX: &str = "::`RTTI Type Descriptor'"; + if let Ok(full) = msvc_demangler::demangle(&format!("??_R0{body}@8"), DemangleFlags::llvm()) + && let Some(qualified) = full.trim().strip_suffix(RTTI_SUFFIX) + { + let name = qualified + .trim_start_matches("class ") + .trim_start_matches("struct ") + .trim_start_matches("enum ") + .trim_start_matches("union ") + .trim(); + if !name.is_empty() { + return Some(name.to_string()); + } + } + + let inner = body[3..].trim_end_matches('@'); + let mut parts: Vec = inner + .split('@') + .filter(|p| !p.is_empty()) + .map(|p| { + if p.starts_with("?A0x") { + "(anonymous namespace)".to_string() + } else { + p.to_string() + } + }) + .collect(); + if parts.is_empty() { + return None; + } + parts.reverse(); + Some(parts.join("::")) +} + +#[cfg(test)] +mod rtti_name_tests { + use super::demangle_type_descriptor; + + #[test] + fn plain_class_in_namespace() { + assert_eq!(demangle_type_descriptor(".?AVSilph@silph@@").as_deref(), Some("silph::Silph")); + } + + #[test] + fn struct_tag() { + assert_eq!( + demangle_type_descriptor(".?AUGAME_PART_PARAM@silph@@").as_deref(), + Some("silph::GAME_PART_PARAM"), + ); + } + + #[test] + fn global_scope_class() { + assert_eq!(demangle_type_descriptor(".?AVexception@std@@").as_deref(), Some("std::exception")); + } + + #[test] + fn anonymous_namespace_is_named() { + let got = demangle_type_descriptor(".?AVAct_Stop@?A0x5cc05762@unnamed_namespaces@@").unwrap(); + assert!(got.ends_with("Act_Stop"), "got {got}"); + assert!(got.starts_with("unnamed_namespaces"), "got {got}"); + } + + #[test] + fn rejects_non_descriptors() { + assert_eq!(demangle_type_descriptor("?Foo@@QAEXXZ"), None); + assert_eq!(demangle_type_descriptor("plain_name"), None); + } +} diff --git a/crates/xenia-analysis/src/disasm.rs b/crates/xenia-analysis/src/disasm.rs index d1a2761..d53f972 100644 --- a/crates/xenia-analysis/src/disasm.rs +++ b/crates/xenia-analysis/src/disasm.rs @@ -5,7 +5,7 @@ //! label name. The three sinks in [`crate::sinks`] (text, JSON, DuckDB) all //! consume `RichDisasmItem`. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use xenia_cpu::disasm::DisasmItem; @@ -18,14 +18,30 @@ pub struct RichDisasmItem<'a> { pub section: &'a str, pub function: Option, pub label: Option<&'a str>, + /// True when this word is data embedded in a code section (a recovered + /// jump table or its index map), so its decoded text is meaningless. + pub is_data: bool, } /// Walk one code section, yielding rich items annotated with section name, -/// rolling-window enclosing function, and label-at-address. +/// enclosing function, and label-at-address. /// -/// The `function` field tracks the most recent function-start the iterator -/// has crossed — matching the legacy `current_func` behaviour in -/// `db.rs::insert_instructions_streaming`. +/// `function` is the function that actually *contains* the address: it is set +/// on crossing a function start and cleared again at that function's +/// `end_address`. It is deliberately `None` in the gaps between functions. +/// +/// It used to be a pure rolling window — set at each start and never cleared — +/// which silently attributed every gap word to whichever function happened to +/// precede it. On the reference title that mislabelled 55,227 instructions, +/// so `WHERE function = X` returned code that is not part of X, and the +/// resulting 100% attribution rate hid the fact that `.pdata` leaves ~450 KB +/// of `.text` unclaimed. +/// +/// `data_words` is the set of 4-byte-aligned addresses inside code sections +/// that are known to hold data (see [`crate::jumptables::data_word_addresses`]). +/// Rows at those addresses are still emitted — their `raw` value is the table +/// entry a consumer wants — but flagged so nothing mistakes the decoded text +/// for a real instruction. pub fn enrich_section<'a>( image: &'a [u8], image_base: u32, @@ -34,18 +50,105 @@ pub fn enrich_section<'a>( va_end: u32, func_analysis: &'a FuncAnalysis, labels: &'a HashMap, + data_words: &'a BTreeSet, ) -> impl Iterator> + 'a { - let mut current_func: Option = None; + // (start, end) of the function currently being walked. + let mut current: Option<(u32, u32)> = None; xenia_cpu::disasm::iter_disasm(image, image_base, va_start, va_end).map(move |item| { - if func_analysis.is_function_start(item.addr) { - current_func = Some(item.addr); + // Leaving the current function must be handled before entering the + // next: a function often starts exactly at its predecessor's end. + if let Some((_, end)) = current + && item.addr >= end + { + current = None; } + if let Some(fi) = func_analysis.functions.get(&item.addr) { + current = Some((item.addr, fi.end)); + } + let current_func = current.map(|(start, _)| start); let label = labels.get(&item.addr).map(|s| s.as_str()); + let is_data = data_words.contains(&item.addr); RichDisasmItem { item, section: section_name, function: current_func, label, + is_data, } }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::func::{FuncAnalysis, FuncInfo}; + use std::collections::BTreeMap; + + fn fi(start: u32, end: u32) -> FuncInfo { + FuncInfo { + start, end, + frame_size: 0, saved_gprs: 0, is_leaf: true, is_saverestore: false, + pdata_validated: true, pdata_length: Some(end - start), + pdata_prolog_length: None, has_eh: false, + } + } + + /// A word in the gap between two functions belongs to neither. Before the + /// containment check this walker carried the *preceding* function forward + /// across the gap, so `WHERE function = X` returned code outside X. + #[test] + fn gap_between_functions_is_unattributed() { + let image_base = 0x82000000u32; + // 6 words: [f0 f0] [gap gap] [f1 f1] + let image = vec![0x60u8; 0x40]; // `ori` — decodes cleanly, value irrelevant + let mut functions = BTreeMap::new(); + functions.insert(image_base, fi(image_base, image_base + 8)); + functions.insert(image_base + 16, fi(image_base + 16, image_base + 24)); + let fa = FuncAnalysis { + functions, + save_gpr_base: None, + restore_gpr_base: None, + pdata_entries: Vec::new(), + }; + let labels = HashMap::new(); + let data_words = BTreeSet::new(); + let got: Vec<(u32, Option)> = enrich_section( + &image, image_base, ".text", image_base, image_base + 24, + &fa, &labels, &data_words, + ).map(|r| (r.item.addr, r.function)).collect(); + + assert_eq!(got, vec![ + (image_base, Some(image_base)), // inside f0 + (image_base + 4, Some(image_base)), // inside f0 + (image_base + 8, None), // gap — was wrongly f0 + (image_base + 12, None), // gap — was wrongly f0 + (image_base + 16, Some(image_base + 16)), // f1 starts + (image_base + 20, Some(image_base + 16)), + ]); + } + + /// A function starting exactly at its predecessor's `end_address` must be + /// entered, not dropped: the leave check runs before the enter check. + #[test] + fn adjacent_functions_hand_over_cleanly() { + let image_base = 0x82000000u32; + let image = vec![0x60u8; 0x40]; + let mut functions = BTreeMap::new(); + functions.insert(image_base, fi(image_base, image_base + 8)); + functions.insert(image_base + 8, fi(image_base + 8, image_base + 16)); + let fa = FuncAnalysis { + functions, save_gpr_base: None, restore_gpr_base: None, + pdata_entries: Vec::new(), + }; + let labels = HashMap::new(); + let data_words = BTreeSet::new(); + let got: Vec> = enrich_section( + &image, image_base, ".text", image_base, image_base + 16, + &fa, &labels, &data_words, + ).map(|r| r.function).collect(); + assert_eq!(got, vec![ + Some(image_base), Some(image_base), + Some(image_base + 8), Some(image_base + 8), + ]); + } +} diff --git a/crates/xenia-analysis/src/formatter.rs b/crates/xenia-analysis/src/formatter.rs index f0746ec..b851451 100644 --- a/crates/xenia-analysis/src/formatter.rs +++ b/crates/xenia-analysis/src/formatter.rs @@ -1,6 +1,6 @@ //! Assembly text output formatter for Xbox 360 disassembly. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::io::Write; use xenia_xex::header::ImportLibrary; @@ -20,6 +20,11 @@ pub struct DisasmInfo<'a> { pub media_id: Option, pub sections: &'a [PeSection], pub import_libraries: &'a [ImportLibrary], + /// Full parsed XEX2 header, when the caller loaded from a XEX/ISO. Drives + /// the extended `metadata` rows (module/system/image flags, image size, + /// compression + encryption, per-library versions, …). `None` when the + /// caller only had a bare PE. + pub xex_header: Option<&'a xenia_xex::header::Xex2Header>, } /// Write full disassembly to the output stream. @@ -32,6 +37,7 @@ pub fn write_asm( import_map: &HashMap, xrefs: &XrefMap, data_annotations: &HashMap, + data_words: &BTreeSet, ) -> anyhow::Result<()> { // Header writeln!(out, "; ============================================================================")?; @@ -95,6 +101,7 @@ pub fn write_asm( let items = enrich_section( pe, info.image_base, §ion.name, abs_start, abs_end, func_analysis, labels, + data_words, ); for ri in items { let abs_addr = ri.item.addr; diff --git a/crates/xenia-analysis/src/func.rs b/crates/xenia-analysis/src/func.rs index cdffdcc..b714c1a 100644 --- a/crates/xenia-analysis/src/func.rs +++ b/crates/xenia-analysis/src/func.rs @@ -39,6 +39,9 @@ pub struct FuncInfo { /// Function size in bytes per `.pdata`'s `function_length` field, if known. /// Absent (None) when this row is prologue-only. pub pdata_length: Option, + /// Prolog size in bytes per `.pdata`'s `prolog_length` field, if known. + /// The linker's own count — more reliable than the prologue pattern match. + pub pdata_prolog_length: Option, /// True when `.pdata`'s exception-flag bit is set on this entry — the /// function has a registered C++ EH (or SEH) frame handler. Always false /// for entries without `.pdata` coverage. (M9) @@ -220,6 +223,19 @@ pub fn analyze( /// - Does not edit the `prolog_length` we'd derive from prologue analysis; /// `frame_size` and `saved_gprs` remain best-effort prologue inferences. /// - Does not infer base/derived call edges — that's M3+M5. +/// - Does not discover functions that are neither in `.pdata` nor the target of +/// a `bl`. Some code does live in the `.pdata` gaps — small leaf helpers +/// reached only through a function-pointer table. Measured against a Ghidra +/// export of the reference title, 217 such entries exist that this pass does +/// not emit. Two obvious heuristics for them were evaluated and **rejected**: +/// "a data word that points into code outside any `.pdata` range" yields 1994 +/// new candidates of which Ghidra confirms 37, and "8-byte-aligned word in a +/// gap, preceded by `blr` + padding" yields 4011 of which Ghidra confirms +/// 106. Either would flood `functions` with several thousand unvalidated +/// rows and destroy the property that every emitted boundary is exact, in +/// exchange for a couple of hundred real ones. If this gap needs closing, it +/// wants a real recursive-descent walk seeded from the function-pointer +/// tables, not a pattern match. #[tracing::instrument(skip_all, fields(image_base = format_args!("{:#010x}", image_base), entry_point = format_args!("{:#010x}", entry_point), pdata_entries = pdata.len()))] pub fn analyze_with_pdata( pe: &[u8], @@ -280,10 +296,79 @@ pub fn analyze_with_pdata( call_targets.insert(e.begin_address); } } + + // Tail-call targets. + // + // `bl ∪ pdata` misses a function that is only ever entered by a tail call: + // it has no `bl` site, and small frameless helpers are frequently absent + // from `.pdata`. `0x82169630` in the reference title is one — it follows a + // `b 0x825F0FDC` that ends the previous function and is itself reached only + // by `b`, so nothing in the union nominates it. + // + // `.pdata` makes the test exact: a non-linking `b` whose target leaves the + // source's own linker-declared range, and that does not land inside any + // other declared range, is entering a *different* function — not branching + // within this one. Intra-function jumps and switch arms both stay inside + // the range and are therefore never nominated. + let pdata_sorted: Vec<(u32, u32)> = { + let mut v: Vec<(u32, u32)> = pdata.iter().map(|e| (e.begin_address, e.end_address())).collect(); + v.sort_unstable(); + v + }; + let containing = |addr: u32| -> Option<(u32, u32)> { + match pdata_sorted.binary_search_by_key(&addr, |&(b, _)| b) { + Ok(i) => Some(pdata_sorted[i]), + Err(0) => None, + Err(i) => { + let (b, e) = pdata_sorted[i - 1]; + (addr < e).then_some((b, e)) + } + } + }; + // + // `.pdata` does not cover the whole of `.text` here — roughly 450 KB of + // code sits in gaps between declared ranges, and both ends of a tail call + // can land there. When the source has no declared range to compare + // against, fall back on the standard entry test: the target is a function + // start if the instruction *before* it ends a function (`blr`, `bctr`, or + // an unconditional `b`). Code placed immediately after a terminator is + // unreachable by fallthrough, so something must enter it there. + let ends_function = |addr: u32| -> bool { + match read_instr(pe, addr, image_base) { + Some(i) => is_blr(i) || is_bctr(i) || is_b(i).is_some(), + None => false, + } + }; + let mut tail_call_targets = 0usize; + for &(start, end) in &code_ranges { + let mut addr = start; + while addr < end { + if let Some(instr) = read_instr(pe, addr, image_base) + && let Some(target) = b_target(instr, addr) + && !saverestore_addrs.contains(&target) + && containing(target).is_none() + && code_ranges.iter().any(|&(s, e)| target >= s && target < e) + && match containing(addr) { + // Source is declared: a jump out of its own range is a + // tail call, one inside it is ordinary control flow. + Some((src_lo, src_hi)) => target < src_lo || target >= src_hi, + // Source is in an undeclared gap: fall back to the + // preceding-terminator test. + None => target >= 4 && ends_function(target - 4), + } + && call_targets.insert(target) + { + tail_call_targets += 1; + } + addr += 4; + } + } + tracing::debug!( candidates = call_targets.len(), pdata_entries = pdata.len(), - "function candidates (bl ∪ pdata)" + tail_call_targets, + "function candidates (bl ∪ pdata ∪ tail-call)" ); // 3. For each candidate, detect prologue and walk to epilogue. Pdata @@ -300,29 +385,38 @@ pub fn analyze_with_pdata( if let Some(p) = pdata_entry { fi.pdata_validated = true; fi.pdata_length = Some(p.function_length); - // bit 0 of the packed flags = exception-handler-present + fi.pdata_prolog_length = Some(p.prolog_length); + // `flags` bit 1 mirrors packed-word bit 31 = exception handler + // registered (see `xenia_xex::pdata`). Bit 0 is the 32-bit-code + // flag, which is set on essentially every PPC entry. fi.has_eh = (p.flags & 0x2) != 0; - // If the prologue walk ended too early, trust pdata's length. - let pdata_end = p.begin_address.wrapping_add(p.function_length); - if pdata_end > fi.end { - fi.end = pdata_end; + // The linker's length is ground truth in BOTH directions: a + // prologue walk that ran past a `blr` into the next function is + // just as wrong as one that stopped early. Only a zero-length + // entry (never observed, but cheap to guard) falls back. + if p.function_length > 0 { + fi.end = p.begin_address.wrapping_add(p.function_length); } } functions.insert(func_addr, fi); } else if let Some(p) = pdata_entry { // Orphan: pdata claims a function here but no prologue matched. // Emit a synthetic entry so the row exists for downstream queries. + let end = p.begin_address.wrapping_add(p.function_length); functions.insert( func_addr, FuncInfo { start: func_addr, - end: p.begin_address.wrapping_add(p.function_length), + end, frame_size: 0, saved_gprs: 0, - is_leaf: false, + // A pdata orphan is usually a hand-written or fully inlined + // leaf; decide it from the body rather than guessing. + is_leaf: !range_has_call(pe, image_base, func_addr, end), is_saverestore: false, pdata_validated: true, pdata_length: Some(p.function_length), + pdata_prolog_length: Some(p.prolog_length), has_eh: (p.flags & 0x2) != 0, }, ); @@ -343,6 +437,7 @@ pub fn analyze_with_pdata( is_saverestore: true, pdata_validated: pe_sb.is_some(), pdata_length: pe_sb.map(|p| p.function_length), + pdata_prolog_length: pe_sb.map(|p| p.prolog_length), has_eh: pe_sb.map(|p| (p.flags & 0x2) != 0).unwrap_or(false), }); } @@ -357,19 +452,57 @@ pub fn analyze_with_pdata( is_saverestore: true, pdata_validated: pe_rb.is_some(), pdata_length: pe_rb.map(|p| p.function_length), + pdata_prolog_length: pe_rb.map(|p| p.prolog_length), has_eh: pe_rb.map(|p| (p.flags & 0x2) != 0).unwrap_or(false), }); } - // 5. Fix up `end_address` collisions: if function A's `end` overlaps - // function B's `start` (B > A), trim A. This catches mis-merged - // prologue walks where pdata revealed an interleaved second prologue. - // We do this in a single forward pass. + // 5. Reconcile candidate starts against the linker's ground truth. + // + // 5a. A `bl` whose target lands *strictly inside* a `.pdata`-validated + // function is not a second function — it is a branch into the middle + // of one (shared epilogue, computed-goto landing pad, or a + // mis-decoded word). Left in place such a candidate would truncate + // the real function at step 5b and orphan the rest of its body. + // The ranges come straight from `.pdata`, which the linker emits + // sorted and non-overlapping — the property the binary search needs. + // (Deriving them from `functions` instead would fold in the + // save/restore stub rows, whose `end` is a fixed block length rather + // than a pdata length and can therefore overlap a neighbour.) + let pdata_ranges: Vec<(u32, u32)> = pdata + .iter() + .filter(|e| e.function_length > 0) + .map(|e| (e.begin_address, e.end_address())) + .collect(); + debug_assert!(pdata_ranges.windows(2).all(|w| w[0].1 <= w[1].0)); + let interior: Vec = functions + .iter() + .filter(|(_, f)| !f.pdata_validated) + .map(|(&addr, _)| addr) + .filter(|&addr| { + pdata_ranges + .binary_search_by(|&(s, e)| { + if addr < s { std::cmp::Ordering::Greater } + else if addr >= e { std::cmp::Ordering::Less } + else { std::cmp::Ordering::Equal } + }) + .is_ok() + }) + .collect(); + let interior_dropped = interior.len(); + for addr in interior { + functions.remove(&addr); + } + + // 5b. Trim overlaps that remain. Only prologue-only rows are trimmed — + // a `.pdata` length is authoritative and must survive intact even + // when a neighbouring heuristic row disagrees. let starts: Vec = functions.keys().copied().collect(); for i in 0..starts.len().saturating_sub(1) { let cur = starts[i]; let next = starts[i + 1]; if let Some(fi) = functions.get_mut(&cur) + && !fi.pdata_validated && fi.end > next { fi.end = next; @@ -383,6 +516,7 @@ pub fn analyze_with_pdata( functions = functions.len(), pdata_entries = pdata.len(), pdata_validated = pdata_validated_count, + interior_candidates_dropped = interior_dropped, elapsed_ms, "function detection complete" ); @@ -395,6 +529,25 @@ pub fn analyze_with_pdata( } } +/// True when `[start, end)` contains any linking branch — `bl`, `bcl`, +/// `bctrl` or `bclrl`. Used to classify pdata-only entries as leaf or not. +fn range_has_call(pe: &[u8], image_base: u32, start: u32, end: u32) -> bool { + let mut addr = start; + while addr < end { + let Some(instr) = read_instr(pe, addr, image_base) else { return false }; + let opcode = op(instr); + // I-form / B-form with LK, and XL-form bclrl / bcctrl. + if (opcode == 18 || opcode == 16) && instr & 1 == 1 { + return true; + } + if opcode == 19 && instr & 1 == 1 && matches!(bits(instr, 30, 21), 16 | 528) { + return true; + } + addr = addr.wrapping_add(4); + } + false +} + /// Analyze a single function starting at `func_addr`. fn analyze_function( pe: &[u8], @@ -509,6 +662,7 @@ fn analyze_function( is_saverestore: false, pdata_validated: false, pdata_length: None, + pdata_prolog_length: None, has_eh: false, }) } diff --git a/crates/xenia-analysis/src/ind_dispatch_typed.rs b/crates/xenia-analysis/src/ind_dispatch_typed.rs index 3b3d82e..cf5d001 100644 --- a/crates/xenia-analysis/src/ind_dispatch_typed.rs +++ b/crates/xenia-analysis/src/ind_dispatch_typed.rs @@ -72,6 +72,10 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use crate::func::FuncAnalysis; use crate::vtables::Vtable; +/// Default ceiling on how many candidates a single dispatch site may +/// materialise. See [`analyze`]. +pub const DEFAULT_MAX_CANDIDATES: usize = 16; + /// One detected dispatch site after typed resolution. #[derive(Debug, Clone)] pub struct TypedDispatch { @@ -79,9 +83,16 @@ pub struct TypedDispatch { pub vptr_offset: u32, pub slot: u32, /// Set of candidate vtable addresses whose `(vptr_offset, slot)` matched. + /// Empty when [`Self::truncated`] is set. pub candidate_vtables: Vec, /// Set of resolved method PCs (one per candidate vtable). + /// Empty when [`Self::truncated`] is set. pub method_pcs: Vec, + /// How many candidates matched, whether or not they were materialised. + pub total_candidates: usize, + /// True when `total_candidates` exceeded the ceiling, so the per-candidate + /// vectors were dropped. The site itself is still reported. + pub truncated: bool, } /// Result of the M5.5 pass. @@ -117,6 +128,7 @@ pub fn analyze( func_analysis: &FuncAnalysis, vtables: &[Vtable], labels: &HashMap, + max_candidates: usize, ) -> TypedIndirectResult { let started = std::time::Instant::now(); @@ -138,13 +150,37 @@ pub fn analyze( } // Phase 3 + 4: scan dispatches and emit edges. - let dispatches = scan_dispatches_and_resolve( + let mut dispatches = scan_dispatches_and_resolve( pe, image_base, func_analysis, &block_boundaries, &vtables_by_offset, &vtable_by_addr, ); + // Drop the per-candidate lists for sites the analysis could not narrow. + // + // A site is resolved by matching `(vptr_offset, slot)` against every class + // seen installing a vtable at that offset. When the offset is 0 — a + // single-inheritance `this->vptr` — that matches essentially every class in + // the binary, so the "resolution" degenerates into a cross product: on the + // reference title 6,556 of 6,983 sites produced 1.80M of the 1.81M + // candidate rows, one site claiming 764 different callees. Those rows are + // not evidence about the callee, and they swamped `xrefs` (84% of it) and + // dominated the database file. + // + // The site row is still emitted with a truthful `total_candidates`, so + // "this is an unresolved virtual call with N possibilities" remains + // queryable — only the meaningless enumeration is dropped. + let mut truncated = 0usize; + for d in &mut dispatches { + if d.total_candidates > max_candidates { + d.candidate_vtables.clear(); + d.method_pcs.clear(); + d.truncated = true; + truncated += 1; + } + } + let elapsed_ms = started.elapsed().as_millis() as f64; - let single_candidate = dispatches.iter().filter(|d| d.candidate_vtables.len() == 1).count(); + let single_candidate = dispatches.iter().filter(|d| d.total_candidates == 1).count(); let multi_candidate = dispatches.len() - single_candidate; let total_edges: usize = dispatches.iter().map(|d| d.method_pcs.len()).sum(); metrics::histogram!("analysis.phase_ms", "phase" => "ind_dispatch_typed").record(elapsed_ms); @@ -154,6 +190,8 @@ pub fn analyze( dispatches = dispatches.len(), single = single_candidate, multi = multi_candidate, + truncated, + max_candidates, edges = total_edges, elapsed_ms, "M5.5 typed indirect-dispatch scan complete", @@ -398,6 +436,7 @@ fn try_resolve_dispatch_site( } } if method_pcs.is_empty() { return None; } + let total_candidates = candidate_vtables.len(); Some(TypedDispatch { dispatch_pc: bcctrl_pc, @@ -405,6 +444,8 @@ fn try_resolve_dispatch_site( slot, candidate_vtables, method_pcs, + total_candidates, + truncated: false, }) } @@ -464,6 +505,7 @@ mod tests { is_saverestore: false, pdata_validated: false, pdata_length: None, + pdata_prolog_length: None, has_eh: false, }); FuncAnalysis { functions, save_gpr_base: None, restore_gpr_base: None, pdata_entries: Vec::new() } @@ -521,12 +563,12 @@ mod tests { fa.functions.insert(disp_pc, FuncInfo { start: disp_pc, end: disp_pc + 0x40, frame_size: 0, saved_gprs: 0, is_leaf: false, is_saverestore: false, - pdata_validated: false, pdata_length: None, has_eh: false, + pdata_validated: false, pdata_length: None, pdata_prolog_length: None, has_eh: false, }); let vt = mk_vtable(0x82010000, vec![0xAA, 0xBB, 0xCC, 0xDD]); let labels: HashMap = HashMap::new(); - let r = analyze(&pe, image_base, &fa, &[vt], &labels); + let r = analyze(&pe, image_base, &fa, &[vt], &labels, usize::MAX); assert_eq!(r.vptr_writes.len(), 1); assert_eq!(r.vptr_writes[0].vtable_addr, 0x82010000); @@ -541,9 +583,11 @@ mod tests { assert_eq!(d.candidate_vtables, vec![0x82010000]); } - #[test] - fn multi_candidate_emits_one_edge_per_match() { - let image_base = 0x82000000u32; + /// Two classes installing different vtables at offset 0, and one dispatch + /// at slot 1 that therefore matches both. + fn multi_candidate_fixture(image_base: u32) + -> (Vec, FuncAnalysis, Vec, HashMap) + { let mut pe = vec![0u8; 0x4000]; // Two ctors, each writing a different vtable at offset 0. @@ -560,12 +604,12 @@ mod tests { fa.functions.insert(ctor_b, FuncInfo { start: ctor_b, end: ctor_b + 0x40, frame_size: 0, saved_gprs: 0, is_leaf: false, is_saverestore: false, - pdata_validated: false, pdata_length: None, has_eh: false, + pdata_validated: false, pdata_length: None, pdata_prolog_length: None, has_eh: false, }); fa.functions.insert(disp, FuncInfo { start: disp, end: disp + 0x40, frame_size: 0, saved_gprs: 0, is_leaf: false, is_saverestore: false, - pdata_validated: false, pdata_length: None, has_eh: false, + pdata_validated: false, pdata_length: None, pdata_prolog_length: None, has_eh: false, }); let vts = vec![ @@ -573,7 +617,14 @@ mod tests { mk_vtable(0x82010040, vec![0x55, 0x66, 0x77, 0x88]), ]; let labels: HashMap = HashMap::new(); - let r = analyze(&pe, image_base, &fa, &vts, &labels); + (pe, fa, vts, labels) + } + + #[test] + fn multi_candidate_emits_one_edge_per_match() { + let image_base = 0x82000000u32; + let (pe, fa, vts, labels) = multi_candidate_fixture(image_base); + let r = analyze(&pe, image_base, &fa, &vts, &labels, usize::MAX); assert_eq!(r.vptr_writes.len(), 2); assert_eq!(r.dispatches.len(), 1); @@ -599,12 +650,12 @@ mod tests { fa.functions.insert(disp, FuncInfo { start: disp, end: disp + 0x40, frame_size: 0, saved_gprs: 0, is_leaf: false, is_saverestore: false, - pdata_validated: false, pdata_length: None, has_eh: false, + pdata_validated: false, pdata_length: None, pdata_prolog_length: None, has_eh: false, }); let vt = mk_vtable(0x82010000, vec![0x11, 0x22, 0x33, 0x44]); let labels: HashMap = HashMap::new(); - let r = analyze(&pe, image_base, &fa, &[vt], &labels); + let r = analyze(&pe, image_base, &fa, &[vt], &labels, usize::MAX); assert_eq!(r.dispatches.len(), 0); } @@ -625,12 +676,36 @@ mod tests { fa.functions.insert(disp, FuncInfo { start: disp, end: disp + 0x40, frame_size: 0, saved_gprs: 0, is_leaf: false, is_saverestore: false, - pdata_validated: false, pdata_length: None, has_eh: false, + pdata_validated: false, pdata_length: None, pdata_prolog_length: None, has_eh: false, }); let vt = mk_vtable(0x82010000, vec![0x11, 0x22, 0x33, 0x44]); let labels: HashMap = HashMap::new(); - let r = analyze(&pe, image_base, &fa, &[vt], &labels); + let r = analyze(&pe, image_base, &fa, &[vt], &labels, usize::MAX); assert_eq!(r.dispatches.len(), 0); } + /// A site the resolver cannot narrow keeps its row and its true count, but + /// stops enumerating: those rows were 84% of `xrefs` and carried no + /// evidence about the callee. + #[test] + fn ceiling_truncates_unresolved_sites_without_losing_them() { + let image_base = 0x82000000u32; + let (pe, fa, vts, labels) = multi_candidate_fixture(image_base); + + let unbounded = analyze(&pe, image_base, &fa, &vts, &labels, usize::MAX); + let d = &unbounded.dispatches[0]; + assert_eq!(d.total_candidates, 2); + assert_eq!(d.method_pcs.len(), 2); + assert!(!d.truncated); + + // Same binary, ceiling of 1: the site survives, the enumeration does not. + let bounded = analyze(&pe, image_base, &fa, &vts, &labels, 1); + let d = &bounded.dispatches[0]; + assert_eq!(bounded.dispatches.len(), unbounded.dispatches.len()); + assert!(d.truncated); + assert_eq!(d.total_candidates, 2, "count stays truthful"); + assert!(d.method_pcs.is_empty(), "no speculative edges"); + assert!(d.candidate_vtables.is_empty()); + } + } diff --git a/crates/xenia-analysis/src/indirect.rs b/crates/xenia-analysis/src/indirect.rs index 506e825..8dbb916 100644 --- a/crates/xenia-analysis/src/indirect.rs +++ b/crates/xenia-analysis/src/indirect.rs @@ -374,6 +374,7 @@ mod tests { is_saverestore: false, pdata_validated: false, pdata_length: None, + pdata_prolog_length: None, has_eh: false, }); let func_analysis = FuncAnalysis { @@ -415,6 +416,7 @@ mod tests { is_saverestore: false, pdata_validated: false, pdata_length: None, + pdata_prolog_length: None, has_eh: false, }); let func_analysis = FuncAnalysis { @@ -450,6 +452,7 @@ mod tests { is_saverestore: false, pdata_validated: false, pdata_length: None, + pdata_prolog_length: None, has_eh: false, }); let func_analysis = FuncAnalysis { diff --git a/crates/xenia-analysis/src/jumptables.rs b/crates/xenia-analysis/src/jumptables.rs new file mode 100644 index 0000000..029bb69 --- /dev/null +++ b/crates/xenia-analysis/src/jumptables.rs @@ -0,0 +1,758 @@ +//! Switch-statement (jump-table) recovery for MSVC PowerPC `bctr` dispatch. +//! +//! The Xbox 360 MSVC compiler lowers a dense `switch` to a table of **absolute +//! target VAs** that is emitted *inline in `.text`*, immediately after the +//! dispatching `bctr` in the common case. The canonical shape is: +//! +//! ```text +//! cmplwi rIdx, N ; bound check — N is the largest case index +//! bgt default ; out-of-range → default label +//! lis r12, tab@h ; addis r12, r0, hi +//! addi r12, r12, tab@l ; r12 = &table +//! rlwinm r0, rIdx, 2, 22, 29 ; r0 = idx * 4 +//! lwzx r0, r12, r0 ; r0 = table[idx] +//! mtctr r0 +//! bctr ; → case body +//! .long case0, case1, ... ; the table itself, inline in .text +//! ``` +//! +//! Sparse switches add a second, byte-wide *index map* read with `lbzx`: +//! `slot = map[idx]` then `target = table[slot]`, which lets several case +//! values share one body without a full-width table. Both tables are +//! recovered; for the two-level form the emitted `targets` vector is already +//! **expanded per case value** (`targets[i] = table[map[i]]`), so consumers +//! never have to redo the indirection. +//! +//! # Why this matters +//! +//! Without this pass the `bctr` is a dead end: the case bodies have no +//! incoming edge (they are unreachable in `v_reachability_from_entry`), and — +//! worse — the table words themselves are linearly disassembled as if they +//! were instructions, so `instructions` carries thousands of bogus rows in the +//! middle of otherwise correct functions. This module fixes both: it emits +//! `jump` xrefs for every case target and reports the table extents as +//! data-in-code regions so the disassembler can mark those words. +//! +//! # Validation +//! +//! A candidate table is accepted only while its entries land **inside the +//! enclosing function** (per the `.pdata`-validated boundary from +//! [`crate::func`]). That is not a heuristic softener — a `switch` always +//! branches within its own function — and on the reference title it holds for +//! 100% of recovered entries, which is what lets the scan terminate a table +//! without needing the `cmplwi` bound. When the enclosing function is unknown +//! the weaker "inside some code section" test is used instead. +//! +//! # Limits +//! +//! - Only `bctr` (tail dispatch) is considered. `bctrl` is a call through a +//! function pointer — that is [`crate::indirect`]'s job, not a switch. +//! - The constant tracker is a straight-line, single-block model over a fixed +//! backward window. Table bases materialised across a branch, or through a +//! register the model conservatively invalidates, are missed (no false +//! positives result — target validation still gates every emission). + +use std::collections::BTreeSet; + +use xenia_xex::pe::PeSection; + +use crate::func::FuncAnalysis; + +/// How far back from a `bctr` the constant tracker looks for the table setup. +/// MSVC emits the whole sequence within a handful of instructions; 48 is far +/// beyond what any observed switch needs and still bounds the scan cost. +const WINDOW_INSTRS: u32 = 48; + +/// Hard ceiling on entries read from a table whose extent cannot be bounded by +/// the enclosing function (only reached when the function is unknown). +const MAX_ENTRIES: u32 = 4096; + +/// Why a `bctr` did not yield a table. Counted per image and logged, so a +/// coverage regression shows up as a shift between buckets rather than as a +/// silently smaller table count. +#[derive(Debug, Default, Clone, Copy)] +pub struct RejectCounts { + /// CTR was not loaded by an `lwzx` in the window — an ordinary indirect + /// tail-call (through a vtable slot or a function-pointer field). + pub not_table_driven: u32, + /// The `lwzx` operands did not resolve to exactly one code-range constant. + pub base_unresolved: u32, + /// A table base resolved, but fewer than two entries validated. + pub too_few_entries: u32, +} + +/// One recovered switch dispatch. +#[derive(Debug, Clone)] +pub struct JumpTable { + /// VA of the dispatching `bctr`. + pub bctr_pc: u32, + /// VA of the enclosing function, when known. + pub function: Option, + /// VA of the table of absolute case targets. + pub table_address: u32, + /// Number of `targets` entries (case values, after index-map expansion). + pub entry_count: u32, + /// Number of 4-byte slots occupied by the target table itself. Equal to + /// `entry_count` for a direct table; for a two-level table it is the + /// highest map slot actually used + 1, which is what bounds the raw table. + pub table_slots: u32, + /// VA of the byte-wide index map for a two-level (sparse) switch. + pub index_map_address: Option, + /// Number of bytes read from the index map (= `bound + 1`). + pub index_map_count: Option, + /// Largest valid case index per the `cmplwi rIdx, N` bound check, when the + /// compare was found in the window. + pub bound: Option, + /// `"direct"` — `target = table[idx]`; `"indexed"` — `target = + /// table[map[idx]]`. + pub kind: &'static str, + /// Case target VA per case value, in case order. May repeat. + pub targets: Vec, +} + +impl JumpTable { + /// Byte extents of the raw tables, for marking data-in-code. + /// Returns `(address, length)` pairs. + pub fn data_regions(&self) -> Vec<(u32, u32)> { + let mut out = Vec::with_capacity(2); + out.push((self.table_address, self.table_slots.saturating_mul(4))); + if let (Some(addr), Some(n)) = (self.index_map_address, self.index_map_count) { + out.push((addr, n)); + } + out + } + + /// Distinct case bodies this dispatch can reach, sorted. + pub fn distinct_targets(&self) -> Vec { + let mut t = self.targets.clone(); + t.sort_unstable(); + t.dedup(); + t + } +} + +// ── Instruction field helpers ────────────────────────────────────────────── + +const BCTR: u32 = 0x4E80_0420; + +fn op(i: u32) -> u32 { i >> 26 } +fn rt(i: u32) -> usize { ((i >> 21) & 0x1F) as usize } +fn ra(i: u32) -> usize { ((i >> 16) & 0x1F) as usize } +fn rb(i: u32) -> usize { ((i >> 11) & 0x1F) as usize } +fn xo(i: u32) -> u32 { (i >> 1) & 0x3FF } +fn simm(i: u32) -> i32 { ((i & 0xFFFF) as i16) as i32 } +fn uimm(i: u32) -> u32 { i & 0xFFFF } + +/// `mtctr rS` — `mtspr` (op 31, xo 467) with the split SPR field naming CTR (9). +fn is_mtctr(i: u32) -> bool { + if op(i) != 31 || xo(i) != 467 { return false; } + let spr_field = (i >> 11) & 0x3FF; + (((spr_field & 0x1F) << 5) | (spr_field >> 5)) == 9 +} + +/// Which GPR (if any) an `op 31` instruction writes. +/// +/// The model errs toward *over*-invalidation: an unrecognised `op 31` form is +/// assumed to clobber its `rT` field. Losing a tracked constant only costs a +/// missed table; it cannot invent one, because every emitted target is +/// range-validated against the enclosing function. +fn op31_dest(i: u32) -> Option { + // Logical / shift / sign-extend X-forms: destination is `rA` (bits 16..20). + const WRITES_RA: &[u32] = &[ + 24, 26, 27, 28, 58, 60, 124, 284, 316, 412, 444, 476, + 536, 539, 792, 794, 824, 826, 827, 922, 954, 986, + ]; + // Stores, compares, traps, cache/sync ops and `mtspr`/`mtcrf`: no GPR write. + const NO_GPR: &[u32] = &[ + 0, 4, 32, 68, // cmp, tw, cmpl, td + 150, 151, 215, 407, 662, 918, 660, 727, 231, // stwcx./stwx/stbx/sthx/stfsx/stfdx/… + 144, 467, 512, 598, 854, 982, 1014, 86, 470, 54, // mtcrf/mtspr/mcrxr/sync/dcb*/icbi + ]; + // Store-*update* forms write back into `rA`. + if matches!(xo(i), 183 | 247 | 439 | 181 | 693 | 759) { + return Some(ra(i)); + } + let x = xo(i); + if NO_GPR.contains(&x) { return None; } + if WRITES_RA.contains(&x) { return Some(ra(i)); } + Some(rt(i)) +} + +// ── Main analysis ────────────────────────────────────────────────────────── + +/// Recover every dense/sparse switch dispatch in the image's code sections. +pub fn analyze( + pe: &[u8], + image_base: u32, + sections: &[PeSection], + func_analysis: &FuncAnalysis, +) -> Vec { + analyze_with_stats(pe, image_base, sections, func_analysis).0 +} + +/// Like [`analyze`], but also returns the per-reason reject tally — the same +/// numbers the pass logs, for callers that want to assert on coverage. +#[tracing::instrument(skip_all, fields(image_base = format_args!("{:#010x}", image_base)))] +pub fn analyze_with_stats( + pe: &[u8], + image_base: u32, + sections: &[PeSection], + func_analysis: &FuncAnalysis, +) -> (Vec, RejectCounts) { + let started = std::time::Instant::now(); + + let code_ranges: Vec<(u32, u32)> = sections + .iter() + .filter(|s| s.is_code()) + .map(|s| (image_base + s.virtual_address, image_base + s.virtual_address + s.virtual_size)) + .collect(); + + let read = |va: u32| -> Option { + let off = va.wrapping_sub(image_base) as usize; + if off.checked_add(4)? > pe.len() { return None; } + Some(u32::from_be_bytes([pe[off], pe[off + 1], pe[off + 2], pe[off + 3]])) + }; + let in_code = |va: u32| code_ranges.iter().any(|&(s, e)| va >= s && va < e); + + let mut out: Vec = Vec::new(); + let mut rejects = RejectCounts::default(); + let mut sites = 0u32; + + for &(sec_start, sec_end) in &code_ranges { + let mut pc = sec_start; + while pc < sec_end { + let Some(instr) = read(pc) else { break }; + if instr == BCTR { + sites += 1; + if let Some(jt) = + recover_at(pc, sec_start, &read, &in_code, func_analysis, &mut rejects) + { + out.push(jt); + } + } + pc = pc.wrapping_add(4); + } + } + + let entries: usize = out.iter().map(|t| t.targets.len()).sum(); + let indexed = out.iter().filter(|t| t.kind == "indexed").count(); + let elapsed_ms = started.elapsed().as_millis() as f64; + metrics::histogram!("analysis.phase_ms", "phase" => "jumptables").record(elapsed_ms); + tracing::info!( + bctr_sites = sites, + jump_tables = out.len(), + indexed, + case_targets = entries, + rejected_not_table_driven = rejects.not_table_driven, + rejected_base_unresolved = rejects.base_unresolved, + rejected_too_few_entries = rejects.too_few_entries, + elapsed_ms, + "jump-table scan complete", + ); + (out, rejects) +} + +/// Try to recover a jump table for the `bctr` at `bctr_pc`. +fn recover_at( + bctr_pc: u32, + sec_start: u32, + read: &impl Fn(u32) -> Option, + in_code: &impl Fn(u32) -> bool, + func_analysis: &FuncAnalysis, + rejects: &mut RejectCounts, +) -> Option { + let containing = func_analysis + .functions + .range(..=bctr_pc) + .next_back() + .filter(|(_, fi)| bctr_pc < fi.end); + + // Only a `.pdata`-validated range is usable as a table bound. A + // prologue-only row's `end` comes from an epilogue walk that stops at the + // first `blr`/`bctr` — i.e. at *this* dispatch — so every case body would + // fall "outside the function" and the table would be rejected wholesale. + let enclosing = containing + .filter(|(_, fi)| fi.pdata_validated) + .map(|(&a, fi)| (a, fi.end)); + + // The window clamp is safe with either kind of row: it only limits how far + // back the constant tracker looks. + let window_start = { + let by_window = bctr_pc.saturating_sub(WINDOW_INSTRS * 4); + let by_func = containing.map(|(&a, _)| a).unwrap_or(sec_start); + by_window.max(by_func).max(sec_start) + }; + + // Straight-line constant propagation over [window_start, bctr_pc). + let mut regs: [Option; 32] = [None; 32]; + let mut lwzx_dest: Option = None; // rT of the last lwzx + let mut lwzx_regs: Option<(Option, Option)> = None; + let mut lwzx_index_tainted = false; // did the index come from the lbzx? + let mut lbzx_regs: Option<(Option, Option)> = None; + let mut ctr_src: Option = None; // rS of the last mtctr + let mut bound: Option = None; + // Taint: "this register holds a value derived from the byte the `lbzx` + // read". Only a register carrying that taint may serve as the jump table's + // index in the two-level form — otherwise any unrelated `lbzx` in the + // window (there are plenty; games read bytes constantly) would be mistaken + // for a case index map. + let mut from_lbzx = [false; 32]; + + let mut pc = window_start; + while pc < bctr_pc { + let Some(i) = read(pc) else { return None }; + match op(i) { + // addis rT, rA, SIMM (lis when rA == 0) + 15 => { + let base = if ra(i) == 0 { Some(0) } else { regs[ra(i)] }; + regs[rt(i)] = base.map(|b| b.wrapping_add(uimm(i) << 16)); + } + // addi rT, rA, SIMM (li when rA == 0) + 14 => { + let base = if ra(i) == 0 { Some(0) } else { regs[ra(i)] }; + regs[rt(i)] = base.map(|b| b.wrapping_add(simm(i) as u32)); + } + // ori / oris rA, rS, UIMM + 24 => regs[ra(i)] = regs[rt(i)].map(|b| b | uimm(i)), + 25 => regs[ra(i)] = regs[rt(i)].map(|b| b | (uimm(i) << 16)), + // cmplwi / cmpwi rA, IMM — the switch's bound check. + 10 | 11 => bound = Some(uimm(i)), + 31 => { + match xo(i) { + 23 => { // lwzx rT, rA, rB — the table read + lwzx_dest = Some(rt(i)); + lwzx_regs = Some((regs[ra(i)], regs[rb(i)])); + lwzx_index_tainted = from_lbzx[ra(i)] || from_lbzx[rb(i)]; + regs[rt(i)] = None; + from_lbzx[rt(i)] = false; + } + 87 => { // lbzx rT, rA, rB — the sparse index-map read + lbzx_regs = Some((regs[ra(i)], regs[rb(i)])); + regs[rt(i)] = None; + from_lbzx = [false; 32]; + from_lbzx[rt(i)] = true; + } + 467 if is_mtctr(i) => ctr_src = Some(rt(i)), + // `mr rA, rS` is `or rA, rS, rS` — propagate constant + taint. + 444 if rt(i) == rb(i) => { + regs[ra(i)] = regs[rt(i)]; + from_lbzx[ra(i)] = from_lbzx[rt(i)]; + } + // `add rT, rA, rB` / `slw rA, rS, rB` also carry the index. + 266 => { + regs[rt(i)] = None; + from_lbzx[rt(i)] = from_lbzx[ra(i)] || from_lbzx[rb(i)]; + } + 24 => { + regs[ra(i)] = None; + from_lbzx[ra(i)] = from_lbzx[rt(i)]; + } + _ => { + if let Some(d) = op31_dest(i) { + regs[d] = None; + from_lbzx[d] = false; + } + } + } + } + // rlwinm / rlwnm / rlwimi write rA — and are how a byte index gets + // scaled to a word offset, so they carry the taint through. + 20 | 21 | 23 => { + regs[ra(i)] = None; + from_lbzx[ra(i)] = from_lbzx[rt(i)]; + } + // D/DS-form GPR loads write rT; the update forms also write rA. + 32 | 34 | 40 | 42 => regs[rt(i)] = None, + 33 | 35 | 41 | 43 => { regs[rt(i)] = None; regs[ra(i)] = None; } + // DS-form: bits 30..31 pick ld(0) / ldu(1) / lwa(2). + 58 => { + regs[rt(i)] = None; + if i & 3 == 1 { regs[ra(i)] = None; } + } + // lmw loads rT..r31. + 46 => for r in rt(i)..32 { regs[r] = None; }, + // FP loads touch no GPR — except the update forms, which write rA. + // Plain stores write no register at all (their `rT` field is the + // *source*), so a tracked base that merely gets spilled survives. + 37 | 39 | 45 | 49 | 51 | 53 | 55 => regs[ra(i)] = None, + // DS-form: bits 30..31 pick std(0) / stdu(1); only stdu writes rA. + 62 if i & 3 == 1 => regs[ra(i)] = None, + // Immediate ALU: 7/8/12/13 write rT, 28/29 (andi./andis.) write rA. + 7 | 8 | 12 | 13 => regs[rt(i)] = None, + 28 | 29 => regs[ra(i)] = None, + _ => {} + } + pc = pc.wrapping_add(4); + } + + // CTR must actually be loaded from the table read — otherwise the `lwzx` + // in the window belongs to unrelated code and the `bctr` is a plain + // indirect tail-call. + if ctr_src.is_none() || ctr_src != lwzx_dest { + rejects.not_table_driven += 1; + return None; + } + let Some((a_val, b_val)) = lwzx_regs else { + rejects.not_table_driven += 1; + return None; + }; + + // Exactly one operand of the table read must be a constant that lands in + // code — the other is the scaled index. Two constants is ambiguous. + let table_address = match (a_val.filter(|&v| in_code(v)), b_val.filter(|&v| in_code(v))) { + (Some(v), None) | (None, Some(v)) => v, + _ => { + rejects.base_unresolved += 1; + return None; + } + }; + + // Same test for the sparse index-map base — but only when the byte that + // `lbzx` produced actually reached the table read as its index. + let index_map_address = lbzx_regs.filter(|_| lwzx_index_tainted).and_then(|(a, b)| { + match (a.filter(|&v| in_code(v)), b.filter(|&v| in_code(v))) { + (Some(v), None) | (None, Some(v)) => Some(v), + _ => None, + } + }); + + // A recovered target is valid only inside the enclosing function; that is + // exact for a `switch`, and on the reference title it holds for 100% of + // recovered entries. Where no `.pdata` range covers the dispatch there is + // no trustworthy bound, so the compiler's own `cmplwi` bound is required + // instead and targets are only checked for being code at all. + if enclosing.is_none() && bound.is_none() { + rejects.base_unresolved += 1; + return None; + } + let valid = |t: u32| match enclosing { + Some((s, e)) => t >= s && t < e, + None => in_code(t), + }; + + if let (Some(map_addr), Some(n)) = (index_map_address, bound) { + // Two-level: expand map[0..=bound] through the table in one shot. + let count = n.saturating_add(1).min(MAX_ENTRIES); + let mut targets = Vec::with_capacity(count as usize); + let mut max_slot = 0u32; + for i in 0..count { + let byte_off = map_addr.wrapping_add(i); + let word = read(byte_off & !3)?; + let slot = (word >> (8 * (3 - (byte_off & 3)))) & 0xFF; + let t = read(table_address.wrapping_add(slot * 4))?; + if !valid(t) { break; } + max_slot = max_slot.max(slot); + targets.push(t); + } + if targets.len() < 2 { + rejects.too_few_entries += 1; + return None; + } + let n_read = targets.len() as u32; + return Some(JumpTable { + bctr_pc, + function: enclosing.map(|(s, _)| s), + table_address, + entry_count: n_read, + table_slots: max_slot + 1, + index_map_address, + index_map_count: Some(n_read), + bound, + kind: "indexed", + targets, + }); + } + + // Dense: read consecutive absolute targets until one leaves the function. + let cap = bound.map(|n| n.saturating_add(1)).unwrap_or(MAX_ENTRIES).min(MAX_ENTRIES); + let mut targets = Vec::new(); + for i in 0..cap { + let Some(t) = read(table_address.wrapping_add(i * 4)) else { break }; + if !valid(t) { break; } + targets.push(t); + } + if targets.len() < 2 { + rejects.too_few_entries += 1; + return None; + } + + Some(JumpTable { + bctr_pc, + function: enclosing.map(|(s, _)| s), + table_address, + entry_count: targets.len() as u32, + table_slots: targets.len() as u32, + index_map_address: None, + index_map_count: None, + bound, + kind: "direct", + targets, + }) +} + +/// Collapse every recovered table's raw extents into a sorted, merged interval +/// list of data-in-code byte ranges. +pub fn data_regions(tables: &[JumpTable]) -> Vec<(u32, u32)> { + let mut regions: Vec<(u32, u32)> = tables + .iter() + .flat_map(|t| t.data_regions()) + .filter(|&(_, len)| len > 0) + .collect(); + regions.sort_unstable(); + + let mut merged: Vec<(u32, u32)> = Vec::with_capacity(regions.len()); + for (addr, len) in regions { + match merged.last_mut() { + Some((p_addr, p_len)) if addr <= p_addr.wrapping_add(*p_len) => { + let end = addr.wrapping_add(len).max(p_addr.wrapping_add(*p_len)); + *p_len = end.wrapping_sub(*p_addr); + } + _ => merged.push((addr, len)), + } + } + merged +} + +/// Expand merged byte ranges into the set of 4-byte-aligned word addresses they +/// cover — the granularity at which `instructions` rows are marked. +pub fn data_word_addresses(tables: &[JumpTable]) -> BTreeSet { + let mut set = BTreeSet::new(); + for (addr, len) in data_regions(tables) { + let start = addr & !3; + let end = addr.wrapping_add(len).div_ceil(4) * 4; + let mut a = start; + while a < end { + set.insert(a); + a = a.wrapping_add(4); + } + } + set +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + use xenia_xex::pe::PeSection; + + use crate::func::{FuncAnalysis, FuncInfo}; + + const BASE: u32 = 0x8200_0000; + const TEXT_RVA: u32 = 0x1000; + const TEXT_VA: u32 = BASE + TEXT_RVA; + + fn text_section(size: u32) -> PeSection { + PeSection { + name: ".text".into(), + virtual_address: TEXT_RVA, + virtual_size: size, + raw_offset: TEXT_RVA, + raw_size: size, + flags: 0x6000_0020, // CODE | EXECUTE | READ + } + } + + fn one_function(start: u32, end: u32) -> FuncAnalysis { + let mut functions = BTreeMap::new(); + functions.insert(start, FuncInfo { + start, + end, + frame_size: 0, + saved_gprs: 0, + is_leaf: false, + is_saverestore: false, + pdata_validated: true, + pdata_length: Some(end - start), + pdata_prolog_length: Some(0), + has_eh: false, + }); + FuncAnalysis { functions, save_gpr_base: None, restore_gpr_base: None, pdata_entries: Vec::new() } + } + + /// Encode the words of the canonical MSVC dense-switch dispatch, ending at + /// the `bctr`, then the inline table. Mirrors the real sequence: + /// cmplwi r10,N / bgt / lis r12 / addi r12 / slwi r0,r10,2 / lwzx r0,r12,r0 + /// / mtctr r0 / bctr / + fn dense_switch(table_va: u32, n_cases: u32) -> Vec { + vec![ + 0x2800_0000 | (10 << 16) | (n_cases - 1), // cmplwi r10, N + 0x4181_0000 | 0x20, // bc (bound check, target irrelevant) + 0x3D80_0000 | (table_va >> 16), // lis r12, hi + 0x398C_0000 | (table_va & 0xFFFF), // addi r12, r12, lo + 0x5540_103A, // slwi r0, r10, 2 + 0x7C0C_002E, // lwzx r0, r12, r0 + 0x7C09_03A6, // mtctr r0 + BCTR, + ] + } + + fn assemble(words: &[u32], size: u32) -> Vec { + let mut pe = vec![0u8; (TEXT_RVA + size) as usize]; + for (i, w) in words.iter().enumerate() { + let off = TEXT_RVA as usize + i * 4; + pe[off..off + 4].copy_from_slice(&w.to_be_bytes()); + } + pe + } + + #[test] + fn recovers_dense_switch_with_inline_table() { + let table_va = TEXT_VA + 8 * 4; + let mut words = dense_switch(table_va, 4); + // Four case bodies, all inside the function. + let cases = [TEXT_VA + 0x40, TEXT_VA + 0x50, TEXT_VA + 0x60, TEXT_VA + 0x70]; + words.extend_from_slice(&cases); + + let pe = assemble(&words, 0x100); + let sections = [text_section(0x100)]; + let fa = one_function(TEXT_VA, TEXT_VA + 0x100); + + let tables = analyze(&pe, BASE, §ions, &fa); + assert_eq!(tables.len(), 1, "expected exactly one recovered table"); + let jt = &tables[0]; + assert_eq!(jt.bctr_pc, TEXT_VA + 7 * 4); + assert_eq!(jt.table_address, table_va); + assert_eq!(jt.kind, "direct"); + assert_eq!(jt.bound, Some(3)); + assert_eq!(jt.targets, cases.to_vec()); + assert_eq!(jt.function, Some(TEXT_VA)); + } + + #[test] + fn table_stops_at_a_target_outside_the_function() { + // Bound says 8 cases but only the first three land inside the function; + // the fourth word is an address in a different function, which must + // terminate the table rather than be emitted as a case. + let table_va = TEXT_VA + 8 * 4; + let mut words = dense_switch(table_va, 8); + words.extend_from_slice(&[ + TEXT_VA + 0x40, TEXT_VA + 0x50, TEXT_VA + 0x60, + 0x8300_0000, // far outside + TEXT_VA + 0x70, + ]); + + let pe = assemble(&words, 0x100); + let sections = [text_section(0x100)]; + let fa = one_function(TEXT_VA, TEXT_VA + 0x80); + + let tables = analyze(&pe, BASE, §ions, &fa); + assert_eq!(tables.len(), 1); + assert_eq!(tables[0].targets.len(), 3); + assert_eq!(tables[0].entry_count, 3); + } + + #[test] + fn plain_indirect_tail_call_is_not_a_switch() { + // `lwz r12, 0(r3); mtctr r12; bctr` — a virtual tail-call, no table. + let words = [ + 0x8183_0000, // lwz r12, 0(r3) + 0x7D89_03A6, // mtctr r12 + BCTR, + ]; + let pe = assemble(&words, 0x100); + let sections = [text_section(0x100)]; + let fa = one_function(TEXT_VA, TEXT_VA + 0x100); + assert!(analyze(&pe, BASE, §ions, &fa).is_empty()); + } + + #[test] + fn recovers_sparse_two_level_switch() { + // cmplwi r10,5 / bc / lis+addi r11 = &map / lbzx r0,r11,r10 + // / lis+addi r12 = &table / slwi r0,r0,2 / lwzx r0,r12,r0 / mtctr / bctr + let map_va = TEXT_VA + 13 * 4; // 6 bytes, then padding + let table_va = TEXT_VA + 17 * 4; // 3 distinct bodies + let words = vec![ + 0x2800_0000 | (10 << 16) | 5, // cmplwi r10, 5 + 0x4181_0000 | 0x20, // bc + 0x3D60_0000 | (map_va >> 16), // lis r11, map@h + 0x396B_0000 | (map_va & 0xFFFF), // addi r11, r11, map@l + 0x7C0B_50AE, // lbzx r0, r11, r10 + 0x3D80_0000 | (table_va >> 16), // lis r12, tab@h + 0x398C_0000 | (table_va & 0xFFFF), // addi r12, r12, tab@l + 0x5400_103A, // slwi r0, r0, 2 + 0x7C0C_002E, // lwzx r0, r12, r0 + 0x7C09_03A6, // mtctr r0 + BCTR, + 0, 0, + // map[0..6] = 0,1,2,2,1,0 packed big-endian, then padding + 0x0001_0202, 0x0100_0000, + 0, 0, + // table[0..3] + TEXT_VA + 0x80, TEXT_VA + 0x90, TEXT_VA + 0xA0, + ]; + let pe = assemble(&words, 0x100); + let sections = [text_section(0x100)]; + let fa = one_function(TEXT_VA, TEXT_VA + 0x100); + + let tables = analyze(&pe, BASE, §ions, &fa); + assert_eq!(tables.len(), 1); + let jt = &tables[0]; + assert_eq!(jt.kind, "indexed"); + assert_eq!(jt.index_map_address, Some(map_va)); + assert_eq!(jt.index_map_count, Some(6)); + assert_eq!(jt.table_slots, 3); + assert_eq!(jt.targets, vec![ + TEXT_VA + 0x80, TEXT_VA + 0x90, TEXT_VA + 0xA0, + TEXT_VA + 0xA0, TEXT_VA + 0x90, TEXT_VA + 0x80, + ]); + } + + /// An unrelated `lbzx` in the window must not be mistaken for a case index + /// map — games read bytes constantly, and a fabricated two-level mapping + /// would silently point every case at the wrong body. + #[test] + fn unrelated_lbzx_does_not_become_an_index_map() { + let table_va = TEXT_VA + 11 * 4; + let mut words = vec![ + 0x2800_0000 | (10 << 16) | 3, // cmplwi r10, 3 + 0x4181_0000 | 0x20, // bc + 0x3D60_0000 | (TEXT_VA >> 16), // lis r11, text@h (a code constant) + 0x396B_0000 | (TEXT_VA & 0xFFFF), // addi r11, r11, text@l + 0x7CEB_44AE, // lbzx r7, r11, r8 — unrelated byte load + 0x3D80_0000 | (table_va >> 16), // lis r12, tab@h + 0x398C_0000 | (table_va & 0xFFFF), // addi r12, r12, tab@l + 0x5540_103A, // slwi r0, r10, 2 — index is r10, NOT r7 + 0x7C0C_002E, // lwzx r0, r12, r0 + 0x7C09_03A6, // mtctr r0 + ]; + words.push(BCTR); + words.extend_from_slice(&[TEXT_VA + 0x80, TEXT_VA + 0x90, TEXT_VA + 0xA0, TEXT_VA + 0xB0]); + + let pe = assemble(&words, 0x100); + let sections = [text_section(0x100)]; + let fa = one_function(TEXT_VA, TEXT_VA + 0x100); + + let tables = analyze(&pe, BASE, §ions, &fa); + assert_eq!(tables.len(), 1); + assert_eq!(tables[0].kind, "direct"); + assert_eq!(tables[0].index_map_address, None); + } + + #[test] + fn data_regions_merge_adjacent_tables() { + let a = JumpTable { + bctr_pc: 0x8200_1000, function: None, table_address: 0x8200_2000, + entry_count: 4, table_slots: 4, index_map_address: None, + index_map_count: None, bound: None, kind: "direct", + targets: vec![0; 4], + }; + let b = JumpTable { table_address: 0x8200_2010, bctr_pc: 0x8200_1004, ..a.clone() }; + let merged = data_regions(&[a, b]); + assert_eq!(merged, vec![(0x8200_2000, 32)]); + } + + #[test] + fn data_word_addresses_covers_every_slot() { + let jt = JumpTable { + bctr_pc: 0x8200_1000, function: None, table_address: 0x8200_2000, + entry_count: 3, table_slots: 3, index_map_address: Some(0x8200_3000), + index_map_count: Some(5), bound: Some(4), kind: "indexed", + targets: vec![0; 5], + }; + let words = data_word_addresses(&[jt]); + // 3 table slots + ceil(5/4) = 2 words of index map. + assert_eq!(words.len(), 5); + assert!(words.contains(&0x8200_2008)); + assert!(words.contains(&0x8200_3004)); + } +} diff --git a/crates/xenia-analysis/src/lib.rs b/crates/xenia-analysis/src/lib.rs index dbe3ec9..078aae0 100644 --- a/crates/xenia-analysis/src/lib.rs +++ b/crates/xenia-analysis/src/lib.rs @@ -15,6 +15,9 @@ pub mod strings; pub mod funcptr_arrays; pub mod eh_scope; pub mod static_init; +pub mod xdbf; +pub mod jumptables; +pub mod rtti; mod ordinals; pub use ordinals::resolve_ordinal; diff --git a/crates/xenia-analysis/src/rtti.rs b/crates/xenia-analysis/src/rtti.rs new file mode 100644 index 0000000..5120e57 --- /dev/null +++ b/crates/xenia-analysis/src/rtti.rs @@ -0,0 +1,453 @@ +//! MSVC RTTI recovery — the authoritative source of C++ class identity. +//! +//! [`crate::vtables`] finds vtables *bottom-up*, by looking for runs of words +//! that happen to be function entries, and only then tries the RTTI walk. That +//! misses every table whose head holds a null / pure-virtual / thunk slot, and +//! it cannot see a class that has no such run at all. This module works +//! *top-down* from the RTTI structures the linker emitted, which is exact: +//! a `CompleteObjectLocator` names its class, and the word that points at a +//! COL is by definition `vftable[-1]`. +//! +//! ## Structure layout (32-bit MSVC, big-endian on Xbox 360) +//! +//! ```text +//! TypeDescriptor (in .data — it is written at startup) +//! +0 void* pVFTable -> type_info's own vftable (identical for all TDs) +//! +4 void* spare +//! +8 char name[] -> ".?AVFoo@Bar@@", NUL-terminated +//! +//! RTTICompleteObjectLocator (in .rdata) +//! +0 u32 signature -> 0 for 32-bit images +//! +4 u32 offset -> this-offset of the subobject this vftable serves +//! +8 u32 cdOffset -> constructor-displacement offset +//! +12 TypeDescriptor* +//! +16 RTTIClassHierarchyDescriptor* +//! +//! RTTIClassHierarchyDescriptor (in .rdata) +//! +0 u32 signature +//! +4 u32 attributes -> bit 0 = multiple inheritance, bit 1 = virtual +//! +8 u32 numBaseClasses +//! +12 RTTIBaseClassDescriptor** pBaseClassArray +//! +//! RTTIBaseClassDescriptor (in .rdata) +//! +0 TypeDescriptor* +//! +4 u32 numContainedBases +//! +8 i32 PMD.mdisp -> member displacement +//! +12 i32 PMD.pdisp -> vbtable displacement (-1 = not virtual) +//! +16 i32 PMD.vdisp -> displacement inside the vbtable +//! +20 u32 attributes +//! ``` +//! +//! A vtable is located at `col_ref + 4` for every word `col_ref` whose value is +//! a validated COL address. `offset` distinguishes the primary vftable +//! (`offset == 0`) from the extra vftables a multiply-inheriting class emits +//! for its secondary base subobjects — those are linked to the same class +//! rather than being reported as unrelated tables. +//! +//! ## Limits +//! +//! - Only statically-emitted RTTI is seen; a class whose RTTI the linker +//! stripped stays anonymous and is left to [`crate::vtables`]. +//! - Vtable *length* is measured by walking forward from `vftable[0]` while the +//! words are plausible method pointers, stopping at the next COL reference or +//! at a known label — the linker does not record it. + +use std::collections::{BTreeMap, BTreeSet}; + +use xenia_xex::pe::PeSection; + +use crate::demangle; + +/// One `TypeDescriptor`: the mangled class name the compiler emitted. +#[derive(Debug, Clone)] +pub struct TypeDescriptor { + /// VA of the descriptor (i.e. of its `pVFTable` word). + pub address: u32, + /// Raw decorated name, e.g. `.?AVSilph@silph@@`. + pub mangled_name: String, + /// Readable form, e.g. `silph::Silph`. Falls back to `mangled_name`. + pub demangled_name: String, +} + +/// One `RTTICompleteObjectLocator` and the vtable it labels. +#[derive(Debug, Clone)] +pub struct CompleteObjectLocator { + pub address: u32, + /// `this`-offset of the subobject whose vftable this is. 0 = primary. + pub offset: u32, + pub cd_offset: u32, + pub type_descriptor: u32, + pub class_hierarchy: u32, + /// VA of `vftable[0]`, when a word pointing at this COL was found. + pub vtable_address: Option, +} + +/// One entry of a class's `RTTIBaseClassArray`, in linearised order. +#[derive(Debug, Clone)] +pub struct BaseClass { + /// VA of the deriving class's `RTTIClassHierarchyDescriptor`. + pub class_hierarchy: u32, + /// Position in the base-class array (index 0 is the class itself). + pub index: u32, + pub type_descriptor: u32, + pub name: String, + pub num_contained_bases: u32, + pub mdisp: i32, + pub pdisp: i32, + pub vdisp: i32, + pub attributes: u32, +} + +/// Everything the RTTI walk recovered. +#[derive(Debug, Default)] +pub struct RttiResult { + pub type_descriptors: Vec, + pub locators: Vec, + pub base_classes: Vec, + /// `vftable[0]` VA → the COL that labels it. + pub vtable_to_locator: BTreeMap, +} + +impl RttiResult { + /// Vtable base VAs the walk proved exist — the anchor set + /// [`crate::vtables`] should treat as authoritative. + pub fn vtable_anchors(&self) -> BTreeSet { + self.vtable_to_locator.keys().copied().collect() + } + + /// `vftable[0]` VA → `(demangled class name, subobject offset)`. + pub fn vtable_class_names(&self) -> BTreeMap { + let td: BTreeMap = + self.type_descriptors.iter().map(|t| (t.address, t)).collect(); + let mut out = BTreeMap::new(); + for col in &self.locators { + if let (Some(vt), Some(t)) = (col.vtable_address, td.get(&col.type_descriptor)) { + out.insert(vt, (t.demangled_name.clone(), col.offset)); + } + } + out + } +} + +// ── Scan ─────────────────────────────────────────────────────────────────── + +/// Walk the image's RTTI. `sections` must be the full PE section list. +#[tracing::instrument(skip_all, fields(image_base = format_args!("{:#010x}", image_base)))] +pub fn analyze(pe: &[u8], image_base: u32, sections: &[PeSection]) -> RttiResult { + let started = std::time::Instant::now(); + + let read = |va: u32| -> Option { + let off = va.wrapping_sub(image_base) as usize; + if off.checked_add(4)? > pe.len() { return None; } + Some(u32::from_be_bytes([pe[off], pe[off + 1], pe[off + 2], pe[off + 3]])) + }; + + // Byte ranges actually backed by file data (a section's tail beyond + // `raw_size` is BSS: reading it yields zeros, never a real structure). + let backed = |s: &PeSection| -> (u32, u32) { + let start = image_base + s.virtual_address; + let len = s.virtual_size.min(s.raw_size); + (start, start + len) + }; + let ranges: Vec<(String, u32, u32)> = sections + .iter() + .map(|s| { let (a, b) = backed(s); (s.name.clone(), a, b) }) + .collect(); + let range_of = |name: &str| -> Option<(u32, u32)> { + ranges.iter().find(|(n, _, _)| n == name).map(|&(_, a, b)| (a, b)) + }; + + // 1. TypeDescriptors. The decorated name lives at descriptor+8 and always + // starts with ".?A". MSVC places these in writable data. + let mut type_descriptors: Vec = Vec::new(); + let mut td_addrs: BTreeSet = BTreeSet::new(); + for (name, start, end) in &ranges { + if !matches!(name.as_str(), ".data" | ".rdata") { continue; } + let s = (*start).wrapping_sub(image_base) as usize; + let e = (*end).wrapping_sub(image_base) as usize; + if e > pe.len() || s >= e { continue; } + let bytes = &pe[s..e]; + let mut i = 0usize; + while i + 3 < bytes.len() { + if &bytes[i..i + 3] != b".?A" { i += 1; continue; } + let name_va = start.wrapping_add(i as u32); + // The descriptor head sits 8 bytes before the name. + let Some(td_va) = name_va.checked_sub(8) else { i += 1; continue }; + if td_va < *start { i += 1; continue; } + let Some(decorated) = read_cstr(bytes, i, 512) else { i += 1; continue }; + i += decorated.len() + 1; + if td_addrs.insert(td_va) { + type_descriptors.push(TypeDescriptor { + address: td_va, + demangled_name: demangle::demangle_type_descriptor(&decorated) + .unwrap_or_else(|| decorated.clone()), + mangled_name: decorated, + }); + } + } + } + + // 2. CompleteObjectLocators. Scan read-only data on a 4-byte grid for the + // 5-word shape whose `pTypeDescriptor` hits a descriptor we just found + // and whose `pClassDescriptor` points back into read-only data. + let rdata = range_of(".rdata"); + let mut locators: Vec = Vec::new(); + let mut col_addrs: BTreeSet = BTreeSet::new(); + if let Some((rd_start, rd_end)) = rdata { + let mut va = rd_start; + while va + 20 <= rd_end { + let (Some(sig), Some(off), Some(cd), Some(ptd), Some(pchd)) = ( + read(va), read(va + 4), read(va + 8), read(va + 12), read(va + 16), + ) else { break }; + if sig == 0 && td_addrs.contains(&ptd) && pchd >= rd_start && pchd < rd_end { + col_addrs.insert(va); + locators.push(CompleteObjectLocator { + address: va, + offset: off, + cd_offset: cd, + type_descriptor: ptd, + class_hierarchy: pchd, + vtable_address: None, + }); + } + va += 4; + } + } + + // 3. `vftable[-1]` sites: any word in initialised data whose value is a COL. + let mut vtable_to_locator: BTreeMap = BTreeMap::new(); + for (name, start, end) in &ranges { + if !matches!(name.as_str(), ".rdata" | ".data") { continue; } + let mut va = *start; + while va + 4 <= *end { + if let Some(w) = read(va) + && col_addrs.contains(&w) + { + vtable_to_locator.insert(va + 4, w); + } + va += 4; + } + } + let locator_to_vtable: BTreeMap = + vtable_to_locator.iter().map(|(&vt, &col)| (col, vt)).collect(); + for col in &mut locators { + col.vtable_address = locator_to_vtable.get(&col.address).copied(); + } + + // 4. Class hierarchies: for each distinct CHD, read its base-class array. + let td_by_addr: BTreeMap = + type_descriptors.iter().map(|t| (t.address, t)).collect(); + let mut base_classes: Vec = Vec::new(); + let chds: BTreeSet = locators.iter().map(|c| c.class_hierarchy).collect(); + if let Some((rd_start, rd_end)) = rdata { + for chd in chds { + let (Some(n_bases), Some(p_array)) = (read(chd + 8), read(chd + 12)) else { continue }; + // A malformed or misidentified descriptor would blow the scan up; + // real hierarchies are small. + if n_bases == 0 || n_bases > 64 { continue; } + if p_array < rd_start || p_array >= rd_end { continue; } + for i in 0..n_bases { + let Some(bcd) = read(p_array + i * 4) else { break }; + if bcd < rd_start || bcd >= rd_end { break; } + let (Some(ptd), Some(ncb), Some(md), Some(pd), Some(vd), Some(attr)) = ( + read(bcd), read(bcd + 4), read(bcd + 8), + read(bcd + 12), read(bcd + 16), read(bcd + 20), + ) else { break }; + let Some(td) = td_by_addr.get(&ptd) else { break }; + base_classes.push(BaseClass { + class_hierarchy: chd, + index: i, + type_descriptor: ptd, + name: td.demangled_name.clone(), + num_contained_bases: ncb, + mdisp: md as i32, + pdisp: pd as i32, + vdisp: vd as i32, + attributes: attr, + }); + } + } + } + + let elapsed_ms = started.elapsed().as_millis() as f64; + metrics::histogram!("analysis.phase_ms", "phase" => "rtti").record(elapsed_ms); + tracing::info!( + type_descriptors = type_descriptors.len(), + locators = locators.len(), + vtables = vtable_to_locator.len(), + base_class_records = base_classes.len(), + elapsed_ms, + "RTTI walk complete", + ); + + RttiResult { type_descriptors, locators, base_classes, vtable_to_locator } +} + +/// Read a NUL-terminated ASCII string starting at `off` in `bytes`. +fn read_cstr(bytes: &[u8], off: usize, max: usize) -> Option { + let end = (off + max).min(bytes.len()); + let slice = &bytes[off..end]; + let nul = slice.iter().position(|&b| b == 0)?; + let s = &slice[..nul]; + if s.is_empty() || !s.iter().all(|&b| (0x20..0x7F).contains(&b)) { + return None; + } + Some(String::from_utf8_lossy(s).into_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const BASE: u32 = 0x8200_0000; + const RDATA_RVA: u32 = 0x1000; + const DATA_RVA: u32 = 0x2000; + const SEC_SIZE: u32 = 0x1000; + + fn sections() -> Vec { + vec![ + PeSection { + name: ".rdata".into(), + virtual_address: RDATA_RVA, virtual_size: SEC_SIZE, + raw_offset: RDATA_RVA, raw_size: SEC_SIZE, + flags: 0x4000_0040, + }, + PeSection { + name: ".data".into(), + virtual_address: DATA_RVA, virtual_size: SEC_SIZE, + raw_offset: DATA_RVA, raw_size: SEC_SIZE, + flags: 0xC000_0040, + }, + ] + } + + struct Image(Vec); + impl Image { + fn new() -> Self { Image(vec![0u8; (DATA_RVA + SEC_SIZE) as usize]) } + fn put_u32(&mut self, va: u32, v: u32) { + let o = (va - BASE) as usize; + self.0[o..o + 4].copy_from_slice(&v.to_be_bytes()); + } + fn put_str(&mut self, va: u32, s: &str) { + let o = (va - BASE) as usize; + self.0[o..o + s.len()].copy_from_slice(s.as_bytes()); + self.0[o + s.len()] = 0; + } + } + + /// Lay down one class: TypeDescriptor in .data, COL + CHD + BCD in .rdata, + /// and the `vftable[-1]` word that points at the COL. + #[allow(clippy::too_many_arguments)] + fn emit_class( + img: &mut Image, td: u32, name: &str, + col: u32, offset: u32, chd: u32, bcd_array: u32, bcd: u32, base_name_td: Option, + vtable_minus_one: u32, + ) { + img.put_u32(td, 0xDEAD_BEEF); // type_info vftable — value is irrelevant + img.put_str(td + 8, name); + + img.put_u32(col, 0); // signature + img.put_u32(col + 4, offset); + img.put_u32(col + 8, 0); // cdOffset + img.put_u32(col + 12, td); + img.put_u32(col + 16, chd); + + let n_bases = if base_name_td.is_some() { 2 } else { 1 }; + img.put_u32(chd, 0); + img.put_u32(chd + 4, 0); + img.put_u32(chd + 8, n_bases); + img.put_u32(chd + 12, bcd_array); + + // Base-class array: entry 0 is the class itself. + img.put_u32(bcd_array, bcd); + img.put_u32(bcd, td); + img.put_u32(bcd + 4, n_bases - 1); + img.put_u32(bcd + 8, 0); // mdisp + img.put_u32(bcd + 12, u32::MAX); // pdisp = -1 + img.put_u32(bcd + 16, 0); // vdisp + img.put_u32(bcd + 20, 0x40); // attributes + if let Some(base_td) = base_name_td { + let bcd2 = bcd + 24; + img.put_u32(bcd_array + 4, bcd2); + img.put_u32(bcd2, base_td); + img.put_u32(bcd2 + 4, 0); + img.put_u32(bcd2 + 8, 4); // mdisp = 4 + img.put_u32(bcd2 + 12, u32::MAX); + img.put_u32(bcd2 + 16, 0); + img.put_u32(bcd2 + 20, 0); + } + + img.put_u32(vtable_minus_one, col); + } + + #[test] + fn recovers_class_name_vtable_and_bases() { + let mut img = Image::new(); + let rd = BASE + RDATA_RVA; + let da = BASE + DATA_RVA; + + // Base class Foo, then Derived : Foo. + emit_class(&mut img, da + 0x100, ".?AVFoo@ns@@", + rd + 0x100, 0, rd + 0x200, rd + 0x280, rd + 0x300, None, + rd + 0x000); + emit_class(&mut img, da + 0x200, ".?AVDerived@ns@@", + rd + 0x400, 0, rd + 0x500, rd + 0x580, rd + 0x600, Some(da + 0x100), + rd + 0x040); + + let r = analyze(&img.0, BASE, §ions()); + + assert_eq!(r.type_descriptors.len(), 2); + let derived = r.type_descriptors.iter() + .find(|t| t.mangled_name.contains("Derived")).unwrap(); + assert_eq!(derived.demangled_name, "ns::Derived"); + + assert_eq!(r.locators.len(), 2); + // vftable[0] is one word past the word holding the COL pointer. + assert_eq!(r.vtable_to_locator.get(&(rd + 0x044)), Some(&(rd + 0x400))); + assert!(r.vtable_anchors().contains(&(rd + 0x004))); + + let names = r.vtable_class_names(); + assert_eq!(names.get(&(rd + 0x044)), Some(&("ns::Derived".to_string(), 0))); + + // Derived's hierarchy lists itself at index 0 and Foo at index 1. + let mut bases: Vec<_> = r.base_classes.iter() + .filter(|b| b.class_hierarchy == rd + 0x500) + .collect(); + bases.sort_by_key(|b| b.index); + assert_eq!(bases.len(), 2); + assert_eq!(bases[1].name, "ns::Foo"); + assert_eq!(bases[1].mdisp, 4); + assert_eq!(bases[1].pdisp, -1); + } + + #[test] + fn secondary_base_vftable_keeps_its_subobject_offset() { + let mut img = Image::new(); + let rd = BASE + RDATA_RVA; + let da = BASE + DATA_RVA; + emit_class(&mut img, da + 0x100, ".?AVMulti@@", + rd + 0x100, 0x8, rd + 0x200, rd + 0x280, rd + 0x300, None, + rd + 0x000); + + let r = analyze(&img.0, BASE, §ions()); + let names = r.vtable_class_names(); + assert_eq!(names.get(&(rd + 0x004)), Some(&("Multi".to_string(), 0x8))); + } + + #[test] + fn ignores_data_that_merely_looks_like_a_locator() { + // A 5-word run with signature 0 but a `pTypeDescriptor` that hits no + // descriptor must not be reported. + let mut img = Image::new(); + let rd = BASE + RDATA_RVA; + img.put_u32(rd + 0x100, 0); + img.put_u32(rd + 0x104, 0); + img.put_u32(rd + 0x108, 0); + img.put_u32(rd + 0x10C, BASE + DATA_RVA + 0x900); // no TD there + img.put_u32(rd + 0x110, rd + 0x200); + + let r = analyze(&img.0, BASE, §ions()); + assert!(r.locators.is_empty()); + assert!(r.type_descriptors.is_empty()); + } +} diff --git a/crates/xenia-analysis/src/sinks/duckdb.rs b/crates/xenia-analysis/src/sinks/duckdb.rs index 1d20b95..40db111 100644 --- a/crates/xenia-analysis/src/sinks/duckdb.rs +++ b/crates/xenia-analysis/src/sinks/duckdb.rs @@ -1,7 +1,8 @@ //! DuckDB sink — appends rich disasm items to the `instructions` table. //! //! Column layout matches [`crate::db`]: address, raw, mnemonic, operands, -//! disasm, ext_mnemonic, ext_operands, ext_disasm, section, function, label. +//! disasm, ext_mnemonic, ext_operands, ext_disasm, target_hex, section, +//! function, label, is_data. use duckdb::{Appender, params}; @@ -30,6 +31,7 @@ pub fn append_instructions<'a>( ri.section, ri.function.map(|f| f as i64), ri.label, + ri.is_data, ])?; count += 1; } diff --git a/crates/xenia-analysis/src/sinks/json.rs b/crates/xenia-analysis/src/sinks/json.rs index 2af660e..ea56da1 100644 --- a/crates/xenia-analysis/src/sinks/json.rs +++ b/crates/xenia-analysis/src/sinks/json.rs @@ -30,6 +30,7 @@ struct JsonRow<'a> { function: Option, #[serde(skip_serializing_if = "Option::is_none")] label: Option<&'a str>, + is_data: bool, } /// Write each item as a single JSON object on its own line. Returns the @@ -54,6 +55,7 @@ pub fn write_jsonl<'a, W: Write>( section: ri.section, function: ri.function, label: ri.label, + is_data: ri.is_data, }; serde_json::to_writer(&mut *out, &row)?; out.write_all(b"\n")?; diff --git a/crates/xenia-analysis/src/sinks/text.rs b/crates/xenia-analysis/src/sinks/text.rs index 008f6e4..2b7de34 100644 --- a/crates/xenia-analysis/src/sinks/text.rs +++ b/crates/xenia-analysis/src/sinks/text.rs @@ -25,6 +25,19 @@ pub fn write_instr_line( image_base: u32, data_annotation: Option<(u32, XrefKind)>, ) -> io::Result<()> { + // A word the analysis proved is data (a recovered jump table or its index + // map) must not be printed as if it decoded to something meaningful. + if item.is_data { + let lbl = labels.get(&item.item.raw) + .map(|s| format!(" ; -> {s}")) + .unwrap_or_default(); + return writeln!( + out, + " {:08X}: {:08X} .long 0x{:08X}{}", + item.item.addr, item.item.raw, item.item.raw, lbl, + ); + } + let disasm_text = item.item.text.display(); // Branch-target → label annotation. Uses the structured `branch_target` diff --git a/crates/xenia-analysis/src/sql_views.rs b/crates/xenia-analysis/src/sql_views.rs index fe27e2e..ddcef4e 100644 --- a/crates/xenia-analysis/src/sql_views.rs +++ b/crates/xenia-analysis/src/sql_views.rs @@ -23,6 +23,42 @@ //! kind-classification CASE drifted out of agreement with `xref.rs`, and //! is worth a one-line warning at log time. + +/// Every XDBF string side-by-side across the languages the title ships, so a +/// piece of UI text can be looked up once and read in all locales. +const V_XDBF_TEXT: &str = " +CREATE OR REPLACE VIEW v_xdbf_text AS +SELECT + s.string_id, + MAX(CASE WHEN s.language = 1 THEN s.value END) AS english, + MAX(CASE WHEN s.language = 2 THEN s.value END) AS japanese, + MAX(CASE WHEN s.language = 3 THEN s.value END) AS german, + MAX(CASE WHEN s.language = 4 THEN s.value END) AS french, + MAX(CASE WHEN s.language = 5 THEN s.value END) AS spanish, + MAX(CASE WHEN s.language = 6 THEN s.value END) AS italian +FROM xdbf_strings s +GROUP BY s.string_id; +"; + +/// Achievements joined to their three strings in every shipped language. +const V_XDBF_ACHIEVEMENTS: &str = " +CREATE OR REPLACE VIEW v_xdbf_achievements AS +SELECT + a.id, + a.gamerscore, + s.language, + s.language_name, + n.value AS name, + u.value AS unlocked_desc, + l.value AS locked_desc, + a.image_id +FROM xdbf_achievements a +JOIN (SELECT DISTINCT language, language_name FROM xdbf_strings) s ON TRUE +LEFT JOIN xdbf_strings n ON n.language = s.language AND n.string_id = a.label_id +LEFT JOIN xdbf_strings u ON u.language = s.language AND u.string_id = a.description_id +LEFT JOIN xdbf_strings l ON l.language = s.language AND l.string_id = a.unachieved_id; +"; + /// `(view_name, CREATE VIEW … SQL)` pairs in the order they must run. /// Later views may depend on earlier ones (e.g. `v_call_graph` reads /// `xrefs`, which is the Rust-pass table; `v_branch_xrefs` is independent). @@ -33,6 +69,12 @@ pub const ALL_VIEWS: &[(&str, &str)] = &[ ("v_indirect_reachability_from_entry", V_INDIRECT_REACHABILITY_FROM_ENTRY), ("v_function_first_instruction", V_FUNCTION_FIRST_INSTRUCTION), ("v_imports_called", V_IMPORTS_CALLED), + ("v_xdbf_text", V_XDBF_TEXT), + ("v_xdbf_achievements", V_XDBF_ACHIEVEMENTS), + ("v_switch_cases", V_SWITCH_CASES), + ("v_class_hierarchy", V_CLASS_HIERARCHY), + ("v_class_methods", V_CLASS_METHODS), + ("v_function_strings", V_FUNCTION_STRINGS), ]; /// Branch cross-references derived purely from `instructions.target_hex`. @@ -105,7 +147,7 @@ WITH RECURSIVE reach(fn) AS ( JOIN instructions src ON src.address = x.source JOIN instructions tgt ON tgt.address = x.target JOIN reach r ON src.function = r.fn - WHERE x.kind IN ('call', 'j', 'br') + WHERE x.kind IN ('call', 'j', 'br', 'jt') AND tgt.function IS NOT NULL ) SELECT fn AS addr FROM reach; @@ -128,7 +170,7 @@ WITH RECURSIVE reach(fn) AS ( JOIN instructions src ON src.address = x.source JOIN instructions tgt ON tgt.address = x.target JOIN reach r ON src.function = r.fn - WHERE x.kind IN ('call', 'ind_call', 'j', 'br') + WHERE x.kind IN ('call', 'ind_call', 'j', 'br', 'jt') AND tgt.function IS NOT NULL ) SELECT fn AS addr FROM reach; @@ -163,3 +205,81 @@ LEFT JOIN functions f ON f.address = x.source_func WHERE x.kind = 'call' AND l.kind = 'import'; "; + +/// Every recovered `switch` case, joined to the dispatching function and to +/// the label on the case body. One row per case *value* — several rows can +/// share a `target_address` when case values fall through to one body. +const V_SWITCH_CASES: &str = " +CREATE OR REPLACE VIEW v_switch_cases AS +SELECT + jt.bctr_pc AS dispatch_pc, + jt.function AS function_addr, + f.name AS function_name, + jt.kind AS table_kind, + jt.table_address AS table_address, + e.case_index AS case_index, + e.target_address AS target_address, + l.name AS target_label +FROM jump_tables jt +JOIN jump_table_entries e ON e.bctr_pc = jt.bctr_pc +LEFT JOIN functions f ON f.address = jt.function +LEFT JOIN labels l ON l.address = e.target_address; +"; + +/// The C++ inheritance graph as recovered from RTTI. Index 0 of a base-class +/// array is the class itself and is excluded, so every row is a genuine +/// `derived -> base` edge carrying the displacement triple needed to find the +/// base subobject inside an instance. +const V_CLASS_HIERARCHY: &str = " +CREATE OR REPLACE VIEW v_class_hierarchy AS +SELECT DISTINCT + dtd.demangled_name AS derived_class, + b.name AS base_class, + b.base_index AS base_index, + b.mdisp AS mdisp, + b.pdisp AS pdisp, + b.vdisp AS vdisp, + c.vtable_address AS derived_vtable +FROM rtti_base_classes b +JOIN rtti_locators c ON c.class_hierarchy = b.class_hierarchy +JOIN rtti_type_descriptors dtd ON dtd.address = c.type_descriptor +WHERE b.base_index > 0; +"; + +/// Virtual methods per class, resolved through the RTTI-named vtable. The +/// authoritative counterpart to querying `methods` by an `ANON_Class_*` name. +const V_CLASS_METHODS: &str = " +CREATE OR REPLACE VIEW v_class_methods AS +SELECT + td.demangled_name AS class_name, + c.subobject_offset AS subobject_offset, + v.address AS vtable_address, + m.slot AS slot, + m.function_address AS method_addr, + f.name AS method_name, + f.has_eh AS method_has_eh +FROM rtti_locators c +JOIN rtti_type_descriptors td ON td.address = c.type_descriptor +JOIN vtables v ON v.address = c.vtable_address +JOIN methods m ON m.vtable_address = v.address +LEFT JOIN functions f ON f.address = m.function_address; +"; + +/// Which function references which string literal. The single most useful +/// orientation query in a stripped binary: it is how you find the code behind +/// a message you can see on screen. +const V_FUNCTION_STRINGS: &str = " +CREATE OR REPLACE VIEW v_function_strings AS +SELECT + x.source_func AS function_addr, + f.name AS function_name, + x.source AS reference_pc, + x.kind AS reference_kind, + s.address AS string_addr, + s.encoding AS encoding, + s.content AS content +FROM xrefs x +JOIN strings s ON s.address = x.target +LEFT JOIN functions f ON f.address = x.source_func +WHERE x.kind IN ('ref', 'read'); +"; diff --git a/crates/xenia-analysis/src/static_init.rs b/crates/xenia-analysis/src/static_init.rs index 02b3e90..c1701ef 100644 --- a/crates/xenia-analysis/src/static_init.rs +++ b/crates/xenia-analysis/src/static_init.rs @@ -347,7 +347,7 @@ mod tests { functions.insert(driver, FuncInfo { start: driver, end: driver + 0x40, frame_size: 0, saved_gprs: 0, is_leaf: false, is_saverestore: false, - pdata_validated: false, pdata_length: None, has_eh: false, + pdata_validated: false, pdata_length: None, pdata_prolog_length: None, has_eh: false, }); let fa = FuncAnalysis { functions, save_gpr_base: None, restore_gpr_base: None, pdata_entries: Vec::new(), @@ -385,7 +385,7 @@ mod tests { functions.insert(driver, FuncInfo { start: driver, end: driver + 0x40, frame_size: 0, saved_gprs: 0, is_leaf: true, is_saverestore: false, - pdata_validated: false, pdata_length: None, has_eh: false, + pdata_validated: false, pdata_length: None, pdata_prolog_length: None, has_eh: false, }); let fa = FuncAnalysis { functions, save_gpr_base: None, restore_gpr_base: None, pdata_entries: Vec::new(), diff --git a/crates/xenia-analysis/src/strings.rs b/crates/xenia-analysis/src/strings.rs index 666a62b..17910fb 100644 --- a/crates/xenia-analysis/src/strings.rs +++ b/crates/xenia-analysis/src/strings.rs @@ -1,4 +1,4 @@ -//! String / constant-pool detection in `.rdata`. +//! String / constant-pool detection in the initialised data sections. //! //! Scans the `.rdata` section for runs of printable ASCII or null-terminated //! UTF-16LE characters of length ≥ 6, emitting one row per discovered string. @@ -9,7 +9,8 @@ //! //! - No UTF-8 multibyte detection — Xbox 360 game binaries reliably use //! ASCII for debug strings and UTF-16LE for localised text. -//! - Strings in `.data` (mutable globals) are not scanned by default. +//! - Only the file-backed part of a section is scanned: the tail of `.data` +//! past `raw_size` is BSS and contains nothing but zeros at rest. //! - Wide strings on Xbox 360 are little-endian (compiler convention even //! on this big-endian platform); we do NOT try big-endian UTF-16. //! - No language detection / classification beyond encoding. @@ -34,27 +35,40 @@ pub struct DetectedString { pub length: u32, /// UTF-8 representation of the string content. pub content: String, + /// Name of the PE section the string lives in (`.rdata` / `.data`). + pub section: String, } -/// Scan all `.rdata` sections (and any other read-only data section the user -/// configures) for ASCII and UTF-16LE strings. +/// Scan the initialised data sections for ASCII / UTF-16LE / Shift_JIS / UTF-8 +/// strings. +/// +/// `.data` is scanned as well as `.rdata`: a lot of a game's string material — +/// mutable tables, and every RTTI type-descriptor name — lives there, and +/// leaving it out is why this table comes back nearly empty on real titles. +/// The `section` column lets a consumer separate the two again. #[tracing::instrument(skip_all, fields(image_base = format_args!("{:#010x}", image_base)))] pub fn analyze(pe: &[u8], image_base: u32, sections: &[PeSection]) -> Vec { let started = std::time::Instant::now(); let mut out: Vec = Vec::new(); for section in sections { - if section.name != ".rdata" { continue; } + if !matches!(section.name.as_str(), ".rdata" | ".data") { continue; } let raw_start = section.virtual_address as usize; - let raw_end = (section.virtual_address + section.virtual_size) as usize; - if raw_end > pe.len() { continue; } - let bytes = &pe[raw_start..raw_end.min(pe.len())]; + // Clamp to the file-backed extent — everything past `raw_size` is BSS. + let backed = section.virtual_size.min(section.raw_size) as usize; + let raw_end = (raw_start + backed).min(pe.len()); + if raw_start >= raw_end { continue; } + let bytes = &pe[raw_start..raw_end]; let va_base = image_base + section.virtual_address; + let before = out.len(); scan_ascii(bytes, va_base, &mut out); scan_utf16le(bytes, va_base, &mut out); scan_shift_jis(bytes, va_base, &mut out); scan_utf8(bytes, va_base, &mut out); + for s in &mut out[before..] { + s.section = section.name.clone(); + } } let elapsed_ms = started.elapsed().as_millis() as f64; @@ -100,6 +114,7 @@ fn scan_ascii(bytes: &[u8], va_base: u32, out: &mut Vec) { encoding: "ascii", length: run_len as u32, content: s.to_string(), + section: String::new(), }); } // Skip the NUL (if any) before continuing. @@ -139,6 +154,7 @@ fn scan_utf16le(bytes: &[u8], va_base: u32, out: &mut Vec) { encoding: "utf16le", length: ((i - start) as u32), content: s, + section: String::new(), }); } // Skip past the terminator. @@ -146,23 +162,82 @@ fn scan_utf16le(bytes: &[u8], va_base: u32, out: &mut Vec) { } } -/// Per JIS X 0208: Shift_JIS first byte ∈ [0x81, 0x9F] ∪ [0xE0, 0xEF]; -/// trail byte ∈ [0x40, 0x7E] ∪ [0x80, 0xFC]. Single-byte ASCII and JIS -/// half-width katakana (0xA1..=0xDF) are passed through. +/// Per JIS X 0208: Shift_JIS lead byte is [0x81, 0x9F] u [0xE0, 0xEF]; +/// trail byte is [0x40, 0x7E] u [0x80, 0xFC]. +/// +/// Half-width katakana (0xA1..=0xDF) is deliberately *not* accepted as string +/// content. It is legal Shift_JIS, but this binary's Japanese text never uses +/// it, while 0xA1..=0xDF is extremely common in the float and pointer tables +/// that share `.rdata` — admitting it turned the scan into a noise generator +/// (837 detections, of which the overwhelming majority were IEEE-754 arrays: +/// `3f 66 66 66` = 0.9f reads as "fff"). fn is_sjis_lead(b: u8) -> bool { (0x81..=0x9F).contains(&b) || (0xE0..=0xEF).contains(&b) } fn is_sjis_trail(b: u8) -> bool { (0x40..=0x7E).contains(&b) || (0x80..=0xFC).contains(&b) } -fn is_sjis_singlebyte(b: u8) -> bool { - is_printable_ascii(b) || (0xA1..=0xDF).contains(&b) + +/// A character that can plausibly appear in a Japanese debug/UI string: +/// printable ASCII, CJK punctuation and kana, CJK ideographs, or full-width +/// ASCII. +fn is_text_like(ch: char) -> bool { + let o = ch as u32; + matches!(o, 0x20..=0x7E) + || matches!(ch, '\t' | '\n' | '\r') + || is_wide(ch) } -/// Scan for Shift_JIS strings — runs of ≥ 6 bytes consisting of valid -/// SJIS code units (single-byte ASCII / half-width katakana, OR a -/// lead+trail pair). At least one multi-byte pair must be present so we -/// don't double-count strings that are purely ASCII. +/// A full-width character — kana, CJK punctuation, ideograph, or full-width +/// ASCII. Used to tell "real text" from a lucky byte pair. +fn is_wide(ch: char) -> bool { + let o = ch as u32; + (0x3000..=0x30FF).contains(&o) || (0x4E00..=0x9FFF).contains(&o) || (0xFF01..=0xFF5E).contains(&o) +} + +/// True when `t` contains a lone ASCII character with a full-width character +/// on *both* sides. +/// +/// This is the Shift_JIS resynchronisation signal. A scan that starts one byte +/// early pairs the wrong lead with the wrong trail and typically produces a +/// stray kanji plus an orphaned ASCII letter before the real text resumes: +/// the run at 0x820a4b9f decodes as `帥Vステムマネージャ開始` when the actual +/// string is `システムマネージャ開始` at 0x820a4ba0. Genuine text mixes ASCII in +/// *runs* (`render_stateスタックオーバーフロー`, `size=%d`), never as a single +/// character wedged between two wide ones. +fn has_isolated_ascii(t: &str) -> bool { + let chars: Vec = t.chars().collect(); + (1..chars.len().saturating_sub(1)).any(|k| { + !is_wide(chars[k]) && is_wide(chars[k - 1]) && is_wide(chars[k + 1]) + }) +} + +/// Decode `raw` as Shift_JIS, rejecting anything that is not convincingly +/// Japanese text. Returns the UTF-8 form on success. +fn decode_sjis(raw: &[u8]) -> Option { + let (text, _, had_errors) = encoding_rs::SHIFT_JIS.decode(raw); + if had_errors { + return None; + } + let t = text.into_owned(); + // Require real kana somewhere. Arbitrary binary readily decodes to + // obscure kanji, but hiragana/katakana (U+3040..U+30FF) essentially never + // appear by accident and are ubiquitous in genuine Japanese. + let has_kana = t.chars().any(|c| ('\u{3040}'..='\u{30FF}').contains(&c)); + if t.chars().count() >= 4 && has_kana && t.chars().all(is_text_like) && !has_isolated_ascii(&t) { + Some(t) + } else { + None + } +} + +/// Scan for Shift_JIS strings — NUL-terminated runs of >= `MIN_LEN` bytes made +/// of printable ASCII and valid lead+trail pairs, with at least one pair. +/// +/// Each accepted run is *resynchronised*: the emitted string starts at the +/// earliest offset within the run whose full decode passes [`decode_sjis`], so +/// a run that begins mid-character reports the true string address rather than +/// a mangled one. fn scan_shift_jis(bytes: &[u8], va_base: u32, out: &mut Vec) { let mut i = 0; while i < bytes.len() { @@ -175,47 +250,29 @@ fn scan_shift_jis(bytes: &[u8], va_base: u32, out: &mut Vec) { has_multibyte = true; nbytes += 2; i += 2; - } else if is_sjis_singlebyte(b) { + } else if is_printable_ascii(b) { nbytes += 1; i += 1; } else { break; } } - // Require NUL terminator + min length + at least one multi-byte char. - if has_multibyte - && nbytes >= MIN_LEN - && i < bytes.len() && bytes[i] == 0 - { - // Decode SJIS → UTF-8 best-effort. We don't ship a full - // SJIS decoder; keep the bytes as a `\u{XX}\u{YY}…` style - // rendering for diagnostic readability, and let downstream - // tooling re-decode if needed. - let raw = &bytes[start..i]; - let mut s = String::with_capacity(raw.len() * 4); - let mut p = 0; - while p < raw.len() { - let b = raw[p]; - if is_sjis_lead(b) && p + 1 < raw.len() && is_sjis_trail(raw[p + 1]) { - // Render as SJIS hex pair so the string is identifiable - // even without a decoder. Real Japanese decoding is a - // future enhancement. - s.push_str(&format!("\\x{:02X}\\x{:02X}", b, raw[p + 1])); - p += 2; - } else { - s.push(b as char); - p += 1; + let end = i; + if has_multibyte && nbytes >= MIN_LEN && end < bytes.len() && bytes[end] == 0 { + for s in start..end { + if let Some(text) = decode_sjis(&bytes[s..end]) { + out.push(DetectedString { + address: va_base + s as u32, + encoding: "shift_jis", + length: (end - s) as u32, + content: text, + section: String::new(), + }); + break; } } - out.push(DetectedString { - address: va_base + start as u32, - encoding: "shift_jis", - length: nbytes as u32, - content: s, - }); - i += 1; // skip NUL + i = end + 1; // skip NUL } else { - // Advance past whatever didn't match. i = start + 1; if i < bytes.len() && bytes[i] == 0 { i += 1; } } @@ -265,6 +322,7 @@ fn scan_utf8(bytes: &[u8], va_base: u32, out: &mut Vec) { encoding: "utf8", length: nbytes as u32, content: s.to_string(), + section: String::new(), }); i += 1; // skip NUL } else { @@ -339,15 +397,54 @@ mod tests { let image_base = 0x82000000u32; let mut pe = vec![0u8; 0x1100]; let off = 0x1000usize; - // "ABC" + (SJIS hiragana 'a' = 0x82 0xA0) + (SJIS 'i' = 0x82 0xA2) + NUL + // "ABC" + SJIS hiragana あ (0x82 0xA0) + い (0x82 0xA2) + NUL. let s: &[u8] = b"ABC\x82\xA0\x82\xA2\0"; pe[off..off + s.len()].copy_from_slice(s); let sections = vec![mk_section(".rdata", 0x1000, 0x100)]; let strings = analyze(&pe, image_base, §ions); let sjis: Vec<_> = strings.iter().filter(|s| s.encoding == "shift_jis").collect(); assert_eq!(sjis.len(), 1); - assert!(sjis[0].content.contains("ABC")); - assert!(sjis[0].content.contains("\\x82\\xA0")); + // Decoded to real UTF-8, not rendered as escaped bytes. + assert_eq!(sjis[0].content, "ABCあい"); + assert_eq!(sjis[0].address, image_base + 0x1000); + } + + #[test] + fn shift_jis_rejects_float_table_noise() { + // Four IEEE-754 floats (0.85, 0.9, 0.8, 0.7). Every byte satisfies the + // Shift_JIS lead/trail ranges, so the byte-range test alone accepts it. + let image_base = 0x82000000u32; + let mut pe = vec![0u8; 0x1100]; + let off = 0x1000usize; + let s: &[u8] = b"\x3f\x59\x99\x9a\x3f\x66\x66\x66\x3f\x4c\xcc\xcd\x3f\x33\x33\x33\0"; + pe[off..off + s.len()].copy_from_slice(s); + let sections = vec![mk_section(".rdata", 0x1000, 0x100)]; + let strings = analyze(&pe, image_base, §ions); + assert!(strings.iter().all(|s| s.encoding != "shift_jis"), + "float table must not be reported as Japanese text"); + } + + #[test] + fn shift_jis_resynchronises_to_true_start() { + // Mirrors 0x820a4b9f in the reference title: binary data runs straight + // into a real string, and a naive forward scan mis-pairs the boundary + // byte, yielding `帥Vステム…` one byte early instead of `システム…`. + let image_base = 0x82000000u32; + let mut pe = vec![0u8; 0x1100]; + let off = 0x1000usize; + // Exact bytes from that site: a trailing 0x90 from the preceding + // float pairs with the string's first byte (0x83) to form 帥, which + // orphans the 0x56 as an ASCII 'V' before the text resumes. + // 0x90 シ ス テ ム + let s: &[u8] = b"\x90\x83\x56\x83\x58\x83\x65\x83\x80\0"; + pe[off..off + s.len()].copy_from_slice(s); + let sections = vec![mk_section(".rdata", 0x1000, 0x100)]; + let strings = analyze(&pe, image_base, §ions); + let sjis: Vec<_> = strings.iter().filter(|s| s.encoding == "shift_jis").collect(); + assert_eq!(sjis.len(), 1); + assert_eq!(sjis[0].content, "システム"); + // Reported at the true start, one byte past the run's beginning. + assert_eq!(sjis[0].address, image_base + 0x1000 + 1); } #[test] diff --git a/crates/xenia-analysis/src/vtables.rs b/crates/xenia-analysis/src/vtables.rs index 35d19b0..f27f619 100644 --- a/crates/xenia-analysis/src/vtables.rs +++ b/crates/xenia-analysis/src/vtables.rs @@ -101,12 +101,22 @@ pub fn analyze_with_anchors( .filter(|s| matches!(s.name.as_str(), ".rdata" | ".data")) .collect(); - // Range table for "is this VA in .rdata or .data?" + // Range table for "is this VA in .rdata?" — where COLs and class-hierarchy + // descriptors live. let rdata_ranges: Vec<(u32, u32)> = sections .iter() .filter(|s| s.name == ".rdata") .map(|s| (image_base + s.virtual_address, image_base + s.virtual_address + s.virtual_size)) .collect(); + // TypeDescriptors are *written at startup* (their first word is + // `type_info`'s vftable), so MSVC emits them into writable `.data`, not + // `.rdata`. Range-checking a TypeDescriptor pointer against `.rdata` alone + // rejects every one of them and leaves the whole inline walk dead. + let typedesc_ranges: Vec<(u32, u32)> = sections + .iter() + .filter(|s| matches!(s.name.as_str(), ".rdata" | ".data")) + .map(|s| (image_base + s.virtual_address, image_base + s.virtual_address + s.virtual_size)) + .collect(); let mut candidates: Vec = Vec::new(); @@ -280,7 +290,7 @@ pub fn analyze_with_anchors( // Try to extract the TypeDescriptor mangled-name string. if let Some((td_ptr, hierarchy_ptr)) = read_col(pe, image_base, col_ptr) - && let Some(mangled) = read_typedescriptor_name(pe, image_base, td_ptr, &rdata_ranges) + && let Some(mangled) = read_typedescriptor_name(pe, image_base, td_ptr, &typedesc_ranges) && let Some(class) = demangle_rtti_typename(&mangled) { v.col_address = Some(col_ptr); @@ -771,3 +781,61 @@ mod tests { assert_eq!(vtables.len(), 0, "runs of 2 must be rejected to keep false-positive rate down"); } } + +// ── RTTI relabelling ─────────────────────────────────────────────────────── + +/// Overwrite heuristic vtable identity with the authoritative RTTI walk. +/// +/// [`analyze_with_anchors`] names a table either from its own inline COL walk +/// or, failing that, with a synthetic `ANON_Class_`. [`crate::rtti`] +/// resolves the same question top-down from the structures the linker emitted, +/// which is exact — so wherever the two disagree, RTTI wins. Rows RTTI knows +/// nothing about keep their heuristic name. +/// +/// `base_classes_json` is rebuilt here as the class's full linearised base list +/// (excluding index 0, which is the class itself), which is strictly more than +/// the first-level list the inline walk produced. +/// +/// Returns the number of vtables that gained a real class name. +pub fn apply_rtti_names(vtables: &mut [Vtable], rtti: &crate::rtti::RttiResult) -> usize { + use std::collections::BTreeMap; + + let names = rtti.vtable_class_names(); + let locator_by_vtable: BTreeMap = rtti + .locators + .iter() + .filter_map(|c| c.vtable_address.map(|v| (v, c))) + .collect(); + + // class-hierarchy VA → base class names, in the linker's order. + let mut bases_by_chd: BTreeMap> = BTreeMap::new(); + for b in &rtti.base_classes { + if b.index == 0 { continue; } // index 0 is the class itself + bases_by_chd.entry(b.class_hierarchy).or_default().push(b.name.as_str()); + } + + let mut named = 0usize; + for vt in vtables.iter_mut() { + let Some((class_name, offset)) = names.get(&vt.address) else { continue }; + // A secondary-base vftable belongs to the same class but is a distinct + // table; keep them apart by suffixing the subobject offset. + vt.class_name = if *offset == 0 { + class_name.clone() + } else { + format!("{class_name}#base+0x{offset:X}") + }; + vt.rtti_present = true; + if let Some(col) = locator_by_vtable.get(&vt.address) { + vt.col_address = Some(col.address); + vt.base_classes_json = bases_by_chd.get(&col.class_hierarchy).map(|names| { + let items: Vec = names + .iter() + .map(|n| format!("\"{}\"", n.replace('\\', "\\\\").replace('"', "\\\""))) + .collect(); + format!("[{}]", items.join(",")) + }); + } + named += 1; + } + named +} diff --git a/crates/xenia-analysis/src/xdbf.rs b/crates/xenia-analysis/src/xdbf.rs new file mode 100644 index 0000000..3811000 --- /dev/null +++ b/crates/xenia-analysis/src/xdbf.rs @@ -0,0 +1,450 @@ +//! XDBF / SPA — the title metadata package embedded in the XEX. +//! +//! A title's `XEX_HEADER_RESOURCE_INFO` names one resource whose body is an +//! **XDBF** ("Xbox DataBase File") container, in its SPA flavour: achievement +//! definitions, one string table per shipped language, PNG images, and the +//! matchmaking / leaderboard / presence schema. +//! +//! ```text +//! XdbfHeader 24 bytes magic 'XDBF', version, entry_count, entry_used, +//! free_count, free_used +//! XdbfEntry[] 18 each namespace u16, id u64, offset u32, size u32 +//! XdbfFileLoc[] 8 each the free-space table +//! data entry offsets are relative to the end of the two tables +//! ``` +//! +//! Each entry's body starts with a section header — `magic, version, size`, +//! plus a `u16 count` for the table-shaped ones. +//! +//! Entries are enumerated from the **entry table**, not by scanning for section +//! magics. Scanning is what the project's earlier `tools/xach_dump.py` does, and +//! on this title it finds a phantom seventh `XSTR` (the byte pattern occurs +//! outside any declared entry) where the entry table declares six — which shifts +//! every language index derived from the scan order. +//! +//! Layouts follow the reference implementation in xenia-canary +//! (`src/xenia/kernel/xam/xdbf/{xdbf_io,spa_info}.h`), which in turn cites +//! freestyledash `Tools/XEX/SPA.{h,cpp}`. + +/// `XDBF` big-endian. +const XDBF_MAGIC: u32 = 0x5844_4246; + +/// The well-known entry id carrying the title's own name (in the string-table +/// namespace) and its icon (in the image namespace) — canary's `kXdbfIdTitle`. +pub const ID_TITLE: u64 = 0x8000; + +const NS_METADATA: u16 = 1; +const NS_IMAGE: u16 = 2; +const NS_STRING_TABLE: u16 = 3; + +/// One row of the container's entry table. +#[derive(Debug, Clone)] +pub struct XdbfEntry { + /// 1 = metadata, 2 = image, 3 = string table. + pub namespace: u16, + /// Entry id. For metadata entries this is the section fourcc as an integer; + /// for string tables it is the [`XLanguage`] value; for images, the image id. + pub id: u64, + /// Absolute offset of the entry body within the image buffer. + pub offset: usize, + /// Entry body length in bytes. + pub size: usize, + /// The body's leading fourcc, when it has one (`XACH`, `XSTR`, …). + pub magic: Option, +} + +/// One achievement definition (`XACH`, 36-byte records). +#[derive(Debug, Clone)] +pub struct Achievement { + pub id: u16, + /// String id of the achievement's name. + pub label_id: u16, + /// String id of the description shown once unlocked. + pub description_id: u16, + /// String id of the description shown while locked. + pub unachieved_id: u16, + pub image_id: u32, + pub gamerscore: u16, + pub flags: u32, +} + +/// One localized string table (`XSTR`). +#[derive(Debug, Clone)] +pub struct StringTable { + /// `XLanguage` value; the entry id. + pub language: u32, + /// `(string id, value)` in table order. + pub strings: Vec<(u16, String)>, +} + +/// `XTHD` — the title header. +#[derive(Debug, Clone, Copy)] +pub struct TitleHeader { + pub title_id: u32, + pub title_type: u32, + pub major: u16, + pub minor: u16, + pub build: u16, + pub revision: u16, + pub flags: u32, +} + +/// An embedded image (namespace 2). Bodies are raw files, in practice PNG. +#[derive(Debug, Clone)] +pub struct Image { + pub id: u64, + pub offset: usize, + pub size: usize, + /// `"png"` when the body carries the PNG signature, else `"unknown"`. + pub format: &'static str, +} + +/// Everything recovered from one XDBF package. +#[derive(Debug, Default)] +pub struct Xdbf { + /// Offset of the container within the image buffer. + pub base: usize, + pub version: u32, + pub entries: Vec, + pub achievements: Vec, + pub string_tables: Vec, + pub images: Vec, + pub title: Option, + /// `XSTC` default language (an `XLanguage` value). + pub default_language: Option, +} + +fn be16(b: &[u8], o: usize) -> Option { + Some(u16::from_be_bytes([*b.get(o)?, *b.get(o + 1)?])) +} +fn be32(b: &[u8], o: usize) -> Option { + Some(u32::from_be_bytes([ + *b.get(o)?, *b.get(o + 1)?, *b.get(o + 2)?, *b.get(o + 3)?, + ])) +} +fn be64(b: &[u8], o: usize) -> Option { + let hi = be32(b, o)? as u64; + let lo = be32(b, o + 4)? as u64; + Some((hi << 32) | lo) +} + +/// Render a fourcc as text when all four bytes are printable ASCII. +fn fourcc(v: u32) -> Option { + let b = v.to_be_bytes(); + b.iter() + .all(|c| (0x20..0x7F).contains(c)) + .then(|| String::from_utf8_lossy(&b).into_owned()) +} + +/// Human-readable name for an `XLanguage` value. +pub fn language_name(v: u32) -> &'static str { + match v { + 1 => "English", + 2 => "Japanese", + 3 => "German", + 4 => "French", + 5 => "Spanish", + 6 => "Italian", + 7 => "Korean", + 8 => "Chinese (Traditional)", + 9 => "Portuguese", + 10 => "Chinese (Simplified)", + 11 => "Polish", + 12 => "Russian", + _ => "unknown", + } +} + +/// Parse the XDBF package at `base` within `image`. +/// +/// Returns `None` when there is no XDBF magic there — callers locate the +/// package via `xenia_xex::resources`, and a title without one is normal. +#[tracing::instrument(skip_all, fields(base = format_args!("{base:#x}")))] +pub fn analyze(image: &[u8], base: usize) -> Option { + let started = std::time::Instant::now(); + if be32(image, base)? != XDBF_MAGIC { + return None; + } + let version = be32(image, base + 4)?; + let entry_count = be32(image, base + 8)? as usize; + let entry_used = be32(image, base + 12)? as usize; + let free_count = be32(image, base + 16)? as usize; + + // Guard against a corrupt header pointing the data region off the end. + if entry_used > entry_count || entry_count > 0x10000 || free_count > 0x10000 { + return None; + } + let entry_table = base + 24; + let data_start = entry_table + entry_count * 18 + free_count * 8; + if data_start > image.len() { + return None; + } + + let mut out = Xdbf { + base, + version, + ..Default::default() + }; + + for i in 0..entry_used { + let p = entry_table + i * 18; + let (Some(namespace), Some(id), Some(off), Some(size)) = + (be16(image, p), be64(image, p + 2), be32(image, p + 10), be32(image, p + 14)) + else { + continue; + }; + let body = data_start + off as usize; + let size = size as usize; + if body + size > image.len() { + continue; + } + let magic = be32(image, body).and_then(fourcc); + out.entries.push(XdbfEntry { + namespace, + id, + offset: body, + size, + magic: magic.clone(), + }); + + match namespace { + NS_IMAGE => out.images.push(Image { + id, + offset: body, + size, + format: if image[body..].starts_with(b"\x89PNG") { "png" } else { "unknown" }, + }), + NS_STRING_TABLE => { + if let Some(t) = parse_string_table(image, body, size, id as u32) { + out.string_tables.push(t); + } + } + NS_METADATA => match magic.as_deref() { + Some("XACH") => out.achievements.extend(parse_achievements(image, body, size)), + Some("XTHD") => out.title = parse_title_header(image, body), + Some("XSTC") => out.default_language = be32(image, body + 12), + _ => {} + }, + _ => {} + } + } + + metrics::histogram!("analysis.phase_ms", "phase" => "xdbf") + .record(started.elapsed().as_millis() as f64); + tracing::info!( + entries = out.entries.len(), + achievements = out.achievements.len(), + string_tables = out.string_tables.len(), + images = out.images.len(), + default_language = out.default_language, + "XDBF package parsed", + ); + Some(out) +} + +/// `XACH`: `magic, version, size, count u16`, then 36-byte records. +fn parse_achievements(image: &[u8], body: usize, size: usize) -> Vec { + let Some(count) = be16(image, body + 12) else { return Vec::new() }; + let mut out = Vec::with_capacity(count as usize); + for i in 0..count as usize { + let p = body + 14 + i * 36; + if p + 36 > body + size { + break; + } + let (Some(id), Some(label_id), Some(description_id), Some(unachieved_id)) = + (be16(image, p), be16(image, p + 2), be16(image, p + 4), be16(image, p + 6)) + else { + break; + }; + out.push(Achievement { + id, + label_id, + description_id, + unachieved_id, + image_id: be32(image, p + 8).unwrap_or(0), + gamerscore: be16(image, p + 12).unwrap_or(0), + flags: be32(image, p + 16).unwrap_or(0), + }); + } + out +} + +/// `XSTR`: `magic, version, size, count u16`, then `id u16, len u16, bytes`. +/// +/// Bodies are UTF-8 (the ASCII subset for most locales; Japanese uses the full +/// range), decoded lossily so one bad table cannot drop a whole language. +fn parse_string_table(image: &[u8], body: usize, size: usize, language: u32) -> Option { + if fourcc(be32(image, body)?)? != "XSTR" { + return None; + } + let count = be16(image, body + 12)?; + let end = body + size; + let mut p = body + 14; + let mut strings = Vec::with_capacity(count as usize); + for _ in 0..count { + let (Some(id), Some(len)) = (be16(image, p), be16(image, p + 2)) else { break }; + let s = p + 4; + let e = s + len as usize; + if e > end || e > image.len() { + break; + } + strings.push((id, String::from_utf8_lossy(&image[s..e]).into_owned())); + p = e; + } + Some(StringTable { language, strings }) +} + +/// `XTHD`: section header then the 32-byte `TitleHeaderData`. +fn parse_title_header(image: &[u8], body: usize) -> Option { + let p = body + 12; + Some(TitleHeader { + title_id: be32(image, p)?, + title_type: be32(image, p + 4)?, + major: be16(image, p + 8)?, + minor: be16(image, p + 10)?, + build: be16(image, p + 12)?, + revision: be16(image, p + 14)?, + flags: be32(image, p + 16)?, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a minimal XDBF: one XACH with a single achievement, one XSTR, one + /// PNG, an XTHD and an XSTC. + fn mk_xdbf() -> (Vec, usize) { + let base = 0x100usize; + let entry_count = 5usize; + let free_count = 1usize; + let data_start = base + 24 + entry_count * 18 + free_count * 8; + + let mut bodies: Vec<(u16, u64, Vec)> = Vec::new(); + + let mut xach = Vec::new(); + xach.extend(b"XACH"); + xach.extend(1u32.to_be_bytes()); + xach.extend(0u32.to_be_bytes()); + xach.extend(1u16.to_be_bytes()); // count + let mut rec = Vec::new(); + rec.extend(7u16.to_be_bytes()); // id + rec.extend(100u16.to_be_bytes()); // label + rec.extend(101u16.to_be_bytes()); // description + rec.extend(102u16.to_be_bytes()); // unachieved + rec.extend(9u32.to_be_bytes()); // image id + rec.extend(20u16.to_be_bytes()); // gamerscore + rec.extend(0u16.to_be_bytes()); + rec.extend(0x0Cu32.to_be_bytes()); // flags + rec.extend([0u8; 16]); + assert_eq!(rec.len(), 36); + xach.extend(rec); + bodies.push((NS_METADATA, u32::from_be_bytes(*b"XACH") as u64, xach)); + + let mut xstr = Vec::new(); + xstr.extend(b"XSTR"); + xstr.extend(1u32.to_be_bytes()); + xstr.extend(0u32.to_be_bytes()); + xstr.extend(2u16.to_be_bytes()); + for (id, s) in [(100u16, "Space Combat Award"), (101u16, "Well done")] { + xstr.extend(id.to_be_bytes()); + xstr.extend((s.len() as u16).to_be_bytes()); + xstr.extend(s.as_bytes()); + } + bodies.push((NS_STRING_TABLE, 1, xstr)); // language 1 = English + + let mut xthd = Vec::new(); + xthd.extend(b"XTHD"); + xthd.extend(1u32.to_be_bytes()); + xthd.extend(0u32.to_be_bytes()); + xthd.extend(0x5351_07D4u32.to_be_bytes()); // title id + xthd.extend(1u32.to_be_bytes()); // type = full + xthd.extend(1u16.to_be_bytes()); + xthd.extend(2u16.to_be_bytes()); + xthd.extend(3u16.to_be_bytes()); + xthd.extend(4u16.to_be_bytes()); + xthd.extend(0u32.to_be_bytes()); + bodies.push((NS_METADATA, u32::from_be_bytes(*b"XTHD") as u64, xthd)); + + let mut xstc = Vec::new(); + xstc.extend(b"XSTC"); + xstc.extend(1u32.to_be_bytes()); + xstc.extend(16u32.to_be_bytes()); + xstc.extend(1u32.to_be_bytes()); // default language = English + bodies.push((NS_METADATA, u32::from_be_bytes(*b"XSTC") as u64, xstc)); + + let png = b"\x89PNG\r\n\x1a\n----".to_vec(); + bodies.push((NS_IMAGE, 9, png)); + + let total: usize = bodies.iter().map(|(_, _, b)| b.len()).sum(); + let mut img = vec![0u8; data_start + total + 0x10]; + img[base..base + 4].copy_from_slice(&XDBF_MAGIC.to_be_bytes()); + img[base + 4..base + 8].copy_from_slice(&0x10000u32.to_be_bytes()); + img[base + 8..base + 12].copy_from_slice(&(entry_count as u32).to_be_bytes()); + img[base + 12..base + 16].copy_from_slice(&(bodies.len() as u32).to_be_bytes()); + img[base + 16..base + 20].copy_from_slice(&(free_count as u32).to_be_bytes()); + + let mut off = 0usize; + for (i, (ns, id, b)) in bodies.iter().enumerate() { + let p = base + 24 + i * 18; + img[p..p + 2].copy_from_slice(&ns.to_be_bytes()); + img[p + 2..p + 10].copy_from_slice(&id.to_be_bytes()); + img[p + 10..p + 14].copy_from_slice(&(off as u32).to_be_bytes()); + img[p + 14..p + 18].copy_from_slice(&(b.len() as u32).to_be_bytes()); + img[data_start + off..data_start + off + b.len()].copy_from_slice(b); + off += b.len(); + } + (img, base) + } + + #[test] + fn parses_container_via_entry_table() { + let (img, base) = mk_xdbf(); + let x = analyze(&img, base).expect("parses"); + assert_eq!(x.entries.len(), 5); + assert_eq!(x.achievements.len(), 1); + assert_eq!(x.string_tables.len(), 1); + assert_eq!(x.images.len(), 1); + assert_eq!(x.default_language, Some(1)); + } + + #[test] + fn achievement_fields_and_string_ids_line_up() { + let (img, base) = mk_xdbf(); + let x = analyze(&img, base).unwrap(); + let a = &x.achievements[0]; + assert_eq!((a.id, a.gamerscore, a.image_id, a.flags), (7, 20, 9, 0x0C)); + let t = &x.string_tables[0]; + assert_eq!(t.language, 1); + assert_eq!(t.strings[0], (100, "Space Combat Award".to_string())); + // The achievement's label resolves through the table. + let name = t.strings.iter().find(|(i, _)| *i == a.label_id).map(|(_, s)| s.as_str()); + assert_eq!(name, Some("Space Combat Award")); + } + + #[test] + fn title_header_and_image_format() { + let (img, base) = mk_xdbf(); + let x = analyze(&img, base).unwrap(); + let t = x.title.expect("XTHD"); + assert_eq!(t.title_id, 0x5351_07D4); + assert_eq!((t.major, t.minor, t.build, t.revision), (1, 2, 3, 4)); + assert_eq!(x.images[0].format, "png"); + assert_eq!(x.images[0].id, 9); + } + + #[test] + fn rejects_non_xdbf() { + let img = vec![0u8; 0x200]; + assert!(analyze(&img, 0x100).is_none()); + } + + #[test] + fn rejects_header_pointing_past_the_buffer() { + let mut img = vec![0u8; 0x200]; + img[0..4].copy_from_slice(&XDBF_MAGIC.to_be_bytes()); + img[8..12].copy_from_slice(&0xFFFFu32.to_be_bytes()); // entry_count + img[12..16].copy_from_slice(&0xFFFFu32.to_be_bytes()); // entry_used + assert!(analyze(&img, 0).is_none()); + } +} diff --git a/crates/xenia-analysis/src/xref.rs b/crates/xenia-analysis/src/xref.rs index 666653a..d5a9c27 100644 --- a/crates/xenia-analysis/src/xref.rs +++ b/crates/xenia-analysis/src/xref.rs @@ -10,6 +10,7 @@ use crate::func::FuncAnalysis; pub enum XrefKind { Call, // bl IndirectCall, // bcctrl through a statically-resolvable vtable slot (M5) + JumpTable, // bctr through a recovered switch jump table (M12) Jump, // b (unconditional) Branch, // bc / bXX (conditional) DataRead, // lwz, lbz, lhz, lha, lfs, lfd, etc. from resolved address @@ -22,6 +23,7 @@ impl XrefKind { match self { XrefKind::Call => "call", XrefKind::IndirectCall => "ind_call", + XrefKind::JumpTable => "jt", XrefKind::Jump => "j", XrefKind::Branch => "br", XrefKind::DataRead => "read", @@ -109,6 +111,31 @@ pub fn analyze_xrefs( sections: &[PeSection], func_analysis: &FuncAnalysis, import_map: &HashMap, +) -> XrefResult { + analyze_xrefs_skipping( + pe, image_base, entry_point, sections, func_analysis, import_map, + &std::collections::BTreeSet::new(), + ) +} + +/// Like [`analyze_xrefs`], but skips the word addresses in `data_words`. +/// +/// Those are data embedded in a code section — recovered jump tables and their +/// index maps (see [`crate::jumptables`]). Decoding them yields whatever +/// instruction their bit pattern happens to spell, and any reference that +/// "instruction" appears to make is fiction. On the reference title every case +/// target begins `0x82…`, which decodes as a `lwz`, so the damage is bogus data +/// reads rather than bogus control flow — but it is damage either way, and it +/// also invents `dat_…` labels in the middle of `.rdata`. +#[tracing::instrument(skip_all, fields(image_base = format_args!("{:#010x}", image_base), data_words = data_words.len()))] +pub fn analyze_xrefs_skipping( + pe: &[u8], + image_base: u32, + entry_point: u32, + sections: &[PeSection], + func_analysis: &FuncAnalysis, + import_map: &HashMap, + data_words: &std::collections::BTreeSet, ) -> XrefResult { let started = std::time::Instant::now(); let func_labels = func_analysis.generate_labels(); @@ -138,7 +165,9 @@ pub fn analyze_xrefs( pe[off], pe[off+1], pe[off+2], pe[off+3] ]); - collect_branch_target(instr, abs_addr, &mut labels, &mut xrefs); + if !data_words.contains(&abs_addr) { + collect_branch_target(instr, abs_addr, &mut labels, &mut xrefs); + } addr += 4; } } @@ -170,6 +199,15 @@ pub fn analyze_xrefs( pe[off], pe[off+1], pe[off+2], pe[off+3] ]); + // A jump-table word is not an instruction. Skip it, and drop the + // tracked constants with it: the words around it belong to + // different basic blocks, so nothing carries across. + if data_words.contains(&abs_addr) { + reg_hi = [None; 32]; + addr += 4; + continue; + } + let opcode = (instr >> 26) & 0x3F; let rd = ((instr >> 21) & 0x1F) as usize; let ra = ((instr >> 16) & 0x1F) as usize; diff --git a/crates/xenia-analysis/tests/db_schema_golden.rs b/crates/xenia-analysis/tests/db_schema_golden.rs index 7211b72..28e7d39 100644 --- a/crates/xenia-analysis/tests/db_schema_golden.rs +++ b/crates/xenia-analysis/tests/db_schema_golden.rs @@ -8,7 +8,7 @@ //! instructions, plus an empty import-library list and one detected //! function. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io::Write; use duckdb::Connection; @@ -16,6 +16,7 @@ use duckdb::Connection; use xenia_analysis::DbWriter; use xenia_analysis::formatter::DisasmInfo; use xenia_analysis::func::{FuncAnalysis, FuncInfo}; +use xenia_analysis::rtti::RttiResult; use xenia_analysis::xref::XrefMap; use xenia_xex::pe::PeSection; @@ -67,6 +68,7 @@ fn synthetic_func_analysis(image_base: u32) -> FuncAnalysis { is_saverestore: false, pdata_validated: false, pdata_length: None, + pdata_prolog_length: None, has_eh: false, }, ); @@ -92,6 +94,7 @@ fn db_schema_matches_expected_columns() { media_id: Some(0xCAFEF00D), sections: §ions, import_libraries: &libs, + xex_header: None, }; let func_analysis = synthetic_func_analysis(image_base); @@ -105,9 +108,12 @@ fn db_schema_matches_expected_columns() { { let mut w = DbWriter::open_fresh(&tmp).expect("open fresh DB"); w.write_base(&info).expect("write_base"); - w.ingest_instructions(&pe, &info, &func_analysis, &labels) + w.ingest_instructions(&pe, &info, &func_analysis, &labels, &BTreeSet::new()) .expect("ingest_instructions"); - w.write_analysis_results(&pe, &info, &func_analysis, &labels, &xrefs, &[], &[], &[], None, &[]) + w.write_analysis_results( + &pe, &info, &func_analysis, &labels, &xrefs, + &[], &[], &[], None, &[], &[], &RttiResult::default(), None, + ) .expect("write_analysis_results"); w.create_sql_views().expect("create_sql_views"); } @@ -149,6 +155,7 @@ fn db_schema_matches_expected_columns() { ("section", "VARCHAR"), ("function", "BIGINT"), ("label", "VARCHAR"), + ("is_data", "BOOLEAN"), ]), ("functions", &[ ("address", "BIGINT"), @@ -160,8 +167,54 @@ fn db_schema_matches_expected_columns() { ("is_saverestore", "BOOLEAN"), ("pdata_validated", "BOOLEAN"), ("pdata_length", "BIGINT"), + ("prolog_length", "BIGINT"), ("has_eh", "BOOLEAN"), ]), + ("jump_tables", &[ + ("bctr_pc", "BIGINT"), + ("function", "BIGINT"), + ("table_address", "BIGINT"), + ("entry_count", "BIGINT"), + ("table_slots", "BIGINT"), + ("index_map_address", "BIGINT"), + ("index_map_count", "BIGINT"), + ("case_bound", "BIGINT"), + ("kind", "VARCHAR"), + ]), + ("jump_table_entries", &[ + ("bctr_pc", "BIGINT"), + ("case_index", "BIGINT"), + ("target_address", "BIGINT"), + ]), + ("data_in_code", &[ + ("address", "BIGINT"), + ("length", "BIGINT"), + ("kind", "VARCHAR"), + ]), + ("rtti_type_descriptors", &[ + ("address", "BIGINT"), + ("mangled_name", "VARCHAR"), + ("demangled_name", "VARCHAR"), + ]), + ("rtti_locators", &[ + ("address", "BIGINT"), + ("subobject_offset", "BIGINT"), + ("cd_offset", "BIGINT"), + ("type_descriptor", "BIGINT"), + ("class_hierarchy", "BIGINT"), + ("vtable_address", "BIGINT"), + ]), + ("rtti_base_classes", &[ + ("class_hierarchy", "BIGINT"), + ("base_index", "BIGINT"), + ("type_descriptor", "BIGINT"), + ("name", "VARCHAR"), + ("num_contained_bases", "BIGINT"), + ("mdisp", "BIGINT"), + ("pdisp", "BIGINT"), + ("vdisp", "BIGINT"), + ("attributes", "BIGINT"), + ]), ("pdata_entries", &[ ("begin_address", "BIGINT"), ("end_address", "BIGINT"), @@ -174,6 +227,39 @@ fn db_schema_matches_expected_columns() { ("name", "VARCHAR"), ("kind", "VARCHAR"), ]), + ("xdbf_entries", &[ + ("namespace", "BIGINT"), + ("namespace_name", "VARCHAR"), + ("id", "BIGINT"), + ("body_offset", "BIGINT"), + ("size", "BIGINT"), + ("magic", "VARCHAR"), + ]), + ("xdbf_achievements", &[ + ("id", "BIGINT"), + ("name", "VARCHAR"), + ("unlocked_desc", "VARCHAR"), + ("locked_desc", "VARCHAR"), + ("label_id", "BIGINT"), + ("description_id", "BIGINT"), + ("unachieved_id", "BIGINT"), + ("image_id", "BIGINT"), + ("gamerscore", "BIGINT"), + ("flags", "BIGINT"), + ]), + ("xdbf_strings", &[ + ("language", "BIGINT"), + ("language_name", "VARCHAR"), + ("string_id", "BIGINT"), + ("value", "VARCHAR"), + ]), + ("xdbf_images", &[ + ("id", "BIGINT"), + ("is_title_icon", "BOOLEAN"), + ("body_offset", "BIGINT"), + ("size", "BIGINT"), + ("format", "VARCHAR"), + ]), ("demangled_names", &[ ("address", "BIGINT"), ("mangled", "VARCHAR"), @@ -209,6 +295,7 @@ fn db_schema_matches_expected_columns() { ("encoding", "VARCHAR"), ("length", "BIGINT"), ("content", "VARCHAR"), + ("section", "VARCHAR"), ]), ("tls_info", &[ ("raw_data_start", "BIGINT"), @@ -237,6 +324,7 @@ fn db_schema_matches_expected_columns() { ("vptr_offset", "BIGINT"), ("slot", "BIGINT"), ("candidate_count", "BIGINT"), + ("truncated", "BOOLEAN"), ]), ("indirect_dispatch_candidates", &[ ("dispatch_pc", "BIGINT"), diff --git a/crates/xenia-app/src/main.rs b/crates/xenia-app/src/main.rs index 5aba5c5..cb114f4 100644 --- a/crates/xenia-app/src/main.rs +++ b/crates/xenia-app/src/main.rs @@ -330,6 +330,18 @@ enum Commands { /// branch xrefs. Disagreement is logged as a warning (non-fatal). #[arg(long, value_enum, default_value_t = AnalyzeMode::Rust)] analyze: AnalyzeMode, + /// Ceiling on candidates materialised per unresolved virtual-call site. + /// + /// A `bcctrl` through `this->vptr` is resolved by matching + /// `(vptr_offset, slot)` against every class installing a vtable at + /// that offset. At offset 0 that matches almost every class, so the + /// result is a cross product rather than an answer — one site can + /// claim 700+ callees. Sites above this ceiling are still recorded in + /// `indirect_dispatch_sites` (with `truncated` set and a truthful + /// `candidate_count`), but emit no `indirect_dispatch_candidates` rows + /// and no `ind_call` xrefs. Raise it to get the full cross product back. + #[arg(long, default_value_t = xenia_analysis::ind_dispatch_typed::DEFAULT_MAX_CANDIDATES)] + max_indirect_candidates: usize, /// Suppress assembly text output (DB-only mode) #[arg(long)] quiet: bool, @@ -491,7 +503,7 @@ fn main() -> Result<()> { Commands::Browse { path } => cmd_browse(&path), Commands::Info { path } => cmd_info(&path), Commands::Extract { path, output, db } => cmd_extract(&path, output.as_deref(), db.as_deref()), - Commands::Dis { path, output, db, json, analyze, quiet } => cmd_dis(&path, output.as_deref(), db.as_deref(), json.as_deref(), analyze, quiet), + Commands::Dis { path, output, db, json, analyze, max_indirect_candidates, quiet } => cmd_dis(&path, output.as_deref(), db.as_deref(), json.as_deref(), analyze, max_indirect_candidates, quiet), Commands::Check { path, max_instructions, @@ -1701,6 +1713,7 @@ fn cmd_exec_inner( media_id: header.execution_info.as_ref().map(|e| e.media_id), sections: §ions, import_libraries: &header.import_libraries, + xex_header: Some(&header), }; info!(db = %db, "writing database"); @@ -5061,7 +5074,13 @@ fn cmd_browse(path: &str) -> Result<()> { /// Helper: load XEX, parse header, decompress PE, resolve imports, parse sections. #[instrument(skip_all, fields(path = %path))] -fn load_and_prepare(path: &str) -> Result<(xenia_xex::Xex2Header, Vec, Vec)> { +/// Load a XEX and prepare it for analysis. +/// +/// Returns the parsed header, the decompressed image, its sections, and the +/// **raw XEX bytes**. The raw bytes are needed because optional-header values +/// are file offsets into the container, not image VAs — the resource table +/// (and so the embedded XDBF package) is only reachable through them. +fn load_and_prepare(path: &str) -> Result<(xenia_xex::Xex2Header, Vec, Vec, Vec)> { let data = load_xex_data(path)?; let mut header = xenia_xex::loader::parse_xex2_header(&data)?; @@ -5086,14 +5105,14 @@ fn load_and_prepare(path: &str) -> Result<(xenia_xex::Xex2Header, Vec, Vec, db_path: Option<&str>) -> Result<()> { use serde::Serialize; - let (header, pe_image, sections) = load_and_prepare(path)?; + let (header, pe_image, sections, _xex_data) = load_and_prepare(path)?; let entry = xenia_xex::loader::get_entry_point(&header).unwrap(); let base = xenia_xex::loader::get_image_base(&header).unwrap(); @@ -5168,6 +5187,7 @@ fn cmd_extract(path: &str, output_dir: Option<&str>, db_path: Option<&str>) -> R media_id: header.execution_info.as_ref().map(|e| e.media_id), sections: §ions, import_libraries: &header.import_libraries, + xex_header: Some(&header), }; info!(db = %db, "writing base tables"); let mut w = xenia_analysis::DbWriter::open_fresh(std::path::Path::new(db))?; @@ -5185,12 +5205,13 @@ fn cmd_dis( db_path: Option<&str>, json_path: Option<&str>, analyze: AnalyzeMode, + max_indirect_candidates: usize, quiet: bool, ) -> Result<()> { use std::collections::HashMap; let started = Instant::now(); - let (header, pe_image, sections) = load_and_prepare(path)?; + let (header, pe_image, sections, xex_data) = load_and_prepare(path)?; let entry = xenia_xex::loader::get_entry_point(&header).unwrap(); let base = xenia_xex::loader::get_image_base(&header).unwrap(); @@ -5225,10 +5246,48 @@ fn cmd_dis( "function detection complete", ); - // Cross-reference analysis - let mut xref_result = xenia_analysis::xref::analyze_xrefs( - &pe_image, base, entry, §ions, &func_analysis, &import_map, + // M12 — switch / jump-table recovery. Emits one `jt` xref per distinct + // case body so the case bodies stop looking unreachable, and reports the + // table extents so the linear disassembler can flag them as data. + let jump_tables = xenia_analysis::jumptables::analyze( + &pe_image, base, §ions, &func_analysis, ); + let jt_data_words = xenia_analysis::jumptables::data_word_addresses(&jump_tables); + info!( + jump_tables = jump_tables.len(), + cases = jump_tables.iter().map(|t| t.targets.len()).sum::(), + data_words = jt_data_words.len(), + "jump-table recovery complete", + ); + + // Cross-reference analysis + let mut xref_result = xenia_analysis::xref::analyze_xrefs_skipping( + &pe_image, base, entry, §ions, &func_analysis, &import_map, &jt_data_words, + ); + + // Feed the recovered `switch` edges into the xref graph, so case bodies + // stop looking unreachable and get a label of their own. + let mut jt_edges = 0usize; + for jt in &jump_tables { + for target in jt.distinct_targets() { + xref_result.xrefs + .entry(target) + .or_default() + .push(xenia_analysis::xref::Xref { + source: jt.bctr_pc, + kind: xenia_analysis::xref::XrefKind::JumpTable, + addr_mode: None, + }); + xref_result.labels + .entry(target) + .or_insert_with(|| format!("case_{target:08X}")); + jt_edges += 1; + } + xref_result.labels + .entry(jt.table_address) + .or_insert_with(|| format!("jpt_{:08X}", jt.table_address)); + } + info!(case_edges = jt_edges, "switch edges added to xref graph"); let total_xrefs: usize = xref_result.xrefs.values().map(|v| v.len()).sum(); info!( labels = xref_result.labels.len(), @@ -5251,17 +5310,34 @@ fn cmd_dis( .collect(); let vptr_block_boundaries: std::collections::HashSet = xref_result.labels.keys().copied().collect(); - let vtable_anchors = xenia_analysis::vtables::scan_vptr_write_constants( + let mut vtable_anchors = xenia_analysis::vtables::scan_vptr_write_constants( &pe_image, base, &vptr_anchor_funcs, §ions, &vptr_block_boundaries, ); info!(vtable_anchors = vtable_anchors.len(), "vptr-write anchor scan complete"); - let vtables = xenia_analysis::vtables::analyze_with_anchors( + + // M13 — authoritative MSVC RTTI walk. Every `vftable[-1] -> COL` link the + // linker emitted is an anchor the heuristic scan must not miss, and the + // class names it recovers override anything the contiguity scan guessed. + let rtti = xenia_analysis::rtti::analyze(&pe_image, base, §ions); + let rtti_anchors = rtti.vtable_anchors(); + let rtti_new_anchors = rtti_anchors.difference(&vtable_anchors).count(); + vtable_anchors.extend(rtti_anchors.iter().copied()); + info!( + rtti_vtables = rtti_anchors.len(), + new_anchors = rtti_new_anchors, + "RTTI anchors merged", + ); + + let mut vtables = xenia_analysis::vtables::analyze_with_anchors( &pe_image, base, §ions, &function_starts, &vtable_anchors, ); + let named = xenia_analysis::vtables::apply_rtti_names(&mut vtables, &rtti); + let vtables = vtables; let rtti_count = vtables.iter().filter(|v| v.rtti_present).count(); info!( vtables = vtables.len(), rtti = rtti_count, + rtti_named = named, anon = vtables.len() - rtti_count, "vtable scan complete", ); @@ -5343,8 +5419,9 @@ fn cmd_dis( // M5.5 — typed indirect-dispatch resolution (this->vptr → method). let typed_ind = xenia_analysis::ind_dispatch_typed::analyze( &pe_image, base, &func_analysis, &vtables, &xref_result.labels, + max_indirect_candidates, ); - let single = typed_ind.dispatches.iter().filter(|d| d.candidate_vtables.len() == 1).count(); + let single = typed_ind.dispatches.iter().filter(|d| d.total_candidates == 1).count(); let multi = typed_ind.dispatches.len() - single; let typed_edges: usize = typed_ind.dispatches.iter().map(|d| d.method_pcs.len()).sum(); info!( @@ -5355,7 +5432,10 @@ fn cmd_dis( edges = typed_edges, "M5.5 typed indirect-dispatch scan complete", ); - // Add ind_call edges for every (dispatch_pc, method) candidate. + // Add ind_call edges for every (dispatch_pc, method) candidate. Sites the + // resolver could not narrow contribute nothing here — `method_pcs` is + // empty for them — which keeps `xrefs` a table of evidence rather than of + // possibilities. for d in &typed_ind.dispatches { for &method_pc in &d.method_pcs { xref_result.xrefs @@ -5369,6 +5449,30 @@ fn cmd_dis( } } + // XDBF/SPA — the title metadata package the XEX names via its resource + // table (achievements, localized strings, images). Located through the + // resource table rather than by scanning for the magic, so the entry + // table's own accounting is what decides what exists. + let resources = xenia_xex::resources::parse_resources(&xex_data, &header); + let xdbf = resources.iter().find_map(|r| { + let off = r.image_offset(base)?; + let x = xenia_analysis::xdbf::analyze(&pe_image, off)?; + info!( + resource = %r.name, + address = format_args!("{:#010x}", r.address), + size = r.size, + entries = x.entries.len(), + achievements = x.achievements.len(), + string_tables = x.string_tables.len(), + images = x.images.len(), + "XDBF package found", + ); + Some(x) + }); + if xdbf.is_none() && !resources.is_empty() { + info!(resources = resources.len(), "resource table present but no XDBF package"); + } + // Build DisasmInfo let disasm_info = xenia_analysis::formatter::DisasmInfo { image_base: base, @@ -5378,6 +5482,7 @@ fn cmd_dis( media_id: header.execution_info.as_ref().map(|e| e.media_id), sections: §ions, import_libraries: &header.import_libraries, + xex_header: Some(&header), }; // SQLite database output (base + ingest + analyze layers) @@ -5385,7 +5490,9 @@ fn cmd_dis( info!(db = %db, analyze = ?analyze, "writing database"); let mut w = xenia_analysis::DbWriter::open_fresh(std::path::Path::new(db))?; w.write_base(&disasm_info)?; - w.ingest_instructions(&pe_image, &disasm_info, &func_analysis, &xref_result.labels)?; + w.ingest_instructions( + &pe_image, &disasm_info, &func_analysis, &xref_result.labels, &jt_data_words, + )?; w.write_analysis_results( &pe_image, &disasm_info, @@ -5397,6 +5504,9 @@ fn cmd_dis( &fparrays, Some(&typed_ind), &eh_records, + &jump_tables, + &rtti, + xdbf.as_ref(), )?; w.write_tls(tls_info.as_ref())?; if matches!(analyze, AnalyzeMode::Sql | AnalyzeMode::Both) { @@ -5430,7 +5540,7 @@ fn cmd_dis( let abs_end = abs_start + section.virtual_size; let items = xenia_analysis::enrich_section( &pe_image, base, §ion.name, abs_start, abs_end, - &func_analysis, &xref_result.labels, + &func_analysis, &xref_result.labels, &jt_data_words, ); total += xenia_analysis::sinks::json::write_jsonl(&mut out, items)?; } @@ -5453,6 +5563,7 @@ fn cmd_dis( &import_map, &xref_result.xrefs, &xref_result.data_annotations, + &jt_data_words, )?; if let Some(path) = output { diff --git a/crates/xenia-xex/src/header.rs b/crates/xenia-xex/src/header.rs index 73ba34c..a52d5d1 100644 --- a/crates/xenia-xex/src/header.rs +++ b/crates/xenia-xex/src/header.rs @@ -120,10 +120,20 @@ pub mod header_keys { pub const ENTRY_POINT: u32 = 0x00010100; pub const IMAGE_BASE_ADDRESS: u32 = 0x00010201; pub const IMPORT_LIBRARIES: u32 = 0x000103FF; - pub const TLS_INFO: u32 = 0x00020200; + // These two were swapped. `0x00020104` is TLS_INFO and `0x00020200` is + // DEFAULT_STACK_SIZE — confirmed against the reference implementation + // (xenia-canary `kernel/util/xex2_info.h`) and against this title, whose + // `0x00020104` header points at a TLS descriptor (slot_count 64) while + // `0x00020200` carries the inline value 0x80000 (512 KiB), a sane stack. + // Swapped, `get_stack_size` returned the TLS descriptor's file offset. + pub const TLS_INFO: u32 = 0x00020104; pub const EXECUTION_INFO: u32 = 0x00040006; - pub const DEFAULT_STACK_SIZE: u32 = 0x00020104; + pub const DEFAULT_STACK_SIZE: u32 = 0x00020200; pub const ORIGINAL_PE_NAME: u32 = 0x000183FF; pub const FILE_FORMAT_INFO: u32 = 0x000003FF; pub const SYSTEM_FLAGS: u32 = 0x00030000; + pub const RESOURCE_INFO: u32 = 0x000002FF; + pub const STATIC_LIBRARIES: u32 = 0x000200FF; + pub const CHECKSUM_TIMESTAMP: u32 = 0x00018002; + pub const GAME_RATINGS: u32 = 0x00040310; } diff --git a/crates/xenia-xex/src/lib.rs b/crates/xenia-xex/src/lib.rs index a77a9f0..b2efca0 100644 --- a/crates/xenia-xex/src/lib.rs +++ b/crates/xenia-xex/src/lib.rs @@ -3,6 +3,7 @@ pub mod loader; pub mod lzx; pub mod pe; pub mod pdata; +pub mod resources; pub mod tls; pub use header::Xex2Header; diff --git a/crates/xenia-xex/src/pdata.rs b/crates/xenia-xex/src/pdata.rs index 8b78466..66c0cb3 100644 --- a/crates/xenia-xex/src/pdata.rs +++ b/crates/xenia-xex/src/pdata.rs @@ -29,7 +29,10 @@ pub struct PdataEntry { pub function_length: u32, /// Prolog size in bytes (prolog_length_dwords * 4). pub prolog_length: u32, - /// Raw 2-bit flags from the packed word (bit 1 = 32-bit-code, bit 0 = exception). + /// Raw 2-bit flags lifted from the packed word's top two bits, i.e. + /// `(meta >> 30) & 3`. So **bit 0 mirrors packed bit 30 (32-bit-code, set + /// on essentially every PPC entry) and bit 1 mirrors packed bit 31 + /// (exception handler registered)** — test `flags & 2` for "has EH". pub flags: u8, } diff --git a/crates/xenia-xex/src/resources.rs b/crates/xenia-xex/src/resources.rs new file mode 100644 index 0000000..1e1c16e --- /dev/null +++ b/crates/xenia-xex/src/resources.rs @@ -0,0 +1,127 @@ +//! XEX `XEX_HEADER_RESOURCE_INFO` (key `0x000002FF`) — the embedded resource table. +//! +//! The header points at a length-prefixed table of fixed 16-byte records: +//! +//! ```text +//! u32 size total table size in bytes, including this field +//! record[] entries (size - 4) / 16 of: +//! char[8] name resource name, NUL-padded (the title's is its +//! title id in uppercase hex, e.g. "535107D4") +//! u32 address absolute VA of the resource inside the loaded image +//! u32 size resource length in bytes +//! ``` +//! +//! For a title the named resource is its **XDBF/SPA package** — achievements, +//! localized strings, and images. See `xenia_analysis::xdbf`. +//! +//! Reference: xenia-canary `kernel/util/xex2_info.h` (`xex2_resource`). + +use crate::header::{Xex2Header, header_keys}; + +/// One entry of the XEX resource table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct XexResource { + /// Resource name from the table, trailing NULs stripped. + pub name: String, + /// Absolute VA of the resource within the loaded image. + pub address: u32, + /// Resource length in bytes. + pub size: u32, +} + +impl XexResource { + /// Offset of this resource within an image-base-relative buffer. + pub fn image_offset(&self, image_base: u32) -> Option { + self.address.checked_sub(image_base).map(|o| o as usize) + } +} + +/// Parse the resource table out of the raw XEX bytes. +/// +/// `data` is the whole XEX file (the optional-header value is a file offset +/// into it, not a VA). Returns an empty vec when the header is absent or the +/// table is truncated — never an error. +pub fn parse_resources(data: &[u8], header: &Xex2Header) -> Vec { + let Some(off) = header + .optional_headers + .iter() + .find(|h| h.key == header_keys::RESOURCE_INFO) + .map(|h| h.value as usize) + else { + return Vec::new(); + }; + if off + 4 > data.len() { + return Vec::new(); + } + let size = u32::from_be_bytes([data[off], data[off + 1], data[off + 2], data[off + 3]]) as usize; + // The size field counts itself; anything smaller than one record is junk. + if size < 4 + 16 || off + size > data.len() { + return Vec::new(); + } + let count = (size - 4) / 16; + let mut out = Vec::with_capacity(count); + for i in 0..count { + let p = off + 4 + i * 16; + let name = String::from_utf8_lossy(&data[p..p + 8]) + .trim_end_matches('\0') + .to_string(); + let address = u32::from_be_bytes([data[p + 8], data[p + 9], data[p + 10], data[p + 11]]); + let rsize = u32::from_be_bytes([data[p + 12], data[p + 13], data[p + 14], data[p + 15]]); + out.push(XexResource { name, address, size: rsize }); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::header::{Xex2Header, Xex2OptionalHeader}; + + fn mk_header(opt: Vec) -> Xex2Header { + Xex2Header { + magic: crate::header::XEX2_MAGIC, + module_flags: 0, + header_size: 0, + security_offset: 0, + header_count: opt.len() as u32, + optional_headers: opt, + security_info: None, + file_format_info: None, + import_libraries: Vec::new(), + execution_info: None, + original_pe_name: None, + } + } + + fn with_resource(value: u32) -> Xex2Header { + mk_header(vec![Xex2OptionalHeader { key: header_keys::RESOURCE_INFO, value }]) + } + + #[test] + fn parses_one_resource() { + let mut data = vec![0u8; 0x100]; + let off = 0x40usize; + data[off..off + 4].copy_from_slice(&(4u32 + 16).to_be_bytes()); + data[off + 4..off + 12].copy_from_slice(b"535107D4"); + data[off + 12..off + 16].copy_from_slice(&0x828F_B900u32.to_be_bytes()); + data[off + 16..off + 20].copy_from_slice(&0x0002_1FCFu32.to_be_bytes()); + let r = parse_resources(&data, &with_resource(off as u32)); + assert_eq!(r.len(), 1); + assert_eq!(r[0].name, "535107D4"); + assert_eq!(r[0].address, 0x828F_B900); + assert_eq!(r[0].size, 0x0002_1FCF); + assert_eq!(r[0].image_offset(0x8200_0000), Some(0x8F_B900)); + } + + #[test] + fn absent_header_yields_nothing() { + assert!(parse_resources(&[0u8; 0x100], &mk_header(Vec::new())).is_empty()); + } + + #[test] + fn truncated_table_yields_nothing() { + let mut data = vec![0u8; 0x20]; + data[0..4].copy_from_slice(&0xFFFF_FFFFu32.to_be_bytes()); + assert!(parse_resources(&data, &with_resource(0)).is_empty()); + } +} diff --git a/zq.py b/zq.py index e334368..934914d 100755 --- a/zq.py +++ b/zq.py @@ -6,13 +6,25 @@ and the fact that the engine vtable / rdata is NOT in the DB (read it from guest memory with `xenia-rs exec ... --dump-addr=0x` instead). Usage: - zq.py dis # disassemble [lo,hi) + zq.py dis # disassemble [lo,hi) (jump-table words shown as .long) zq.py fn # function containing pc (address,name,end) zq.py xref # xrefs whose target == addr (callers) zq.py callers # call-sites of vtable slot at byte offset N # (finds `lwz r11, N(r11)` + reports the fn) - zq.py grep # instructions whose operands LIKE %substr% - zq.py find # instructions whose raw word == value (e.g. a ptr) + zq.py grep # instructions whose operands LIKE %substr% + zq.py find # instructions whose raw word == value (e.g. a ptr) + + zq.py switch # recovered switch cases for the bctr at/near pc + zq.py switches [fn_hex] # every recovered switch (optionally in one function) + zq.py classes [substr] # RTTI class names (+ vtable, method count) + zq.py class # one class: bases, vtable, virtual methods + zq.py str # string literals matching, with referencing functions + + zq.py xdbf [substr] # XDBF title text (all locales); substr filters + zq.py ach # XDBF achievements (id, gamerscore, name, descriptions) + +A command that needs a table the current DB predates prints what to regenerate +rather than a SQL error. """ import duckdb, sys @@ -20,6 +32,23 @@ DB = '/home/fabi/RE - Project Sylpheed/xenia-rs/sylpheed.db' c = duckdb.connect(DB, read_only=True) H = lambda x: '0x%08x' % x +REGEN = ("xenia-rs dis --db sylpheed.db --analyze sql") + + +def _need(*tables): + """Exit with a regeneration hint if any table is missing from this DB.""" + have = {r[0] for r in c.execute( + "SELECT table_name FROM information_schema.tables").fetchall()} + missing = [t for t in tables if t not in have] + if missing: + sys.exit(f"this db predates {', '.join(missing)} — regenerate with:\n {REGEN}") + + +def _has_col(table, col): + return any(r[0] == col for r in c.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name=?", + [table]).fetchall()) + def _fn(pc): r = c.execute('SELECT address,name,end_address FROM functions WHERE address<=? AND end_address>? ' @@ -27,40 +56,161 @@ def _fn(pc): return f'{r[0][1]}({H(r[0][0])})' if r else '?' +def cmd_dis(lo, hi): + data_col = 'is_data' if _has_col('instructions', 'is_data') else 'false' + rows = c.execute(f'SELECT address,mnemonic,operands,raw,{data_col} FROM instructions ' + 'WHERE address>=? AND address=? ORDER BY bctr_pc LIMIT 1', [pc]).fetchall() + if not r: + sys.exit('no recovered switch at or after %s' % H(pc)) + bctr, fn, tbl, kind, n = r[0] + print(f'bctr {H(bctr)} in {_fn(bctr)} table={H(tbl)} kind={kind} cases={n}') + for ci, tgt in c.execute('SELECT case_index,target_address FROM jump_table_entries ' + 'WHERE bctr_pc=? ORDER BY case_index', [bctr]).fetchall(): + print(f' case {ci:>3} -> {H(tgt)}') + + +def cmd_switches(fn): + _need('jump_tables') + q = ('SELECT bctr_pc,function,kind,entry_count,table_address FROM jump_tables ' + + ('WHERE function=? ' if fn is not None else '') + 'ORDER BY bctr_pc') + for bctr, f, kind, n, tbl in c.execute(q, [fn] if fn is not None else []).fetchall(): + print(H(bctr), f'{kind:<8}', f'cases={n:<4}', 'table=' + H(tbl), 'in', _fn(bctr)) + + +def cmd_classes(sub): + _need('rtti_type_descriptors', 'rtti_locators') + q = """SELECT td.demangled_name, c.vtable_address, c.subobject_offset, + (SELECT count(*) FROM methods m WHERE m.vtable_address = c.vtable_address) + FROM rtti_locators c + JOIN rtti_type_descriptors td ON td.address = c.type_descriptor + {} ORDER BY td.demangled_name, c.subobject_offset""" + q = q.format('WHERE td.demangled_name ILIKE ?' if sub else '') + for name, vt, off, nm in c.execute(q, [f'%{sub}%'] if sub else []).fetchall(): + loc = H(vt) if vt is not None else '-' + print(f'{name:<60} vtable={loc} +0x{off:x} methods={nm}') + + +def cmd_class(name): + _need('rtti_type_descriptors', 'rtti_locators', 'rtti_base_classes') + rows = c.execute("""SELECT c.address, c.vtable_address, c.class_hierarchy, c.subobject_offset + FROM rtti_locators c + JOIN rtti_type_descriptors td ON td.address = c.type_descriptor + WHERE td.demangled_name = ?""", [name]).fetchall() + if not rows: + sys.exit(f'no RTTI class named {name!r} (try: zq.py classes {name})') + for col, vt, chd, off in rows: + print(f'== {name} (COL {H(col)}, subobject +0x{off:x})') + bases = c.execute('SELECT base_index,name,mdisp,pdisp,vdisp FROM rtti_base_classes ' + 'WHERE class_hierarchy=? AND base_index>0 ORDER BY base_index', + [chd]).fetchall() + for _, bn, md, pd, vd in bases: + print(f' base {bn} mdisp={md} pdisp={pd} vdisp={vd}') + if vt is None: + print(' (no vtable located)') + continue + for slot, fa in c.execute('SELECT slot,function_address FROM methods ' + 'WHERE vtable_address=? ORDER BY slot', [vt]).fetchall(): + print(f' vf{slot:<3} {H(fa)} {_fn(fa)}') + + +def cmd_str(sub): + sec = ', section' if _has_col('strings', 'section') else ", ''" + rows = c.execute(f'SELECT address, encoding, content{sec} FROM strings ' + 'WHERE content ILIKE ? ORDER BY address', [f'%{sub}%']).fetchall() + for a, enc, content, section in rows: + refs = c.execute("SELECT DISTINCT source_func FROM xrefs WHERE target=? AND source_func IS NOT NULL", + [a]).fetchall() + where = ', '.join(_fn(r[0]) for r in refs[:4]) or '(no xref)' + print(f'{H(a)} [{enc}{"/" + section if section else ""}] {content!r}\n <- {where}') + + +def cmd_xdbf(args): + """XDBF title text across every shipped locale.""" + sub = args[0] if args else "" + rows = c.execute( + "SELECT string_id, english, japanese FROM v_xdbf_text " + "WHERE (? = '' OR english ILIKE '%' || ? || '%' OR japanese ILIKE '%' || ? || '%') " + "ORDER BY string_id", + [sub, sub, sub], + ).fetchall() + for sid, en, ja in rows: + print(f"{sid:6} {en or ''}") + if ja and ja != en: + print(f" ja: {ja}") + print(f"({len(rows)} strings)") + + +def cmd_ach(_args): + """XDBF achievements in the title's default language.""" + rows = c.execute( + "SELECT id, gamerscore, name, unlocked_desc, locked_desc " + "FROM xdbf_achievements ORDER BY id" + ).fetchall() + total = 0 + for aid, gs, name, unlocked, locked in rows: + total += gs or 0 + print(f"{aid:3} | {gs:3}G | {name}") + print(f" unlocked: {unlocked}") + print(f" locked : {locked}") + print(f"\n{len(rows)} achievements, {total}G") + + def main(): if len(sys.argv) < 2: print(__doc__); return - cmd = sys.argv[1] + cmd, args = sys.argv[1], sys.argv[2:] if cmd == 'dis': - lo, hi = int(sys.argv[2], 16), int(sys.argv[3], 16) - for a, m, o in c.execute('SELECT address,mnemonic,operands FROM instructions ' - 'WHERE address>=? AND address