From 62b0f79590181e344efb3eb89ad43d9b1e311ff2 Mon Sep 17 00:00:00 2001 From: MechaCat02 Date: Sun, 13 Sep 2026 19:42:26 +0200 Subject: [PATCH] wip: tool README, DuckDB wording, zq.py escape hatch --- crates/sylpheed-ppc/src/lib.rs | 4 + crates/sylpheed-xexdb/Cargo.toml | 3 + crates/sylpheed-xexdb/README.md | 51 +++++++ crates/sylpheed-xexdb/build.rs | 139 +++++++++---------- crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs | 34 ++++- crates/sylpheed-xexdb/src/sinks/text.rs | 2 +- tools/zq.py | 13 +- 7 files changed, 163 insertions(+), 83 deletions(-) create mode 100644 crates/sylpheed-xexdb/README.md diff --git a/crates/sylpheed-ppc/src/lib.rs b/crates/sylpheed-ppc/src/lib.rs index 6aa847ec..37940c96 100644 --- a/crates/sylpheed-ppc/src/lib.rs +++ b/crates/sylpheed-ppc/src/lib.rs @@ -10,3 +10,7 @@ pub mod decoder; pub mod disasm; pub mod opcode; + +pub use decoder::decode; +pub use disasm::{DisasmItem, DisasmText, disassemble, format as disasm_format, iter_disasm}; +pub use opcode::PpcOpcode; diff --git a/crates/sylpheed-xexdb/Cargo.toml b/crates/sylpheed-xexdb/Cargo.toml index 3502c62d..d1410d40 100644 --- a/crates/sylpheed-xexdb/Cargo.toml +++ b/crates/sylpheed-xexdb/Cargo.toml @@ -21,3 +21,6 @@ duckdb = { version = "1", features = ["bundled"] } msvc-demangler = "0.11" encoding_rs = "0.8" clap = { version = "4", features = ["derive"] } + +[build-dependencies] +serde_json = "1" diff --git a/crates/sylpheed-xexdb/README.md b/crates/sylpheed-xexdb/README.md new file mode 100644 index 00000000..3a2494d9 --- /dev/null +++ b/crates/sylpheed-xexdb/README.md @@ -0,0 +1,51 @@ +# `sylph-xexdb` — the title's XEX, as a queryable database + +```bash +sylph-xexdb dis --db sylpheed.db --analyze sql +tools/zq.py dis 0x82341a20 0x82341b00 # then query it +``` + +Extract the PE image, disassemble it, detect functions, resolve cross-references +and RTTI, and write the lot to **DuckDB**. + +## Where it came from, and what did not come with it + +This was `xenia-rs` — a from-scratch Rust Xbox 360 emulator, started because +Canary would not run on Linux. Canary later did, with fixes from us, and the +emulator was retired. **The static analysis is the part that stayed useful**, so +it came here and the emulator did not: no interpreter, no JIT, no scheduler, no +GPU, no kernel. About 17,000 lines of ~71,000. + +That was possible because the coupling was three symbols — `decoder::decode`, +`disasm::DisasmItem`, `disasm::format` — now `sylpheed-ppc`. + +| crate | what it is | +|---|---| +| `sylpheed-xex` | the XEX2 container: decrypt, LZX, PE image, resources, and the disc image it may live in | +| `sylpheed-ppc` | PowerPC decode and disassembly | +| `sylpheed-xexdb` | the analysis passes, the schema, and this binary | + +## ⚠️ Two things that will mislead you + +**The database is DuckDB.** `xenia-rs`'s own `--db` help said "SQLite" in two +places and was wrong. Query it with `python3 -c "import duckdb"`, or `zq.py`. + +**`indirect_dispatch_candidates` is deliberately not a cross product.** A +`bcctrl` through `this->vptr` at offset 0 matches nearly every class, so one +site can claim 700+ callees. Sites past `--max-indirect-candidates` record a +truthful `candidate_count` with `truncated` set and emit **no** candidate rows. +On this title that is 6,556 of 6,983 sites, standing for 1,801,075 candidates +that are counted rather than materialised. An older database built without the +ceiling has ~1.8M more rows in that table and in `xrefs`, and they say nothing +extra. + +## The export table + +Import names come from `docs/reference/xbox360-exports.json` — 2,913 ordinals, +compiled in by `build.rs`. + +🔴 It used to come from Canary's `xboxkrnl_table.inc` through **a relative path +to a sibling checkout**, which printed a warning and produced an empty table +whenever that checkout was not there. Every import in the database then resolved +to nothing, and the build still succeeded. Now the source is in this repository +and a missing file **fails the build**. diff --git a/crates/sylpheed-xexdb/build.rs b/crates/sylpheed-xexdb/build.rs index 0a8ef346..da7a8002 100644 --- a/crates/sylpheed-xexdb/build.rs +++ b/crates/sylpheed-xexdb/build.rs @@ -1,87 +1,76 @@ -//! Build script: parse xenia's xboxkrnl_table.inc and xam_table.inc to generate -//! ordinal->name lookup tables at compile time. +//! Generate the ordinal → export-name table that import resolution needs. +//! +//! 🔴 **THE SOURCE CHANGED, AND THE OLD ONE COULD FAIL SILENTLY.** In `xenia-rs` +//! this parsed Canary's `xboxkrnl_table.inc` / `xam_table.inc` through a +//! **relative path to a sibling checkout** — `../xenia-canary/src/…`. When that +//! path was not there (a container, another machine, a fresh clone) it printed a +//! `cargo:warning` and returned an empty table, and every import in the database +//! came out unresolved. A build that quietly produces a worse artifact is the +//! failure mode this project keeps meeting. +//! +//! The source is now `docs/reference/xbox360-exports.json`, which is **in this +//! repository** — 2,913 exports across `xboxkrnl.exe`, `xam.xex` and `xbdm.xex`, +//! adopted in Phase 2 of `docs/agents/CONSOLIDATION.md`. There is no sibling +//! checkout to be missing, and if the file is unreadable this **fails the +//! build** rather than degrading. -use std::env; -use std::fs; -use std::io::Write; use std::path::Path; - -fn parse_table(path: &Path) -> Vec<(u32, String, String)> { - let content = match fs::read_to_string(path) { - Ok(c) => c, - Err(e) => { - eprintln!("cargo:warning=could not read {}: {}", path.display(), e); - return Vec::new(); - } - }; - - let mut entries = Vec::new(); - for line in content.lines() { - let line = line.trim(); - // XE_EXPORT(module, 0xNNNNNNNN, Name, kType), - if !line.starts_with("XE_EXPORT(") { continue; } - let inner = match line.strip_prefix("XE_EXPORT(").and_then(|s| s.strip_suffix("),")) { - Some(s) => s, - None => continue, - }; - let parts: Vec<&str> = inner.splitn(4, ',').map(|s| s.trim()).collect(); - if parts.len() < 4 { continue; } - let module = parts[0].to_string(); - let ordinal = match u32::from_str_radix(parts[1].trim_start_matches("0x").trim_start_matches("0X"), 16) { - Ok(n) => n, - Err(_) => continue, - }; - let name = parts[2].to_string(); - entries.push((ordinal, name, module)); - } - entries -} +use std::{env, fs, io::Write}; fn main() { - let out_dir = env::var("OUT_DIR").unwrap(); - let dest = Path::new(&out_dir).join("ordinals.rs"); - let mut f = fs::File::create(&dest).unwrap(); - - // Locate xenia tables relative to the workspace root - // crates/xenia-analysis/ -> ../../ -> workspace root -> ../xenia-canary/ let manifest = env::var("CARGO_MANIFEST_DIR").unwrap(); - let workspace_root = Path::new(&manifest).parent().unwrap().parent().unwrap(); - let project_root = workspace_root.parent().unwrap(); + let json_path = Path::new(&manifest) + .parent() + .and_then(Path::parent) + .unwrap() + .join("docs/reference/xbox360-exports.json"); + println!("cargo:rerun-if-changed={}", json_path.display()); - let krnl_path = project_root - .join("xenia-canary/src/xenia/kernel/xboxkrnl/xboxkrnl_table.inc"); - let xam_path = project_root - .join("xenia-canary/src/xenia/kernel/xam/xam_table.inc"); + let raw = fs::read_to_string(&json_path).unwrap_or_else(|e| { + panic!( + "sylpheed-xexdb: cannot read the export table at {} ({e}).\n\ + Import names would all resolve to None and the database would be \ + silently poorer, so this is a hard error rather than a warning.", + json_path.display() + ) + }); + let doc: serde_json::Value = serde_json::from_str(&raw).expect("export table is not valid JSON"); - println!("cargo:rerun-if-changed={}", krnl_path.display()); - println!("cargo:rerun-if-changed={}", xam_path.display()); - - let krnl = parse_table(&krnl_path); - let xam = parse_table(&xam_path); - - writeln!(f, "/// Auto-generated from xenia's export tables.").unwrap(); - writeln!(f, "pub fn resolve_ordinal(lib: &str, ordinal: u16) -> Option<&'static str> {{").unwrap(); + let out = Path::new(&env::var("OUT_DIR").unwrap()).join("ordinals.rs"); + let mut f = fs::File::create(&out).unwrap(); + writeln!(f, "/// Auto-generated from `docs/reference/xbox360-exports.json`.").unwrap(); + writeln!( + f, + "pub fn resolve_ordinal(lib: &str, ordinal: u16) -> Option<&'static str> {{" + ) + .unwrap(); writeln!(f, " match lib {{").unwrap(); - // xboxkrnl.exe - writeln!(f, " \"xboxkrnl.exe\" => match ordinal {{").unwrap(); - for (ord, name, _) in &krnl { - writeln!(f, " 0x{ord:04X} => Some(\"{name}\"),").unwrap(); + let mut total = 0usize; + for (module, file) in [ + ("xboxkrnl", "xboxkrnl.exe"), + ("xam", "xam.xex"), + ("xbdm", "xbdm.xex"), + ] { + writeln!(f, " \"{file}\" => match ordinal {{").unwrap(); + let exports = doc["modules"][module]["exports"] + .as_array() + .unwrap_or_else(|| panic!("no exports array for module {module}")); + for e in exports { + let (Some(ord), Some(name)) = (e["ordinal"].as_u64(), e["name"].as_str()) else { + continue; + }; + if ord > u16::MAX as u64 { + continue; + } + writeln!(f, " 0x{ord:04X} => Some(\"{name}\"),").unwrap(); + total += 1; + } + writeln!(f, " _ => None,\n }},").unwrap(); } - writeln!(f, " _ => None,").unwrap(); - writeln!(f, " }},").unwrap(); + writeln!(f, " _ => None,\n }}\n}}").unwrap(); - // xam.xex - writeln!(f, " \"xam.xex\" => match ordinal {{").unwrap(); - for (ord, name, _) in &xam { - writeln!(f, " 0x{ord:04X} => Some(\"{name}\"),").unwrap(); - } - writeln!(f, " _ => None,").unwrap(); - writeln!(f, " }},").unwrap(); - - writeln!(f, " _ => None,").unwrap(); - writeln!(f, " }}").unwrap(); - writeln!(f, "}}").unwrap(); - - eprintln!("ordinals.rs: {} xboxkrnl + {} xam entries", krnl.len(), xam.len()); + // A count in the build log, so "the table is there" is observable rather + // than assumed. 2,913 is what Phase 2 adopted. + println!("cargo:warning=sylpheed-xexdb: {total} ordinals from xbox360-exports.json"); } diff --git a/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs b/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs index efbeddec..8d8bc894 100644 --- a/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs +++ b/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs @@ -8,6 +8,8 @@ //! ⚠️ The database is **DuckDB**, not SQLite. `xenia-rs`'s own `--db` help said //! SQLite in two places and was wrong; `docs/agents/CONSOLIDATION.md` Phase 3. +use std::time::Instant; + use anyhow::Result; use clap::{Parser, Subcommand, ValueEnum}; use tracing::{debug, info, instrument, warn}; @@ -22,6 +24,19 @@ struct Cli { log_filter: Option, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +enum AnalyzeMode { + /// Rust passes only (`func.rs`, `xref.rs`). No SQL views. + Rust, + /// Rust passes + additive SQL views (`v_branch_xrefs`, `v_call_graph`, + /// `v_reachability_from_entry`, `v_function_first_instruction`, + /// `v_imports_called`). + Sql, + /// Same as `sql`, plus a Rust-vs-SQL cross-check on branch xrefs at + /// the end. Disagreement is logged as a warning (non-fatal). + Both, +} + #[derive(Subcommand)] enum Commands { @@ -59,7 +74,7 @@ enum Commands { /// Output directory (default: same directory as input) #[arg(short, long)] output: Option, - /// Write base tables (metadata, sections, imports) to a SQLite database + /// Write base tables (metadata, sections, imports) to a DuckDB database #[arg(long)] db: Option, }, @@ -71,7 +86,7 @@ enum Commands { /// Output .asm file (default: stdout) #[arg(short, long)] output: Option, - /// Output SQLite database (also includes the base extract tables) + /// Output DuckDB database (also includes the base extract tables) #[arg(long)] db: Option, /// Output JSON Lines file: one structured row per instruction with @@ -110,9 +125,18 @@ enum Commands { }, } -fn parse_hex_u32(s: &str) -> Result { - let t = s.trim_start_matches("0x").trim_start_matches("0X"); - u32::from_str_radix(t, 16).map_err(|e| format!("bad hex address `{s}`: {e}")) +fn load_xex_data(path: &str) -> Result> { + let lower = path.to_lowercase(); + if lower.ends_with(".iso") || lower.ends_with(".xiso") { + use sylpheed_xex::vfs::VfsDevice; + info!("detected disc image, extracting default.xex"); + let disc = sylpheed_xex::vfs::disc_image::DiscImageDevice::open("disc", std::path::Path::new(path)) + .map_err(|e| anyhow::anyhow!("Failed to open disc image: {}", e))?; + disc.read_file("default.xex") + .map_err(|e| anyhow::anyhow!("Failed to extract default.xex from disc image: {}", e)) + } else { + Ok(std::fs::read(path)?) + } } fn main() -> Result<()> { diff --git a/crates/sylpheed-xexdb/src/sinks/text.rs b/crates/sylpheed-xexdb/src/sinks/text.rs index 2b7de344..b073d0e3 100644 --- a/crates/sylpheed-xexdb/src/sinks/text.rs +++ b/crates/sylpheed-xexdb/src/sinks/text.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use std::io::{self, Write}; -use xenia_xex::pe::PeSection; +use sylpheed_xex::pe::PeSection; use crate::disasm::RichDisasmItem; use crate::xref::{XrefKind, section_for_addr}; diff --git a/tools/zq.py b/tools/zq.py index 934914d0..7b4b0c8b 100755 --- a/tools/zq.py +++ b/tools/zq.py @@ -2,8 +2,17 @@ """Sylpheed static-analysis helper over DuckDB `sylpheed.db`. Hides the gotchas: DECIMAL bounds (DuckDB rejects 0x literals), read-only connect, -and the fact that the engine vtable / rdata is NOT in the DB (read it from guest -memory with `xenia-rs exec ... --dump-addr=0x` instead). +and the fact that the engine vtable / rdata is NOT in the DB. + +🔴 THAT LAST LINE USED TO SAY "read it from guest memory with +`xenia-rs exec ... --dump-addr=0x`". That emulator is retired, and the +command no longer exists -- see docs/agents/CONSOLIDATION.md. Two replacements, +both static, neither needing anything to run: + + * the extracted PE is a FLAT VA DUMP: byte offset = VA - 0x82000000, so + `dd`/`xxd` on `*.pe` reads any address directly; + * `sylph-xexdb extract ` regenerates that `.pe` (byte-identical, verified) + alongside a metadata JSON. Usage: zq.py dis # disassemble [lo,hi) (jump-table words shown as .long)