wip: tool README, DuckDB wording, zq.py escape hatch

crates/sylpheed-ppc/src/lib.rs

@@ -10,3 +10,7 @@
 pub mod decoder;
 pub mod disasm;
 pub mod opcode;
+
+pub use decoder::decode;
+pub use disasm::{DisasmItem, DisasmText, disassemble, format as disasm_format, iter_disasm};
+pub use opcode::PpcOpcode;

crates/sylpheed-xexdb/Cargo.toml

@@ -21,3 +21,6 @@ duckdb = { version = "1", features = ["bundled"] }
 msvc-demangler = "0.11"
 encoding_rs = "0.8"
 clap = { version = "4", features = ["derive"] }
+
+[build-dependencies]
+serde_json = "1"

crates/sylpheed-xexdb/README.md

@@ -0,0 +1,51 @@
+# `sylph-xexdb` — the title's XEX, as a queryable database
+
+```bash
+sylph-xexdb dis <disc.iso|default.xex> --db sylpheed.db --analyze sql
+tools/zq.py dis 0x82341a20 0x82341b00      # then query it
+```
+
+Extract the PE image, disassemble it, detect functions, resolve cross-references
+and RTTI, and write the lot to **DuckDB**.
+
+## Where it came from, and what did not come with it
+
+This was `xenia-rs` — a from-scratch Rust Xbox 360 emulator, started because
+Canary would not run on Linux. Canary later did, with fixes from us, and the
+emulator was retired. **The static analysis is the part that stayed useful**, so
+it came here and the emulator did not: no interpreter, no JIT, no scheduler, no
+GPU, no kernel. About 17,000 lines of ~71,000.
+
+That was possible because the coupling was three symbols — `decoder::decode`,
+`disasm::DisasmItem`, `disasm::format` — now `sylpheed-ppc`.
+
+| crate | what it is |
+|---|---|
+| `sylpheed-xex` | the XEX2 container: decrypt, LZX, PE image, resources, and the disc image it may live in |
+| `sylpheed-ppc` | PowerPC decode and disassembly |
+| `sylpheed-xexdb` | the analysis passes, the schema, and this binary |
+
+## ⚠️ Two things that will mislead you
+
+**The database is DuckDB.** `xenia-rs`'s own `--db` help said "SQLite" in two
+places and was wrong. Query it with `python3 -c "import duckdb"`, or `zq.py`.
+
+**`indirect_dispatch_candidates` is deliberately not a cross product.** A
+`bcctrl` through `this->vptr` at offset 0 matches nearly every class, so one
+site can claim 700+ callees. Sites past `--max-indirect-candidates` record a
+truthful `candidate_count` with `truncated` set and emit **no** candidate rows.
+On this title that is 6,556 of 6,983 sites, standing for 1,801,075 candidates
+that are counted rather than materialised. An older database built without the
+ceiling has ~1.8M more rows in that table and in `xrefs`, and they say nothing
+extra.
+
+## The export table
+
+Import names come from `docs/reference/xbox360-exports.json` — 2,913 ordinals,
+compiled in by `build.rs`.
+
+🔴 It used to come from Canary's `xboxkrnl_table.inc` through **a relative path
+to a sibling checkout**, which printed a warning and produced an empty table
+whenever that checkout was not there. Every import in the database then resolved
+to nothing, and the build still succeeded. Now the source is in this repository
+and a missing file **fails the build**.

crates/sylpheed-xexdb/build.rs

@@ -1,87 +1,76 @@
-//! Build script: parse xenia's xboxkrnl_table.inc and xam_table.inc to generate
-//! ordinal->name lookup tables at compile time.
+//! Generate the ordinal → export-name table that import resolution needs.
+//!
+//! 🔴 **THE SOURCE CHANGED, AND THE OLD ONE COULD FAIL SILENTLY.** In `xenia-rs`
+//! this parsed Canary's `xboxkrnl_table.inc` / `xam_table.inc` through a
+//! **relative path to a sibling checkout** — `../xenia-canary/src/…`. When that
+//! path was not there (a container, another machine, a fresh clone) it printed a
+//! `cargo:warning` and returned an empty table, and every import in the database
+//! came out unresolved. A build that quietly produces a worse artifact is the
+//! failure mode this project keeps meeting.
+//!
+//! The source is now `docs/reference/xbox360-exports.json`, which is **in this
+//! repository** — 2,913 exports across `xboxkrnl.exe`, `xam.xex` and `xbdm.xex`,
+//! adopted in Phase 2 of `docs/agents/CONSOLIDATION.md`. There is no sibling
+//! checkout to be missing, and if the file is unreadable this **fails the
+//! build** rather than degrading.
 
-use std::env;
-use std::fs;
-use std::io::Write;
 use std::path::Path;
-
-fn parse_table(path: &Path) -> Vec<(u32, String, String)> {
-    let content = match fs::read_to_string(path) {
-        Ok(c) => c,
-        Err(e) => {
-            eprintln!("cargo:warning=could not read {}: {}", path.display(), e);
-            return Vec::new();
-        }
-    };
-
-    let mut entries = Vec::new();
-    for line in content.lines() {
-        let line = line.trim();
-        // XE_EXPORT(module, 0xNNNNNNNN, Name, kType),
-        if !line.starts_with("XE_EXPORT(") { continue; }
-        let inner = match line.strip_prefix("XE_EXPORT(").and_then(|s| s.strip_suffix("),")) {
-            Some(s) => s,
-            None => continue,
-        };
-        let parts: Vec<&str> = inner.splitn(4, ',').map(|s| s.trim()).collect();
-        if parts.len() < 4 { continue; }
-        let module = parts[0].to_string();
-        let ordinal = match u32::from_str_radix(parts[1].trim_start_matches("0x").trim_start_matches("0X"), 16) {
-            Ok(n) => n,
-            Err(_) => continue,
-        };
-        let name = parts[2].to_string();
-        entries.push((ordinal, name, module));
-    }
-    entries
-}
+use std::{env, fs, io::Write};
 
 fn main() {
-    let out_dir = env::var("OUT_DIR").unwrap();
-    let dest = Path::new(&out_dir).join("ordinals.rs");
-    let mut f = fs::File::create(&dest).unwrap();
-
-    // Locate xenia tables relative to the workspace root
-    // crates/xenia-analysis/ -> ../../ -> workspace root -> ../xenia-canary/
     let manifest = env::var("CARGO_MANIFEST_DIR").unwrap();
-    let workspace_root = Path::new(&manifest).parent().unwrap().parent().unwrap();
-    let project_root = workspace_root.parent().unwrap();
+    let json_path = Path::new(&manifest)
+        .parent()
+        .and_then(Path::parent)
+        .unwrap()
+        .join("docs/reference/xbox360-exports.json");
+    println!("cargo:rerun-if-changed={}", json_path.display());
 
-    let krnl_path = project_root
-        .join("xenia-canary/src/xenia/kernel/xboxkrnl/xboxkrnl_table.inc");
-    let xam_path = project_root
-        .join("xenia-canary/src/xenia/kernel/xam/xam_table.inc");
+    let raw = fs::read_to_string(&json_path).unwrap_or_else(|e| {
+        panic!(
+            "sylpheed-xexdb: cannot read the export table at {} ({e}).\n\
+             Import names would all resolve to None and the database would be \
+             silently poorer, so this is a hard error rather than a warning.",
+            json_path.display()
+        )
+    });
+    let doc: serde_json::Value = serde_json::from_str(&raw).expect("export table is not valid JSON");
 
-    println!("cargo:rerun-if-changed={}", krnl_path.display());
-    println!("cargo:rerun-if-changed={}", xam_path.display());
-
-    let krnl = parse_table(&krnl_path);
-    let xam = parse_table(&xam_path);
-
-    writeln!(f, "/// Auto-generated from xenia's export tables.").unwrap();
-    writeln!(f, "pub fn resolve_ordinal(lib: &str, ordinal: u16) -> Option<&'static str> {{").unwrap();
+    let out = Path::new(&env::var("OUT_DIR").unwrap()).join("ordinals.rs");
+    let mut f = fs::File::create(&out).unwrap();
+    writeln!(f, "/// Auto-generated from `docs/reference/xbox360-exports.json`.").unwrap();
+    writeln!(
+        f,
+        "pub fn resolve_ordinal(lib: &str, ordinal: u16) -> Option<&'static str> {{"
+    )
+    .unwrap();
     writeln!(f, "    match lib {{").unwrap();
 
-    // xboxkrnl.exe
-    writeln!(f, "        \"xboxkrnl.exe\" => match ordinal {{").unwrap();
-    for (ord, name, _) in &krnl {
-        writeln!(f, "            0x{ord:04X} => Some(\"{name}\"),").unwrap();
+    let mut total = 0usize;
+    for (module, file) in [
+        ("xboxkrnl", "xboxkrnl.exe"),
+        ("xam", "xam.xex"),
+        ("xbdm", "xbdm.xex"),
+    ] {
+        writeln!(f, "        \"{file}\" => match ordinal {{").unwrap();
+        let exports = doc["modules"][module]["exports"]
+            .as_array()
+            .unwrap_or_else(|| panic!("no exports array for module {module}"));
+        for e in exports {
+            let (Some(ord), Some(name)) = (e["ordinal"].as_u64(), e["name"].as_str()) else {
+                continue;
+            };
+            if ord > u16::MAX as u64 {
+                continue;
+            }
+            writeln!(f, "            0x{ord:04X} => Some(\"{name}\"),").unwrap();
+            total += 1;
+        }
+        writeln!(f, "            _ => None,\n        }},").unwrap();
     }
-    writeln!(f, "            _ => None,").unwrap();
-    writeln!(f, "        }},").unwrap();
+    writeln!(f, "        _ => None,\n    }}\n}}").unwrap();
 
-    // xam.xex
-    writeln!(f, "        \"xam.xex\" => match ordinal {{").unwrap();
-    for (ord, name, _) in &xam {
-        writeln!(f, "            0x{ord:04X} => Some(\"{name}\"),").unwrap();
-    }
-    writeln!(f, "            _ => None,").unwrap();
-    writeln!(f, "        }},").unwrap();
-
-    writeln!(f, "        _ => None,").unwrap();
-    writeln!(f, "    }}").unwrap();
-    writeln!(f, "}}").unwrap();
-
-    eprintln!("ordinals.rs: {} xboxkrnl + {} xam entries", krnl.len(), xam.len());
+    // A count in the build log, so "the table is there" is observable rather
+    // than assumed. 2,913 is what Phase 2 adopted.
+    println!("cargo:warning=sylpheed-xexdb: {total} ordinals from xbox360-exports.json");
 }

crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs

@@ -8,6 +8,8 @@
 //! ⚠️ The database is **DuckDB**, not SQLite. `xenia-rs`'s own `--db` help said
 //! SQLite in two places and was wrong; `docs/agents/CONSOLIDATION.md` Phase 3.
 
+use std::time::Instant;
+
 use anyhow::Result;
 use clap::{Parser, Subcommand, ValueEnum};
 use tracing::{debug, info, instrument, warn};
@@ -22,6 +24,19 @@ struct Cli {
     log_filter: Option<String>,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)]
+enum AnalyzeMode {
+    /// Rust passes only (`func.rs`, `xref.rs`). No SQL views.
+    Rust,
+    /// Rust passes + additive SQL views (`v_branch_xrefs`, `v_call_graph`,
+    /// `v_reachability_from_entry`, `v_function_first_instruction`,
+    /// `v_imports_called`).
+    Sql,
+    /// Same as `sql`, plus a Rust-vs-SQL cross-check on branch xrefs at
+    /// the end. Disagreement is logged as a warning (non-fatal).
+    Both,
+}
+
 #[derive(Subcommand)]
 enum Commands {
 
@@ -59,7 +74,7 @@ enum Commands {
         /// Output directory (default: same directory as input)
         #[arg(short, long)]
         output: Option<String>,
-        /// Write base tables (metadata, sections, imports) to a SQLite database
+        /// Write base tables (metadata, sections, imports) to a DuckDB database
         #[arg(long)]
         db: Option<String>,
     },
@@ -71,7 +86,7 @@ enum Commands {
         /// Output .asm file (default: stdout)
         #[arg(short, long)]
         output: Option<String>,
-        /// Output SQLite database (also includes the base extract tables)
+        /// Output DuckDB database (also includes the base extract tables)
         #[arg(long)]
         db: Option<String>,
         /// Output JSON Lines file: one structured row per instruction with
@@ -110,9 +125,18 @@ enum Commands {
     },
 }
 
-fn parse_hex_u32(s: &str) -> Result<u32, String> {
-    let t = s.trim_start_matches("0x").trim_start_matches("0X");
-    u32::from_str_radix(t, 16).map_err(|e| format!("bad hex address `{s}`: {e}"))
+fn load_xex_data(path: &str) -> Result<Vec<u8>> {
+    let lower = path.to_lowercase();
+    if lower.ends_with(".iso") || lower.ends_with(".xiso") {
+        use sylpheed_xex::vfs::VfsDevice;
+        info!("detected disc image, extracting default.xex");
+        let disc = sylpheed_xex::vfs::disc_image::DiscImageDevice::open("disc", std::path::Path::new(path))
+            .map_err(|e| anyhow::anyhow!("Failed to open disc image: {}", e))?;
+        disc.read_file("default.xex")
+            .map_err(|e| anyhow::anyhow!("Failed to extract default.xex from disc image: {}", e))
+    } else {
+        Ok(std::fs::read(path)?)
+    }
 }
 
 fn main() -> Result<()> {

crates/sylpheed-xexdb/src/sinks/text.rs

@@ -8,7 +8,7 @@
 use std::collections::HashMap;
 use std::io::{self, Write};
 
-use xenia_xex::pe::PeSection;
+use sylpheed_xex::pe::PeSection;
 
 use crate::disasm::RichDisasmItem;
 use crate::xref::{XrefKind, section_for_addr};

tools/zq.py

@@ -2,8 +2,17 @@
 """Sylpheed static-analysis helper over DuckDB `sylpheed.db`.
 
 Hides the gotchas: DECIMAL bounds (DuckDB rejects 0x literals), read-only connect,
-and the fact that the engine vtable / rdata is NOT in the DB (read it from guest
-memory with `xenia-rs exec ... --dump-addr=0x<va>` instead).
+and the fact that the engine vtable / rdata is NOT in the DB.
+
+🔴 THAT LAST LINE USED TO SAY "read it from guest memory with
+`xenia-rs exec ... --dump-addr=0x<va>`". That emulator is retired, and the
+command no longer exists -- see docs/agents/CONSOLIDATION.md. Two replacements,
+both static, neither needing anything to run:
+
+  * the extracted PE is a FLAT VA DUMP: byte offset = VA - 0x82000000, so
+    `dd`/`xxd` on `*.pe` reads any address directly;
+  * `sylph-xexdb extract <xex>` regenerates that `.pe` (byte-identical, verified)
+    alongside a metadata JSON.
 
 Usage:
   zq.py dis  <lo_hex> <hi_hex>      # disassemble [lo,hi)  (jump-table words shown as .long)