diff --git a/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs b/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs index af6b5db4..48fda0de 100644 --- a/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs +++ b/crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs @@ -919,6 +919,16 @@ fn cmd_dis( info!(db = %db, "database written"); } + // Word-level index of the import thunks, used to annotate the instruction + // stream: which words are loader-patched import records, and which + // branches are really kernel/xam calls. The database path builds its own + // inside `ingest_instructions`; these two output paths need it here. + let import_sites = sylpheed_xexdb::ImportSites::build( + disasm_info.import_libraries, + &pe_image, + disasm_info.image_base, + ); + // JSON Lines output: one row per instruction, structured columns. if let Some(json) = json_path { info!(json = %json, "writing JSON Lines"); @@ -939,6 +949,7 @@ fn cmd_dis( &func_analysis, &xref_result.labels, &jt_data_words, + &import_sites, ); total += sylpheed_xexdb::sinks::json::write_jsonl(&mut out, items)?; } @@ -962,6 +973,7 @@ fn cmd_dis( &xref_result.xrefs, &xref_result.data_annotations, &jt_data_words, + &import_sites, )?; if let Some(path) = output { diff --git a/crates/sylpheed-xexdb/src/db.rs b/crates/sylpheed-xexdb/src/db.rs index 76ea3bd8..6f3ba5f7 100644 --- a/crates/sylpheed-xexdb/src/db.rs +++ b/crates/sylpheed-xexdb/src/db.rs @@ -15,6 +15,33 @@ //! - `"jump"` : bcctrx without LK //! - `"branch"` : bx/bcx without LK //! +//! # Referential integrity +//! +//! The schema declares foreign keys wherever a column is *derived from* the +//! row it points at, so a build that would produce a dangling reference fails +//! loudly instead of shipping a broken database. Two consequences: +//! +//! - **Insert order is load-bearing.** DuckDB checks each row as it is +//! appended, so a parent must be populated before its children, and +//! `CREATE TABLE` statements must be ordered so a table exists before +//! anything references it. `write_analysis_results` documents its order. +//! - **`functions` is deliberately NOT a foreign-key parent.** DuckDB +//! implements `UPDATE` as delete-then-insert, so *any* FK onto +//! `functions(address)` makes the table un-updatable — even an update that +//! does not touch `address`. That would break `apply_re_symbols.sql`, which +//! re-stamps reverse-engineered names onto `functions.name` after every +//! regeneration. `xrefs.source_func`, `vptr_writes.writer_function` and +//! `jump_tables.function` are therefore plain columns. The same restriction +//! applies to any table a human post-edits. +//! +//! Some references are intentionally left undeclared because the relationship +//! genuinely does not hold: see `rtti_locators.vtable_address` (41 RTTI-named +//! vftables the M3 scanner misses), `methods.function_address` and +//! `indirect_dispatch_candidates.method_address` (vtable slots pointing at +//! code no `.pdata` record or `bl` ever declared), and `xrefs.target` (which +//! is frequently data). Those orphan sets are findings, not noise — querying +//! them is how the gaps stay visible. +//! //! # Schema //! //! ## `metadata` @@ -48,8 +75,9 @@ //! - `library` — Module name (e.g. `xboxkrnl.exe`, `xam.xex`) //! - `ordinal` — Numeric ordinal identifying the export within the library //! - `name` — Resolved human-readable symbol name; `NULL` if not in symbol table -//! - `record_type` — XEX import record type: `0` = function thunk, `1` = variable -//! - `address` — Absolute VA of the import thunk or variable in the binary +//! - `record_type` — XEX import record type: `0` = variable, `1` = function thunk +//! - `address` — Absolute VA of the import thunk or variable in the binary. +//! UNIQUE, because `instructions.import_address` is a foreign key onto it. //! //! ## `functions` //! One row per detected function. Candidates are `bl` targets ∪ `.pdata` @@ -94,9 +122,18 @@ //! - `function` — VA of the enclosing function; `NULL` if not inside a detected function //! - `label` — Label name at this address; `NULL` if none //! - `is_data` — `true` when this word is data embedded in a code section (a -//! recovered jump table or index map). The decoded `mnemonic` / -//! `disasm` columns are meaningless on such rows; filter them out -//! (`WHERE NOT is_data`) for any instruction-level analysis. +//! recovered jump table or index map, or an XEX import record). The decoded +//! `mnemonic` / `disasm` columns are meaningless on such rows; filter them +//! out (`WHERE NOT is_data`) for any instruction-level analysis. +//! - `import_address` — FK onto `imports.address` (a thunk head) when this word is +//! part of an XEX import thunk, or is a direct branch into one; `NULL` +//! otherwise. Join it to name the callee. +//! - `import_role` — Which of the three this row is: `'record'` (one of the two +//! loader-patched words at the thunk head — also `is_data`), `'thunk'` (the +//! `mtctr r11` / `bctr` body), or `'call'` (a `bl`/`b` elsewhere in the binary +//! whose target is that thunk). `NULL` iff `import_address` is `NULL`. Without +//! these columns an import call reads as a branch to a bare address and the +//! thunk head as two meaningless `.long`s. //! //! ## `jump_tables` / `jump_table_entries` / `data_in_code` //! Recovered `switch` dispatches (see [`crate::jumptables`]). `jump_tables` has @@ -153,7 +190,7 @@ //! - `kind` — `call`, `return`, `jump`, or `branch` (see top-level doc) //! - `lr` — Link register value at time of branch -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::Path; use duckdb::{Connection, params}; @@ -220,6 +257,11 @@ pub struct DbWriter { trace_instructions: bool, trace_imports: bool, trace_branches: bool, + /// Import-thunk index, built during `ingest_instructions` and reused when + /// `write_analysis_results` records the thunk records in `data_in_code`. + /// Empty until then — with no instruction rows there is nothing to + /// cross-reference it against. + import_sites: crate::imports::ImportSites, } impl DbWriter { @@ -241,6 +283,7 @@ impl DbWriter { trace_instructions: false, trace_imports: false, trace_branches: false, + import_sites: crate::imports::ImportSites::default(), }) } @@ -270,8 +313,14 @@ impl DbWriter { library VARCHAR NOT NULL, -- module name (e.g. xboxkrnl.exe, xam.xex) ordinal BIGINT NOT NULL, -- ordinal identifying the export within the library name VARCHAR, -- resolved symbol name; NULL if not in symbol table - record_type BIGINT NOT NULL, -- 0 = function thunk, 1 = variable - address BIGINT NOT NULL -- absolute VA of the thunk or variable + -- 0 = variable (a pointer slot in .rdata the loader fills in), + -- 1 = function thunk (four words of code in .text). This was + -- documented the wrong way round; `ImportEntry::record_type` + -- in xenia-xex has always had it right. + record_type BIGINT NOT NULL, + -- Absolute VA of the thunk or variable. UNIQUE because + -- `instructions.import_address` is a foreign key onto it. + address BIGINT NOT NULL UNIQUE ); ", )?; @@ -322,11 +371,31 @@ impl DbWriter { section VARCHAR NOT NULL, -- PE section name containing this instruction function BIGINT, -- VA of the enclosing function; NULL if unknown label VARCHAR, -- label at this address; NULL if none - is_data BOOLEAN NOT NULL -- M12: word is data embedded in code (jump table / index map), NOT an instruction + is_data BOOLEAN NOT NULL, -- M12: word is data embedded in code (jump table / index map / import record), NOT an instruction + -- XEX import this word references, as a foreign key into `imports` + -- (the `record_type = 1` row, whose address is the thunk head). + -- NULL for everything that is not part of, or a branch into, an + -- import thunk. See `import_role` for which of the three it is. + import_address BIGINT REFERENCES imports(address), + import_role VARCHAR -- 'record' | 'thunk' | 'call'; NULL iff import_address IS NULL ); ")?; - insert_instructions_streaming(&self.conn, pe, info, func_analysis, labels, data_words)?; + // Built here rather than passed in: everything it needs (the parsed + // import table and the loaded image) is already on `info`/`pe`, and + // the index is only ever consumed by this one streaming pass. + self.import_sites = + crate::imports::ImportSites::build(info.import_libraries, pe, info.image_base); + + insert_instructions_streaming( + &self.conn, + pe, + info, + func_analysis, + labels, + data_words, + &self.import_sites, + )?; let indices = [ ( @@ -357,6 +426,10 @@ impl DbWriter { "idx_instructions_is_data", "CREATE INDEX idx_instructions_is_data ON instructions(is_data)", ), + ( + "idx_instructions_import", + "CREATE INDEX idx_instructions_import ON instructions(import_address)", + ), ]; for (name, sql) in indices { tracing::debug!(index = name, "creating instructions index"); @@ -422,17 +495,63 @@ impl DbWriter { kind VARCHAR NOT NULL -- function | import | saverestore | local | data | other ); + -- M13 — MSVC RTTI. `rtti_type_descriptors` is the authoritative + -- source of C++ class identity: the linker wrote these names, they + -- are not inferred. + CREATE TABLE rtti_type_descriptors ( + address BIGINT PRIMARY KEY, -- VA of the TypeDescriptor + mangled_name VARCHAR NOT NULL, -- decorated name, e.g. .?AVSilph@silph@@ + demangled_name VARCHAR NOT NULL -- readable form, e.g. silph::Silph + ); + + -- M13 — RTTICompleteObjectLocator. One per emitted vftable; the + -- `subobject_offset` column separates a class's primary vftable (0) + -- from the extra vftables it emits for secondary base subobjects. + CREATE TABLE rtti_locators ( + address BIGINT PRIMARY KEY, + subobject_offset BIGINT NOT NULL, -- this-offset of the subobject this vftable serves + cd_offset BIGINT NOT NULL, -- constructor-displacement offset + type_descriptor BIGINT NOT NULL REFERENCES rtti_type_descriptors(address), + class_hierarchy BIGINT NOT NULL, -- VA of the RTTIClassHierarchyDescriptor + -- VA of vftable[0]; NULL if no referencing word was found. + -- Deliberately NOT a foreign key: on the reference title 41 + -- locators name a vftable the M3 scanner did not detect (see + -- `crate::vtables`), so declaring one would fail the build on + -- a real gap rather than reporting it. Query the orphans with + -- SELECT * FROM rtti_locators l WHERE NOT EXISTS + -- (SELECT 1 FROM vtables v WHERE v.address = l.vtable_address) + vtable_address BIGINT + ); + + -- M13 — linearised RTTIBaseClassArray. Index 0 is the class itself; + -- the remaining rows are its bases in MSVC's depth-first order, + -- each with the PMD displacement triple needed to locate the base + -- subobject inside an instance. + CREATE TABLE rtti_base_classes ( + class_hierarchy BIGINT NOT NULL, -- VA of the deriving class's hierarchy descriptor + base_index BIGINT NOT NULL, -- position in the base-class array + type_descriptor BIGINT NOT NULL REFERENCES rtti_type_descriptors(address), + name VARCHAR NOT NULL, + num_contained_bases BIGINT NOT NULL, + mdisp BIGINT NOT NULL, -- member displacement + pdisp BIGINT NOT NULL, -- vbtable displacement (-1 = non-virtual base) + vdisp BIGINT NOT NULL, -- displacement inside the vbtable + attributes BIGINT NOT NULL, + PRIMARY KEY (class_hierarchy, base_index) + ); + CREATE TABLE vtables ( address BIGINT PRIMARY KEY, -- absolute VA of vtable[0] length BIGINT NOT NULL, -- number of method slots - col_address BIGINT, -- VA of CompleteObjectLocator (NULL when no RTTI) + -- VA of the CompleteObjectLocator; NULL when the class has no RTTI. + col_address BIGINT REFERENCES rtti_locators(address), class_name VARCHAR NOT NULL, -- demangled class name OR ANON_Class_ when stripped rtti_present BOOLEAN NOT NULL, -- true when COL → TypeDescriptor walk succeeded base_classes_json VARCHAR -- JSON array of base class names (NULL if none / parse failure) ); CREATE TABLE methods ( - vtable_address BIGINT NOT NULL, -- vtable this slot belongs to + vtable_address BIGINT NOT NULL REFERENCES vtables(address), -- vtable this slot belongs to slot BIGINT NOT NULL, -- 0-based slot index function_address BIGINT NOT NULL, -- VA of the function this slot points at mangled_name VARCHAR, -- raw label name when mangled (?...) @@ -442,7 +561,7 @@ impl DbWriter { CREATE TABLE classes ( name VARCHAR PRIMARY KEY, -- class name (demangled or ANON_*) - vtable_address BIGINT NOT NULL, -- representative vtable (first detected) + vtable_address BIGINT NOT NULL REFERENCES vtables(address), -- representative vtable (first detected) rtti_present BOOLEAN NOT NULL, base_classes_json VARCHAR -- JSON of base class names (NULL when stripped) ); @@ -476,7 +595,7 @@ impl DbWriter { ); CREATE TABLE function_pointer_array_entries ( - array_address BIGINT NOT NULL, -- FK to function_pointer_arrays.address + array_address BIGINT NOT NULL REFERENCES function_pointer_arrays(address), slot BIGINT NOT NULL, -- 0-based slot index function_address BIGINT NOT NULL, -- VA of the function this slot points at PRIMARY KEY (array_address, slot) @@ -503,8 +622,10 @@ impl DbWriter { -- ind_call xref edges in the `xrefs` table are derived from -- this; this view lets you join back to vtable / method info. CREATE TABLE indirect_dispatch_candidates ( - dispatch_pc BIGINT NOT NULL, - vtable_address BIGINT NOT NULL, + dispatch_pc BIGINT NOT NULL REFERENCES indirect_dispatch_sites(dispatch_pc), + vtable_address BIGINT NOT NULL REFERENCES vtables(address), + -- No FK: a vtable slot may point at code `.pdata` never declared + -- and no `bl` ever targets, so this is not always in `functions`. method_address BIGINT NOT NULL, PRIMARY KEY (dispatch_pc, vtable_address) ); @@ -514,7 +635,7 @@ impl DbWriter { -- has (or does not have) coverage in the dispatch resolver. CREATE TABLE vptr_writes ( writer_pc BIGINT NOT NULL, - vtable_address BIGINT NOT NULL, + vtable_address BIGINT NOT NULL REFERENCES vtables(address), vptr_offset BIGINT NOT NULL, writer_function BIGINT NOT NULL, PRIMARY KEY (writer_pc, vtable_address, vptr_offset) @@ -536,7 +657,7 @@ impl DbWriter { ); CREATE TABLE eh_unwind_map ( - funcinfo_address BIGINT NOT NULL, -- FK to eh_funcinfo.address + funcinfo_address BIGINT NOT NULL REFERENCES eh_funcinfo(address), state_index BIGINT NOT NULL, to_state BIGINT NOT NULL, action_pc BIGINT NOT NULL, @@ -544,7 +665,7 @@ impl DbWriter { ); CREATE TABLE eh_try_blocks ( - funcinfo_address BIGINT NOT NULL, -- FK to eh_funcinfo.address + funcinfo_address BIGINT NOT NULL REFERENCES eh_funcinfo(address), try_index BIGINT NOT NULL, try_low BIGINT NOT NULL, try_high BIGINT NOT NULL, @@ -566,6 +687,14 @@ impl DbWriter { PRIMARY KEY (namespace, id) ); + CREATE TABLE xdbf_images ( + id BIGINT PRIMARY KEY, -- image id referenced by achievements + is_title_icon BOOLEAN NOT NULL, -- id 0x8000 — the title's own icon + body_offset BIGINT NOT NULL, -- offset within the image buffer + size BIGINT NOT NULL, + format VARCHAR NOT NULL -- 'png' when the body carries the PNG signature + ); + CREATE TABLE xdbf_achievements ( id BIGINT PRIMARY KEY, -- 1-based achievement id name VARCHAR, -- resolved via the default language's string table @@ -574,7 +703,7 @@ impl DbWriter { label_id BIGINT NOT NULL, -- string ids, for joining other languages description_id BIGINT NOT NULL, unachieved_id BIGINT NOT NULL, - image_id BIGINT NOT NULL, -- FK to xdbf_images.id + image_id BIGINT NOT NULL REFERENCES xdbf_images(id), gamerscore BIGINT NOT NULL, flags BIGINT NOT NULL ); @@ -589,14 +718,6 @@ impl DbWriter { PRIMARY KEY (language, string_id) ); - CREATE TABLE xdbf_images ( - id BIGINT PRIMARY KEY, -- image id referenced by achievements - is_title_icon BOOLEAN NOT NULL, -- id 0x8000 — the title's own icon - body_offset BIGINT NOT NULL, -- offset within the image buffer - size BIGINT NOT NULL, - format VARCHAR NOT NULL -- 'png' when the body carries the PNG signature - ); - CREATE TABLE demangled_names ( address BIGINT, -- VA the mangled name is associated with; NULL when from a non-address source (e.g. RTTI-only string) mangled VARCHAR NOT NULL, -- original mangled symbol (e.g. ?Foo@Bar@@QEAAXXZ) @@ -624,7 +745,7 @@ impl DbWriter { -- M12 — one row per case value, in case order. `target_address` -- repeats when several case values share a body. CREATE TABLE jump_table_entries ( - bctr_pc BIGINT NOT NULL, -- FK to jump_tables.bctr_pc + bctr_pc BIGINT NOT NULL REFERENCES jump_tables(bctr_pc), case_index BIGINT NOT NULL, -- 0-based case value target_address BIGINT NOT NULL, -- VA of the case body PRIMARY KEY (bctr_pc, case_index) @@ -634,52 +755,16 @@ impl DbWriter { -- instructions. Anything listed here is a decoding hazard: linear -- disassembly of these words produces garbage rows. CREATE TABLE data_in_code ( - address BIGINT PRIMARY KEY, -- VA of the first byte + address BIGINT PRIMARY KEY REFERENCES instructions(address), -- VA of the first byte length BIGINT NOT NULL, -- byte length - kind VARCHAR NOT NULL -- 'jump_table' | 'jump_index_map' - ); - - -- M13 — MSVC RTTI. `rtti_type_descriptors` is the authoritative - -- source of C++ class identity: the linker wrote these names, they - -- are not inferred. - CREATE TABLE rtti_type_descriptors ( - address BIGINT PRIMARY KEY, -- VA of the TypeDescriptor - mangled_name VARCHAR NOT NULL, -- decorated name, e.g. .?AVSilph@silph@@ - demangled_name VARCHAR NOT NULL -- readable form, e.g. silph::Silph - ); - - -- M13 — RTTICompleteObjectLocator. One per emitted vftable; the - -- `subobject_offset` column separates a class's primary vftable (0) - -- from the extra vftables it emits for secondary base subobjects. - CREATE TABLE rtti_locators ( - address BIGINT PRIMARY KEY, - subobject_offset BIGINT NOT NULL, -- this-offset of the subobject this vftable serves - cd_offset BIGINT NOT NULL, -- constructor-displacement offset - type_descriptor BIGINT NOT NULL, -- FK to rtti_type_descriptors.address - class_hierarchy BIGINT NOT NULL, -- VA of the RTTIClassHierarchyDescriptor - vtable_address BIGINT -- VA of vftable[0]; NULL if no referencing word was found - ); - - -- M13 — linearised RTTIBaseClassArray. Index 0 is the class itself; - -- the remaining rows are its bases in MSVC's depth-first order, - -- each with the PMD displacement triple needed to locate the base - -- subobject inside an instance. - CREATE TABLE rtti_base_classes ( - class_hierarchy BIGINT NOT NULL, -- VA of the deriving class's hierarchy descriptor - base_index BIGINT NOT NULL, -- position in the base-class array - type_descriptor BIGINT NOT NULL, - name VARCHAR NOT NULL, - num_contained_bases BIGINT NOT NULL, - mdisp BIGINT NOT NULL, -- member displacement - pdisp BIGINT NOT NULL, -- vbtable displacement (-1 = non-virtual base) - vdisp BIGINT NOT NULL, -- displacement inside the vbtable - attributes BIGINT NOT NULL, - PRIMARY KEY (class_hierarchy, base_index) + kind VARCHAR NOT NULL -- 'jump_table' | 'jump_index_map' | 'import_record' ); CREATE TABLE xrefs ( - source BIGINT NOT NULL, -- VA of the referencing instruction - target BIGINT NOT NULL, -- VA of the referenced destination + source BIGINT NOT NULL REFERENCES instructions(address), + -- No FK: a target may be data (.rdata/.data) or an address with + -- no decoded word, so it is not always an `instructions` row. + target BIGINT NOT NULL, kind VARCHAR NOT NULL, -- call | ind_call | j | br | read | write | ref addr_mode VARCHAR, -- M6 sub-classification of how source computes target (NULL for control-flow) instruction VARCHAR, -- mnemonic of source instruction; NULL if not in binary @@ -703,17 +788,24 @@ impl DbWriter { // which is both fast and flat in memory. It bypasses the SQL layer, // so `ON CONFLICT DO NOTHING` is unavailable and each converted sink // documents why its key cannot collide (or dedupes explicitly). + // Insertion order is load-bearing: the schema declares foreign keys, + // and DuckDB checks them per row, so a parent table must be populated + // before any child that references it. `insert_rtti` runs ahead of + // `insert_vtables` because `vtables.col_address` points into + // `rtti_locators`; everything else follows the natural dependency + // order (functions -> vtables -> methods/classes -> dispatch). insert_functions(&self.conn, func_analysis, labels)?; insert_pdata_entries(&self.conn, &func_analysis.pdata_entries)?; insert_labels(&self.conn, labels)?; insert_demangled_from_labels(&self.conn, labels, info.import_libraries)?; + insert_rtti(&self.conn, rtti)?; insert_vtables(&self.conn, vtables, pe, info.image_base)?; insert_methods_and_classes(&self.conn, vtables, labels)?; insert_strings(&self.conn, strings)?; insert_funcptr_arrays(&self.conn, funcptr_arrays)?; insert_eh_records(&self.conn, eh_records)?; insert_jump_tables(&self.conn, jump_tables)?; - insert_rtti(&self.conn, rtti)?; + insert_import_record_extents(&self.conn, &self.import_sites)?; insert_xdbf(&self.conn, xdbf)?; if let Some(t) = typed_ind { insert_typed_ind_dispatch(&self.conn, t)?; @@ -1363,8 +1455,24 @@ fn insert_imports(conn: &Connection, info: &DisasmInfo) -> anyhow::Result<()> { "INSERT INTO imports (library, ordinal, name, record_type, address) VALUES (?, ?, ?, ?, ?)", )?; + // `address` is UNIQUE so that `instructions.import_address` can key off + // it. Two entries sharing an address would be a malformed import table — + // one slot cannot hold two imports — but a duplicate must not abort the + // whole database build, so keep the first and say what was dropped. + let mut seen: HashSet = HashSet::new(); + let mut duplicates = 0usize; for lib in info.import_libraries { for imp in &lib.imports { + if !seen.insert(imp.address) { + duplicates += 1; + tracing::warn!( + library = %lib.name, + ordinal = imp.ordinal, + address = format_args!("{:#010x}", imp.address), + "duplicate import address; keeping the first entry", + ); + continue; + } let resolved = crate::resolve_ordinal(&lib.name, imp.ordinal); stmt.execute(params![ lib.name, @@ -1375,6 +1483,9 @@ fn insert_imports(conn: &Connection, info: &DisasmInfo) -> anyhow::Result<()> { ])?; } } + if duplicates > 0 { + tracing::warn!(duplicates, "import table has entries sharing an address"); + } Ok(()) } @@ -1756,6 +1867,23 @@ fn insert_xdbf(conn: &Connection, xdbf: Option<&crate::xdbf::Xdbf>) -> anyhow::R .map(|(_, s)| s.clone()) }; + // Images before achievements: `xdbf_achievements.image_id` is a foreign + // key onto `xdbf_images.id`, so the icon rows must already exist. + let mut stmt = conn.prepare( + "INSERT INTO xdbf_images (id, is_title_icon, body_offset, size, format) + VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING", + )?; + for i in &x.images { + stmt.execute(params![ + i.id as i64, + i.id == crate::xdbf::ID_TITLE, + i.offset as i64, + i.size as i64, + i.format, + ])?; + } + drop(stmt); + let mut stmt = conn.prepare( "INSERT INTO xdbf_achievements (id, name, unlocked_desc, locked_desc, label_id, description_id, @@ -1778,21 +1906,6 @@ fn insert_xdbf(conn: &Connection, xdbf: Option<&crate::xdbf::Xdbf>) -> anyhow::R } drop(stmt); - let mut stmt = conn.prepare( - "INSERT INTO xdbf_images (id, is_title_icon, body_offset, size, format) - VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING", - )?; - for i in &x.images { - stmt.execute(params![ - i.id as i64, - i.id == crate::xdbf::ID_TITLE, - i.offset as i64, - i.size as i64, - i.format, - ])?; - } - drop(stmt); - let mut meta = conn.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)")?; meta.execute(params!["xdbf.entry_count", x.entries.len().to_string()])?; if let Some(l) = x.default_language { @@ -1991,6 +2104,7 @@ fn insert_instructions_streaming( func_analysis: &FuncAnalysis, labels: &HashMap, data_words: &std::collections::BTreeSet, + import_sites: &crate::imports::ImportSites, ) -> anyhow::Result<()> { let mut appender = conn.appender("instructions")?; let mut total: u64 = 0; @@ -2010,6 +2124,7 @@ fn insert_instructions_streaming( func_analysis, labels, data_words, + import_sites, ); total += crate::sinks::duckdb::append_instructions(&mut appender, items)?; } @@ -2020,6 +2135,34 @@ fn insert_instructions_streaming( Ok(()) } +/// Record each import thunk's two loader-rewritten words in `data_in_code`. +/// +/// `data_in_code` already lists recovered jump tables; import records are the +/// other kind of non-instruction word sitting in a code section, so a consumer +/// asking "which bytes of `.text` are not code?" gets a complete answer from +/// one table instead of having to know about thunk layout. +fn insert_import_record_extents( + conn: &Connection, + import_sites: &crate::imports::ImportSites, +) -> anyhow::Result<()> { + let mut heads: Vec = import_sites.thunk_heads().collect(); + heads.sort_unstable(); + let mut stmt = + conn.prepare("INSERT INTO data_in_code (address, length, kind) VALUES (?, ?, ?)")?; + for head in &heads { + // The two records are contiguous at the head of the thunk; the two + // instructions that follow them are real code. + stmt.execute(params![*head as i64, 8i64, "import_record"])?; + } + metrics::counter!("db.rows", "table" => "data_in_code").increment(heads.len() as u64); + tracing::info!( + rows = heads.len(), + kind = "import_record", + "data_in_code write complete" + ); + Ok(()) +} + /// Write the M12 jump-table tables plus the data-in-code extent list. fn insert_jump_tables( conn: &Connection, diff --git a/crates/sylpheed-xexdb/src/disasm.rs b/crates/sylpheed-xexdb/src/disasm.rs index adcb9436..0a2a1e39 100644 --- a/crates/sylpheed-xexdb/src/disasm.rs +++ b/crates/sylpheed-xexdb/src/disasm.rs @@ -10,6 +10,7 @@ use std::collections::{BTreeSet, HashMap}; use sylpheed_ppc::disasm::DisasmItem; use crate::func::FuncAnalysis; +use crate::imports::{ImportRole, ImportSites}; /// `DisasmItem` plus the analysis context (section/function/label). #[derive(Debug, Clone)] @@ -19,8 +20,13 @@ pub struct RichDisasmItem<'a> { pub function: Option, pub label: Option<&'a str>, /// True when this word is data embedded in a code section (a recovered - /// jump table or its index map), so its decoded text is meaningless. + /// jump table or its index map, or an XEX import record), so its decoded + /// text is meaningless. pub is_data: bool, + /// The XEX import this word belongs to, as `(thunk head VA, role)`. The + /// head VA is `imports.address` for that import, so it is written out as + /// a foreign key. `None` for the overwhelming majority of instructions. + pub import: Option<(u32, ImportRole)>, } /// Walk one code section, yielding rich items annotated with section name, @@ -42,6 +48,12 @@ pub struct RichDisasmItem<'a> { /// Rows at those addresses are still emitted — their `raw` value is the table /// entry a consumer wants — but flagged so nothing mistakes the decoded text /// for a real instruction. +/// +/// `import_sites` resolves XEX import thunks. It contributes two things: the +/// two import-record words at each thunk head are also flagged `is_data` +/// (they are loader-rewritten data, not code, and decode to nonsense), and +/// every word of a thunk plus every direct branch into one carries the +/// import it belongs to. pub fn enrich_section<'a>( image: &'a [u8], image_base: u32, @@ -51,6 +63,7 @@ pub fn enrich_section<'a>( func_analysis: &'a FuncAnalysis, labels: &'a HashMap, data_words: &'a BTreeSet, + import_sites: &'a ImportSites, ) -> impl Iterator> + 'a { // (start, end) of the function currently being walked. let mut current: Option<(u32, u32)> = None; @@ -67,13 +80,16 @@ pub fn enrich_section<'a>( } let current_func = current.map(|(start, _)| start); let label = labels.get(&item.addr).map(|s| s.as_str()); - let is_data = data_words.contains(&item.addr); + let import = import_sites.lookup(item.addr, item.text.branch_target); + let is_data = + data_words.contains(&item.addr) || matches!(import, Some((_, ImportRole::Record))); RichDisasmItem { item, section: section_name, function: current_func, label, is_data, + import, } }) } @@ -127,6 +143,7 @@ mod tests { &fa, &labels, &data_words, + &ImportSites::default(), ) .map(|r| (r.item.addr, r.function)) .collect(); @@ -170,6 +187,7 @@ mod tests { &fa, &labels, &data_words, + &ImportSites::default(), ) .map(|r| r.function) .collect(); diff --git a/crates/sylpheed-xexdb/src/formatter.rs b/crates/sylpheed-xexdb/src/formatter.rs index 5e573976..fe6f3dcc 100644 --- a/crates/sylpheed-xexdb/src/formatter.rs +++ b/crates/sylpheed-xexdb/src/formatter.rs @@ -8,6 +8,7 @@ use sylpheed_xex::pe::PeSection; use crate::disasm::enrich_section; use crate::func::FuncAnalysis; +use crate::imports::{ImportRole, ImportSites}; use crate::sinks::text::write_instr_line; use crate::xref::{Xref, XrefKind, XrefMap, resolve_source_label}; @@ -38,6 +39,7 @@ pub fn write_asm( xrefs: &XrefMap, data_annotations: &HashMap, data_words: &BTreeSet, + import_sites: &ImportSites, ) -> anyhow::Result<()> { // Header writeln!( @@ -136,6 +138,7 @@ pub fn write_asm( func_analysis, labels, data_words, + import_sites, ); for ri in items { let abs_addr = ri.item.addr; @@ -208,10 +211,26 @@ pub fn write_asm( } } - // Import thunk annotation + // Import annotations. The thunk head keeps its banner; call + // sites get a trailing note so a `bl` into a thunk reads as + // the kernel call it actually is rather than a bare address. if let Some(imp_name) = import_map.get(&abs_addr) { writeln!(out, " ; IMPORT: {imp_name}")?; } + if let Some((head, role)) = ri.import { + match role { + ImportRole::Record => writeln!( + out, + " ; import record (loader-patched; not an instruction)", + )?, + ImportRole::Call => { + if let Some(name) = import_map.get(&head) { + writeln!(out, " ; -> IMPORT {name}")?; + } + } + ImportRole::Thunk => {} + } + } let data_annot = data_annotations.get(&abs_addr).copied(); write_instr_line(out, &ri, labels, info.sections, info.image_base, data_annot)?; diff --git a/crates/sylpheed-xexdb/src/imports.rs b/crates/sylpheed-xexdb/src/imports.rs new file mode 100644 index 00000000..eaf27940 --- /dev/null +++ b/crates/sylpheed-xexdb/src/imports.rs @@ -0,0 +1,354 @@ +//! Import-thunk recognition — makes XEX imports visible in the code stream. +//! +//! An XEX import is not a PLT-style jump. The linker emits a four-word thunk +//! whose first two words are *import records*, which the loader rewrites at +//! module load into `lis r11, hi(target)` / `ori r11, r11, lo(target)`. On +//! disc they are still records, so a decoder that only knows PowerPC renders +//! them as two meaningless `.long`s in front of an indirect branch: +//! +//! ```text +//! 8284DA7C .long 0x0100028C <- record: type 1, library 0, ordinal 652 +//! 8284DA80 .long 0x0200028C <- record: type 2, library 0, ordinal 652 +//! 8284DA84 mtctr r11 +//! 8284DA88 bctr +//! ``` +//! +//! Record layout, matching xenia-canary `XexModule::SetupLibraryImports`: +//! +//! | bits | meaning | +//! |---------|------------------------------------------------------------| +//! | 31..24 | record type — `1` and `2` mark the two halves of a thunk | +//! | 23..16 | index of the owning import library | +//! | 15..0 | ordinal within that library | +//! +//! This module maps every word of every thunk — and every direct branch that +//! targets one — back to the `imports` row it belongs to, so the disassembly +//! can say `xboxkrnl.exe::RtlEnterCriticalSection` where it used to say +//! `.long 0x01010194`. +//! +//! The mapping is deliberately *shape-validated* rather than trusted: an +//! import entry is only indexed when the four words it points at actually +//! look like a thunk for that exact ordinal. A malformed or already-patched +//! image annotates nothing rather than mislabelling real code. + +use std::collections::HashMap; + +use sylpheed_xex::header::ImportLibrary; + +/// `mtspr CTR, r11` — third word of every import thunk. +const MTCTR_R11: u32 = 0x7D69_03A6; +/// `bcctr 20, lt` (`bctr`) — fourth word of every import thunk. +const BCTR: u32 = 0x4E80_0420; +/// Record type in the first thunk word. +const RECORD_TYPE_FIRST: u32 = 1; +/// Record type in the second thunk word. +const RECORD_TYPE_SECOND: u32 = 2; +/// A thunk is four 4-byte words. +pub const THUNK_SIZE: u32 = 16; + +/// How an instruction relates to the import it references. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ImportRole { + /// One of the two import-record words at the head of a thunk. The word is + /// data the loader rewrites, not an instruction — its decoded text is + /// meaningless and it is flagged `is_data` alongside jump-table words. + Record, + /// A real instruction in the thunk body (`mtctr r11` / `bctr`). + Thunk, + /// A direct branch (`bl` or a tail-call `b`) whose target is this + /// import's thunk head. + Call, +} + +impl ImportRole { + /// Stable lowercase tag, as stored in `instructions.import_role`. + pub fn as_str(self) -> &'static str { + match self { + Self::Record => "record", + Self::Thunk => "thunk", + Self::Call => "call", + } + } +} + +/// Word-level index from code addresses to the import they belong to. +/// +/// Values are the thunk head VA, which is exactly `imports.address` for the +/// `record_type = 1` row — so it doubles as the foreign key written into +/// `instructions.import_address`. +#[derive(Debug, Default)] +pub struct ImportSites { + /// Every word inside a validated thunk -> (thunk head VA, role). + words: HashMap, +} + +impl ImportSites { + /// Index every import thunk in `libraries` that `image` actually backs. + /// + /// `record_type = 0` entries are plain pointer slots in `.rdata`: the + /// loader writes the resolved address there and no code is emitted, so + /// there is nothing in the instruction stream to annotate. + pub fn build(libraries: &[ImportLibrary], image: &[u8], image_base: u32) -> Self { + let read = |va: u32| -> Option { + let off = va.wrapping_sub(image_base) as usize; + if off.checked_add(4)? > image.len() { + return None; + } + Some(u32::from_be_bytes([ + image[off], + image[off + 1], + image[off + 2], + image[off + 3], + ])) + }; + + let mut words = HashMap::new(); + let mut indexed = 0usize; + let mut rejected = 0usize; + + for lib in libraries { + for imp in &lib.imports { + if imp.record_type != 1 { + continue; + } + let head = imp.address; + let Some(w0) = read(head) else { + rejected += 1; + continue; + }; + let (Some(w1), Some(w2), Some(w3)) = + (read(head + 4), read(head + 8), read(head + 12)) + else { + rejected += 1; + continue; + }; + if !is_thunk([w0, w1, w2, w3], imp.ordinal) { + rejected += 1; + continue; + } + + words.insert(head, (head, ImportRole::Record)); + words.insert(head + 4, (head, ImportRole::Record)); + words.insert(head + 8, (head, ImportRole::Thunk)); + words.insert(head + 12, (head, ImportRole::Thunk)); + indexed += 1; + } + } + + tracing::info!( + thunks = indexed, + rejected, + words = words.len(), + "import thunk index built", + ); + Self { words } + } + + /// The import this instruction references, and in what capacity. + /// + /// `branch_target` is the resolved target of a direct branch. A `bl` into + /// a thunk head yields [`ImportRole::Call`], which is what lets a query + /// find *call sites* of a kernel function rather than just the thunk. + /// Landing mid-thunk is not a call, so only the head matches. + pub fn lookup(&self, addr: u32, branch_target: Option) -> Option<(u32, ImportRole)> { + if let Some(&hit) = self.words.get(&addr) { + return Some(hit); + } + let target = branch_target?; + match self.words.get(&target) { + // `head == target` rejects a branch into the second record word. + Some(&(head, _)) if head == target => Some((head, ImportRole::Call)), + _ => None, + } + } + + /// Addresses of the import-record words — data embedded in a code section. + pub fn record_words(&self) -> impl Iterator + '_ { + self.words + .iter() + .filter(|(_, (_, role))| *role == ImportRole::Record) + .map(|(&addr, _)| addr) + } + + /// Thunk head VAs, one per indexed import. + pub fn thunk_heads(&self) -> impl Iterator + '_ { + self.words + .iter() + .filter(|(addr, (head, _))| *addr == head) + .map(|(&addr, _)| addr) + } + + /// Number of thunks indexed. + pub fn len(&self) -> usize { + self.words.len() / 4 + } + + pub fn is_empty(&self) -> bool { + self.words.is_empty() + } +} + +/// Does this four-word run look like an import thunk for `ordinal`? +/// +/// The library-index byte is checked for *consistency between the two +/// records* rather than against the library's position in the header: the +/// ordinal match plus the fixed `mtctr`/`bctr` tail is already conclusive, +/// and not depending on header ordering keeps this robust for titles whose +/// import libraries are listed in a different order. +fn is_thunk(w: [u32; 4], ordinal: u16) -> bool { + w[0] >> 24 == RECORD_TYPE_FIRST + && w[1] >> 24 == RECORD_TYPE_SECOND + && (w[0] & 0xFFFF) as u16 == ordinal + && (w[1] & 0xFFFF) as u16 == ordinal + && (w[0] >> 16) & 0xFF == (w[1] >> 16) & 0xFF + && w[2] == MTCTR_R11 + && w[3] == BCTR +} + +#[cfg(test)] +mod tests { + use super::*; + use sylpheed_xex::header::ImportEntry; + + const BASE: u32 = 0x8200_0000; + + /// Build a one-library header plus a backing image containing `thunks` + /// laid out back to back from `BASE`. + fn fixture(thunks: &[(u16, u8)]) -> (Vec, Vec) { + let mut image = Vec::new(); + let mut imports = Vec::new(); + for (i, &(ordinal, lib_index)) in thunks.iter().enumerate() { + let head = BASE + i as u32 * THUNK_SIZE; + let li = (lib_index as u32) << 16; + for w in [ + (RECORD_TYPE_FIRST << 24) | li | ordinal as u32, + (RECORD_TYPE_SECOND << 24) | li | ordinal as u32, + MTCTR_R11, + BCTR, + ] { + image.extend_from_slice(&w.to_be_bytes()); + } + imports.push(ImportEntry { + ordinal, + record_type: 1, + address: head, + }); + } + let lib = ImportLibrary { + name: "xboxkrnl.exe".into(), + id: 0, + version_min: 0, + version_cur: 0, + imports, + }; + (vec![lib], image) + } + + #[test] + fn indexes_every_word_of_a_thunk() { + let (libs, image) = fixture(&[(0x28C, 0)]); + let sites = ImportSites::build(&libs, &image, BASE); + assert_eq!(sites.len(), 1); + assert_eq!(sites.lookup(BASE, None), Some((BASE, ImportRole::Record))); + assert_eq!( + sites.lookup(BASE + 4, None), + Some((BASE, ImportRole::Record)) + ); + assert_eq!( + sites.lookup(BASE + 8, None), + Some((BASE, ImportRole::Thunk)) + ); + assert_eq!( + sites.lookup(BASE + 12, None), + Some((BASE, ImportRole::Thunk)) + ); + assert_eq!(sites.lookup(BASE + 16, None), None); + } + + /// The point of the whole exercise: a `bl` somewhere else in the binary + /// must resolve to the import it ends up calling. + #[test] + fn direct_branch_to_a_thunk_head_is_a_call() { + let (libs, image) = fixture(&[(0x28C, 0)]); + let sites = ImportSites::build(&libs, &image, BASE); + assert_eq!( + sites.lookup(0x8215_0000, Some(BASE)), + Some((BASE, ImportRole::Call)), + ); + } + + /// A branch into the *second* record word is not a call to the import; + /// treating it as one would invent a call edge that does not exist. + #[test] + fn branch_into_thunk_interior_is_not_a_call() { + let (libs, image) = fixture(&[(0x28C, 0)]); + let sites = ImportSites::build(&libs, &image, BASE); + assert_eq!(sites.lookup(0x8215_0000, Some(BASE + 4)), None); + assert_eq!(sites.lookup(0x8215_0000, Some(BASE + 8)), None); + } + + /// A word inside a thunk keeps its own role even when it also carries a + /// branch target, so the thunk body is never relabelled as a call. + #[test] + fn own_role_wins_over_branch_target() { + let (libs, image) = fixture(&[(0x28C, 0), (0x194, 1)]); + let sites = ImportSites::build(&libs, &image, BASE); + let second = BASE + THUNK_SIZE; + assert_eq!( + sites.lookup(BASE + 8, Some(second)), + Some((BASE, ImportRole::Thunk)) + ); + } + + /// Shape validation must reject an entry whose ordinal does not match the + /// record it points at — otherwise a stale import table would silently + /// stamp the wrong kernel name onto real code. + #[test] + fn ordinal_mismatch_is_rejected() { + let (mut libs, image) = fixture(&[(0x28C, 0)]); + libs[0].imports[0].ordinal = 0x999; + let sites = ImportSites::build(&libs, &image, BASE); + assert!(sites.is_empty()); + } + + /// An already-patched image (records rewritten to `lis`/`ori`) has no + /// records left to recognise, and must annotate nothing rather than + /// guessing. + #[test] + fn patched_thunk_is_rejected() { + let (libs, mut image) = fixture(&[(0x28C, 0)]); + image[0..4].copy_from_slice(&0x3D60_8200u32.to_be_bytes()); // lis r11, 0x8200 + let sites = ImportSites::build(&libs, &image, BASE); + assert!(sites.is_empty()); + } + + /// A `record_type = 0` entry is a data slot in `.rdata`, not code. + #[test] + fn variable_records_are_not_indexed() { + let (mut libs, image) = fixture(&[(0x28C, 0)]); + libs[0].imports[0].record_type = 0; + let sites = ImportSites::build(&libs, &image, BASE); + assert!(sites.is_empty()); + } + + /// An import pointing outside the image must not panic or index garbage. + #[test] + fn out_of_bounds_thunk_is_rejected() { + let (mut libs, image) = fixture(&[(0x28C, 0)]); + libs[0].imports[0].address = BASE + 0x10_0000; + let sites = ImportSites::build(&libs, &image, BASE); + assert!(sites.is_empty()); + } + + #[test] + fn record_words_lists_both_halves() { + let (libs, image) = fixture(&[(0x28C, 0), (0x194, 1)]); + let sites = ImportSites::build(&libs, &image, BASE); + let mut got: Vec = sites.record_words().collect(); + got.sort_unstable(); + assert_eq!(got, vec![BASE, BASE + 4, BASE + 16, BASE + 20]); + let mut heads: Vec = sites.thunk_heads().collect(); + heads.sort_unstable(); + assert_eq!(heads, vec![BASE, BASE + 16]); + } +} diff --git a/crates/sylpheed-xexdb/src/lib.rs b/crates/sylpheed-xexdb/src/lib.rs index a0a3025b..ed426568 100644 --- a/crates/sylpheed-xexdb/src/lib.rs +++ b/crates/sylpheed-xexdb/src/lib.rs @@ -25,6 +25,7 @@ pub mod eh_scope; pub mod formatter; pub mod func; pub mod funcptr_arrays; +pub mod imports; pub mod ind_dispatch_typed; pub mod indirect; pub mod jumptables; @@ -42,5 +43,6 @@ pub mod xref; mod ordinals; pub use db::{BranchTraceEntry, DbWriter, ExecTraceEntry, ImportCallEntry}; pub use disasm::{RichDisasmItem, enrich_section}; +pub use imports::{ImportRole, ImportSites}; pub use ordinals::resolve_ordinal; pub use xref::{Xref, XrefKind, XrefMap, resolve_source_label}; diff --git a/crates/sylpheed-xexdb/src/sinks/duckdb.rs b/crates/sylpheed-xexdb/src/sinks/duckdb.rs index 40db1117..23306937 100644 --- a/crates/sylpheed-xexdb/src/sinks/duckdb.rs +++ b/crates/sylpheed-xexdb/src/sinks/duckdb.rs @@ -32,6 +32,8 @@ pub fn append_instructions<'a>( ri.function.map(|f| f as i64), ri.label, ri.is_data, + ri.import.map(|(head, _)| head as i64), + ri.import.map(|(_, role)| role.as_str()), ])?; count += 1; } diff --git a/crates/sylpheed-xexdb/src/sql_views.rs b/crates/sylpheed-xexdb/src/sql_views.rs index ec3ede27..083c0269 100644 --- a/crates/sylpheed-xexdb/src/sql_views.rs +++ b/crates/sylpheed-xexdb/src/sql_views.rs @@ -192,20 +192,34 @@ FROM functions f JOIN instructions i ON i.address = f.address; "; -/// Per-function summary of which kernel/library imports it calls. Joins -/// xrefs (call edges) against the labels table to surface import names. +/// Every call site of a kernel/library import, one row per branch. +/// +/// Driven by `instructions.import_address` — the foreign key onto `imports` +/// laid down by [`crate::imports::ImportSites`] — rather than by matching +/// `xrefs` against import-kind labels. That change fixes two things: +/// +/// - a tail-call `b` into a thunk is a call to that import, but `xrefs` tags +/// it `'j'`, so the label-matching version silently dropped all of them +/// (64 sites on the reference title, against 3,217 `bl`); +/// - the library and ordinal are now available without re-parsing the name, +/// so `WHERE library = 'xboxkrnl.exe'` works directly. +/// +/// `call_site` is the branch instruction; `import_addr` is the thunk head. const V_IMPORTS_CALLED: &str = " CREATE OR REPLACE VIEW v_imports_called AS SELECT - x.source_func AS function_addr, + i.address AS call_site, + i.function AS function_addr, f.name AS function_name, - x.target AS import_addr, - l.name AS import_name -FROM xrefs x -JOIN labels l ON l.address = x.target -LEFT JOIN functions f ON f.address = x.source_func -WHERE x.kind = 'call' - AND l.kind = 'import'; + i.mnemonic AS instruction, + i.import_address AS import_addr, + im.library AS library, + im.name AS import_name, + im.ordinal AS ordinal +FROM instructions i +JOIN imports im ON im.address = i.import_address +LEFT JOIN functions f ON f.address = i.function +WHERE i.import_role = 'call'; "; /// Every recovered `switch` case, joined to the dispatching function and to diff --git a/crates/sylpheed-xexdb/tests/db_schema_golden.rs b/crates/sylpheed-xexdb/tests/db_schema_golden.rs index 303cf12f..12611695 100644 --- a/crates/sylpheed-xexdb/tests/db_schema_golden.rs +++ b/crates/sylpheed-xexdb/tests/db_schema_golden.rs @@ -176,6 +176,8 @@ fn db_schema_matches_expected_columns() { ("function", "BIGINT"), ("label", "VARCHAR"), ("is_data", "BOOLEAN"), + ("import_address", "BIGINT"), + ("import_role", "VARCHAR"), ], ), ( @@ -564,5 +566,63 @@ fn db_schema_matches_expected_columns() { assert_eq!(exists, 1, "missing SQL view: {v}"); } + // ── Foreign keys ──────────────────────────────────────────────────── + // + // Lock the declared FK set. These are correctness assertions baked into + // the schema: a build that would produce a dangling reference fails at + // insert time rather than shipping a broken database. + let expected_fks = [ + ("classes", "vtables"), + ("data_in_code", "instructions"), + ("eh_try_blocks", "eh_funcinfo"), + ("eh_unwind_map", "eh_funcinfo"), + ("function_pointer_array_entries", "function_pointer_arrays"), + ("indirect_dispatch_candidates", "indirect_dispatch_sites"), + ("indirect_dispatch_candidates", "vtables"), + ("instructions", "imports"), + ("jump_table_entries", "jump_tables"), + ("methods", "vtables"), + ("rtti_base_classes", "rtti_type_descriptors"), + ("rtti_locators", "rtti_type_descriptors"), + ("vptr_writes", "vtables"), + ("vtables", "rtti_locators"), + ("xdbf_achievements", "xdbf_images"), + ("xrefs", "instructions"), + ]; + for (child, parent) in expected_fks { + let n: i64 = conn + .query_row( + "SELECT COUNT(*) FROM duckdb_constraints() \ + WHERE constraint_type = 'FOREIGN KEY' AND table_name = ? \ + AND referenced_table = ?", + [child, parent], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(n, 1, "missing foreign key: {child} -> {parent}"); + } + + // `functions` must never become a foreign-key parent. DuckDB implements + // UPDATE as delete-then-insert, so a single inbound FK makes the table + // un-updatable — even for an update that does not touch `address` — and + // that breaks `apply_re_symbols.sql`, which re-stamps reverse-engineered + // names onto `functions.name` after every regeneration. Adding one back + // looks harmless in review and only fails at symbol-stamping time, hence + // this guard. + let inbound_to_functions: i64 = conn + .query_row( + "SELECT COUNT(*) FROM duckdb_constraints() \ + WHERE constraint_type = 'FOREIGN KEY' \ + AND referenced_table = 'functions'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + inbound_to_functions, 0, + "no table may declare a foreign key onto `functions`: it would make \ + functions.name un-updatable and break apply_re_symbols.sql", + ); + let _ = std::fs::remove_file(&tmp); } diff --git a/tools/zq.py b/tools/zq.py index 7b4b0c8b..0dc6dbc2 100755 --- a/tools/zq.py +++ b/tools/zq.py @@ -15,7 +15,7 @@ both static, neither needing anything to run: alongside a metadata JSON. Usage: - zq.py dis # disassemble [lo,hi) (jump-table words shown as .long) + zq.py dis # disassemble [lo,hi) (data words as .long; imports named) zq.py fn # function containing pc (address,name,end) zq.py xref # xrefs whose target == addr (callers) zq.py callers # call-sites of vtable slot at byte offset N @@ -29,13 +29,21 @@ Usage: zq.py class # one class: bases, vtable, virtual methods zq.py str # string literals matching, with referencing functions + zq.py gaps # dangling refs = known detector gaps (see db.rs schema docs) + zq.py imp [substr] # imports + how many call sites each has + zq.py impcalls # every call site of the matching imports + zq.py xdbf [substr] # XDBF title text (all locales); substr filters zq.py ach # XDBF achievements (id, gamerscore, name, descriptions) A command that needs a table the current DB predates prints what to regenerate rather than a SQL error. """ -import duckdb, sys +import duckdb, sys, signal + +# Output is routinely piped into `head`; without this Python prints a +# BrokenPipeError traceback when the reader closes early. +signal.signal(signal.SIGPIPE, signal.SIG_DFL) DB = '/home/fabi/RE - Project Sylpheed/xenia-rs/sylpheed.db' c = duckdb.connect(DB, read_only=True) @@ -65,17 +73,108 @@ def _fn(pc): return f'{r[0][1]}({H(r[0][0])})' if r else '?' +def _import_names(): + """thunk head VA -> 'library::name' for every import, or {} on an old db.""" + if not _has_col('instructions', 'import_address'): + return {} + return {a: (f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}') + for a, lib, nm, o in c.execute( + 'SELECT address,library,name,ordinal FROM imports').fetchall()} + + def cmd_dis(lo, hi): data_col = 'is_data' if _has_col('instructions', 'is_data') else 'false' - rows = c.execute(f'SELECT address,mnemonic,operands,raw,{data_col} FROM instructions ' - 'WHERE address>=? AND address=? AND address {who}' if imp_role == 'record' else 'jump-table data' + print(H(a), '.long', H(raw & 0xffffffff), f' ; {why}') + elif imp_role == 'call': + print(H(a), m, o, f' ; -> IMPORT {who}') + elif imp_role == 'thunk': + print(H(a), m, o, f' ; import thunk {who}') else: print(H(a), m, o) +def cmd_gaps(): + """Dangling references the schema deliberately does NOT declare as FKs. + + Each of these is a real gap in a detector, kept queryable rather than + hidden: declaring the FK would abort the build instead of reporting it. + """ + checks = [ + ("vtables RTTI names but M3 missed", + "SELECT count(*) FROM rtti_locators l WHERE l.vtable_address IS NOT NULL " + "AND NOT EXISTS (SELECT 1 FROM vtables v WHERE v.address=l.vtable_address)"), + ("vtable slots -> undetected function", + "SELECT count(DISTINCT function_address) FROM methods m WHERE NOT EXISTS " + "(SELECT 1 FROM functions f WHERE f.address=m.function_address)"), + ("funcptr-array slots -> undetected function", + "SELECT count(DISTINCT function_address) FROM function_pointer_array_entries e " + "WHERE NOT EXISTS (SELECT 1 FROM functions f WHERE f.address=e.function_address)"), + ("dispatch candidates -> undetected function", + "SELECT count(DISTINCT method_address) FROM indirect_dispatch_candidates x " + "WHERE NOT EXISTS (SELECT 1 FROM functions f WHERE f.address=x.method_address)"), + ("virtual sites with no candidates (truncated)", + "SELECT count(*) FROM indirect_dispatch_sites WHERE truncated"), + ] + for label, q in checks: + try: + print(f"{c.execute(q).fetchone()[0]:>7} {label}") + except Exception as e: + print(f" ? {label} ({str(e).splitlines()[0][:50]})") + print("\n -- RTTI-named vftables the vtable scanner missed --") + try: + rows = c.execute( + "SELECT l.vtable_address, t.demangled_name FROM rtti_locators l " + "JOIN rtti_type_descriptors t ON t.address=l.type_descriptor " + "WHERE l.vtable_address IS NOT NULL AND NOT EXISTS " + "(SELECT 1 FROM vtables v WHERE v.address=l.vtable_address) " + "ORDER BY 2 LIMIT 15").fetchall() + for a, nm in rows: + print(f" {H(a)} {nm}") + except Exception as e: + print(" ", str(e).splitlines()[0][:70]) + + +def cmd_imp(pat=None): + """Import call sites, grouped by import. `pat` filters on library::name.""" + _need('imports') + if not _has_col('instructions', 'import_address'): + sys.exit(f'this db predates instructions.import_address — regenerate with:\n {REGEN}') + q = ("SELECT im.library, im.name, im.ordinal, im.address, count(i.address) " + "FROM imports im LEFT JOIN instructions i " + " ON i.import_address = im.address AND i.import_role = 'call' " + "WHERE im.record_type = 1 GROUP BY 1,2,3,4 ORDER BY 5 DESC, 1, 2") + for lib, nm, o, addr, n in c.execute(q).fetchall(): + label = f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}' + if pat and pat.lower() not in label.lower(): + continue + print(f'{n:>5} calls {H(addr)} {label}') + + +def cmd_impcalls(pat): + """Every call site of the imports matching `pat`.""" + if not _has_col('instructions', 'import_address'): + sys.exit(f'this db predates instructions.import_address — regenerate with:\n {REGEN}') + q = ("SELECT i.address, i.mnemonic, im.library, im.name, im.ordinal " + "FROM instructions i JOIN imports im ON im.address = i.import_address " + "WHERE i.import_role = 'call' ORDER BY i.address") + for a, m, lib, nm, o in c.execute(q).fetchall(): + label = f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}' + if pat.lower() not in label.lower(): + continue + print(H(a), m, 'in', _fn(a), '->', label) + + def cmd_switch(pc): _need('jump_tables', 'jump_table_entries') r = c.execute('SELECT bctr_pc,function,table_address,kind,entry_count FROM jump_tables ' @@ -194,6 +293,12 @@ def main(): for (a,) in c.execute("SELECT address FROM instructions WHERE mnemonic='lwz' AND operands=? " 'ORDER BY address', [pat]).fetchall(): print(H(a), 'in', _fn(a)) + elif cmd == 'gaps': + cmd_gaps() + elif cmd == 'imp': + cmd_imp(args[0] if args else None) + elif cmd == 'impcalls': + cmd_impcalls(args[0]) elif cmd == 'grep': for a, m, o in c.execute("SELECT address,mnemonic,operands FROM instructions " "WHERE operands LIKE ? ORDER BY address", [f'%{args[0]}%']).fetchall():