fabi/Sylpheed
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'feat(xexdb): re-land #35 onto main — import-thunk naming, FKs, zq.py fixes, RE symbols' (#39) from feat/xexdb-import-naming into main

Browse Source 
Reviewed-on: #39
This commit is contained in:
fabi
2026-09-16 05:09:15 +00:00
parent f33a2ff646 0679abb9f9
commit cb731f2e96
13 changed files with 1029 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@@ -18,6 +18,13 @@ Thumbs.db
# Local dev overrides
.env
# The static-analysis database `tools/zq.py` reads by default. A build artefact
# of several hundred MB (`sylph-xexdb dis ... --db sylpheed.db`), and zq.py
# now tells people to put it exactly here -- so it must never be committable.
# One `git add -A` would otherwise put it in public history for good.
/sylpheed.db
/sylpheed.db.wal
# Trunk build output
dist/
__pycache__/

12
crates/sylpheed-xexdb/src/bin/sylph-xexdb.rs
View File

@@ -919,6 +919,16 @@ fn cmd_dis(
info!(db = %db, "database written");
}
// Word-level index of the import thunks, used to annotate the instruction
// stream: which words are loader-patched import records, and which
// branches are really kernel/xam calls. The database path builds its own
// inside `ingest_instructions`; these two output paths need it here.
let import_sites = sylpheed_xexdb::ImportSites::build(
disasm_info.import_libraries,
&pe_image,
disasm_info.image_base,
);
// JSON Lines output: one row per instruction, structured columns.
if let Some(json) = json_path {
info!(json = %json, "writing JSON Lines");
@@ -939,6 +949,7 @@ fn cmd_dis(
&func_analysis,
&xref_result.labels,
&jt_data_words,
&import_sites,
);
total += sylpheed_xexdb::sinks::json::write_jsonl(&mut out, items)?;
}
@@ -962,6 +973,7 @@ fn cmd_dis(
&xref_result.xrefs,
&xref_result.data_annotations,
&jt_data_words,
&import_sites,
)?;
if let Some(path) = output {

317
crates/sylpheed-xexdb/src/db.rs
View File

@@ -15,6 +15,33 @@
//! - `"jump"` : bcctrx without LK
//! - `"branch"` : bx/bcx without LK
//!
//! # Referential integrity
//!
//! The schema declares foreign keys wherever a column is *derived from* the
//! row it points at, so a build that would produce a dangling reference fails
//! loudly instead of shipping a broken database. Two consequences:
//!
//! - **Insert order is load-bearing.** DuckDB checks each row as it is
//! appended, so a parent must be populated before its children, and
//! `CREATE TABLE` statements must be ordered so a table exists before
//! anything references it. `write_analysis_results` documents its order.
//! - **`functions` is deliberately NOT a foreign-key parent.** DuckDB
//! implements `UPDATE` as delete-then-insert, so *any* FK onto
//! `functions(address)` makes the table un-updatable — even an update that
//! does not touch `address`. That would break `apply_re_symbols.sql`, which
//! re-stamps reverse-engineered names onto `functions.name` after every
//! regeneration. `xrefs.source_func`, `vptr_writes.writer_function` and
//! `jump_tables.function` are therefore plain columns. The same restriction
//! applies to any table a human post-edits.
//!
//! Some references are intentionally left undeclared because the relationship
//! genuinely does not hold: see `rtti_locators.vtable_address` (41 RTTI-named
//! vftables the M3 scanner misses), `methods.function_address` and
//! `indirect_dispatch_candidates.method_address` (vtable slots pointing at
//! code no `.pdata` record or `bl` ever declared), and `xrefs.target` (which
//! is frequently data). Those orphan sets are findings, not noise — querying
//! them is how the gaps stay visible.
//!
//! # Schema
//!
//! ## `metadata`
@@ -48,8 +75,9 @@
//! - `library` — Module name (e.g. `xboxkrnl.exe`, `xam.xex`)
//! - `ordinal` — Numeric ordinal identifying the export within the library
//! - `name` — Resolved human-readable symbol name; `NULL` if not in symbol table
//! - `record_type` — XEX import record type: `0` = function thunk, `1` = variable
//! - `address` — Absolute VA of the import thunk or variable in the binary
//! - `record_type` — XEX import record type: `0` = variable, `1` = function thunk
//! - `address` — Absolute VA of the import thunk or variable in the binary.
//! UNIQUE, because `instructions.import_address` is a foreign key onto it.
//!
//! ## `functions`
//! One row per detected function. Candidates are `bl` targets ∪ `.pdata`
@@ -94,9 +122,18 @@
//! - `function` — VA of the enclosing function; `NULL` if not inside a detected function
//! - `label` — Label name at this address; `NULL` if none
//! - `is_data` — `true` when this word is data embedded in a code section (a
//! recovered jump table or index map). The decoded `mnemonic` /
//! `disasm` columns are meaningless on such rows; filter them out
//! (`WHERE NOT is_data`) for any instruction-level analysis.
//! recovered jump table or index map, or an XEX import record). The decoded
//! `mnemonic` / `disasm` columns are meaningless on such rows; filter them
//! out (`WHERE NOT is_data`) for any instruction-level analysis.
//! - `import_address` — FK onto `imports.address` (a thunk head) when this word is
//! part of an XEX import thunk, or is a direct branch into one; `NULL`
//! otherwise. Join it to name the callee.
//! - `import_role` — Which of the three this row is: `'record'` (one of the two
//! loader-patched words at the thunk head — also `is_data`), `'thunk'` (the
//! `mtctr r11` / `bctr` body), or `'call'` (a `bl`/`b` elsewhere in the binary
//! whose target is that thunk). `NULL` iff `import_address` is `NULL`. Without
//! these columns an import call reads as a branch to a bare address and the
//! thunk head as two meaningless `.long`s.
//!
//! ## `jump_tables` / `jump_table_entries` / `data_in_code`
//! Recovered `switch` dispatches (see [`crate::jumptables`]). `jump_tables` has
@@ -153,7 +190,7 @@
//! - `kind` — `call`, `return`, `jump`, or `branch` (see top-level doc)
//! - `lr` — Link register value at time of branch
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::path::Path;
use duckdb::{Connection, params};
@@ -220,6 +257,11 @@ pub struct DbWriter {
trace_instructions: bool,
trace_imports: bool,
trace_branches: bool,
/// Import-thunk index, built during `ingest_instructions` and reused when
/// `write_analysis_results` records the thunk records in `data_in_code`.
/// Empty until then — with no instruction rows there is nothing to
/// cross-reference it against.
import_sites: crate::imports::ImportSites,
}
impl DbWriter {
@@ -241,6 +283,7 @@ impl DbWriter {
trace_instructions: false,
trace_imports: false,
trace_branches: false,
import_sites: crate::imports::ImportSites::default(),
})
}
@@ -270,8 +313,14 @@ impl DbWriter {
library VARCHAR NOT NULL, -- module name (e.g. xboxkrnl.exe, xam.xex)
ordinal BIGINT NOT NULL, -- ordinal identifying the export within the library
name VARCHAR, -- resolved symbol name; NULL if not in symbol table
record_type BIGINT NOT NULL, -- 0 = function thunk, 1 = variable
address BIGINT NOT NULL -- absolute VA of the thunk or variable
-- 0 = variable (a pointer slot in .rdata the loader fills in),
-- 1 = function thunk (four words of code in .text). This was
-- documented the wrong way round; `ImportEntry::record_type`
-- in xenia-xex has always had it right.
record_type BIGINT NOT NULL,
-- Absolute VA of the thunk or variable. UNIQUE because
-- `instructions.import_address` is a foreign key onto it.
address BIGINT NOT NULL UNIQUE
);
",
)?;
@@ -322,11 +371,31 @@ impl DbWriter {
section VARCHAR NOT NULL, -- PE section name containing this instruction
function BIGINT, -- VA of the enclosing function; NULL if unknown
label VARCHAR, -- label at this address; NULL if none
is_data BOOLEAN NOT NULL -- M12: word is data embedded in code (jump table / index map), NOT an instruction
is_data BOOLEAN NOT NULL, -- M12: word is data embedded in code (jump table / index map / import record), NOT an instruction
-- XEX import this word references, as a foreign key into `imports`
-- (the `record_type = 1` row, whose address is the thunk head).
-- NULL for everything that is not part of, or a branch into, an
-- import thunk. See `import_role` for which of the three it is.
import_address BIGINT REFERENCES imports(address),
import_role VARCHAR -- 'record' | 'thunk' | 'call'; NULL iff import_address IS NULL
);
")?;
insert_instructions_streaming(&self.conn, pe, info, func_analysis, labels, data_words)?;
// Built here rather than passed in: everything it needs (the parsed
// import table and the loaded image) is already on `info`/`pe`, and
// the index is only ever consumed by this one streaming pass.
self.import_sites =
crate::imports::ImportSites::build(info.import_libraries, pe, info.image_base);
insert_instructions_streaming(
&self.conn,
pe,
info,
func_analysis,
labels,
data_words,
&self.import_sites,
)?;
let indices = [
(
@@ -357,6 +426,10 @@ impl DbWriter {
"idx_instructions_is_data",
"CREATE INDEX idx_instructions_is_data ON instructions(is_data)",
),
(
"idx_instructions_import",
"CREATE INDEX idx_instructions_import ON instructions(import_address)",
),
];
for (name, sql) in indices {
tracing::debug!(index = name, "creating instructions index");
@@ -422,17 +495,63 @@ impl DbWriter {
kind VARCHAR NOT NULL -- function | import | saverestore | local | data | other
);
-- M13 — MSVC RTTI. `rtti_type_descriptors` is the authoritative
-- source of C++ class identity: the linker wrote these names, they
-- are not inferred.
CREATE TABLE rtti_type_descriptors (
address BIGINT PRIMARY KEY, -- VA of the TypeDescriptor
mangled_name VARCHAR NOT NULL, -- decorated name, e.g. .?AVSilph@silph@@
demangled_name VARCHAR NOT NULL -- readable form, e.g. silph::Silph
);
-- M13 — RTTICompleteObjectLocator. One per emitted vftable; the
-- `subobject_offset` column separates a class's primary vftable (0)
-- from the extra vftables it emits for secondary base subobjects.
CREATE TABLE rtti_locators (
address BIGINT PRIMARY KEY,
subobject_offset BIGINT NOT NULL, -- this-offset of the subobject this vftable serves
cd_offset BIGINT NOT NULL, -- constructor-displacement offset
type_descriptor BIGINT NOT NULL REFERENCES rtti_type_descriptors(address),
class_hierarchy BIGINT NOT NULL, -- VA of the RTTIClassHierarchyDescriptor
-- VA of vftable[0]; NULL if no referencing word was found.
-- Deliberately NOT a foreign key: on the reference title 41
-- locators name a vftable the M3 scanner did not detect (see
-- `crate::vtables`), so declaring one would fail the build on
-- a real gap rather than reporting it. Query the orphans with
-- SELECT * FROM rtti_locators l WHERE NOT EXISTS
-- (SELECT 1 FROM vtables v WHERE v.address = l.vtable_address)
vtable_address BIGINT
);
-- M13 — linearised RTTIBaseClassArray. Index 0 is the class itself;
-- the remaining rows are its bases in MSVC's depth-first order,
-- each with the PMD displacement triple needed to locate the base
-- subobject inside an instance.
CREATE TABLE rtti_base_classes (
class_hierarchy BIGINT NOT NULL, -- VA of the deriving class's hierarchy descriptor
base_index BIGINT NOT NULL, -- position in the base-class array
type_descriptor BIGINT NOT NULL REFERENCES rtti_type_descriptors(address),
name VARCHAR NOT NULL,
num_contained_bases BIGINT NOT NULL,
mdisp BIGINT NOT NULL, -- member displacement
pdisp BIGINT NOT NULL, -- vbtable displacement (-1 = non-virtual base)
vdisp BIGINT NOT NULL, -- displacement inside the vbtable
attributes BIGINT NOT NULL,
PRIMARY KEY (class_hierarchy, base_index)
);
CREATE TABLE vtables (
address BIGINT PRIMARY KEY, -- absolute VA of vtable[0]
length BIGINT NOT NULL, -- number of method slots
col_address BIGINT, -- VA of CompleteObjectLocator (NULL when no RTTI)
-- VA of the CompleteObjectLocator; NULL when the class has no RTTI.
col_address BIGINT REFERENCES rtti_locators(address),
class_name VARCHAR NOT NULL, -- demangled class name OR ANON_Class_<hash> when stripped
rtti_present BOOLEAN NOT NULL, -- true when COL → TypeDescriptor walk succeeded
base_classes_json VARCHAR -- JSON array of base class names (NULL if none / parse failure)
);
CREATE TABLE methods (
vtable_address BIGINT NOT NULL, -- vtable this slot belongs to
vtable_address BIGINT NOT NULL REFERENCES vtables(address), -- vtable this slot belongs to
slot BIGINT NOT NULL, -- 0-based slot index
function_address BIGINT NOT NULL, -- VA of the function this slot points at
mangled_name VARCHAR, -- raw label name when mangled (?...)
@@ -442,7 +561,7 @@ impl DbWriter {
CREATE TABLE classes (
name VARCHAR PRIMARY KEY, -- class name (demangled or ANON_*)
vtable_address BIGINT NOT NULL, -- representative vtable (first detected)
vtable_address BIGINT NOT NULL REFERENCES vtables(address), -- representative vtable (first detected)
rtti_present BOOLEAN NOT NULL,
base_classes_json VARCHAR -- JSON of base class names (NULL when stripped)
);
@@ -476,7 +595,7 @@ impl DbWriter {
);
CREATE TABLE function_pointer_array_entries (
array_address BIGINT NOT NULL, -- FK to function_pointer_arrays.address
array_address BIGINT NOT NULL REFERENCES function_pointer_arrays(address),
slot BIGINT NOT NULL, -- 0-based slot index
function_address BIGINT NOT NULL, -- VA of the function this slot points at
PRIMARY KEY (array_address, slot)
@@ -503,8 +622,10 @@ impl DbWriter {
-- ind_call xref edges in the `xrefs` table are derived from
-- this; this view lets you join back to vtable / method info.
CREATE TABLE indirect_dispatch_candidates (
dispatch_pc BIGINT NOT NULL,
vtable_address BIGINT NOT NULL,
dispatch_pc BIGINT NOT NULL REFERENCES indirect_dispatch_sites(dispatch_pc),
vtable_address BIGINT NOT NULL REFERENCES vtables(address),
-- No FK: a vtable slot may point at code `.pdata` never declared
-- and no `bl` ever targets, so this is not always in `functions`.
method_address BIGINT NOT NULL,
PRIMARY KEY (dispatch_pc, vtable_address)
);
@@ -514,7 +635,7 @@ impl DbWriter {
-- has (or does not have) coverage in the dispatch resolver.
CREATE TABLE vptr_writes (
writer_pc BIGINT NOT NULL,
vtable_address BIGINT NOT NULL,
vtable_address BIGINT NOT NULL REFERENCES vtables(address),
vptr_offset BIGINT NOT NULL,
writer_function BIGINT NOT NULL,
PRIMARY KEY (writer_pc, vtable_address, vptr_offset)
@@ -536,7 +657,7 @@ impl DbWriter {
);
CREATE TABLE eh_unwind_map (
funcinfo_address BIGINT NOT NULL, -- FK to eh_funcinfo.address
funcinfo_address BIGINT NOT NULL REFERENCES eh_funcinfo(address),
state_index BIGINT NOT NULL,
to_state BIGINT NOT NULL,
action_pc BIGINT NOT NULL,
@@ -544,7 +665,7 @@ impl DbWriter {
);
CREATE TABLE eh_try_blocks (
funcinfo_address BIGINT NOT NULL, -- FK to eh_funcinfo.address
funcinfo_address BIGINT NOT NULL REFERENCES eh_funcinfo(address),
try_index BIGINT NOT NULL,
try_low BIGINT NOT NULL,
try_high BIGINT NOT NULL,
@@ -566,6 +687,14 @@ impl DbWriter {
PRIMARY KEY (namespace, id)
);
CREATE TABLE xdbf_images (
id BIGINT PRIMARY KEY, -- image id referenced by achievements
is_title_icon BOOLEAN NOT NULL, -- id 0x8000 — the title's own icon
body_offset BIGINT NOT NULL, -- offset within the image buffer
size BIGINT NOT NULL,
format VARCHAR NOT NULL -- 'png' when the body carries the PNG signature
);
CREATE TABLE xdbf_achievements (
id BIGINT PRIMARY KEY, -- 1-based achievement id
name VARCHAR, -- resolved via the default language's string table
@@ -574,7 +703,7 @@ impl DbWriter {
label_id BIGINT NOT NULL, -- string ids, for joining other languages
description_id BIGINT NOT NULL,
unachieved_id BIGINT NOT NULL,
image_id BIGINT NOT NULL, -- FK to xdbf_images.id
image_id BIGINT NOT NULL REFERENCES xdbf_images(id),
gamerscore BIGINT NOT NULL,
flags BIGINT NOT NULL
);
@@ -589,14 +718,6 @@ impl DbWriter {
PRIMARY KEY (language, string_id)
);
CREATE TABLE xdbf_images (
id BIGINT PRIMARY KEY, -- image id referenced by achievements
is_title_icon BOOLEAN NOT NULL, -- id 0x8000 — the title's own icon
body_offset BIGINT NOT NULL, -- offset within the image buffer
size BIGINT NOT NULL,
format VARCHAR NOT NULL -- 'png' when the body carries the PNG signature
);
CREATE TABLE demangled_names (
address BIGINT, -- VA the mangled name is associated with; NULL when from a non-address source (e.g. RTTI-only string)
mangled VARCHAR NOT NULL, -- original mangled symbol (e.g. ?Foo@Bar@@QEAAXXZ)
@@ -624,7 +745,7 @@ impl DbWriter {
-- M12 — one row per case value, in case order. `target_address`
-- repeats when several case values share a body.
CREATE TABLE jump_table_entries (
bctr_pc BIGINT NOT NULL, -- FK to jump_tables.bctr_pc
bctr_pc BIGINT NOT NULL REFERENCES jump_tables(bctr_pc),
case_index BIGINT NOT NULL, -- 0-based case value
target_address BIGINT NOT NULL, -- VA of the case body
PRIMARY KEY (bctr_pc, case_index)
@@ -634,52 +755,16 @@ impl DbWriter {
-- instructions. Anything listed here is a decoding hazard: linear
-- disassembly of these words produces garbage rows.
CREATE TABLE data_in_code (
address BIGINT PRIMARY KEY, -- VA of the first byte
address BIGINT PRIMARY KEY REFERENCES instructions(address), -- VA of the first byte
length BIGINT NOT NULL, -- byte length
kind VARCHAR NOT NULL -- 'jump_table' | 'jump_index_map'
);
-- M13 — MSVC RTTI. `rtti_type_descriptors` is the authoritative
-- source of C++ class identity: the linker wrote these names, they
-- are not inferred.
CREATE TABLE rtti_type_descriptors (
address BIGINT PRIMARY KEY, -- VA of the TypeDescriptor
mangled_name VARCHAR NOT NULL, -- decorated name, e.g. .?AVSilph@silph@@
demangled_name VARCHAR NOT NULL -- readable form, e.g. silph::Silph
);
-- M13 — RTTICompleteObjectLocator. One per emitted vftable; the
-- `subobject_offset` column separates a class's primary vftable (0)
-- from the extra vftables it emits for secondary base subobjects.
CREATE TABLE rtti_locators (
address BIGINT PRIMARY KEY,
subobject_offset BIGINT NOT NULL, -- this-offset of the subobject this vftable serves
cd_offset BIGINT NOT NULL, -- constructor-displacement offset
type_descriptor BIGINT NOT NULL, -- FK to rtti_type_descriptors.address
class_hierarchy BIGINT NOT NULL, -- VA of the RTTIClassHierarchyDescriptor
vtable_address BIGINT -- VA of vftable[0]; NULL if no referencing word was found
);
-- M13 — linearised RTTIBaseClassArray. Index 0 is the class itself;
-- the remaining rows are its bases in MSVC's depth-first order,
-- each with the PMD displacement triple needed to locate the base
-- subobject inside an instance.
CREATE TABLE rtti_base_classes (
class_hierarchy BIGINT NOT NULL, -- VA of the deriving class's hierarchy descriptor
base_index BIGINT NOT NULL, -- position in the base-class array
type_descriptor BIGINT NOT NULL,
name VARCHAR NOT NULL,
num_contained_bases BIGINT NOT NULL,
mdisp BIGINT NOT NULL, -- member displacement
pdisp BIGINT NOT NULL, -- vbtable displacement (-1 = non-virtual base)
vdisp BIGINT NOT NULL, -- displacement inside the vbtable
attributes BIGINT NOT NULL,
PRIMARY KEY (class_hierarchy, base_index)
kind VARCHAR NOT NULL -- 'jump_table' | 'jump_index_map' | 'import_record'
);
CREATE TABLE xrefs (
source BIGINT NOT NULL, -- VA of the referencing instruction
target BIGINT NOT NULL, -- VA of the referenced destination
source BIGINT NOT NULL REFERENCES instructions(address),
-- No FK: a target may be data (.rdata/.data) or an address with
-- no decoded word, so it is not always an `instructions` row.
target BIGINT NOT NULL,
kind VARCHAR NOT NULL, -- call | ind_call | j | br | read | write | ref
addr_mode VARCHAR, -- M6 sub-classification of how source computes target (NULL for control-flow)
instruction VARCHAR, -- mnemonic of source instruction; NULL if not in binary
@@ -703,17 +788,24 @@ impl DbWriter {
// which is both fast and flat in memory. It bypasses the SQL layer,
// so `ON CONFLICT DO NOTHING` is unavailable and each converted sink
// documents why its key cannot collide (or dedupes explicitly).
// Insertion order is load-bearing: the schema declares foreign keys,
// and DuckDB checks them per row, so a parent table must be populated
// before any child that references it. `insert_rtti` runs ahead of
// `insert_vtables` because `vtables.col_address` points into
// `rtti_locators`; everything else follows the natural dependency
// order (functions -> vtables -> methods/classes -> dispatch).
insert_functions(&self.conn, func_analysis, labels)?;
insert_pdata_entries(&self.conn, &func_analysis.pdata_entries)?;
insert_labels(&self.conn, labels)?;
insert_demangled_from_labels(&self.conn, labels, info.import_libraries)?;
insert_rtti(&self.conn, rtti)?;
insert_vtables(&self.conn, vtables, pe, info.image_base)?;
insert_methods_and_classes(&self.conn, vtables, labels)?;
insert_strings(&self.conn, strings)?;
insert_funcptr_arrays(&self.conn, funcptr_arrays)?;
insert_eh_records(&self.conn, eh_records)?;
insert_jump_tables(&self.conn, jump_tables)?;
insert_rtti(&self.conn, rtti)?;
insert_import_record_extents(&self.conn, &self.import_sites)?;
insert_xdbf(&self.conn, xdbf)?;
if let Some(t) = typed_ind {
insert_typed_ind_dispatch(&self.conn, t)?;
@@ -1363,8 +1455,24 @@ fn insert_imports(conn: &Connection, info: &DisasmInfo) -> anyhow::Result<()> {
"INSERT INTO imports (library, ordinal, name, record_type, address)
VALUES (?, ?, ?, ?, ?)",
)?;
// `address` is UNIQUE so that `instructions.import_address` can key off
// it. Two entries sharing an address would be a malformed import table —
// one slot cannot hold two imports — but a duplicate must not abort the
// whole database build, so keep the first and say what was dropped.
let mut seen: HashSet<u32> = HashSet::new();
let mut duplicates = 0usize;
for lib in info.import_libraries {
for imp in &lib.imports {
if !seen.insert(imp.address) {
duplicates += 1;
tracing::warn!(
library = %lib.name,
ordinal = imp.ordinal,
address = format_args!("{:#010x}", imp.address),
"duplicate import address; keeping the first entry",
);
continue;
}
let resolved = crate::resolve_ordinal(&lib.name, imp.ordinal);
stmt.execute(params![
lib.name,
@@ -1375,6 +1483,9 @@ fn insert_imports(conn: &Connection, info: &DisasmInfo) -> anyhow::Result<()> {
])?;
}
}
if duplicates > 0 {
tracing::warn!(duplicates, "import table has entries sharing an address");
}
Ok(())
}
@@ -1756,6 +1867,23 @@ fn insert_xdbf(conn: &Connection, xdbf: Option<&crate::xdbf::Xdbf>) -> anyhow::R
.map(|(_, s)| s.clone())
};
// Images before achievements: `xdbf_achievements.image_id` is a foreign
// key onto `xdbf_images.id`, so the icon rows must already exist.
let mut stmt = conn.prepare(
"INSERT INTO xdbf_images (id, is_title_icon, body_offset, size, format)
VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING",
)?;
for i in &x.images {
stmt.execute(params![
i.id as i64,
i.id == crate::xdbf::ID_TITLE,
i.offset as i64,
i.size as i64,
i.format,
])?;
}
drop(stmt);
let mut stmt = conn.prepare(
"INSERT INTO xdbf_achievements
(id, name, unlocked_desc, locked_desc, label_id, description_id,
@@ -1778,21 +1906,6 @@ fn insert_xdbf(conn: &Connection, xdbf: Option<&crate::xdbf::Xdbf>) -> anyhow::R
}
drop(stmt);
let mut stmt = conn.prepare(
"INSERT INTO xdbf_images (id, is_title_icon, body_offset, size, format)
VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING",
)?;
for i in &x.images {
stmt.execute(params![
i.id as i64,
i.id == crate::xdbf::ID_TITLE,
i.offset as i64,
i.size as i64,
i.format,
])?;
}
drop(stmt);
let mut meta = conn.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)")?;
meta.execute(params!["xdbf.entry_count", x.entries.len().to_string()])?;
if let Some(l) = x.default_language {
@@ -1991,6 +2104,7 @@ fn insert_instructions_streaming(
func_analysis: &FuncAnalysis,
labels: &HashMap<u32, String>,
data_words: &std::collections::BTreeSet<u32>,
import_sites: &crate::imports::ImportSites,
) -> anyhow::Result<()> {
let mut appender = conn.appender("instructions")?;
let mut total: u64 = 0;
@@ -2010,6 +2124,7 @@ fn insert_instructions_streaming(
func_analysis,
labels,
data_words,
import_sites,
);
total += crate::sinks::duckdb::append_instructions(&mut appender, items)?;
}
@@ -2020,6 +2135,34 @@ fn insert_instructions_streaming(
Ok(())
}
/// Record each import thunk's two loader-rewritten words in `data_in_code`.
///
/// `data_in_code` already lists recovered jump tables; import records are the
/// other kind of non-instruction word sitting in a code section, so a consumer
/// asking "which bytes of `.text` are not code?" gets a complete answer from
/// one table instead of having to know about thunk layout.
fn insert_import_record_extents(
conn: &Connection,
import_sites: &crate::imports::ImportSites,
) -> anyhow::Result<()> {
let mut heads: Vec<u32> = import_sites.thunk_heads().collect();
heads.sort_unstable();
let mut stmt =
conn.prepare("INSERT INTO data_in_code (address, length, kind) VALUES (?, ?, ?)")?;
for head in &heads {
// The two records are contiguous at the head of the thunk; the two
// instructions that follow them are real code.
stmt.execute(params![*head as i64, 8i64, "import_record"])?;
}
metrics::counter!("db.rows", "table" => "data_in_code").increment(heads.len() as u64);
tracing::info!(
rows = heads.len(),
kind = "import_record",
"data_in_code write complete"
);
Ok(())
}
/// Write the M12 jump-table tables plus the data-in-code extent list.
fn insert_jump_tables(
conn: &Connection,

22
crates/sylpheed-xexdb/src/disasm.rs
View File

@@ -10,6 +10,7 @@ use std::collections::{BTreeSet, HashMap};
use sylpheed_ppc::disasm::DisasmItem;
use crate::func::FuncAnalysis;
use crate::imports::{ImportRole, ImportSites};
/// `DisasmItem` plus the analysis context (section/function/label).
#[derive(Debug, Clone)]
@@ -19,8 +20,13 @@ pub struct RichDisasmItem<'a> {
pub function: Option<u32>,
pub label: Option<&'a str>,
/// True when this word is data embedded in a code section (a recovered
/// jump table or its index map), so its decoded text is meaningless.
/// jump table or its index map, or an XEX import record), so its decoded
/// text is meaningless.
pub is_data: bool,
/// The XEX import this word belongs to, as `(thunk head VA, role)`. The
/// head VA is `imports.address` for that import, so it is written out as
/// a foreign key. `None` for the overwhelming majority of instructions.
pub import: Option<(u32, ImportRole)>,
}
/// Walk one code section, yielding rich items annotated with section name,
@@ -42,6 +48,12 @@ pub struct RichDisasmItem<'a> {
/// Rows at those addresses are still emitted — their `raw` value is the table
/// entry a consumer wants — but flagged so nothing mistakes the decoded text
/// for a real instruction.
///
/// `import_sites` resolves XEX import thunks. It contributes two things: the
/// two import-record words at each thunk head are also flagged `is_data`
/// (they are loader-rewritten data, not code, and decode to nonsense), and
/// every word of a thunk plus every direct branch into one carries the
/// import it belongs to.
pub fn enrich_section<'a>(
image: &'a [u8],
image_base: u32,
@@ -51,6 +63,7 @@ pub fn enrich_section<'a>(
func_analysis: &'a FuncAnalysis,
labels: &'a HashMap<u32, String>,
data_words: &'a BTreeSet<u32>,
import_sites: &'a ImportSites,
) -> impl Iterator<Item = RichDisasmItem<'a>> + 'a {
// (start, end) of the function currently being walked.
let mut current: Option<(u32, u32)> = None;
@@ -67,13 +80,16 @@ pub fn enrich_section<'a>(
}
let current_func = current.map(|(start, _)| start);
let label = labels.get(&item.addr).map(|s| s.as_str());
let is_data = data_words.contains(&item.addr);
let import = import_sites.lookup(item.addr, item.text.branch_target);
let is_data =
data_words.contains(&item.addr) || matches!(import, Some((_, ImportRole::Record)));
RichDisasmItem {
item,
section: section_name,
function: current_func,
label,
is_data,
import,
}
})
}
@@ -127,6 +143,7 @@ mod tests {
&fa,
&labels,
&data_words,
&ImportSites::default(),
)
.map(|r| (r.item.addr, r.function))
.collect();
@@ -170,6 +187,7 @@ mod tests {
&fa,
&labels,
&data_words,
&ImportSites::default(),
)
.map(|r| r.function)
.collect();

21
crates/sylpheed-xexdb/src/formatter.rs
View File

@@ -8,6 +8,7 @@ use sylpheed_xex::pe::PeSection;
use crate::disasm::enrich_section;
use crate::func::FuncAnalysis;
use crate::imports::{ImportRole, ImportSites};
use crate::sinks::text::write_instr_line;
use crate::xref::{Xref, XrefKind, XrefMap, resolve_source_label};
@@ -38,6 +39,7 @@ pub fn write_asm(
xrefs: &XrefMap,
data_annotations: &HashMap<u32, (u32, XrefKind)>,
data_words: &BTreeSet<u32>,
import_sites: &ImportSites,
) -> anyhow::Result<()> {
// Header
writeln!(
@@ -136,6 +138,7 @@ pub fn write_asm(
func_analysis,
labels,
data_words,
import_sites,
);
for ri in items {
let abs_addr = ri.item.addr;
@@ -208,10 +211,26 @@ pub fn write_asm(
}
}
// Import thunk annotation
// Import annotations. The thunk head keeps its banner; call
// sites get a trailing note so a `bl` into a thunk reads as
// the kernel call it actually is rather than a bare address.
if let Some(imp_name) = import_map.get(&abs_addr) {
writeln!(out, " ; IMPORT: {imp_name}")?;
}
if let Some((head, role)) = ri.import {
match role {
ImportRole::Record => writeln!(
out,
" ; import record (loader-patched; not an instruction)",
)?,
ImportRole::Call => {
if let Some(name) = import_map.get(&head) {
writeln!(out, " ; -> IMPORT {name}")?;
}
}
ImportRole::Thunk => {}
}
}
let data_annot = data_annotations.get(&abs_addr).copied();
write_instr_line(out, &ri, labels, info.sections, info.image_base, data_annot)?;

354
crates/sylpheed-xexdb/src/imports.rs Normal file
View File

@@ -0,0 +1,354 @@
//! Import-thunk recognition — makes XEX imports visible in the code stream.
//!
//! An XEX import is not a PLT-style jump. The linker emits a four-word thunk
//! whose first two words are *import records*, which the loader rewrites at
//! module load into `lis r11, hi(target)` / `ori r11, r11, lo(target)`. On
//! disc they are still records, so a decoder that only knows PowerPC renders
//! them as two meaningless `.long`s in front of an indirect branch:
//!
//! ```text
//! 8284DA7C .long 0x0100028C <- record: type 1, library 0, ordinal 652
//! 8284DA80 .long 0x0200028C <- record: type 2, library 0, ordinal 652
//! 8284DA84 mtctr r11
//! 8284DA88 bctr
//! ```
//!
//! Record layout, matching xenia-canary `XexModule::SetupLibraryImports`:
//!
//! | bits | meaning |
//! |---------|------------------------------------------------------------|
//! | 31..24 | record type — `1` and `2` mark the two halves of a thunk |
//! | 23..16 | index of the owning import library |
//! | 15..0 | ordinal within that library |
//!
//! This module maps every word of every thunk — and every direct branch that
//! targets one — back to the `imports` row it belongs to, so the disassembly
//! can say `xboxkrnl.exe::RtlEnterCriticalSection` where it used to say
//! `.long 0x01010194`.
//!
//! The mapping is deliberately *shape-validated* rather than trusted: an
//! import entry is only indexed when the four words it points at actually
//! look like a thunk for that exact ordinal. A malformed or already-patched
//! image annotates nothing rather than mislabelling real code.
use std::collections::HashMap;
use sylpheed_xex::header::ImportLibrary;
/// `mtspr CTR, r11` — third word of every import thunk.
const MTCTR_R11: u32 = 0x7D69_03A6;
/// `bcctr 20, lt` (`bctr`) — fourth word of every import thunk.
const BCTR: u32 = 0x4E80_0420;
/// Record type in the first thunk word.
const RECORD_TYPE_FIRST: u32 = 1;
/// Record type in the second thunk word.
const RECORD_TYPE_SECOND: u32 = 2;
/// A thunk is four 4-byte words.
pub const THUNK_SIZE: u32 = 16;
/// How an instruction relates to the import it references.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ImportRole {
/// One of the two import-record words at the head of a thunk. The word is
/// data the loader rewrites, not an instruction — its decoded text is
/// meaningless and it is flagged `is_data` alongside jump-table words.
Record,
/// A real instruction in the thunk body (`mtctr r11` / `bctr`).
Thunk,
/// A direct branch (`bl` or a tail-call `b`) whose target is this
/// import's thunk head.
Call,
}
impl ImportRole {
/// Stable lowercase tag, as stored in `instructions.import_role`.
pub fn as_str(self) -> &'static str {
match self {
Self::Record => "record",
Self::Thunk => "thunk",
Self::Call => "call",
}
}
}
/// Word-level index from code addresses to the import they belong to.
///
/// Values are the thunk head VA, which is exactly `imports.address` for the
/// `record_type = 1` row — so it doubles as the foreign key written into
/// `instructions.import_address`.
#[derive(Debug, Default)]
pub struct ImportSites {
/// Every word inside a validated thunk -> (thunk head VA, role).
words: HashMap<u32, (u32, ImportRole)>,
}
impl ImportSites {
/// Index every import thunk in `libraries` that `image` actually backs.
///
/// `record_type = 0` entries are plain pointer slots in `.rdata`: the
/// loader writes the resolved address there and no code is emitted, so
/// there is nothing in the instruction stream to annotate.
pub fn build(libraries: &[ImportLibrary], image: &[u8], image_base: u32) -> Self {
let read = |va: u32| -> Option<u32> {
let off = va.wrapping_sub(image_base) as usize;
if off.checked_add(4)? > image.len() {
return None;
}
Some(u32::from_be_bytes([
image[off],
image[off + 1],
image[off + 2],
image[off + 3],
]))
};
let mut words = HashMap::new();
let mut indexed = 0usize;
let mut rejected = 0usize;
for lib in libraries {
for imp in &lib.imports {
if imp.record_type != 1 {
continue;
}
let head = imp.address;
let Some(w0) = read(head) else {
rejected += 1;
continue;
};
let (Some(w1), Some(w2), Some(w3)) =
(read(head + 4), read(head + 8), read(head + 12))
else {
rejected += 1;
continue;
};
if !is_thunk([w0, w1, w2, w3], imp.ordinal) {
rejected += 1;
continue;
}
words.insert(head, (head, ImportRole::Record));
words.insert(head + 4, (head, ImportRole::Record));
words.insert(head + 8, (head, ImportRole::Thunk));
words.insert(head + 12, (head, ImportRole::Thunk));
indexed += 1;
}
}
tracing::info!(
thunks = indexed,
rejected,
words = words.len(),
"import thunk index built",
);
Self { words }
}
/// The import this instruction references, and in what capacity.
///
/// `branch_target` is the resolved target of a direct branch. A `bl` into
/// a thunk head yields [`ImportRole::Call`], which is what lets a query
/// find *call sites* of a kernel function rather than just the thunk.
/// Landing mid-thunk is not a call, so only the head matches.
pub fn lookup(&self, addr: u32, branch_target: Option<u32>) -> Option<(u32, ImportRole)> {
if let Some(&hit) = self.words.get(&addr) {
return Some(hit);
}
let target = branch_target?;
match self.words.get(&target) {
// `head == target` rejects a branch into the second record word.
Some(&(head, _)) if head == target => Some((head, ImportRole::Call)),
_ => None,
}
}
/// Addresses of the import-record words — data embedded in a code section.
pub fn record_words(&self) -> impl Iterator<Item = u32> + '_ {
self.words
.iter()
.filter(|(_, (_, role))| *role == ImportRole::Record)
.map(|(&addr, _)| addr)
}
/// Thunk head VAs, one per indexed import.
pub fn thunk_heads(&self) -> impl Iterator<Item = u32> + '_ {
self.words
.iter()
.filter(|(addr, (head, _))| *addr == head)
.map(|(&addr, _)| addr)
}
/// Number of thunks indexed.
pub fn len(&self) -> usize {
self.words.len() / 4
}
pub fn is_empty(&self) -> bool {
self.words.is_empty()
}
}
/// Does this four-word run look like an import thunk for `ordinal`?
///
/// The library-index byte is checked for *consistency between the two
/// records* rather than against the library's position in the header: the
/// ordinal match plus the fixed `mtctr`/`bctr` tail is already conclusive,
/// and not depending on header ordering keeps this robust for titles whose
/// import libraries are listed in a different order.
fn is_thunk(w: [u32; 4], ordinal: u16) -> bool {
w[0] >> 24 == RECORD_TYPE_FIRST
&& w[1] >> 24 == RECORD_TYPE_SECOND
&& (w[0] & 0xFFFF) as u16 == ordinal
&& (w[1] & 0xFFFF) as u16 == ordinal
&& (w[0] >> 16) & 0xFF == (w[1] >> 16) & 0xFF
&& w[2] == MTCTR_R11
&& w[3] == BCTR
}
#[cfg(test)]
mod tests {
use super::*;
use sylpheed_xex::header::ImportEntry;
const BASE: u32 = 0x8200_0000;
/// Build a one-library header plus a backing image containing `thunks`
/// laid out back to back from `BASE`.
fn fixture(thunks: &[(u16, u8)]) -> (Vec<ImportLibrary>, Vec<u8>) {
let mut image = Vec::new();
let mut imports = Vec::new();
for (i, &(ordinal, lib_index)) in thunks.iter().enumerate() {
let head = BASE + i as u32 * THUNK_SIZE;
let li = (lib_index as u32) << 16;
for w in [
(RECORD_TYPE_FIRST << 24) | li | ordinal as u32,
(RECORD_TYPE_SECOND << 24) | li | ordinal as u32,
MTCTR_R11,
BCTR,
] {
image.extend_from_slice(&w.to_be_bytes());
}
imports.push(ImportEntry {
ordinal,
record_type: 1,
address: head,
});
}
let lib = ImportLibrary {
name: "xboxkrnl.exe".into(),
id: 0,
version_min: 0,
version_cur: 0,
imports,
};
(vec![lib], image)
}
#[test]
fn indexes_every_word_of_a_thunk() {
let (libs, image) = fixture(&[(0x28C, 0)]);
let sites = ImportSites::build(&libs, &image, BASE);
assert_eq!(sites.len(), 1);
assert_eq!(sites.lookup(BASE, None), Some((BASE, ImportRole::Record)));
assert_eq!(
sites.lookup(BASE + 4, None),
Some((BASE, ImportRole::Record))
);
assert_eq!(
sites.lookup(BASE + 8, None),
Some((BASE, ImportRole::Thunk))
);
assert_eq!(
sites.lookup(BASE + 12, None),
Some((BASE, ImportRole::Thunk))
);
assert_eq!(sites.lookup(BASE + 16, None), None);
}
/// The point of the whole exercise: a `bl` somewhere else in the binary
/// must resolve to the import it ends up calling.
#[test]
fn direct_branch_to_a_thunk_head_is_a_call() {
let (libs, image) = fixture(&[(0x28C, 0)]);
let sites = ImportSites::build(&libs, &image, BASE);
assert_eq!(
sites.lookup(0x8215_0000, Some(BASE)),
Some((BASE, ImportRole::Call)),
);
}
/// A branch into the *second* record word is not a call to the import;
/// treating it as one would invent a call edge that does not exist.
#[test]
fn branch_into_thunk_interior_is_not_a_call() {
let (libs, image) = fixture(&[(0x28C, 0)]);
let sites = ImportSites::build(&libs, &image, BASE);
assert_eq!(sites.lookup(0x8215_0000, Some(BASE + 4)), None);
assert_eq!(sites.lookup(0x8215_0000, Some(BASE + 8)), None);
}
/// A word inside a thunk keeps its own role even when it also carries a
/// branch target, so the thunk body is never relabelled as a call.
#[test]
fn own_role_wins_over_branch_target() {
let (libs, image) = fixture(&[(0x28C, 0), (0x194, 1)]);
let sites = ImportSites::build(&libs, &image, BASE);
let second = BASE + THUNK_SIZE;
assert_eq!(
sites.lookup(BASE + 8, Some(second)),
Some((BASE, ImportRole::Thunk))
);
}
/// Shape validation must reject an entry whose ordinal does not match the
/// record it points at — otherwise a stale import table would silently
/// stamp the wrong kernel name onto real code.
#[test]
fn ordinal_mismatch_is_rejected() {
let (mut libs, image) = fixture(&[(0x28C, 0)]);
libs[0].imports[0].ordinal = 0x999;
let sites = ImportSites::build(&libs, &image, BASE);
assert!(sites.is_empty());
}
/// An already-patched image (records rewritten to `lis`/`ori`) has no
/// records left to recognise, and must annotate nothing rather than
/// guessing.
#[test]
fn patched_thunk_is_rejected() {
let (libs, mut image) = fixture(&[(0x28C, 0)]);
image[0..4].copy_from_slice(&0x3D60_8200u32.to_be_bytes()); // lis r11, 0x8200
let sites = ImportSites::build(&libs, &image, BASE);
assert!(sites.is_empty());
}
/// A `record_type = 0` entry is a data slot in `.rdata`, not code.
#[test]
fn variable_records_are_not_indexed() {
let (mut libs, image) = fixture(&[(0x28C, 0)]);
libs[0].imports[0].record_type = 0;
let sites = ImportSites::build(&libs, &image, BASE);
assert!(sites.is_empty());
}
/// An import pointing outside the image must not panic or index garbage.
#[test]
fn out_of_bounds_thunk_is_rejected() {
let (mut libs, image) = fixture(&[(0x28C, 0)]);
libs[0].imports[0].address = BASE + 0x10_0000;
let sites = ImportSites::build(&libs, &image, BASE);
assert!(sites.is_empty());
}
#[test]
fn record_words_lists_both_halves() {
let (libs, image) = fixture(&[(0x28C, 0), (0x194, 1)]);
let sites = ImportSites::build(&libs, &image, BASE);
let mut got: Vec<u32> = sites.record_words().collect();
got.sort_unstable();
assert_eq!(got, vec![BASE, BASE + 4, BASE + 16, BASE + 20]);
let mut heads: Vec<u32> = sites.thunk_heads().collect();
heads.sort_unstable();
assert_eq!(heads, vec![BASE, BASE + 16]);
}
}

2
crates/sylpheed-xexdb/src/lib.rs
View File

@@ -25,6 +25,7 @@ pub mod eh_scope;
pub mod formatter;
pub mod func;
pub mod funcptr_arrays;
pub mod imports;
pub mod ind_dispatch_typed;
pub mod indirect;
pub mod jumptables;
@@ -42,5 +43,6 @@ pub mod xref;
mod ordinals;
pub use db::{BranchTraceEntry, DbWriter, ExecTraceEntry, ImportCallEntry};
pub use disasm::{RichDisasmItem, enrich_section};
pub use imports::{ImportRole, ImportSites};
pub use ordinals::resolve_ordinal;
pub use xref::{Xref, XrefKind, XrefMap, resolve_source_label};

2
crates/sylpheed-xexdb/src/sinks/duckdb.rs
View File

@@ -32,6 +32,8 @@ pub fn append_instructions<'a>(
ri.function.map(|f| f as i64),
ri.label,
ri.is_data,
ri.import.map(|(head, _)| head as i64),
ri.import.map(|(_, role)| role.as_str()),
])?;
count += 1;
}

34
crates/sylpheed-xexdb/src/sql_views.rs
View File

@@ -192,20 +192,34 @@ FROM functions f
JOIN instructions i ON i.address = f.address;
";
/// Per-function summary of which kernel/library imports it calls. Joins
/// xrefs (call edges) against the labels table to surface import names.
/// Every call site of a kernel/library import, one row per branch.
///
/// Driven by `instructions.import_address` — the foreign key onto `imports`
/// laid down by [`crate::imports::ImportSites`] — rather than by matching
/// `xrefs` against import-kind labels. That change fixes two things:
///
/// - a tail-call `b` into a thunk is a call to that import, but `xrefs` tags
/// it `'j'`, so the label-matching version silently dropped all of them
/// (64 sites on the reference title, against 3,217 `bl`);
/// - the library and ordinal are now available without re-parsing the name,
/// so `WHERE library = 'xboxkrnl.exe'` works directly.
///
/// `call_site` is the branch instruction; `import_addr` is the thunk head.
const V_IMPORTS_CALLED: &str = "
CREATE OR REPLACE VIEW v_imports_called AS
SELECT
x.source_func AS function_addr,
i.address AS call_site,
i.function AS function_addr,
f.name AS function_name,
x.target AS import_addr,
l.name AS import_name
FROM xrefs x
JOIN labels l ON l.address = x.target
LEFT JOIN functions f ON f.address = x.source_func
WHERE x.kind = 'call'
AND l.kind = 'import';
i.mnemonic AS instruction,
i.import_address AS import_addr,
im.library AS library,
im.name AS import_name,
im.ordinal AS ordinal
FROM instructions i
JOIN imports im ON im.address = i.import_address
LEFT JOIN functions f ON f.address = i.function
WHERE i.import_role = 'call';
";
/// Every recovered `switch` case, joined to the dispatching function and to

60
crates/sylpheed-xexdb/tests/db_schema_golden.rs
View File

@@ -176,6 +176,8 @@ fn db_schema_matches_expected_columns() {
("function", "BIGINT"),
("label", "VARCHAR"),
("is_data", "BOOLEAN"),
("import_address", "BIGINT"),
("import_role", "VARCHAR"),
],
),
(
@@ -564,5 +566,63 @@ fn db_schema_matches_expected_columns() {
assert_eq!(exists, 1, "missing SQL view: {v}");
}
// ── Foreign keys ────────────────────────────────────────────────────
//
// Lock the declared FK set. These are correctness assertions baked into
// the schema: a build that would produce a dangling reference fails at
// insert time rather than shipping a broken database.
let expected_fks = [
("classes", "vtables"),
("data_in_code", "instructions"),
("eh_try_blocks", "eh_funcinfo"),
("eh_unwind_map", "eh_funcinfo"),
("function_pointer_array_entries", "function_pointer_arrays"),
("indirect_dispatch_candidates", "indirect_dispatch_sites"),
("indirect_dispatch_candidates", "vtables"),
("instructions", "imports"),
("jump_table_entries", "jump_tables"),
("methods", "vtables"),
("rtti_base_classes", "rtti_type_descriptors"),
("rtti_locators", "rtti_type_descriptors"),
("vptr_writes", "vtables"),
("vtables", "rtti_locators"),
("xdbf_achievements", "xdbf_images"),
("xrefs", "instructions"),
];
for (child, parent) in expected_fks {
let n: i64 = conn
.query_row(
"SELECT COUNT(*) FROM duckdb_constraints() \
WHERE constraint_type = 'FOREIGN KEY' AND table_name = ? \
AND referenced_table = ?",
[child, parent],
|r| r.get(0),
)
.unwrap();
assert_eq!(n, 1, "missing foreign key: {child} -> {parent}");
}
// `functions` must never become a foreign-key parent. DuckDB implements
// UPDATE as delete-then-insert, so a single inbound FK makes the table
// un-updatable — even for an update that does not touch `address` — and
// that breaks `apply_re_symbols.sql`, which re-stamps reverse-engineered
// names onto `functions.name` after every regeneration. Adding one back
// looks harmless in review and only fails at symbol-stamping time, hence
// this guard.
let inbound_to_functions: i64 = conn
.query_row(
"SELECT COUNT(*) FROM duckdb_constraints() \
WHERE constraint_type = 'FOREIGN KEY' \
AND referenced_table = 'functions'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(
inbound_to_functions, 0,
"no table may declare a foreign key onto `functions`: it would make \
functions.name un-updatable and break apply_re_symbols.sql",
);
let _ = std::fs::remove_file(&tmp);
}

121
docs/re/RE_SYMBOLS.md Normal file
View File

@@ -0,0 +1,121 @@
# Project Sylpheed — reverse-engineered symbol map
Durable record of guest functions/data in the retail title (`default.xex`,
base `0x82000000`) that have been positively identified by RE. Addresses are
guest virtual addresses. This is the source of truth for names; the DuckDB
`sylpheed.db` (this dir) `functions.name` column can be re-stamped from here
with `apply_re_symbols.sql` (names are wiped on a full DB regen).
Confidence: **H** = behaviour proven (reproduced/verified against data), **M** =
strongly inferred from disassembly, **L** = tentative.
## Archive (IPFB `.pak`) subsystem
| Address | Name | Conf | Description |
|---|---|---|---|
| `0x824609C8` | `Pak_FindEntryByName` | H | Look up a TOC entry by path. Args `(r3 = pak object, r4 = char* path)`. Calls `Pak_HashPathName(path)`, then binary-searches the sorted 12-byte TOC (base `pak+4`, count `pak+8`, stride 12; compares the 32-bit `name_hash`). Returns the matching record pointer or 0. |
| `0x82460928` | `Pak_HashPathName` | H | Duplicate `r3 = char* path`, lowercase it in place (`Str_ToLowerAscii`), hash with `Sylph_NameHash`, free the copy, return the 32-bit hash. This is the path→key bridge. |
| `0x82455C78` | `Sylph_NameHash` | H | The IPFB TOC / cache-path name-hash. See [Name-hash](#name-hash-sylph_namehash). Input must already be lowercased. `r3 = char*` → `r3 = u32 hash`. |
| `0x825F4F90` | `Str_ToLowerAscii` | H | In-place ASCII lowercase (`A`–`Z` → `+0x20`); other bytes pass through. `r3 = char*`. (When `r3 == 0` it takes an unrelated init branch.) |
| `0x824607B0` | `Pak_IsIPFBHeader` | H | Validate an IPFB header. Loads the header's first word and checks it equals `"IPFB"` (`0x49504642`) via a delta trick: `first_word - 0x49504620 == 0x22` (`'B' - ' '`). `r5 = header ptr`. Returns bool-ish in `r3`. |
| `0x82460AD0`… | *(pak header parse, unregistered container)* | M | The pak-open/validate path around `0x82460DC0`–`0x82460EC4` builds an expected-header template (`"IPFB"`, `0x10000000` flags) on the stack and calls `Pak_IsIPFBHeader`. Function-boundary detection missed the enclosing frame (entry ≈ `0x82460E34`). |
| `0x82458508` | `Archive_StreamReadCrc32` | M | Streaming block reader that validates content with a reflected CRC-32 (init `0xFFFFFFFF`, final `~`, table at `g_Crc32Table`). Compares the computed CRC to a stored expected value (`state+144`). This is a **content-integrity** CRC, *not* the name-hash. |
| `0x828992F0` | `g_Crc32Table` (data) | H | 256-entry reflected CRC-32 lookup table. Referenced only by `Archive_StreamReadCrc32` and its sibling loop at `0x82457AF8`. |
## IDXD reflective serialization framework
Generic tagged-object (de)serializer shared by ~15 object families (craft,
weapon, message/character, effects, …). The `.pak` IDXD payloads are read
through this.
| Address | Name | Conf | Description |
|---|---|---|---|
| `0x824486C0` | `IdxdLoad_Dispatch` | M | Entry: reads the magic and dispatches across variants `IDXD` (`0x49445844`), `IDX2`, `IDX3`, `IDXC`, `IXUD` → the matching parser. **15 callers** (one per object family). The target object is passed in `r3` — already constructed (defaults set) by the caller. |
| `0x82448D00` | `IdxdLoad_Variant2` | M | Sibling variant loader (also materializes the `IDXD` magic). |
| `0x82449640` | `Idxd_Parse` | M | The IDXD body parser: skips a BOM, then loops the string pool — per field, reads the name token, `Reflect_FindFieldIndex`, then dispatches on the **value's first char** (jump table at `0x824497F8`, index `firstchar - 0x22`) to type-specific writers. Only fields **present** in the pool are written. |
| `0x8244A2F0` | `Reflect_FindFieldIndex` | M | Look up a field-name string in a global name-vector at `0x820B4F18` (std::vector-like: count `+20`, data `+24`/SSO), via a `strncmp`-style compare (`0x825EDCE0`). Returns index or `-1`. |
| `0x8244A4B0` | `Reflect_SetField` | L | Apply a value to `object` using its per-instance property registry at `object+64` and a config string at `0x820B4F28`. |
| `0x820B4F18` | `g_FieldNameRegistry` (data) | L | Global reflection name-vector (runtime-built) used by `Reflect_FindFieldIndex`. |
**Defaulted fields:** a field is written to the IDXD pool only when it differs
from its default, so `Idxd_Parse` never sets defaulted fields — their values are
established by each **type's constructor** (in the 15 `IdxdLoad_Dispatch`
callers) before load, via this same registry. Recovering them statically means
reversing those constructors and joining `field-name → struct-offset →
default-store`; a runtime dump of a loaded object's registry gives
`name → value` directly. (Open — see below.)
## 3D resource subsystem
| Address | Name | Conf | Description |
|---|---|---|---|
| `0x82640290` | `Res3D_LoadMeshChunk` | L | 3D-resource loader: walks 12-byte records, checks chunk tags (`0x1A22AA26`, `0x2DA2AA24`), and copies `float` triples (x/y/z vertices) via `lfs`/`stfs`. Part of the DefTables/`machines\<model>\…` mesh path. |
## Name-hash (`Sylph_NameHash`)
Per-byte Barrett-reduced polynomial hash over the **lowercased** name:
```
A = 0 ; B = 0
for each (sign-extended) byte c of the lowercased name:
A = (A << 8) + c # 32-bit
A = A - (((A * 0x8003_1493) >> 32) rol 9 & 0x1FF) * 0x00FF_F9D7 # A mod 0x00FF_F9D7 (no final fixup)
B = B + c
hash = ((B & 0xFF) << 24) | (A & 0x00FF_FFFF)
```
- Modulus `0x00FF_F9D7`, reciprocal `0x8003_1493` (the `mulhwu`/`mullw` Barrett step).
- Low 24 bits = modular polynomial hash; top byte = 8-bit additive checksum of the bytes.
- No trailing conditional subtract — the value is defined by the exact op sequence.
- Faithful Rust reproduction: `sylpheed-formats/src/hash.rs` (`name_hash`).
**Verified** against retail TOCs: `name_hash("files.tbl") == 0x8342_1153`,
`name_hash("eng\\weapon.tbl") == 0x900C_8DCD`, `name_hash("eng\\strings.tbl") == 0x10C8_0B87`,
`name_hash("jpn\\weapon.tbl") == 0x9E85_FEFF`.
## TOC key path conventions (preimages)
The 32-bit TOC key is `name_hash` of a **backslash** path with the entry's
internal identity string. Confirmed schemes:
| Scheme | Example | Where |
|---|---|---|
| `unit\<ID>.tbl` | `unit\UN_f001_TCAF_DeltaSaber_T_EX5.tbl` → `0x7C96296C` | craft/ship entries in `GP_MAIN_GAME_*.pak` |
| `weapon\<ID>.tbl` | `weapon\Weapon_DSaber_P_wep_26_Missile.tbl` | weapon entries |
| `message\<ID>.tbl` | `message\CharacterCARL.tbl` | character/dialog entries |
| `effect\<ID>.tbl` | — | effect definitions |
| `<lang>\<name>.tbl` | `eng\weapon.tbl`, `jpn\strings.tbl` | localized loadout/text tables |
| `<name>.tbl` | `files.tbl` | root manifest / DefTables resource entries |
`<ID>` = the entry's internal `ID` field (IDXD string pool). Craft/weapon
entries also self-describe by content, so stats are readable without the key.
Coverage with these schemes (IDXD entries, whole-pool ID extraction):
GP_MAIN_GAME_E 308/1004 (≈31%), DefTables 804/1425 (≈56%). Shipped in
`sylpheed-formats` (`hash::recover_toc_name`, `pak list` prints resolved
paths). The unresolved remainder use deeper cross-referenced directory
paths (e.g. per-mission dialog `MSG_*` entries) — a later hunt, not an
extraction limit.
## Open threads
- **Defaulted IDXD fields.** A field is omitted from the pool exactly when it
equals the schema default, so present values are all *overrides* and the
defaults cannot be inferred from them. Gap is large: across the 534 craft
(`schema 0x43FAA517`) entries, explicit-presence is `Acceleration` 42%,
`MaximumVelocity` 51%, `Turn_AngularVelocity` 31%, `FCSRange` 34%,
`ShieldRatio` 49% (`Size_X` 100%, `HP` 85%). So 40–70% of craft rely on
unknown defaults for core flight/defense stats.
- Ruled out: the big 16-byte-record binary node table inside a craft IDXD is
**spatial/mesh data**, not defaults (not keyed by `name_hash(field_key)`).
- `schema_hash` is **not** a code immediate and **not** `name_hash(typename)`.
- Defaults are set by each type's **constructor** (one per `IdxdLoad_Dispatch`
caller) via the reflection registry, not by `Idxd_Parse`.
- **Two finish routes.** (A, static) reverse the craft constructor: read its
field registrations (`name → offset`) and default-init stores
(`offset → value`), join them — heavy, and offset↔name correlation is
error-prone. (B, runtime) dump a loaded craft object's registry
(`obj+64`) / struct from xenia-rs or Canary — a fully-loaded craft already
holds every default; one dump yields `name → value`. **B is the efficient
finish** given the framework is generic; A's framework map above is the
prerequisite either way.

21
tools/apply_re_symbols.sql Normal file
View File

@@ -0,0 +1,21 @@
-- Re-stamp reverse-engineered function names onto sylpheed.db.
-- Source of truth: docs/re/RE_SYMBOLS.md. Re-run after any full DB regen
-- (regen wipes functions.name back to sub_XXXX).
-- python3 -c "import duckdb; duckdb.connect('sylpheed.db').execute(open('tools/apply_re_symbols.sql').read())"
-- (from the repo root; `sylpheed.db` is where `tools/zq.py` looks by default)
-- Addresses are decimal (DuckDB rejects 0x literals in some contexts); hex in comments.
UPDATE functions SET name = 'Pak_FindEntryByName' WHERE address = 2185628104; -- 0x824609C8
UPDATE functions SET name = 'Pak_HashPathName' WHERE address = 2185627944; -- 0x82460928
UPDATE functions SET name = 'Sylph_NameHash' WHERE address = 2185583736; -- 0x82455C78
UPDATE functions SET name = 'Str_ToLowerAscii' WHERE address = 2187284368; -- 0x825F4F90
UPDATE functions SET name = 'Pak_IsIPFBHeader' WHERE address = 2185627568; -- 0x824607B0
UPDATE functions SET name = 'Archive_StreamReadCrc32' WHERE address = 2185594120; -- 0x82458508
UPDATE functions SET name = 'Res3D_LoadMeshChunk' WHERE address = 2187592336; -- 0x82640290
-- IDXD reflective serialization framework (defaulted-field hunt, 2026-07-09)
UPDATE functions SET name = 'IdxdLoad_Dispatch' WHERE address = 2185529024; -- 0x824486C0 (magic dispatch IDXD/IDX2/IDX3/IDXC/IXUD; 15 callers)
UPDATE functions SET name = 'IdxdLoad_Variant2' WHERE address = 2185530624; -- 0x82448D00 (sibling variant loader)
UPDATE functions SET name = 'Idxd_Parse' WHERE address = 2185532992; -- 0x82449640 (tokenize pool; per-field lookup+set)
UPDATE functions SET name = 'Reflect_FindFieldIndex' WHERE address = 2185536240; -- 0x8244A2F0 (field-name -> index in name-vector @0x820B4F18)
UPDATE functions SET name = 'Reflect_SetField' WHERE address = 2185536688; -- 0x8244A4B0 (apply value via object registry @obj+64)

166
tools/zq.py
View File

@@ -15,7 +15,7 @@ both static, neither needing anything to run:
alongside a metadata JSON.
Usage:
zq.py dis <lo_hex> <hi_hex> # disassemble [lo,hi) (jump-table words shown as .long)
zq.py dis <lo_hex> <hi_hex> # disassemble [lo,hi) (data words as .long; imports named)
zq.py fn <pc_hex> # function containing pc (address,name,end)
zq.py xref <target_hex> # xrefs whose target == addr (callers)
zq.py callers <vtable_off_dec> # call-sites of vtable slot at byte offset N
@@ -29,20 +29,69 @@ Usage:
zq.py class <name> # one class: bases, vtable, virtual methods
zq.py str <substr> # string literals matching, with referencing functions
zq.py gaps # dangling refs = known detector gaps (see db.rs schema docs)
zq.py imp [substr] # imports + how many call sites each has
zq.py impcalls <substr> # every call site of the matching imports
zq.py xdbf [substr] # XDBF title text (all locales); substr filters
zq.py ach # XDBF achievements (id, gamerscore, name, descriptions)
A command that needs a table the current DB predates prints what to regenerate
rather than a SQL error.
The database is found via `$SYLPH_XEXDB`, else `<repo root>/sylpheed.db`. It is a
build artefact, not a tracked file; if neither exists, zq.py says so and prints
the command that builds one.
"""
import duckdb, sys
import duckdb, sys, os, pathlib, signal
DB = '/home/fabi/RE - Project Sylpheed/xenia-rs/sylpheed.db'
c = duckdb.connect(DB, read_only=True)
# Output is routinely piped into `head`; without this Python prints a
# BrokenPipeError traceback when the reader closes early.
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
REGEN = "sylph-xexdb dis <xex|iso> --db sylpheed.db --analyze sql"
def _resolve_db():
"""Locate the database: `$SYLPH_XEXDB`, else `<repo root>/sylpheed.db`.
🔴 This used to be one hardcoded absolute path, and it pointed *inside*
`xenia-rs` -- a repository `docs/agents/CONSOLIDATION.md` archives in Phase 5
and drops in Phase 7. It resolved on exactly one machine and would have
started failing there too, with `duckdb` raising about a missing file rather
than anything saying why.
The database is a build artefact of several hundred MB and is not in the
repository, so there is nothing to fall back *to*: if neither location has
one, say so and print how to build it. A default that silently resolves to
the wrong database is worse than no default -- see issue #16.
"""
env = os.environ.get('SYLPH_XEXDB')
if env:
p = pathlib.Path(env).expanduser()
if not p.is_file():
sys.exit(f'$SYLPH_XEXDB is set but is not a file:\n {p}')
return p
# Relative to this script, not to the caller's cwd: `zq.py` is run from
# wherever the investigation happens to be.
p = pathlib.Path(__file__).resolve().parent.parent / 'sylpheed.db'
if p.is_file():
return p
sys.exit(
f'no database found.\n'
f' looked for: {p}\n'
f' set $SYLPH_XEXDB to an existing one, or build it with:\n'
f' {REGEN}'
)
# Resolved lazily enough that `zq.py` with no command still prints its usage on
# a machine that has no database yet.
_WANTS_DB = len(sys.argv) > 1 and sys.argv[1] not in ('-h', '--help', 'help')
DB = _resolve_db() if _WANTS_DB else None
c = duckdb.connect(str(DB), read_only=True) if DB is not None else None
H = lambda x: '0x%08x' % x
REGEN = ("xenia-rs dis <xex|iso> --db sylpheed.db --analyze sql")
def _need(*tables):
"""Exit with a regeneration hint if any table is missing from this DB."""
@@ -65,17 +114,108 @@ def _fn(pc):
return f'{r[0][1]}({H(r[0][0])})' if r else '?'
def _import_names():
"""thunk head VA -> 'library::name' for every import, or {} on an old db."""
if not _has_col('instructions', 'import_address'):
return {}
return {a: (f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}')
for a, lib, nm, o in c.execute(
'SELECT address,library,name,ordinal FROM imports').fetchall()}
def cmd_dis(lo, hi):
data_col = 'is_data' if _has_col('instructions', 'is_data') else 'false'
rows = c.execute(f'SELECT address,mnemonic,operands,raw,{data_col} FROM instructions '
'WHERE address>=? AND address<? ORDER BY address', [lo, hi]).fetchall()
for a, m, o, raw, is_data in rows:
has_imp = _has_col('instructions', 'import_address')
imp_cols = 'import_address,import_role' if has_imp else 'NULL,NULL'
names = _import_names()
rows = c.execute(f'SELECT address,mnemonic,operands,raw,{data_col},{imp_cols} '
'FROM instructions WHERE address>=? AND address<? ORDER BY address',
[lo, hi]).fetchall()
for a, m, o, raw, is_data, imp_a, imp_role in rows:
who = names.get(imp_a, H(imp_a) if imp_a else None)
if is_data:
print(H(a), '.long', H(raw & 0xffffffff), ' ; jump-table data')
# is_data covers two different hazards; say which one this is.
why = f'import record -> {who}' if imp_role == 'record' else 'jump-table data'
print(H(a), '.long', H(raw & 0xffffffff), f' ; {why}')
elif imp_role == 'call':
print(H(a), m, o, f' ; -> IMPORT {who}')
elif imp_role == 'thunk':
print(H(a), m, o, f' ; import thunk {who}')
else:
print(H(a), m, o)
def cmd_gaps():
"""Dangling references the schema deliberately does NOT declare as FKs.
Each of these is a real gap in a detector, kept queryable rather than
hidden: declaring the FK would abort the build instead of reporting it.
"""
checks = [
("vtables RTTI names but M3 missed",
"SELECT count(*) FROM rtti_locators l WHERE l.vtable_address IS NOT NULL "
"AND NOT EXISTS (SELECT 1 FROM vtables v WHERE v.address=l.vtable_address)"),
("vtable slots -> undetected function",
"SELECT count(DISTINCT function_address) FROM methods m WHERE NOT EXISTS "
"(SELECT 1 FROM functions f WHERE f.address=m.function_address)"),
("funcptr-array slots -> undetected function",
"SELECT count(DISTINCT function_address) FROM function_pointer_array_entries e "
"WHERE NOT EXISTS (SELECT 1 FROM functions f WHERE f.address=e.function_address)"),
("dispatch candidates -> undetected function",
"SELECT count(DISTINCT method_address) FROM indirect_dispatch_candidates x "
"WHERE NOT EXISTS (SELECT 1 FROM functions f WHERE f.address=x.method_address)"),
("virtual sites with no candidates (truncated)",
"SELECT count(*) FROM indirect_dispatch_sites WHERE truncated"),
]
for label, q in checks:
try:
print(f"{c.execute(q).fetchone()[0]:>7} {label}")
except Exception as e:
print(f" ? {label} ({str(e).splitlines()[0][:50]})")
print("\n -- RTTI-named vftables the vtable scanner missed --")
try:
rows = c.execute(
"SELECT l.vtable_address, t.demangled_name FROM rtti_locators l "
"JOIN rtti_type_descriptors t ON t.address=l.type_descriptor "
"WHERE l.vtable_address IS NOT NULL AND NOT EXISTS "
"(SELECT 1 FROM vtables v WHERE v.address=l.vtable_address) "
"ORDER BY 2 LIMIT 15").fetchall()
for a, nm in rows:
print(f" {H(a)} {nm}")
except Exception as e:
print(" ", str(e).splitlines()[0][:70])
def cmd_imp(pat=None):
"""Import call sites, grouped by import. `pat` filters on library::name."""
_need('imports')
if not _has_col('instructions', 'import_address'):
sys.exit(f'this db predates instructions.import_address — regenerate with:\n {REGEN}')
q = ("SELECT im.library, im.name, im.ordinal, im.address, count(i.address) "
"FROM imports im LEFT JOIN instructions i "
" ON i.import_address = im.address AND i.import_role = 'call' "
"WHERE im.record_type = 1 GROUP BY 1,2,3,4 ORDER BY 5 DESC, 1, 2")
for lib, nm, o, addr, n in c.execute(q).fetchall():
label = f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}'
if pat and pat.lower() not in label.lower():
continue
print(f'{n:>5} calls {H(addr)} {label}')
def cmd_impcalls(pat):
"""Every call site of the imports matching `pat`."""
if not _has_col('instructions', 'import_address'):
sys.exit(f'this db predates instructions.import_address — regenerate with:\n {REGEN}')
q = ("SELECT i.address, i.mnemonic, im.library, im.name, im.ordinal "
"FROM instructions i JOIN imports im ON im.address = i.import_address "
"WHERE i.import_role = 'call' ORDER BY i.address")
for a, m, lib, nm, o in c.execute(q).fetchall():
label = f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}'
if pat.lower() not in label.lower():
continue
print(H(a), m, 'in', _fn(a), '->', label)
def cmd_switch(pc):
_need('jump_tables', 'jump_table_entries')
r = c.execute('SELECT bctr_pc,function,table_address,kind,entry_count FROM jump_tables '
@@ -194,6 +334,12 @@ def main():
for (a,) in c.execute("SELECT address FROM instructions WHERE mnemonic='lwz' AND operands=? "
'ORDER BY address', [pat]).fetchall():
print(H(a), 'in', _fn(a))
elif cmd == 'gaps':
cmd_gaps()
elif cmd == 'imp':
cmd_imp(args[0] if args else None)
elif cmd == 'impcalls':
cmd_impcalls(args[0])
elif cmd == 'grep':
for a, m, o in c.execute("SELECT address,mnemonic,operands FROM instructions "
"WHERE operands LIKE ? ORDER BY address", [f'%{args[0]}%']).fetchall():