Merge pull request 'feat(xexdb): re-land #35 onto main — import-thunk naming, FKs, zq.py fixes, RE symbols' (#39) from feat/xexdb-import-naming into main
Reviewed-on: #39
This commit is contained in:
21
tools/apply_re_symbols.sql
Normal file
21
tools/apply_re_symbols.sql
Normal file
@@ -0,0 +1,21 @@
|
||||
-- Re-stamp reverse-engineered function names onto sylpheed.db.
|
||||
-- Source of truth: docs/re/RE_SYMBOLS.md. Re-run after any full DB regen
|
||||
-- (regen wipes functions.name back to sub_XXXX).
|
||||
-- python3 -c "import duckdb; duckdb.connect('sylpheed.db').execute(open('tools/apply_re_symbols.sql').read())"
|
||||
-- (from the repo root; `sylpheed.db` is where `tools/zq.py` looks by default)
|
||||
-- Addresses are decimal (DuckDB rejects 0x literals in some contexts); hex in comments.
|
||||
|
||||
UPDATE functions SET name = 'Pak_FindEntryByName' WHERE address = 2185628104; -- 0x824609C8
|
||||
UPDATE functions SET name = 'Pak_HashPathName' WHERE address = 2185627944; -- 0x82460928
|
||||
UPDATE functions SET name = 'Sylph_NameHash' WHERE address = 2185583736; -- 0x82455C78
|
||||
UPDATE functions SET name = 'Str_ToLowerAscii' WHERE address = 2187284368; -- 0x825F4F90
|
||||
UPDATE functions SET name = 'Pak_IsIPFBHeader' WHERE address = 2185627568; -- 0x824607B0
|
||||
UPDATE functions SET name = 'Archive_StreamReadCrc32' WHERE address = 2185594120; -- 0x82458508
|
||||
UPDATE functions SET name = 'Res3D_LoadMeshChunk' WHERE address = 2187592336; -- 0x82640290
|
||||
|
||||
-- IDXD reflective serialization framework (defaulted-field hunt, 2026-07-09)
|
||||
UPDATE functions SET name = 'IdxdLoad_Dispatch' WHERE address = 2185529024; -- 0x824486C0 (magic dispatch IDXD/IDX2/IDX3/IDXC/IXUD; 15 callers)
|
||||
UPDATE functions SET name = 'IdxdLoad_Variant2' WHERE address = 2185530624; -- 0x82448D00 (sibling variant loader)
|
||||
UPDATE functions SET name = 'Idxd_Parse' WHERE address = 2185532992; -- 0x82449640 (tokenize pool; per-field lookup+set)
|
||||
UPDATE functions SET name = 'Reflect_FindFieldIndex' WHERE address = 2185536240; -- 0x8244A2F0 (field-name -> index in name-vector @0x820B4F18)
|
||||
UPDATE functions SET name = 'Reflect_SetField' WHERE address = 2185536688; -- 0x8244A4B0 (apply value via object registry @obj+64)
|
||||
166
tools/zq.py
166
tools/zq.py
@@ -15,7 +15,7 @@ both static, neither needing anything to run:
|
||||
alongside a metadata JSON.
|
||||
|
||||
Usage:
|
||||
zq.py dis <lo_hex> <hi_hex> # disassemble [lo,hi) (jump-table words shown as .long)
|
||||
zq.py dis <lo_hex> <hi_hex> # disassemble [lo,hi) (data words as .long; imports named)
|
||||
zq.py fn <pc_hex> # function containing pc (address,name,end)
|
||||
zq.py xref <target_hex> # xrefs whose target == addr (callers)
|
||||
zq.py callers <vtable_off_dec> # call-sites of vtable slot at byte offset N
|
||||
@@ -29,20 +29,69 @@ Usage:
|
||||
zq.py class <name> # one class: bases, vtable, virtual methods
|
||||
zq.py str <substr> # string literals matching, with referencing functions
|
||||
|
||||
zq.py gaps # dangling refs = known detector gaps (see db.rs schema docs)
|
||||
zq.py imp [substr] # imports + how many call sites each has
|
||||
zq.py impcalls <substr> # every call site of the matching imports
|
||||
|
||||
zq.py xdbf [substr] # XDBF title text (all locales); substr filters
|
||||
zq.py ach # XDBF achievements (id, gamerscore, name, descriptions)
|
||||
|
||||
A command that needs a table the current DB predates prints what to regenerate
|
||||
rather than a SQL error.
|
||||
|
||||
The database is found via `$SYLPH_XEXDB`, else `<repo root>/sylpheed.db`. It is a
|
||||
build artefact, not a tracked file; if neither exists, zq.py says so and prints
|
||||
the command that builds one.
|
||||
"""
|
||||
import duckdb, sys
|
||||
import duckdb, sys, os, pathlib, signal
|
||||
|
||||
DB = '/home/fabi/RE - Project Sylpheed/xenia-rs/sylpheed.db'
|
||||
c = duckdb.connect(DB, read_only=True)
|
||||
# Output is routinely piped into `head`; without this Python prints a
|
||||
# BrokenPipeError traceback when the reader closes early.
|
||||
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
||||
|
||||
REGEN = "sylph-xexdb dis <xex|iso> --db sylpheed.db --analyze sql"
|
||||
|
||||
|
||||
def _resolve_db():
|
||||
"""Locate the database: `$SYLPH_XEXDB`, else `<repo root>/sylpheed.db`.
|
||||
|
||||
🔴 This used to be one hardcoded absolute path, and it pointed *inside*
|
||||
`xenia-rs` -- a repository `docs/agents/CONSOLIDATION.md` archives in Phase 5
|
||||
and drops in Phase 7. It resolved on exactly one machine and would have
|
||||
started failing there too, with `duckdb` raising about a missing file rather
|
||||
than anything saying why.
|
||||
|
||||
The database is a build artefact of several hundred MB and is not in the
|
||||
repository, so there is nothing to fall back *to*: if neither location has
|
||||
one, say so and print how to build it. A default that silently resolves to
|
||||
the wrong database is worse than no default -- see issue #16.
|
||||
"""
|
||||
env = os.environ.get('SYLPH_XEXDB')
|
||||
if env:
|
||||
p = pathlib.Path(env).expanduser()
|
||||
if not p.is_file():
|
||||
sys.exit(f'$SYLPH_XEXDB is set but is not a file:\n {p}')
|
||||
return p
|
||||
# Relative to this script, not to the caller's cwd: `zq.py` is run from
|
||||
# wherever the investigation happens to be.
|
||||
p = pathlib.Path(__file__).resolve().parent.parent / 'sylpheed.db'
|
||||
if p.is_file():
|
||||
return p
|
||||
sys.exit(
|
||||
f'no database found.\n'
|
||||
f' looked for: {p}\n'
|
||||
f' set $SYLPH_XEXDB to an existing one, or build it with:\n'
|
||||
f' {REGEN}'
|
||||
)
|
||||
|
||||
|
||||
# Resolved lazily enough that `zq.py` with no command still prints its usage on
|
||||
# a machine that has no database yet.
|
||||
_WANTS_DB = len(sys.argv) > 1 and sys.argv[1] not in ('-h', '--help', 'help')
|
||||
DB = _resolve_db() if _WANTS_DB else None
|
||||
c = duckdb.connect(str(DB), read_only=True) if DB is not None else None
|
||||
H = lambda x: '0x%08x' % x
|
||||
|
||||
REGEN = ("xenia-rs dis <xex|iso> --db sylpheed.db --analyze sql")
|
||||
|
||||
|
||||
def _need(*tables):
|
||||
"""Exit with a regeneration hint if any table is missing from this DB."""
|
||||
@@ -65,17 +114,108 @@ def _fn(pc):
|
||||
return f'{r[0][1]}({H(r[0][0])})' if r else '?'
|
||||
|
||||
|
||||
def _import_names():
|
||||
"""thunk head VA -> 'library::name' for every import, or {} on an old db."""
|
||||
if not _has_col('instructions', 'import_address'):
|
||||
return {}
|
||||
return {a: (f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}')
|
||||
for a, lib, nm, o in c.execute(
|
||||
'SELECT address,library,name,ordinal FROM imports').fetchall()}
|
||||
|
||||
|
||||
def cmd_dis(lo, hi):
|
||||
data_col = 'is_data' if _has_col('instructions', 'is_data') else 'false'
|
||||
rows = c.execute(f'SELECT address,mnemonic,operands,raw,{data_col} FROM instructions '
|
||||
'WHERE address>=? AND address<? ORDER BY address', [lo, hi]).fetchall()
|
||||
for a, m, o, raw, is_data in rows:
|
||||
has_imp = _has_col('instructions', 'import_address')
|
||||
imp_cols = 'import_address,import_role' if has_imp else 'NULL,NULL'
|
||||
names = _import_names()
|
||||
rows = c.execute(f'SELECT address,mnemonic,operands,raw,{data_col},{imp_cols} '
|
||||
'FROM instructions WHERE address>=? AND address<? ORDER BY address',
|
||||
[lo, hi]).fetchall()
|
||||
for a, m, o, raw, is_data, imp_a, imp_role in rows:
|
||||
who = names.get(imp_a, H(imp_a) if imp_a else None)
|
||||
if is_data:
|
||||
print(H(a), '.long', H(raw & 0xffffffff), ' ; jump-table data')
|
||||
# is_data covers two different hazards; say which one this is.
|
||||
why = f'import record -> {who}' if imp_role == 'record' else 'jump-table data'
|
||||
print(H(a), '.long', H(raw & 0xffffffff), f' ; {why}')
|
||||
elif imp_role == 'call':
|
||||
print(H(a), m, o, f' ; -> IMPORT {who}')
|
||||
elif imp_role == 'thunk':
|
||||
print(H(a), m, o, f' ; import thunk {who}')
|
||||
else:
|
||||
print(H(a), m, o)
|
||||
|
||||
|
||||
def cmd_gaps():
|
||||
"""Dangling references the schema deliberately does NOT declare as FKs.
|
||||
|
||||
Each of these is a real gap in a detector, kept queryable rather than
|
||||
hidden: declaring the FK would abort the build instead of reporting it.
|
||||
"""
|
||||
checks = [
|
||||
("vtables RTTI names but M3 missed",
|
||||
"SELECT count(*) FROM rtti_locators l WHERE l.vtable_address IS NOT NULL "
|
||||
"AND NOT EXISTS (SELECT 1 FROM vtables v WHERE v.address=l.vtable_address)"),
|
||||
("vtable slots -> undetected function",
|
||||
"SELECT count(DISTINCT function_address) FROM methods m WHERE NOT EXISTS "
|
||||
"(SELECT 1 FROM functions f WHERE f.address=m.function_address)"),
|
||||
("funcptr-array slots -> undetected function",
|
||||
"SELECT count(DISTINCT function_address) FROM function_pointer_array_entries e "
|
||||
"WHERE NOT EXISTS (SELECT 1 FROM functions f WHERE f.address=e.function_address)"),
|
||||
("dispatch candidates -> undetected function",
|
||||
"SELECT count(DISTINCT method_address) FROM indirect_dispatch_candidates x "
|
||||
"WHERE NOT EXISTS (SELECT 1 FROM functions f WHERE f.address=x.method_address)"),
|
||||
("virtual sites with no candidates (truncated)",
|
||||
"SELECT count(*) FROM indirect_dispatch_sites WHERE truncated"),
|
||||
]
|
||||
for label, q in checks:
|
||||
try:
|
||||
print(f"{c.execute(q).fetchone()[0]:>7} {label}")
|
||||
except Exception as e:
|
||||
print(f" ? {label} ({str(e).splitlines()[0][:50]})")
|
||||
print("\n -- RTTI-named vftables the vtable scanner missed --")
|
||||
try:
|
||||
rows = c.execute(
|
||||
"SELECT l.vtable_address, t.demangled_name FROM rtti_locators l "
|
||||
"JOIN rtti_type_descriptors t ON t.address=l.type_descriptor "
|
||||
"WHERE l.vtable_address IS NOT NULL AND NOT EXISTS "
|
||||
"(SELECT 1 FROM vtables v WHERE v.address=l.vtable_address) "
|
||||
"ORDER BY 2 LIMIT 15").fetchall()
|
||||
for a, nm in rows:
|
||||
print(f" {H(a)} {nm}")
|
||||
except Exception as e:
|
||||
print(" ", str(e).splitlines()[0][:70])
|
||||
|
||||
|
||||
def cmd_imp(pat=None):
|
||||
"""Import call sites, grouped by import. `pat` filters on library::name."""
|
||||
_need('imports')
|
||||
if not _has_col('instructions', 'import_address'):
|
||||
sys.exit(f'this db predates instructions.import_address — regenerate with:\n {REGEN}')
|
||||
q = ("SELECT im.library, im.name, im.ordinal, im.address, count(i.address) "
|
||||
"FROM imports im LEFT JOIN instructions i "
|
||||
" ON i.import_address = im.address AND i.import_role = 'call' "
|
||||
"WHERE im.record_type = 1 GROUP BY 1,2,3,4 ORDER BY 5 DESC, 1, 2")
|
||||
for lib, nm, o, addr, n in c.execute(q).fetchall():
|
||||
label = f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}'
|
||||
if pat and pat.lower() not in label.lower():
|
||||
continue
|
||||
print(f'{n:>5} calls {H(addr)} {label}')
|
||||
|
||||
|
||||
def cmd_impcalls(pat):
|
||||
"""Every call site of the imports matching `pat`."""
|
||||
if not _has_col('instructions', 'import_address'):
|
||||
sys.exit(f'this db predates instructions.import_address — regenerate with:\n {REGEN}')
|
||||
q = ("SELECT i.address, i.mnemonic, im.library, im.name, im.ordinal "
|
||||
"FROM instructions i JOIN imports im ON im.address = i.import_address "
|
||||
"WHERE i.import_role = 'call' ORDER BY i.address")
|
||||
for a, m, lib, nm, o in c.execute(q).fetchall():
|
||||
label = f'{lib}::{nm}' if nm else f'{lib}::ordinal_{o:#06x}'
|
||||
if pat.lower() not in label.lower():
|
||||
continue
|
||||
print(H(a), m, 'in', _fn(a), '->', label)
|
||||
|
||||
|
||||
def cmd_switch(pc):
|
||||
_need('jump_tables', 'jump_table_entries')
|
||||
r = c.execute('SELECT bctr_pc,function,table_address,kind,entry_count FROM jump_tables '
|
||||
@@ -194,6 +334,12 @@ def main():
|
||||
for (a,) in c.execute("SELECT address FROM instructions WHERE mnemonic='lwz' AND operands=? "
|
||||
'ORDER BY address', [pat]).fetchall():
|
||||
print(H(a), 'in', _fn(a))
|
||||
elif cmd == 'gaps':
|
||||
cmd_gaps()
|
||||
elif cmd == 'imp':
|
||||
cmd_imp(args[0] if args else None)
|
||||
elif cmd == 'impcalls':
|
||||
cmd_impcalls(args[0])
|
||||
elif cmd == 'grep':
|
||||
for a, m, o in c.execute("SELECT address,mnemonic,operands FROM instructions "
|
||||
"WHERE operands LIKE ? ORDER BY address", [f'%{args[0]}%']).fetchall():
|
||||
|
||||
Reference in New Issue
Block a user