docs(reference): adopt the last reference files from the project root
Three untracked files in the project root had no home in either repository: - the Xbox 360 technical reference report (a research compilation) becomes docs/reference/xbox360-re-technical-reference.md, unchanged; - XBOX360_ARCHITECTURE.md becomes docs/reference/xbox360-architecture.md, trimmed to its platform facts. Its format "status" sections (PAK unknown, mesh unknown, audio TODO) and the `just sniff` workflow predate every decoder in this repo and were wrong; - generate_export_docs.py becomes tools/generate_export_docs.py, the path the committed xbox360-exports.* already name as their generator. It lived inside a Canary checkout, so it now takes --canary and --out and fails loudly on a tree that is not Canary. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
sim
368
tools/generate_export_docs.py
Executable file
368
tools/generate_export_docs.py
Executable file
@@ -0,0 +1,368 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate documentation of all Xbox 360 system exports in Xenia Canary.
|
||||
|
||||
Parses the export table .inc files and implementation .cc files to produce
|
||||
a comprehensive listing of all exports with their implementation status.
|
||||
|
||||
Reads a Xenia Canary working tree and writes
|
||||
`docs/reference/xbox360-exports.{md,json}`. The committed copies were generated
|
||||
from an older Canary, so a fresh run changes some statuses.
|
||||
|
||||
Usage:
|
||||
python3 tools/generate_export_docs.py [--canary ../xenia-canary] [--out docs/reference]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
from collections import OrderedDict
|
||||
|
||||
SYLPHEED_ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||||
CANARY_DEFAULT = SYLPHEED_ROOT.parent / "xenia-canary"
|
||||
|
||||
# --- Configuration ---
|
||||
|
||||
# Set by configure() from --canary, before anything reads them.
|
||||
REPO_ROOT = CANARY_DEFAULT
|
||||
TABLE_FILES = {}
|
||||
IMPL_DIRS = {}
|
||||
|
||||
|
||||
def configure(canary_root):
|
||||
"""Point the table and implementation paths at a Canary checkout."""
|
||||
global REPO_ROOT, TABLE_FILES, IMPL_DIRS
|
||||
REPO_ROOT = canary_root
|
||||
TABLE_FILES = {
|
||||
"xboxkrnl": REPO_ROOT / "src/xenia/kernel/xboxkrnl/xboxkrnl_table.inc",
|
||||
"xam": REPO_ROOT / "src/xenia/kernel/xam/xam_table.inc",
|
||||
"xbdm": REPO_ROOT / "src/xenia/kernel/xbdm/xbdm_table.inc",
|
||||
}
|
||||
IMPL_DIRS = {
|
||||
"xboxkrnl": REPO_ROOT / "src/xenia/kernel/xboxkrnl",
|
||||
"xam": REPO_ROOT / "src/xenia/kernel/xam",
|
||||
"xbdm": REPO_ROOT / "src/xenia/kernel/xbdm",
|
||||
}
|
||||
missing = [str(p) for p in TABLE_FILES.values() if not p.is_file()]
|
||||
if missing:
|
||||
raise SystemExit(
|
||||
f"not a Xenia Canary checkout: {canary_root} "
|
||||
f"(missing {', '.join(missing)}); pass --canary"
|
||||
)
|
||||
|
||||
MODULE_NAMES = {
|
||||
"xboxkrnl": "xboxkrnl.exe",
|
||||
"xam": "xam.xex",
|
||||
"xbdm": "xbdm.xex",
|
||||
}
|
||||
|
||||
MODULE_MAP = {"XBOXKRNL": "xboxkrnl", "XAM": "xam", "XBDM": "xbdm"}
|
||||
|
||||
# Regex patterns
|
||||
XE_EXPORT_RE = re.compile(
|
||||
r"XE_EXPORT\(\s*(\w+)\s*,\s*(0x[0-9A-Fa-f]+)\s*,\s*(\w+)\s*,\s*k(\w+)\s*\)"
|
||||
)
|
||||
|
||||
DECLARE_RE = re.compile(
|
||||
r"DECLARE_(XBOXKRNL|XAM|XBDM)_EXPORT\d*\(\s*(\w+)\s*,"
|
||||
r"\s*k(\w+)\s*,\s*(.+?)\)\s*;",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
TAG_RE = re.compile(r"k(\w+)")
|
||||
|
||||
SHIM_RE = re.compile(r'SHIM_SET_MAPPING\(\s*"[^"]+"\s*,\s*(\w+)\s*,')
|
||||
|
||||
VAR_MAPPING_RE = re.compile(r"ordinals::(\w+)")
|
||||
EXPORT_KVAR_RE = re.compile(r"EXPORT_KVAR\((\w+)\)")
|
||||
|
||||
|
||||
def parse_table_files():
|
||||
"""Phase A: Parse all XE_EXPORT entries from *_table.inc files."""
|
||||
exports = OrderedDict()
|
||||
|
||||
for module, path in TABLE_FILES.items():
|
||||
content = path.read_text()
|
||||
for match in XE_EXPORT_RE.finditer(content):
|
||||
mod, ordinal_hex, name, typ = match.groups()
|
||||
exports[(mod, name)] = {
|
||||
"module": mod,
|
||||
"ordinal": int(ordinal_hex, 16),
|
||||
"ordinal_hex": ordinal_hex.upper().replace("0X", "0x"),
|
||||
"name": name,
|
||||
"type": "variable" if typ == "Variable" else "function",
|
||||
"status": "not_implemented",
|
||||
"tags": [],
|
||||
"category": "",
|
||||
"source_file": "",
|
||||
}
|
||||
|
||||
return exports
|
||||
|
||||
|
||||
def scan_declare_macros(exports):
|
||||
"""Phase B: Scan DECLARE_*_EXPORT macros in .cc files."""
|
||||
for mod_key, dir_path in IMPL_DIRS.items():
|
||||
for cc_file in sorted(dir_path.glob("*.cc")):
|
||||
content = cc_file.read_text()
|
||||
for match in DECLARE_RE.finditer(content):
|
||||
mod_prefix, name, category, tags_str = match.groups()
|
||||
module = MODULE_MAP[mod_prefix]
|
||||
tags = TAG_RE.findall(tags_str)
|
||||
key = (module, name)
|
||||
|
||||
if key in exports:
|
||||
entry = exports[key]
|
||||
entry["category"] = category
|
||||
entry["tags"] = tags
|
||||
entry["source_file"] = str(
|
||||
cc_file.relative_to(REPO_ROOT)
|
||||
)
|
||||
|
||||
if "Implemented" in tags:
|
||||
entry["status"] = "implemented"
|
||||
elif "Stub" in tags:
|
||||
entry["status"] = "stub"
|
||||
elif "Sketchy" in tags:
|
||||
entry["status"] = "sketchy"
|
||||
else:
|
||||
entry["status"] = "stub"
|
||||
|
||||
|
||||
def scan_legacy_patterns(exports):
|
||||
"""Phase C: Handle SHIM_SET_MAPPING and SetVariableMapping."""
|
||||
# SHIM_SET_MAPPING in xboxkrnl_strings.cc
|
||||
strings_file = IMPL_DIRS["xboxkrnl"] / "xboxkrnl_strings.cc"
|
||||
if strings_file.exists():
|
||||
content = strings_file.read_text()
|
||||
for match in SHIM_RE.finditer(content):
|
||||
name = match.group(1)
|
||||
key = ("xboxkrnl", name)
|
||||
if key in exports:
|
||||
exports[key]["status"] = "implemented"
|
||||
exports[key]["source_file"] = str(
|
||||
strings_file.relative_to(REPO_ROOT)
|
||||
)
|
||||
if not exports[key]["tags"]:
|
||||
exports[key]["tags"] = ["Implemented"]
|
||||
|
||||
# SetVariableMapping / EXPORT_KVAR in xboxkrnl_module.cc
|
||||
module_file = IMPL_DIRS["xboxkrnl"] / "xboxkrnl_module.cc"
|
||||
if module_file.exists():
|
||||
content = module_file.read_text()
|
||||
for match in VAR_MAPPING_RE.finditer(content):
|
||||
name = match.group(1)
|
||||
key = ("xboxkrnl", name)
|
||||
if key in exports:
|
||||
exports[key]["status"] = "implemented"
|
||||
exports[key]["source_file"] = str(
|
||||
module_file.relative_to(REPO_ROOT)
|
||||
)
|
||||
if not exports[key]["tags"]:
|
||||
exports[key]["tags"] = ["Implemented"]
|
||||
|
||||
for match in EXPORT_KVAR_RE.finditer(content):
|
||||
name = match.group(1)
|
||||
key = ("xboxkrnl", name)
|
||||
if key in exports:
|
||||
exports[key]["status"] = "implemented"
|
||||
exports[key]["source_file"] = str(
|
||||
module_file.relative_to(REPO_ROOT)
|
||||
)
|
||||
if not exports[key]["tags"]:
|
||||
exports[key]["tags"] = ["Implemented"]
|
||||
|
||||
|
||||
def compute_statistics(exports):
|
||||
"""Compute per-module statistics."""
|
||||
stats = {}
|
||||
for module in TABLE_FILES:
|
||||
module_exports = [e for e in exports.values() if e["module"] == module]
|
||||
stats[module] = {
|
||||
"total": len(module_exports),
|
||||
"implemented": sum(
|
||||
1 for e in module_exports if e["status"] == "implemented"
|
||||
),
|
||||
"stub": sum(1 for e in module_exports if e["status"] == "stub"),
|
||||
"sketchy": sum(
|
||||
1 for e in module_exports if e["status"] == "sketchy"
|
||||
),
|
||||
"not_implemented": sum(
|
||||
1
|
||||
for e in module_exports
|
||||
if e["status"] == "not_implemented"
|
||||
),
|
||||
"functions": sum(
|
||||
1 for e in module_exports if e["type"] == "function"
|
||||
),
|
||||
"variables": sum(
|
||||
1 for e in module_exports if e["type"] == "variable"
|
||||
),
|
||||
}
|
||||
return stats
|
||||
|
||||
|
||||
def generate_markdown(exports, stats):
|
||||
"""Generate the Markdown documentation."""
|
||||
lines = []
|
||||
lines.append("# Xbox 360 System Exports - Xenia Canary")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"Auto-generated by `tools/generate_export_docs.py`. "
|
||||
"Do not edit manually."
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# Overall summary
|
||||
lines.append("## Summary")
|
||||
lines.append("")
|
||||
total_all = sum(s["total"] for s in stats.values())
|
||||
impl_all = sum(s["implemented"] for s in stats.values())
|
||||
stub_all = sum(s["stub"] for s in stats.values())
|
||||
sketchy_all = sum(s["sketchy"] for s in stats.values())
|
||||
not_impl_all = sum(s["not_implemented"] for s in stats.values())
|
||||
|
||||
lines.append(
|
||||
f"**Total exports: {total_all}** | "
|
||||
f"Implemented: {impl_all} | "
|
||||
f"Stub: {stub_all} | "
|
||||
f"Sketchy: {sketchy_all} | "
|
||||
f"Not implemented: {not_impl_all}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
lines.append(
|
||||
"| Module | File | Total | Implemented | Stub | Sketchy | Not Implemented | Functions | Variables |"
|
||||
)
|
||||
lines.append(
|
||||
"|--------|------|------:|------------:|-----:|--------:|----------------:|----------:|----------:|"
|
||||
)
|
||||
for module in TABLE_FILES:
|
||||
s = stats[module]
|
||||
lines.append(
|
||||
f"| {module} | {MODULE_NAMES[module]} | {s['total']} | "
|
||||
f"{s['implemented']} | {s['stub']} | {s['sketchy']} | "
|
||||
f"{s['not_implemented']} | {s['functions']} | {s['variables']} |"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# Per-module tables
|
||||
for module in TABLE_FILES:
|
||||
s = stats[module]
|
||||
lines.append(f"## {module} ({MODULE_NAMES[module]})")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
f"Total: {s['total']} | "
|
||||
f"Implemented: {s['implemented']} | "
|
||||
f"Stub: {s['stub']} | "
|
||||
f"Sketchy: {s['sketchy']} | "
|
||||
f"Not implemented: {s['not_implemented']}"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
lines.append(
|
||||
"| Ordinal | Name | Type | Status | Category | Source |"
|
||||
)
|
||||
lines.append(
|
||||
"|--------:|------|------|--------|----------|--------|"
|
||||
)
|
||||
|
||||
module_exports = sorted(
|
||||
[e for e in exports.values() if e["module"] == module],
|
||||
key=lambda e: e["ordinal"],
|
||||
)
|
||||
|
||||
for e in module_exports:
|
||||
ordinal = f"0x{e['ordinal']:03X}"
|
||||
status_icon = {
|
||||
"implemented": "implemented",
|
||||
"stub": "stub",
|
||||
"sketchy": "sketchy",
|
||||
"not_implemented": "-",
|
||||
}.get(e["status"], e["status"])
|
||||
|
||||
source = e["source_file"]
|
||||
if source:
|
||||
# Show just the filename for brevity
|
||||
source = pathlib.Path(source).name
|
||||
|
||||
lines.append(
|
||||
f"| {ordinal} | {e['name']} | {e['type']} | "
|
||||
f"{status_icon} | {e['category']} | {source} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_json(exports, stats):
|
||||
"""Generate the JSON documentation."""
|
||||
data = {
|
||||
"description": "Xbox 360 system exports implemented in Xenia Canary",
|
||||
"generator": "tools/generate_export_docs.py",
|
||||
"statistics": stats,
|
||||
"modules": {},
|
||||
}
|
||||
|
||||
for module in TABLE_FILES:
|
||||
module_exports = sorted(
|
||||
[e for e in exports.values() if e["module"] == module],
|
||||
key=lambda e: e["ordinal"],
|
||||
)
|
||||
data["modules"][module] = {
|
||||
"file": MODULE_NAMES[module],
|
||||
"exports": module_exports,
|
||||
}
|
||||
|
||||
return json.dumps(data, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--canary", type=pathlib.Path, default=CANARY_DEFAULT,
|
||||
help="Xenia Canary working tree (default: %(default)s)")
|
||||
parser.add_argument("--out", type=pathlib.Path,
|
||||
default=SYLPHEED_ROOT / "docs" / "reference",
|
||||
help="output directory (default: %(default)s)")
|
||||
args = parser.parse_args()
|
||||
configure(args.canary.resolve())
|
||||
|
||||
print("Parsing export tables...")
|
||||
exports = parse_table_files()
|
||||
print(f" Found {len(exports)} exports across {len(TABLE_FILES)} modules")
|
||||
|
||||
print("Scanning DECLARE_*_EXPORT macros...")
|
||||
scan_declare_macros(exports)
|
||||
|
||||
print("Scanning legacy registration patterns...")
|
||||
scan_legacy_patterns(exports)
|
||||
|
||||
stats = compute_statistics(exports)
|
||||
|
||||
for module, s in stats.items():
|
||||
print(
|
||||
f" {module}: {s['total']} total, "
|
||||
f"{s['implemented']} implemented, "
|
||||
f"{s['stub']} stub, "
|
||||
f"{s['sketchy']} sketchy, "
|
||||
f"{s['not_implemented']} not implemented"
|
||||
)
|
||||
|
||||
# Generate outputs
|
||||
docs_dir = args.out
|
||||
docs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
md_path = docs_dir / "xbox360-exports.md"
|
||||
md_content = generate_markdown(exports, stats)
|
||||
md_path.write_text(md_content)
|
||||
print(f"Written: {md_path}")
|
||||
|
||||
json_path = docs_dir / "xbox360-exports.json"
|
||||
json_content = generate_json(exports, stats)
|
||||
json_path.write_text(json_content)
|
||||
print(f"Written: {json_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user