[WIP] Audio codec probe/detection + CLI audio-info + viewer audio panel

Snapshot of local WIP before rebasing onto the movie/voice remote work.
- audio.rs: AudioCodec enum, AudioInfo::probe (RIFF/WAVE parse, raw-XMA2
  Shannon-entropy heuristic looks_like_raw_xma2), from_wav, riff_data_span.
- cli/main.rs: `audio-info` subcommand (cmd_audio_info).
- viewer ui.rs/iso_loader.rs: draw_audio_detail panel wiring.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MechaCat02
2026-07-20 20:42:48 +02:00
parent 3e1040b472
commit 8ed8c85f18
5 changed files with 496 additions and 62 deletions

View File

@@ -1,85 +1,405 @@
//! Audio format parsing — XMA and XWB handling.
//! Audio format parsing — WAV/PCM decode + Xbox 360 XMA/XMA2 recognition.
//!
//! Xbox 360 games use **XMA** (Xbox Media Audio) as their primary audio codec.
//! XMA is a proprietary Microsoft codec derived from WMA Pro.
//! ## What the game actually ships
//! Project Sylpheed's in-game audio lives in `dat/sound.pak` (IPFB): ~9500
//! entries of **raw, header-less XMA2** stream data — no per-entry `RIFF`/`fmt `
//! header, no compression wrapper (identical sound effects are byte-for-byte
//! duplicate entries). The per-stream format (channel count, sample rate, loop
//! points) is therefore **not** in the stream data; it lives in a separate
//! sound-bank descriptor that has not been reverse engineered yet. So this
//! module can *identify* those streams (and count their XMA packets) but cannot
//! yet decode them to PCM — that needs (a) the bank descriptor and (b) an XMA2
//! decoder.
//!
//! ## The Challenge
//! XMA decoding requires Microsoft's proprietary XMA decoder, which is only
//! available in the Windows DirectX runtime. There are two approaches:
//! ## What this module does today
//! - Fully parses **RIFF/WAVE PCM** (8/16/24-bit int, 32-bit float) → `GameAudio`
//! (interleaved `f32`), ready for playback/export of any converted audio.
//! - Reads metadata from **RIFF/WAVE XMA (`0x0165`) / XMA2 (`0x0166`)** headers
//! (channels, sample rate) — decode still unsupported.
//! - Recognizes **raw XMA2** stream blobs by entropy + packet alignment and
//! reports the 2048-byte packet count.
//!
//! ### Option A: Convert on extraction (recommended for Milestone 1)
//! Use `xma2encode.exe` (from Xbox 360 SDK) or `ffmpeg` (has partial XMA support)
//! to pre-convert all audio to OGG/WAV during the extraction step.
//!
//! ```bash
//! # Convert a single XMA file to WAV using ffmpeg
//! ffmpeg -i audio.xma output.wav
//!
//! # Or use VGMStream's test.exe for more accurate XMA decoding
//! ```
//!
//! ### Option B: XMA2 software decoder
//! The `xma2dec` project provides an open-source XMA2 decoder.
//! GitHub: https://github.com/koolkdev/xmalib (research-quality)
//!
//! ### XWB Wave Bank format
//! Xbox 360 games typically bundle audio into XWB (Xbox Wave Bank) files.
//! These are containers that hold multiple XMA streams.
//! Reference: https://github.com/microsoft/DirectXTK/tree/main/Audio
//! XMA framing constants are from xenia-canary `src/xenia/apu/xma_context.h`
//! (`kBytesPerPacket = 2048`, `kSamplesPerFrame = 512`, `kBytesPerSample = 2`).
use thiserror::Error;
// XMA framing (xenia-canary xma_context.h).
/// One XMA2 packet is 2048 bytes (4-byte header + 2044 bytes of frame data).
pub const XMA_BYTES_PER_PACKET: usize = 2048;
/// Decoded PCM samples produced per XMA frame, per channel.
pub const XMA_SAMPLES_PER_FRAME: u32 = 512;
// WAVE format tags.
const WAVE_FORMAT_PCM: u16 = 0x0001;
const WAVE_FORMAT_IEEE_FLOAT: u16 = 0x0003;
const WAVE_FORMAT_EXTENSIBLE: u16 = 0xFFFE;
const WAVE_FORMAT_XMA: u16 = 0x0165;
const WAVE_FORMAT_XMA2: u16 = 0x0166;
#[derive(Debug, Error)]
pub enum AudioError {
#[error("Unknown audio format magic: {0:?}")]
UnknownMagic([u8; 4]),
#[error("XMA decoding requires pre-conversion. See audio.rs for instructions.")]
XmaNotSupported,
#[error("Parse error: {0}")]
Parse(String),
#[error("not a recognized audio container")]
Unrecognized,
#[error("malformed audio: {0}")]
Malformed(&'static str),
#[error("codec needs a decoder this crate does not provide: {0:?}")]
NeedsDecoder(AudioCodec),
#[error("unsupported PCM sample format: tag {tag:#06x}, {bits} bits")]
UnsupportedPcm { tag: u16, bits: u16 },
}
/// An audio clip decoded to raw PCM, ready for Bevy's audio system.
/// Recognized audio container / codec.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AudioCodec {
/// RIFF/WAVE integer PCM (`WAVE_FORMAT_PCM`).
Pcm,
/// RIFF/WAVE IEEE-float PCM (`WAVE_FORMAT_IEEE_FLOAT`).
PcmFloat,
/// RIFF/WAVE XMA (`0x0165`).
Xma,
/// RIFF/WAVE XMA2 (`0x0166`).
Xma2,
/// Header-less XMA2 stream (the game's `sound.pak` entries), identified
/// heuristically — no embedded channel/rate metadata.
RawXma2,
Unknown,
}
impl AudioCodec {
/// Does decoding this codec require support this crate does not (yet) have?
pub fn needs_decoder(&self) -> bool {
matches!(self, Self::Xma | Self::Xma2 | Self::RawXma2)
}
pub fn label(&self) -> &'static str {
match self {
Self::Pcm => "PCM",
Self::PcmFloat => "PCM float",
Self::Xma => "XMA (RIFF)",
Self::Xma2 => "XMA2 (RIFF)",
Self::RawXma2 => "XMA2 (raw stream)",
Self::Unknown => "unknown",
}
}
}
/// Non-decoding metadata describing an audio blob.
#[derive(Debug, Clone)]
pub struct AudioInfo {
pub codec: AudioCodec,
pub channels: Option<u16>,
pub sample_rate: Option<u32>,
pub bits_per_sample: Option<u16>,
/// PCM samples per channel, when derivable (WAV only).
pub samples_per_channel: Option<u64>,
pub duration_secs: Option<f32>,
pub size_bytes: usize,
/// 2048-byte XMA packet count, for XMA/raw-XMA streams.
pub xma_packets: Option<u32>,
}
impl AudioInfo {
fn empty(codec: AudioCodec, size: usize) -> Self {
Self {
codec,
channels: None,
sample_rate: None,
bits_per_sample: None,
samples_per_channel: None,
duration_secs: None,
size_bytes: size,
xma_packets: None,
}
}
/// Best-effort classification of an audio blob. Never fails: unrecognized
/// input yields [`AudioCodec::Unknown`].
///
/// Order matters: a `RIFF/WAVE` header is authoritative; only header-less,
/// high-entropy, packet-sized blobs fall through to the raw-XMA2 heuristic.
pub fn probe(bytes: &[u8]) -> Self {
if let Some(info) = parse_riff_wave(bytes) {
return info;
}
if looks_like_raw_xma2(bytes) {
let mut info = Self::empty(AudioCodec::RawXma2, bytes.len());
info.xma_packets = Some(bytes.len().div_ceil(XMA_BYTES_PER_PACKET) as u32);
return info;
}
Self::empty(AudioCodec::Unknown, bytes.len())
}
}
/// True when `bytes` is likely a raw, header-less XMA2 stream: no known audio
/// magic, but a large, near-incompressible (high-entropy) payload — the shape
/// of the game's `sound.pak` audio entries. Heuristic: not conclusive, but the
/// probe runs only after every structured format has been ruled out.
pub fn looks_like_raw_xma2(bytes: &[u8]) -> bool {
// XMA streams are always at least a couple of packets long.
if bytes.len() < XMA_BYTES_PER_PACKET * 2 {
return false;
}
if bytes.starts_with(b"RIFF") || bytes.starts_with(b"XMA2") {
return false; // handled by the RIFF path
}
shannon_entropy_bits(&bytes[..bytes.len().min(64 * 1024)]) >= 7.8
}
/// Shannon entropy in bits/byte over `data` (0.0..=8.0). Compressed audio sits
/// very close to 8; structured/text data is much lower.
fn shannon_entropy_bits(data: &[u8]) -> f64 {
if data.is_empty() {
return 0.0;
}
let mut counts = [0u32; 256];
for &b in data {
counts[b as usize] += 1;
}
let n = data.len() as f64;
counts
.iter()
.filter(|&&c| c > 0)
.map(|&c| {
let p = c as f64 / n;
-p * p.log2()
})
.sum()
}
// ── RIFF/WAVE ──────────────────────────────────────────────────────────────────
/// Parse a RIFF/WAVE header for metadata. Returns `None` if `bytes` is not a
/// `RIFF....WAVE` container. All RIFF fields are little-endian.
fn parse_riff_wave(bytes: &[u8]) -> Option<AudioInfo> {
if bytes.len() < 12 || &bytes[..4] != b"RIFF" || &bytes[8..12] != b"WAVE" {
return None;
}
let le16 = |o: usize| u16::from_le_bytes([bytes[o], bytes[o + 1]]);
let le32 =
|o: usize| u32::from_le_bytes([bytes[o], bytes[o + 1], bytes[o + 2], bytes[o + 3]]);
let mut pos = 12;
let (mut tag, mut channels, mut rate, mut bits) = (0u16, 0u16, 0u32, 0u16);
let mut data_bytes: Option<u64> = None;
let mut have_fmt = false;
while pos + 8 <= bytes.len() {
let id = &bytes[pos..pos + 4];
let size = le32(pos + 4) as usize;
let body = pos + 8;
match id {
b"fmt " if body + 16 <= bytes.len() => {
tag = le16(body);
channels = le16(body + 2);
rate = le32(body + 4);
bits = le16(body + 14);
// WAVE_FORMAT_EXTENSIBLE stores the real tag in the GUID's first
// two bytes, right after cbSize (+2) → +24 from the fmt body.
if tag == WAVE_FORMAT_EXTENSIBLE && body + 26 <= bytes.len() {
tag = le16(body + 24);
}
have_fmt = true;
}
b"data" => data_bytes = Some(size as u64),
_ => {}
}
// Chunks are word-aligned (pad byte when size is odd).
pos = body + size + (size & 1);
}
if !have_fmt {
return None;
}
let codec = match tag {
WAVE_FORMAT_PCM => AudioCodec::Pcm,
WAVE_FORMAT_IEEE_FLOAT => AudioCodec::PcmFloat,
WAVE_FORMAT_XMA => AudioCodec::Xma,
WAVE_FORMAT_XMA2 => AudioCodec::Xma2,
_ => AudioCodec::Unknown,
};
let mut info = AudioInfo::empty(codec, bytes.len());
info.channels = Some(channels).filter(|&c| c > 0);
info.sample_rate = Some(rate).filter(|&r| r > 0);
info.bits_per_sample = Some(bits).filter(|&b| b > 0);
match codec {
AudioCodec::Pcm | AudioCodec::PcmFloat => {
if let (Some(d), true) = (data_bytes, channels > 0 && bits > 0) {
let frame = channels as u64 * (bits as u64 / 8);
if frame > 0 {
let spc = d / frame;
info.samples_per_channel = Some(spc);
if rate > 0 {
info.duration_secs = Some(spc as f32 / rate as f32);
}
}
}
}
AudioCodec::Xma | AudioCodec::Xma2 => {
if let Some(d) = data_bytes {
info.xma_packets = Some((d / XMA_BYTES_PER_PACKET as u64) as u32);
}
}
_ => {}
}
Some(info)
}
// ── PCM decode → GameAudio ─────────────────────────────────────────────────────
/// An audio clip decoded to raw interleaved `f32` PCM (`L R L R …`).
#[derive(Debug, Clone)]
pub struct GameAudio {
/// Raw PCM samples (interleaved for stereo: L R L R ...)
pub samples: Vec<f32>,
pub channels: u16,
pub sample_rate: u32,
}
impl GameAudio {
/// Parse audio from raw bytes.
///
/// Currently only WAV/PCM is supported. For XMA files, pre-convert using:
/// `ffmpeg -i file.xma file.wav`
pub fn from_bytes(bytes: &[u8]) -> Result<Self, AudioError> {
// Check for WAV magic
if bytes.len() >= 4 && &bytes[..4] == b"RIFF" {
return Self::from_wav(bytes);
/// Decode a RIFF/WAVE **PCM** file (int 8/16/24-bit or 32-bit float) to
/// interleaved `f32`. XMA/XMA2 return [`AudioError::NeedsDecoder`].
pub fn from_wav(bytes: &[u8]) -> Result<Self, AudioError> {
let info = parse_riff_wave(bytes).ok_or(AudioError::Unrecognized)?;
if info.codec.needs_decoder() {
return Err(AudioError::NeedsDecoder(info.codec));
}
let channels = info.channels.ok_or(AudioError::Malformed("no channels"))?;
let rate = info.sample_rate.ok_or(AudioError::Malformed("no sample rate"))?;
let bits = info.bits_per_sample.ok_or(AudioError::Malformed("no bit depth"))?;
// XMA magic bytes
if bytes.len() >= 4 && (
&bytes[..4] == b"XWB\0" || // Wave bank
&bytes[..4] == b"XMA2" // Raw XMA2
) {
return Err(AudioError::XmaNotSupported);
}
// Locate the `data` chunk body.
let (off, len) = riff_data_span(bytes).ok_or(AudioError::Malformed("no data chunk"))?;
let data = &bytes[off..off + len];
Err(AudioError::UnknownMagic(
bytes[..4.min(bytes.len())].try_into().unwrap_or([0u8; 4])
))
}
fn from_wav(bytes: &[u8]) -> Result<Self, AudioError> {
// Minimal WAV parser for PCM files
// A proper implementation should use the `hound` crate:
// https://crates.io/crates/hound
//
// TODO: Add `hound = "3"` to Cargo.toml and implement properly
Err(AudioError::Parse(
"WAV parsing TODO: add `hound` crate and implement".to_string()
))
let samples: Vec<f32> = match (info.codec, bits) {
(AudioCodec::Pcm, 8) => data.iter().map(|&b| (b as f32 - 128.0) / 128.0).collect(),
(AudioCodec::Pcm, 16) => data
.chunks_exact(2)
.map(|c| i16::from_le_bytes([c[0], c[1]]) as f32 / 32768.0)
.collect(),
(AudioCodec::Pcm, 24) => data
.chunks_exact(3)
.map(|c| {
let v = ((c[2] as i32) << 16) | ((c[1] as i32) << 8) | c[0] as i32;
let v = (v << 8) >> 8; // sign-extend 24→32
v as f32 / 8_388_608.0
})
.collect(),
(AudioCodec::PcmFloat, 32) => data
.chunks_exact(4)
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect(),
_ => return Err(AudioError::UnsupportedPcm { tag: 0, bits }),
};
Ok(Self { samples, channels, sample_rate: rate })
}
}
/// Byte span (offset, length) of the WAVE `data` chunk body, if present.
fn riff_data_span(bytes: &[u8]) -> Option<(usize, usize)> {
if bytes.len() < 12 || &bytes[..4] != b"RIFF" || &bytes[8..12] != b"WAVE" {
return None;
}
let le32 =
|o: usize| u32::from_le_bytes([bytes[o], bytes[o + 1], bytes[o + 2], bytes[o + 3]]) as usize;
let mut pos = 12;
while pos + 8 <= bytes.len() {
let size = le32(pos + 4);
let body = pos + 8;
if &bytes[pos..pos + 4] == b"data" {
let len = size.min(bytes.len().saturating_sub(body));
return Some((body, len));
}
pos = body + size + (size & 1);
}
None
}
#[cfg(test)]
mod tests {
use super::*;
/// Build a minimal 16-bit PCM WAV in memory (stereo, 2 frames).
fn tiny_wav() -> Vec<u8> {
let mut v = Vec::new();
let data: [i16; 4] = [1000, -1000, 32767, -32768]; // L R L R
let data_bytes: Vec<u8> = data.iter().flat_map(|s| s.to_le_bytes()).collect();
v.extend_from_slice(b"RIFF");
v.extend_from_slice(&(36 + data_bytes.len() as u32).to_le_bytes());
v.extend_from_slice(b"WAVE");
v.extend_from_slice(b"fmt ");
v.extend_from_slice(&16u32.to_le_bytes());
v.extend_from_slice(&WAVE_FORMAT_PCM.to_le_bytes());
v.extend_from_slice(&2u16.to_le_bytes()); // channels
v.extend_from_slice(&48000u32.to_le_bytes());
v.extend_from_slice(&(48000u32 * 2 * 2).to_le_bytes());
v.extend_from_slice(&4u16.to_le_bytes()); // block align
v.extend_from_slice(&16u16.to_le_bytes()); // bits
v.extend_from_slice(b"data");
v.extend_from_slice(&(data_bytes.len() as u32).to_le_bytes());
v.extend_from_slice(&data_bytes);
v
}
#[test]
fn probe_and_decode_pcm_wav() {
let wav = tiny_wav();
let info = AudioInfo::probe(&wav);
assert_eq!(info.codec, AudioCodec::Pcm);
assert_eq!(info.channels, Some(2));
assert_eq!(info.sample_rate, Some(48000));
assert_eq!(info.samples_per_channel, Some(2));
let audio = GameAudio::from_wav(&wav).unwrap();
assert_eq!(audio.channels, 2);
assert_eq!(audio.samples.len(), 4);
assert!((audio.samples[2] - 0.99997).abs() < 1e-3); // 32767/32768
}
#[test]
fn probe_xma2_riff_reports_metadata_not_decode() {
// Minimal RIFF/WAVE with an XMA2 fmt tag.
let mut v = Vec::new();
v.extend_from_slice(b"RIFF");
v.extend_from_slice(&200u32.to_le_bytes());
v.extend_from_slice(b"WAVE");
v.extend_from_slice(b"fmt ");
v.extend_from_slice(&16u32.to_le_bytes());
v.extend_from_slice(&WAVE_FORMAT_XMA2.to_le_bytes());
v.extend_from_slice(&2u16.to_le_bytes());
v.extend_from_slice(&44100u32.to_le_bytes());
v.extend_from_slice(&0u32.to_le_bytes());
v.extend_from_slice(&0u16.to_le_bytes());
v.extend_from_slice(&0u16.to_le_bytes());
let info = AudioInfo::probe(&v);
assert_eq!(info.codec, AudioCodec::Xma2);
assert_eq!(info.channels, Some(2));
assert!(info.codec.needs_decoder());
assert!(matches!(
GameAudio::from_wav(&v),
Err(AudioError::NeedsDecoder(AudioCodec::Xma2))
));
}
#[test]
fn raw_high_entropy_blob_reads_as_raw_xma2() {
// A pseudo-random 8 KB blob (no magic) → RawXma2 with a packet count.
let mut b = vec![0u8; 8192];
let mut x = 0x2545_F491u32;
for v in b.iter_mut() {
x ^= x << 13;
x ^= x >> 17;
x ^= x << 5;
*v = (x & 0xFF) as u8;
}
let info = AudioInfo::probe(&b);
assert_eq!(info.codec, AudioCodec::RawXma2);
assert_eq!(info.xma_packets, Some(4)); // 8192 / 2048
}
#[test]
fn structured_low_entropy_blob_is_unknown_not_audio() {
let b = b"IDXD............a bunch of readable ASCII text fields....".repeat(40);
assert_eq!(AudioInfo::probe(&b).codec, AudioCodec::Unknown);
}
}

View File

@@ -51,6 +51,7 @@ pub mod mesh;
pub mod audio;
/// Re-export the most commonly used types at the crate root.
pub use audio::{AudioCodec, AudioInfo, GameAudio};
pub use font::FontInfo;
pub use idxd::{IdxdError, IdxdObject};
pub use ixud::{Cue, Subtitle};