480 lines
19 KiB
Rust
480 lines
19 KiB
Rust
//! String / constant-pool detection in the initialised data sections.
|
|
//!
|
|
//! Scans the `.rdata` section for runs of printable ASCII or null-terminated
|
|
//! UTF-16LE characters of length ≥ 6, emitting one row per discovered string.
|
|
//! Cross-references against `xrefs.target` are computed by the caller —
|
|
//! this module only finds the strings; downstream queries can join.
|
|
//!
|
|
//! ### What this layer does NOT do
|
|
//!
|
|
//! - No UTF-8 multibyte detection — Xbox 360 game binaries reliably use
|
|
//! ASCII for debug strings and UTF-16LE for localised text.
|
|
//! - Only the file-backed part of a section is scanned: the tail of `.data`
|
|
//! past `raw_size` is BSS and contains nothing but zeros at rest.
|
|
//! - Wide strings on Xbox 360 are little-endian (compiler convention even
|
|
//! on this big-endian platform); we do NOT try big-endian UTF-16.
|
|
//! - No language detection / classification beyond encoding.
|
|
//!
|
|
//! Extends the original ASCII / UTF-16LE pass with Shift_JIS detection
|
|
//! (Sylpheed is originally Japanese — likely yields mission/UI text
|
|
//! invisible to ASCII-only) and UTF-8 multi-byte detection.
|
|
//!
|
|
//! Reference: `objdump -s` `.rdata` walks rely on the same heuristic;
|
|
//! Shift_JIS lead/trail byte ranges per JIS X 0208.
|
|
|
|
use sylpheed_xex::pe::PeSection;
|
|
|
|
/// One detected string.
|
|
#[derive(Debug, Clone)]
|
|
pub struct DetectedString {
|
|
/// Absolute VA of the first byte.
|
|
pub address: u32,
|
|
/// `"ascii"` | `"utf16le"` | `"shift_jis"` | `"utf8"`.
|
|
pub encoding: &'static str,
|
|
/// Length in bytes (excluding the NUL terminator).
|
|
pub length: u32,
|
|
/// UTF-8 representation of the string content.
|
|
pub content: String,
|
|
/// Name of the PE section the string lives in (`.rdata` / `.data`).
|
|
pub section: String,
|
|
}
|
|
|
|
/// Scan the initialised data sections for ASCII / UTF-16LE / Shift_JIS / UTF-8
|
|
/// strings.
|
|
///
|
|
/// `.data` is scanned as well as `.rdata`: a lot of a game's string material —
|
|
/// mutable tables, and every RTTI type-descriptor name — lives there, and
|
|
/// leaving it out is why this table comes back nearly empty on real titles.
|
|
/// The `section` column lets a consumer separate the two again.
|
|
#[tracing::instrument(skip_all, fields(image_base = format_args!("{:#010x}", image_base)))]
|
|
pub fn analyze(pe: &[u8], image_base: u32, sections: &[PeSection]) -> Vec<DetectedString> {
|
|
let started = std::time::Instant::now();
|
|
let mut out: Vec<DetectedString> = Vec::new();
|
|
|
|
for section in sections {
|
|
if !matches!(section.name.as_str(), ".rdata" | ".data") { continue; }
|
|
let raw_start = section.virtual_address as usize;
|
|
// Clamp to the file-backed extent — everything past `raw_size` is BSS.
|
|
let backed = section.virtual_size.min(section.raw_size) as usize;
|
|
let raw_end = (raw_start + backed).min(pe.len());
|
|
if raw_start >= raw_end { continue; }
|
|
let bytes = &pe[raw_start..raw_end];
|
|
let va_base = image_base + section.virtual_address;
|
|
|
|
let before = out.len();
|
|
scan_ascii(bytes, va_base, &mut out);
|
|
scan_utf16le(bytes, va_base, &mut out);
|
|
scan_shift_jis(bytes, va_base, &mut out);
|
|
scan_utf8(bytes, va_base, &mut out);
|
|
for s in &mut out[before..] {
|
|
s.section = section.name.clone();
|
|
}
|
|
}
|
|
|
|
let elapsed_ms = started.elapsed().as_millis() as f64;
|
|
let n_ascii = out.iter().filter(|s| s.encoding == "ascii").count();
|
|
let n_utf16 = out.iter().filter(|s| s.encoding == "utf16le").count();
|
|
let n_sjis = out.iter().filter(|s| s.encoding == "shift_jis").count();
|
|
let n_utf8 = out.iter().filter(|s| s.encoding == "utf8").count();
|
|
metrics::histogram!("analysis.phase_ms", "phase" => "strings").record(elapsed_ms);
|
|
tracing::info!(
|
|
ascii = n_ascii,
|
|
utf16le = n_utf16,
|
|
shift_jis = n_sjis,
|
|
utf8 = n_utf8,
|
|
total = out.len(),
|
|
elapsed_ms,
|
|
"string scan complete"
|
|
);
|
|
out
|
|
}
|
|
|
|
const MIN_LEN: usize = 6;
|
|
|
|
fn is_printable_ascii(b: u8) -> bool {
|
|
// Printable + the common whitespace characters used in real strings.
|
|
matches!(b, 0x20..=0x7E | b'\t' | b'\n' | b'\r')
|
|
}
|
|
|
|
fn scan_ascii(bytes: &[u8], va_base: u32, out: &mut Vec<DetectedString>) {
|
|
let mut i = 0;
|
|
while i < bytes.len() {
|
|
if !is_printable_ascii(bytes[i]) {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
let start = i;
|
|
while i < bytes.len() && is_printable_ascii(bytes[i]) { i += 1; }
|
|
let run_len = i - start;
|
|
// Require NUL termination and minimum length.
|
|
if run_len >= MIN_LEN && i < bytes.len() && bytes[i] == 0 {
|
|
let s = std::str::from_utf8(&bytes[start..i]).unwrap_or("");
|
|
out.push(DetectedString {
|
|
address: va_base + start as u32,
|
|
encoding: "ascii",
|
|
length: run_len as u32,
|
|
content: s.to_string(),
|
|
section: String::new(),
|
|
});
|
|
}
|
|
// Skip the NUL (if any) before continuing.
|
|
if i < bytes.len() && bytes[i] == 0 { i += 1; }
|
|
}
|
|
}
|
|
|
|
fn scan_utf16le(bytes: &[u8], va_base: u32, out: &mut Vec<DetectedString>) {
|
|
// UTF-16LE strings are 2-byte aligned in MSVC output. Walk on even
|
|
// offsets to avoid misaligned hits.
|
|
let mut i = 0;
|
|
while i + 2 <= bytes.len() {
|
|
if !i.is_multiple_of(2) { i += 1; continue; }
|
|
let lo = bytes[i];
|
|
let hi = bytes[i + 1];
|
|
// Restrict scan-start to printable ASCII range with a zero high byte —
|
|
// this is what real Xbox 360 wide strings look like.
|
|
if hi != 0 || !is_printable_ascii(lo) {
|
|
i += 2;
|
|
continue;
|
|
}
|
|
let start = i;
|
|
let mut codeunits: Vec<u16> = Vec::new();
|
|
while i + 2 <= bytes.len() {
|
|
let l = bytes[i];
|
|
let h = bytes[i + 1];
|
|
if h != 0 || !is_printable_ascii(l) { break; }
|
|
codeunits.push((h as u16) << 8 | l as u16);
|
|
i += 2;
|
|
}
|
|
// Require NUL u16 terminator.
|
|
let nul_terminated = i + 2 <= bytes.len() && bytes[i] == 0 && bytes[i + 1] == 0;
|
|
if codeunits.len() >= MIN_LEN && nul_terminated {
|
|
let s: String = String::from_utf16_lossy(&codeunits);
|
|
out.push(DetectedString {
|
|
address: va_base + start as u32,
|
|
encoding: "utf16le",
|
|
length: ((i - start) as u32),
|
|
content: s,
|
|
section: String::new(),
|
|
});
|
|
}
|
|
// Skip past the terminator.
|
|
if nul_terminated { i += 2; }
|
|
}
|
|
}
|
|
|
|
/// Per JIS X 0208: Shift_JIS lead byte is [0x81, 0x9F] u [0xE0, 0xEF];
|
|
/// trail byte is [0x40, 0x7E] u [0x80, 0xFC].
|
|
///
|
|
/// Half-width katakana (0xA1..=0xDF) is deliberately *not* accepted as string
|
|
/// content. It is legal Shift_JIS, but this binary's Japanese text never uses
|
|
/// it, while 0xA1..=0xDF is extremely common in the float and pointer tables
|
|
/// that share `.rdata` — admitting it turned the scan into a noise generator
|
|
/// (837 detections, of which the overwhelming majority were IEEE-754 arrays:
|
|
/// `3f 66 66 66` = 0.9f reads as "fff").
|
|
fn is_sjis_lead(b: u8) -> bool {
|
|
(0x81..=0x9F).contains(&b) || (0xE0..=0xEF).contains(&b)
|
|
}
|
|
fn is_sjis_trail(b: u8) -> bool {
|
|
(0x40..=0x7E).contains(&b) || (0x80..=0xFC).contains(&b)
|
|
}
|
|
|
|
/// A character that can plausibly appear in a Japanese debug/UI string:
|
|
/// printable ASCII, CJK punctuation and kana, CJK ideographs, or full-width
|
|
/// ASCII.
|
|
fn is_text_like(ch: char) -> bool {
|
|
let o = ch as u32;
|
|
matches!(o, 0x20..=0x7E)
|
|
|| matches!(ch, '\t' | '\n' | '\r')
|
|
|| is_wide(ch)
|
|
}
|
|
|
|
/// A full-width character — kana, CJK punctuation, ideograph, or full-width
|
|
/// ASCII. Used to tell "real text" from a lucky byte pair.
|
|
fn is_wide(ch: char) -> bool {
|
|
let o = ch as u32;
|
|
(0x3000..=0x30FF).contains(&o) || (0x4E00..=0x9FFF).contains(&o) || (0xFF01..=0xFF5E).contains(&o)
|
|
}
|
|
|
|
/// True when `t` contains a lone ASCII character with a full-width character
|
|
/// on *both* sides.
|
|
///
|
|
/// This is the Shift_JIS resynchronisation signal. A scan that starts one byte
|
|
/// early pairs the wrong lead with the wrong trail and typically produces a
|
|
/// stray kanji plus an orphaned ASCII letter before the real text resumes:
|
|
/// the run at 0x820a4b9f decodes as `帥Vステムマネージャ開始` when the actual
|
|
/// string is `システムマネージャ開始` at 0x820a4ba0. Genuine text mixes ASCII in
|
|
/// *runs* (`render_stateスタックオーバーフロー`, `size=%d`), never as a single
|
|
/// character wedged between two wide ones.
|
|
fn has_isolated_ascii(t: &str) -> bool {
|
|
let chars: Vec<char> = t.chars().collect();
|
|
(1..chars.len().saturating_sub(1)).any(|k| {
|
|
!is_wide(chars[k]) && is_wide(chars[k - 1]) && is_wide(chars[k + 1])
|
|
})
|
|
}
|
|
|
|
/// Decode `raw` as Shift_JIS, rejecting anything that is not convincingly
|
|
/// Japanese text. Returns the UTF-8 form on success.
|
|
fn decode_sjis(raw: &[u8]) -> Option<String> {
|
|
let (text, _, had_errors) = encoding_rs::SHIFT_JIS.decode(raw);
|
|
if had_errors {
|
|
return None;
|
|
}
|
|
let t = text.into_owned();
|
|
// Require real kana somewhere. Arbitrary binary readily decodes to
|
|
// obscure kanji, but hiragana/katakana (U+3040..U+30FF) essentially never
|
|
// appear by accident and are ubiquitous in genuine Japanese.
|
|
let has_kana = t.chars().any(|c| ('\u{3040}'..='\u{30FF}').contains(&c));
|
|
if t.chars().count() >= 4 && has_kana && t.chars().all(is_text_like) && !has_isolated_ascii(&t) {
|
|
Some(t)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
/// Scan for Shift_JIS strings — NUL-terminated runs of >= `MIN_LEN` bytes made
|
|
/// of printable ASCII and valid lead+trail pairs, with at least one pair.
|
|
///
|
|
/// Each accepted run is *resynchronised*: the emitted string starts at the
|
|
/// earliest offset within the run whose full decode passes [`decode_sjis`], so
|
|
/// a run that begins mid-character reports the true string address rather than
|
|
/// a mangled one.
|
|
fn scan_shift_jis(bytes: &[u8], va_base: u32, out: &mut Vec<DetectedString>) {
|
|
let mut i = 0;
|
|
while i < bytes.len() {
|
|
let start = i;
|
|
let mut has_multibyte = false;
|
|
let mut nbytes = 0;
|
|
while i < bytes.len() {
|
|
let b = bytes[i];
|
|
if is_sjis_lead(b) && i + 1 < bytes.len() && is_sjis_trail(bytes[i + 1]) {
|
|
has_multibyte = true;
|
|
nbytes += 2;
|
|
i += 2;
|
|
} else if is_printable_ascii(b) {
|
|
nbytes += 1;
|
|
i += 1;
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
let end = i;
|
|
if has_multibyte && nbytes >= MIN_LEN && end < bytes.len() && bytes[end] == 0 {
|
|
for s in start..end {
|
|
if let Some(text) = decode_sjis(&bytes[s..end]) {
|
|
out.push(DetectedString {
|
|
address: va_base + s as u32,
|
|
encoding: "shift_jis",
|
|
length: (end - s) as u32,
|
|
content: text,
|
|
section: String::new(),
|
|
});
|
|
break;
|
|
}
|
|
}
|
|
i = end + 1; // skip NUL
|
|
} else {
|
|
i = start + 1;
|
|
if i < bytes.len() && bytes[i] == 0 { i += 1; }
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Scan for UTF-8 strings carrying multi-byte sequences (we already
|
|
/// catch pure-ASCII via `scan_ascii`). Validates 2/3-byte sequences;
|
|
/// 4-byte (supplementary plane) is uncommon in game text and skipped.
|
|
fn scan_utf8(bytes: &[u8], va_base: u32, out: &mut Vec<DetectedString>) {
|
|
let mut i = 0;
|
|
while i < bytes.len() {
|
|
let start = i;
|
|
let mut has_multibyte = false;
|
|
let mut nbytes = 0;
|
|
while i < bytes.len() {
|
|
let b = bytes[i];
|
|
if b < 0x80 {
|
|
if !is_printable_ascii(b) { break; }
|
|
nbytes += 1;
|
|
i += 1;
|
|
} else if (b & 0xE0) == 0xC0 {
|
|
// 2-byte: 110xxxxx 10xxxxxx
|
|
if i + 1 >= bytes.len() || (bytes[i + 1] & 0xC0) != 0x80 { break; }
|
|
has_multibyte = true;
|
|
nbytes += 2;
|
|
i += 2;
|
|
} else if (b & 0xF0) == 0xE0 {
|
|
// 3-byte: 1110xxxx 10xxxxxx 10xxxxxx
|
|
if i + 2 >= bytes.len()
|
|
|| (bytes[i + 1] & 0xC0) != 0x80
|
|
|| (bytes[i + 2] & 0xC0) != 0x80 { break; }
|
|
has_multibyte = true;
|
|
nbytes += 3;
|
|
i += 3;
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
if has_multibyte
|
|
&& nbytes >= MIN_LEN
|
|
&& i < bytes.len() && bytes[i] == 0
|
|
&& let Ok(s) = std::str::from_utf8(&bytes[start..i])
|
|
{
|
|
out.push(DetectedString {
|
|
address: va_base + start as u32,
|
|
encoding: "utf8",
|
|
length: nbytes as u32,
|
|
content: s.to_string(),
|
|
section: String::new(),
|
|
});
|
|
i += 1; // skip NUL
|
|
} else {
|
|
i = start + 1;
|
|
if i < bytes.len() && bytes[i] == 0 { i += 1; }
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn mk_section(name: &str, va: u32, size: u32) -> PeSection {
|
|
PeSection {
|
|
name: name.into(),
|
|
virtual_address: va,
|
|
virtual_size: size,
|
|
raw_offset: va,
|
|
raw_size: size,
|
|
flags: 0x4000_0040,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn detects_ascii_string() {
|
|
let image_base = 0x82000000u32;
|
|
let mut pe = vec![0u8; 0x1100];
|
|
let off = 0x1000usize;
|
|
let s = b"Hello, world!\0";
|
|
pe[off..off + s.len()].copy_from_slice(s);
|
|
let sections = vec![mk_section(".rdata", 0x1000, 0x100)];
|
|
let strings = analyze(&pe, image_base, §ions);
|
|
assert_eq!(strings.len(), 1);
|
|
assert_eq!(strings[0].encoding, "ascii");
|
|
assert_eq!(strings[0].content, "Hello, world!");
|
|
assert_eq!(strings[0].address, image_base + 0x1000);
|
|
}
|
|
|
|
#[test]
|
|
fn rejects_short_runs() {
|
|
let image_base = 0x82000000u32;
|
|
let mut pe = vec![0u8; 0x1100];
|
|
let off = 0x1000usize;
|
|
let s = b"Hi\0longer string here\0";
|
|
pe[off..off + s.len()].copy_from_slice(s);
|
|
let sections = vec![mk_section(".rdata", 0x1000, 0x100)];
|
|
let strings = analyze(&pe, image_base, §ions);
|
|
assert_eq!(strings.len(), 1);
|
|
assert_eq!(strings[0].content, "longer string here");
|
|
}
|
|
|
|
#[test]
|
|
fn detects_utf16le_string() {
|
|
let image_base = 0x82000000u32;
|
|
let mut pe = vec![0u8; 0x1100];
|
|
let off = 0x1000usize;
|
|
// "Hello!" in UTF-16LE + NUL u16
|
|
let s: &[u8] = b"H\0e\0l\0l\0o\0!\0\0\0";
|
|
pe[off..off + s.len()].copy_from_slice(s);
|
|
let sections = vec![mk_section(".rdata", 0x1000, 0x100)];
|
|
let strings = analyze(&pe, image_base, §ions);
|
|
// Both ASCII and UTF-16 may detect — UTF-16 should find it as wide;
|
|
// ASCII pass scans bytes and won't see this as a contiguous run
|
|
// because of the interleaved 0 bytes (non-printable).
|
|
let utf16: Vec<_> = strings.iter().filter(|s| s.encoding == "utf16le").collect();
|
|
assert!(utf16.iter().any(|s| s.content == "Hello!"));
|
|
}
|
|
|
|
#[test]
|
|
fn detects_shift_jis_string() {
|
|
let image_base = 0x82000000u32;
|
|
let mut pe = vec![0u8; 0x1100];
|
|
let off = 0x1000usize;
|
|
// "ABC" + SJIS hiragana あ (0x82 0xA0) + い (0x82 0xA2) + NUL.
|
|
let s: &[u8] = b"ABC\x82\xA0\x82\xA2\0";
|
|
pe[off..off + s.len()].copy_from_slice(s);
|
|
let sections = vec![mk_section(".rdata", 0x1000, 0x100)];
|
|
let strings = analyze(&pe, image_base, §ions);
|
|
let sjis: Vec<_> = strings.iter().filter(|s| s.encoding == "shift_jis").collect();
|
|
assert_eq!(sjis.len(), 1);
|
|
// Decoded to real UTF-8, not rendered as escaped bytes.
|
|
assert_eq!(sjis[0].content, "ABCあい");
|
|
assert_eq!(sjis[0].address, image_base + 0x1000);
|
|
}
|
|
|
|
#[test]
|
|
fn shift_jis_rejects_float_table_noise() {
|
|
// Four IEEE-754 floats (0.85, 0.9, 0.8, 0.7). Every byte satisfies the
|
|
// Shift_JIS lead/trail ranges, so the byte-range test alone accepts it.
|
|
let image_base = 0x82000000u32;
|
|
let mut pe = vec![0u8; 0x1100];
|
|
let off = 0x1000usize;
|
|
let s: &[u8] = b"\x3f\x59\x99\x9a\x3f\x66\x66\x66\x3f\x4c\xcc\xcd\x3f\x33\x33\x33\0";
|
|
pe[off..off + s.len()].copy_from_slice(s);
|
|
let sections = vec![mk_section(".rdata", 0x1000, 0x100)];
|
|
let strings = analyze(&pe, image_base, §ions);
|
|
assert!(strings.iter().all(|s| s.encoding != "shift_jis"),
|
|
"float table must not be reported as Japanese text");
|
|
}
|
|
|
|
#[test]
|
|
fn shift_jis_resynchronises_to_true_start() {
|
|
// Mirrors 0x820a4b9f in the reference title: binary data runs straight
|
|
// into a real string, and a naive forward scan mis-pairs the boundary
|
|
// byte, yielding `帥Vステム…` one byte early instead of `システム…`.
|
|
let image_base = 0x82000000u32;
|
|
let mut pe = vec![0u8; 0x1100];
|
|
let off = 0x1000usize;
|
|
// Exact bytes from that site: a trailing 0x90 from the preceding
|
|
// float pairs with the string's first byte (0x83) to form 帥, which
|
|
// orphans the 0x56 as an ASCII 'V' before the text resumes.
|
|
// 0x90 シ ス テ ム
|
|
let s: &[u8] = b"\x90\x83\x56\x83\x58\x83\x65\x83\x80\0";
|
|
pe[off..off + s.len()].copy_from_slice(s);
|
|
let sections = vec![mk_section(".rdata", 0x1000, 0x100)];
|
|
let strings = analyze(&pe, image_base, §ions);
|
|
let sjis: Vec<_> = strings.iter().filter(|s| s.encoding == "shift_jis").collect();
|
|
assert_eq!(sjis.len(), 1);
|
|
assert_eq!(sjis[0].content, "システム");
|
|
// Reported at the true start, one byte past the run's beginning.
|
|
assert_eq!(sjis[0].address, image_base + 0x1000 + 1);
|
|
}
|
|
|
|
#[test]
|
|
fn detects_utf8_multibyte_string() {
|
|
let image_base = 0x82000000u32;
|
|
let mut pe = vec![0u8; 0x1100];
|
|
let off = 0x1000usize;
|
|
// "Café" = 'C', 'a', 'f', 0xC3 0xA9 (é), then more ASCII to reach min length
|
|
let s: &[u8] = b"Caf\xC3\xA9eteria\0";
|
|
pe[off..off + s.len()].copy_from_slice(s);
|
|
let sections = vec![mk_section(".rdata", 0x1000, 0x100)];
|
|
let strings = analyze(&pe, image_base, §ions);
|
|
let u8s: Vec<_> = strings.iter().filter(|s| s.encoding == "utf8").collect();
|
|
assert_eq!(u8s.len(), 1);
|
|
assert_eq!(u8s[0].content, "Café".to_string() + "eteria");
|
|
}
|
|
|
|
#[test]
|
|
fn requires_nul_terminator() {
|
|
let image_base = 0x82000000u32;
|
|
let mut pe = vec![0u8; 0x1100];
|
|
// No trailing NUL — should NOT be detected.
|
|
let off = 0x1000usize;
|
|
let s = b"abcdefghij";
|
|
pe[off..off + s.len()].copy_from_slice(s);
|
|
// Fill rest of section with 0xFF so the run terminates cleanly without NUL.
|
|
for j in off + s.len()..off + 0x100 { pe[j] = 0xFF; }
|
|
let sections = vec![mk_section(".rdata", 0x1000, 0x100)];
|
|
let strings = analyze(&pe, image_base, §ions);
|
|
assert_eq!(strings.len(), 0);
|
|
}
|
|
}
|