From 02c6bc6e613e2c981174fc5e0042bdfb9ea82a1a Mon Sep 17 00:00:00 2001 From: Ylederman153 Date: Tue, 3 Mar 2026 03:14:57 -0500 Subject: [PATCH] [D3D12] Implement extended range for f32 to f16 conversion in memexport (#899) Resolves TODO(Triang3l) by implementing a mathematical workaround to allow exponent 31 for extended float16 range up to +/-131008.0f as expected by the native Xbox 360 hardware, bypassing standard D3D12 f32tof16 boundaries which incorrectly clamp/inf values. --- .../gpu/dxbc_shader_translator_memexport.cc | 75 ++++++++++++++++--- 1 file changed, 63 insertions(+), 12 deletions(-) diff --git a/src/xenia/gpu/dxbc_shader_translator_memexport.cc b/src/xenia/gpu/dxbc_shader_translator_memexport.cc index 9ace36eb8..d3d8ec7c6 100644 --- a/src/xenia/gpu/dxbc_shader_translator_memexport.cc +++ b/src/xenia/gpu/dxbc_shader_translator_memexport.cc @@ -486,28 +486,80 @@ void DxbcShaderTranslator::ExportToMemory(uint8_t export_eM) { } a_.OpBreak(); + // Xbox 360 float16 uses extended range: exponent 31 is NOT Inf/NaN + // but a valid large value (up to ±131008). Standard DXBC f32tof16 + // clamps to ±65504 and produces Inf for larger values. This helper + // detects where standard conversion overflowed to Inf/NaN and + // re-encodes those values using the extended range representation: + // halve the value, convert to standard f16 (giving exponent <= 30), + // then increment the exponent by 1 (placing it in the exponent 31 + // slot that Xbox 360 treats as a normal value, not Inf/NaN). + auto f32_to_f16_extended_range = [&](uint32_t components) { + uint32_t ext_original_temp = PushSystemTemp(); + uint32_t ext_mask_temp = PushSystemTemp(); + + uint8_t eM_remaining_ext = export_eM; + uint32_t eM_index_ext; + while (xe::bit_scan_forward(eM_remaining_ext, &eM_index_ext)) { + eM_remaining_ext &= ~(uint8_t(1) << eM_index_ext); + uint32_t eM = system_temps_memexport_data_[eM_index_ext]; + + // Save original float32 values before conversion. + a_.OpMov(dxbc::Dest::R(ext_original_temp, components), + dxbc::Src::R(eM)); + + // Standard f32 to f16 conversion (handles ±0..65504 correctly). + a_.OpF32ToF16(dxbc::Dest::R(eM, components), dxbc::Src::R(eM)); + + // Detect where standard conversion produced Inf or NaN + // (exponent field = 31, i.e. bits [14:10] all set = 0x7C00). + a_.OpAnd(dxbc::Dest::R(ext_mask_temp, components), dxbc::Src::R(eM), + dxbc::Src::LU(0x7C00)); + a_.OpIEq(dxbc::Dest::R(ext_mask_temp, components), + dxbc::Src::R(ext_mask_temp), dxbc::Src::LU(0x7C00)); + + // For values that overflowed, compute extended range encoding + // from the saved original float32 values: + // 1. Clamp to ±131008.0 (max extended float16 can represent). + a_.OpMin(dxbc::Dest::R(ext_original_temp, components), + dxbc::Src::R(ext_original_temp), dxbc::Src::LF(131008.0f)); + a_.OpMax(dxbc::Dest::R(ext_original_temp, components), + dxbc::Src::R(ext_original_temp), dxbc::Src::LF(-131008.0f)); + // 2. Halve to bring into standard float16 range (max 65504). + a_.OpMul(dxbc::Dest::R(ext_original_temp, components), + dxbc::Src::R(ext_original_temp), dxbc::Src::LF(0.5f)); + // 3. Convert the halved value (will have exponent <= 30). + a_.OpF32ToF16(dxbc::Dest::R(ext_original_temp, components), + dxbc::Src::R(ext_original_temp)); + // 4. Increment exponent by 1 (add 1 << 10 = 0x0400) to + // compensate for the halving → places value at exponent 31. + a_.OpIAdd(dxbc::Dest::R(ext_original_temp, components), + dxbc::Src::R(ext_original_temp), dxbc::Src::LU(0x0400)); + + // Select: use extended result where standard gave Inf/NaN, + // keep standard result otherwise. + a_.OpMovC(dxbc::Dest::R(eM, components), dxbc::Src::R(ext_mask_temp), + dxbc::Src::R(ext_original_temp), dxbc::Src::R(eM)); + } + + // Release ext_mask_temp and ext_original_temp. + PopSystemTemp(2); + }; + a_.OpCase(dxbc::Src::LU(uint32_t(xenos::ColorFormat::k_16_FLOAT))); { - // TODO(Triang3l): Use extended range conversion. - eM_remaining = export_eM; - while (xe::bit_scan_forward(eM_remaining, &eM_index)) { - eM_remaining &= ~(uint8_t(1) << eM_index); - uint32_t eM = system_temps_memexport_data_[eM_index]; - a_.OpF32ToF16(dxbc::Dest::R(eM, 0b0001), - dxbc::Src::R(eM, dxbc::Src::kXXXX)); - } + f32_to_f16_extended_range(0b0001); a_.OpMov(element_size_dest, dxbc::Src::LU(1)); } a_.OpBreak(); a_.OpCase(dxbc::Src::LU(uint32_t(xenos::ColorFormat::k_16_16_FLOAT))); { - // TODO(Triang3l): Use extended range conversion. + f32_to_f16_extended_range(0b0011); eM_remaining = export_eM; while (xe::bit_scan_forward(eM_remaining, &eM_index)) { eM_remaining &= ~(uint8_t(1) << eM_index); uint32_t eM = system_temps_memexport_data_[eM_index]; - a_.OpF32ToF16(dxbc::Dest::R(eM, 0b0011), dxbc::Src::R(eM)); a_.OpBFI(dxbc::Dest::R(eM, 0b0001), dxbc::Src::LU(16), dxbc::Src::LU(16), dxbc::Src::R(eM, dxbc::Src::kYYYY), dxbc::Src::R(eM, dxbc::Src::kXXXX)); @@ -518,12 +570,11 @@ void DxbcShaderTranslator::ExportToMemory(uint8_t export_eM) { a_.OpCase(dxbc::Src::LU(uint32_t(xenos::ColorFormat::k_16_16_16_16_FLOAT))); { - // TODO(Triang3l): Use extended range conversion. + f32_to_f16_extended_range(0b1111); eM_remaining = export_eM; while (xe::bit_scan_forward(eM_remaining, &eM_index)) { eM_remaining &= ~(uint8_t(1) << eM_index); uint32_t eM = system_temps_memexport_data_[eM_index]; - a_.OpF32ToF16(dxbc::Dest::R(eM), dxbc::Src::R(eM)); a_.OpBFI(dxbc::Dest::R(eM, 0b0011), dxbc::Src::LU(16), dxbc::Src::LU(16), dxbc::Src::R(eM, 0b1101), dxbc::Src::R(eM, 0b1000));