/** ****************************************************************************** * Xenia : Xbox 360 Emulator Research Project * ****************************************************************************** * Copyright 2024 Ben Vanik. All rights reserved. * * Released under the BSD license - see LICENSE in the root for more details. * ****************************************************************************** */ #include "xenia/gpu/spirv_shader_translator.h" #include "third_party/glslang/SPIRV/GLSL.std.450.h" #include "xenia/base/assert.h" #include "xenia/base/math.h" #include "xenia/gpu/spirv_compatibility.h" #include "xenia/gpu/ucode.h" namespace xe { namespace gpu { void SpirvShaderTranslator::ExportToMemory(uint8_t export_eM) { if (!export_eM) { return; } assert_zero(export_eM & ~current_shader().memexport_eM_written()); if (!IsMemoryExportSupported()) { return; } // Check if memory export is allowed in this guest shader invocation. std::optional if_memexport_allowed; spv::Id memexport_allowed = main_memexport_allowed_; // For pixel shaders with resolution scaling, only allow memory export from // the center host pixel to avoid duplicate exports. if (is_pixel_shader() && (draw_resolution_scale_x_ > 1 || draw_resolution_scale_y_ > 1)) { assert_true(input_fragment_coordinates_ != spv::NoResult); // Check if we're at the center pixel (scale/2 for both X and Y). spv::Id is_center_pixel = builder_->makeBoolConstant(true); // Check X coordinate. if (draw_resolution_scale_x_ > 1) { id_vector_temp_.clear(); id_vector_temp_.push_back(const_int_0_); spv::Id pixel_x = builder_->createUnaryOp( spv::OpConvertFToU, type_uint_, builder_->createLoad( builder_->createAccessChain(spv::StorageClassInput, input_fragment_coordinates_, id_vector_temp_), spv::NoPrecision)); spv::Id pixel_x_remainder = builder_->createBinOp( spv::OpUMod, type_uint_, pixel_x, builder_->makeUintConstant(draw_resolution_scale_x_)); is_center_pixel = builder_->createBinOp( spv::OpLogicalAnd, type_bool_, is_center_pixel, builder_->createBinOp( spv::OpIEqual, type_bool_, pixel_x_remainder, builder_->makeUintConstant(draw_resolution_scale_x_ >> 1))); } // Check Y coordinate. if (draw_resolution_scale_y_ > 1) { id_vector_temp_.clear(); id_vector_temp_.push_back(builder_->makeIntConstant(1)); spv::Id pixel_y = builder_->createUnaryOp( spv::OpConvertFToU, type_uint_, builder_->createLoad( builder_->createAccessChain(spv::StorageClassInput, input_fragment_coordinates_, id_vector_temp_), spv::NoPrecision)); spv::Id pixel_y_remainder = builder_->createBinOp( spv::OpUMod, type_uint_, pixel_y, builder_->makeUintConstant(draw_resolution_scale_y_)); is_center_pixel = builder_->createBinOp( spv::OpLogicalAnd, type_bool_, is_center_pixel, builder_->createBinOp( spv::OpIEqual, type_bool_, pixel_y_remainder, builder_->makeUintConstant(draw_resolution_scale_y_ >> 1))); } // Combine with existing memexport_allowed condition. memexport_allowed = memexport_allowed != spv::NoResult ? builder_->createBinOp(spv::OpLogicalAnd, type_bool_, memexport_allowed, is_center_pixel) : is_center_pixel; } if (memexport_allowed != spv::NoResult) { if_memexport_allowed.emplace( memexport_allowed, spv::SelectionControlDontFlattenMask, *builder_); } // If the pixel was killed (but the actual killing on the SPIR-V side has not // been performed yet because the device doesn't support demotion to helper // invocation that doesn't interfere with control flow), the current // invocation is not considered active anymore. std::optional if_pixel_not_killed; if (var_main_kill_pixel_ != spv::NoResult) { if_pixel_not_killed.emplace( builder_->createUnaryOp( spv::OpLogicalNot, type_bool_, builder_->createLoad(var_main_kill_pixel_, spv::NoPrecision)), spv::SelectionControlDontFlattenMask, *builder_); } // Check if the address with the correct sign and exponent was written, and // that the index doesn't overflow the mantissa bits. // all((eA_vector >> uvec4(30, 23, 23, 23)) == uvec4(0x1, 0x96, 0x96, 0x96)) spv::Id eA_vector = builder_->createUnaryOp( spv::OpBitcast, type_uint4_, builder_->createLoad(var_main_memexport_address_, spv::NoPrecision)); id_vector_temp_.clear(); id_vector_temp_.push_back(builder_->makeUintConstant(30)); id_vector_temp_.push_back(builder_->makeUintConstant(23)); id_vector_temp_.push_back(id_vector_temp_.back()); id_vector_temp_.push_back(id_vector_temp_.back()); spv::Id address_validation_shift = builder_->makeCompositeConstant(type_uint4_, id_vector_temp_); id_vector_temp_.clear(); id_vector_temp_.push_back(builder_->makeUintConstant(0x1)); id_vector_temp_.push_back(builder_->makeUintConstant(0x96)); id_vector_temp_.push_back(id_vector_temp_.back()); id_vector_temp_.push_back(id_vector_temp_.back()); spv::Id address_validation_value = builder_->makeCompositeConstant(type_uint4_, id_vector_temp_); SpirvBuilder::IfBuilder if_address_valid( builder_->createUnaryOp( spv::OpAll, type_bool_, builder_->createBinOp( spv::OpIEqual, type_bool4_, builder_->createBinOp(spv::OpShiftRightLogical, type_uint4_, eA_vector, address_validation_shift), address_validation_value)), spv::SelectionControlDontFlattenMask, *builder_, 2, 1); using EMIdArray = std::array; auto for_each_eM = [&](std::function fn) { uint8_t eM_remaining = export_eM; uint32_t eM_index; while (xe::bit_scan_forward(eM_remaining, &eM_index)) { eM_remaining &= ~(uint8_t(1) << eM_index); fn(eM_index); } }; // Load the original eM. EMIdArray eM_original; for_each_eM([&](uint32_t eM_index) { eM_original[eM_index] = builder_->createLoad( var_main_memexport_data_[eM_index], spv::NoPrecision); }); // Swap red and blue if needed. spv::Id format_info = builder_->createCompositeExtract(eA_vector, type_uint_, 2); spv::Id swap_red_blue = builder_->createBinOp( spv::OpINotEqual, type_bool_, builder_->createBinOp(spv::OpBitwiseAnd, type_uint_, format_info, builder_->makeUintConstant(uint32_t(1) << 19)), const_uint_0_); EMIdArray eM_swapped; uint_vector_temp_.clear(); uint_vector_temp_.push_back(2); uint_vector_temp_.push_back(1); uint_vector_temp_.push_back(0); uint_vector_temp_.push_back(3); spv::Id swap_red_blue_bool4 = builder_->smearScalar(spv::NoPrecision, swap_red_blue, type_bool4_); for_each_eM([&](uint32_t eM_index) { eM_swapped[eM_index] = builder_->createTriOp( spv::OpSelect, type_float4_, swap_red_blue_bool4, builder_->createRvalueSwizzle(spv::NoPrecision, type_float4_, eM_original[eM_index], uint_vector_temp_), eM_original[eM_index]); }); // Extract the numeric format. spv::Id is_signed = builder_->createBinOp( spv::OpINotEqual, type_bool_, builder_->createBinOp(spv::OpBitwiseAnd, type_uint_, format_info, builder_->makeUintConstant(uint32_t(1) << 16)), const_uint_0_); spv::Id is_norm = builder_->createBinOp( spv::OpIEqual, type_bool_, builder_->createBinOp(spv::OpBitwiseAnd, type_uint_, format_info, builder_->makeUintConstant(uint32_t(1) << 17)), const_uint_0_); // Perform format packing. auto flush_nan = [&](const EMIdArray& eM) -> EMIdArray { EMIdArray eM_flushed; for_each_eM([&](uint32_t eM_index) { spv::Id element_unflushed = eM[eM_index]; unsigned int component_count = builder_->getNumComponents(element_unflushed); eM_flushed[eM_index] = builder_->createTriOp( spv::OpSelect, type_float_vectors_[component_count - 1], builder_->createUnaryOp(spv::OpIsNan, type_bool_vectors_[component_count - 1], element_unflushed), const_float_vectors_0_[component_count - 1], element_unflushed); }); return eM_flushed; }; auto make_float_constant_vectors = [&](float value) -> std::array { std::array const_vectors; const_vectors[0] = builder_->makeFloatConstant(value); id_vector_temp_.clear(); id_vector_temp_.push_back(const_vectors[0]); for (unsigned int component_count_minus_1 = 1; component_count_minus_1 < 4; ++component_count_minus_1) { id_vector_temp_.push_back(const_vectors[0]); const_vectors[component_count_minus_1] = builder_->makeCompositeConstant( type_float_vectors_[component_count_minus_1], id_vector_temp_); } return const_vectors; }; std::array const_float_vectors_minus_1 = make_float_constant_vectors(-1.0f); std::array const_float_vectors_minus_0_5 = make_float_constant_vectors(-0.5f); std::array const_float_vectors_0_5 = make_float_constant_vectors(0.5f); // The widths must be without holes (R, RG, RGB, RGBA), and expecting the // widths to add up to the size of the stored texel (8, 16 or 32 bits), as the // unused upper bits will contain junk from the sign extension of X if the // number is signed. auto pack_8_16_32 = [&](std::array widths) -> EMIdArray { unsigned int component_count; std::array offsets{}; for (component_count = 0; component_count < widths.size(); ++component_count) { if (!widths[component_count]) { break; } // Only formats for which max + 0.5 can be represented exactly. assert(widths[component_count] <= 23); if (component_count) { offsets[component_count] = offsets[component_count - 1] + widths[component_count - 1]; } } assert_not_zero(component_count); // Extract the needed components. EMIdArray eM_unflushed = eM_swapped; if (component_count < 4) { if (component_count == 1) { for_each_eM([&](uint32_t eM_index) { eM_unflushed[eM_index] = builder_->createCompositeExtract( eM_unflushed[eM_index], type_float_, 0); }); } else { uint_vector_temp_.clear(); for (unsigned int component_index = 0; component_index < component_count; ++component_index) { uint_vector_temp_.push_back(component_index); } for_each_eM([&](uint32_t eM_index) { eM_unflushed[eM_index] = builder_->createRvalueSwizzle( spv::NoPrecision, type_float_vectors_[component_count - 1], eM_unflushed[eM_index], uint_vector_temp_); }); } } // Flush NaNs. EMIdArray eM_flushed = flush_nan(eM_unflushed); // Convert to integers. SpirvBuilder::IfBuilder if_signed( is_signed, spv::SelectionControlDontFlattenMask, *builder_); EMIdArray eM_signed; { // Signed. SpirvBuilder::IfBuilder if_norm( is_norm, spv::SelectionControlDontFlattenMask, *builder_); EMIdArray eM_norm; { // Signed normalized. id_vector_temp_.clear(); for (unsigned int component_index = 0; component_index < component_count; ++component_index) { id_vector_temp_.push_back(builder_->makeFloatConstant( float((uint32_t(1) << (widths[component_index] - 1)) - 1))); } spv::Id const_max_value = component_count > 1 ? builder_->makeCompositeConstant( type_float_vectors_[component_count - 1], id_vector_temp_) : id_vector_temp_.front(); for_each_eM([&](uint32_t eM_index) { eM_norm[eM_index] = builder_->createNoContractionBinOp( spv::OpFMul, type_float_vectors_[component_count - 1], builder_->createTriBuiltinCall( type_float_vectors_[component_count - 1], ext_inst_glsl_std_450_, GLSLstd450FClamp, eM_flushed[eM_index], const_float_vectors_minus_1[component_count - 1], const_float_vectors_1_[component_count - 1]), const_max_value); }); } if_norm.makeEndIf(); // All phi instructions must be in the beginning of the block. for_each_eM([&](uint32_t eM_index) { eM_signed[eM_index] = if_norm.createMergePhi(eM_norm[eM_index], eM_flushed[eM_index]); }); // Convert to signed integer, adding plus/minus 0.5 before truncating // according to the Direct3D format conversion rules. for_each_eM([&](uint32_t eM_index) { eM_signed[eM_index] = builder_->createUnaryOp( spv::OpBitcast, type_uint_vectors_[component_count - 1], builder_->createUnaryOp( spv::OpConvertFToS, type_int_vectors_[component_count - 1], builder_->createNoContractionBinOp( spv::OpFAdd, type_float_vectors_[component_count - 1], eM_signed[eM_index], builder_->createTriOp( spv::OpSelect, type_float_vectors_[component_count - 1], builder_->createBinOp( spv::OpFOrdLessThan, type_bool_vectors_[component_count - 1], eM_signed[eM_index], const_float_vectors_0_[component_count - 1]), const_float_vectors_minus_0_5[component_count - 1], const_float_vectors_0_5[component_count - 1])))); }); } if_signed.makeBeginElse(); EMIdArray eM_unsigned; { SpirvBuilder::IfBuilder if_norm( is_norm, spv::SelectionControlDontFlattenMask, *builder_); EMIdArray eM_norm; { // Unsigned normalized. id_vector_temp_.clear(); for (unsigned int component_index = 0; component_index < component_count; ++component_index) { id_vector_temp_.push_back(builder_->makeFloatConstant( float((uint32_t(1) << widths[component_index]) - 1))); } spv::Id const_max_value = component_count > 1 ? builder_->makeCompositeConstant( type_float_vectors_[component_count - 1], id_vector_temp_) : id_vector_temp_.front(); for_each_eM([&](uint32_t eM_index) { eM_norm[eM_index] = builder_->createNoContractionBinOp( spv::OpFMul, type_float_vectors_[component_count - 1], builder_->createTriBuiltinCall( type_float_vectors_[component_count - 1], ext_inst_glsl_std_450_, GLSLstd450FClamp, eM_flushed[eM_index], const_float_vectors_0_[component_count - 1], const_float_vectors_1_[component_count - 1]), const_max_value); }); } if_norm.makeEndIf(); // All phi instructions must be in the beginning of the block. for_each_eM([&](uint32_t eM_index) { eM_unsigned[eM_index] = if_norm.createMergePhi(eM_norm[eM_index], eM_flushed[eM_index]); }); // Convert to unsigned integer, adding 0.5 before truncating according to // the Direct3D format conversion rules. for_each_eM([&](uint32_t eM_index) { eM_unsigned[eM_index] = builder_->createUnaryOp( spv::OpConvertFToU, type_uint_vectors_[component_count - 1], builder_->createNoContractionBinOp( spv::OpFAdd, type_float_vectors_[component_count - 1], eM_unsigned[eM_index], const_float_vectors_0_5[component_count - 1])); }); } if_signed.makeEndIf(); EMIdArray eM_unpacked; for_each_eM([&](uint32_t eM_index) { eM_unpacked[eM_index] = if_signed.createMergePhi(eM_signed[eM_index], eM_unsigned[eM_index]); }); // Pack into a 32-bit value, and pad to a 4-component vector for the phi. EMIdArray eM_packed; for_each_eM([&](uint32_t eM_index) { spv::Id element_unpacked = eM_unpacked[eM_index]; eM_packed[eM_index] = component_count > 1 ? builder_->createCompositeExtract( element_unpacked, type_uint_, 0) : element_unpacked; for (unsigned int component_index = 1; component_index < component_count; ++component_index) { eM_packed[eM_index] = builder_->createQuadOp( spv::OpBitFieldInsert, type_uint_, eM_packed[eM_index], builder_->createCompositeExtract(element_unpacked, type_uint_, component_index), builder_->makeUintConstant(offsets[component_index]), builder_->makeUintConstant(widths[component_index])); } id_vector_temp_.clear(); id_vector_temp_.resize(4, const_uint_0_); id_vector_temp_.front() = eM_packed[eM_index]; eM_packed[eM_index] = builder_->createCompositeConstruct(type_uint4_, id_vector_temp_); }); return eM_packed; }; SpirvBuilder::SwitchBuilder format_switch( builder_->createTriOp(spv::OpBitFieldUExtract, type_uint_, format_info, builder_->makeUintConstant(8), builder_->makeUintConstant(6)), spv::SelectionControlDontFlattenMask, *builder_); struct FormatCase { EMIdArray eM_packed; uint32_t element_bytes_log2; spv::Id phi_parent; }; std::vector format_cases; // Must be called at the end of the switch case segment for the correct phi // parent. auto add_format_case = [&](const EMIdArray& eM_packed, uint32_t element_bytes_log2) { FormatCase& format_case = format_cases.emplace_back(); format_case.eM_packed = eM_packed; format_case.element_bytes_log2 = element_bytes_log2; format_case.phi_parent = builder_->getBuildPoint()->getId(); }; // k_8, k_8_A, k_8_B format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_8)); // TODO(Triang3l): Investigate how input should be treated for k_8_A, k_8_B. format_switch.addCurrentCaseLiteral( static_cast(xenos::ColorFormat::k_8_A)); format_switch.addCurrentCaseLiteral( static_cast(xenos::ColorFormat::k_8_B)); add_format_case(pack_8_16_32({8}), 0); // k_1_5_5_5 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_1_5_5_5)); add_format_case(pack_8_16_32({5, 5, 5, 1}), 1); // k_5_6_5 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_5_6_5)); add_format_case(pack_8_16_32({5, 6, 5}), 1); // k_6_5_5 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_6_5_5)); add_format_case(pack_8_16_32({5, 5, 6}), 1); // k_8_8_8_8, k_8_8_8_8_A, k_8_8_8_8_AS_16_16_16_16 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_8_8_8_8)); // TODO(Triang3l): Investigate how input should be treated for k_8_8_8_8_A. format_switch.addCurrentCaseLiteral( static_cast(xenos::ColorFormat::k_8_8_8_8_A)); format_switch.addCurrentCaseLiteral( static_cast(xenos::ColorFormat::k_8_8_8_8_AS_16_16_16_16)); add_format_case(pack_8_16_32({8, 8, 8, 8}), 2); // k_2_10_10_10, k_2_10_10_10_AS_16_16_16_16 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_2_10_10_10)); format_switch.addCurrentCaseLiteral(static_cast( xenos::ColorFormat::k_2_10_10_10_AS_16_16_16_16)); add_format_case(pack_8_16_32({10, 10, 10, 2}), 2); // k_8_8 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_8_8)); add_format_case(pack_8_16_32({8, 8}), 1); // k_4_4_4_4 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_4_4_4_4)); add_format_case(pack_8_16_32({4, 4, 4, 4}), 1); // k_10_11_11, k_10_11_11_AS_16_16_16_16 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_10_11_11)); format_switch.addCurrentCaseLiteral( static_cast(xenos::ColorFormat::k_10_11_11_AS_16_16_16_16)); add_format_case(pack_8_16_32({11, 11, 10}), 2); // k_11_11_10, k_11_11_10_AS_16_16_16_16 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_11_11_10)); format_switch.addCurrentCaseLiteral( static_cast(xenos::ColorFormat::k_11_11_10_AS_16_16_16_16)); add_format_case(pack_8_16_32({10, 11, 11}), 2); // k_16 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_16)); add_format_case(pack_8_16_32({16}), 1); // k_16_16 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_16_16)); add_format_case(pack_8_16_32({16, 16}), 2); // k_16_16_16_16 format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_16_16_16_16)); { // Flush NaNs. EMIdArray fixed16_flushed = flush_nan(eM_swapped); // Convert to integers. SpirvBuilder::IfBuilder if_signed( is_signed, spv::SelectionControlDontFlattenMask, *builder_); EMIdArray fixed16_signed; { // Signed. SpirvBuilder::IfBuilder if_norm( is_norm, spv::SelectionControlDontFlattenMask, *builder_); EMIdArray fixed16_norm; { // Signed normalized. id_vector_temp_.clear(); id_vector_temp_.resize(4, builder_->makeFloatConstant( float((uint32_t(1) << (16 - 1)) - 1))); spv::Id const_snorm16_max_value = builder_->makeCompositeConstant(type_float4_, id_vector_temp_); for_each_eM([&](uint32_t eM_index) { fixed16_norm[eM_index] = builder_->createNoContractionBinOp( spv::OpFMul, type_float4_, builder_->createTriBuiltinCall( type_float4_, ext_inst_glsl_std_450_, GLSLstd450FClamp, fixed16_flushed[eM_index], const_float_vectors_minus_1[3], const_float4_1_), const_snorm16_max_value); }); } if_norm.makeEndIf(); // All phi instructions must be in the beginning of the block. for_each_eM([&](uint32_t eM_index) { fixed16_signed[eM_index] = if_norm.createMergePhi( fixed16_norm[eM_index], fixed16_flushed[eM_index]); }); // Convert to signed integer, adding plus/minus 0.5 before truncating // according to the Direct3D format conversion rules. for_each_eM([&](uint32_t eM_index) { fixed16_signed[eM_index] = builder_->createUnaryOp( spv::OpBitcast, type_uint4_, builder_->createUnaryOp( spv::OpConvertFToS, type_int4_, builder_->createNoContractionBinOp( spv::OpFAdd, type_float4_, fixed16_signed[eM_index], builder_->createTriOp( spv::OpSelect, type_float4_, builder_->createBinOp(spv::OpFOrdLessThan, type_bool4_, fixed16_signed[eM_index], const_float4_0_), const_float_vectors_minus_0_5[3], const_float_vectors_0_5[3])))); }); } if_signed.makeBeginElse(); EMIdArray fixed16_unsigned; { // Unsigned. SpirvBuilder::IfBuilder if_norm( is_norm, spv::SelectionControlDontFlattenMask, *builder_); EMIdArray fixed16_norm; { // Unsigned normalized. id_vector_temp_.clear(); id_vector_temp_.resize( 4, builder_->makeFloatConstant(float((uint32_t(1) << 16) - 1))); spv::Id const_unorm16_max_value = builder_->makeCompositeConstant(type_float4_, id_vector_temp_); for_each_eM([&](uint32_t eM_index) { fixed16_norm[eM_index] = builder_->createNoContractionBinOp( spv::OpFMul, type_float4_, builder_->createTriBuiltinCall( type_float4_, ext_inst_glsl_std_450_, GLSLstd450FClamp, fixed16_flushed[eM_index], const_float4_0_, const_float4_1_), const_unorm16_max_value); }); } if_norm.makeEndIf(); // All phi instructions must be in the beginning of the block. for_each_eM([&](uint32_t eM_index) { fixed16_unsigned[eM_index] = if_norm.createMergePhi( fixed16_norm[eM_index], fixed16_flushed[eM_index]); }); // Convert to unsigned integer, adding 0.5 before truncating according to // the Direct3D format conversion rules. for_each_eM([&](uint32_t eM_index) { fixed16_unsigned[eM_index] = builder_->createUnaryOp( spv::OpConvertFToU, type_uint4_, builder_->createNoContractionBinOp(spv::OpFAdd, type_float4_, fixed16_unsigned[eM_index], const_float_vectors_0_5[3])); }); } if_signed.makeEndIf(); EMIdArray fixed16_unpacked; for_each_eM([&](uint32_t eM_index) { fixed16_unpacked[eM_index] = if_signed.createMergePhi( fixed16_signed[eM_index], fixed16_unsigned[eM_index]); }); // Pack into two 32-bit values, and pad to a 4-component vector for the phi. EMIdArray fixed16_packed; spv::Id const_uint_16 = builder_->makeUintConstant(16); for_each_eM([&](uint32_t eM_index) { spv::Id fixed16_element_unpacked = fixed16_unpacked[eM_index]; id_vector_temp_.clear(); for (uint32_t component_index = 0; component_index < 2; ++component_index) { id_vector_temp_.push_back(builder_->createQuadOp( spv::OpBitFieldInsert, type_uint_, builder_->createCompositeExtract(fixed16_element_unpacked, type_uint_, 2 * component_index), builder_->createCompositeExtract( fixed16_element_unpacked, type_uint_, 2 * component_index + 1), const_uint_16, const_uint_16)); } for (uint32_t component_index = 2; component_index < 4; ++component_index) { id_vector_temp_.push_back(const_uint_0_); } fixed16_packed[eM_index] = builder_->createCompositeConstruct(type_uint4_, id_vector_temp_); }); add_format_case(fixed16_packed, 3); } // Xbox 360 float16 uses extended range: exponent 31 is NOT Inf/NaN // but a valid large value (up to +/-131008). Standard PackHalf2x16 // clamps to +/-65504 and produces Inf for larger values. This helper // detects where standard conversion overflowed to Inf/NaN and // re-encodes those values using the extended range representation: // halve the value, convert to standard f16 (giving exponent <= 30), // then increment the exponent by 1 (placing it in the exponent 31 // slot that Xbox 360 treats as a normal value, not Inf/NaN). // Operates on a float2 packed via PackHalf2x16 into a uint32, with // per-lane overflow detection and selection. auto pack_half_2x16_extended_range = [&](spv::Id float2_value) -> spv::Id { // Standard f32 to f16 conversion (handles +/-0..65504 correctly). spv::Id standard = builder_->createUnaryBuiltinCall(type_uint_, ext_inst_glsl_std_450_, GLSLstd450PackHalf2x16, float2_value); // Detect where standard conversion produced Inf or NaN // (exponent field = 31, i.e. bits [14:10] all set = 0x7C00) // in each 16-bit lane of the packed result. spv::Id const_0x7C00 = builder_->makeUintConstant(0x7C00); spv::Id lower_exp = builder_->createBinOp(spv::OpBitwiseAnd, type_uint_, standard, const_0x7C00); spv::Id lower_overflow = builder_->createBinOp(spv::OpIEqual, type_bool_, lower_exp, const_0x7C00); spv::Id upper_bits = builder_->createBinOp(spv::OpShiftRightLogical, type_uint_, standard, builder_->makeUintConstant(16)); spv::Id upper_exp = builder_->createBinOp(spv::OpBitwiseAnd, type_uint_, upper_bits, const_0x7C00); spv::Id upper_overflow = builder_->createBinOp(spv::OpIEqual, type_bool_, upper_exp, const_0x7C00); // For values that overflowed, compute extended range encoding: // 1. Clamp to +/-131008.0 (max extended float16 can represent). spv::Id const_131008 = builder_->makeFloatConstant(131008.0f); spv::Id const_neg_131008 = builder_->makeFloatConstant(-131008.0f); id_vector_temp_.clear(); id_vector_temp_.push_back(const_neg_131008); id_vector_temp_.push_back(const_neg_131008); spv::Id const_neg_131008_vec2 = builder_->makeCompositeConstant(type_float2_, id_vector_temp_); id_vector_temp_.clear(); id_vector_temp_.push_back(const_131008); id_vector_temp_.push_back(const_131008); spv::Id const_131008_vec2 = builder_->makeCompositeConstant(type_float2_, id_vector_temp_); spv::Id clamped = builder_->createTriBuiltinCall( type_float2_, ext_inst_glsl_std_450_, GLSLstd450FClamp, float2_value, const_neg_131008_vec2, const_131008_vec2); // 2. Halve to bring into standard float16 range (max 65504). spv::Id const_half = builder_->makeFloatConstant(0.5f); id_vector_temp_.clear(); id_vector_temp_.push_back(const_half); id_vector_temp_.push_back(const_half); spv::Id const_half_vec2 = builder_->makeCompositeConstant(type_float2_, id_vector_temp_); spv::Id halved = builder_->createBinOp(spv::OpFMul, type_float2_, clamped, const_half_vec2); // 3. Convert the halved value (will have exponent <= 30). spv::Id halved_packed = builder_->createUnaryBuiltinCall( type_uint_, ext_inst_glsl_std_450_, GLSLstd450PackHalf2x16, halved); // 4. Increment exponent by 1 in both lanes (add 0x0400 to each // 16-bit half = 0x04000400) to compensate for the halving. spv::Id extended = builder_->createBinOp(spv::OpIAdd, type_uint_, halved_packed, builder_->makeUintConstant(0x04000400)); // Select: use extended result where standard gave Inf/NaN, // keep standard result otherwise. Per-lane selection via masking. spv::Id const_0xFFFF = builder_->makeUintConstant(0xFFFF); spv::Id const_0xFFFF0000 = builder_->makeUintConstant(0xFFFF0000); spv::Id result_lower = builder_->createTriOp( spv::OpSelect, type_uint_, lower_overflow, builder_->createBinOp(spv::OpBitwiseAnd, type_uint_, extended, const_0xFFFF), builder_->createBinOp(spv::OpBitwiseAnd, type_uint_, standard, const_0xFFFF)); spv::Id result_upper = builder_->createTriOp( spv::OpSelect, type_uint_, upper_overflow, builder_->createBinOp(spv::OpBitwiseAnd, type_uint_, extended, const_0xFFFF0000), builder_->createBinOp(spv::OpBitwiseAnd, type_uint_, standard, const_0xFFFF0000)); return builder_->createBinOp(spv::OpBitwiseOr, type_uint_, result_lower, result_upper); }; // k_16_FLOAT format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_16_FLOAT)); { EMIdArray format_packed_16_float; for_each_eM([&](uint32_t eM_index) { id_vector_temp_.clear(); id_vector_temp_.push_back(builder_->createCompositeExtract( eM_swapped[eM_index], type_float_, 0)); id_vector_temp_.push_back(const_float_0_); spv::Id format_packed_16_float_x = pack_half_2x16_extended_range( builder_->createCompositeConstruct(type_float2_, id_vector_temp_)); id_vector_temp_.clear(); id_vector_temp_.resize(4, const_uint_0_); id_vector_temp_.front() = format_packed_16_float_x; format_packed_16_float[eM_index] = builder_->createCompositeConstruct(type_uint4_, id_vector_temp_); }); add_format_case(format_packed_16_float, 1); } // k_16_16_FLOAT format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_16_16_FLOAT)); { EMIdArray format_packed_16_16_float; for_each_eM([&](uint32_t eM_index) { uint_vector_temp_.clear(); uint_vector_temp_.push_back(0); uint_vector_temp_.push_back(1); spv::Id format_packed_16_16_float_xy = pack_half_2x16_extended_range(builder_->createRvalueSwizzle( spv::NoPrecision, type_float2_, eM_swapped[eM_index], uint_vector_temp_)); id_vector_temp_.clear(); id_vector_temp_.resize(4, const_uint_0_); id_vector_temp_.front() = format_packed_16_16_float_xy; format_packed_16_16_float[eM_index] = builder_->createCompositeConstruct(type_uint4_, id_vector_temp_); }); add_format_case(format_packed_16_16_float, 2); } // k_16_16_16_16_FLOAT format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_16_16_16_16_FLOAT)); { EMIdArray format_packed_16_16_16_16_float; for_each_eM([&](uint32_t eM_index) { spv::Id format_packed_16_16_16_16_float_xy_zw[2]; for (uint32_t component_index = 0; component_index < 2; ++component_index) { uint_vector_temp_.clear(); uint_vector_temp_.push_back(2 * component_index); uint_vector_temp_.push_back(2 * component_index + 1); format_packed_16_16_16_16_float_xy_zw[component_index] = pack_half_2x16_extended_range(builder_->createRvalueSwizzle( spv::NoPrecision, type_float2_, eM_swapped[eM_index], uint_vector_temp_)); } id_vector_temp_.clear(); id_vector_temp_.push_back(format_packed_16_16_16_16_float_xy_zw[0]); id_vector_temp_.push_back(format_packed_16_16_16_16_float_xy_zw[1]); id_vector_temp_.push_back(const_uint_0_); id_vector_temp_.push_back(const_uint_0_); format_packed_16_16_16_16_float[eM_index] = builder_->createCompositeConstruct(type_uint4_, id_vector_temp_); }); add_format_case(format_packed_16_16_16_16_float, 3); } // k_32_FLOAT format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_32_FLOAT)); { EMIdArray format_packed_32_float; for_each_eM([&](uint32_t eM_index) { format_packed_32_float[eM_index] = builder_->createUnaryOp( spv::OpBitcast, type_uint4_, eM_swapped[eM_index]); }); add_format_case(format_packed_32_float, 2); } // k_32_32_FLOAT format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_32_32_FLOAT)); { EMIdArray format_packed_32_32_float; for_each_eM([&](uint32_t eM_index) { format_packed_32_32_float[eM_index] = builder_->createUnaryOp( spv::OpBitcast, type_uint4_, eM_swapped[eM_index]); }); add_format_case(format_packed_32_32_float, 3); } // k_32_32_32_32_FLOAT format_switch.makeBeginCase( static_cast(xenos::ColorFormat::k_32_32_32_32_FLOAT)); { EMIdArray format_packed_32_32_32_32_float; for_each_eM([&](uint32_t eM_index) { format_packed_32_32_32_32_float[eM_index] = builder_->createUnaryOp( spv::OpBitcast, type_uint4_, eM_swapped[eM_index]); }); add_format_case(format_packed_32_32_32_32_float, 4); } format_switch.makeEndSwitch(); // Select the result and the element size based on the format. // Phi must be the first instructions in a block. EMIdArray eM_packed; for_each_eM([&](uint32_t eM_index) { auto eM_packed_phi = std::make_unique( builder_->getUniqueId(), type_uint4_, spv::OpPhi); // Default case for an invalid format. eM_packed_phi->addIdOperand(const_uint4_0_); eM_packed_phi->addIdOperand(format_switch.getDefaultPhiParent()); for (const FormatCase& format_case : format_cases) { eM_packed_phi->addIdOperand(format_case.eM_packed[eM_index]); eM_packed_phi->addIdOperand(format_case.phi_parent); } eM_packed[eM_index] = eM_packed_phi->getResultId(); builder_->getBuildPoint()->addInstruction(std::move(eM_packed_phi)); }); spv::Id element_bytes_log2; { auto element_bytes_log2_phi = std::make_unique( builder_->getUniqueId(), type_uint_, spv::OpPhi); // Default case for an invalid format (doesn't enter any element size // conditional, skipped). element_bytes_log2_phi->addIdOperand(builder_->makeUintConstant(5)); element_bytes_log2_phi->addIdOperand(format_switch.getDefaultPhiParent()); for (const FormatCase& format_case : format_cases) { element_bytes_log2_phi->addIdOperand( builder_->makeUintConstant(format_case.element_bytes_log2)); element_bytes_log2_phi->addIdOperand(format_case.phi_parent); } element_bytes_log2 = element_bytes_log2_phi->getResultId(); builder_->getBuildPoint()->addInstruction( std::move(element_bytes_log2_phi)); } // Endian-swap. spv::Id endian = builder_->createTriOp(spv::OpBitFieldUExtract, type_uint_, format_info, const_uint_0_, builder_->makeUintConstant(3)); for_each_eM([&](uint32_t eM_index) { eM_packed[eM_index] = EndianSwap128Uint4(eM_packed[eM_index], endian); }); // Load the index of eM0 in the stream. spv::Id eM0_index = builder_->createTriOp( spv::OpBitFieldUExtract, type_uint_, builder_->createCompositeExtract(eA_vector, type_uint_, 1), const_uint_0_, builder_->makeUintConstant(23)); // Check how many elements starting from eM0 are within the bounds of the // stream, and from the eM# that were written, exclude the out-of-bounds ones. // The index can't be negative, and the index and the count are limited to 23 // bits, so it's safe to use 32-bit signed subtraction and clamping to get the // remaining eM# count. spv::Id eM_indices_to_store = builder_->createTriOp( spv::OpBitFieldUExtract, type_uint_, builder_->createLoad(var_main_memexport_data_written_, spv::NoPrecision), const_uint_0_, builder_->createUnaryOp( spv::OpBitcast, type_uint_, builder_->createTriBuiltinCall( type_int_, ext_inst_glsl_std_450_, GLSLstd450SClamp, builder_->createBinOp( spv::OpISub, type_int_, builder_->createUnaryOp( spv::OpBitcast, type_int_, builder_->createTriOp(spv::OpBitFieldUExtract, type_uint_, builder_->createCompositeExtract( eA_vector, type_uint_, 3), const_uint_0_, builder_->makeUintConstant(23))), builder_->createUnaryOp(spv::OpBitcast, type_int_, eM0_index)), const_int_0_, builder_->makeIntConstant(ucode::kMaxMemExportElementCount)))); // Get the eM0 address in bytes. // Left-shift the stream base address by 2 to both convert it from dwords to // bytes and drop the upper bits. spv::Id const_uint_2 = builder_->makeUintConstant(2); spv::Id eM0_address_bytes = builder_->createBinOp( spv::OpIAdd, type_uint_, builder_->createBinOp( spv::OpShiftLeftLogical, type_uint_, builder_->createCompositeExtract(eA_vector, type_uint_, 0), const_uint_2), builder_->createBinOp(spv::OpShiftLeftLogical, type_uint_, eM0_index, element_bytes_log2)); // Store based on the element size. auto store_needed_eM = [&](std::function fn) { for_each_eM([&](uint32_t eM_index) { SpirvBuilder::IfBuilder if_eM_needed( builder_->createBinOp( spv::OpINotEqual, type_bool_, builder_->createBinOp(spv::OpBitwiseAnd, type_uint_, eM_indices_to_store, builder_->makeUintConstant(1u << eM_index)), const_uint_0_), spv::SelectionControlDontFlattenMask, *builder_, 2, 1); fn(eM_index); if_eM_needed.makeEndIf(); }); }; SpirvBuilder::SwitchBuilder element_size_switch( element_bytes_log2, spv::SelectionControlDontFlattenMask, *builder_); element_size_switch.makeBeginCase(0); { store_needed_eM([&](uint32_t eM_index) { spv::Id element_address_bytes = eM_index != 0 ? builder_->createBinOp( spv::OpIAdd, type_uint_, eM0_address_bytes, builder_->makeUintConstant(eM_index)) : eM0_address_bytes; // replace_shift = 8 * (element_address_bytes & 3) spv::Id replace_shift = builder_->createQuadOp( spv::OpBitFieldInsert, type_uint_, const_uint_0_, element_address_bytes, builder_->makeUintConstant(3), const_uint_2); StoreUint32ToSharedMemory( builder_->createBinOp(spv::OpShiftLeftLogical, type_uint_, builder_->createCompositeExtract( eM_packed[eM_index], type_uint_, 0), replace_shift), builder_->createUnaryOp( spv::OpBitcast, type_int_, builder_->createBinOp(spv::OpShiftRightLogical, type_uint_, element_address_bytes, const_uint_2)), builder_->createBinOp(spv::OpShiftLeftLogical, type_uint_, builder_->makeUintConstant(0xFFu), replace_shift)); }); } element_size_switch.makeBeginCase(1); { spv::Id const_uint_1 = builder_->makeUintConstant(1); spv::Id eM0_address_words = builder_->createBinOp( spv::OpShiftRightLogical, type_uint_, eM0_address_bytes, const_uint_1); store_needed_eM([&](uint32_t eM_index) { spv::Id element_address_words = eM_index != 0 ? builder_->createBinOp( spv::OpIAdd, type_uint_, eM0_address_words, builder_->makeUintConstant(eM_index)) : eM0_address_words; // replace_shift = 16 * (element_address_words & 1) spv::Id replace_shift = builder_->createQuadOp( spv::OpBitFieldInsert, type_uint_, const_uint_0_, element_address_words, builder_->makeUintConstant(4), const_uint_1); StoreUint32ToSharedMemory( builder_->createBinOp(spv::OpShiftLeftLogical, type_uint_, builder_->createCompositeExtract( eM_packed[eM_index], type_uint_, 0), replace_shift), builder_->createUnaryOp( spv::OpBitcast, type_int_, builder_->createBinOp(spv::OpShiftRightLogical, type_uint_, element_address_words, const_uint_1)), builder_->createBinOp(spv::OpShiftLeftLogical, type_uint_, builder_->makeUintConstant(0xFFFFu), replace_shift)); }); } element_size_switch.makeBeginCase(2); { spv::Id eM0_address_dwords = builder_->createBinOp( spv::OpShiftRightLogical, type_uint_, eM0_address_bytes, const_uint_2); store_needed_eM([&](uint32_t eM_index) { StoreUint32ToSharedMemory( builder_->createCompositeExtract(eM_packed[eM_index], type_uint_, 0), builder_->createUnaryOp( spv::OpBitcast, type_int_, eM_index != 0 ? builder_->createBinOp( spv::OpIAdd, type_uint_, eM0_address_dwords, builder_->makeUintConstant(eM_index)) : eM0_address_dwords)); }); } element_size_switch.makeBeginCase(3); { spv::Id eM0_address_dwords = builder_->createBinOp( spv::OpShiftRightLogical, type_uint_, eM0_address_bytes, const_uint_2); store_needed_eM([&](uint32_t eM_index) { spv::Id element_value = eM_packed[eM_index]; spv::Id element_address_dwords_int = builder_->createUnaryOp( spv::OpBitcast, type_int_, eM_index != 0 ? builder_->createBinOp( spv::OpIAdd, type_uint_, eM0_address_dwords, builder_->makeUintConstant(2 * eM_index)) : eM0_address_dwords); StoreUint32ToSharedMemory( builder_->createCompositeExtract(element_value, type_uint_, 0), element_address_dwords_int); StoreUint32ToSharedMemory( builder_->createCompositeExtract(element_value, type_uint_, 1), builder_->createBinOp(spv::OpIAdd, type_int_, element_address_dwords_int, builder_->makeIntConstant(1))); }); } element_size_switch.makeBeginCase(4); { spv::Id eM0_address_dwords = builder_->createBinOp( spv::OpShiftRightLogical, type_uint_, eM0_address_bytes, const_uint_2); store_needed_eM([&](uint32_t eM_index) { spv::Id element_value = eM_packed[eM_index]; spv::Id element_address_dwords_int = builder_->createUnaryOp( spv::OpBitcast, type_int_, eM_index != 0 ? builder_->createBinOp( spv::OpIAdd, type_uint_, eM0_address_dwords, builder_->makeUintConstant(4 * eM_index)) : eM0_address_dwords); StoreUint32ToSharedMemory( builder_->createCompositeExtract(element_value, type_uint_, 0), element_address_dwords_int); for (uint32_t element_dword_index = 1; element_dword_index < 4; ++element_dword_index) { StoreUint32ToSharedMemory( builder_->createCompositeExtract(element_value, type_uint_, element_dword_index), builder_->createBinOp(spv::OpIAdd, type_int_, element_address_dwords_int, builder_->makeIntConstant( static_cast(element_dword_index)))); } }); } element_size_switch.makeEndSwitch(); // Close the conditionals for whether memory export is allowed in this // invocation. if_address_valid.makeEndIf(); if (if_pixel_not_killed.has_value()) { if_pixel_not_killed->makeEndIf(); } if (if_memexport_allowed.has_value()) { if_memexport_allowed->makeEndIf(); } } } // namespace gpu } // namespace xe