[Vulkan] fix FBO path reading from output variables for alpha test

Makes FBO follow the same pattern as FSI by using function scoped
intermediate variables

src/xenia/gpu/spirv_shader_translator.cc

@@ -143,6 +143,8 @@ void SpirvShaderTranslator::Reset() {
   var_main_point_size_edge_flag_kill_vertex_ = spv::NoResult;
   var_main_kill_pixel_ = spv::NoResult;
   var_main_fsi_color_written_ = spv::NoResult;
+  std::fill(output_fragment_data_.begin(), output_fragment_data_.end(),
+            spv::NoResult);
 
   main_switch_op_.reset();
   main_switch_next_pc_phi_operands_.clear();
@@ -2234,10 +2236,14 @@ void SpirvShaderTranslator::StartFragmentShaderBeforeMain() {
   }
 
   if (!is_depth_only_fragment_shader_) {
-    // Framebuffer color attachment outputs.
+    // Framebuffer color attachment outputs (FBO path only).
+    // For FBO, we create Output variables here and Function-scoped variables
+    // in StartFragmentShaderInMain. The Function-scoped variables are used
+    // throughout the shader (so we can read them for alpha test), and copied
+    // to the Output variables at the end.
     if (!edram_fragment_shader_interlock_) {
-      std::fill(output_or_var_fragment_data_.begin(),
+      std::fill(output_fragment_data_.begin(), output_fragment_data_.end(),
-                output_or_var_fragment_data_.end(), spv::NoResult);
+                spv::NoResult);
       static const char* const kFragmentDataOutputNames[] = {
           "xe_out_fragment_data_0",
           "xe_out_fragment_data_1",
@@ -2253,8 +2259,7 @@ void SpirvShaderTranslator::StartFragmentShaderBeforeMain() {
         spv::Id output_fragment_data_rt = builder_->createVariable(
             spv::NoPrecision, spv::StorageClassOutput, type_float4_,
             kFragmentDataOutputNames[color_target_index]);
-        output_or_var_fragment_data_[color_target_index] =
+        output_fragment_data_[color_target_index] = output_fragment_data_rt;
-            output_fragment_data_rt;
         builder_->addDecoration(output_fragment_data_rt,
                                 spv::DecorationLocation,
                                 int(color_target_index));
@@ -2311,33 +2316,45 @@ void SpirvShaderTranslator::StartFragmentShaderInMain() {
     // to the execution mask GPUs naturally have.
   }
 
+  // Initialize color output variables as Function-scoped for both FSI and FBO.
+  // For FBO, this allows reading the color values back (e.g., for alpha test),
+  // which isn't possible with Output storage class. The values are copied to
+  // the actual Output variables at the end of the shader for FBO.
+  std::fill(output_or_var_fragment_data_.begin(),
+            output_or_var_fragment_data_.end(), spv::NoResult);
+  var_main_fsi_color_written_ = spv::NoResult;
+  uint32_t color_targets_written = current_shader().writes_color_targets();
+  if (color_targets_written && !is_depth_only_fragment_shader_) {
+    static const char* const kFragmentDataVariableNames[] = {
+        "xe_var_fragment_data_0",
+        "xe_var_fragment_data_1",
+        "xe_var_fragment_data_2",
+        "xe_var_fragment_data_3",
+    };
+    uint32_t color_targets_remaining = color_targets_written;
+    uint32_t color_target_index;
+    while (xe::bit_scan_forward(color_targets_remaining, &color_target_index)) {
+      color_targets_remaining &= ~(UINT32_C(1) << color_target_index);
+      output_or_var_fragment_data_[color_target_index] =
+          builder_->createVariable(
+              spv::NoPrecision, spv::StorageClassFunction, type_float4_,
+              kFragmentDataVariableNames[color_target_index], const_float4_0_);
+    }
+    // Color write tracking for both FSI and FBO paths.
+    // This is used to conditionally skip alpha test / alpha-to-coverage if
+    // render target 0 wasn't written on the execution path.
+    var_main_fsi_color_written_ = builder_->createVariable(
+        spv::NoPrecision, spv::StorageClassFunction, type_uint_,
+        "xe_var_color_written", const_uint_0_);
+  }
+
   if (edram_fragment_shader_interlock_) {
-    // Initialize color output variables with fragment shader interlock.
+    // Initialize depth output variable with fragment shader interlock.
-    std::fill(output_or_var_fragment_data_.begin(),
+    output_or_var_fragment_depth_ = spv::NoResult;
-              output_or_var_fragment_data_.end(), spv::NoResult);
+    if (current_shader().writes_depth()) {
-    var_main_fsi_color_written_ = spv::NoResult;
+      output_or_var_fragment_depth_ = builder_->createVariable(
-    uint32_t color_targets_written = current_shader().writes_color_targets();
+          spv::NoPrecision, spv::StorageClassFunction, type_float_,
-    if (color_targets_written) {
+          "xe_var_fragment_depth", const_float_0_);
-      static const char* const kFragmentDataVariableNames[] = {
-          "xe_var_fragment_data_0",
-          "xe_var_fragment_data_1",
-          "xe_var_fragment_data_2",
-          "xe_var_fragment_data_3",
-      };
-      uint32_t color_targets_remaining = color_targets_written;
-      uint32_t color_target_index;
-      while (
-          xe::bit_scan_forward(color_targets_remaining, &color_target_index)) {
-        color_targets_remaining &= ~(UINT32_C(1) << color_target_index);
-        output_or_var_fragment_data_[color_target_index] =
-            builder_->createVariable(
-                spv::NoPrecision, spv::StorageClassFunction, type_float4_,
-                kFragmentDataVariableNames[color_target_index],
-                const_float4_0_);
-      }
-      var_main_fsi_color_written_ = builder_->createVariable(
-          spv::NoPrecision, spv::StorageClassFunction, type_uint_,
-          "xe_var_fsi_color_written", const_uint_0_);
     }
   }
 
@@ -2554,16 +2571,6 @@ void SpirvShaderTranslator::StartFragmentShaderInMain() {
                                          spv::StorageClassFunction,
                                          var_main_registers_, id_vector_temp_));
   }
-
-  if (!edram_fragment_shader_interlock_) {
-    // Initialize the colors for safety.
-    for (uint32_t i = 0; i < xenos::kMaxColorRenderTargets; ++i) {
-      spv::Id output_fragment_data_rt = output_or_var_fragment_data_[i];
-      if (output_fragment_data_rt != spv::NoResult) {
-        builder_->createStore(const_float4_0_, output_fragment_data_rt);
-      }
-    }
-  }
 }
 
 void SpirvShaderTranslator::UpdateExecConditionals(
@@ -2923,8 +2930,7 @@ void SpirvShaderTranslator::StoreResult(const InstructionResult& result,
       assert_not_zero(used_write_mask);
       assert_true(current_shader().writes_color_target(result.storage_index));
       target_pointer = output_or_var_fragment_data_[result.storage_index];
-      if (edram_fragment_shader_interlock_) {
+      if (var_main_fsi_color_written_ != spv::NoResult) {
-        assert_true(var_main_fsi_color_written_ != spv::NoResult);
         builder_->createStore(
             builder_->createBinOp(
                 spv::OpBitwiseOr, type_uint_,

src/xenia/gpu/spirv_shader_translator.h

@@ -947,11 +947,23 @@ class SpirvShaderTranslator : public ShaderTranslator {
   unsigned int output_per_vertex_clip_distance_member_index_ = 0;
   unsigned int output_per_vertex_cull_distance_member_index_ = 0;
 
-  // With fragment shader interlock, variables in the main function.
+  // Function-scoped variables for fragment color data.
-  // Otherwise, framebuffer color attachment outputs.
+  // Used by both FSI and FBO paths so that color values can be read back
+  // (e.g., for alpha test). For FBO, these are copied to output_fragment_data_
+  // at the end of the shader.
   std::array<spv::Id, xenos::kMaxColorRenderTargets>
       output_or_var_fragment_data_;
 
+  // FBO only: Actual framebuffer color attachment outputs (Output storage).
+  // These are write-only and populated at the end of the shader from
+  // output_or_var_fragment_data_.
+  std::array<spv::Id, xenos::kMaxColorRenderTargets> output_fragment_data_;
+
+  // Fragment shader depth output (gl_FragDepth).
+  // With fragment shader interlock, a variable in the main function.
+  // Otherwise, the depth output (only created if shader writes depth).
+  spv::Id output_or_var_fragment_depth_;
+
   // Fragment shader sample mask output (gl_SampleMask).
   // Only used for alpha-to-coverage in non-FSI mode.
   // For FSI mode, sample mask is handled via main_fsi_sample_mask_.
@@ -999,11 +1011,12 @@ class SpirvShaderTranslator : public ShaderTranslator {
   spv::Id var_main_point_size_edge_flag_kill_vertex_;
   // PS, only when needed - bool.
   spv::Id var_main_kill_pixel_;
-  // PS, only when writing to color render targets with fragment shader
+  // PS, when writing to color render targets - uint.
-  // interlock - uint.
   // Whether color buffers have been written to, if not written on the taken
   // execution path, don't export according to Direct3D 9 register documentation
   // (some games rely on this behavior).
+  // Used by both FSI and FBO paths for proper alpha test / alpha-to-coverage
+  // behavior.
   spv::Id var_main_fsi_color_written_;
   // Loaded by FSI_LoadSampleMask.
   // Can be modified on the outermost control flow level in the main function.

src/xenia/gpu/spirv_shader_translator_rb.cc

@@ -472,13 +472,15 @@ void SpirvShaderTranslator::CompleteFragmentShaderInMain() {
 
   if ((color_targets_written & 0b1) && !IsExecutionModeEarlyFragmentTests()) {
     spv::Id fsi_sample_mask_in_rt_0_alpha_tests = spv::NoResult;
-    spv::Block* block_fsi_rt_0_alpha_tests_rt_written_head = nullptr;
+    spv::Block* block_rt_0_alpha_tests_rt_written_head = nullptr;
-    spv::Block* block_fsi_rt_0_alpha_tests_rt_written_merge = nullptr;
+    spv::Block* block_rt_0_alpha_tests_rt_written_merge = nullptr;
     builder_->makeNewBlock();
-    if (edram_fragment_shader_interlock_) {
+    if (var_main_fsi_color_written_ != spv::NoResult) {
       // Skip the alpha test and alpha to coverage if the render target 0 is not
-      // written to dynamically.
+      // written to dynamically. This check is used by both FSI and FBO paths.
-      fsi_sample_mask_in_rt_0_alpha_tests = main_fsi_sample_mask_;
+      if (edram_fragment_shader_interlock_) {
+        fsi_sample_mask_in_rt_0_alpha_tests = main_fsi_sample_mask_;
+      }
       spv::Id rt_0_written = builder_->createBinOp(
           spv::OpINotEqual, type_bool_,
           builder_->createBinOp(
@@ -487,32 +489,30 @@ void SpirvShaderTranslator::CompleteFragmentShaderInMain() {
                                    spv::NoPrecision),
               builder_->makeUintConstant(0b1)),
           const_uint_0_);
-      block_fsi_rt_0_alpha_tests_rt_written_head = builder_->getBuildPoint();
+      block_rt_0_alpha_tests_rt_written_head = builder_->getBuildPoint();
-      spv::Block& block_fsi_rt_0_alpha_tests_rt_written =
+      spv::Block& block_rt_0_alpha_tests_rt_written = builder_->makeNewBlock();
-          builder_->makeNewBlock();
+      block_rt_0_alpha_tests_rt_written_merge = &builder_->makeNewBlock();
-      block_fsi_rt_0_alpha_tests_rt_written_merge = &builder_->makeNewBlock();
+      builder_->createSelectionMerge(block_rt_0_alpha_tests_rt_written_merge,
-      builder_->createSelectionMerge(
+                                     spv::SelectionControlDontFlattenMask);
-          block_fsi_rt_0_alpha_tests_rt_written_merge,
-          spv::SelectionControlDontFlattenMask);
       {
         std::unique_ptr<spv::Instruction> rt_0_written_branch_conditional_op =
             std::make_unique<spv::Instruction>(spv::OpBranchConditional);
         rt_0_written_branch_conditional_op->addIdOperand(rt_0_written);
         rt_0_written_branch_conditional_op->addIdOperand(
-            block_fsi_rt_0_alpha_tests_rt_written.getId());
+            block_rt_0_alpha_tests_rt_written.getId());
         rt_0_written_branch_conditional_op->addIdOperand(
-            block_fsi_rt_0_alpha_tests_rt_written_merge->getId());
+            block_rt_0_alpha_tests_rt_written_merge->getId());
         // More likely to write to the render target 0 than not.
         rt_0_written_branch_conditional_op->addImmediateOperand(2);
         rt_0_written_branch_conditional_op->addImmediateOperand(1);
         builder_->getBuildPoint()->addInstruction(
             std::move(rt_0_written_branch_conditional_op));
       }
-      block_fsi_rt_0_alpha_tests_rt_written.addPredecessor(
+      block_rt_0_alpha_tests_rt_written.addPredecessor(
-          block_fsi_rt_0_alpha_tests_rt_written_head);
+          block_rt_0_alpha_tests_rt_written_head);
-      block_fsi_rt_0_alpha_tests_rt_written_merge->addPredecessor(
+      block_rt_0_alpha_tests_rt_written_merge->addPredecessor(
-          block_fsi_rt_0_alpha_tests_rt_written_head);
+          block_rt_0_alpha_tests_rt_written_head);
-      builder_->setBuildPoint(&block_fsi_rt_0_alpha_tests_rt_written);
+      builder_->setBuildPoint(&block_rt_0_alpha_tests_rt_written);
     }
 
     // Alpha test.
@@ -533,10 +533,9 @@ void SpirvShaderTranslator::CompleteFragmentShaderInMain() {
       id_vector_temp_.clear();
       id_vector_temp_.push_back(builder_->makeIntConstant(3));
       spv::Id alpha_test_alpha = builder_->createLoad(
-          builder_->createAccessChain(
+          builder_->createAccessChain(spv::StorageClassFunction,
-              edram_fragment_shader_interlock_ ? spv::StorageClassFunction
+                                      output_or_var_fragment_data_[0],
-                                               : spv::StorageClassOutput,
+                                      id_vector_temp_),
-              output_or_var_fragment_data_[0], id_vector_temp_),
           spv::NoPrecision);
       id_vector_temp_.clear();
       id_vector_temp_.push_back(
@@ -627,22 +626,23 @@ void SpirvShaderTranslator::CompleteFragmentShaderInMain() {
     // Alpha to coverage.
     FSI_AlphaToMask();
 
-    if (edram_fragment_shader_interlock_) {
+    if (block_rt_0_alpha_tests_rt_written_merge) {
-      // Close the render target 0 written check.
+      // Close the render target 0 written check (used by both FSI and FBO).
-      builder_->createBranch(block_fsi_rt_0_alpha_tests_rt_written_merge);
+      builder_->createBranch(block_rt_0_alpha_tests_rt_written_merge);
-      spv::Block& block_fsi_rt_0_alpha_tests_rt_written_end =
+      spv::Block& block_rt_0_alpha_tests_rt_written_end =
           *builder_->getBuildPoint();
-      builder_->setBuildPoint(block_fsi_rt_0_alpha_tests_rt_written_merge);
+      builder_->setBuildPoint(block_rt_0_alpha_tests_rt_written_merge);
-      if (!features_.demote_to_helper_invocation) {
+      if (edram_fragment_shader_interlock_ &&
+          !features_.demote_to_helper_invocation) {
         // The tests might have modified the sample mask via
         // fsi_sample_mask_in_rt_0_alpha_tests.
         id_vector_temp_.clear();
         id_vector_temp_.push_back(fsi_sample_mask_in_rt_0_alpha_tests);
         id_vector_temp_.push_back(
-            block_fsi_rt_0_alpha_tests_rt_written_end.getId());
+            block_rt_0_alpha_tests_rt_written_end.getId());
         id_vector_temp_.push_back(main_fsi_sample_mask_);
         id_vector_temp_.push_back(
-            block_fsi_rt_0_alpha_tests_rt_written_head->getId());
+            block_rt_0_alpha_tests_rt_written_head->getId());
         main_fsi_sample_mask_ =
             builder_->createOp(spv::OpPhi, type_uint_, id_vector_temp_);
       }
@@ -1297,6 +1297,23 @@ void SpirvShaderTranslator::CompleteFragmentShaderInMain() {
     }
   }
 
+  if (!edram_fragment_shader_interlock_) {
+    // FBO path: Copy from Function-scoped variables to Output variables.
+    // This is done at the end after alpha test/coverage so we can read the
+    // color values during those operations.
+    uint32_t color_targets_to_copy = current_shader().writes_color_targets();
+    uint32_t color_target_index;
+    while (xe::bit_scan_forward(color_targets_to_copy, &color_target_index)) {
+      color_targets_to_copy &= ~(UINT32_C(1) << color_target_index);
+      spv::Id var_color = output_or_var_fragment_data_[color_target_index];
+      spv::Id out_color = output_fragment_data_[color_target_index];
+      if (var_color != spv::NoResult && out_color != spv::NoResult) {
+        builder_->createStore(builder_->createLoad(var_color, spv::NoPrecision),
+                              out_color);
+      }
+    }
+  }
+
   if (edram_fragment_shader_interlock_) {
     if (block_fsi_if_after_depth_stencil_merge) {
       builder_->createBranch(block_fsi_if_after_depth_stencil_merge);
@@ -3808,10 +3825,9 @@ void SpirvShaderTranslator::FSI_AlphaToMask() {
   id_vector_temp_.clear();
   id_vector_temp_.push_back(builder_->makeIntConstant(3));  // W component
   spv::Id alpha = builder_->createLoad(
-      builder_->createAccessChain(
+      builder_->createAccessChain(spv::StorageClassFunction,
-          edram_fragment_shader_interlock_ ? spv::StorageClassFunction
+                                  output_or_var_fragment_data_[0],
-                                           : spv::StorageClassOutput,
+                                  id_vector_temp_),
-          output_or_var_fragment_data_[0], id_vector_temp_),
       spv::NoPrecision);
 
   // Load MSAA sample count to determine which mode to use.