[Vulkan] Route guest oDepth through gl_FragDepth in FBO and into FSI depth test

src/xenia/gpu/spirv_shader_translator.cc

@@ -145,6 +145,8 @@ void SpirvShaderTranslator::Reset() {
   var_main_fsi_color_written_ = spv::NoResult;
   std::fill(output_fragment_data_.begin(), output_fragment_data_.end(),
             spv::NoResult);
+  output_or_var_fragment_depth_ = spv::NoResult;
+  output_fragment_depth_ = spv::NoResult;
 
   main_switch_op_.reset();
   main_switch_next_pc_phi_operands_.clear();
@@ -723,7 +725,7 @@ std::vector<uint8_t> SpirvShaderTranslator::CompleteTranslation() {
       builder_->addExecutionMode(function_main_,
                                  spv::ExecutionModeEarlyFragmentTests);
     }
-    if (current_shader().writes_depth()) {
+    if (current_shader().writes_depth() && !edram_fragment_shader_interlock_) {
       builder_->addExecutionMode(function_main_,
                                  spv::ExecutionModeDepthReplacing);
     }
@@ -2285,6 +2287,19 @@ void SpirvShaderTranslator::StartFragmentShaderBeforeMain() {
     }
   }
 
+  // Fragment depth output (gl_FragDepth) for the FBO path.
+  // Created when the guest pixel shader writes oDepth.
+  // FSI manages its own depth and does not need an Output.
+  if (!edram_fragment_shader_interlock_ && !is_depth_only_fragment_shader_ &&
+      current_shader().writes_depth()) {
+    output_fragment_depth_ = builder_->createVariable(
+        spv::NoPrecision, spv::StorageClassOutput, type_float_, "gl_FragDepth");
+    builder_->addDecoration(output_fragment_depth_, spv::DecorationBuiltIn,
+                            static_cast<int>(spv::BuiltIn::FragDepth));
+    builder_->addDecoration(output_fragment_depth_, spv::DecorationInvariant);
+    main_interface_.push_back(output_fragment_depth_);
+  }
+
   // Sample mask output for alpha-to-coverage.
   // Only needed for non-FSI mode. FSI uses main_fsi_sample_mask_ instead.
   output_fragment_sample_mask_ = spv::NoResult;
@@ -2361,14 +2376,14 @@ void SpirvShaderTranslator::StartFragmentShaderInMain() {
         "xe_var_color_written", const_uint_0_);
   }
 
-  if (edram_fragment_shader_interlock_) {
+  // Staging variable for guest oDepth writes.
-    // Initialize depth output variable with fragment shader interlock.
+  // Created whenever the shader uses oDepth:
-    output_or_var_fragment_depth_ = spv::NoResult;
+  //   * FSI reads it during its EDRAM depth write inside the interlock.
-    if (current_shader().writes_depth()) {
+  //   * FBO copies it to gl_FragDepth at the end of the shader.
-      output_or_var_fragment_depth_ = builder_->createVariable(
+  if (current_shader().writes_depth()) {
-          spv::NoPrecision, spv::StorageClassFunction, type_float_,
+    output_or_var_fragment_depth_ = builder_->createVariable(
-          "xe_var_fragment_depth", const_float_0_);
+        spv::NoPrecision, spv::StorageClassFunction, type_float_,
-    }
+        "xe_var_fragment_depth", const_float_0_);
   }
 
   if (edram_fragment_shader_interlock_ && FSI_IsDepthStencilEarly()) {
@@ -2974,6 +2989,26 @@ void SpirvShaderTranslator::StoreResult(const InstructionResult& result,
             var_main_memexport_data_written_);
       }
     } break;
+    case InstructionStorageTarget::kDepth: {
+      // oDepth is scalar. The FBO path copies it to gl_FragDepth (in
+      // CompleteFragmentShader_DSV_DepthTo24Bit), while FSI writes it to a
+      // depth variable consumed by FSI_DepthStencilTest.
+      assert_true(is_pixel_shader());
+      assert_true(used_write_mask == 0b0001);
+      assert_true(current_shader().writes_depth());
+      assert_true(output_or_var_fragment_depth_ != spv::NoResult);
+      target_pointer = output_or_var_fragment_depth_;
+      // Depth outside [0, 1] needs to be clamped for safety, similar to D3D12.
+      // Though 20e4 float depth can store values between 1 and 2, it's a very
+      // unusual case. In Vulkan, gl_FragDepth can accept any values when the
+      // depth buffer is floating-point, but we clamp for consistency.
+      // Clamp the depth value to [0, 1] if not already saturated.
+      if (value != spv::NoResult && !result.is_clamped) {
+        value = builder_->createTriBuiltinCall(
+            type_float_, ext_inst_glsl_std_450_, GLSLstd450NClamp, value,
+            const_float_0_, const_float_1_);
+      }
+    } break;
     default:
       // TODO(Triang3l): All storage targets.
       break;

src/xenia/gpu/spirv_shader_translator.h

@@ -563,6 +563,11 @@ class SpirvShaderTranslator : public ShaderTranslator {
   void StartFragmentShaderInMain();
   void CompleteFragmentShaderInMain();
 
+  // Writes gl_FragDepth at the end of an FBO pixel shader, remapping the
+  // guest 0...1 oDepth value to host 0...0.5 when the bound depth buffer is
+  // float24. No-op for FSI and for shaders that don't write oDepth.
+  void CompleteFragmentShader_DSV_DepthTo24Bit();
+
   // Updates the current flow control condition (to be called in the beginning
   // of exec and in jumps), closing the previous conditionals if needed.
   // However, if the condition is not different, the instruction-level predicate
@@ -960,11 +965,16 @@ class SpirvShaderTranslator : public ShaderTranslator {
   // output_or_var_fragment_data_.
   std::array<spv::Id, xenos::kMaxColorRenderTargets> output_fragment_data_;
 
-  // Fragment shader depth output (gl_FragDepth).
+  // Function-scoped staging variable for guest oDepth writes. Used by both
-  // With fragment shader interlock, a variable in the main function.
+  // FSI (which writes the value to the EDRAM buffer inside the interlock)
-  // Otherwise, the depth output (only created if shader writes depth).
+  // and FBO (copied to output_fragment_depth_ at the end of the shader,
+  // remapping guest 0...1 to host 0...0.5 when the depth format is float24).
   spv::Id output_or_var_fragment_depth_;
 
+  // FBO only: actual gl_FragDepth Output.
+  // Written at the end of the pixel shader from output_or_var_fragment_depth_.
+  spv::Id output_fragment_depth_;
+
   // Fragment shader sample mask output (gl_SampleMask).
   // Only used for alpha-to-coverage in non-FSI mode.
   // For FSI mode, sample mask is handled via main_fsi_sample_mask_.

src/xenia/gpu/spirv_shader_translator_rb.cc

@@ -1483,6 +1483,10 @@ void SpirvShaderTranslator::CompleteFragmentShaderInMain() {
     }
   }
 
+  // Copies the staged depth to the FBO gl_FragDepth output.
+  // No-op for FSI and when no host depth output was declared.
+  CompleteFragmentShader_DSV_DepthTo24Bit();
+
   if (edram_fragment_shader_interlock_) {
     if (block_fsi_if_after_depth_stencil_merge) {
       builder_->createBranch(block_fsi_if_after_depth_stencil_merge);
@@ -1503,6 +1507,37 @@ void SpirvShaderTranslator::CompleteFragmentShaderInMain() {
   }
 }
 
+void SpirvShaderTranslator::CompleteFragmentShader_DSV_DepthTo24Bit() {
+  // FSI manages its own depth via the EDRAM buffer - this hook is FBO-only.
+  // Likewise, if no Output FragDepth was declared, there is nothing to write.
+  if (edram_fragment_shader_interlock_ ||
+      output_fragment_depth_ == spv::NoResult) {
+    return;
+  }
+  if (!current_shader().writes_depth()) {
+    return;
+  }
+  assert_true(output_or_var_fragment_depth_ != spv::NoResult);
+  // The shader writes depth explicitly, for float24, need to scale it from
+  // guest 0...1 to host 0...0.5 to support reinterpretation round trips as
+  // viewport scaling doesn't apply to oDepth.
+  spv::Id depth_value =
+      builder_->createLoad(output_or_var_fragment_depth_, spv::NoPrecision);
+  spv::Id depth_float24_flag = builder_->createBinOp(
+      spv::OpINotEqual, type_bool_,
+      builder_->createBinOp(spv::OpBitwiseAnd, type_uint_,
+                            main_system_constant_flags_,
+                            builder_->makeUintConstant(kSysFlag_DepthFloat24)),
+      const_uint_0_);
+  spv::Id depth_scaled = builder_->createBinOp(
+      spv::OpFMul, type_float_, depth_value, builder_->makeFloatConstant(0.5f));
+  spv::Id depth_remapped =
+      builder_->createTriOp(spv::OpSelect, type_float_, depth_float24_flag,
+                            depth_scaled, depth_value);
+  // Write the depth from the temporary to the system depth output.
+  builder_->createStore(depth_remapped, output_fragment_depth_);
+}
+
 spv::Id SpirvShaderTranslator::LoadMsaaSamplesFromFlags() {
   return builder_->createTriOp(
       spv::OpBitFieldUExtract, type_uint_, main_system_constant_flags_,
@@ -1798,21 +1833,35 @@ void SpirvShaderTranslator::FSI_DepthStencilTest(
   SpirvBuilder::IfBuilder if_depth_stencil_enabled(
       depth_stencil_enabled, spv::SelectionControlDontFlattenMask, *builder_);
 
-  // Load the depth in the center of the pixel and calculate the derivatives of
+  spv::Id center_depth32_unbiased;
-  // the depth outside non-uniform control flow.
-  assert_true(input_fragment_coordinates_ != spv::NoResult);
-  id_vector_temp_.clear();
-  id_vector_temp_.push_back(builder_->makeIntConstant(2));
-  spv::Id center_depth32_unbiased = builder_->createLoad(
-      builder_->createAccessChain(spv::StorageClassInput,
-                                  input_fragment_coordinates_, id_vector_temp_),
-      spv::NoPrecision);
-  builder_->addCapability(spv::CapabilityDerivativeControl);
   std::array<spv::Id, 2> depth_dxy;
-  depth_dxy[0] = builder_->createUnaryOp(spv::OpDPdxCoarse, type_float_,
+
-                                         center_depth32_unbiased);
+  // Guest oDepth replaces the depth value FSI tests. It's not the raster depth
-  depth_dxy[1] = builder_->createUnaryOp(spv::OpDPdyCoarse, type_float_,
+  // plane, so don't take FragCoord for it.
-                                         center_depth32_unbiased);
+  if (current_shader().writes_depth()) {
+    assert_false(is_early);
+    assert_true(output_or_var_fragment_depth_ != spv::NoResult);
+    center_depth32_unbiased =
+        builder_->createLoad(output_or_var_fragment_depth_, spv::NoPrecision);
+    depth_dxy[0] = const_float_0_;
+    depth_dxy[1] = const_float_0_;
+  } else {
+    // Load the depth in the center of the pixel and calculate the derivatives
+    // of the depth outside non-uniform control flow.
+    assert_true(input_fragment_coordinates_ != spv::NoResult);
+    id_vector_temp_.clear();
+    id_vector_temp_.push_back(builder_->makeIntConstant(2));
+    center_depth32_unbiased =
+        builder_->createLoad(builder_->createAccessChain(
+                                 spv::StorageClassInput,
+                                 input_fragment_coordinates_, id_vector_temp_),
+                             spv::NoPrecision);
+    builder_->addCapability(spv::CapabilityDerivativeControl);
+    depth_dxy[0] = builder_->createUnaryOp(spv::OpDPdxCoarse, type_float_,
+                                           center_depth32_unbiased);
+    depth_dxy[1] = builder_->createUnaryOp(spv::OpDPdyCoarse, type_float_,
+                                           center_depth32_unbiased);
+  }
 
   // Skip everything if potentially discarded all the samples previously in the
   // shader.
@@ -1981,27 +2030,36 @@ void SpirvShaderTranslator::FSI_DepthStencilTest(
                             builder_->makeUintConstant(uint32_t(1) << 2)),
       const_uint_0_);
 
-  // Get the maximum depth slope for the polygon offset.
+  spv::Id center_depth32_biased;
-  // https://docs.microsoft.com/en-us/windows/desktop/direct3d9/depth-bias
+
-  std::array<spv::Id, 2> depth_dxy_abs;
+  // When the guest shader replaces depth, don't apply offset from the original
-  for (uint32_t i = 0; i < 2; ++i) {
+  // FragCoord.z plane. That would mix two different depth sources.
-    depth_dxy_abs[i] = builder_->createUnaryBuiltinCall(
+  if (current_shader().writes_depth()) {
-        type_float_, ext_inst_glsl_std_450_, GLSLstd450FAbs, depth_dxy[i]);
+    center_depth32_biased = center_depth32_unbiased;
+  } else {
+    // Get the maximum depth slope for the polygon offset.
+    // https://docs.microsoft.com/en-us/windows/desktop/direct3d9/depth-bias
+    std::array<spv::Id, 2> depth_dxy_abs;
+    for (uint32_t i = 0; i < 2; ++i) {
+      depth_dxy_abs[i] = builder_->createUnaryBuiltinCall(
+          type_float_, ext_inst_glsl_std_450_, GLSLstd450FAbs, depth_dxy[i]);
+    }
+    spv::Id depth_max_slope = builder_->createBinBuiltinCall(
+        type_float_, ext_inst_glsl_std_450_, GLSLstd450FMax, depth_dxy_abs[0],
+        depth_dxy_abs[1]);
+    // Calculate the polygon offset.
+    spv::Id slope_scaled_poly_offset = builder_->createNoContractionBinOp(
+        spv::OpFMul, type_float_, poly_offset_scale, depth_max_slope);
+    spv::Id poly_offset = builder_->createNoContractionBinOp(
+        spv::OpFAdd, type_float_, slope_scaled_poly_offset, poly_offset_offset);
+    // Apply the post-clip and post-viewport polygon offset to the fragment's
+    // depth. Not clamping yet as this is at the center, which is not
+    // necessarily covered and not necessarily inside the bounds - derivatives
+    // scaled by sample locations will be added to this value, and it must be
+    // linear.
+    center_depth32_biased = builder_->createNoContractionBinOp(
+        spv::OpFAdd, type_float_, center_depth32_unbiased, poly_offset);
   }
-  spv::Id depth_max_slope = builder_->createBinBuiltinCall(
-      type_float_, ext_inst_glsl_std_450_, GLSLstd450FMax, depth_dxy_abs[0],
-      depth_dxy_abs[1]);
-  // Calculate the polygon offset.
-  spv::Id slope_scaled_poly_offset = builder_->createNoContractionBinOp(
-      spv::OpFMul, type_float_, poly_offset_scale, depth_max_slope);
-  spv::Id poly_offset = builder_->createNoContractionBinOp(
-      spv::OpFAdd, type_float_, slope_scaled_poly_offset, poly_offset_offset);
-  // Apply the post-clip and post-viewport polygon offset to the fragment's
-  // depth. Not clamping yet as this is at the center, which is not necessarily
-  // covered and not necessarily inside the bounds - derivatives scaled by
-  // sample locations will be added to this value, and it must be linear.
-  spv::Id center_depth32_biased = builder_->createNoContractionBinOp(
-      spv::OpFAdd, type_float_, center_depth32_unbiased, poly_offset);
 
   // Perform depth and stencil testing for each covered sample.
   spv::Id new_sample_mask = main_fsi_sample_mask_;