[D3D12] Force early Z with DSV, fix blend disabled flag in rb_colorcontrol ignored

src/xenia/gpu/d3d12/d3d12_command_processor.cc

@@ -2199,8 +2199,8 @@ void D3D12CommandProcessor::UpdateSystemConstantValues(
 
   // Alpha test.
   int32_t alpha_test;
-  if (rb_colorcontrol & 0x8) {
-    uint32_t alpha_test_function = rb_colorcontrol & 0x7;
+  uint32_t alpha_test_function = rb_colorcontrol & 0x7;
+  if ((rb_colorcontrol & 0x8) && alpha_test_function != 0x7) {
     // 0: Never - fail in [-inf, +inf].
     // 1: Less - fail in [ref, +inf].
     // 2: Equal - pass in [ref, ref].

src/xenia/gpu/d3d12/d3d12_shader.h

@@ -40,6 +40,17 @@ class D3D12Shader : public Shader {
       const DxbcShaderTranslator::SamplerBinding* sampler_bindings,
       uint32_t sampler_binding_count);
 
+  void SetForcedEarlyZShaderObject(const std::vector<uint8_t>& shader_object) {
+    forced_early_z_shader_ = shader_object;
+  }
+  // Returns the shader with forced early depth/stencil set with
+  // SetForcedEarlyZShader after translation. If there's none (for example,
+  // if the shader discards pixels or writes to the depth buffer), an empty
+  // vector is returned.
+  const std::vector<uint8_t>& GetForcedEarlyZShaderObject() const {
+    return forced_early_z_shader_;
+  }
+
   bool DisassembleDxbc(const ui::d3d12::D3D12Provider* provider);
 
   static constexpr uint32_t kMaxTextureSRVIndexBits =
@@ -78,9 +89,12 @@ class D3D12Shader : public Shader {
 
  private:
   PrimitiveType domain_shader_primitive_type_ = PrimitiveType::kNone;
+
   std::vector<TextureSRV> texture_srvs_;
   uint32_t used_texture_mask_ = 0;
   std::vector<SamplerBinding> sampler_bindings_;
+
+  std::vector<uint8_t> forced_early_z_shader_;
 };
 
 }  // namespace d3d12

src/xenia/gpu/d3d12/pipeline_cache.cc

@@ -331,6 +331,15 @@ bool PipelineCache::TranslateShader(D3D12Shader* shader,
              shader->ucode_disassembly().c_str());
   }
 
+  // If may be useful, create a version of the shader with early depth/stencil
+  // forced.
+  if (shader->type() == ShaderType::kPixel && !edram_rov_used_ &&
+      shader->early_z_allowed()) {
+    shader->SetForcedEarlyZShaderObject(
+        std::move(DxbcShaderTranslator::ForceEarlyDepthStencil(
+            shader->translated_binary().data())));
+  }
+
   // Disassemble the shader for dumping.
   if (FLAGS_d3d12_dxbc_disasm) {
     auto provider = command_processor_->GetD3D12Context()->GetD3D12Provider();
@@ -569,6 +578,8 @@ bool PipelineCache::GetCurrentStateDescription(
   }
 
   if (!edram_rov_used_) {
+    uint32_t rb_colorcontrol = regs[XE_GPU_REG_RB_COLORCONTROL].u32;
+
     // Depth/stencil. No stencil, always passing depth test and no depth writing
     // means depth disabled.
     if (render_targets[4].format != DXGI_FORMAT_UNKNOWN) {
@@ -616,6 +627,16 @@ bool PipelineCache::GetCurrentStateDescription(
       description_out.depth_func = 0b111;
     }
 
+    // Forced early Z if the shader allows that and alpha testing is disabled.
+    // TODO(Triang3l): For memexporting shaders, possibly choose this according
+    // to the early Z toggle in RB_DEPTHCONTROL (the correct behavior is still
+    // unknown).
+    if (pixel_shader != nullptr &&
+        pixel_shader->GetForcedEarlyZShaderObject().size() != 0 &&
+        (!(rb_colorcontrol & 0x8) || (rb_colorcontrol & 0x7) == 0x7)) {
+      description_out.force_early_z = 1;
+    }
+
     // Render targets and blending state. 32 because of 0x1F mask, for safety
     // (all unknown to zero).
     uint32_t color_mask = command_processor_->GetCurrentColorMask(pixel_shader);
@@ -695,7 +716,7 @@ bool PipelineCache::GetCurrentStateDescription(
       rt.format = RenderTargetCache::GetBaseColorFormat(
           ColorRenderTargetFormat((color_info >> 16) & 0xF));
       rt.write_mask = (color_mask >> (guest_rt_index * 4)) & 0xF;
-      if (rt.write_mask) {
+      if (!(rb_colorcontrol & 0x20) && rt.write_mask) {
         rt.src_blend = kBlendFactorMap[blendcontrol & 0x1F];
         rt.dest_blend = kBlendFactorMap[(blendcontrol >> 8) & 0x1F];
         rt.blend_op = BlendOp((blendcontrol >> 5) & 0x7);
@@ -874,10 +895,17 @@ ID3D12PipelineState* PipelineCache::CreatePipelineState(
       assert_always();
       return nullptr;
     }
-    state_desc.PS.pShaderBytecode =
-        description.pixel_shader->translated_binary().data();
-    state_desc.PS.BytecodeLength =
-        description.pixel_shader->translated_binary().size();
+    const auto& forced_early_z_shader =
+        description.pixel_shader->GetForcedEarlyZShaderObject();
+    if (description.force_early_z && forced_early_z_shader.size() != 0) {
+      state_desc.PS.pShaderBytecode = forced_early_z_shader.data();
+      state_desc.PS.BytecodeLength = forced_early_z_shader.size();
+    } else {
+      state_desc.PS.pShaderBytecode =
+          description.pixel_shader->translated_binary().data();
+      state_desc.PS.BytecodeLength =
+          description.pixel_shader->translated_binary().size();
+    }
   } else if (edram_rov_used_) {
     state_desc.PS.pShaderBytecode = depth_only_pixel_shader_.data();
     state_desc.PS.BytecodeLength = depth_only_pixel_shader_.size();

src/xenia/gpu/d3d12/pipeline_cache.h

@@ -155,6 +155,7 @@ class PipelineCache {
     uint32_t depth_write : 1;                                   // 21
     uint32_t stencil_enable : 1;                                // 22
     uint32_t stencil_read_mask : 8;                             // 30
+    uint32_t force_early_z : 1;                                 // 31
 
     uint32_t stencil_write_mask : 8;           // 8
     uint32_t stencil_front_fail_op : 3;        // 11