[GPU] PS and rasterization disabling cleanup

src/xenia/gpu/draw_util.cc

@@ -111,6 +111,78 @@ int32_t FloatToD3D11Fixed16p8(float f32) {
   return result.s;
 }
 
+bool IsRasterizationPotentiallyDone(const RegisterFile& regs,
+                                    bool primitive_polygonal) {
+  // TODO(Triang3l): Investigate ModeControl::kIgnore better, with respect to
+  // sample counting. Let's assume sample counting is a part of depth / stencil,
+  // thus disabled too.
+  xenos::ModeControl edram_mode = regs.Get<reg::RB_MODECONTROL>().edram_mode;
+  if (edram_mode != xenos::ModeControl::kColorDepth &&
+      edram_mode != xenos::ModeControl::kDepth) {
+    return false;
+  }
+  auto sq_program_cntl = regs.Get<reg::SQ_PROGRAM_CNTL>();
+  if (sq_program_cntl.vs_export_mode ==
+      xenos::VertexShaderExportMode::kMultipass) {
+    return false;
+  }
+  if (primitive_polygonal) {
+    auto pa_su_sc_mode_cntl = regs.Get<reg::PA_SU_SC_MODE_CNTL>();
+    if (pa_su_sc_mode_cntl.cull_front && pa_su_sc_mode_cntl.cull_back) {
+      // Both faces are culled.
+      return false;
+    }
+  }
+  return true;
+}
+
+bool IsPixelShaderNeededWithRasterization(const Shader& shader,
+                                          const RegisterFile& regs) {
+  assert_true(shader.type() == xenos::ShaderType::kPixel);
+  assert_true(shader.is_ucode_analyzed());
+
+  // See xenos::ModeControl for explanation why the pixel shader is only used
+  // when it's kColorDepth here.
+  if (regs.Get<reg::RB_MODECONTROL>().edram_mode !=
+      xenos::ModeControl::kColorDepth) {
+    return false;
+  }
+
+  // Discarding (explicitly or through alphatest or alpha to coverage) has side
+  // effects on pixel counting.
+  //
+  // Depth output only really matters if depth test is active, but it's used
+  // extremely rarely, and pretty much always intentionally - for simplicity,
+  // consider it as always mattering.
+  //
+  // Memory export is an obvious intentional side effect.
+  if (shader.kills_pixels() || shader.writes_depth() ||
+      !shader.memexport_stream_constants().empty() ||
+      (shader.writes_color_target(0) &&
+       DoesCoverageDependOnAlpha(regs.Get<reg::RB_COLORCONTROL>()))) {
+    return true;
+  }
+
+  // Check if a color target is actually written.
+  uint32_t rb_color_mask = regs[XE_GPU_REG_RB_COLOR_MASK].u32;
+  uint32_t rts_remaining = shader.writes_color_targets();
+  uint32_t rt_index;
+  while (xe::bit_scan_forward(rts_remaining, &rt_index)) {
+    rts_remaining &= ~(uint32_t(1) << rt_index);
+    uint32_t format_component_count = GetColorRenderTargetFormatComponentCount(
+        regs.Get<reg::RB_COLOR_INFO>(
+                reg::RB_COLOR_INFO::rt_register_indices[rt_index])
+            .color_format);
+    if ((rb_color_mask >> (rt_index * 4)) &
+        ((uint32_t(1) << format_component_count) - 1)) {
+      return true;
+    }
+  }
+
+  // Only depth / stencil passthrough potentially.
+  return false;
+}
+
 void GetHostViewportInfo(const RegisterFile& regs, float pixel_size_x,
                          float pixel_size_y, bool origin_bottom_left,
                          float x_max, float y_max, bool allow_reverse_z,
@@ -271,7 +343,8 @@ void GetHostViewportInfo(const RegisterFile& regs, float pixel_size_x,
     ndc_scale_z = -ndc_scale_z;
     ndc_offset_z = 1.0f - ndc_offset_z;
   }
-  if (convert_z_to_float24 && regs.Get<reg::RB_DEPTHCONTROL>().z_enable &&
+  if (convert_z_to_float24 &&
+      GetDepthControlForCurrentEdramMode(regs).z_enable &&
       regs.Get<reg::RB_DEPTH_INFO>().depth_format ==
           xenos::DepthRenderTargetFormat::kD24FS8) {
     // Need to adjust the bounds that the resulting depth values will be clamped