/**
 ******************************************************************************
 * Xenia : Xbox 360 Emulator Research Project                                 *
 ******************************************************************************
 * Copyright 2022 Ben Vanik. All rights reserved.                             *
 * Released under the BSD license - see LICENSE in the root for more details. *
 ******************************************************************************
 */

#include "xenia/gpu/command_processor.h"

#include "third_party/fmt/include/fmt/format.h"
#include "xenia/base/byte_stream.h"
#include "xenia/base/clock.h"
#include "xenia/base/cvar.h"
#include "xenia/base/logging.h"
#include "xenia/base/profiling.h"
#include "xenia/gpu/gpu_flags.h"
#include "xenia/gpu/graphics_system.h"
#include "xenia/gpu/packet_disassembler.h"
#include "xenia/gpu/sampler_info.h"
#include "xenia/gpu/texture_info.h"
#include "xenia/gpu/xenos_zpd_report.h"
#include "xenia/kernel/kernel_state.h"
#include "xenia/kernel/user_module.h"
#if !defined(NDEBUG)

#define XE_ENABLE_GPU_REG_WRITE_LOGGING 1
#endif
DEFINE_bool(
    log_guest_driven_gpu_register_written_values, false,
    "Only does anything in debug builds, if set will log every write to a gpu "
    "register done by a guest. Does not log writes that are done by the CP on "
    "its own, just ones the guest makes or instructs it to make.",
    "GPU");

DEFINE_bool(disassemble_pm4, false,
            "Only does anything in debug builds, if set will disassemble and "
            "log all PM4 packets sent to the CP.",
            "GPU");

DEFINE_bool(
    log_ringbuffer_kickoff_initiator_bts, false,
    "Only does anything in debug builds, if set will log the pseudo-stacktrace "
    "of the guest thread that wrote the new read position.",
    "GPU");

DEFINE_bool(clear_memory_page_state, false,
            "Refresh state of memory pages to enable gpu written data. (Use "
            "for 'Team Ninja' Games to fix missing character models)",
            "GPU");

DEFINE_string(
    occlusion_query, "fast",
    "Controls hardware occlusion query behavior for EVENT_WRITE_ZPD.\n"
    "Used for effects like lens flares, object culling, and auto-exposure.\n"
    " fake: Write a fake result without asking the GPU. Safe for most games,\n"
    "       though some effects may look slightly wrong.\n"
    " fast: Ask the GPU but don't wait for the answer. Writes a cached\n"
    "       result immediately and updates it when the GPU catches up.\n"
    "       Cached results bias toward visible when guessing. (default)\n"
    " fast-alt: Variant of fast mode that keeps cached zero results for\n"
    "           unresolved reports. May improve effects relying on precise\n"
    "           visibility, but may be less stable for occlusion culling.\n"
    " strict: Ask the GPU and wait for the real result before continuing.\n"
    "         Most accurate, but may be somewhat less performant.",
    "GPU");

DEFINE_string(
    readback_resolve, "none",
    "Controls CPU readback of render-to-texture resolve results.\n"
    " fast: Read from previous frame (1 frame delay, no GPU stall, slight "
    "performance hit)\n"
    " full: Wait for GPU to finish (accurate but slow, GPU-CPU sync stall)\n"
    " none: Disable readback completely (some games render better without it)",
    "GPU");

UPDATE_from_string(readback_resolve, 2025, 12, 4, 21, "fast");

DEFINE_bool(
    readback_memexport, false,
    "Read data written by memory export in shaders on the CPU. "
    "This may be needed in some games (but many only access exported data on "
    "the GPU, and this flag isn't needed to handle such behavior), but causes "
    "mid-frame synchronization, so it has a huge performance impact.",
    "GPU");

namespace xe {
namespace gpu {

// This should be written completely differently with support for different
// types.
void SaveGPUSetting(GPUSetting setting, uint64_t value) {
  switch (setting) {
    case GPUSetting::ClearMemoryPageState:
      OVERRIDE_bool(clear_memory_page_state, static_cast<bool>(value));
      break;
    case GPUSetting::ReadbackMemexport:
      OVERRIDE_bool(readback_memexport, static_cast<bool>(value));
      break;
  }
}

bool GetGPUSetting(GPUSetting setting) {
  switch (setting) {
    case GPUSetting::ClearMemoryPageState:
      return cvars::clear_memory_page_state;
    case GPUSetting::ReadbackMemexport:
      return cvars::readback_memexport;
  }
  return false;
}

ReadbackResolveMode GetReadbackResolveMode() {
  const std::string& mode = cvars::readback_resolve;
  if (mode == "full") {
    return ReadbackResolveMode::kFull;
  } else if (mode == "none") {
    return ReadbackResolveMode::kDisabled;
  } else {
    // Default to "fast" for any unrecognized value
    return ReadbackResolveMode::kFast;
  }
}

void SetReadbackResolveMode(const std::string& mode) {
  OVERRIDE_string(readback_resolve, mode);
}

ZPDMode GetZPDMode() {
  const std::string& mode = cvars::occlusion_query;
  if (mode == "fake") {
    return ZPDMode::kFake;
  } else if (mode == "strict") {
    return ZPDMode::kStrict;
  } else if (mode == "fast-alt") {
    return ZPDMode::kFastAlt;
  }
  return ZPDMode::kFast;
}

void SetZPDMode(const std::string& mode) {
  OVERRIDE_string(occlusion_query, mode);
}

using namespace xe::gpu::xenos;

CommandProcessor::CommandProcessor(GraphicsSystem* graphics_system,
                                   kernel::KernelState* kernel_state)
    : reader_(nullptr, 0),
      memory_(graphics_system->memory()),
      kernel_state_(kernel_state),
      graphics_system_(graphics_system),
      register_file_(graphics_system_->register_file()),
      trace_writer_(graphics_system->memory()->physical_membase()),
      worker_running_(true),
      write_ptr_index_event_(xe::threading::Event::CreateAutoResetEvent(false)),
      write_ptr_index_(0) {
  assert_not_null(write_ptr_index_event_);
}

CommandProcessor::~CommandProcessor() = default;

bool CommandProcessor::Initialize() {
  // Initialize the gamma ramps to their default (linear) values - taken from
  // what games set when starting with the sRGB (return value 1)
  // VdGetCurrentDisplayGamma.
  for (uint32_t i = 0; i < 256; ++i) {
    uint32_t value = i * 0x3FF / 0xFF;
    reg::DC_LUT_30_COLOR& gamma_ramp_entry = gamma_ramp_256_entry_table_[i];
    gamma_ramp_entry.color_10_blue = value;
    gamma_ramp_entry.color_10_green = value;
    gamma_ramp_entry.color_10_red = value;
  }
  for (uint32_t i = 0; i < 128; ++i) {
    reg::DC_LUT_PWL_DATA gamma_ramp_entry = {};
    gamma_ramp_entry.base = (i * 0xFFFF / 0x7F) & ~UINT32_C(0x3F);
    gamma_ramp_entry.delta = i < 0x7F ? 0x200 : 0;
    for (uint32_t j = 0; j < 3; ++j) {
      gamma_ramp_pwl_rgb_[i][j] = gamma_ramp_entry;
    }
  }

  worker_running_ = true;
  worker_thread_ =
      kernel::object_ref<kernel::XHostThread>(new kernel::XHostThread(
          kernel_state_, 128 * 1024, 0,
          [this]() {
            WorkerThreadMain();
            return 0;
          },
          kernel_state_->GetIdleProcess()));
  worker_thread_->set_name("GPU Commands");
  worker_thread_->Create();

  return true;
}

void CommandProcessor::Shutdown() {
  EndTracing();

  worker_running_ = false;
  write_ptr_index_event_->Set();
  worker_thread_->Wait(0, 0, 0, nullptr);
  worker_thread_.reset();
}

void CommandProcessor::InitializeShaderStorage(
    const std::filesystem::path& cache_root, uint32_t title_id, bool blocking,
    std::function<void()> completion_callback) {
  if (completion_callback) {
    completion_callback();
  }
}

void CommandProcessor::RequestFrameTrace(
    const std::filesystem::path& root_path) {
  if (trace_state_ == TraceState::kStreaming) {
    XELOGE("Streaming trace; cannot also trace frame.");
    return;
  }
  if (trace_state_ == TraceState::kSingleFrame) {
    XELOGE("Frame trace already pending; ignoring.");
    return;
  }
  trace_state_ = TraceState::kSingleFrame;
  trace_frame_path_ = root_path;
}

void CommandProcessor::BeginTracing(const std::filesystem::path& root_path) {
  if (trace_state_ == TraceState::kStreaming) {
    XELOGE("Streaming already active; ignoring request.");
    return;
  }
  if (trace_state_ == TraceState::kSingleFrame) {
    XELOGE("Frame trace pending; ignoring streaming request.");
    return;
  }
  // Streaming starts on the next primary buffer execute.
  trace_state_ = TraceState::kStreaming;
  trace_stream_path_ = root_path;
}

void CommandProcessor::EndTracing() {
  if (!trace_writer_.is_open()) {
    return;
  }
  assert_true(trace_state_ == TraceState::kStreaming);
  trace_state_ = TraceState::kDisabled;
  trace_writer_.Close();
}

void CommandProcessor::RestoreRegisters(uint32_t first_register,
                                        const uint32_t* register_values,
                                        uint32_t register_count,
                                        bool execute_callbacks) {
  if (first_register > RegisterFile::kRegisterCount ||
      RegisterFile::kRegisterCount - first_register < register_count) {
    XELOGW(
        "CommandProcessor::RestoreRegisters out of bounds (0x{:X} registers "
        "starting with 0x{:X}, while a total of 0x{:X} registers are stored)",
        register_count, first_register, RegisterFile::kRegisterCount);
    if (first_register > RegisterFile::kRegisterCount) {
      return;
    }
    register_count =
        std::min(uint32_t(RegisterFile::kRegisterCount) - first_register,
                 register_count);
  }
  if (execute_callbacks) {
    for (uint32_t i = 0; i < register_count; ++i) {
      WriteRegister(first_register + i, register_values[i]);
    }
  } else {
    std::memcpy(register_file_->values + first_register, register_values,
                sizeof(uint32_t) * register_count);
  }
}

void CommandProcessor::RestoreGammaRamp(
    const reg::DC_LUT_30_COLOR* new_gamma_ramp_256_entry_table,
    const reg::DC_LUT_PWL_DATA* new_gamma_ramp_pwl_rgb,
    uint32_t new_gamma_ramp_rw_component) {
  std::memcpy(gamma_ramp_256_entry_table_, new_gamma_ramp_256_entry_table,
              sizeof(reg::DC_LUT_30_COLOR) * 256);
  std::memcpy(gamma_ramp_pwl_rgb_, new_gamma_ramp_pwl_rgb,
              sizeof(reg::DC_LUT_PWL_DATA) * 3 * 128);
  gamma_ramp_rw_component_ = new_gamma_ramp_rw_component;
  OnGammaRamp256EntryTableValueWritten();
  OnGammaRampPWLValueWritten();
}

void CommandProcessor::CallInThread(std::function<void()> fn) {
  if (pending_fns_.empty() &&
      kernel::XThread::IsInThread(worker_thread_.get())) {
    fn();
  } else {
    pending_fns_.push(std::move(fn));
  }
}

void CommandProcessor::ClearCaches() {}

void CommandProcessor::SetDesiredSwapPostEffect(
    SwapPostEffect swap_post_effect) {
  if (swap_post_effect_desired_ == swap_post_effect) {
    return;
  }
  swap_post_effect_desired_ = swap_post_effect;
  CallInThread([this, swap_post_effect]() {
    swap_post_effect_actual_ = swap_post_effect;
  });
}

void CommandProcessor::WorkerThreadMain() {
  if (!SetupContext()) {
    xe::FatalError("Unable to setup command processor internal state");
    return;
  }

  while (worker_running_) {
    while (!pending_fns_.empty()) {
      auto fn = std::move(pending_fns_.front());
      pending_fns_.pop();
      fn();
    }

    uint32_t write_ptr_index = write_ptr_index_.load();
    if (write_ptr_index == 0xBAADF00D || read_ptr_index_ == write_ptr_index) {
      SCOPE_profile_cpu_i("gpu", "xe::gpu::CommandProcessor::Stall");
      // We've run out of commands to execute.
      // We spin here waiting for new ones, as the overhead of waiting on our
      // event is too high.
      PrepareForWait();
      uint32_t loop_count = 0;
      do {
        // If we spin around too much, revert to a "low-power" state.
        if (loop_count > 500) {
          constexpr int wait_time_ms = 2;
          xe::threading::Wait(write_ptr_index_event_.get(), true,
                              std::chrono::milliseconds(wait_time_ms));
        } else {
          xe::threading::MaybeYield();
        }
        loop_count++;
        write_ptr_index = write_ptr_index_.load();
      } while (worker_running_ && pending_fns_.empty() &&
               (write_ptr_index == 0xBAADF00D ||
                read_ptr_index_ == write_ptr_index));
      ReturnFromWait();
      if (!worker_running_ || !pending_fns_.empty()) {
        continue;
      }
    }
    assert_true(read_ptr_index_ != write_ptr_index);

    // Execute. Note that we handle wraparound transparently.
    read_ptr_index_ = ExecutePrimaryBuffer(read_ptr_index_, write_ptr_index);

    // TODO(benvanik): use reader->Read_update_freq_ and only issue after moving
    //     that many indices.
    // Keep in mind that the gpu also updates the cpu-side copy if the write
    // pointer and read pointer would be equal
    if (read_ptr_writeback_ptr_) {
      xe::store_and_swap<uint32_t>(
          memory_->TranslatePhysical(read_ptr_writeback_ptr_), read_ptr_index_);
    }

    // FIXME: We're supposed to process the WAIT_UNTIL register at this point,
    // but no games seem to actually use it.
  }

  ShutdownContext();
}

void CommandProcessor::Pause() {
  if (paused_) {
    return;
  }
  paused_ = true;

  threading::Fence fence;
  CallInThread([&fence]() {
    fence.Signal();
    threading::Thread::GetCurrentThread()->Suspend();
  });

  fence.Wait();
}

void CommandProcessor::Resume() {
  if (!paused_) {
    return;
  }
  paused_ = false;

  worker_thread_->thread()->Resume();
}

bool CommandProcessor::Save(ByteStream* stream) {
  assert_true(paused_);

  stream->Write<uint32_t>(primary_buffer_ptr_);
  stream->Write<uint32_t>(primary_buffer_size_);
  stream->Write<uint32_t>(read_ptr_index_);
  stream->Write<uint32_t>(read_ptr_update_freq_);
  stream->Write<uint32_t>(read_ptr_writeback_ptr_);
  stream->Write<uint32_t>(write_ptr_index_.load());

  return true;
}

bool CommandProcessor::Restore(ByteStream* stream) {
  assert_true(paused_);

  primary_buffer_ptr_ = stream->Read<uint32_t>();
  primary_buffer_size_ = stream->Read<uint32_t>();
  read_ptr_index_ = stream->Read<uint32_t>();
  read_ptr_update_freq_ = stream->Read<uint32_t>();
  read_ptr_writeback_ptr_ = stream->Read<uint32_t>();
  write_ptr_index_.store(stream->Read<uint32_t>());

  return true;
}

bool CommandProcessor::SetupContext() {
  ResetZPDState();
  return true;
}

void CommandProcessor::ShutdownContext() { ResetZPDState(); }

void CommandProcessor::InitializeRingBuffer(uint32_t ptr, uint32_t size_log2) {
  read_ptr_index_ = 0;
  primary_buffer_ptr_ = ptr;
  primary_buffer_size_ = uint32_t(1) << (size_log2 + 3);

  std::memset(kernel_state_->memory()->TranslatePhysical(primary_buffer_ptr_),
              0, primary_buffer_size_);
}

void CommandProcessor::EnableReadPointerWriteBack(uint32_t ptr,
                                                  uint32_t block_size_log2) {
  // CP_RB_RPTR_ADDR Ring Buffer Read Pointer Address 0x70C
  // ptr = RB_RPTR_ADDR, pointer to write back the address to.
  read_ptr_writeback_ptr_ = ptr;
  // CP_RB_CNTL Ring Buffer Control 0x704
  // block_size = RB_BLKSZ, log2 of number of quadwords read between updates of
  //              the read pointer.
  read_ptr_update_freq_ = uint32_t(1) << block_size_log2 >> 2;
}

XE_NOINLINE XE_COLD void CommandProcessor::LogKickoffInitator(uint32_t value) {
  cpu::backend::GuestPseudoStackTrace st;

  if (logging::ShouldLog(LogLevel::Debug) &&
      kernel_state_->processor()->backend()->PopulatePseudoStacktrace(&st)) {
    logging::LoggerBatch<LogLevel::Debug> log_initiator{};

    log_initiator("Updating read ptr to {}, initiator stacktrace below\n",
                  value);

    for (uint32_t i = 0; i < st.count; ++i) {
      log_initiator("\t{:08X}\n", st.return_addrs[i]);
    }

    if (st.truncated_flag) {
      log_initiator("\t(Truncated stacktrace to {} entries)\n",
                    cpu::backend::MAX_GUEST_PSEUDO_STACKTRACE_ENTRIES);
    }
    log_initiator.submit('d');
  }
}

void CommandProcessor::UpdateWritePointer(uint32_t value) {
  XE_UNLIKELY_IF(cvars::log_ringbuffer_kickoff_initiator_bts) {
    LogKickoffInitator(value);
  }
  write_ptr_index_ = value;
  write_ptr_index_event_->SetBoostPriority();
}

void CommandProcessor::LogRegisterSet(uint32_t register_index, uint32_t value) {
#if XE_ENABLE_GPU_REG_WRITE_LOGGING == 1
  if (cvars::log_guest_driven_gpu_register_written_values &&
      logging::ShouldLog(LogLevel::Debug)) {
    const RegisterInfo* reginfo = RegisterFile::GetRegisterInfo(register_index);

    if (!reginfo) {
      XELOGD("Unknown_Reg{:04X} <- {:08X}\n", register_index, value);
    } else {
      XELOGD("{} <- {:08X}\n", reginfo->name, value);
    }
  }
#endif
}

void CommandProcessor::LogRegisterSets(uint32_t base_register_index,
                                       const uint32_t* values,
                                       uint32_t n_values) {
#if XE_ENABLE_GPU_REG_WRITE_LOGGING == 1
  if (cvars::log_guest_driven_gpu_register_written_values &&
      logging::ShouldLog(LogLevel::Debug)) {
    auto target = logging::internal::GetThreadBuffer();

    auto target_ptr = target.first;

    size_t total_size = 0;

    size_t rem_size = target.second;

    for (uint32_t i = 0; i < n_values; ++i) {
      uint32_t register_index = base_register_index + i;

      uint32_t value = xe::load_and_swap<uint32_t>(&values[i]);

      const RegisterInfo* reginfo =
          RegisterFile::GetRegisterInfo(register_index);

      if (!reginfo) {
        auto tmpres = fmt::format_to_n(target_ptr, rem_size,
                                       "Unknown_Reg{:04X} <- {:08X}\n",
                                       register_index, value);
        target_ptr = tmpres.out;
        rem_size -= tmpres.size;
        total_size += tmpres.size;

      } else {
        auto tmpres = fmt::format_to_n(target_ptr, rem_size, "{} <- {:08X}\n",
                                       reginfo->name, value);
        rem_size -= tmpres.size;
        target_ptr = tmpres.out;
        total_size += tmpres.size;
      }
    }
    logging::internal::AppendLogLine(LogLevel::Debug, 'd', total_size);
  }
#endif
}

void CommandProcessor::HandleSpecialRegisterWrite(uint32_t index,
                                                  uint32_t value) {
  RegisterFile& regs = *register_file_;
  // Scratch register writeback.
  if (index >= XE_GPU_REG_SCRATCH_REG0 && index <= XE_GPU_REG_SCRATCH_REG7) {
    uint32_t scratch_reg = index - XE_GPU_REG_SCRATCH_REG0;
    if ((1 << scratch_reg) & regs.values[XE_GPU_REG_SCRATCH_UMSK]) {
      // Enabled - write to address.
      uint32_t scratch_addr = regs.values[XE_GPU_REG_SCRATCH_ADDR];
      uint32_t mem_addr = scratch_addr + (scratch_reg * 4);
      xe::store_and_swap<uint32_t>(memory_->TranslatePhysical(mem_addr), value);
    }
  } else {
    switch (index) {
      // If this is a COHER register, set the dirty flag.
      // This will block the command processor the next time it WAIT_MEM_REGs
      // and allow us to synchronize the memory.
      case XE_GPU_REG_COHER_STATUS_HOST: {
        regs.values[index] |= UINT32_C(0x80000000);
      } break;

      case XE_GPU_REG_DC_LUT_RW_INDEX: {
        // Reset the sequential read / write component index (see the M56
        // DC_LUT_SEQ_COLOR documentation).
        gamma_ramp_rw_component_ = 0;
      } break;

      case XE_GPU_REG_DC_LUT_SEQ_COLOR: {
        // Should be in the 256-entry table writing mode.
        assert_zero(regs[XE_GPU_REG_DC_LUT_RW_MODE] & 0b1);
        auto gamma_ramp_rw_index = regs.Get<reg::DC_LUT_RW_INDEX>();
        // DC_LUT_SEQ_COLOR is in the red, green, blue order, but the write
        // enable mask is blue, green, red.
        bool write_gamma_ramp_component =
            (regs[XE_GPU_REG_DC_LUT_WRITE_EN_MASK] &
             (UINT32_C(1) << (2 - gamma_ramp_rw_component_))) != 0;
        if (write_gamma_ramp_component) {
          reg::DC_LUT_30_COLOR& gamma_ramp_entry =
              gamma_ramp_256_entry_table_[gamma_ramp_rw_index.rw_index];
          // Bits 0:5 are hardwired to zero.
          uint32_t gamma_ramp_seq_color =
              regs.Get<reg::DC_LUT_SEQ_COLOR>().seq_color >> 6;
          switch (gamma_ramp_rw_component_) {
            case 0:
              gamma_ramp_entry.color_10_red = gamma_ramp_seq_color;
              break;
            case 1:
              gamma_ramp_entry.color_10_green = gamma_ramp_seq_color;
              break;
            case 2:
              gamma_ramp_entry.color_10_blue = gamma_ramp_seq_color;
              break;
          }
        }
        if (++gamma_ramp_rw_component_ >= 3) {
          gamma_ramp_rw_component_ = 0;
          reg::DC_LUT_RW_INDEX new_gamma_ramp_rw_index = gamma_ramp_rw_index;
          ++new_gamma_ramp_rw_index.rw_index;
          WriteRegister(
              XE_GPU_REG_DC_LUT_RW_INDEX,
              xe::memory::Reinterpret<uint32_t>(new_gamma_ramp_rw_index));
        }
        if (write_gamma_ramp_component) {
          OnGammaRamp256EntryTableValueWritten();
        }
      } break;

      case XE_GPU_REG_DC_LUT_PWL_DATA: {
        // Should be in the PWL writing mode.
        assert_not_zero(regs[XE_GPU_REG_DC_LUT_RW_MODE] & 0b1);
        auto gamma_ramp_rw_index = regs.Get<reg::DC_LUT_RW_INDEX>();
        // Bit 7 of the index is ignored for PWL.
        uint32_t gamma_ramp_rw_index_pwl = gamma_ramp_rw_index.rw_index & 0x7F;
        // DC_LUT_PWL_DATA is likely in the red, green, blue order because
        // DC_LUT_SEQ_COLOR is, but the write enable mask is blue, green, red.
        bool write_gamma_ramp_component =
            (regs[XE_GPU_REG_DC_LUT_WRITE_EN_MASK] &
             (UINT32_C(1) << (2 - gamma_ramp_rw_component_))) != 0;
        if (write_gamma_ramp_component) {
          reg::DC_LUT_PWL_DATA& gamma_ramp_entry =
              gamma_ramp_pwl_rgb_[gamma_ramp_rw_index_pwl]
                                 [gamma_ramp_rw_component_];
          auto gamma_ramp_value = regs.Get<reg::DC_LUT_PWL_DATA>();
          // Bits 0:5 are hardwired to zero.
          gamma_ramp_entry.base = gamma_ramp_value.base & ~UINT32_C(0x3F);
          gamma_ramp_entry.delta = gamma_ramp_value.delta & ~UINT32_C(0x3F);
        }
        if (++gamma_ramp_rw_component_ >= 3) {
          gamma_ramp_rw_component_ = 0;
          reg::DC_LUT_RW_INDEX new_gamma_ramp_rw_index = gamma_ramp_rw_index;
          // TODO(Triang3l): Should this increase beyond 7 bits for PWL?
          // Direct3D 9 explicitly sets rw_index to 0x80 after writing the last
          // PWL entry. However, the DC_LUT_RW_INDEX documentation says that for
          // PWL, the bit 7 is ignored.
          new_gamma_ramp_rw_index.rw_index =
              (gamma_ramp_rw_index.rw_index & ~UINT32_C(0x7F)) |
              ((gamma_ramp_rw_index_pwl + 1) & 0x7F);
          WriteRegister(
              XE_GPU_REG_DC_LUT_RW_INDEX,
              xe::memory::Reinterpret<uint32_t>(new_gamma_ramp_rw_index));
        }
        if (write_gamma_ramp_component) {
          OnGammaRampPWLValueWritten();
        }
      } break;

      case XE_GPU_REG_DC_LUT_30_COLOR: {
        // Should be in the 256-entry table writing mode.
        assert_zero(regs[XE_GPU_REG_DC_LUT_RW_MODE] & 0b1);
        auto gamma_ramp_rw_index = regs.Get<reg::DC_LUT_RW_INDEX>();
        uint32_t gamma_ramp_write_enable_mask =
            regs[XE_GPU_REG_DC_LUT_WRITE_EN_MASK] & 0b111;
        if (gamma_ramp_write_enable_mask) {
          reg::DC_LUT_30_COLOR& gamma_ramp_entry =
              gamma_ramp_256_entry_table_[gamma_ramp_rw_index.rw_index];
          auto gamma_ramp_value = regs.Get<reg::DC_LUT_30_COLOR>();
          if (gamma_ramp_write_enable_mask & 0b001) {
            gamma_ramp_entry.color_10_blue = gamma_ramp_value.color_10_blue;
          }
          if (gamma_ramp_write_enable_mask & 0b010) {
            gamma_ramp_entry.color_10_green = gamma_ramp_value.color_10_green;
          }
          if (gamma_ramp_write_enable_mask & 0b100) {
            gamma_ramp_entry.color_10_red = gamma_ramp_value.color_10_red;
          }
        }
        // TODO(Triang3l): Should this reset the component write index? If this
        // increase is assumed to behave like a full DC_LUT_RW_INDEX write, it
        // probably should. Currently this also calls WriteRegister for
        // DC_LUT_RW_INDEX, which resets gamma_ramp_rw_component_ as well.
        gamma_ramp_rw_component_ = 0;
        reg::DC_LUT_RW_INDEX new_gamma_ramp_rw_index = gamma_ramp_rw_index;
        ++new_gamma_ramp_rw_index.rw_index;
        WriteRegister(
            XE_GPU_REG_DC_LUT_RW_INDEX,
            xe::memory::Reinterpret<uint32_t>(new_gamma_ramp_rw_index));
        if (gamma_ramp_write_enable_mask) {
          OnGammaRamp256EntryTableValueWritten();
        }
      } break;
    }
  }
}
void CommandProcessor::WriteRegister(uint32_t index, uint32_t value) {
  // chrispy: rearrange check order, place set after checks

  if (XE_LIKELY(index < RegisterFile::kRegisterCount)) {
    register_file_->values[index] = value;

    // quick pre-test
    // todo: figure out just how unlikely this is. if very (it ought to be,
    // theres a ton of registers other than these) make this predicate
    // branchless and mark with unlikely, then make HandleSpecialRegisterWrite
    // noinline yep, its very unlikely. these ORS here are meant to be bitwise
    // ors, so that we do not do branching evaluation of the conditions (we will
    // almost always take all of the branches)

    unsigned expr = (index - XE_GPU_REG_SCRATCH_REG0 < 8) |
                    (index == XE_GPU_REG_COHER_STATUS_HOST) |
                    ((index - XE_GPU_REG_DC_LUT_RW_INDEX) <=
                     (XE_GPU_REG_DC_LUT_30_COLOR - XE_GPU_REG_DC_LUT_RW_INDEX));
    // chrispy: reordered for msvc branch probability (assumes if is taken and
    // else is not)
    if (XE_LIKELY(expr == 0)) {
      XE_MSVC_REORDER_BARRIER();

    } else {
      HandleSpecialRegisterWrite(index, value);
    }
  } else {
    XELOGW("CommandProcessor::WriteRegister index out of bounds: {}", index);
    return;
  }
}
void CommandProcessor::WriteRegistersFromMem(uint32_t start_index,
                                             uint32_t* base,
                                             uint32_t num_registers) {
  for (uint32_t i = 0; i < num_registers; ++i) {
    uint32_t data = xe::load_and_swap<uint32_t>(base + i);
    this->WriteRegister(start_index + i, data);
  }
}

void CommandProcessor::WriteRegisterRangeFromRing(xe::RingBuffer* ring,
                                                  uint32_t base,
                                                  uint32_t num_registers) {
  for (uint32_t i = 0; i < num_registers; ++i) {
    uint32_t data = ring->ReadAndSwap<uint32_t>();
    WriteRegister(base + i, data);
  }
}

void CommandProcessor::WriteALURangeFromRing(xe::RingBuffer* ring,
                                             uint32_t base,
                                             uint32_t num_times) {
  WriteRegisterRangeFromRing(ring, base + 0x4000, num_times);
}

void CommandProcessor::WriteFetchRangeFromRing(xe::RingBuffer* ring,
                                               uint32_t base,
                                               uint32_t num_times) {
  WriteRegisterRangeFromRing(ring, base + 0x4800, num_times);
}

void CommandProcessor::WriteBoolRangeFromRing(xe::RingBuffer* ring,
                                              uint32_t base,
                                              uint32_t num_times) {
  WriteRegisterRangeFromRing(ring, base + 0x4900, num_times);
}

void CommandProcessor::WriteLoopRangeFromRing(xe::RingBuffer* ring,
                                              uint32_t base,
                                              uint32_t num_times) {
  WriteRegisterRangeFromRing(ring, base + 0x4908, num_times);
}

void CommandProcessor::WriteREGISTERSRangeFromRing(xe::RingBuffer* ring,
                                                   uint32_t base,
                                                   uint32_t num_times) {
  WriteRegisterRangeFromRing(ring, base + 0x2000, num_times);
}

void CommandProcessor::WriteALURangeFromMem(uint32_t start_index,
                                            uint32_t* base,
                                            uint32_t num_registers) {
  WriteRegistersFromMem(start_index + 0x4000, base, num_registers);
}

void CommandProcessor::WriteFetchRangeFromMem(uint32_t start_index,
                                              uint32_t* base,
                                              uint32_t num_registers) {
  WriteRegistersFromMem(start_index + 0x4800, base, num_registers);
}

void CommandProcessor::WriteBoolRangeFromMem(uint32_t start_index,
                                             uint32_t* base,
                                             uint32_t num_registers) {
  WriteRegistersFromMem(start_index + 0x4900, base, num_registers);
}

void CommandProcessor::WriteLoopRangeFromMem(uint32_t start_index,
                                             uint32_t* base,
                                             uint32_t num_registers) {
  WriteRegistersFromMem(start_index + 0x4908, base, num_registers);
}

void CommandProcessor::WriteREGISTERSRangeFromMem(uint32_t start_index,
                                                  uint32_t* base,
                                                  uint32_t num_registers) {
  WriteRegistersFromMem(start_index + 0x2000, base, num_registers);
}
XE_NOINLINE
void CommandProcessor::WriteOneRegisterFromRing(uint32_t base,
                                                uint32_t num_times) {
  for (uint32_t m = 0; m < num_times; m++) {
    uint32_t reg_data = reader_.ReadAndSwap<uint32_t>();
    uint32_t target_index = base;
    WriteRegister(target_index, reg_data);
  }
}
void CommandProcessor::MakeCoherent() {
  SCOPE_profile_cpu_f("gpu");

  // Status host often has 0x01000000 or 0x03000000.
  // This is likely toggling VC (vertex cache) or TC (texture cache).
  // Or, it also has a direction in here maybe - there is probably
  // some way to check for dest coherency (what all the COHER_DEST_BASE_*
  // registers are for).
  // Best docs I've found on this are here:
  // https://web.archive.org/web/20160711162346/https://amd-dev.wpengine.netdna-cdn.com/wordpress/media/2013/10/R6xx_R7xx_3D.pdf
  // https://cgit.freedesktop.org/xorg/driver/xf86-video-radeonhd/tree/src/r6xx_accel.c?id=3f8b6eccd9dba116cc4801e7f80ce21a879c67d2#n454

  volatile uint32_t* regs_volatile = register_file_->values;
  auto status_host = xe::memory::Reinterpret<reg::COHER_STATUS_HOST>(
      uint32_t(regs_volatile[XE_GPU_REG_COHER_STATUS_HOST]));
  uint32_t base_host = regs_volatile[XE_GPU_REG_COHER_BASE_HOST];
  uint32_t size_host = regs_volatile[XE_GPU_REG_COHER_SIZE_HOST];

  if (!status_host.status) {
    return;
  }

  const char* action = "N/A";
  if (status_host.vc_action_ena && status_host.tc_action_ena) {
    action = "VC | TC";
  } else if (status_host.tc_action_ena) {
    action = "TC";
  } else if (status_host.vc_action_ena) {
    action = "VC";
  }

  // TODO(benvanik): notify resource cache of base->size and type.
  XELOGGPU("Make {:08X} -> {:08X} ({}b) coherent, action = {}", base_host,
           base_host + size_host, size_host, action);

  // Mark coherent.
  regs_volatile[XE_GPU_REG_COHER_STATUS_HOST] = 0;
}

void CommandProcessor::PrepareForWait() {
  trace_writer_.Flush();
  // Only refresh completion if there is a strict ZPD retire pending so
  // PumpPendingRetire sees the latest progress without adding extra overhead.
  if (zpd_pending_retire_handle_ != kInvalidReportHandle) {
    PollCompletedSubmission();
  }
  // Give strict ZPD a chance to retire a pending report before the guest's
  // loop polls again.
  PumpPendingRetire();
}

void CommandProcessor::ReturnFromWait() {}

void CommandProcessor::InitializeTrace() {
  // Write the initial register values, to be loaded directly into the
  // RegisterFile since all registers, including those that may have side
  // effects on setting, will be saved.
  trace_writer_.WriteRegisters(
      0, reinterpret_cast<const uint32_t*>(register_file_->values),
      RegisterFile::kRegisterCount, false);

  trace_writer_.WriteGammaRamp(gamma_ramp_256_entry_table(),
                               gamma_ramp_pwl_rgb(), gamma_ramp_rw_component_);
}

CommandProcessor::PendingZPDSlot CommandProcessor::GetPendingZPDSlot(
    uint32_t slot_base, uint32_t end_record) const {
  PendingZPDSlot pending_slot;

  for (const auto& report_pair : logical_zpd_reports_) {
    const ZPDReport& report = report_pair.second;
    if (!report.ended || report.pending_segments == 0 ||
        report.slot_base != slot_base) {
      continue;
    }

    // Wait on the oldest unresolved report for this slot first.
    if (pending_slot.report_handle == kInvalidReportHandle ||
        report_pair.first < pending_slot.report_handle) {
      pending_slot.report_handle = report_pair.first;
    }

    // Slot reuse needs to be handled carefully in fast mode. Keep the biggest
    // cached delta, not the newest one. A stale zero is a lot more dangerous
    // than a stale nonzero.
    if (report.has_cached_delta) {
      if (!pending_slot.has_cached_delta ||
          report.cached_delta > pending_slot.cached_delta) {
        pending_slot.cached_delta = report.cached_delta;
      }
      pending_slot.has_cached_delta = true;
    }

    if (report.end_record) {
      auto report_cache_it =
          fast_zpd_report_cached_values_.find(report.end_record);
      if (report_cache_it != fast_zpd_report_cached_values_.end()) {
        if (!pending_slot.has_cached_delta ||
            report_cache_it->second > pending_slot.cached_delta) {
          pending_slot.cached_delta = report_cache_it->second;
        }
        pending_slot.has_cached_delta = true;
      }
    }
  }

  auto end_record_cache_it = fast_zpd_report_cached_values_.find(end_record);
  if (end_record_cache_it != fast_zpd_report_cached_values_.end()) {
    if (!pending_slot.has_cached_delta ||
        end_record_cache_it->second > pending_slot.cached_delta) {
      pending_slot.cached_delta = end_record_cache_it->second;
    }
    pending_slot.has_cached_delta = true;
  }

  return pending_slot;
}

bool CommandProcessor::BeginZPDReport(uint32_t report_address) {
  if (GetZPDMode() == ZPDMode::kFake) {
    return false;
  }

  // Track any delta to carry forward if the same slot is immediately reused.
  uint32_t carried_cached_delta = 0;
  bool has_carried_cached_delta = false;
  uint32_t carried_from_slot_base = 0;

  if (zpd_active_segment_.logical_active) {
    // New BEGIN while a prior report is open. Hardware has one register for
    // the query address, so a new BEGIN implicitly ends the prior one.
    if (zpd_active_segment_.end_record) {
      EndZPDReport(zpd_active_segment_.end_record, true);
    } else {
      carried_from_slot_base = zpd_active_segment_.slot_base;

      auto dying_report =
          logical_zpd_reports_.find(zpd_active_segment_.report_handle);
      if (dying_report != logical_zpd_reports_.end() &&
          dying_report->second.has_cached_delta) {
        carried_cached_delta = dying_report->second.cached_delta;
        has_carried_cached_delta = true;
      }

      if (zpd_active_segment_.segment_active) {
        // Deactivate the segment before DiscardZPDQuery so that
        // EndSubmission -> CloseQuerySegment does not re-enter and
        // issue a second EndQuery on the same slot.
        zpd_active_segment_.segment_active = false;
        DiscardZPDQuery();
      }
      logical_zpd_reports_.erase(zpd_active_segment_.report_handle);
      zpd_active_segment_ = {};
    }
  }

  uint32_t slot_base = XenosZPDReport::GetSlotBase(report_address);
  uint32_t begin_record = XenosZPDReport::GetBeginRecordBase(slot_base);
  uint32_t end_record = XenosZPDReport::GetEndRecordBase(slot_base);
  if (!slot_base) {
    return false;
  }

  // Resolve same slot hazards before invalidating pending writes from the prior
  // lifetime. For finished strict queries with unpolled completion, refresh now
  // and drain, avoiding unnecessary AwaitQueryResolve blocking.
  if (GetZPDMode() == ZPDMode::kStrict) {
    PollCompletedSubmission();
  } else {
    PumpQueryResolves();
  }

  PendingZPDSlot pending_slot = GetPendingZPDSlot(slot_base, end_record);

  if (pending_slot.report_handle != kInvalidReportHandle) {
    if (GetZPDMode() == ZPDMode::kFast || GetZPDMode() == ZPDMode::kFastAlt) {
      if (pending_slot.has_cached_delta) {
        carried_cached_delta = pending_slot.cached_delta;
        has_carried_cached_delta = true;
        carried_from_slot_base = slot_base;
      }
    } else {
      while (pending_slot.report_handle != kInvalidReportHandle) {
        auto report_it = logical_zpd_reports_.find(pending_slot.report_handle);
        if (report_it == logical_zpd_reports_.end()) {
          break;
        }

        uint64_t wait_for_submission =
            report_it->second.last_segment_end_submission;

        bool wait_succeeded =
            AwaitQueryResolve(pending_slot.report_handle, wait_for_submission);

        if (!wait_succeeded) {
          if (pending_slot.cached_delta != 0) {
            carried_cached_delta = pending_slot.cached_delta;
            carried_from_slot_base = slot_base;
          }
          break;
        }

        PumpQueryResolves();

        pending_slot = GetPendingZPDSlot(slot_base, end_record);
      }
    }
  }

  // Bump slot sequence - invalidates pending writes from prior lifetime.
  uint64_t slot_sequence_id = ++zpd_slot_sequences_[slot_base];

  // By default, BEGIN drops the cached value so an orphaned END doesn't replay
  // something from a prior lifetime. The alternate fast path keeps it around
  // long enough for an async zero to help the next unresolved write.
  if (GetZPDMode() != ZPDMode::kFastAlt) {
    auto cache_it = fast_zpd_report_cached_values_.find(end_record);
    if (cache_it != fast_zpd_report_cached_values_.end() &&
        cache_it->second == 0) {
      fast_zpd_report_cached_values_.erase(cache_it);
    }
  }

  ReportHandle report_handle = zpd_next_report_handle_++;
  if (report_handle == kInvalidReportHandle) {
    report_handle = zpd_next_report_handle_++;
  }

  ZPDReport& logical = logical_zpd_reports_[report_handle];
  logical.slot_base = slot_base;
  logical.slot_sequence_id = slot_sequence_id;
  logical.begin_record = begin_record;
  logical.end_record = end_record;
  logical.begin_value = zpd_slot_values_[slot_base];
  logical.accumulated_samples = 0;
  logical.first_segment_end_submission = 0;
  logical.last_segment_end_submission = 0;
  logical.pending_segments = 0;
  logical.cached_delta = 0;
  logical.has_cached_delta = false;
  logical.ended = false;

  if (slot_base == carried_from_slot_base && has_carried_cached_delta) {
    logical.cached_delta = carried_cached_delta;
    logical.has_cached_delta = true;
  }

  zpd_active_segment_.report_handle = report_handle;
  zpd_active_segment_.slot_base = slot_base;
  zpd_active_segment_.begin_record = begin_record;
  zpd_active_segment_.end_record = end_record;
  zpd_active_segment_.segment_active = false;
  // Opens lazily. OpenQuerySegment will open it at the next valid opportunity.
  zpd_active_segment_.segment_pending_begin = true;
  zpd_active_segment_.logical_active = true;

  OpenQuerySegment(true);
  return true;
}

// Guest END closes the logical lifetime, but the final value may still depend
// on in flight query segments.
bool CommandProcessor::EndZPDReport(uint32_t report_address,
                                    bool guest_forced_end) {
  if (GetZPDMode() == ZPDMode::kFake) {
    return false;
  }

  CommandProcessor::ReportHandle report_handle =
      zpd_active_segment_.report_handle;
  uint32_t stored_end_record = zpd_active_segment_.end_record;
  uint32_t report_record_base = XenosZPDReport::GetRecordBase(report_address);
  if (!report_record_base) {
    report_record_base = stored_end_record;
  }

  if (zpd_active_segment_.segment_active) {
    CloseQuerySegment();
  }

  zpd_active_segment_.segment_pending_begin = false;

  if (!report_record_base) {
    logical_zpd_reports_.erase(report_handle);
    zpd_active_segment_ = {};
    return false;
  }

  bool resolved_immediately = false;
  uint32_t begin_record = 0;
  uint32_t begin_value = 0;
  uint32_t final_value = 0;
  uint32_t cached_delta = 0;
  bool has_cached_delta = false;

  auto it = logical_zpd_reports_.find(report_handle);
  if (it == logical_zpd_reports_.end()) {
    zpd_active_segment_ = {};
    return false;
  }

  ZPDReport& logical = it->second;
  logical.ended = true;
  logical.end_record = report_record_base;
  begin_record = logical.begin_record;
  begin_value = logical.begin_value;

  if (logical.pending_segments == 0) {
    resolved_immediately = true;
    final_value = NormalizeSampleCount(logical.accumulated_samples);

    cached_delta = final_value;
    has_cached_delta = true;
    logical.cached_delta = cached_delta;
    logical.has_cached_delta = true;
    if (fast_zpd_report_cached_values_.size() >= kFastZPDCacheMaxEntries &&
        !fast_zpd_report_cached_values_.count(report_record_base)) {
      fast_zpd_report_cached_values_.clear();
    }
    fast_zpd_report_cached_values_[report_record_base] = cached_delta;
    final_value = cached_delta;
  } else {
    if (logical.has_cached_delta) {
      cached_delta = logical.cached_delta;
      has_cached_delta = true;
    }
    auto cache_it = fast_zpd_report_cached_values_.find(report_record_base);
    if (cache_it != fast_zpd_report_cached_values_.end()) {
      cached_delta = cache_it->second;
      has_cached_delta = true;
    }
  }

  if (resolved_immediately) {
    CommitZPDReport(logical, final_value);
    logical_zpd_reports_.erase(it);
  }

  bool has_cross_slot_end =
      stored_end_record && stored_end_record != report_record_base;
  if (has_cross_slot_end) {
    // The guest ended a different ZPD record than the one opened by BEGIN.
    // Preserve the prior running slot value in the stored END record before
    // writing the actual END selected by the packet.
    WriteZPDReport(0, stored_end_record, 0, begin_value, false);
  }

  if (GetZPDMode() == ZPDMode::kFast || GetZPDMode() == ZPDMode::kFastAlt) {
    bool write_begin = begin_record && report_record_base &&
                       begin_record != report_record_base;
    // Unknown still means visible in fast mode. Reusing cached zeroes can help
    // flares stop shining through walls (545107FC, 454108D4, 4D5307D2), but it
    // also tends to break occlusion culling (4D5308AB, 4D530805), so only do it
    // in the alternate fast path.
    uint32_t speculative = cached_delta;
    if (!resolved_immediately) {
      speculative = 1;
      if (has_cached_delta &&
          (cached_delta != 0 || GetZPDMode() == ZPDMode::kFastAlt)) {
        speculative = cached_delta;
      }
    }
    WriteZPDReport(begin_record, report_record_base, begin_value, speculative,
                   write_begin);
  } else if (!resolved_immediately) {
    PumpQueryResolves();

    // Recheck after the drain. The report may have resolved synchronously if
    // all segments were already complete by the time we got here.
    if (!logical_zpd_reports_.count(report_handle)) {
      // OnZPDQueryResolved already committed and erased the report; nothing
      // left to defer.
    } else if (zpd_pending_retire_handle_ != report_handle) {
      zpd_pending_retire_handle_ = report_handle;
      zpd_pending_retire_stalls_ = 0;
      zpd_pending_retire_start_ms_ = Clock::QueryHostUptimeMillis();
    }
  }

  zpd_active_segment_ = {};
  return true;
}

void CommandProcessor::OpenQuerySegment(bool can_close_submission) {
  if (GetZPDMode() == ZPDMode::kFake || zpd_force_fake_fallback_ ||
      !zpd_active_segment_.logical_active ||
      !zpd_active_segment_.segment_pending_begin || !CanOpenZPDQuery()) {
    return;
  }

  EnsureZPDQueryResources();

  // Resource setup failed. Drop the logical report and fall back to fake mode.
  if (!IsZPDQueryPoolReady()) {
    zpd_force_fake_fallback_ = true;
    logical_zpd_reports_.erase(zpd_active_segment_.report_handle);
    zpd_active_segment_ = {};
    return;
  }

  // Frees any slots from completed submissions before asking for new ones.
  PumpQueryResolves();

  QueryOpenResult open_result =
      OpenZPDQuery(zpd_active_segment_.report_handle, can_close_submission);
  switch (open_result) {
    case QueryOpenResult::kOpened:
      break;
    case QueryOpenResult::kDeferred:
      return;
    case QueryOpenResult::kPoolExhausted: {
      if (GetZPDMode() == ZPDMode::kFast || GetZPDMode() == ZPDMode::kFastAlt) {
        // Fast mode favors forward progress over accuracy. Keep a minimal
        // accumulated value instead of waiting for a slot to become available.
        auto it = logical_zpd_reports_.find(zpd_active_segment_.report_handle);
        if (it != logical_zpd_reports_.end()) {
          it->second.accumulated_samples =
              std::max<uint64_t>(it->second.accumulated_samples, uint64_t{1});
        }
        zpd_active_segment_.segment_pending_begin = false;
        return;
      }
      return;
    }
    case QueryOpenResult::kFailed:
    default:
      return;
  }

  zpd_active_segment_.segment_active = true;
  zpd_active_segment_.segment_pending_begin = false;
}

// Closes the active host segment without ending the logical report.
// BeginQuery/EndQuery can't cross D3D12 command list or Vulkan render pass
// boundaries. The result accumulates across all pieces.
void CommandProcessor::CloseQuerySegment() {
  if (GetZPDMode() == ZPDMode::kFake || !zpd_active_segment_.segment_active) {
    return;
  }

  uint64_t submission = 0;
  if (!CloseZPDQuery(zpd_active_segment_.report_handle, submission)) {
    zpd_active_segment_.segment_active = false;
    zpd_active_segment_.segment_pending_begin =
        zpd_active_segment_.logical_active;
    return;
  }

  auto it = logical_zpd_reports_.find(zpd_active_segment_.report_handle);
  if (it != logical_zpd_reports_.end()) {
    // Lets PumpPendingRetire drain early segments without blocking on the
    // final segment's submission.
    if (it->second.pending_segments == 0) {
      it->second.first_segment_end_submission = submission;
    }
    it->second.pending_segments++;
    it->second.last_segment_end_submission = submission;
  }

  zpd_active_segment_.segment_active = false;

  zpd_active_segment_.segment_pending_begin =
      zpd_active_segment_.logical_active;
}

void CommandProcessor::OnZPDQueryResolved(ReportHandle report_handle,
                                          uint64_t raw_samples) {
  auto it = logical_zpd_reports_.find(report_handle);
  if (it == logical_zpd_reports_.end()) {
    return;
  }

  ZPDReport& logical = it->second;

  if (logical.pending_segments) {
    logical.pending_segments--;
  }

  logical.accumulated_samples += raw_samples;

  if (logical.ended && logical.pending_segments == 0) {
    uint32_t final_value = NormalizeSampleCount(logical.accumulated_samples);

    logical.cached_delta = final_value;
    logical.has_cached_delta = true;
    if (logical.end_record) {
      if (fast_zpd_report_cached_values_.size() >= kFastZPDCacheMaxEntries &&
          !fast_zpd_report_cached_values_.count(logical.end_record)) {
        fast_zpd_report_cached_values_.clear();
      }
      fast_zpd_report_cached_values_[logical.end_record] = final_value;
    }
    if (IsZPDReportCurrent(logical)) {
      CommitZPDReport(logical, final_value);
    }
    logical_zpd_reports_.erase(it);
  }
}

void CommandProcessor::PumpPendingRetire() {
  ReportHandle handle_to_await = zpd_pending_retire_handle_;
  if (handle_to_await == kInvalidReportHandle) {
    return;
  }

  auto logical_report = logical_zpd_reports_.find(handle_to_await);
  if (logical_report == logical_zpd_reports_.end()) {
    // If the report is already gone it retired through another path.
    // Clear so we don't spin on a handle that no longer exists.
    zpd_pending_retire_handle_ = kInvalidReportHandle;
    zpd_pending_retire_stalls_ = 0;
    return;
  }

  uint64_t wait_for_submission =
      logical_report->second.last_segment_end_submission;
  uint64_t first_submission =
      logical_report->second.first_segment_end_submission;

  // Early segments can be retired here and, in the best case, the report
  // fully resolves without any wait.
  if (first_submission != 0 && first_submission < wait_for_submission &&
      first_submission <= GetCompletedSubmission()) {
    PumpQueryResolves();
    logical_report = logical_zpd_reports_.find(handle_to_await);
    if (logical_report == logical_zpd_reports_.end()) {
      zpd_pending_retire_handle_ = kInvalidReportHandle;
      zpd_pending_retire_stalls_ = 0;
      return;
    }
    wait_for_submission = logical_report->second.last_segment_end_submission;
  }

  if (AwaitQueryResolve(handle_to_await, wait_for_submission)) {
    zpd_pending_retire_handle_ = kInvalidReportHandle;
    zpd_pending_retire_stalls_ = 0;
    return;
  }

  if (wait_for_submission == 0 ||
      GetCompletedSubmission() >= wait_for_submission) {
    ++zpd_pending_retire_stalls_;
  }

  // Abandon if the deadline has elapsed or the stall limit has been reached.
  // Both are checked to account for varied guest polling behavior.
  bool deadline_exceeded =
      (Clock::QueryHostUptimeMillis() - zpd_pending_retire_start_ms_ >=
       kStrictZPDRetireDeadlineMs);
  if (deadline_exceeded ||
      zpd_pending_retire_stalls_ >= kStrictZPDRetireMaxStalls) {
    // Write the cached delta to guest memory to avoid a sudden occlusion flash.
    if (IsZPDReportCurrent(logical_report->second)) {
      uint32_t fallback_delta = logical_report->second.cached_delta
                                    ? logical_report->second.cached_delta
                                    : 1;
      CommitZPDReport(logical_report->second, fallback_delta);
    }
    logical_zpd_reports_.erase(logical_report);
    zpd_pending_retire_handle_ = kInvalidReportHandle;
    zpd_pending_retire_stalls_ = 0;
  }
}

void CommandProcessor::WriteZPDReport(uint32_t begin_record,
                                      uint32_t end_record, uint32_t begin_value,
                                      uint32_t delta_value,
                                      bool write_begin_record) {
  xenos::xe_gpu_depth_sample_counts* begin =
      begin_record
          ? memory_->TranslatePhysical<xenos::xe_gpu_depth_sample_counts*>(
                begin_record)
          : nullptr;
  if (!end_record) {
    return;
  }
  xenos::xe_gpu_depth_sample_counts* end =
      memory_->TranslatePhysical<xenos::xe_gpu_depth_sample_counts*>(
          end_record);

  XenosZPDReport::WriteReportDelta(begin, end, begin_value, delta_value,
                                   write_begin_record);
}

void CommandProcessor::CommitZPDReport(ZPDReport& report,
                                       uint32_t delta_value) {
  uint32_t end_record =
      report.end_record ? report.end_record
                        : XenosZPDReport::GetEndRecordBase(report.slot_base);
  WriteZPDReport(report.begin_record, end_record, report.begin_value,
                 delta_value, report.begin_record != 0);

  // Advance running total so the next BeginReport on this slot picks up
  // the correct begin_value.
  uint32_t saturated_delta = XenosZPDReport::SaturateSampleCount(delta_value);
  uint32_t end_value = report.begin_value + saturated_delta;
  zpd_slot_values_[report.slot_base] = end_value;
}

bool CommandProcessor::IsZPDReportCurrent(const ZPDReport& report) const {
  auto it = zpd_slot_sequences_.find(report.slot_base);
  uint64_t current_sequence = it != zpd_slot_sequences_.end() ? it->second : 0;
  return current_sequence == report.slot_sequence_id;
}

uint32_t CommandProcessor::NormalizeSampleCount(uint64_t samples) const {
  if (samples == 0) {
    return 0;
  }

  uint64_t scale = zpd_draw_resolution_scale_x_ * zpd_draw_resolution_scale_y_;
  // Round, don't truncate. 1 guest sample at 2x = 4 host samples, need >= 1.
  uint64_t normalized = scale <= 1 ? samples : (samples + (scale >> 1)) / scale;

  return static_cast<uint32_t>(std::min<uint64_t>(normalized, UINT32_MAX));
}
#define COMMAND_PROCESSOR CommandProcessor
#include "pm4_command_processor_implement.h"
}  // namespace gpu
}  // namespace xe