diff --git a/src/xenia/gpu/command_processor.cc b/src/xenia/gpu/command_processor.cc index deffb2049..bf1ccfdc8 100644 --- a/src/xenia/gpu/command_processor.cc +++ b/src/xenia/gpu/command_processor.cc @@ -11,6 +11,7 @@ #include "third_party/fmt/include/fmt/format.h" #include "xenia/base/byte_stream.h" +#include "xenia/base/clock.h" #include "xenia/base/cvar.h" #include "xenia/base/logging.h" #include "xenia/base/profiling.h" @@ -19,6 +20,7 @@ #include "xenia/gpu/packet_disassembler.h" #include "xenia/gpu/sampler_info.h" #include "xenia/gpu/texture_info.h" +#include "xenia/gpu/xenos_zpd_report.h" #include "xenia/kernel/kernel_state.h" #include "xenia/kernel/user_module.h" #if !defined(NDEBUG) @@ -48,6 +50,19 @@ DEFINE_bool(clear_memory_page_state, false, "for 'Team Ninja' Games to fix missing character models)", "GPU"); +DEFINE_string( + occlusion_query, "fast", + "Controls hardware occlusion query behavior for EVENT_WRITE_ZPD.\n" + "Used for effects like lens flares, object culling, and auto-exposure.\n" + " fake: Write a fake result without asking the GPU. Safe for most games,\n" + " though some effects may look slightly wrong.\n" + " fast: Ask the GPU but don't wait for the answer. Writes a cached\n" + " result immediately and updates it when the GPU catches up.\n" + " (default)\n" + " strict: Ask the GPU and wait for the real result before continuing.\n" + " Most accurate, but may be somewhat less performant.", + "GPU"); + DEFINE_string( readback_resolve, "none", "Controls CPU readback of render-to-texture resolve results.\n" @@ -109,6 +124,20 @@ void SetReadbackResolveMode(const std::string& mode) { OVERRIDE_string(readback_resolve, mode); } +ZPDMode GetZPDMode() { + const std::string& mode = cvars::occlusion_query; + if (mode == "fake") { + return ZPDMode::kFake; + } else if (mode == "strict") { + return ZPDMode::kStrict; + } + return ZPDMode::kFast; +} + +void SetZPDMode(const std::string& mode) { + OVERRIDE_string(occlusion_query, mode); +} + using namespace xe::gpu::xenos; CommandProcessor::CommandProcessor(GraphicsSystem* graphics_system, @@ -389,9 +418,12 @@ bool CommandProcessor::Restore(ByteStream* stream) { return true; } -bool CommandProcessor::SetupContext() { return true; } +bool CommandProcessor::SetupContext() { + ResetZPDState(); + return true; +} -void CommandProcessor::ShutdownContext() {} +void CommandProcessor::ShutdownContext() { ResetZPDState(); } void CommandProcessor::InitializeRingBuffer(uint32_t ptr, uint32_t size_log2) { read_ptr_index_ = 0; @@ -800,7 +832,17 @@ void CommandProcessor::MakeCoherent() { regs_volatile[XE_GPU_REG_COHER_STATUS_HOST] = 0; } -void CommandProcessor::PrepareForWait() { trace_writer_.Flush(); } +void CommandProcessor::PrepareForWait() { + trace_writer_.Flush(); + // Only refresh completion if there is a strict ZPD retire pending so + // PumpPendingRetire sees the latest progress without adding extra overhead. + if (zpd_pending_retire_handle_ != kInvalidReportHandle) { + PollCompletedSubmission(); + } + // Give strict ZPD a chance to retire a pending report before the guest's + // loop polls again. + PumpPendingRetire(); +} void CommandProcessor::ReturnFromWait() {} @@ -815,6 +857,556 @@ void CommandProcessor::InitializeTrace() { trace_writer_.WriteGammaRamp(gamma_ramp_256_entry_table(), gamma_ramp_pwl_rgb(), gamma_ramp_rw_component_); } + +CommandProcessor::PendingZPDSlot CommandProcessor::GetPendingZPDSlot( + uint32_t slot_base, uint32_t end_record) const { + PendingZPDSlot pending_slot; + + for (const auto& report_pair : logical_zpd_reports_) { + const ZPDReport& report = report_pair.second; + if (!report.ended || report.pending_segments == 0 || + report.slot_base != slot_base) { + continue; + } + + // Wait on the oldest unresolved report for this slot first. + if (pending_slot.report_handle == kInvalidReportHandle || + report_pair.first < pending_slot.report_handle) { + pending_slot.report_handle = report_pair.first; + } + + // Slot reuse needs to be handled carefully in fast mode. Keep the biggest + // cached delta, not the newest one. A stale zero is a lot more dangerous + // than a stale nonzero. + if (report.has_cached_delta) { + if (!pending_slot.has_cached_delta || + report.cached_delta > pending_slot.cached_delta) { + pending_slot.cached_delta = report.cached_delta; + } + pending_slot.has_cached_delta = true; + } + + if (report.end_record) { + auto report_cache_it = + fast_zpd_report_cached_values_.find(report.end_record); + if (report_cache_it != fast_zpd_report_cached_values_.end()) { + if (!pending_slot.has_cached_delta || + report_cache_it->second > pending_slot.cached_delta) { + pending_slot.cached_delta = report_cache_it->second; + } + pending_slot.has_cached_delta = true; + } + } + } + + auto end_record_cache_it = fast_zpd_report_cached_values_.find(end_record); + if (end_record_cache_it != fast_zpd_report_cached_values_.end()) { + if (!pending_slot.has_cached_delta || + end_record_cache_it->second > pending_slot.cached_delta) { + pending_slot.cached_delta = end_record_cache_it->second; + } + pending_slot.has_cached_delta = true; + } + + return pending_slot; +} + +bool CommandProcessor::BeginZPDReport(uint32_t report_address) { + if (GetZPDMode() == ZPDMode::kFake) { + return false; + } + + // Track any delta to carry forward if the same slot is immediately reused. + uint32_t carried_cached_delta = 0; + bool has_carried_cached_delta = false; + uint32_t carried_from_slot_base = 0; + + if (zpd_active_segment_.logical_active) { + // New BEGIN while a prior report is open. Hardware has one register for + // the query address, so a new BEGIN implicitly ends the prior one. + if (zpd_active_segment_.end_record) { + EndZPDReport(zpd_active_segment_.end_record, true); + } else { + carried_from_slot_base = zpd_active_segment_.slot_base; + + auto dying_report = + logical_zpd_reports_.find(zpd_active_segment_.report_handle); + if (dying_report != logical_zpd_reports_.end() && + dying_report->second.has_cached_delta) { + carried_cached_delta = dying_report->second.cached_delta; + has_carried_cached_delta = true; + } + + if (zpd_active_segment_.segment_active) { + // Deactivate the segment before DiscardZPDQuery so that + // EndSubmission -> CloseQuerySegment does not re-enter and + // issue a second EndQuery on the same slot. + zpd_active_segment_.segment_active = false; + DiscardZPDQuery(); + } + logical_zpd_reports_.erase(zpd_active_segment_.report_handle); + zpd_active_segment_ = {}; + } + } + + uint32_t slot_base = XenosZPDReport::GetSlotBase(report_address); + uint32_t begin_record = XenosZPDReport::GetBeginRecordBase(slot_base); + uint32_t end_record = XenosZPDReport::GetEndRecordBase(slot_base); + if (!slot_base) { + return false; + } + + // Resolve same slot hazards before invalidating pending writes from the prior + // lifetime. For finished strict queries with unpolled completion, refresh now + // and drain, avoiding unnecessary AwaitQueryResolve blocking. + if (GetZPDMode() == ZPDMode::kStrict) { + PollCompletedSubmission(); + } else { + PumpQueryResolves(); + } + + PendingZPDSlot pending_slot = GetPendingZPDSlot(slot_base, end_record); + + if (pending_slot.report_handle != kInvalidReportHandle) { + if (GetZPDMode() == ZPDMode::kFast) { + if (pending_slot.has_cached_delta) { + carried_cached_delta = pending_slot.cached_delta; + has_carried_cached_delta = true; + carried_from_slot_base = slot_base; + } + } else { + while (pending_slot.report_handle != kInvalidReportHandle) { + auto report_it = logical_zpd_reports_.find(pending_slot.report_handle); + if (report_it == logical_zpd_reports_.end()) { + break; + } + + uint64_t wait_for_submission = + report_it->second.last_segment_end_submission; + + bool wait_succeeded = + AwaitQueryResolve(pending_slot.report_handle, wait_for_submission); + + if (!wait_succeeded) { + if (pending_slot.cached_delta != 0) { + carried_cached_delta = pending_slot.cached_delta; + carried_from_slot_base = slot_base; + } + break; + } + + PumpQueryResolves(); + + pending_slot = GetPendingZPDSlot(slot_base, end_record); + } + } + } + + // Bump slot sequence - invalidates pending writes from prior lifetime. + uint64_t slot_sequence_id = ++zpd_slot_sequences_[slot_base]; + + // By default, BEGIN drops the cached value so an orphaned END doesn't replay + // something from a prior lifetime. The alternate fast path keeps it around + // long enough for an async zero to help the next unresolved write. + if (!cvars::occlusion_query_fast_preserve_cached_zero) { + fast_zpd_report_cached_values_.erase(end_record); + } + + ReportHandle report_handle = zpd_next_report_handle_++; + if (report_handle == kInvalidReportHandle) { + report_handle = zpd_next_report_handle_++; + } + + ZPDReport& logical = logical_zpd_reports_[report_handle]; + logical.slot_base = slot_base; + logical.slot_sequence_id = slot_sequence_id; + logical.begin_record = begin_record; + logical.end_record = end_record; + logical.begin_value = zpd_slot_values_[slot_base]; + logical.accumulated_samples = 0; + logical.first_segment_end_submission = 0; + logical.last_segment_end_submission = 0; + logical.pending_segments = 0; + logical.cached_delta = 0; + logical.has_cached_delta = false; + logical.ended = false; + + if (slot_base == carried_from_slot_base && has_carried_cached_delta) { + logical.cached_delta = carried_cached_delta; + logical.has_cached_delta = true; + } + + zpd_active_segment_.report_handle = report_handle; + zpd_active_segment_.slot_base = slot_base; + zpd_active_segment_.begin_record = begin_record; + zpd_active_segment_.end_record = end_record; + zpd_active_segment_.segment_active = false; + // Opens lazily. OpenQuerySegment will open it at the next valid opportunity. + zpd_active_segment_.segment_pending_begin = true; + zpd_active_segment_.logical_active = true; + + OpenQuerySegment(true); + return true; +} + +// Guest END closes the logical lifetime, but the final value may still depend +// on in flight query segments. +bool CommandProcessor::EndZPDReport(uint32_t report_address, + bool guest_forced_end) { + if (GetZPDMode() == ZPDMode::kFake) { + return false; + } + + CommandProcessor::ReportHandle report_handle = + zpd_active_segment_.report_handle; + uint32_t stored_end_record = zpd_active_segment_.end_record; + uint32_t report_record_base = XenosZPDReport::GetRecordBase(report_address); + if (!report_record_base) { + report_record_base = stored_end_record; + } + + if (zpd_active_segment_.segment_active) { + CloseQuerySegment(); + } + + zpd_active_segment_.segment_pending_begin = false; + + if (!report_record_base) { + logical_zpd_reports_.erase(report_handle); + zpd_active_segment_ = {}; + return false; + } + + bool resolved_immediately = false; + uint32_t begin_record = 0; + uint32_t begin_value = 0; + uint32_t final_value = 0; + uint32_t cached_delta = 0; + bool has_cached_delta = false; + + auto it = logical_zpd_reports_.find(report_handle); + if (it == logical_zpd_reports_.end()) { + zpd_active_segment_ = {}; + return false; + } + + ZPDReport& logical = it->second; + logical.ended = true; + logical.end_record = report_record_base; + begin_record = logical.begin_record; + begin_value = logical.begin_value; + + if (logical.pending_segments == 0) { + resolved_immediately = true; + final_value = NormalizeSampleCount(logical.accumulated_samples); + + cached_delta = final_value; + has_cached_delta = true; + logical.cached_delta = cached_delta; + logical.has_cached_delta = true; + if (fast_zpd_report_cached_values_.size() >= kFastZPDCacheMaxEntries && + !fast_zpd_report_cached_values_.count(report_record_base)) { + fast_zpd_report_cached_values_.clear(); + } + fast_zpd_report_cached_values_[report_record_base] = cached_delta; + final_value = cached_delta; + } else { + if (logical.has_cached_delta) { + cached_delta = logical.cached_delta; + has_cached_delta = true; + } + auto cache_it = fast_zpd_report_cached_values_.find(report_record_base); + if (cache_it != fast_zpd_report_cached_values_.end()) { + cached_delta = cache_it->second; + has_cached_delta = true; + } + } + + if (resolved_immediately) { + CommitZPDReport(logical, final_value); + logical_zpd_reports_.erase(it); + } + + bool has_cross_slot_end = + stored_end_record && stored_end_record != report_record_base; + if (has_cross_slot_end) { + // The guest ended a different ZPD record than the one opened by BEGIN. + // Preserve the prior running slot value in the stored END record before + // writing the actual END selected by the packet. + WriteZPDReport(0, stored_end_record, 0, begin_value, false); + } + + if (GetZPDMode() == ZPDMode::kFast) { + bool write_begin = begin_record && report_record_base && + begin_record != report_record_base; + // Unknown still means visible in fast mode. Reusing cached zeroes can help + // flares stop shining through walls (545107FC, 454108D4, 4D5307D2), but it + // also tends to break occlusion culling (4D5308AB, 4D530805), so only do it + // in the alternate fast path. + uint32_t speculative = cached_delta; + if (!resolved_immediately) { + speculative = 1; + if (has_cached_delta && + (cached_delta != 0 || + cvars::occlusion_query_fast_preserve_cached_zero)) { + speculative = cached_delta; + } + } + WriteZPDReport(begin_record, report_record_base, begin_value, speculative, + write_begin); + } else if (!resolved_immediately) { + PumpQueryResolves(); + + // Recheck after the drain. The report may have resolved synchronously if + // all segments were already complete by the time we got here. + if (!logical_zpd_reports_.count(report_handle)) { + // OnZPDQueryResolved already committed and erased the report; nothing + // left to defer. + } else if (zpd_pending_retire_handle_ != report_handle) { + zpd_pending_retire_handle_ = report_handle; + zpd_pending_retire_stalls_ = 0; + zpd_pending_retire_start_ms_ = Clock::QueryHostUptimeMillis(); + } + } + + zpd_active_segment_ = {}; + return true; +} + +void CommandProcessor::OpenQuerySegment(bool can_close_submission) { + if (GetZPDMode() == ZPDMode::kFake || zpd_force_fake_fallback_ || + !zpd_active_segment_.logical_active || + !zpd_active_segment_.segment_pending_begin || !CanOpenZPDQuery()) { + return; + } + + EnsureZPDQueryResources(); + + // Resource setup failed. Drop the logical report and fall back to fake mode. + if (!IsZPDQueryPoolReady()) { + zpd_force_fake_fallback_ = true; + logical_zpd_reports_.erase(zpd_active_segment_.report_handle); + zpd_active_segment_ = {}; + return; + } + + // Frees any slots from completed submissions before asking for new ones. + PumpQueryResolves(); + + QueryOpenResult open_result = + OpenZPDQuery(zpd_active_segment_.report_handle, can_close_submission); + switch (open_result) { + case QueryOpenResult::kOpened: + break; + case QueryOpenResult::kDeferred: + return; + case QueryOpenResult::kPoolExhausted: { + if (GetZPDMode() == ZPDMode::kFast) { + // Fast mode favors forward progress over accuracy. Keep a minimal + // accumulated value instead of waiting for a slot to become available. + auto it = logical_zpd_reports_.find(zpd_active_segment_.report_handle); + if (it != logical_zpd_reports_.end()) { + it->second.accumulated_samples = + std::max(it->second.accumulated_samples, uint64_t{1}); + } + zpd_active_segment_.segment_pending_begin = false; + return; + } + return; + } + case QueryOpenResult::kFailed: + default: + return; + } + + zpd_active_segment_.segment_active = true; + zpd_active_segment_.segment_pending_begin = false; +} + +// Closes the active host segment without ending the logical report. +// BeginQuery/EndQuery can't cross D3D12 command list or Vulkan render pass +// boundaries. The result accumulates across all pieces. +void CommandProcessor::CloseQuerySegment() { + if (GetZPDMode() == ZPDMode::kFake || !zpd_active_segment_.segment_active) { + return; + } + + uint64_t submission = 0; + if (!CloseZPDQuery(zpd_active_segment_.report_handle, submission)) { + zpd_active_segment_.segment_active = false; + zpd_active_segment_.segment_pending_begin = + zpd_active_segment_.logical_active; + return; + } + + auto it = logical_zpd_reports_.find(zpd_active_segment_.report_handle); + if (it != logical_zpd_reports_.end()) { + // Lets PumpPendingRetire drain early segments without blocking on the + // final segment's submission. + if (it->second.pending_segments == 0) { + it->second.first_segment_end_submission = submission; + } + it->second.pending_segments++; + it->second.last_segment_end_submission = submission; + } + + zpd_active_segment_.segment_active = false; + + zpd_active_segment_.segment_pending_begin = + zpd_active_segment_.logical_active; +} + +void CommandProcessor::OnZPDQueryResolved(ReportHandle report_handle, + uint64_t raw_samples) { + auto it = logical_zpd_reports_.find(report_handle); + if (it == logical_zpd_reports_.end()) { + return; + } + + ZPDReport& logical = it->second; + + if (logical.pending_segments) { + logical.pending_segments--; + } + + logical.accumulated_samples += raw_samples; + + if (logical.ended && logical.pending_segments == 0) { + uint32_t final_value = NormalizeSampleCount(logical.accumulated_samples); + + logical.cached_delta = final_value; + logical.has_cached_delta = true; + if (logical.end_record) { + if (fast_zpd_report_cached_values_.size() >= kFastZPDCacheMaxEntries && + !fast_zpd_report_cached_values_.count(logical.end_record)) { + fast_zpd_report_cached_values_.clear(); + } + fast_zpd_report_cached_values_[logical.end_record] = final_value; + } + if (IsZPDReportCurrent(logical)) { + CommitZPDReport(logical, final_value); + } + logical_zpd_reports_.erase(it); + } +} + +void CommandProcessor::PumpPendingRetire() { + ReportHandle handle_to_await = zpd_pending_retire_handle_; + if (handle_to_await == kInvalidReportHandle) { + return; + } + + auto logical_report = logical_zpd_reports_.find(handle_to_await); + if (logical_report == logical_zpd_reports_.end()) { + // If the report is already gone it retired through another path. + // Clear so we don't spin on a handle that no longer exists. + zpd_pending_retire_handle_ = kInvalidReportHandle; + zpd_pending_retire_stalls_ = 0; + return; + } + + uint64_t wait_for_submission = + logical_report->second.last_segment_end_submission; + uint64_t first_submission = + logical_report->second.first_segment_end_submission; + + // Early segments can be retired here and, in the best case, the report + // fully resolves without any wait. + if (first_submission != 0 && first_submission < wait_for_submission && + first_submission <= GetCompletedSubmission()) { + PumpQueryResolves(); + logical_report = logical_zpd_reports_.find(handle_to_await); + if (logical_report == logical_zpd_reports_.end()) { + zpd_pending_retire_handle_ = kInvalidReportHandle; + zpd_pending_retire_stalls_ = 0; + return; + } + wait_for_submission = logical_report->second.last_segment_end_submission; + } + + if (AwaitQueryResolve(handle_to_await, wait_for_submission)) { + zpd_pending_retire_handle_ = kInvalidReportHandle; + zpd_pending_retire_stalls_ = 0; + return; + } + + if (wait_for_submission == 0 || + GetCompletedSubmission() >= wait_for_submission) { + ++zpd_pending_retire_stalls_; + } + + // Abandon if the deadline has elapsed or the stall limit has been reached. + // Both are checked to account for varied guest polling behavior. + bool deadline_exceeded = + (Clock::QueryHostUptimeMillis() - zpd_pending_retire_start_ms_ >= + kStrictZPDRetireDeadlineMs); + if (deadline_exceeded || + zpd_pending_retire_stalls_ >= kStrictZPDRetireMaxStalls) { + // Write the cached delta to guest memory to avoid a sudden occlusion flash. + if (IsZPDReportCurrent(logical_report->second)) { + uint32_t fallback_delta = logical_report->second.cached_delta + ? logical_report->second.cached_delta + : 1; + CommitZPDReport(logical_report->second, fallback_delta); + } + logical_zpd_reports_.erase(logical_report); + zpd_pending_retire_handle_ = kInvalidReportHandle; + zpd_pending_retire_stalls_ = 0; + } +} + +void CommandProcessor::WriteZPDReport(uint32_t begin_record, + uint32_t end_record, uint32_t begin_value, + uint32_t delta_value, + bool write_begin_record) { + xenos::xe_gpu_depth_sample_counts* begin = + begin_record + ? memory_->TranslatePhysical( + begin_record) + : nullptr; + if (!end_record) { + return; + } + xenos::xe_gpu_depth_sample_counts* end = + memory_->TranslatePhysical( + end_record); + + XenosZPDReport::WriteReportDelta(begin, end, begin_value, delta_value, + write_begin_record); +} + +void CommandProcessor::CommitZPDReport(ZPDReport& report, + uint32_t delta_value) { + uint32_t end_record = + report.end_record ? report.end_record + : XenosZPDReport::GetEndRecordBase(report.slot_base); + WriteZPDReport(report.begin_record, end_record, report.begin_value, + delta_value, report.begin_record != 0); + + // Advance running total so the next BeginReport on this slot picks up + // the correct begin_value. + uint32_t saturated_delta = XenosZPDReport::SaturateSampleCount(delta_value); + uint32_t end_value = report.begin_value + saturated_delta; + zpd_slot_values_[report.slot_base] = end_value; +} + +bool CommandProcessor::IsZPDReportCurrent(const ZPDReport& report) const { + auto it = zpd_slot_sequences_.find(report.slot_base); + uint64_t current_sequence = it != zpd_slot_sequences_.end() ? it->second : 0; + return current_sequence == report.slot_sequence_id; +} + +uint32_t CommandProcessor::NormalizeSampleCount(uint64_t samples) const { + if (samples == 0) { + return 0; + } + + uint64_t scale = zpd_draw_resolution_scale_x_ * zpd_draw_resolution_scale_y_; + // Round, don't truncate. 1 guest sample at 2x = 4 host samples, need >= 1. + uint64_t normalized = scale <= 1 ? samples : (samples + (scale >> 1)) / scale; + + return static_cast(std::min(normalized, UINT32_MAX)); +} #define COMMAND_PROCESSOR CommandProcessor #include "pm4_command_processor_implement.h" } // namespace gpu diff --git a/src/xenia/gpu/command_processor.h b/src/xenia/gpu/command_processor.h index 1fa5bcaeb..113e221b5 100644 --- a/src/xenia/gpu/command_processor.h +++ b/src/xenia/gpu/command_processor.h @@ -17,8 +17,10 @@ #include #include #include +#include #include +#include "xenia/base/math.h" #include "xenia/base/ring_buffer.h" #include "xenia/gpu/register_file.h" #include "xenia/gpu/trace_writer.h" @@ -41,10 +43,40 @@ enum class ReadbackResolveMode { kFull // Immediate sync with GPU stall (full) }; +// Occlusion queries - ZPD report mode. +enum class ZPDMode { + kFake, // Fake sample counts, no real GPU queries (fake) + kFast, // Real queries with speculative cached writes (fast) + kStrict, // Real queries, waits before writeback (strict) +}; + void SaveGPUSetting(GPUSetting setting, uint64_t value); bool GetGPUSetting(GPUSetting setting); ReadbackResolveMode GetReadbackResolveMode(); void SetReadbackResolveMode(const std::string& mode); +ZPDMode GetZPDMode(); +void SetZPDMode(const std::string& mode); + +// Shared pool capacity for D3D12 and Vulkan. +constexpr uint32_t kZPDQueryPoolCapacity = 8192; + +// Contiguous range of query indices for batched resolve/copy operations. +struct ResolveRange { + uint32_t start; + uint32_t count; +}; + +// Backstop for strict mode. Abandon any pending retires after this many polls +// so EVENT_WRITE_ZPD doesn't keep spinning on an unresolved report. +constexpr uint32_t kStrictZPDRetireMaxStalls = 16; +// Clock backstop used for strict retire if guest polling is sparse. +constexpr uint64_t kStrictZPDRetireDeadlineMs = 2; + +// Cap for the fast-mode cached delta map. Games reuse a small set of report +// addresses so this should never be hit, but prevents unbounded growth if a +// title cycles through unique addresses. Clearing the cache has no +// correctness impact - it only removes speculative writeback hints. +constexpr size_t kFastZPDCacheMaxEntries = 1024; class GraphicsSystem; class Shader; @@ -256,8 +288,148 @@ class CommandProcessor { virtual void PrepareForWait(); virtual void ReturnFromWait(); + virtual void PollCompletedSubmission() {} + + // Used by strict ZPD to distinguish normal in flight latency from a + // genuinely stuck report. + virtual uint64_t GetCompletedSubmission() const { return 0; } + virtual void OnPrimaryBufferEnd() {} + // TODO(boma): Add tracking for EVENT_WRITE_EXT reports. + using ReportHandle = uint64_t; + static constexpr ReportHandle kInvalidReportHandle = 0; + + enum class QueryOpenResult { + kOpened, + kDeferred, + kPoolExhausted, + kFailed, + }; + + // One active guest report slot. May span multiple host query segments split + // across submissions or render passes, final value is the normalized sum. + struct ZPDReport { + // Raw host count across all segments, normalized at retirement. + uint64_t accumulated_samples = 0; + // Submission of the first closed segment. + uint64_t first_segment_end_submission = 0; + // Submission containing the most recently closed segment's resolve. + uint64_t last_segment_end_submission = 0; + uint64_t slot_sequence_id = 0; + uint32_t slot_base = 0; + uint32_t begin_record = 0; + uint32_t end_record = 0; + // Snapshotted at BEGIN from zpd_slot_values_. + uint32_t begin_value = 0; + uint32_t pending_segments = 0; + // Last known delta. Carried forward on forced close so slot doesn't + // briefly look fully occluded. 0 is a valid delta if the alternate fast + // cvar is enabled. + uint32_t cached_delta = 0; + bool has_cached_delta = false; + bool ended = false; + }; + + // Currently open guest lifetime. Retired reports are tracked separately + // by handle until their query segments resolve. This intentionally models + // only one logical report at a time. That's enough for conventional ZPD + // reports, but QueryBatch can have multiple slots in flight, so it doesn't + // fit this layout. Eventually this probably wants to become something more + // like a map of active reports keyed by slot and sequence instead. + struct ActiveZPDSegment { + ReportHandle report_handle = kInvalidReportHandle; + uint32_t slot_base = 0; + uint32_t begin_record = 0; + uint32_t end_record = 0; + bool segment_active = false; + bool segment_pending_begin = false; + bool logical_active = false; + }; + + struct PendingZPDSlot { + ReportHandle report_handle = kInvalidReportHandle; + uint32_t cached_delta = 0; + bool has_cached_delta = false; + }; + + virtual void EnsureZPDQueryResources() {} + virtual void ShutdownZPDQueryResources() {} + + virtual bool IsZPDQueryPoolReady() const { return false; } + virtual bool CanOpenZPDQuery() const { return true; } + + // Backend acquires a pool slot, records BeginQuery, tracks it internally. + virtual QueryOpenResult OpenZPDQuery(ReportHandle report_handle, + bool can_close_submission) { + return QueryOpenResult::kFailed; + } + // Backend records EndQuery, queues a resolve for the active slot. + virtual bool CloseZPDQuery(ReportHandle report_handle, + uint64_t& out_submission) { + return false; + } + // Backend discards the active query without resolving. + virtual bool DiscardZPDQuery() { return false; } + + // Backend drains completed resolves and calls OnZPDQueryResolved for each. + virtual void PumpQueryResolves() {} + // Backend waits for all pending segments of report_handle to resolve. + virtual bool AwaitQueryResolve(ReportHandle report_handle, + uint64_t wait_for_submission) { + return false; + } + + bool BeginZPDReport(uint32_t report_address); + bool EndZPDReport(uint32_t report_address, bool guest_forced_end); + // Opens a new host query segment when CanOpenZPDQuery is true. + void OpenQuerySegment(bool can_close_submission); + // Closes the current segment at a submission or render pass boundary. + // The logical report stays open and a new segment will open at the next + // opportunity. + void CloseQuerySegment(); + + // Called by backends when a host query resolve completes. Accumulates + // the raw sample count, and if all segments are done, commits the report + // to guest memory. + void OnZPDQueryResolved(ReportHandle report_handle, uint64_t raw_samples); + + // Writes guest report with begin_value read from guest memory. + // Orphan END path only when no controller snapshot is available. + void WriteZPDReport(uint32_t begin_record, uint32_t end_record, + uint32_t begin_value, uint32_t delta_value, + bool write_begin_record); + + // Called from PrepareForWait so strict mode can retire before guest loops + // again. Gives up after kStrictZPDRetireMaxStalls. + void PumpPendingRetire(); + + // Divides host count by draw resolution scale. + uint32_t NormalizeSampleCount(uint64_t samples) const; + + // Writes the final report to guest memory and advances the slot running + // total. Called when a report fully resolves or is abandoned. + void CommitZPDReport(ZPDReport& report, uint32_t delta_value); + // Checks that the report's slot sequence is still current (not reused). + bool IsZPDReportCurrent(const ZPDReport& report) const; + PendingZPDSlot GetPendingZPDSlot(uint32_t slot_base, + uint32_t end_record) const; + + void ResetZPDState() { + zpd_active_segment_ = {}; + zpd_next_report_handle_ = 1; + zpd_slot_sequences_.clear(); + zpd_slot_values_.clear(); + logical_zpd_reports_.clear(); + fast_zpd_report_cached_values_.clear(); + fake_zpd_sample_count_ = 0; + querybatch_zpd_sample_count_ = UINT32_MAX; + zpd_pending_retire_handle_ = kInvalidReportHandle; + zpd_pending_retire_stalls_ = 0; + zpd_pending_retire_start_ms_ = 0; + zpd_force_fake_fallback_ = false; + } + #include "pm4_command_processor_declare.h" virtual Shader* LoadShader(xenos::ShaderType shader_type, @@ -287,6 +459,38 @@ class CommandProcessor { GraphicsSystem* graphics_system_ = nullptr; RegisterFile* XE_RESTRICT register_file_ = nullptr; + ReportHandle zpd_next_report_handle_ = 1; + std::unordered_map zpd_slot_sequences_; + std::unordered_map zpd_slot_values_; + std::unordered_map logical_zpd_reports_; + ActiveZPDSegment zpd_active_segment_{}; + + // Cached delta per END. + // Fast mode uses this for speculative writeback and orphaned END replay. + std::unordered_map fast_zpd_report_cached_values_; + + uint32_t querybatch_zpd_sample_count_ = UINT32_MAX; + bool zpd_force_fake_fallback_ = false; + + // Strict mode defers guest completion until the queued END has retired. + ReportHandle zpd_pending_retire_handle_ = kInvalidReportHandle; + uint32_t zpd_pending_retire_stalls_ = 0; + // Uptime in ms when zpd_pending_retire_handle_ was first set. + uint64_t zpd_pending_retire_start_ms_ = 0; + + // Set by the backend when resolution scale changes. + uint32_t zpd_draw_resolution_scale_x_ = 1; + uint32_t zpd_draw_resolution_scale_y_ = 1; + + uint32_t zpd_draw_resolution_scale_x() const { + return zpd_draw_resolution_scale_x_; + } + uint32_t zpd_draw_resolution_scale_y() const { + return zpd_draw_resolution_scale_y_; + } + + uint32_t fake_zpd_sample_count_ = 0; + TraceWriter trace_writer_; enum class TraceState { kDisabled, diff --git a/src/xenia/gpu/d3d12/d3d12_command_processor.cc b/src/xenia/gpu/d3d12/d3d12_command_processor.cc index 73f71e630..81550a70d 100644 --- a/src/xenia/gpu/d3d12/d3d12_command_processor.cc +++ b/src/xenia/gpu/d3d12/d3d12_command_processor.cc @@ -24,6 +24,7 @@ #include "xenia/gpu/packet_disassembler.h" #include "xenia/gpu/registers.h" #include "xenia/gpu/xenos.h" +#include "xenia/gpu/xenos_zpd_report.h" #include "xenia/kernel/kernel_state.h" #include "xenia/ui/d3d12/d3d12_presenter.h" #include "xenia/ui/d3d12/d3d12_util.h" @@ -98,6 +99,13 @@ void D3D12CommandProcessor::RestoreEdramSnapshot(const void* snapshot) { render_target_cache_->RestoreEdramSnapshot(snapshot); } +void D3D12CommandProcessor::PollCompletedSubmission() { + // Strict ZPD just needs the completion timeline updated and any ready query + // resolves drained here. + completion_timeline_->AwaitSubmissionAndUpdateCompleted(0); + PumpQueryResolves(); +} + bool D3D12CommandProcessor::PushTransitionBarrier( ID3D12Resource* resource, D3D12_RESOURCE_STATES old_state, D3D12_RESOURCE_STATES new_state, UINT subresource) { @@ -250,8 +258,8 @@ ID3D12RootSignature* D3D12CommandProcessor::GetRootSignature( parameter.ShaderVisibility = D3D12_SHADER_VISIBILITY_ALL; } - // Shared memory and, if ROVs are used, EDRAM. - D3D12_DESCRIPTOR_RANGE shared_memory_and_edram_ranges[3]; + // Shared memory and, if ROVs are used, EDRAM and the ZPD counter. + D3D12_DESCRIPTOR_RANGE shared_memory_and_edram_ranges[4]; { auto& parameter = parameters[kRootParameter_Bindful_SharedMemoryAndEdram]; parameter.ParameterType = D3D12_ROOT_PARAMETER_TYPE_DESCRIPTOR_TABLE; @@ -284,6 +292,14 @@ ID3D12RootSignature* D3D12CommandProcessor::GetRootSignature( UINT(DxbcShaderTranslator::UAVRegister::kEdram); shared_memory_and_edram_ranges[2].RegisterSpace = 0; shared_memory_and_edram_ranges[2].OffsetInDescriptorsFromTableStart = 2; + ++parameter.DescriptorTable.NumDescriptorRanges; + shared_memory_and_edram_ranges[3].RangeType = + D3D12_DESCRIPTOR_RANGE_TYPE_UAV; + shared_memory_and_edram_ranges[3].NumDescriptors = 1; + shared_memory_and_edram_ranges[3].BaseShaderRegister = + UINT(DxbcShaderTranslator::UAVRegister::kZpdRovCounter); + shared_memory_and_edram_ranges[3].RegisterSpace = 0; + shared_memory_and_edram_ranges[3].OffsetInDescriptorsFromTableStart = 3; } } @@ -805,8 +821,8 @@ bool D3D12CommandProcessor::SetupContext() { } // Initially in open state, wait until a deferred command list submission. command_list_->Close(); - // Optional - added in Creators Update (SDK 10.0.15063.0). command_list_->QueryInterface(IID_PPV_ARGS(&command_list_1_)); + command_list_->QueryInterface(IID_PPV_ARGS(&command_list_2_)); bindless_resources_used_ = cvars::d3d12_bindless && @@ -1032,7 +1048,7 @@ bool D3D12CommandProcessor::SetupContext() { root_bindless_sampler_range.OffsetInDescriptorsFromTableStart = 0; } // View heap. - D3D12_DESCRIPTOR_RANGE root_bindless_view_ranges[4]; + D3D12_DESCRIPTOR_RANGE root_bindless_view_ranges[5]; { auto& parameter = root_parameters_bindless[kRootParameter_Bindless_ViewHeap]; @@ -1052,9 +1068,22 @@ bool D3D12CommandProcessor::SetupContext() { range.NumDescriptors = 1; range.BaseShaderRegister = UINT(DxbcShaderTranslator::UAVRegister::kEdram); + // ROV ZPD counter. range.RegisterSpace = 0; range.OffsetInDescriptorsFromTableStart = UINT(SystemBindlessView::kEdramR32UintUAV); + assert_true(parameter.DescriptorTable.NumDescriptorRanges < + xe::countof(root_bindless_view_ranges)); + auto& counter_range = + root_bindless_view_ranges[parameter.DescriptorTable + .NumDescriptorRanges++]; + counter_range.RangeType = D3D12_DESCRIPTOR_RANGE_TYPE_UAV; + counter_range.NumDescriptors = 1; + counter_range.BaseShaderRegister = + UINT(DxbcShaderTranslator::UAVRegister::kZpdRovCounter); + counter_range.RegisterSpace = 0; + counter_range.OffsetInDescriptorsFromTableStart = + UINT(SystemBindlessView::kZpdROVCounterRawUAV); } // Used UAV and SRV ranges must not overlap on Nvidia Fermi, so textures // have OffsetInDescriptorsFromTableStart after all static descriptors of @@ -1139,6 +1168,10 @@ bool D3D12CommandProcessor::SetupContext() { return false; } + // Needed by NormalizeSampleCount. + zpd_draw_resolution_scale_x_ = draw_resolution_scale_x; + zpd_draw_resolution_scale_y_ = draw_resolution_scale_y; + pipeline_cache_ = std::make_unique(*this, *register_file_, *render_target_cache_.get(), bindless_resources_used_); @@ -1514,6 +1547,10 @@ bool D3D12CommandProcessor::SetupContext() { view_bindless_heap_cpu_start_, uint32_t(SystemBindlessView::kEdramR32G32B32A32UintUAV)), 4); + // kZpdROVCounterRawUAV. + WriteZPDROVCounterRawUAVDescriptor(provider.OffsetViewDescriptor( + view_bindless_heap_cpu_start_, + uint32_t(SystemBindlessView::kZpdROVCounterRawUAV))); // kGammaRampTableSRV. WriteGammaRampSRV(false, provider.OffsetViewDescriptor( @@ -1526,6 +1563,10 @@ bool D3D12CommandProcessor::SetupContext() { uint32_t(SystemBindlessView::kGammaRampPWLSRV))); } + // Initialize the ZPD occlusion query pool and resources. + zpd_host_query_pool_ = std::make_unique(); + EnsureZPDQueryResources(); + pix_capture_requested_.store(false, std::memory_order_relaxed); pix_capturing_ = false; @@ -1547,6 +1588,9 @@ void D3D12CommandProcessor::ShutdownContext() { ui::d3d12::util::ReleaseAndNull(memexport_readback_buffer_); memexport_readback_buffer_size_ = 0; + ShutdownZPDQueryResources(); + zpd_host_query_pool_.reset(); + ui::d3d12::util::ReleaseAndNull(scratch_buffer_); scratch_buffer_size_ = 0; @@ -1615,6 +1659,7 @@ void D3D12CommandProcessor::ShutdownContext() { deferred_command_list_.Reset(); ui::d3d12::util::ReleaseAndNull(command_list_1_); + ui::d3d12::util::ReleaseAndNull(command_list_2_); ui::d3d12::util::ReleaseAndNull(command_list_); ClearCommandAllocatorCache(); @@ -2438,6 +2483,10 @@ void D3D12CommandProcessor::IssueSwap(uint32_t frontbuffer_ptr, } void D3D12CommandProcessor::OnPrimaryBufferEnd() { + // Pump any completed resolves now since the guest is likely about to poll. + PumpQueryResolves(); + PumpPendingRetire(); + if (cvars::d3d12_submit_on_primary_buffer_end && submission_open_ && CanEndSubmissionImmediately()) { EndSubmission(false); @@ -3220,6 +3269,9 @@ void D3D12CommandProcessor::CheckSubmissionCompletion( primitive_processor_->CompletedSubmissionUpdated(); texture_cache_->CompletedSubmissionUpdated(completed_submission); + + // Pull completed query resolves so logical ZPD reports can retire. + PumpQueryResolves(); } bool D3D12CommandProcessor::BeginSubmission(bool is_guest_command) { @@ -3280,6 +3332,11 @@ bool D3D12CommandProcessor::BeginSubmission(bool is_guest_command) { // fulfilled). deferred_command_list_.Reset(); + // Resume the active query segment. + if (GetZPDMode() != ZPDMode::kFake && zpd_active_segment_.logical_active) { + OpenQuerySegment(false); + } + // Reset cached state of the command list. ff_viewport_update_needed_ = true; ff_scissor_update_needed_ = true; @@ -3419,6 +3476,13 @@ bool D3D12CommandProcessor::EndSubmission(bool is_swap) { if (submission_open_) { assert_false(scratch_buffer_used_); + // We can't close the command list with an active query - D3D12 requirement. + // Close the active segment and emit ResolveQueryData before executing. + if (GetZPDMode() != ZPDMode::kFake) { + CloseQuerySegment(); + RecordZPDResolveBatch(); + } + pipeline_cache_->EndSubmission(); // Submit barriers now because resources with the queued barriers may be @@ -3438,7 +3502,8 @@ bool D3D12CommandProcessor::EndSubmission(bool is_swap) { command_allocator_writable_first_->command_allocator; command_allocator->Reset(); command_list_->Reset(command_allocator, nullptr); - deferred_command_list_.Execute(command_list_, command_list_1_); + deferred_command_list_.Execute(command_list_, command_list_1_, + command_list_2_); command_list_->Close(); ID3D12CommandList* execute_command_lists[] = {command_list_}; direct_queue->ExecuteCommandLists(1, execute_command_lists); @@ -3460,6 +3525,12 @@ bool D3D12CommandProcessor::EndSubmission(bool is_swap) { submission_open_ = false; + // Pump ZPD query process. This drains any resolves that became readable + // from completed work and retires reports unblocked by those resolves. + // Strict mode may block here before any guest visible progress continues. + PumpQueryResolves(); + PumpPendingRetire(); + // Queue operations done directly (like UpdateTileMappings) will be awaited // alongside the last submission if needed. queue_operations_done_since_submission_signal_ = false; @@ -4069,6 +4140,15 @@ XE_NOINLINE void D3D12CommandProcessor::UpdateSystemConstantValues_Impl( system_constants_.edram_depth_base_dwords_scaled = depth_base_dwords_scaled; + uint32_t zpd_rov_counter_index = UINT32_MAX; + if (zpd_active_query_is_rov_ && zpd_active_query_index_ != UINT32_MAX && + zpd_host_query_pool_->rov_initialized()) { + zpd_rov_counter_index = zpd_active_query_index_; + } + update_dirty_uint32_cmp(system_constants_.zpd_rov_counter_index, + zpd_rov_counter_index); + system_constants_.zpd_rov_counter_index = zpd_rov_counter_index; + // For non-polygons, front polygon offset is used, and it's enabled if // POLY_OFFSET_PARA_ENABLED is set, for polygons, separate front and back // are used. @@ -4941,12 +5021,14 @@ bool D3D12CommandProcessor::UpdateBindings_BindfulPath( if (write_textures_pixel) { view_count_partial_update += texture_count_pixel; } - // All the constants + shared memory SRV and UAV + textures. + // Shared memory SRV and null UAV + null SRV and shared memory UAV + + // textures. size_t view_count_full_update = - 2 + texture_count_vertex + texture_count_pixel; + 4 + texture_count_vertex + texture_count_pixel; if (edram_rov_used) { - // + EDRAM UAV. - ++view_count_full_update; + // + EDRAM UAV and ZPD counter UAV in two tables (with the shared memory + // SRV and with the shared memory UAV). + view_count_full_update += 4; } D3D12_CPU_DESCRIPTOR_HANDLE view_cpu_handle; D3D12_GPU_DESCRIPTOR_HANDLE view_gpu_handle; @@ -4990,20 +5072,25 @@ bool D3D12CommandProcessor::UpdateBindings_BindfulPath( bindful_textures_written_vertex_ = false; bindful_textures_written_pixel_ = false; // If updating fully, write the shared memory SRV and UAV descriptors and, - // if needed, the EDRAM descriptor. + // if needed, the EDRAM and ZPD counter descriptors. + // SRV + null UAV + EDRAM + ZPD counter. gpu_handle_shared_memory_srv_and_edram_ = view_gpu_handle; shared_memory_->WriteRawSRVDescriptor(view_cpu_handle); view_cpu_handle.ptr += descriptor_size_view; view_gpu_handle.ptr += descriptor_size_view; - shared_memory_->WriteRawUAVDescriptor(view_cpu_handle); + ui::d3d12::util::CreateBufferRawUAV(provider.GetDevice(), view_cpu_handle, + nullptr, 0); view_cpu_handle.ptr += descriptor_size_view; view_gpu_handle.ptr += descriptor_size_view; if (edram_rov_used) { render_target_cache_->WriteEdramUintPow2UAVDescriptor(view_cpu_handle, 2); view_cpu_handle.ptr += descriptor_size_view; view_gpu_handle.ptr += descriptor_size_view; + WriteZPDROVCounterRawUAVDescriptor(view_cpu_handle); + view_cpu_handle.ptr += descriptor_size_view; + view_gpu_handle.ptr += descriptor_size_view; } - // Null SRV + UAV + EDRAM. + // Null SRV + UAV + EDRAM + ZPD counter. gpu_handle_shared_memory_uav_and_edram_ = view_gpu_handle; ui::d3d12::util::CreateBufferRawSRV(provider.GetDevice(), view_cpu_handle, nullptr, 0); @@ -5016,6 +5103,9 @@ bool D3D12CommandProcessor::UpdateBindings_BindfulPath( render_target_cache_->WriteEdramUintPow2UAVDescriptor(view_cpu_handle, 2); view_cpu_handle.ptr += descriptor_size_view; view_gpu_handle.ptr += descriptor_size_view; + WriteZPDROVCounterRawUAVDescriptor(view_cpu_handle); + view_cpu_handle.ptr += descriptor_size_view; + view_gpu_handle.ptr += descriptor_size_view; } current_graphics_root_up_to_date_ &= ~(1u << kRootParameter_Bindful_SharedMemoryAndEdram); @@ -5141,6 +5231,271 @@ ID3D12Resource* D3D12CommandProcessor::RequestReadbackBuffer(uint32_t size) { return memexport_readback_buffer_; } +void D3D12CommandProcessor::EnsureZPDQueryResources() { + if (GetZPDMode() == ZPDMode::kFake || !zpd_host_query_pool_) { + return; + } + + bool can_recreate = !zpd_active_segment_.logical_active && + !zpd_active_segment_.segment_active && + zpd_active_query_index_ == UINT32_MAX && + !zpd_active_query_is_rov_ && + !zpd_host_query_pool_->has_pending_resolve_batch() && + zpd_resolves_in_flight_.empty(); + bool needs_rov_counter = render_target_cache_ && + render_target_cache_->GetPath() == + RenderTargetCache::Path::kPixelShaderInterlock; + zpd_query_pool_needs_rov_counter_ = needs_rov_counter; + // The ROV counter clear uses WriteBufferImmediate, so only initialize when + // CommandList2 is available. + bool can_initialize_rov_counter = needs_rov_counter && command_list_2_; + + bool resources_initialized = zpd_host_query_pool_->EnsureInitialized( + GetD3D12Provider(), kZPDQueryPoolCapacity, can_recreate, + can_initialize_rov_counter); + ID3D12Resource* rov_counter_buffer = nullptr; + uint32_t rov_counter_capacity = 0; + if (resources_initialized && zpd_host_query_pool_->rov_initialized()) { + rov_counter_buffer = zpd_host_query_pool_->rov_counter_buffer(); + rov_counter_capacity = zpd_host_query_pool_->capacity(); + } + if (bindless_resources_used_) { + WriteZPDROVCounterRawUAVDescriptor(GetD3D12Provider().OffsetViewDescriptor( + view_bindless_heap_cpu_start_, + uint32_t(SystemBindlessView::kZpdROVCounterRawUAV))); + } else if (bindful_zpd_rov_counter_buffer_ != rov_counter_buffer || + bindful_zpd_rov_counter_capacity_ != rov_counter_capacity) { + // If the ROV counter appears or changes after a bindful page was built, + // then an old page can end up counting into a null/stale UAV. So invalidate + // it and let the normal bindful rebuild pick up the current counter. + bindful_zpd_rov_counter_buffer_ = rov_counter_buffer; + bindful_zpd_rov_counter_capacity_ = rov_counter_capacity; + draw_view_bindful_heap_index_ = + ui::d3d12::D3D12DescriptorHeapPool::kHeapIndexInvalid; + } + if (zpd_query_pool_needs_rov_counter_ && !IsZPDQueryPoolReady()) { + if (!command_list_2_) { + XELOGW( + "ZPD/D3D12: ROV counter unavailable because CommandList2 is not " + "available; keeping counter index sentinel active"); + } else { + XELOGW( + "ZPD/D3D12: ROV counter resources unavailable; keeping counter index " + "sentinel active"); + } + } +} + +bool D3D12CommandProcessor::IsZPDQueryPoolReady() const { + if (!zpd_host_query_pool_ || !zpd_host_query_pool_->rtv_initialized()) { + return false; + } + if (!zpd_query_pool_needs_rov_counter_) { + return true; + } + return zpd_host_query_pool_->rov_initialized(); +} + +bool D3D12CommandProcessor::CanOpenZPDQuery() const { return submission_open_; } + +CommandProcessor::QueryOpenResult D3D12CommandProcessor::OpenZPDQuery( + ReportHandle report_handle, bool can_close_submission) { + bool use_rov_counter_path = zpd_query_pool_needs_rov_counter_ && + zpd_host_query_pool_->rov_initialized(); + bool is_pool_exhausted = !zpd_host_query_pool_->has_free_indices(); + + if (is_pool_exhausted) { + PumpQueryResolves(); + is_pool_exhausted = !zpd_host_query_pool_->has_free_indices(); + } + + bool waited_for_submission = false; + + if (is_pool_exhausted) { + if (GetZPDMode() == ZPDMode::kFast) { + return QueryOpenResult::kPoolExhausted; + } + + uint64_t wait_for = 0; + if (!zpd_resolves_in_flight_.empty()) { + wait_for = zpd_resolves_in_flight_.front().submission; + } + + uint64_t completed_submission = GetCompletedSubmission(); + if (wait_for > completed_submission) { + if (wait_for >= GetCurrentSubmission()) { + if (can_close_submission) { + if (!EndSubmission(false)) { + return QueryOpenResult::kFailed; + } + } + return QueryOpenResult::kDeferred; + } + + completion_timeline_->AwaitSubmissionAndUpdateCompleted(wait_for); + waited_for_submission = true; + PumpQueryResolves(); + is_pool_exhausted = !zpd_host_query_pool_->has_free_indices(); + } + } + + if (is_pool_exhausted) { + return waited_for_submission ? QueryOpenResult::kPoolExhausted + : QueryOpenResult::kDeferred; + } + + if (!zpd_host_query_pool_->AcquireQueryIndex(zpd_active_query_index_, + zpd_active_query_generation_)) { + return QueryOpenResult::kFailed; + } + + zpd_active_query_is_rov_ = use_rov_counter_path; + + // ROV queries don't use D3D12 occlusion queries at all. + // While the segment is open, the translated pixel shader accumulates passed + // MSAA samples into one counter slot selected via zpd_rov_counter_index. + // Clear the slot here so a recycled index never inherits old counts. + if (zpd_active_query_is_rov_) { + zpd_host_query_pool_->ClearROVCounter(deferred_command_list_, + zpd_active_query_index_); + return QueryOpenResult::kOpened; + } + + zpd_host_query_pool_->BeginQuery(deferred_command_list_, + zpd_active_query_index_); + return QueryOpenResult::kOpened; +} + +bool D3D12CommandProcessor::CloseZPDQuery(ReportHandle report_handle, + uint64_t& out_submission) { + if (zpd_active_query_is_rov_) { + zpd_host_query_pool_->QueueQueryResolve(zpd_active_query_index_, true); + } else { + zpd_host_query_pool_->EndQuery(deferred_command_list_, + zpd_active_query_index_); + zpd_host_query_pool_->QueueQueryResolve(zpd_active_query_index_, false); + } + + PendingQueryResolve resolve; + resolve.submission = GetCurrentSubmission(); + resolve.query_index = zpd_active_query_index_; + resolve.query_generation = zpd_active_query_generation_; + resolve.uses_rov_counter = zpd_active_query_is_rov_; + resolve.report_handle = report_handle; + zpd_resolves_in_flight_.push_back(resolve); + + out_submission = resolve.submission; + + zpd_active_query_index_ = UINT32_MAX; + zpd_active_query_generation_ = 0; + zpd_active_query_is_rov_ = false; + return true; +} + +bool D3D12CommandProcessor::DiscardZPDQuery() { + if (zpd_active_query_is_rov_) { + // The slot counter may be dirty if draws ran between OpenZPDQuery and here, + // but the next OpenZPDQuery will zero it before any new shader accumulates. + zpd_host_query_pool_->ReleaseQueryIndex(zpd_active_query_index_, + zpd_active_query_generation_); + zpd_active_query_index_ = UINT32_MAX; + zpd_active_query_generation_ = 0; + zpd_active_query_is_rov_ = false; + return true; + } + + // D3D12 requires a paired EndQuery before the slot can be released. + // EndSubmission flushes it so the slot can be freed without a resolve. + zpd_host_query_pool_->EndQuery(deferred_command_list_, + zpd_active_query_index_); + if (!EndSubmission(false)) { + return false; + } + zpd_host_query_pool_->ReleaseQueryIndex(zpd_active_query_index_, + zpd_active_query_generation_); + zpd_active_query_index_ = UINT32_MAX; + zpd_active_query_generation_ = 0; + zpd_active_query_is_rov_ = false; + return true; +} + +void D3D12CommandProcessor::PumpQueryResolves() { + if (GetZPDMode() == ZPDMode::kFake || !zpd_host_query_pool_) { + return; + } + + uint64_t completed = GetCompletedSubmission(); + if (completed == 0) { + return; + } + + while (!zpd_resolves_in_flight_.empty()) { + if (zpd_resolves_in_flight_.front().submission > completed) { + break; + } + PendingQueryResolve resolve = zpd_resolves_in_flight_.front(); + zpd_resolves_in_flight_.pop_front(); + + if (zpd_host_query_pool_->GenerationMatches(resolve.query_index, + resolve.query_generation)) { + uint64_t raw_samples = zpd_host_query_pool_->GetQueryReadbackValue( + resolve.query_index, resolve.uses_rov_counter); + zpd_host_query_pool_->ReleaseQueryIndex(resolve.query_index, + resolve.query_generation); + OnZPDQueryResolved(resolve.report_handle, raw_samples); + } + } +} + +bool D3D12CommandProcessor::AwaitQueryResolve(ReportHandle report_handle, + uint64_t wait_for_submission) { + if (GetZPDMode() == ZPDMode::kFake) { + return false; + } + + PumpQueryResolves(); + + auto it = logical_zpd_reports_.find(report_handle); + if (it == logical_zpd_reports_.end()) { + return true; + } + if (it->second.pending_segments == 0 && it->second.ended) { + return true; + } + if (wait_for_submission == 0) { + return false; + } + + // Ensure the submission is flushed. + if (wait_for_submission >= GetCurrentSubmission()) { + if (!submission_open_) { + return false; + } + if (!CanEndSubmissionImmediately()) { + pipeline_cache_->AwaitPipelineCompletion(); + } + if (!CanEndSubmissionImmediately() || !EndSubmission(false)) { + return false; + } + } + + if (wait_for_submission > GetCompletedSubmission()) { + completion_timeline_->AwaitSubmissionAndUpdateCompleted( + wait_for_submission); + } + + PumpQueryResolves(); + + it = logical_zpd_reports_.find(report_handle); + return it == logical_zpd_reports_.end() || + (it->second.pending_segments == 0 && it->second.ended); +} + +void D3D12CommandProcessor::RecordZPDResolveBatch() { + zpd_host_query_pool_->FlushResolveBatch(deferred_command_list_, + submission_open_); +} + void D3D12CommandProcessor::WriteGammaRampSRV( bool is_pwl, D3D12_CPU_DESCRIPTOR_HANDLE handle) const { ID3D12Device* device = GetD3D12Provider().GetDevice(); @@ -5162,6 +5517,19 @@ void D3D12CommandProcessor::WriteGammaRampSRV( device->CreateShaderResourceView(gamma_ramp_buffer_.Get(), &desc, handle); } +void D3D12CommandProcessor::WriteZPDROVCounterRawUAVDescriptor( + D3D12_CPU_DESCRIPTOR_HANDLE handle) const { + ID3D12Device* device = GetD3D12Provider().GetDevice(); + if (zpd_host_query_pool_ && zpd_host_query_pool_->rov_initialized()) { + ui::d3d12::util::CreateBufferRawUAV( + device, handle, zpd_host_query_pool_->rov_counter_buffer(), + sizeof(uint32_t) * zpd_host_query_pool_->capacity()); + return; + } + + ui::d3d12::util::CreateBufferRawUAV(device, handle, nullptr, 0); +} + #define COMMAND_PROCESSOR D3D12CommandProcessor #include "../pm4_command_processor_implement.h" diff --git a/src/xenia/gpu/d3d12/d3d12_command_processor.h b/src/xenia/gpu/d3d12/d3d12_command_processor.h index 557d42479..95687cd98 100644 --- a/src/xenia/gpu/d3d12/d3d12_command_processor.h +++ b/src/xenia/gpu/d3d12/d3d12_command_processor.h @@ -27,6 +27,7 @@ #include "xenia/gpu/d3d12/d3d12_render_target_cache.h" #include "xenia/gpu/d3d12/d3d12_shared_memory.h" #include "xenia/gpu/d3d12/d3d12_texture_cache.h" +#include "xenia/gpu/d3d12/d3d12_zpd_query_pool.h" #include "xenia/gpu/d3d12/deferred_command_list.h" #include "xenia/gpu/d3d12/pipeline_cache.h" #include "xenia/gpu/draw_util.h" @@ -78,6 +79,8 @@ class D3D12CommandProcessor final : public CommandProcessor { void RestoreEdramSnapshot(const void* snapshot) override; + void PollCompletedSubmission() override; + ui::d3d12::D3D12Provider& GetD3D12Provider() const { return *static_cast( graphics_system_->provider()); @@ -93,7 +96,7 @@ class D3D12CommandProcessor final : public CommandProcessor { uint64_t GetCurrentSubmission() const { return completion_timeline_->GetUpcomingSubmission(); } - uint64_t GetCompletedSubmission() const { + uint64_t GetCompletedSubmission() const override { return completion_timeline_->GetCompletedSubmissionFromLastUpdate(); } @@ -169,6 +172,7 @@ class D3D12CommandProcessor final : public CommandProcessor { kEdramR32UintUAV, kEdramR32G32UintUAV, kEdramR32G32B32A32UintUAV, + kZpdROVCounterRawUAV, kGammaRampTableSRV, kGammaRampPWLSRV, @@ -490,11 +494,63 @@ class D3D12CommandProcessor final : public CommandProcessor { ID3D12Resource* RequestReadbackBuffer(uint32_t size); void WriteGammaRampSRV(bool is_pwl, D3D12_CPU_DESCRIPTOR_HANDLE handle) const; + void WriteZPDROVCounterRawUAVDescriptor( + D3D12_CPU_DESCRIPTOR_HANDLE handle) const; + + // ZPD occlusion queries backend. + // BeginQuery/EndQuery must be in the same command list, segments split at + // EndSubmission, resume at BeginSubmission. Discarded queries still need + // EndQuery or the heap slot breaks on some drivers. RecordZPDResolveBatch + // emits coalesced ResolveQueryData and ROV counter copies at submit. + void EnsureZPDQueryResources() override; + void ShutdownZPDQueryResources() override { + zpd_resolves_in_flight_.clear(); + zpd_active_query_index_ = UINT32_MAX; + zpd_active_query_generation_ = 0; + zpd_active_query_is_rov_ = false; + zpd_query_pool_needs_rov_counter_ = false; + bindful_zpd_rov_counter_buffer_ = nullptr; + bindful_zpd_rov_counter_capacity_ = 0; + if (!bindless_resources_used_) { + draw_view_bindful_heap_index_ = + ui::d3d12::D3D12DescriptorHeapPool::kHeapIndexInvalid; + } + if (zpd_host_query_pool_) { + zpd_host_query_pool_->Shutdown(); + } + } + + bool IsZPDQueryPoolReady() const override; + bool CanOpenZPDQuery() const override; + + QueryOpenResult OpenZPDQuery(ReportHandle report_handle, + bool can_close_submission) override; + bool CloseZPDQuery(ReportHandle report_handle, + uint64_t& out_submission) override; + bool DiscardZPDQuery() override; + void PumpQueryResolves() override; + bool AwaitQueryResolve(ReportHandle report_handle, + uint64_t wait_for_submission) override; + + void RecordZPDResolveBatch(); bool device_removed_ = false; bool cache_clear_requested_ = false; + struct PendingQueryResolve { + uint64_t submission = 0; + uint32_t query_index = UINT32_MAX; + uint32_t query_generation = 0; + bool uses_rov_counter = false; + ReportHandle report_handle = kInvalidReportHandle; + }; + uint32_t zpd_active_query_index_ = UINT32_MAX; + uint32_t zpd_active_query_generation_ = 0; + bool zpd_active_query_is_rov_ = false; + bool zpd_query_pool_needs_rov_counter_ = false; + std::deque zpd_resolves_in_flight_; + std::unique_ptr completion_timeline_; bool submission_open_ = false; @@ -525,6 +581,7 @@ class D3D12CommandProcessor final : public CommandProcessor { CommandAllocator* command_allocator_submitted_last_ = nullptr; ID3D12GraphicsCommandList* command_list_ = nullptr; ID3D12GraphicsCommandList1* command_list_1_ = nullptr; + ID3D12GraphicsCommandList2* command_list_2_ = nullptr; DeferredCommandList deferred_command_list_; // Should bindless textures and samplers be used - many times faster @@ -537,6 +594,10 @@ class D3D12CommandProcessor final : public CommandProcessor { std::unique_ptr render_target_cache_; + std::unique_ptr zpd_host_query_pool_; + ID3D12Resource* bindful_zpd_rov_counter_buffer_ = nullptr; + uint32_t bindful_zpd_rov_counter_capacity_ = 0; + std::unique_ptr constant_buffer_pool_; static constexpr uint32_t kViewBindfulHeapSize = 32768; diff --git a/src/xenia/gpu/d3d12/d3d12_zpd_query_pool.cc b/src/xenia/gpu/d3d12/d3d12_zpd_query_pool.cc new file mode 100644 index 000000000..9dbf5f798 --- /dev/null +++ b/src/xenia/gpu/d3d12/d3d12_zpd_query_pool.cc @@ -0,0 +1,421 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/gpu/d3d12/d3d12_zpd_query_pool.h" + +#include + +#include "xenia/base/logging.h" +#include "xenia/gpu/d3d12/deferred_command_list.h" +#include "xenia/ui/d3d12/d3d12_provider.h" +#include "xenia/ui/d3d12/d3d12_util.h" + +namespace xe { +namespace gpu { +namespace d3d12 { + +bool D3D12ZPDQueryPool::EnsureInitialized( + const ui::d3d12::D3D12Provider& provider, uint32_t requested_capacity, + bool can_recreate, bool initialize_rov_counter) { + if (rtv_initialized() && (!initialize_rov_counter || rov_initialized()) && + (capacity_ == requested_capacity || !can_recreate)) { + return true; + } + + if (rtv_initialized() && capacity_ != requested_capacity) { + if (!can_recreate) { + requested_capacity = capacity_; + } else { + // Can't recreate while resolves are in-flight, that would destroy the + // backing resources under pending resolve or copy work. + assert_true(!has_pending_resolve_batch()); + Shutdown(); + } + } + + ID3D12Device* device = provider.GetDevice(); + + if (!rtv_initialized()) { + D3D12_QUERY_HEAP_DESC heap_desc = {}; + heap_desc.Type = D3D12_QUERY_HEAP_TYPE_OCCLUSION; + heap_desc.Count = requested_capacity; + heap_desc.NodeMask = 0; + + if (FAILED( + device->CreateQueryHeap(&heap_desc, IID_PPV_ARGS(&query_heap_)))) { + XELOGW( + "D3D12ZPDQueryPool: Failed to create the ZPD query " + "heap, falling back to fake sample counts."); + return false; + } + + D3D12_RESOURCE_DESC buffer_desc; + ui::d3d12::util::FillBufferResourceDesc( + buffer_desc, sizeof(uint64_t) * requested_capacity, + D3D12_RESOURCE_FLAG_NONE); + + if (FAILED(device->CreateCommittedResource( + &ui::d3d12::util::kHeapPropertiesReadback, + provider.GetHeapFlagCreateNotZeroed(), &buffer_desc, + D3D12_RESOURCE_STATE_COPY_DEST, nullptr, + IID_PPV_ARGS(&readback_buffer_)))) { + XELOGW( + "D3D12ZPDQueryPool: Failed to allocate the ZPD query " + "readback buffer, falling back to fake sample counts."); + Shutdown(); + return false; + } + + D3D12_RANGE read_range = {}; + read_range.Begin = 0; + read_range.End = sizeof(uint64_t) * requested_capacity; + + void* mapping = nullptr; + if (FAILED(readback_buffer_->Map(0, &read_range, &mapping))) { + XELOGW( + "D3D12ZPDQueryPool: Failed to map the ZPD query " + "readback buffer, falling back to fake sample counts."); + Shutdown(); + return false; + } + + readback_mapping_ = reinterpret_cast(mapping); + capacity_ = requested_capacity; + + resolve_batch_pending_.assign(requested_capacity, 0); + resolve_batch_indices_.clear(); + rov_counter_resolve_batch_pending_.assign(requested_capacity, 0); + rov_counter_resolve_batch_indices_.clear(); + resolve_batch_ranges_.clear(); + + free_indices_.clear(); + free_indices_.reserve(requested_capacity); + for (uint32_t i = requested_capacity; i > 0; --i) { + free_indices_.push_back(i - 1); + } + index_generations_.assign(requested_capacity, 0); + } + + if (!initialize_rov_counter || rov_initialized()) { + return true; + } + + if (rov_counter_readback_mapping_ && rov_counter_readback_buffer_) { + D3D12_RANGE written_range = {0, 0}; + rov_counter_readback_buffer_->Unmap(0, &written_range); + } + rov_counter_readback_mapping_ = nullptr; + rov_counter_readback_buffer_.Reset(); + rov_counter_buffer_.Reset(); + rov_counter_resolve_batch_pending_.assign(requested_capacity, 0); + rov_counter_resolve_batch_indices_.clear(); + + D3D12_RESOURCE_DESC counter_buffer_desc; + ui::d3d12::util::FillBufferResourceDesc( + counter_buffer_desc, sizeof(uint32_t) * requested_capacity, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS); + if (FAILED(device->CreateCommittedResource( + &ui::d3d12::util::kHeapPropertiesDefault, + provider.GetHeapFlagCreateNotZeroed(), &counter_buffer_desc, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS, nullptr, + IID_PPV_ARGS(&rov_counter_buffer_)))) { + XELOGW( + "D3D12ZPDQueryPool: Failed to create the ZPD ROV counter " + "buffer, falling back to fake sample counts."); + return false; + } + + D3D12_RESOURCE_DESC readback_buffer_desc; + ui::d3d12::util::FillBufferResourceDesc(readback_buffer_desc, + sizeof(uint32_t) * requested_capacity, + D3D12_RESOURCE_FLAG_NONE); + if (FAILED(device->CreateCommittedResource( + &ui::d3d12::util::kHeapPropertiesReadback, + provider.GetHeapFlagCreateNotZeroed(), &readback_buffer_desc, + D3D12_RESOURCE_STATE_COPY_DEST, nullptr, + IID_PPV_ARGS(&rov_counter_readback_buffer_)))) { + XELOGW( + "D3D12ZPDQueryPool: Failed to create the ZPD ROV counter readback " + "buffer, falling back to fake sample counts."); + rov_counter_buffer_.Reset(); + return false; + } + + D3D12_RANGE read_range = {}; + read_range.Begin = 0; + read_range.End = sizeof(uint32_t) * requested_capacity; + + void* mapping = nullptr; + if (FAILED(rov_counter_readback_buffer_->Map(0, &read_range, &mapping))) { + XELOGW( + "D3D12ZPDQueryPool: Failed to map the ZPD ROV counter readback " + "buffer, falling back to fake sample counts."); + rov_counter_readback_buffer_.Reset(); + rov_counter_buffer_.Reset(); + return false; + } + + rov_counter_readback_mapping_ = reinterpret_cast(mapping); + return true; +} + +void D3D12ZPDQueryPool::Shutdown() { + resolve_batch_pending_.clear(); + resolve_batch_indices_.clear(); + resolve_batch_ranges_.clear(); + rov_counter_resolve_batch_pending_.clear(); + rov_counter_resolve_batch_indices_.clear(); + free_indices_.clear(); + index_generations_.clear(); + + capacity_ = 0; + + if (readback_mapping_ && readback_buffer_) { + // CPU never writes to this READBACK buffer - empty written range. + D3D12_RANGE written_range = {0, 0}; + readback_buffer_->Unmap(0, &written_range); + } + + readback_mapping_ = nullptr; + readback_buffer_.Reset(); + query_heap_.Reset(); + + if (rov_counter_readback_mapping_ && rov_counter_readback_buffer_) { + D3D12_RANGE written_range = {0, 0}; + rov_counter_readback_buffer_->Unmap(0, &written_range); + } + + rov_counter_readback_mapping_ = nullptr; + rov_counter_readback_buffer_.Reset(); + rov_counter_buffer_.Reset(); +} + +bool D3D12ZPDQueryPool::AcquireQueryIndex(uint32_t& query_index, + uint32_t& query_generation) { + if (free_indices_.empty()) { + query_index = UINT32_MAX; + query_generation = 0; + return false; + } + + query_index = free_indices_.back(); + free_indices_.pop_back(); + + assert_true(query_index < index_generations_.size()); + // Bump the generation. Any in-flight readbacks for the slot's previous + // occupants are ignored. + query_generation = ++index_generations_[query_index]; + return true; +} + +void D3D12ZPDQueryPool::ReleaseQueryIndex(uint32_t query_index, + uint32_t query_generation) { + if (query_index >= capacity_) { + return; + } + + if (!GenerationMatches(query_index, query_generation)) { + return; + } + + // Bump generation so a second release with the same generation is rejected. + ++index_generations_[query_index]; + free_indices_.push_back(query_index); +} + +bool D3D12ZPDQueryPool::GenerationMatches(uint32_t query_index, + uint32_t query_generation) const { + return query_index < index_generations_.size() && + index_generations_[query_index] == query_generation; +} + +void D3D12ZPDQueryPool::BeginQuery(DeferredCommandList& deferred_command_list, + uint32_t query_index) const { + if (!query_heap_ || query_index >= capacity_) { + return; + } + + deferred_command_list.D3DBeginQuery(query_heap_.Get(), + D3D12_QUERY_TYPE_OCCLUSION, query_index); +} + +void D3D12ZPDQueryPool::EndQuery(DeferredCommandList& deferred_command_list, + uint32_t query_index) const { + if (!query_heap_ || query_index >= capacity_) { + return; + } + + deferred_command_list.D3DEndQuery(query_heap_.Get(), + D3D12_QUERY_TYPE_OCCLUSION, query_index); +} + +void D3D12ZPDQueryPool::QueueQueryResolve(uint32_t query_index, + bool uses_rov_counter) { + if (query_index >= capacity_) { + return; + } + + // Guard against duplicates. Split paths can touch the same index twice before + // the batch drains at EndSubmission. + if (uses_rov_counter) { + if (!rov_counter_resolve_batch_pending_[query_index]) { + rov_counter_resolve_batch_pending_[query_index] = 1; + rov_counter_resolve_batch_indices_.push_back(query_index); + } + return; + } + + if (!resolve_batch_pending_[query_index]) { + resolve_batch_pending_[query_index] = 1; + resolve_batch_indices_.push_back(query_index); + } +} + +void D3D12ZPDQueryPool::ClearROVCounter( + DeferredCommandList& deferred_command_list, uint32_t query_index) const { + if (!rov_initialized() || query_index >= capacity_) { + return; + } + + // This buffer stays in UNORDERED_ACCESS for the duration of its use. Before + // reusing a slot, order this write after any atomic adds issued by the + // previous query that owned the same index. + D3D12_RESOURCE_BARRIER uav_barrier = {}; + uav_barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_UAV; + uav_barrier.UAV.pResource = rov_counter_buffer_.Get(); + deferred_command_list.D3DResourceBarrier(1, &uav_barrier); + + // Only the selected 32 bit slot needs to be reset, so use + // WriteBufferImmediate instead of transitioning the whole buffer through a + // copy path. + deferred_command_list.D3DWriteBufferImmediate( + rov_counter_buffer_->GetGPUVirtualAddress() + + static_cast(query_index) * sizeof(uint32_t), + 0u); + + // Order the zero write before any upcoming PS atomic adds so the next query + // using this slot sees the cleared counter value. + deferred_command_list.D3DResourceBarrier(1, &uav_barrier); +} + +void D3D12ZPDQueryPool::FlushResolveBatch( + DeferredCommandList& deferred_command_list, bool submission_open) { + if (!submission_open || (resolve_batch_indices_.empty() && + rov_counter_resolve_batch_indices_.empty())) { + return; + } + + // Sorts indices, coalesces contiguous runs into resolve_batch_ranges_, resets + // pending flags, and clears the index list. + auto build_ranges = [this](std::vector& indices, + std::vector& pending) { + std::sort(indices.begin(), indices.end()); + resolve_batch_ranges_.clear(); + uint32_t range_start = 0; + uint32_t range_count = 0; + for (uint32_t index : indices) { + if (range_count == 0) { + range_start = index; + range_count = 1; + continue; + } + if (index == range_start + range_count) { + ++range_count; + continue; + } + resolve_batch_ranges_.push_back({range_start, range_count}); + range_start = index; + range_count = 1; + } + if (range_count != 0) { + resolve_batch_ranges_.push_back({range_start, range_count}); + } + for (uint32_t index : indices) { + pending[index] = 0; + } + indices.clear(); + }; + + if (!resolve_batch_indices_.empty()) { + if (!rtv_initialized()) { + for (uint32_t index : resolve_batch_indices_) { + resolve_batch_pending_[index] = 0; + } + resolve_batch_indices_.clear(); + } else { + build_ranges(resolve_batch_indices_, resolve_batch_pending_); + for (const ResolveRange& range : resolve_batch_ranges_) { + deferred_command_list.D3DResolveQueryData( + query_heap_.Get(), D3D12_QUERY_TYPE_OCCLUSION, range.start, + range.count, readback_buffer_.Get(), + range.start * sizeof(uint64_t)); + } + } + } + + if (rov_counter_resolve_batch_indices_.empty()) { + return; + } + + if (!rov_initialized()) { + for (uint32_t index : rov_counter_resolve_batch_indices_) { + rov_counter_resolve_batch_pending_[index] = 0; + } + rov_counter_resolve_batch_indices_.clear(); + return; + } + + // The shader path writes counters through UAV atomics, so resolve on + // this path means copying the finished 32 bit slots out of the UAV buffer. + // The whole buffer is transitioned for the copy and then returned to + // UNORDERED_ACCESS since D3D12 state is tracked per resource, not per range. + D3D12_RESOURCE_BARRIER barrier = {}; + barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + barrier.Transition.pResource = rov_counter_buffer_.Get(); + barrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + barrier.Transition.StateBefore = D3D12_RESOURCE_STATE_UNORDERED_ACCESS; + barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; + deferred_command_list.D3DResourceBarrier(1, &barrier); + + build_ranges(rov_counter_resolve_batch_indices_, + rov_counter_resolve_batch_pending_); + for (const ResolveRange& range : resolve_batch_ranges_) { + uint64_t offset = static_cast(range.start) * sizeof(uint32_t); + uint64_t size = static_cast(range.count) * sizeof(uint32_t); + deferred_command_list.D3DCopyBufferRegion( + rov_counter_readback_buffer_.Get(), offset, rov_counter_buffer_.Get(), + offset, size); + } + + barrier.Transition.StateBefore = D3D12_RESOURCE_STATE_COPY_SOURCE; + barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_UNORDERED_ACCESS; + deferred_command_list.D3DResourceBarrier(1, &barrier); +} + +uint64_t D3D12ZPDQueryPool::GetQueryReadbackValue(uint32_t query_index, + bool uses_rov_counter) const { + if (query_index >= capacity_) { + return 0; + } + + if (uses_rov_counter) { + // ROV queries read back a translated 32 bit sample count. Widen here so + // paths feed the uint64_t resolve. + return rov_counter_readback_mapping_ + ? static_cast( + rov_counter_readback_mapping_[query_index]) + : 0; + } + + return readback_mapping_ ? readback_mapping_[query_index] : 0; +} + +} // namespace d3d12 +} // namespace gpu +} // namespace xe diff --git a/src/xenia/gpu/d3d12/d3d12_zpd_query_pool.h b/src/xenia/gpu/d3d12/d3d12_zpd_query_pool.h new file mode 100644 index 000000000..94710710e --- /dev/null +++ b/src/xenia/gpu/d3d12/d3d12_zpd_query_pool.h @@ -0,0 +1,129 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_GPU_D3D12_D3D12_ZPD_QUERY_POOL_H_ +#define XENIA_GPU_D3D12_D3D12_ZPD_QUERY_POOL_H_ + +#include +#include + +#include "xenia/gpu/command_processor.h" +#include "xenia/ui/d3d12/d3d12_api.h" + +namespace xe { +namespace ui { +namespace d3d12 { +class D3D12Provider; +} +} // namespace ui + +namespace gpu { +namespace d3d12 { + +class DeferredCommandList; + +// D3D12 occlusion query pool for ZPD reports. Queries live in ID3D12QueryHeap, +// results are copied to a persistent readback buffer via ResolveQueryData. +// +// D3D12 requires BeginQuery and EndQuery to be recorded in the same command +// list, so segments split at EndSubmission. Discarded queries still need a +// paired EndQuery or the heap slot may become undefined on some drivers. +// +// FlushResolveBatch coalesces pending indices into contiguous ranges to cut +// down on ResolveQueryData call count. +// +// ROV queries use a separate path instead of normal D3D12 results. They write +// surviving MSAA coverage into a dedicated buffer, one slot per active query. +// QueueQueryResolve + ClearROVCounter are used instead of BeginQuery and +// EndQuery. +class D3D12ZPDQueryPool { + public: + D3D12ZPDQueryPool() = default; + D3D12ZPDQueryPool(const D3D12ZPDQueryPool&) = delete; + D3D12ZPDQueryPool& operator=(const D3D12ZPDQueryPool&) = delete; + ~D3D12ZPDQueryPool() { Shutdown(); } + + bool EnsureInitialized(const ui::d3d12::D3D12Provider& provider, + uint32_t requested_capacity, bool can_recreate, + bool initialize_rov_counter); + void Shutdown(); + + bool rtv_initialized() const { + return query_heap_ && readback_buffer_ && readback_mapping_ != nullptr && + capacity_ != 0; + } + + uint32_t capacity() const { return capacity_; } + + bool has_pending_resolve_batch() const { + return !resolve_batch_indices_.empty() || + !rov_counter_resolve_batch_indices_.empty(); + } + + bool rov_initialized() const { + return rov_counter_buffer_ && rov_counter_readback_buffer_ && + rov_counter_readback_mapping_ != nullptr && capacity_ != 0; + } + + ID3D12Resource* rov_counter_buffer() const { + return rov_counter_buffer_.Get(); + } + + bool has_free_indices() const { return !free_indices_.empty(); } + + bool AcquireQueryIndex(uint32_t& query_index, uint32_t& query_generation); + void ReleaseQueryIndex(uint32_t query_index, uint32_t query_generation); + bool GenerationMatches(uint32_t query_index, uint32_t query_generation) const; + + void BeginQuery(DeferredCommandList& deferred_command_list, + uint32_t query_index) const; + void EndQuery(DeferredCommandList& deferred_command_list, + uint32_t query_index) const; + void QueueQueryResolve(uint32_t query_index, bool uses_rov_counter); + void ClearROVCounter(DeferredCommandList& deferred_command_list, + uint32_t query_index) const; + + void FlushResolveBatch(DeferredCommandList& deferred_command_list, + bool submission_open); + + uint64_t GetQueryReadbackValue(uint32_t query_index, + bool uses_rov_counter) const; + + private: + Microsoft::WRL::ComPtr query_heap_; + + // Persistently mapped. Results readable once the fence signals. + Microsoft::WRL::ComPtr readback_buffer_; + uint64_t* readback_mapping_ = nullptr; + + Microsoft::WRL::ComPtr rov_counter_buffer_; + Microsoft::WRL::ComPtr rov_counter_readback_buffer_; + uint32_t* rov_counter_readback_mapping_ = nullptr; + + uint32_t capacity_ = 0; + std::vector free_indices_; + + // Bumped on each acquire so stale readbacks from a recycled slot get dropped. + std::vector index_generations_; + + std::vector resolve_batch_pending_; + // Active indices with resolve_batch_pending_[i] == 1, so flush iterates + // only the active entries instead of scanning the full capacity. + std::vector resolve_batch_indices_; + std::vector rov_counter_resolve_batch_pending_; + std::vector rov_counter_resolve_batch_indices_; + // Reusable scratch for coalesced contiguous ranges during flush. + std::vector resolve_batch_ranges_; +}; + +} // namespace d3d12 +} // namespace gpu +} // namespace xe + +#endif // XENIA_GPU_D3D12_D3D12_ZPD_QUERY_POOL_H_ diff --git a/src/xenia/gpu/d3d12/deferred_command_list.cc b/src/xenia/gpu/d3d12/deferred_command_list.cc index feb58e7b1..d0e11928f 100644 --- a/src/xenia/gpu/d3d12/deferred_command_list.cc +++ b/src/xenia/gpu/d3d12/deferred_command_list.cc @@ -28,7 +28,8 @@ DeferredCommandList::DeferredCommandList( void DeferredCommandList::Reset() { command_stream_.clear(); } void DeferredCommandList::Execute(ID3D12GraphicsCommandList* command_list, - ID3D12GraphicsCommandList1* command_list_1) { + ID3D12GraphicsCommandList1* command_list_1, + ID3D12GraphicsCommandList2* command_list_2) { #if XE_GPU_FINE_GRAINED_DRAW_SCOPES SCOPE_profile_cpu_f("gpu"); #endif // XE_GPU_FINE_GRAINED_DRAW_SCOPES @@ -117,6 +118,23 @@ void DeferredCommandList::Execute(ID3D12GraphicsCommandList* command_list, args.start_vertex_location, args.start_instance_location); } } break; + case Command::kD3DBeginQuery: { + auto& args = *reinterpret_cast(stream); + command_list->BeginQuery(args.query_heap, args.query_type, + args.query_index); + } break; + case Command::kD3DEndQuery: { + auto& args = *reinterpret_cast(stream); + command_list->EndQuery(args.query_heap, args.query_type, + args.query_index); + } break; + case Command::kD3DResolveQueryData: { + auto& args = + *reinterpret_cast(stream); + command_list->ResolveQueryData( + args.query_heap, args.query_type, args.start_index, + args.query_count, args.destination_buffer, args.destination_offset); + } break; case Command::kD3DIASetIndexBuffer: { auto view = reinterpret_cast(stream); command_list->IASetIndexBuffer( @@ -278,6 +296,19 @@ void DeferredCommandList::Execute(ID3D12GraphicsCommandList* command_list, : nullptr); } } break; + case Command::kD3DWriteBufferImmediate: { + if (command_list_2 != nullptr) { + auto& args = + *reinterpret_cast( + stream); + D3D12_WRITEBUFFERIMMEDIATE_PARAMETER param; + param.Dest = args.dest; + param.Value = args.value; + D3D12_WRITEBUFFERIMMEDIATE_MODE mode = + D3D12_WRITEBUFFERIMMEDIATE_MODE_DEFAULT; + command_list_2->WriteBufferImmediate(1, ¶m, &mode); + } + } break; default: assert_unhandled_case(header.command); break; diff --git a/src/xenia/gpu/d3d12/deferred_command_list.h b/src/xenia/gpu/d3d12/deferred_command_list.h index 4cf7f1b38..2d66d4c67 100644 --- a/src/xenia/gpu/d3d12/deferred_command_list.h +++ b/src/xenia/gpu/d3d12/deferred_command_list.h @@ -40,7 +40,8 @@ class DeferredCommandList { void Reset(); void Execute(ID3D12GraphicsCommandList* command_list, - ID3D12GraphicsCommandList1* command_list_1); + ID3D12GraphicsCommandList1* command_list_1, + ID3D12GraphicsCommandList2* command_list_2); D3D12_RECT* ClearDepthStencilViewAllocatedRects( D3D12_CPU_DESCRIPTOR_HANDLE depth_stencil_view, @@ -184,6 +185,36 @@ class DeferredCommandList { args.start_instance_location = start_instance_location; } + void D3DBeginQuery(ID3D12QueryHeap* heap, D3D12_QUERY_TYPE type, UINT index) { + auto& args = *reinterpret_cast( + WriteCommand(Command::kD3DBeginQuery, sizeof(D3DQueryArguments))); + args.query_heap = heap; + args.query_type = type; + args.query_index = index; + } + + void D3DEndQuery(ID3D12QueryHeap* heap, D3D12_QUERY_TYPE type, UINT index) { + auto& args = *reinterpret_cast( + WriteCommand(Command::kD3DEndQuery, sizeof(D3DQueryArguments))); + args.query_heap = heap; + args.query_type = type; + args.query_index = index; + } + + void D3DResolveQueryData(ID3D12QueryHeap* heap, D3D12_QUERY_TYPE type, + UINT start_index, UINT query_count, + ID3D12Resource* destination_buffer, + UINT64 destination_offset) { + auto& args = *reinterpret_cast(WriteCommand( + Command::kD3DResolveQueryData, sizeof(D3DResolveQueryDataArguments))); + args.query_heap = heap; + args.query_type = type; + args.start_index = start_index; + args.query_count = query_count; + args.destination_buffer = destination_buffer; + args.destination_offset = destination_offset; + } + void D3DIASetIndexBuffer(const D3D12_INDEX_BUFFER_VIEW* view) { auto& args = *reinterpret_cast(WriteCommand( Command::kD3DIASetIndexBuffer, sizeof(D3D12_INDEX_BUFFER_VIEW))); @@ -441,6 +472,14 @@ class DeferredCommandList { sizeof(D3D12_SAMPLE_POSITION)); } + void D3DWriteBufferImmediate(D3D12_GPU_VIRTUAL_ADDRESS dest, UINT value) { + auto& args = *reinterpret_cast( + WriteCommand(Command::kD3DWriteBufferImmediate, + sizeof(D3DWriteBufferImmediateArguments))); + args.dest = dest; + args.value = value; + } + private: enum class Command { kD3DClearDepthStencilView, @@ -453,6 +492,9 @@ class DeferredCommandList { kD3DDispatch, kD3DDrawIndexedInstanced, kD3DDrawInstanced, + kD3DBeginQuery, + kD3DEndQuery, + kD3DResolveQueryData, kD3DIASetIndexBuffer, kD3DIASetPrimitiveTopology, kD3DIASetVertexBuffers, @@ -478,6 +520,7 @@ class DeferredCommandList { kD3DSetPipelineState, kSetPipelineStateHandle, kD3DSetSamplePositions, + kD3DWriteBufferImmediate, }; struct CommandHeader { @@ -561,6 +604,21 @@ class DeferredCommandList { UINT start_instance_location; }; + struct D3DQueryArguments { + ID3D12QueryHeap* query_heap; + D3D12_QUERY_TYPE query_type; + UINT query_index; + }; + + struct D3DResolveQueryDataArguments { + ID3D12QueryHeap* query_heap; + D3D12_QUERY_TYPE query_type; + UINT start_index; + UINT query_count; + ID3D12Resource* destination_buffer; + UINT64 destination_offset; + }; + struct D3DIASetVertexBuffersHeader { UINT start_slot; UINT num_views; @@ -602,6 +660,11 @@ class DeferredCommandList { D3D12_SAMPLE_POSITION sample_positions[16]; }; + struct D3DWriteBufferImmediateArguments { + D3D12_GPU_VIRTUAL_ADDRESS dest; + UINT value; + }; + void* WriteCommand(Command command, size_t arguments_size_bytes); const D3D12CommandProcessor& command_processor_; diff --git a/src/xenia/gpu/d3d12/pipeline_cache.cc b/src/xenia/gpu/d3d12/pipeline_cache.cc index 8b6b4d1c5..6122979d1 100644 --- a/src/xenia/gpu/d3d12/pipeline_cache.cc +++ b/src/xenia/gpu/d3d12/pipeline_cache.cc @@ -556,6 +556,37 @@ bool PipelineCache::IsCreatingPipelines() { return !creation_queue_.empty() || creation_threads_busy_ != 0; } +void PipelineCache::AwaitPipelineCompletion() { + if (creation_threads_.empty()) { + return; + } + + bool await_creation_completion_event; + { + std::lock_guard lock(creation_request_lock_); + await_creation_completion_event = + !creation_queue_.empty() || creation_threads_busy_ != 0; + if (await_creation_completion_event) { + creation_completion_event_->Reset(); + creation_completion_set_event_ = true; + } + } + + if (await_creation_completion_event) { + creation_request_cond_.notify_one(); + xe::threading::Wait(creation_completion_event_.get(), false); + } +} + +ID3D12PipelineState* PipelineCache::AwaitD3D12PipelineByHandle(void* handle) { + ID3D12PipelineState* pipeline = GetD3D12PipelineByHandle(handle); + if (pipeline != nullptr) { + return pipeline; + } + AwaitPipelineCompletion(); + return GetD3D12PipelineByHandle(handle); +} + D3D12Shader* PipelineCache::LoadShader(xenos::ShaderType shader_type, const uint32_t* host_address, uint32_t dword_count) { diff --git a/src/xenia/gpu/d3d12/pipeline_cache.h b/src/xenia/gpu/d3d12/pipeline_cache.h index 6771fb771..172a1ee29 100644 --- a/src/xenia/gpu/d3d12/pipeline_cache.h +++ b/src/xenia/gpu/d3d12/pipeline_cache.h @@ -71,6 +71,10 @@ class PipelineCache { void EndSubmission(); bool IsCreatingPipelines(); + // Waits for any pipeline creation needed by the current draw path to finish + // before state is consumed. This was added so strict ZPD query paths stop + // racing pipeline compilation and then blocking work on incomplete state. + void AwaitPipelineCompletion(); D3D12Shader* LoadShader(xenos::ShaderType shader_type, const uint32_t* host_address, uint32_t dword_count); @@ -109,6 +113,7 @@ class PipelineCache { return reinterpret_cast(handle)->state.load( std::memory_order_acquire); } + ID3D12PipelineState* AwaitD3D12PipelineByHandle(void* handle); ID3D12RootSignature* GetRootSignatureByHandle(void* handle) const { return reinterpret_cast(handle) diff --git a/src/xenia/gpu/dxbc.h b/src/xenia/gpu/dxbc.h index ca3dfd1e3..af2401b32 100644 --- a/src/xenia/gpu/dxbc.h +++ b/src/xenia/gpu/dxbc.h @@ -1496,6 +1496,7 @@ enum class Opcode : uint32_t { kRcp = 129, kF32ToF16 = 130, kF16ToF32 = 131, + kCountBits = 134, kFirstBitHi = 135, kFirstBitLo = 136, kUBFE = 138, @@ -1515,6 +1516,7 @@ enum class Opcode : uint32_t { kStoreRaw = 166, kAtomicAnd = 169, kAtomicOr = 170, + kAtomicIAdd = 173, kEvalSampleIndex = 204, kEvalCentroid = 205, }; @@ -2248,6 +2250,10 @@ class Assembler { EmitAluOp(Opcode::kF16ToF32, 0b1, dest, src); ++stat_.conversion_instruction_count; } + void OpCountBits(const Dest& dest, const Src& src) { + EmitAluOp(Opcode::kCountBits, 0b1, dest, src); + ++stat_.uint_instruction_count; + } void OpFirstBitHi(const Dest& dest, const Src& src) { EmitAluOp(Opcode::kFirstBitHi, 0b1, dest, src); ++stat_.uint_instruction_count; @@ -2411,6 +2417,10 @@ class Assembler { uint32_t address_components, const Src& value) { EmitAtomicOp(Opcode::kAtomicOr, dest, address, address_components, value); } + void OpAtomicIAdd(const Dest& dest, const Src& address, + uint32_t address_components, const Src& value) { + EmitAtomicOp(Opcode::kAtomicIAdd, dest, address, address_components, value); + } void OpEvalSampleIndex(const Dest& dest, const Src& value, const Src& sample_index) { uint32_t dest_write_mask = dest.GetMask(); diff --git a/src/xenia/gpu/dxbc_shader_translator.cc b/src/xenia/gpu/dxbc_shader_translator.cc index 7af15bc95..4d01159e9 100644 --- a/src/xenia/gpu/dxbc_shader_translator.cc +++ b/src/xenia/gpu/dxbc_shader_translator.cc @@ -173,6 +173,7 @@ void DxbcShaderTranslator::Reset() { uav_count_ = 0; uav_index_shared_memory_ = kBindingIndexUnallocated; uav_index_edram_ = kBindingIndexUnallocated; + uav_index_zpd_rov_counter_ = kBindingIndexUnallocated; sampler_bindings_.clear(); @@ -2146,7 +2147,9 @@ constexpr DxbcShaderTranslator::SystemConstantRdef {"xe_edram_32bpp_tile_pitch_dwords_scaled", ShaderRdefTypeIndex::kUint, sizeof(uint32_t)}, {"xe_edram_depth_base_dwords_scaled", ShaderRdefTypeIndex::kUint, - sizeof(uint32_t), sizeof(uint32_t)}, + sizeof(uint32_t)}, + {"xe_zpd_rov_counter_index", ShaderRdefTypeIndex::kUint, + sizeof(uint32_t)}, {"xe_color_exp_bias", ShaderRdefTypeIndex::kFloat4, sizeof(float) * 4}, @@ -2548,6 +2551,11 @@ void DxbcShaderTranslator::WriteResourceDefinition() { if (uav_index_edram_ != kBindingIndexUnallocated) { name_ptr += dxbc::AppendAlignedString(shader_object_, "xe_edram"); } + uint32_t zpd_rov_counter_name_ptr = name_ptr; + if (uav_index_zpd_rov_counter_ != kBindingIndexUnallocated) { + name_ptr += + dxbc::AppendAlignedString(shader_object_, "xe_zpd_rov_counter_uav"); + } uint32_t bindings_position_dwords = uint32_t(shader_object_.size()); @@ -2678,6 +2686,13 @@ void DxbcShaderTranslator::WriteResourceDefinition() { uav.dimension = dxbc::RdefDimension::kUAVBuffer; uav.sample_count = UINT32_MAX; uav.bind_point = uint32_t(UAVRegister::kEdram); + } else if (i == uav_index_zpd_rov_counter_) { + // ROV ZPD counter buffer. + uav.name_ptr = zpd_rov_counter_name_ptr; + uav.type = dxbc::RdefInputType::kUAVRWByteAddress; + uav.return_type = dxbc::ResourceReturnType::kMixed; + uav.dimension = dxbc::RdefDimension::kUAVBuffer; + uav.bind_point = uint32_t(UAVRegister::kZpdRovCounter); } else { assert_unhandled_case(i); } @@ -3565,6 +3580,12 @@ void DxbcShaderTranslator::WriteShaderCode() { dxbc::Src::U(dxbc::Src::Dcl, uav_index_edram_, uint32_t(UAVRegister::kEdram), uint32_t(UAVRegister::kEdram))); + } else if (i == uav_index_zpd_rov_counter_) { + // ROV ZPD counter buffer. + ao_.OpDclUnorderedAccessViewRaw( + 0, dxbc::Src::U(dxbc::Src::Dcl, uav_index_zpd_rov_counter_, + uint32_t(UAVRegister::kZpdRovCounter), + uint32_t(UAVRegister::kZpdRovCounter))); } else { assert_unhandled_case(i); } diff --git a/src/xenia/gpu/dxbc_shader_translator.h b/src/xenia/gpu/dxbc_shader_translator.h index 2b5e8599c..f71473b43 100644 --- a/src/xenia/gpu/dxbc_shader_translator.h +++ b/src/xenia/gpu/dxbc_shader_translator.h @@ -329,7 +329,9 @@ class DxbcShaderTranslator : public ShaderTranslator { uint32_t alpha_to_mask; uint32_t edram_32bpp_tile_pitch_dwords_scaled; uint32_t edram_depth_base_dwords_scaled; - uint32_t padding_edram_depth_base_dwords_scaled; + // UINT32_MAX when this draw is outside an active ZPD segment. The shader + // helper should treat that as a skip sentinel. + uint32_t zpd_rov_counter_index; float color_exp_bias[4]; @@ -431,6 +433,7 @@ class DxbcShaderTranslator : public ShaderTranslator { kAlphaToMask, kEdram32bppTilePitchDwordsScaled, kEdramDepthBaseDwordsScaled, + kZpdRovCounterIndex, kColorExpBias, @@ -516,6 +519,7 @@ class DxbcShaderTranslator : public ShaderTranslator { enum class UAVRegister { kSharedMemory, kEdram, + kZpdRovCounter, }; uint64_t GetDefaultVertexShaderModification( @@ -762,6 +766,9 @@ class DxbcShaderTranslator : public ShaderTranslator { // unchanged or known that it's safe not to await kills/alphatest/AtoC), // returns from the shader. void ROV_DepthStencilTest(); + // Adds the surviving coverage MSAA counts from ROV params to the active ZPD + // counter slot after the final PS depth/stencil decision. + void ROV_AddPassedMSAASamplesToZPD(); // Unpacks a 32bpp or a 64bpp color in packed_temp.packed_temp_components to // color_temp, using 2 temporary VGPRs. void ROV_UnpackColor(uint32_t rt_index, uint32_t packed_temp, @@ -1207,6 +1214,7 @@ class DxbcShaderTranslator : public ShaderTranslator { uint32_t uav_count_; uint32_t uav_index_shared_memory_; uint32_t uav_index_edram_; + uint32_t uav_index_zpd_rov_counter_; std::vector sampler_bindings_; }; diff --git a/src/xenia/gpu/dxbc_shader_translator_om.cc b/src/xenia/gpu/dxbc_shader_translator_om.cc index a181008ed..e46a7ed06 100644 --- a/src/xenia/gpu/dxbc_shader_translator_om.cc +++ b/src/xenia/gpu/dxbc_shader_translator_om.cc @@ -2120,6 +2120,52 @@ void DxbcShaderTranslator::CompletePixelShader_AlphaToMask() { a_.OpEndIf(); } +void DxbcShaderTranslator::ROV_AddPassedMSAASamplesToZPD() { + if (uav_index_zpd_rov_counter_ == kBindingIndexUnallocated) { + uav_index_zpd_rov_counter_ = uav_count_++; + } + + uint32_t temp = PushSystemTemp(); + dxbc::Dest temp_x_dest(dxbc::Dest::R(temp, 0b0001)); + dxbc::Src temp_x_src(dxbc::Src::R(temp, dxbc::Src::kXXXX)); + dxbc::Dest temp_y_dest(dxbc::Dest::R(temp, 0b0010)); + dxbc::Src temp_y_src(dxbc::Src::R(temp, dxbc::Src::kYYYY)); + + dxbc::Src counter_index_src(LoadSystemConstant( + SystemConstants::Index::kZpdRovCounterIndex, + offsetof(SystemConstants, zpd_rov_counter_index), dxbc::Src::kXXXX)); + + // UINT32_MAX means no ZPD segment is currently open for this draw. + a_.OpINE(temp_x_dest, counter_index_src, dxbc::Src::LU(UINT32_MAX)); + a_.OpIf(true, temp_x_src); + + { + // Only bits 0:3 are surviving coverage. 4:7 are deferred depth/stencil + // writes and don't contribute to the counter. + a_.OpAnd(temp_x_dest, + dxbc::Src::R(system_temp_rov_params_, dxbc::Src::kXXXX), + dxbc::Src::LU((uint32_t(1) << 4) - 1)); + a_.OpCountBits(temp_x_dest, temp_x_src); + a_.OpIf(true, temp_x_src); + { + // The counter UAV is raw, so address it in bytes. + // One counter slot is one uint32_t. + a_.OpUMul(dxbc::Dest::Null(), temp_y_dest, counter_index_src, + dxbc::Src::LU(sizeof(uint32_t))); + // Add the number of samples that survived depth/stencil for this pixel to + // the active query slot. This slot is copied to the readback buffer when + // the ZPD segment is closed. + a_.OpAtomicIAdd(dxbc::Dest::U(uav_index_zpd_rov_counter_, + uint32_t(UAVRegister::kZpdRovCounter), 0), + temp_y_src, 0b0001, temp_x_src); + } + a_.OpEndIf(); + } + a_.OpEndIf(); + + PopSystemTemp(); +} + void DxbcShaderTranslator::CompletePixelShader_WriteToROV() { uint32_t temp = PushSystemTemp(); dxbc::Dest temp_x_dest(dxbc::Dest::R(temp, 0b0001)); @@ -2176,6 +2222,8 @@ void DxbcShaderTranslator::CompletePixelShader_WriteToROV() { // system_temp_rov_params_.y (the depth / stencil sample address) is not // needed anymore, can be used for color writing. + ROV_AddPassedMSAASamplesToZPD(); + if (!is_depth_only_pixel_shader_) { // Check if any sample is still covered after depth testing and writing, // skip color writing completely in this case. diff --git a/src/xenia/gpu/gpu_flags.cc b/src/xenia/gpu/gpu_flags.cc index a2e24ee5b..839eb8b8e 100644 --- a/src/xenia/gpu/gpu_flags.cc +++ b/src/xenia/gpu/gpu_flags.cc @@ -59,21 +59,39 @@ DEFINE_bool( "when MSAA is used with fullscreen passes.", "GPU"); -DEFINE_int32(query_occlusion_querybatch_range, 0, - "Range of fake ZPD sample count values to cycle for titles that " - "use D3D QueryBatch. 0 disables this behavior entirely. Any non-" - "zero values should only be used if required for specific titles.", +DEFINE_int32(occlusion_query_fake_lower_threshold, 80, + "Lower end of the fake sample count value written on " + "EVENT_WRITE_ZPD when real occlusion queries are disabled.\n" + "-1 writes nothing, resulting in some games that sit and hang.\n" + "0 means the fake result stays fully occluded.", "GPU"); -DEFINE_int32(query_occlusion_sample_lower_threshold, 80, - "If set to -1 no sample counts are written, games may hang. Else, " - "the sample count of every tile will be incremented on every " - "EVENT_WRITE_ZPD by this number. Setting this to 0 means " - "everything is reported as occluded.", +DEFINE_int32(occlusion_query_fake_upper_threshold, 100, + "Upper end of the fake sample count value written on " + "EVENT_WRITE_ZPD when real occlusion queries are disabled.\n" + "Keep this higher than occlusion_query_fake_lower_threshold.\n" + "Ignored if occlusion_query_fake_lower_threshold is -1.", "GPU"); -DEFINE_int32( - query_occlusion_sample_upper_threshold, 100, - "Set to higher number than query_occlusion_sample_lower_threshold. This " - "value is ignored if query_occlusion_sample_lower_threshold is set to -1.", +DEFINE_bool(occlusion_query_fast_preserve_cached_zero, false, + "Alternate fast ZPD behavior that allows saved zero results to be " + "used for unresolved fast writes instead of forcing them visible.\n" + "This can materially improve flare accuracy in titles that might " + "otherwise need strict mode, but it also tends to break occlusion " + "culling in some titles.", + "GPU"); +DEFINE_int32(occlusion_query_querybatch_range, 0, + "Range of fake sample count values to walk for titles using the " + "D3D QueryBatch standard before wrapping back to " + "occlusion_query_fake_lower_threshold.\n" + "This shouldn't be changed from the default value of 0 (disabled) " + "unless necessary for a specific title.", + "GPU"); +DEFINE_double( + occlusion_query_sample_count_saturation, 1.0, + "Compress higher occlusion query sample counts before guest writeback.\n" + "This can be useful if effects such as lens flares appear too strong.\n" + "1.0 = default behavior\n" + "0.0 = collapse all nonzero sample counts to 1\n" + "Values around 0.90 are a good starting point for subtle tuning.", "GPU"); DEFINE_int32(anisotropic_override, -1, diff --git a/src/xenia/gpu/gpu_flags.h b/src/xenia/gpu/gpu_flags.h index 6d92fc57c..701903ff6 100644 --- a/src/xenia/gpu/gpu_flags.h +++ b/src/xenia/gpu/gpu_flags.h @@ -26,11 +26,17 @@ DECLARE_bool(non_seamless_cube_map); DECLARE_bool(half_pixel_offset); -DECLARE_int32(query_occlusion_querybatch_range); +DECLARE_string(occlusion_query); -DECLARE_int32(query_occlusion_sample_lower_threshold); +DECLARE_int32(occlusion_query_fake_lower_threshold); -DECLARE_int32(query_occlusion_sample_upper_threshold); +DECLARE_int32(occlusion_query_fake_upper_threshold); + +DECLARE_bool(occlusion_query_fast_preserve_cached_zero); + +DECLARE_int32(occlusion_query_querybatch_range); + +DECLARE_double(occlusion_query_sample_count_saturation); DECLARE_int32(anisotropic_override); diff --git a/src/xenia/gpu/pm4_command_processor_implement.h b/src/xenia/gpu/pm4_command_processor_implement.h index 870e85833..c66d9fbf3 100644 --- a/src/xenia/gpu/pm4_command_processor_implement.h +++ b/src/xenia/gpu/pm4_command_processor_implement.h @@ -954,65 +954,95 @@ bool COMMAND_PROCESSOR::ExecutePacketType3_EVENT_WRITE_EXT( return true; } -static uint32_t samples = cvars::query_occlusion_sample_upper_threshold; -static uint32_t batched_samples = 0; - XE_NOINLINE bool COMMAND_PROCESSOR::ExecutePacketType3_EVENT_WRITE_ZPD( uint32_t packet, uint32_t count) XE_RESTRICT { - // Set by D3D as BE but struct ABI is LE - const uint32_t kQueryFinished = xe::byte_swap(0xFFFFFEED); assert_true(count == 1); uint32_t initiator = reader_.ReadAndSwap(); // Writeback initiator. COMMAND_PROCESSOR::WriteEventInitiator(initiator & 0x3F); - if (cvars::query_occlusion_sample_lower_threshold < 0) { + uint32_t report_address = + register_file_->values[XE_GPU_REG_RB_SAMPLE_COUNT_ADDR]; + uint32_t report_record_base = XenosZPDReport::GetRecordBase(report_address); + bool is_begin_record = XenosZPDReport::IsBeginRecord(report_address); + bool is_end_record = XenosZPDReport::IsEndRecord(report_address); + + xe_gpu_depth_sample_counts* report = + report_record_base + ? memory_->TranslatePhysical( + report_record_base) + : nullptr; + + // True if the record has the pending D3D sentinel. + // Useful as a hint, but not authoritative for report boundaries. + // QueryBatch titles can have multiple pending sentinels in a row and don't + // necessarily update in an order we currently observe. + bool guest_marks_end = report && XenosZPDReport::HasPendingSentinel(report); + bool logical_active = zpd_active_segment_.logical_active; + + // QueryBatch fake fallback, which ignores record layout and just returns an + // incrementing sample count on each event. + if (cvars::occlusion_query_querybatch_range > 0) { + uint32_t sample_count = + XenosZPDReport::QueryBatchFakeSamples(querybatch_zpd_sample_count_); + if (report) { + // Both QueryBatch and conventional fake samples skip elective saturation. + XenosZPDReport::WriteSampleCount(report, sample_count, false); + } return true; } - // Occlusion queries: - // This command is send on query begin and end. - // As a workaround report some fixed amount of passed samples. - auto* pSampleCounts = memory_->TranslatePhysical( - register_file_->values[XE_GPU_REG_RB_SAMPLE_COUNT_ADDR]); - // 0xFFFFFEED is written to this two locations by D3D only on D3DISSUE_END - // and used to detect a finished query. - bool is_end_via_z_pass = pSampleCounts->ZPass_A == kQueryFinished || - pSampleCounts->ZPass_B == kQueryFinished; - // Older versions of D3D also checks for ZFail (4D5307D5). - bool is_end_via_z_fail = pSampleCounts->ZFail_A == kQueryFinished || - pSampleCounts->ZFail_B == kQueryFinished; - std::memset(pSampleCounts, 0, sizeof(xe_gpu_depth_sample_counts)); - // Titles that use QueryBatch (4D5309B1, 4E4D0801) don't use an END result for - // every query. Each ISSUE snapshots the sample count into a new slot and - // recovers the total by looking at differences between slots. - // END detection isn't sufficient here. - if (cvars::query_occlusion_querybatch_range > 0) { - // Mimic batched behavior by reporting a running count and writing it on - // every event. - const uint32_t base = - static_cast(cvars::query_occlusion_sample_lower_threshold); - const uint32_t range = - static_cast(cvars::query_occlusion_querybatch_range); - if (batched_samples < base || batched_samples - base + 1 >= range) { - batched_samples = base; - } else { - ++batched_samples; + if (GetZPDMode() != ZPDMode::kFake && !zpd_force_fake_fallback_) { + if (logical_active && is_end_record) { + COMMAND_PROCESSOR::EndZPDReport(report_address, false); + return true; } - pSampleCounts->ZPass_A = batched_samples; - pSampleCounts->Total_A = batched_samples; - } else if (is_end_via_z_pass || is_end_via_z_fail) { - pSampleCounts->ZPass_A = samples; - pSampleCounts->Total_A = samples; + if (is_begin_record) { + // Clear the record so the game knows the BEGIN was processed and + // stale sentinel data from a prior query lifetime doesn't persist. + if (report) { + std::memset(report, 0, sizeof(xe_gpu_depth_sample_counts)); + } + COMMAND_PROCESSOR::BeginZPDReport(report_address); + return true; + } + if (!logical_active && is_end_record) { + // No logical report is active for this slot, so this is likely an + // orphaned END. In fast mode, replay the last cached delta so polling + // code does not sit on the sentinel forever. + if (GetZPDMode() == ZPDMode::kFast) { + uint32_t cached_delta = 1; + auto cache_it = fast_zpd_report_cached_values_.find(report_record_base); + if (cache_it != fast_zpd_report_cached_values_.end()) { + cached_delta = cache_it->second; + } + COMMAND_PROCESSOR::WriteZPDReport(0, report_record_base, 0, + cached_delta, false); + } else { + // In strict mode, just pump in case a previous report has resolved. + COMMAND_PROCESSOR::PumpQueryResolves(); + } + return true; + } + // Address is neither BEGIN nor END (non-standard layout). Fall through + // to the fake path so the guest at least gets a result written rather + // than leaving the sentinel in place forever. } - samples = - samples <= static_cast( - cvars::query_occlusion_sample_lower_threshold) - ? static_cast(cvars::query_occlusion_sample_upper_threshold) - : samples - 1; + // Conventional fake fallback, which only touches records marked as pending. + if (cvars::occlusion_query_fake_lower_threshold < 0 || !report_record_base || + !guest_marks_end) { + return true; + } + fake_zpd_sample_count_ = + (fake_zpd_sample_count_ <= + static_cast(cvars::occlusion_query_fake_lower_threshold)) + ? static_cast(cvars::occlusion_query_fake_upper_threshold) + : fake_zpd_sample_count_ - 1; + + XenosZPDReport::WriteSampleCount(report, fake_zpd_sample_count_, false); return true; } diff --git a/src/xenia/gpu/spirv_shader_translator.cc b/src/xenia/gpu/spirv_shader_translator.cc index 83ba94461..1171e365a 100644 --- a/src/xenia/gpu/spirv_shader_translator.cc +++ b/src/xenia/gpu/spirv_shader_translator.cc @@ -283,12 +283,14 @@ void SpirvShaderTranslator::StartTranslation() { type_uint_}, {"alpha_test_reference", offsetof(SystemConstants, alpha_test_reference), type_float_}, - {"alpha_to_mask", offsetof(SystemConstants, alpha_to_mask), type_uint_}, {"edram_32bpp_tile_pitch_dwords_scaled", offsetof(SystemConstants, edram_32bpp_tile_pitch_dwords_scaled), type_uint_}, {"edram_depth_base_dwords_scaled", offsetof(SystemConstants, edram_depth_base_dwords_scaled), type_uint_}, + {"alpha_to_mask", offsetof(SystemConstants, alpha_to_mask), type_uint_}, + {"zpd_fsi_counter_index", + offsetof(SystemConstants, zpd_fsi_counter_index), type_uint_}, {"color_exp_bias", offsetof(SystemConstants, color_exp_bias), type_float4_}, {"edram_poly_offset_front_scale", @@ -2161,6 +2163,37 @@ void SpirvShaderTranslator::StartFragmentShaderBeforeMain() { if (features_.spirv_version >= spv::Spv_1_4) { main_interface_.push_back(buffer_edram_); } + + // ZPD FSI counter buffer uint[]. + id_vector_temp_.clear(); + id_vector_temp_.push_back(builder_->makeRuntimeArray(type_uint_)); + builder_->addDecoration(id_vector_temp_.back(), spv::DecorationArrayStride, + sizeof(uint32_t)); + spv::Id type_zpd_fsi_counter = + builder_->makeStructType(id_vector_temp_, "XeZPDFSICounter"); + builder_->addMemberName(type_zpd_fsi_counter, 0, "counter"); + builder_->addMemberDecoration(type_zpd_fsi_counter, 0, + spv::DecorationCoherent); + builder_->addMemberDecoration(type_zpd_fsi_counter, 0, + spv::DecorationRestrict); + builder_->addMemberDecoration(type_zpd_fsi_counter, 0, + spv::DecorationOffset, 0); + builder_->addDecoration(type_zpd_fsi_counter, + features_.spirv_version >= spv::Spv_1_3 + ? spv::DecorationBlock + : spv::DecorationBufferBlock); + buffer_zpd_fsi_counter_ = builder_->createVariable( + spv::NoPrecision, + features_.spirv_version >= spv::Spv_1_3 ? spv::StorageClassStorageBuffer + : spv::StorageClassUniform, + type_zpd_fsi_counter, "xe_zpd_fsi_counter"); + builder_->addDecoration(buffer_zpd_fsi_counter_, + spv::DecorationDescriptorSet, + int(kDescriptorSetSharedMemoryAndEdram)); + builder_->addDecoration(buffer_zpd_fsi_counter_, spv::DecorationBinding, 2); + if (features_.spirv_version >= spv::Spv_1_4) { + main_interface_.push_back(buffer_zpd_fsi_counter_); + } } bool param_gen_needed = !is_depth_only_fragment_shader_ && diff --git a/src/xenia/gpu/spirv_shader_translator.h b/src/xenia/gpu/spirv_shader_translator.h index db7365389..f3214bdd0 100644 --- a/src/xenia/gpu/spirv_shader_translator.h +++ b/src/xenia/gpu/spirv_shader_translator.h @@ -224,6 +224,11 @@ class SpirvShaderTranslator : public ShaderTranslator { // If alpha to mask is enabled, bits 0:7 are sample offsets, and bit 8 must // be 1. uint32_t alpha_to_mask; + // UINT32_MAX when the draw is outside an active ZPD segment, which is used + // as a skip writing sentinel to the FSI counter buffer. + uint32_t zpd_fsi_counter_index; + // Align for std140. + uint32_t color_exp_bias_padding[2]; float color_exp_bias[4]; @@ -726,6 +731,9 @@ class SpirvShaderTranslator : public ShaderTranslator { // flow because of taking derivatives of the fragment depth. void FSI_DepthStencilTest(spv::Id msaa_samples, bool sample_mask_potentially_narrowed_previouly); + // Adds the surviving coverage MSAA counts from FSI to the active ZPD counter + // slot after final PS depth/stencil. + void FSI_AddPassedMSAASamplesToZPD(); // Alpha to coverage helper - tests one sample. // coverage_out is modified to include this sample if it passes. @@ -878,9 +886,10 @@ class SpirvShaderTranslator : public ShaderTranslator { kSystemConstantTextureSwizzles, kSystemConstantTexturesResolved, kSystemConstantAlphaTestReference, - kSystemConstantAlphaToMask, kSystemConstantEdram32bppTilePitchDwordsScaled, kSystemConstantEdramDepthBaseDwordsScaled, + kSystemConstantAlphaToMask, + kSystemConstantZpdFsiCounterIndex, kSystemConstantColorExpBias, kSystemConstantEdramPolyOffsetFrontScale, kSystemConstantEdramPolyOffsetBackScale, @@ -905,6 +914,7 @@ class SpirvShaderTranslator : public ShaderTranslator { spv::Id buffers_shared_memory_; spv::Id buffer_edram_; + spv::Id buffer_zpd_fsi_counter_; // Not using combined images and samplers because // maxPerStageDescriptorSamplers is often lower than diff --git a/src/xenia/gpu/spirv_shader_translator_rb.cc b/src/xenia/gpu/spirv_shader_translator_rb.cc index 5efeafe7b..3e0ca7574 100644 --- a/src/xenia/gpu/spirv_shader_translator_rb.cc +++ b/src/xenia/gpu/spirv_shader_translator_rb.cc @@ -743,6 +743,8 @@ void SpirvShaderTranslator::CompleteFragmentShaderInMain() { } } + FSI_AddPassedMSAASamplesToZPD(); + if (color_write_depth_stencil_condition != spv::NoResult) { // Skip all color operations if the pixel has failed the tests entirely. block_fsi_if_after_depth_stencil = &builder_->makeNewBlock(); @@ -1815,6 +1817,61 @@ spv::Id SpirvShaderTranslator::FSI_AddSampleOffset(spv::Id sample_0_address, sample_offset); } +void SpirvShaderTranslator::FSI_AddPassedMSAASamplesToZPD() { + assert_true(edram_fragment_shader_interlock_); + assert_true(buffer_zpd_fsi_counter_ != spv::NoResult); + + // UINT32_MAX means no ZPD segment is currently open for this draw. + id_vector_temp_.clear(); + id_vector_temp_.push_back( + builder_->makeIntConstant(kSystemConstantZpdFsiCounterIndex)); + spv::Id counter_index = builder_->createLoad( + builder_->createAccessChain(spv::StorageClassUniform, + uniform_system_constants_, id_vector_temp_), + spv::NoPrecision); + SpirvBuilder::IfBuilder if_counter_open( + builder_->createBinOp(spv::OpINotEqual, type_bool_, counter_index, + builder_->makeUintConstant(UINT32_MAX)), + spv::SelectionControlDontFlattenMask, *builder_); + + // Only bits 0:3 are surviving coverage. 4:7 are deferred depth/stencil and + // don't contribute to the counter. + spv::Id passed_sample_count = builder_->createUnaryOp( + spv::OpBitCount, type_uint_, + builder_->createBinOp( + spv::OpBitwiseAnd, type_uint_, main_fsi_sample_mask_, + builder_->makeUintConstant((uint32_t(1) << 4) - 1))); + + SpirvBuilder::IfBuilder if_any_samples( + builder_->createBinOp(spv::OpINotEqual, type_bool_, passed_sample_count, + const_uint_0_), + spv::SelectionControlDontFlattenMask, *builder_); + + spv::StorageClass storage_class = features_.spirv_version >= spv::Spv_1_3 + ? spv::StorageClassStorageBuffer + : spv::StorageClassUniform; + spv::Id const_scope_device = + builder_->makeUintConstant(static_cast(spv::ScopeDevice)); + spv::Id const_semantics_relaxed = const_uint_0_; + + id_vector_temp_.clear(); + id_vector_temp_.push_back(const_int_0_); + id_vector_temp_.push_back( + builder_->createUnaryOp(spv::OpBitcast, type_int_, counter_index)); + spv::Id counter_ptr = builder_->createAccessChain( + storage_class, buffer_zpd_fsi_counter_, id_vector_temp_); + + // Add the number of samples that survived final depth/stencil for this + // fragment to the active query slot, which is copied to the ZPD readback + // buffer when the query segment is closed. + builder_->createQuadOp(spv::OpAtomicIAdd, type_uint_, counter_ptr, + const_scope_device, const_semantics_relaxed, + passed_sample_count); + + if_any_samples.makeEndIf(); + if_counter_open.makeEndIf(); +} + void SpirvShaderTranslator::FSI_DepthStencilTest( spv::Id msaa_samples, bool sample_mask_potentially_narrowed_previouly) { bool is_early = FSI_IsDepthStencilEarly(); diff --git a/src/xenia/gpu/vulkan/deferred_command_buffer.cc b/src/xenia/gpu/vulkan/deferred_command_buffer.cc index 0ddfef765..b8355a4ee 100644 --- a/src/xenia/gpu/vulkan/deferred_command_buffer.cc +++ b/src/xenia/gpu/vulkan/deferred_command_buffer.cc @@ -116,6 +116,27 @@ void DeferredCommandBuffer::Execute(VkCommandBuffer command_buffer) { dfn.vkCmdBindVertexBuffers(command_buffer, args.first_binding, args.binding_count, buffers, offsets); } break; + case Command::kVkBeginQuery: { + auto& args = *reinterpret_cast(stream); + dfn.vkCmdBeginQuery(command_buffer, args.query_pool, args.query, + args.flags); + } break; + case Command::kVkEndQuery: { + auto& args = *reinterpret_cast(stream); + dfn.vkCmdEndQuery(command_buffer, args.query_pool, args.query); + } break; + case Command::kVkCopyQueryPoolResults: { + auto& args = + *reinterpret_cast(stream); + dfn.vkCmdCopyQueryPoolResults( + command_buffer, args.query_pool, args.first_query, args.query_count, + args.dst_buffer, args.dst_offset, args.stride, args.flags); + } break; + case Command::kVkResetQueryPool: { + auto& args = *reinterpret_cast(stream); + dfn.vkCmdResetQueryPool(command_buffer, args.query_pool, + args.first_query, args.query_count); + } break; case Command::kVkClearAttachments: { auto& args = *reinterpret_cast(stream); @@ -153,6 +174,12 @@ void DeferredCommandBuffer::Execute(VkCommandBuffer command_buffer) { xe::align(sizeof(ArgsVkCopyBuffer), alignof(VkBufferCopy)))); } break; + case Command::kVkFillBuffer: { + auto& args = *reinterpret_cast(stream); + dfn.vkCmdFillBuffer(command_buffer, args.dst_buffer, args.dst_offset, + args.size, args.data); + } break; + case Command::kVkCopyBufferToImage: { auto& args = *reinterpret_cast(stream); dfn.vkCmdCopyBufferToImage( diff --git a/src/xenia/gpu/vulkan/deferred_command_buffer.h b/src/xenia/gpu/vulkan/deferred_command_buffer.h index 4fef1ee6f..a0629507f 100644 --- a/src/xenia/gpu/vulkan/deferred_command_buffer.h +++ b/src/xenia/gpu/vulkan/deferred_command_buffer.h @@ -131,6 +131,46 @@ class DeferredCommandBuffer { sizeof(VkDeviceSize) * binding_count); } + void CmdVkBeginQuery(VkQueryPool query_pool, uint32_t query, + VkQueryControlFlags flags) { + auto& args = *reinterpret_cast( + WriteCommand(Command::kVkBeginQuery, sizeof(ArgsVkBeginQuery))); + args.query_pool = query_pool; + args.query = query; + args.flags = flags; + } + + void CmdVkEndQuery(VkQueryPool query_pool, uint32_t query) { + auto& args = *reinterpret_cast( + WriteCommand(Command::kVkEndQuery, sizeof(ArgsVkEndQuery))); + args.query_pool = query_pool; + args.query = query; + } + + void CmdVkCopyQueryPoolResults(VkQueryPool query_pool, uint32_t first_query, + uint32_t query_count, VkBuffer dst_buffer, + VkDeviceSize dst_offset, VkDeviceSize stride, + VkQueryResultFlags flags) { + auto& args = *reinterpret_cast(WriteCommand( + Command::kVkCopyQueryPoolResults, sizeof(ArgsVkCopyQueryPoolResults))); + args.query_pool = query_pool; + args.first_query = first_query; + args.query_count = query_count; + args.dst_buffer = dst_buffer; + args.dst_offset = dst_offset; + args.stride = stride; + args.flags = flags; + } + + void CmdVkResetQueryPool(VkQueryPool query_pool, uint32_t first_query, + uint32_t query_count) { + auto& args = *reinterpret_cast( + WriteCommand(Command::kVkResetQueryPool, sizeof(ArgsVkResetQueryPool))); + args.query_pool = query_pool; + args.first_query = first_query; + args.query_count = query_count; + } + void CmdClearAttachmentsEmplace(uint32_t attachment_count, VkClearAttachment*& attachments_out, uint32_t rect_count, @@ -232,6 +272,16 @@ class DeferredCommandBuffer { regions, sizeof(VkBufferImageCopy) * region_count); } + void CmdVkFillBuffer(VkBuffer dst_buffer, VkDeviceSize dst_offset, + VkDeviceSize size, uint32_t data) { + auto& args = *reinterpret_cast( + WriteCommand(Command::kVkFillBuffer, sizeof(ArgsVkFillBuffer))); + args.dst_buffer = dst_buffer; + args.dst_offset = dst_offset; + args.size = size; + args.data = data; + } + VkImageBlit* CmdBlitImageEmplace(VkImage src_image, VkImageLayout src_image_layout, VkImage dst_image, @@ -393,10 +443,15 @@ class DeferredCommandBuffer { kVkBindIndexBuffer, kVkBindPipeline, kVkBindVertexBuffers, + kVkBeginQuery, + kVkEndQuery, + kVkCopyQueryPoolResults, + kVkResetQueryPool, kVkClearAttachments, kVkClearColorImage, kVkCopyBuffer, kVkCopyBufferToImage, + kVkFillBuffer, kVkBlitImage, kVkDispatch, kVkDraw, @@ -459,6 +514,33 @@ class DeferredCommandBuffer { static_assert(alignof(VkDeviceSize) <= alignof(uintmax_t)); }; + struct ArgsVkBeginQuery { + VkQueryPool query_pool; + uint32_t query; + VkQueryControlFlags flags; + }; + + struct ArgsVkEndQuery { + VkQueryPool query_pool; + uint32_t query; + }; + + struct ArgsVkCopyQueryPoolResults { + VkQueryPool query_pool; + uint32_t first_query; + uint32_t query_count; + VkBuffer dst_buffer; + VkDeviceSize dst_offset; + VkDeviceSize stride; + VkQueryResultFlags flags; + }; + + struct ArgsVkResetQueryPool { + VkQueryPool query_pool; + uint32_t first_query; + uint32_t query_count; + }; + struct ArgsVkClearAttachments { uint32_t attachment_count; uint32_t rect_count; @@ -493,6 +575,13 @@ class DeferredCommandBuffer { static_assert(alignof(VkBufferImageCopy) <= alignof(uintmax_t)); }; + struct ArgsVkFillBuffer { + VkBuffer dst_buffer; + VkDeviceSize dst_offset; + VkDeviceSize size; + uint32_t data; + }; + struct ArgsVkBlitImage { VkImage src_image; VkImageLayout src_image_layout; diff --git a/src/xenia/gpu/vulkan/vulkan_command_processor.cc b/src/xenia/gpu/vulkan/vulkan_command_processor.cc index d7b548999..81bcb0b5d 100644 --- a/src/xenia/gpu/vulkan/vulkan_command_processor.cc +++ b/src/xenia/gpu/vulkan/vulkan_command_processor.cc @@ -30,6 +30,7 @@ #include "xenia/gpu/vulkan/vulkan_shader.h" #include "xenia/gpu/vulkan/vulkan_shared_memory.h" #include "xenia/gpu/xenos.h" +#include "xenia/gpu/xenos_zpd_report.h" #include "xenia/kernel/kernel_state.h" #include "xenia/kernel/user_module.h" #include "xenia/ui/vulkan/vulkan_presenter.h" @@ -333,12 +334,12 @@ bool VulkanCommandProcessor::SetupContext() { return false; } - // Shared memory and EDRAM descriptor set layout. + // Shared memory, EDRAM, and ZPD FSI counter descriptor set layout. bool edram_fragment_shader_interlock = render_target_cache_->GetPath() == RenderTargetCache::Path::kPixelShaderInterlock; VkDescriptorSetLayoutBinding - shared_memory_and_edram_descriptor_set_layout_bindings[2]; + shared_memory_and_edram_descriptor_set_layout_bindings[3]; shared_memory_and_edram_descriptor_set_layout_bindings[0].binding = 0; shared_memory_and_edram_descriptor_set_layout_bindings[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; @@ -368,6 +369,17 @@ bool VulkanCommandProcessor::SetupContext() { shared_memory_and_edram_descriptor_set_layout_bindings[1] .pImmutableSamplers = nullptr; shared_memory_and_edram_descriptor_set_layout_create_info.bindingCount = 2; + // ZPD FSI counter. + shared_memory_and_edram_descriptor_set_layout_bindings[2].binding = 2; + shared_memory_and_edram_descriptor_set_layout_bindings[2].descriptorType = + VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + shared_memory_and_edram_descriptor_set_layout_bindings[2].descriptorCount = + 1; + shared_memory_and_edram_descriptor_set_layout_bindings[2].stageFlags = + VK_SHADER_STAGE_FRAGMENT_BIT; + shared_memory_and_edram_descriptor_set_layout_bindings[2] + .pImmutableSamplers = nullptr; + shared_memory_and_edram_descriptor_set_layout_create_info.bindingCount = 3; } else { shared_memory_and_edram_descriptor_set_layout_create_info.bindingCount = 1; } @@ -399,11 +411,30 @@ bool VulkanCommandProcessor::SetupContext() { return false; } - // Shared memory and EDRAM common bindings. + // Needed by NormalizeSampleCount. + zpd_draw_resolution_scale_x_ = draw_resolution_scale_x; + zpd_draw_resolution_scale_y_ = draw_resolution_scale_y; + + const VkDeviceSize zpd_fsi_counter_sink_range = + sizeof(uint32_t) * kZPDQueryPoolCapacity; + if (edram_fragment_shader_interlock) { + if (!ui::vulkan::util::CreateDedicatedAllocationBuffer( + vulkan_device, zpd_fsi_counter_sink_range, + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, + ui::vulkan::util::MemoryPurpose::kDeviceLocal, + zpd_fsi_counter_sink_buffer_, + zpd_fsi_counter_sink_buffer_memory_)) { + XELOGE("Failed to create the ZPD FSI counter sink buffer"); + return false; + } + } + + // Shared memory, EDRAM, and ZPD FSI counter common bindings. VkDescriptorPoolSize descriptor_pool_sizes[1]; descriptor_pool_sizes[0].type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; descriptor_pool_sizes[0].descriptorCount = - shared_memory_binding_count + uint32_t(edram_fragment_shader_interlock); + shared_memory_binding_count + + 2u * uint32_t(edram_fragment_shader_interlock); VkDescriptorPoolCreateInfo descriptor_pool_create_info; descriptor_pool_create_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; @@ -450,7 +481,7 @@ bool VulkanCommandProcessor::SetupContext() { shared_memory_binding_range * i; shared_memory_descriptor_buffer_info.range = shared_memory_binding_range; } - VkWriteDescriptorSet write_descriptor_sets[2]; + VkWriteDescriptorSet write_descriptor_sets[3]; VkWriteDescriptorSet& write_descriptor_set_shared_memory = write_descriptor_sets[0]; write_descriptor_set_shared_memory.sType = @@ -485,10 +516,37 @@ bool VulkanCommandProcessor::SetupContext() { write_descriptor_set_edram.pImageInfo = nullptr; write_descriptor_set_edram.pBufferInfo = &edram_descriptor_buffer_info; write_descriptor_set_edram.pTexelBufferView = nullptr; + // ZPD FSI counter. + VkDescriptorBufferInfo zpd_fsi_counter_descriptor_buffer_info; + zpd_fsi_counter_descriptor_buffer_info.buffer = + zpd_fsi_counter_sink_buffer_; + zpd_fsi_counter_descriptor_buffer_info.offset = 0; + zpd_fsi_counter_descriptor_buffer_info.range = zpd_fsi_counter_sink_range; + // Keep binding 2 valid until the real counter buffer is ready. + VkWriteDescriptorSet& write_descriptor_set_zpd_fsi_counter_init = + write_descriptor_sets[2]; + write_descriptor_set_zpd_fsi_counter_init.sType = + VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write_descriptor_set_zpd_fsi_counter_init.pNext = nullptr; + write_descriptor_set_zpd_fsi_counter_init.dstSet = + shared_memory_and_edram_descriptor_set_; + write_descriptor_set_zpd_fsi_counter_init.dstBinding = 2; + write_descriptor_set_zpd_fsi_counter_init.dstArrayElement = 0; + write_descriptor_set_zpd_fsi_counter_init.descriptorCount = 1; + write_descriptor_set_zpd_fsi_counter_init.descriptorType = + VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write_descriptor_set_zpd_fsi_counter_init.pImageInfo = nullptr; + write_descriptor_set_zpd_fsi_counter_init.pBufferInfo = + &zpd_fsi_counter_descriptor_buffer_info; + write_descriptor_set_zpd_fsi_counter_init.pTexelBufferView = nullptr; } dfn.vkUpdateDescriptorSets(device, - 1 + uint32_t(edram_fragment_shader_interlock), + 1 + 2 * uint32_t(edram_fragment_shader_interlock), write_descriptor_sets, 0, nullptr); + if (edram_fragment_shader_interlock) { + zpd_fsi_counter_descriptor_buffer_ = zpd_fsi_counter_sink_buffer_; + zpd_fsi_counter_descriptor_range_ = zpd_fsi_counter_sink_range; + } // Swap objects. @@ -1068,8 +1126,15 @@ bool VulkanCommandProcessor::SetupContext() { return false; } + // Initialize the ZPD occlusion query pool and resources. + zpd_host_query_pool_ = std::make_unique(); + EnsureZPDQueryResources(); + // Just not to expose uninitialized memory. std::memset(&system_constants_, 0, sizeof(system_constants_)); + // ZPD FSI counter uses UINT32_MAX as its skip sentinel outside query draws. + system_constants_.zpd_fsi_counter_index = UINT32_MAX; + zpd_fsi_counter_index_force_update_ = true; return true; } @@ -1077,6 +1142,9 @@ bool VulkanCommandProcessor::SetupContext() { void VulkanCommandProcessor::ShutdownContext() { AwaitAllQueueOperationsCompletion(); + ShutdownZPDQueryResources(); + zpd_host_query_pool_.reset(); + const ui::vulkan::VulkanDevice* const vulkan_device = GetVulkanDevice(); const ui::vulkan::VulkanDevice::Functions& dfn = vulkan_device->functions(); const VkDevice device = vulkan_device->device(); @@ -1142,6 +1210,12 @@ void VulkanCommandProcessor::ShutdownContext() { ui::vulkan::util::DestroyAndNullHandle( dfn.vkDestroyDescriptorPool, device, shared_memory_and_edram_descriptor_pool_); + zpd_fsi_counter_descriptor_buffer_ = VK_NULL_HANDLE; + zpd_fsi_counter_descriptor_range_ = 0; + ui::vulkan::util::DestroyAndNullHandle(dfn.vkDestroyBuffer, device, + zpd_fsi_counter_sink_buffer_); + ui::vulkan::util::DestroyAndNullHandle(dfn.vkFreeMemory, device, + zpd_fsi_counter_sink_buffer_memory_); texture_cache_.reset(); @@ -1860,10 +1934,25 @@ void VulkanCommandProcessor::SubmitBarriersAndEnterRenderTargetCacheRenderPass( render_pass_begin_info.pClearValues = nullptr; deferred_command_buffer_.CmdVkBeginRenderPass(&render_pass_begin_info, VK_SUBPASS_CONTENTS_INLINE); + + // Resume any pending ZPD segment now that the pass is open. + OpenQuerySegment(false); } void VulkanCommandProcessor::EndRenderPass() { assert_true(submission_open_); + + // Close native Vulkan occlusion queries before ending the pass. FSI counter + // segments don't use vkCmdBeginQuery / vkCmdEndQuery and can stay logically + // open across render passes. + if (GetZPDMode() != ZPDMode::kFake && zpd_active_segment_.segment_active && + !zpd_active_query_is_fsi_) { + CloseQuerySegment(); + if (zpd_active_segment_.logical_active) { + zpd_active_segment_.segment_pending_begin = true; + } + } + if (current_render_pass_ == VK_NULL_HANDLE) { return; } @@ -3116,6 +3205,399 @@ VkBuffer VulkanCommandProcessor::RequestReadbackBuffer(uint32_t size) { return memexport_readback_buffer_; } +void VulkanCommandProcessor::EnsureZPDQueryResources() { + if (GetZPDMode() == ZPDMode::kFake || !zpd_host_query_pool_) { + return; + } + + bool can_recreate = + !zpd_active_segment_.logical_active && + !zpd_active_segment_.segment_active && + zpd_active_query_index_ == UINT32_MAX && !zpd_active_query_is_fsi_ && + !zpd_host_query_pool_->has_pending_resolve_batch() && + zpd_resolves_in_flight_.empty() && zpd_deferred_releases_.empty(); + + bool needs_fsi_counter = render_target_cache_ && + render_target_cache_->GetPath() == + RenderTargetCache::Path::kPixelShaderInterlock; + zpd_query_pool_needs_fsi_counter_ = needs_fsi_counter; + + bool resources_initialized = zpd_host_query_pool_->EnsureInitialized( + GetVulkanDevice(), kZPDQueryPoolCapacity, can_recreate, + needs_fsi_counter); + + if (resources_initialized && needs_fsi_counter && + zpd_host_query_pool_->fsi_initialized()) { + VkBuffer fsi_counter_buffer = zpd_host_query_pool_->fsi_counter_buffer(); + VkDeviceSize fsi_counter_range = + sizeof(uint32_t) * zpd_host_query_pool_->capacity(); + if (zpd_fsi_counter_descriptor_buffer_ != fsi_counter_buffer || + zpd_fsi_counter_descriptor_range_ != fsi_counter_range) { + VkDescriptorBufferInfo fsi_counter_descriptor_buffer_info; + fsi_counter_descriptor_buffer_info.buffer = fsi_counter_buffer; + fsi_counter_descriptor_buffer_info.offset = 0; + fsi_counter_descriptor_buffer_info.range = fsi_counter_range; + + VkWriteDescriptorSet fsi_counter_descriptor_write; + fsi_counter_descriptor_write.sType = + VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + fsi_counter_descriptor_write.pNext = nullptr; + fsi_counter_descriptor_write.dstSet = + shared_memory_and_edram_descriptor_set_; + fsi_counter_descriptor_write.dstBinding = 2; + fsi_counter_descriptor_write.dstArrayElement = 0; + fsi_counter_descriptor_write.descriptorCount = 1; + fsi_counter_descriptor_write.descriptorType = + VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + fsi_counter_descriptor_write.pImageInfo = nullptr; + fsi_counter_descriptor_write.pBufferInfo = + &fsi_counter_descriptor_buffer_info; + fsi_counter_descriptor_write.pTexelBufferView = nullptr; + + const ui::vulkan::VulkanDevice::Functions& dfn = + GetVulkanDevice()->functions(); + dfn.vkUpdateDescriptorSets(GetVulkanDevice()->device(), 1, + &fsi_counter_descriptor_write, 0, nullptr); + + zpd_fsi_counter_descriptor_buffer_ = fsi_counter_buffer; + zpd_fsi_counter_descriptor_range_ = fsi_counter_range; + } + } else if (!IsZPDQueryPoolReady()) { + XELOGW( + "ZPD/Vulkan: FSI counter resources unavailable; keeping counter " + "index sentinel active"); + } + zpd_fsi_counter_index_force_update_ = true; +} + +bool VulkanCommandProcessor::IsZPDQueryPoolReady() const { + if (!zpd_host_query_pool_ || !zpd_host_query_pool_->fbo_initialized()) { + return false; + } + if (!zpd_query_pool_needs_fsi_counter_) { + return true; + } + VkDeviceSize fsi_counter_range = + sizeof(uint32_t) * zpd_host_query_pool_->capacity(); + return zpd_host_query_pool_->fsi_initialized() && + zpd_fsi_counter_descriptor_buffer_ == + zpd_host_query_pool_->fsi_counter_buffer() && + zpd_fsi_counter_descriptor_range_ == fsi_counter_range; +} + +bool VulkanCommandProcessor::CanOpenZPDQuery() const { + if (!submission_open_) { + return false; + } + bool use_fsi_counter_path = + render_target_cache_ && + render_target_cache_->GetPath() == + RenderTargetCache::Path::kPixelShaderInterlock; + return use_fsi_counter_path || current_render_pass_ != VK_NULL_HANDLE; +} + +CommandProcessor::QueryOpenResult VulkanCommandProcessor::OpenZPDQuery( + ReportHandle report_handle, bool can_close_submission) { + bool use_fsi_counter_path = zpd_query_pool_needs_fsi_counter_ && + zpd_host_query_pool_->fsi_initialized(); + + if (!BeginSubmission(true)) { + return QueryOpenResult::kFailed; + } + + if (!use_fsi_counter_path && current_render_pass_ == VK_NULL_HANDLE) { + return QueryOpenResult::kDeferred; + } + + bool retried_after_submission_flip = false; + while (true) { + bool is_pool_exhausted = !zpd_host_query_pool_->has_free_indices(); + + if (is_pool_exhausted) { + PumpQueryResolves(); + is_pool_exhausted = !zpd_host_query_pool_->has_free_indices(); + } + + if (is_pool_exhausted && GetZPDMode() == ZPDMode::kFast) { + return QueryOpenResult::kPoolExhausted; + } + + uint64_t wait_for = 0; + if (is_pool_exhausted && !zpd_resolves_in_flight_.empty()) { + wait_for = zpd_resolves_in_flight_.front().submission; + } + + if (wait_for == 0) { + break; + } + + if (submission_open_ && wait_for == GetCurrentSubmission()) { + if (retried_after_submission_flip || !can_close_submission || + !CanEndSubmissionImmediately()) { + return QueryOpenResult::kDeferred; + } + + VkRenderPass saved_render_pass = VK_NULL_HANDLE; + const VulkanRenderTargetCache::Framebuffer* saved_framebuffer = nullptr; + if (current_render_pass_ != VK_NULL_HANDLE) { + saved_render_pass = current_render_pass_; + saved_framebuffer = current_framebuffer_; + } + + if (current_render_pass_ != VK_NULL_HANDLE) { + EndRenderPass(); + } + if (!EndSubmission(false)) { + return QueryOpenResult::kFailed; + } + if (!BeginSubmission(true)) { + return QueryOpenResult::kFailed; + } + + if (saved_framebuffer) { + bool saved_pending_begin = zpd_active_segment_.segment_pending_begin; + if (use_fsi_counter_path) { + zpd_active_segment_.segment_pending_begin = false; + } + SubmitBarriersAndEnterRenderTargetCacheRenderPass(saved_render_pass, + saved_framebuffer); + zpd_active_segment_.segment_pending_begin = saved_pending_begin; + if (current_render_pass_ == VK_NULL_HANDLE) { + return QueryOpenResult::kDeferred; + } + } + + retried_after_submission_flip = true; + continue; + } + + uint64_t completed_submission = GetCompletedSubmission(); + if (wait_for > completed_submission) { + completion_timeline_.AwaitSubmissionAndUpdateCompleted(wait_for); + PumpQueryResolves(); + } + + break; + } + + if (!use_fsi_counter_path && current_render_pass_ == VK_NULL_HANDLE) { + return QueryOpenResult::kDeferred; + } + + if (!zpd_host_query_pool_->AcquireQueryIndex(zpd_active_query_index_, + zpd_active_query_generation_)) { + return QueryOpenResult::kFailed; + } + + zpd_active_query_is_fsi_ = use_fsi_counter_path; + + // FSI queries don't use Vulkan occlusion queries at all. + // While the segment is open, the translated pixel shader accumulates passed + // MSAA samples into one counter slot selected via zpd_fsi_counter_index. + // Clear the slot here so a recycled index never inherits old counts. + if (zpd_active_query_is_fsi_) { + bool fsi_counter_cleared = false; + if (zpd_host_query_pool_->fsi_initialized()) { + if (current_render_pass_ != VK_NULL_HANDLE) { + VkRenderPass saved_render_pass = current_render_pass_; + const VulkanRenderTargetCache::Framebuffer* saved_framebuffer = + current_framebuffer_; + EndRenderPass(); + zpd_host_query_pool_->ClearFSICounter(deferred_command_buffer_, + zpd_active_query_index_); + + bool saved_pending_begin = zpd_active_segment_.segment_pending_begin; + zpd_active_segment_.segment_pending_begin = false; + SubmitBarriersAndEnterRenderTargetCacheRenderPass(saved_render_pass, + saved_framebuffer); + zpd_active_segment_.segment_pending_begin = saved_pending_begin; + fsi_counter_cleared = current_render_pass_ != VK_NULL_HANDLE; + } else { + zpd_host_query_pool_->ClearFSICounter(deferred_command_buffer_, + zpd_active_query_index_); + fsi_counter_cleared = true; + } + } + if (!fsi_counter_cleared) { + zpd_host_query_pool_->ReleaseQueryIndex(zpd_active_query_index_, + zpd_active_query_generation_); + zpd_active_query_index_ = UINT32_MAX; + zpd_active_query_generation_ = 0; + zpd_active_query_is_fsi_ = false; + zpd_fsi_counter_index_force_update_ = true; + return QueryOpenResult::kFailed; + } + zpd_fsi_counter_index_force_update_ = true; + return QueryOpenResult::kOpened; + } + + zpd_host_query_pool_->BeginQuery(deferred_command_buffer_, + zpd_active_query_index_); + return QueryOpenResult::kOpened; +} + +bool VulkanCommandProcessor::CloseZPDQuery(ReportHandle report_handle, + uint64_t& out_submission) { + if (!zpd_active_query_is_fsi_ && current_render_pass_ == VK_NULL_HANDLE) { + return false; + } + + if (zpd_active_query_is_fsi_) { + zpd_host_query_pool_->QueueQueryResolve(zpd_active_query_index_, true); + } else { + zpd_host_query_pool_->EndQuery(deferred_command_buffer_, + zpd_active_query_index_); + zpd_host_query_pool_->QueueQueryResolve(zpd_active_query_index_, false); + } + + PendingQueryResolve resolve; + resolve.submission = GetCurrentSubmission(); + resolve.query_index = zpd_active_query_index_; + resolve.query_generation = zpd_active_query_generation_; + resolve.uses_fsi_counter = zpd_active_query_is_fsi_; + resolve.report_handle = report_handle; + zpd_resolves_in_flight_.push_back(resolve); + + out_submission = resolve.submission; + + zpd_active_query_index_ = UINT32_MAX; + zpd_active_query_generation_ = 0; + + bool closed_fsi_counter = zpd_active_query_is_fsi_; + zpd_active_query_is_fsi_ = false; + + if (closed_fsi_counter) { + zpd_fsi_counter_index_force_update_ = true; + } + return true; +} + +bool VulkanCommandProcessor::DiscardZPDQuery() { + if (zpd_active_query_is_fsi_) { + // The slot counter may be dirty if draws ran between OpenZPDQuery and + // here, but the next OpenZPDQuery clears it before any new shader adds. + zpd_host_query_pool_->ReleaseQueryIndex(zpd_active_query_index_, + zpd_active_query_generation_); + zpd_active_query_index_ = UINT32_MAX; + zpd_active_query_generation_ = 0; + zpd_active_query_is_fsi_ = false; + zpd_fsi_counter_index_force_update_ = true; + return true; + } + + if (current_render_pass_ == VK_NULL_HANDLE) { + // vkCmdEndQuery is invalid outside a render pass for occlusion queries. + // Defer the release until the submission containing the stale BeginQuery + // completes on the GPU. + zpd_deferred_releases_.push_back({GetCurrentSubmission(), + zpd_active_query_index_, + zpd_active_query_generation_}); + zpd_active_query_index_ = UINT32_MAX; + zpd_active_query_generation_ = 0; + zpd_active_query_is_fsi_ = false; + return true; + } + + // Inside a render pass, EndQuery must be issued before releasing the slot. + zpd_host_query_pool_->EndQuery(deferred_command_buffer_, + zpd_active_query_index_); + zpd_host_query_pool_->ReleaseQueryIndex(zpd_active_query_index_, + zpd_active_query_generation_); + zpd_active_query_index_ = UINT32_MAX; + zpd_active_query_generation_ = 0; + zpd_active_query_is_fsi_ = false; + return true; +} + +void VulkanCommandProcessor::PumpQueryResolves() { + if (GetZPDMode() == ZPDMode::kFake || !zpd_host_query_pool_) { + return; + } + + uint64_t completed = GetCompletedSubmission(); + if (completed == 0) { + return; + } + + // Drain deferred releases first. + while (!zpd_deferred_releases_.empty()) { + auto& entry = zpd_deferred_releases_.front(); + if (entry.submission > completed) { + break; + } + zpd_host_query_pool_->ReleaseQueryIndex(entry.query_index, + entry.query_generation); + zpd_deferred_releases_.pop_front(); + } + + // Invalidate CPU cache before reading results on non-coherent memory. + if (!zpd_resolves_in_flight_.empty() && + zpd_resolves_in_flight_.front().submission <= completed) { + zpd_host_query_pool_->InvalidateReadback(); + } + + while (!zpd_resolves_in_flight_.empty()) { + if (zpd_resolves_in_flight_.front().submission > completed) { + break; + } + PendingQueryResolve resolve = zpd_resolves_in_flight_.front(); + zpd_resolves_in_flight_.pop_front(); + + if (zpd_host_query_pool_->GenerationMatches(resolve.query_index, + resolve.query_generation)) { + uint64_t raw_samples = zpd_host_query_pool_->GetQueryReadbackValue( + resolve.query_index, resolve.uses_fsi_counter); + zpd_host_query_pool_->ReleaseQueryIndex(resolve.query_index, + resolve.query_generation); + OnZPDQueryResolved(resolve.report_handle, raw_samples); + } + } +} + +bool VulkanCommandProcessor::AwaitQueryResolve(ReportHandle report_handle, + uint64_t wait_for_submission) { + if (GetZPDMode() == ZPDMode::kFake) { + return false; + } + + PumpQueryResolves(); + + auto it = logical_zpd_reports_.find(report_handle); + if (it == logical_zpd_reports_.end()) { + return true; + } + if (it->second.pending_segments == 0 && it->second.ended) { + return true; + } + if (wait_for_submission == 0) { + return false; + } + + // Ensure the submission is flushed. + if (wait_for_submission >= GetCurrentSubmission()) { + if (!submission_open_) { + return false; + } + if (!CanEndSubmissionImmediately()) { + pipeline_cache_->AwaitPipelineCompletion(); + } + EndRenderPass(); + if (!EndSubmission(false)) { + return false; + } + } + + if (wait_for_submission > GetCompletedSubmission()) { + completion_timeline_.AwaitSubmissionAndUpdateCompleted(wait_for_submission); + } + + PumpQueryResolves(); + + it = logical_zpd_reports_.find(report_handle); + return it == logical_zpd_reports_.end() || + (it->second.pending_segments == 0 && it->second.ended); +} + void VulkanCommandProcessor::InitializeTrace() { CommandProcessor::InitializeTrace(); @@ -3134,6 +3616,17 @@ void VulkanCommandProcessor::InitializeTrace() { } } +void VulkanCommandProcessor::PollCompletedSubmission() { + // Strict ZPD can skip unnecessary work here that can wait for the next full + // CheckSubmissionCompletionAndDeviceLoss and it's not needed for retirement. + if (device_lost_) { + return; + } + completion_timeline_.AwaitSubmissionAndUpdateCompleted( + GetCompletedSubmission()); + PumpQueryResolves(); +} + void VulkanCommandProcessor::CheckSubmissionCompletionAndDeviceLoss( uint64_t await_submission) { // Only report once, no need to retry a wait that won't succeed anyway. @@ -3191,6 +3684,8 @@ void VulkanCommandProcessor::CheckSubmissionCompletionAndDeviceLoss( texture_cache_->CompletedSubmissionUpdated(completed_submission); + PumpQueryResolves(); + // Destroy objects scheduled for destruction. const ui::vulkan::VulkanDevice::Functions& dfn = vulkan_device->functions(); const VkDevice device = vulkan_device->device(); @@ -3527,6 +4022,9 @@ bool VulkanCommandProcessor::EndSubmission(bool is_swap) { sparse_memory_binds_.clear(); } + // Can't cross command buffer boundaries. Close the active segment first. + CloseQuerySegment(); + SubmitBarriers(true); assert_false(command_buffers_writable_.empty()); @@ -3548,6 +4046,12 @@ bool VulkanCommandProcessor::EndSubmission(bool is_swap) { return false; } deferred_command_buffer_.Execute(command_buffer.buffer); + + // Record ZPD resolves before submitting. + if (zpd_host_query_pool_) { + zpd_host_query_pool_->RecordResolveBatch(command_buffer.buffer); + } + if (dfn.vkEndCommandBuffer(command_buffer.buffer) != VK_SUCCESS) { XELOGE("Failed to end a Vulkan command buffer"); return false; @@ -3586,6 +4090,11 @@ bool VulkanCommandProcessor::EndSubmission(bool is_swap) { command_buffers_writable_.pop_back(); submission_open_ = false; + + // Process any ZPD resolves that completed with this submission. + // Block if strict mode has a pending result waiting on the guest sentinel. + PumpQueryResolves(); + PumpPendingRetire(); } if (is_closing_frame) { @@ -3634,6 +4143,10 @@ bool VulkanCommandProcessor::EndSubmission(bool is_swap) { return true; } +bool VulkanCommandProcessor::CanEndSubmissionImmediately() const { + return !submission_open_ || !pipeline_cache_->IsCreatingPipelines(); +} + void VulkanCommandProcessor::ClearTransientDescriptorPools() { texture_transient_descriptor_sets_free_.clear(); texture_transient_descriptor_sets_used_.clear(); @@ -4243,6 +4756,18 @@ void VulkanCommandProcessor::UpdateSystemConstantValues( dirty |= system_constants_.alpha_to_mask != alpha_to_mask; system_constants_.alpha_to_mask = alpha_to_mask; + // FSI ZPD counter. + uint32_t zpd_fsi_counter_index = UINT32_MAX; + if (edram_fragment_shader_interlock && + zpd_active_query_index_ != UINT32_MAX && zpd_active_query_is_fsi_ && + zpd_host_query_pool_->fsi_initialized()) { + zpd_fsi_counter_index = zpd_active_query_index_; + } + dirty |= zpd_fsi_counter_index_force_update_ || + system_constants_.zpd_fsi_counter_index != zpd_fsi_counter_index; + system_constants_.zpd_fsi_counter_index = zpd_fsi_counter_index; + zpd_fsi_counter_index_force_update_ = false; + uint32_t edram_tile_dwords_scaled = xenos::kEdramTileWidthSamples * xenos::kEdramTileHeightSamples * (draw_resolution_scale_x * draw_resolution_scale_y); diff --git a/src/xenia/gpu/vulkan/vulkan_command_processor.h b/src/xenia/gpu/vulkan/vulkan_command_processor.h index 9768527f5..8d5d6e68f 100644 --- a/src/xenia/gpu/vulkan/vulkan_command_processor.h +++ b/src/xenia/gpu/vulkan/vulkan_command_processor.h @@ -35,6 +35,7 @@ #include "xenia/gpu/vulkan/vulkan_shader.h" #include "xenia/gpu/vulkan/vulkan_shared_memory.h" #include "xenia/gpu/vulkan/vulkan_texture_cache.h" +#include "xenia/gpu/vulkan/vulkan_zpd_query_pool.h" #include "xenia/gpu/xenos.h" #include "xenia/kernel/kernel_state.h" #include "xenia/ui/vulkan/linked_type_descriptor_set_allocator.h" @@ -151,6 +152,8 @@ class VulkanCommandProcessor final : public CommandProcessor { void RestoreEdramSnapshot(const void* snapshot) override; + void PollCompletedSubmission() override; + ui::vulkan::VulkanDevice* GetVulkanDevice() const { return static_cast( graphics_system_->provider()) @@ -168,7 +171,7 @@ class VulkanCommandProcessor final : public CommandProcessor { uint64_t GetCurrentSubmission() const { return completion_timeline_.GetUpcomingSubmission(); } - uint64_t GetCompletedSubmission() const { + uint64_t GetCompletedSubmission() const override { return completion_timeline_.GetCompletedSubmissionFromLastUpdate(); } @@ -426,6 +429,7 @@ class VulkanCommandProcessor final : public CommandProcessor { // clearing and stopping capturing. Returns whether the submission was done // successfully, if it has failed, leaves it open. bool EndSubmission(bool is_swap); + bool CanEndSubmissionImmediately() const; bool AwaitAllQueueOperationsCompletion() { CheckSubmissionCompletionAndDeviceLoss(GetCurrentSubmission()); return !submission_open_ && @@ -441,6 +445,39 @@ class VulkanCommandProcessor final : public CommandProcessor { void DestroyScratchBuffer(); + // ZPD occlusion queries backend. + // vkCmdBeginQuery is only valid inside a render pass, so segments split at + // pass end and resume at the next pass begin. If BEGIN fires outside a pass, + // segment_pending_begin waits for the next. Outside a render pass, + // DiscardZPDQuery defers the slot release until the submission completes. + // FSI queries clear a dedicated counter with vkCmdFillBuffer, so they may + // need to open before a pass begins or split an active pass around the clear. + void EnsureZPDQueryResources() override; + void ShutdownZPDQueryResources() override { + zpd_resolves_in_flight_.clear(); + zpd_deferred_releases_.clear(); + zpd_active_query_index_ = UINT32_MAX; + zpd_active_query_generation_ = 0; + zpd_active_query_is_fsi_ = false; + zpd_query_pool_needs_fsi_counter_ = false; + zpd_fsi_counter_index_force_update_ = true; + if (zpd_host_query_pool_) { + zpd_host_query_pool_->Shutdown(); + } + } + + bool IsZPDQueryPoolReady() const override; + bool CanOpenZPDQuery() const override; + + QueryOpenResult OpenZPDQuery(ReportHandle report_handle, + bool can_close_submission) override; + bool CloseZPDQuery(ReportHandle report_handle, + uint64_t& out_submission) override; + bool DiscardZPDQuery() override; + void PumpQueryResolves() override; + bool AwaitQueryResolve(ReportHandle report_handle, + uint64_t wait_for_submission) override; + void UpdateDynamicState(const draw_util::ViewportInfo& viewport_info, bool primitive_polygonal, reg::RB_DEPTHCONTROL normalized_depth_control, @@ -481,6 +518,26 @@ class VulkanCommandProcessor final : public CommandProcessor { std::vector semaphores_free_; + struct PendingQueryResolve { + uint64_t submission = 0; + uint32_t query_index = UINT32_MAX; + uint32_t query_generation = 0; + bool uses_fsi_counter = false; + ReportHandle report_handle = kInvalidReportHandle; + }; + uint32_t zpd_active_query_index_ = UINT32_MAX; + uint32_t zpd_active_query_generation_ = 0; + bool zpd_active_query_is_fsi_ = false; + bool zpd_query_pool_needs_fsi_counter_ = false; + bool zpd_fsi_counter_index_force_update_ = true; + std::deque zpd_resolves_in_flight_; + // Fallback buffer for EDRAM descriptor binding 2. + VkBuffer zpd_fsi_counter_sink_buffer_ = VK_NULL_HANDLE; + VkDeviceMemory zpd_fsi_counter_sink_buffer_memory_ = VK_NULL_HANDLE; + // Currently installed binding 2 buffer. + VkBuffer zpd_fsi_counter_descriptor_buffer_ = VK_NULL_HANDLE; + VkDeviceSize zpd_fsi_counter_descriptor_range_ = 0; + ui::vulkan::VulkanGPUCompletionTimeline completion_timeline_; bool submission_open_ = false; // In case vkQueueSubmit fails after something like a successful @@ -576,6 +633,18 @@ class VulkanCommandProcessor final : public CommandProcessor { std::unique_ptr render_target_cache_; + std::unique_ptr zpd_host_query_pool_; + + // Deferred query slot releases for discards that happen outside a render + // pass, where vkCmdEndQuery cannot be issued. The slot is held until the + // submission containing the stale BeginQuery completes on the GPU. + struct DeferredQueryRelease { + uint64_t submission; + uint32_t query_index; + uint32_t query_generation; + }; + std::deque zpd_deferred_releases_; + std::unique_ptr pipeline_cache_; std::unique_ptr texture_cache_; diff --git a/src/xenia/gpu/vulkan/vulkan_pipeline_cache.cc b/src/xenia/gpu/vulkan/vulkan_pipeline_cache.cc index 100729680..003ef86b2 100644 --- a/src/xenia/gpu/vulkan/vulkan_pipeline_cache.cc +++ b/src/xenia/gpu/vulkan/vulkan_pipeline_cache.cc @@ -766,6 +766,28 @@ bool VulkanPipelineCache::IsCreatingPipelines() { return !creation_queue_.empty() || creation_threads_busy_ != 0; } +void VulkanPipelineCache::AwaitPipelineCompletion() { + if (creation_threads_.empty()) { + return; + } + + bool await_creation_completion_event; + { + std::lock_guard lock(creation_request_lock_); + await_creation_completion_event = + !creation_queue_.empty() || creation_threads_busy_ != 0; + if (await_creation_completion_event) { + creation_completion_event_->Reset(); + creation_completion_set_event_.store(true, std::memory_order_release); + } + } + + if (await_creation_completion_event) { + creation_request_cond_.notify_one(); + xe::threading::Wait(creation_completion_event_.get(), false); + } +} + void VulkanPipelineCache::CreationThread() { for (;;) { PipelineCreationArguments creation_arguments; diff --git a/src/xenia/gpu/vulkan/vulkan_pipeline_cache.h b/src/xenia/gpu/vulkan/vulkan_pipeline_cache.h index 1e33c5a5e..f251beb80 100644 --- a/src/xenia/gpu/vulkan/vulkan_pipeline_cache.h +++ b/src/xenia/gpu/vulkan/vulkan_pipeline_cache.h @@ -113,6 +113,10 @@ class VulkanPipelineCache { void EndSubmission(); bool IsCreatingPipelines(); + // Waits for any pipeline creation needed by the current draw path to finish + // before state is consumed. This was added so strict ZPD query paths stop + // racing pipeline compilation and then blocking work on incomplete state. + void AwaitPipelineCompletion(); VulkanShader* LoadShader(xenos::ShaderType shader_type, const uint32_t* host_address, uint32_t dword_count); diff --git a/src/xenia/gpu/vulkan/vulkan_zpd_query_pool.cc b/src/xenia/gpu/vulkan/vulkan_zpd_query_pool.cc new file mode 100644 index 000000000..66cc01956 --- /dev/null +++ b/src/xenia/gpu/vulkan/vulkan_zpd_query_pool.cc @@ -0,0 +1,796 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/gpu/vulkan/vulkan_zpd_query_pool.h" + +#include + +#include "xenia/base/logging.h" +#include "xenia/gpu/vulkan/deferred_command_buffer.h" +#include "xenia/ui/vulkan/vulkan_device.h" +#include "xenia/ui/vulkan/vulkan_util.h" + +namespace xe { +namespace gpu { +namespace vulkan { + +bool VulkanZPDQueryPool::EnsureInitialized( + const ui::vulkan::VulkanDevice* vulkan_device, uint32_t requested_capacity, + bool can_recreate, bool initialize_fsi_counter) { + vulkan_device_ = vulkan_device; + if (!vulkan_device_ || !requested_capacity) { + return false; + } + + if (capacity_ != 0 && capacity_ != requested_capacity) { + if (!can_recreate) { + return fbo_initialized() || (initialize_fsi_counter && fsi_initialized()); + } + Shutdown(); + } + + if (capacity_ == requested_capacity && fbo_initialized() && + (!initialize_fsi_counter || fsi_initialized())) { + return true; + } + + const ui::vulkan::VulkanDevice::Functions& dfn = vulkan_device_->functions(); + const VkDevice device = vulkan_device_->device(); + + auto initialize_shared_capacity = [this, requested_capacity]() { + if (capacity_ == requested_capacity) { + return; + } + capacity_ = requested_capacity; + + free_indices_.clear(); + free_indices_.reserve(requested_capacity); + for (uint32_t i = 0; i < requested_capacity; ++i) { + free_indices_.push_back(requested_capacity - 1 - i); + } + index_generations_.assign(requested_capacity, 0); + + resolve_batch_pending_.assign(requested_capacity, 0); + resolve_batch_indices_.clear(); + fsi_counter_resolve_batch_pending_.assign(requested_capacity, 0); + fsi_counter_resolve_batch_indices_.clear(); + }; + + auto initialize_host_query_path = [&]() -> bool { + if (fbo_initialized()) { + return true; + } + + // Need VK_EXT_host_query_reset (1.2 core) to reset slots on the CPU + // without a paired vkCmdEndQuery. May be unavailable on older drivers or + // devices. + if (!vulkan_device_->properties().hostQueryReset || + dfn.vkResetQueryPool == nullptr) { + return false; + } + + VkQueryPoolCreateInfo pool_info; + pool_info.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO; + pool_info.pNext = nullptr; + pool_info.flags = 0; + pool_info.queryType = VK_QUERY_TYPE_OCCLUSION; + pool_info.queryCount = requested_capacity; + pool_info.pipelineStatistics = 0; + if (dfn.vkCreateQueryPool(device, &pool_info, nullptr, &query_pool_) != + VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to create the ZPD query " + "pool, falling back to fake sample counts."); + query_pool_ = VK_NULL_HANDLE; + return false; + } + + VkBufferCreateInfo readback_buffer_info; + readback_buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + readback_buffer_info.pNext = nullptr; + readback_buffer_info.flags = 0; + readback_buffer_info.size = sizeof(uint64_t) * requested_capacity; + readback_buffer_info.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + readback_buffer_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + readback_buffer_info.queueFamilyIndexCount = 0; + readback_buffer_info.pQueueFamilyIndices = nullptr; + if (dfn.vkCreateBuffer(device, &readback_buffer_info, nullptr, + &readback_buffer_) != VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to create the ZPD query " + "readback buffer, falling back to fake sample counts."); + dfn.vkDestroyQueryPool(device, query_pool_, nullptr); + query_pool_ = VK_NULL_HANDLE; + return false; + } + + VkMemoryRequirements readback_mem_reqs; + dfn.vkGetBufferMemoryRequirements(device, readback_buffer_, + &readback_mem_reqs); + + VkMemoryAllocateInfo readback_alloc_info; + readback_alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + readback_alloc_info.pNext = nullptr; + readback_alloc_info.allocationSize = readback_mem_reqs.size; + readback_alloc_info.memoryTypeIndex = ui::vulkan::util::ChooseMemoryType( + vulkan_device_->memory_types(), readback_mem_reqs.memoryTypeBits, + ui::vulkan::util::MemoryPurpose::kReadback); + if (readback_alloc_info.memoryTypeIndex == UINT32_MAX || + dfn.vkAllocateMemory(device, &readback_alloc_info, nullptr, + &readback_memory_) != VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to allocate ZPD query " + "readback memory, falling back to fake sample counts."); + dfn.vkDestroyBuffer(device, readback_buffer_, nullptr); + readback_buffer_ = VK_NULL_HANDLE; + dfn.vkDestroyQueryPool(device, query_pool_, nullptr); + query_pool_ = VK_NULL_HANDLE; + return false; + } + + readback_is_coherent_ = (vulkan_device_->memory_types().host_coherent & + (1u << readback_alloc_info.memoryTypeIndex)) != 0; + + if (dfn.vkBindBufferMemory(device, readback_buffer_, readback_memory_, 0) != + VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to bind ZPD query readback " + "buffer memory, falling back to fake sample counts."); + dfn.vkFreeMemory(device, readback_memory_, nullptr); + readback_memory_ = VK_NULL_HANDLE; + dfn.vkDestroyBuffer(device, readback_buffer_, nullptr); + readback_buffer_ = VK_NULL_HANDLE; + dfn.vkDestroyQueryPool(device, query_pool_, nullptr); + query_pool_ = VK_NULL_HANDLE; + return false; + } + + void* mapping = nullptr; + if (dfn.vkMapMemory(device, readback_memory_, 0, VK_WHOLE_SIZE, 0, + &mapping) != VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to map ZPD query readback " + "memory, falling back to fake sample counts."); + dfn.vkFreeMemory(device, readback_memory_, nullptr); + readback_memory_ = VK_NULL_HANDLE; + dfn.vkDestroyBuffer(device, readback_buffer_, nullptr); + readback_buffer_ = VK_NULL_HANDLE; + dfn.vkDestroyQueryPool(device, query_pool_, nullptr); + query_pool_ = VK_NULL_HANDLE; + return false; + } + + readback_mapping_ = reinterpret_cast(mapping); + + dfn.vkResetQueryPool(device, query_pool_, 0, requested_capacity); + initialize_shared_capacity(); + return true; + }; + + auto initialize_fsi_counter_path = [&]() -> bool { + if (fsi_initialized()) { + return true; + } + + VkBufferCreateInfo counter_buffer_info; + counter_buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + counter_buffer_info.pNext = nullptr; + counter_buffer_info.flags = 0; + counter_buffer_info.size = sizeof(uint32_t) * requested_capacity; + counter_buffer_info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | + VK_BUFFER_USAGE_TRANSFER_DST_BIT; + counter_buffer_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + counter_buffer_info.queueFamilyIndexCount = 0; + counter_buffer_info.pQueueFamilyIndices = nullptr; + if (dfn.vkCreateBuffer(device, &counter_buffer_info, nullptr, + &fsi_counter_buffer_) != VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to create the ZPD FSI counter " + "buffer, falling back to fake sample counts."); + fsi_counter_buffer_ = VK_NULL_HANDLE; + return false; + } + + VkMemoryRequirements counter_mem_reqs; + dfn.vkGetBufferMemoryRequirements(device, fsi_counter_buffer_, + &counter_mem_reqs); + + VkMemoryAllocateInfo counter_alloc_info; + counter_alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + counter_alloc_info.pNext = nullptr; + counter_alloc_info.allocationSize = counter_mem_reqs.size; + counter_alloc_info.memoryTypeIndex = ui::vulkan::util::ChooseMemoryType( + vulkan_device_->memory_types(), counter_mem_reqs.memoryTypeBits, + ui::vulkan::util::MemoryPurpose::kDeviceLocal); + if (counter_alloc_info.memoryTypeIndex == UINT32_MAX || + dfn.vkAllocateMemory(device, &counter_alloc_info, nullptr, + &fsi_counter_memory_) != VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to allocate ZPD FSI counter " + "memory, falling back to fake sample counts."); + dfn.vkDestroyBuffer(device, fsi_counter_buffer_, nullptr); + fsi_counter_buffer_ = VK_NULL_HANDLE; + return false; + } + + if (dfn.vkBindBufferMemory(device, fsi_counter_buffer_, fsi_counter_memory_, + 0) != VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to bind ZPD FSI counter " + "buffer memory, falling back to fake sample counts."); + dfn.vkFreeMemory(device, fsi_counter_memory_, nullptr); + fsi_counter_memory_ = VK_NULL_HANDLE; + dfn.vkDestroyBuffer(device, fsi_counter_buffer_, nullptr); + fsi_counter_buffer_ = VK_NULL_HANDLE; + return false; + } + + VkBufferCreateInfo readback_buffer_info; + readback_buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + readback_buffer_info.pNext = nullptr; + readback_buffer_info.flags = 0; + readback_buffer_info.size = sizeof(uint32_t) * requested_capacity; + readback_buffer_info.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + readback_buffer_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + readback_buffer_info.queueFamilyIndexCount = 0; + readback_buffer_info.pQueueFamilyIndices = nullptr; + if (dfn.vkCreateBuffer(device, &readback_buffer_info, nullptr, + &fsi_counter_readback_buffer_) != VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to create the ZPD FSI counter " + "readback buffer, falling back to fake sample counts."); + dfn.vkFreeMemory(device, fsi_counter_memory_, nullptr); + fsi_counter_memory_ = VK_NULL_HANDLE; + dfn.vkDestroyBuffer(device, fsi_counter_buffer_, nullptr); + fsi_counter_buffer_ = VK_NULL_HANDLE; + return false; + } + + VkMemoryRequirements readback_mem_reqs; + dfn.vkGetBufferMemoryRequirements(device, fsi_counter_readback_buffer_, + &readback_mem_reqs); + + VkMemoryAllocateInfo readback_alloc_info; + readback_alloc_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + readback_alloc_info.pNext = nullptr; + readback_alloc_info.allocationSize = readback_mem_reqs.size; + readback_alloc_info.memoryTypeIndex = ui::vulkan::util::ChooseMemoryType( + vulkan_device_->memory_types(), readback_mem_reqs.memoryTypeBits, + ui::vulkan::util::MemoryPurpose::kReadback); + if (readback_alloc_info.memoryTypeIndex == UINT32_MAX || + dfn.vkAllocateMemory(device, &readback_alloc_info, nullptr, + &fsi_counter_readback_memory_) != VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to allocate ZPD FSI counter " + "readback memory, falling back to fake sample counts."); + dfn.vkDestroyBuffer(device, fsi_counter_readback_buffer_, nullptr); + fsi_counter_readback_buffer_ = VK_NULL_HANDLE; + dfn.vkFreeMemory(device, fsi_counter_memory_, nullptr); + fsi_counter_memory_ = VK_NULL_HANDLE; + dfn.vkDestroyBuffer(device, fsi_counter_buffer_, nullptr); + fsi_counter_buffer_ = VK_NULL_HANDLE; + return false; + } + + fsi_counter_readback_is_coherent_ = + (vulkan_device_->memory_types().host_coherent & + (1u << readback_alloc_info.memoryTypeIndex)) != 0; + + if (dfn.vkBindBufferMemory(device, fsi_counter_readback_buffer_, + fsi_counter_readback_memory_, 0) != VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to bind ZPD FSI counter " + "readback buffer memory, falling back to fake sample counts."); + dfn.vkFreeMemory(device, fsi_counter_readback_memory_, nullptr); + fsi_counter_readback_memory_ = VK_NULL_HANDLE; + dfn.vkDestroyBuffer(device, fsi_counter_readback_buffer_, nullptr); + fsi_counter_readback_buffer_ = VK_NULL_HANDLE; + dfn.vkFreeMemory(device, fsi_counter_memory_, nullptr); + fsi_counter_memory_ = VK_NULL_HANDLE; + dfn.vkDestroyBuffer(device, fsi_counter_buffer_, nullptr); + fsi_counter_buffer_ = VK_NULL_HANDLE; + return false; + } + + void* fsi_counter_readback_mapping = nullptr; + if (dfn.vkMapMemory(device, fsi_counter_readback_memory_, 0, VK_WHOLE_SIZE, + 0, &fsi_counter_readback_mapping) != VK_SUCCESS) { + XELOGW( + "VulkanZPDQueryPool: Failed to map ZPD FSI counter " + "readback memory, falling back to fake sample counts."); + dfn.vkFreeMemory(device, fsi_counter_readback_memory_, nullptr); + fsi_counter_readback_memory_ = VK_NULL_HANDLE; + dfn.vkDestroyBuffer(device, fsi_counter_readback_buffer_, nullptr); + fsi_counter_readback_buffer_ = VK_NULL_HANDLE; + dfn.vkFreeMemory(device, fsi_counter_memory_, nullptr); + fsi_counter_memory_ = VK_NULL_HANDLE; + dfn.vkDestroyBuffer(device, fsi_counter_buffer_, nullptr); + fsi_counter_buffer_ = VK_NULL_HANDLE; + return false; + } + + fsi_counter_readback_mapping_ = + reinterpret_cast(fsi_counter_readback_mapping); + + initialize_shared_capacity(); + return true; + }; + + bool any_initialized = false; + if (initialize_host_query_path()) { + any_initialized = true; + } + if (initialize_fsi_counter && initialize_fsi_counter_path()) { + any_initialized = true; + } + return any_initialized; +} + +void VulkanZPDQueryPool::Shutdown() { + if (!vulkan_device_) { + query_pool_ = VK_NULL_HANDLE; + readback_buffer_ = VK_NULL_HANDLE; + readback_memory_ = VK_NULL_HANDLE; + readback_mapping_ = nullptr; + readback_is_coherent_ = true; + fsi_counter_buffer_ = VK_NULL_HANDLE; + fsi_counter_memory_ = VK_NULL_HANDLE; + fsi_counter_readback_buffer_ = VK_NULL_HANDLE; + fsi_counter_readback_memory_ = VK_NULL_HANDLE; + fsi_counter_readback_mapping_ = nullptr; + fsi_counter_readback_is_coherent_ = true; + capacity_ = 0; + free_indices_.clear(); + index_generations_.clear(); + resolve_batch_pending_.clear(); + resolve_batch_indices_.clear(); + fsi_counter_resolve_batch_pending_.clear(); + fsi_counter_resolve_batch_indices_.clear(); + resolve_batch_ranges_.clear(); + fsi_resolve_barrier_scratch_.clear(); + fsi_resolve_copy_scratch_.clear(); + return; + } + + const ui::vulkan::VulkanDevice::Functions& dfn = vulkan_device_->functions(); + const VkDevice device = vulkan_device_->device(); + + free_indices_.clear(); + index_generations_.clear(); + resolve_batch_pending_.clear(); + resolve_batch_indices_.clear(); + fsi_counter_resolve_batch_pending_.clear(); + fsi_counter_resolve_batch_indices_.clear(); + resolve_batch_ranges_.clear(); + fsi_resolve_barrier_scratch_.clear(); + fsi_resolve_copy_scratch_.clear(); + + capacity_ = 0; + readback_is_coherent_ = true; + fsi_counter_readback_is_coherent_ = true; + + if (fsi_counter_readback_mapping_ && + fsi_counter_readback_memory_ != VK_NULL_HANDLE) { + dfn.vkUnmapMemory(device, fsi_counter_readback_memory_); + } + fsi_counter_readback_mapping_ = nullptr; + + if (fsi_counter_readback_buffer_ != VK_NULL_HANDLE) { + dfn.vkDestroyBuffer(device, fsi_counter_readback_buffer_, nullptr); + } + fsi_counter_readback_buffer_ = VK_NULL_HANDLE; + + if (fsi_counter_readback_memory_ != VK_NULL_HANDLE) { + dfn.vkFreeMemory(device, fsi_counter_readback_memory_, nullptr); + } + fsi_counter_readback_memory_ = VK_NULL_HANDLE; + + if (fsi_counter_buffer_ != VK_NULL_HANDLE) { + dfn.vkDestroyBuffer(device, fsi_counter_buffer_, nullptr); + } + fsi_counter_buffer_ = VK_NULL_HANDLE; + + if (fsi_counter_memory_ != VK_NULL_HANDLE) { + dfn.vkFreeMemory(device, fsi_counter_memory_, nullptr); + } + fsi_counter_memory_ = VK_NULL_HANDLE; + + if (readback_mapping_ && readback_memory_ != VK_NULL_HANDLE) { + dfn.vkUnmapMemory(device, readback_memory_); + } + readback_mapping_ = nullptr; + + if (readback_buffer_ != VK_NULL_HANDLE) { + dfn.vkDestroyBuffer(device, readback_buffer_, nullptr); + } + readback_buffer_ = VK_NULL_HANDLE; + + if (readback_memory_ != VK_NULL_HANDLE) { + dfn.vkFreeMemory(device, readback_memory_, nullptr); + } + readback_memory_ = VK_NULL_HANDLE; + + if (query_pool_ != VK_NULL_HANDLE) { + dfn.vkDestroyQueryPool(device, query_pool_, nullptr); + } + query_pool_ = VK_NULL_HANDLE; +} + +bool VulkanZPDQueryPool::AcquireQueryIndex(uint32_t& query_index, + uint32_t& query_generation) { + if (free_indices_.empty()) { + query_index = UINT32_MAX; + query_generation = 0; + return false; + } + + query_index = free_indices_.back(); + free_indices_.pop_back(); + + assert_true(query_index < index_generations_.size()); + // Bump before returning - invalidates in-flight copies from prior occupant. + query_generation = ++index_generations_[query_index]; + return true; +} + +void VulkanZPDQueryPool::ReleaseQueryIndex(uint32_t query_index, + uint32_t query_generation) { + if (!vulkan_device_ || query_index >= capacity_) { + return; + } + + if (!GenerationMatches(query_index, query_generation)) { + return; + } + + // Bump generation so a second release with the same generation is rejected. + ++index_generations_[query_index]; + + if (query_pool_ != VK_NULL_HANDLE) { + const ui::vulkan::VulkanDevice::Functions& dfn = + vulkan_device_->functions(); + const VkDevice device = vulkan_device_->device(); + + // Immediately reset the slot on the CPU so it's ready for the next + // AcquireQueryIndex without requiring a paired EndQuery. + dfn.vkResetQueryPool(device, query_pool_, query_index, 1); + } + + free_indices_.push_back(query_index); +} + +bool VulkanZPDQueryPool::GenerationMatches(uint32_t query_index, + uint32_t query_generation) const { + return query_index < index_generations_.size() && + index_generations_[query_index] == query_generation; +} + +void VulkanZPDQueryPool::BeginQuery( + DeferredCommandBuffer& deferred_command_buffer, + uint32_t query_index) const { + if (query_pool_ == VK_NULL_HANDLE || query_index >= capacity_) { + return; + } + + // Precise bit is crucial. Most titles tested actually care about the sample + // counts, not just 0 vs non-zero. + deferred_command_buffer.CmdVkBeginQuery(query_pool_, query_index, + VK_QUERY_CONTROL_PRECISE_BIT); +} + +void VulkanZPDQueryPool::EndQuery( + DeferredCommandBuffer& deferred_command_buffer, + uint32_t query_index) const { + if (query_pool_ == VK_NULL_HANDLE || query_index >= capacity_) { + return; + } + + deferred_command_buffer.CmdVkEndQuery(query_pool_, query_index); +} + +void VulkanZPDQueryPool::QueueQueryResolve(uint32_t query_index, + bool uses_fsi_counter) { + if (query_index >= capacity_) { + return; + } + + // Guard against duplicates within the same submission. + if (uses_fsi_counter) { + if (!fsi_counter_resolve_batch_pending_[query_index]) { + fsi_counter_resolve_batch_pending_[query_index] = 1; + fsi_counter_resolve_batch_indices_.push_back(query_index); + } + return; + } + + if (!resolve_batch_pending_[query_index]) { + resolve_batch_pending_[query_index] = 1; + resolve_batch_indices_.push_back(query_index); + } +} + +void VulkanZPDQueryPool::ClearFSICounter( + DeferredCommandBuffer& deferred_command_buffer, + uint32_t query_index) const { + if (!fsi_initialized() || query_index >= capacity_) { + return; + } + + VkDeviceSize offset = + static_cast(query_index) * sizeof(uint32_t); + + VkBufferMemoryBarrier drain_barrier; + drain_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + drain_barrier.pNext = nullptr; + drain_barrier.srcAccessMask = + VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + drain_barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + drain_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + drain_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + drain_barrier.buffer = fsi_counter_buffer_; + drain_barrier.offset = offset; + drain_barrier.size = sizeof(uint32_t); + deferred_command_buffer.CmdVkPipelineBarrier( + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &drain_barrier, 0, + nullptr); + + deferred_command_buffer.CmdVkFillBuffer(fsi_counter_buffer_, offset, + sizeof(uint32_t), 0); + + VkBufferMemoryBarrier ready_barrier; + ready_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + ready_barrier.pNext = nullptr; + ready_barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + ready_barrier.dstAccessMask = + VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + ready_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + ready_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + ready_barrier.buffer = fsi_counter_buffer_; + ready_barrier.offset = offset; + ready_barrier.size = sizeof(uint32_t); + + deferred_command_buffer.CmdVkPipelineBarrier( + VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, + 0, nullptr, 1, &ready_barrier, 0, nullptr); +} + +void VulkanZPDQueryPool::RecordResolveBatch(VkCommandBuffer command_buffer) { + if (resolve_batch_indices_.empty() && + fsi_counter_resolve_batch_indices_.empty()) { + return; + } + + auto build_ranges = [this](std::vector& indices, + std::vector& pending) { + std::sort(indices.begin(), indices.end()); + resolve_batch_ranges_.clear(); + uint32_t range_start = 0; + uint32_t range_count = 0; + for (uint32_t index : indices) { + if (range_count == 0) { + range_start = index; + range_count = 1; + continue; + } + if (index == range_start + range_count) { + ++range_count; + continue; + } + resolve_batch_ranges_.push_back({range_start, range_count}); + range_start = index; + range_count = 1; + } + if (range_count != 0) { + resolve_batch_ranges_.push_back({range_start, range_count}); + } + for (uint32_t index : indices) { + pending[index] = 0; + } + indices.clear(); + }; + + const ui::vulkan::VulkanDevice::Functions& dfn = vulkan_device_->functions(); + + if (!resolve_batch_indices_.empty()) { + if (!fbo_initialized()) { + for (uint32_t index : resolve_batch_indices_) { + resolve_batch_pending_[index] = 0; + } + resolve_batch_indices_.clear(); + } else { + build_ranges(resolve_batch_indices_, resolve_batch_pending_); + + VkDeviceSize barrier_offset = VK_WHOLE_SIZE; + VkDeviceSize barrier_end = 0; + for (const ResolveRange& range : resolve_batch_ranges_) { + if (range.start >= capacity_) { + continue; + } + + uint32_t count = std::min(range.count, capacity_ - range.start); + VkDeviceSize offset = + static_cast(range.start) * sizeof(uint64_t); + VkDeviceSize size = static_cast(count) * sizeof(uint64_t); + + // WAIT_BIT blocks until available. No separate availability check + // needed. + dfn.vkCmdCopyQueryPoolResults( + command_buffer, query_pool_, range.start, count, readback_buffer_, + offset, sizeof(uint64_t), + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); + + barrier_offset = std::min(barrier_offset, offset); + barrier_end = std::max(barrier_end, offset + size); + } + + if (barrier_offset != VK_WHOLE_SIZE && barrier_end > barrier_offset) { + VkBufferMemoryBarrier readback_barrier; + readback_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + readback_barrier.pNext = nullptr; + readback_barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + readback_barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + readback_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + readback_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + readback_barrier.buffer = readback_buffer_; + readback_barrier.offset = barrier_offset; + readback_barrier.size = barrier_end - barrier_offset; + dfn.vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_HOST_BIT, 0, 0, nullptr, 1, + &readback_barrier, 0, nullptr); + } + } + } + + if (fsi_counter_resolve_batch_indices_.empty()) { + return; + } + + if (!fsi_initialized()) { + for (uint32_t index : fsi_counter_resolve_batch_indices_) { + fsi_counter_resolve_batch_pending_[index] = 0; + } + fsi_counter_resolve_batch_indices_.clear(); + return; + } + + build_ranges(fsi_counter_resolve_batch_indices_, + fsi_counter_resolve_batch_pending_); + + VkDeviceSize readback_barrier_offset = VK_WHOLE_SIZE; + VkDeviceSize readback_barrier_end = 0; + fsi_resolve_barrier_scratch_.clear(); + fsi_resolve_copy_scratch_.clear(); + fsi_resolve_barrier_scratch_.reserve(resolve_batch_ranges_.size()); + fsi_resolve_copy_scratch_.reserve(resolve_batch_ranges_.size()); + for (const ResolveRange& range : resolve_batch_ranges_) { + if (range.start >= capacity_) { + continue; + } + + uint32_t count = std::min(range.count, capacity_ - range.start); + VkDeviceSize offset = + static_cast(range.start) * sizeof(uint32_t); + VkDeviceSize size = static_cast(count) * sizeof(uint32_t); + + VkBufferMemoryBarrier counter_barrier; + counter_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + counter_barrier.pNext = nullptr; + // Include TRANSFER_WRITE so that slots cleared by vkCmdFillBuffer with no + // subsequent fragment writes are also covered. A fully occluded query sees + // the fill as its last write, not a shader atomic. + counter_barrier.srcAccessMask = + VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + counter_barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + counter_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + counter_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + counter_barrier.buffer = fsi_counter_buffer_; + counter_barrier.offset = offset; + counter_barrier.size = size; + fsi_resolve_barrier_scratch_.push_back(counter_barrier); + + VkBufferCopy copy_region; + copy_region.srcOffset = offset; + copy_region.dstOffset = offset; + copy_region.size = size; + fsi_resolve_copy_scratch_.push_back(copy_region); + + readback_barrier_offset = std::min(readback_barrier_offset, offset); + readback_barrier_end = std::max(readback_barrier_end, offset + size); + } + + if (!fsi_resolve_barrier_scratch_.empty()) { + // Source stage includes TRANSFER_BIT alongside FRAGMENT_SHADER_BIT so + // that the barrier correctly covers the vkCmdFillBuffer clear case. + dfn.vkCmdPipelineBarrier( + command_buffer, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, + uint32_t(fsi_resolve_barrier_scratch_.size()), + fsi_resolve_barrier_scratch_.data(), 0, nullptr); + dfn.vkCmdCopyBuffer(command_buffer, fsi_counter_buffer_, + fsi_counter_readback_buffer_, + uint32_t(fsi_resolve_copy_scratch_.size()), + fsi_resolve_copy_scratch_.data()); + } + + if (readback_barrier_offset == VK_WHOLE_SIZE || + readback_barrier_end <= readback_barrier_offset) { + return; + } + + VkBufferMemoryBarrier readback_barrier; + readback_barrier.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + readback_barrier.pNext = nullptr; + readback_barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + readback_barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + readback_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + readback_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + readback_barrier.buffer = fsi_counter_readback_buffer_; + readback_barrier.offset = readback_barrier_offset; + readback_barrier.size = readback_barrier_end - readback_barrier_offset; + dfn.vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_HOST_BIT, 0, 0, nullptr, 1, + &readback_barrier, 0, nullptr); +} + +void VulkanZPDQueryPool::InvalidateReadback() { + if (vulkan_device_) { + const ui::vulkan::VulkanDevice::Functions& dfn = + vulkan_device_->functions(); + const VkDevice device = vulkan_device_->device(); + + if (!readback_is_coherent_ && readback_memory_ != VK_NULL_HANDLE && + readback_mapping_) { + // Invalidates the CPU-side cache so the persistent mapping reflects the + // GPU writes made by vkCmdCopyQueryPoolResults. Not needed on + // HOST_COHERENT. + VkMappedMemoryRange range; + range.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE; + range.pNext = nullptr; + range.memory = readback_memory_; + range.offset = 0; + range.size = VK_WHOLE_SIZE; + dfn.vkInvalidateMappedMemoryRanges(device, 1, &range); + } + + if (!fsi_counter_readback_is_coherent_ && + fsi_counter_readback_memory_ != VK_NULL_HANDLE && + fsi_counter_readback_mapping_) { + VkMappedMemoryRange range; + range.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE; + range.pNext = nullptr; + range.memory = fsi_counter_readback_memory_; + range.offset = 0; + range.size = VK_WHOLE_SIZE; + dfn.vkInvalidateMappedMemoryRanges(device, 1, &range); + } + } +} + +uint64_t VulkanZPDQueryPool::GetQueryReadbackValue( + uint32_t query_index, bool uses_fsi_counter) const { + if (query_index >= capacity_) { + return 0; + } + + if (uses_fsi_counter) { + return fsi_counter_readback_mapping_ + ? static_cast( + fsi_counter_readback_mapping_[query_index]) + : 0; + } + + return readback_mapping_ ? readback_mapping_[query_index] : 0; +} + +} // namespace vulkan +} // namespace gpu +} // namespace xe diff --git a/src/xenia/gpu/vulkan/vulkan_zpd_query_pool.h b/src/xenia/gpu/vulkan/vulkan_zpd_query_pool.h new file mode 100644 index 000000000..2bb43d4ac --- /dev/null +++ b/src/xenia/gpu/vulkan/vulkan_zpd_query_pool.h @@ -0,0 +1,139 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_GPU_VULKAN_VULKAN_ZPD_QUERY_POOL_H_ +#define XENIA_GPU_VULKAN_VULKAN_ZPD_QUERY_POOL_H_ + +#include +#include + +#include "xenia/gpu/command_processor.h" +#include "xenia/ui/vulkan/vulkan_api.h" + +namespace xe { +namespace ui { +namespace vulkan { +class VulkanDevice; +} +} // namespace ui + +namespace gpu { +namespace vulkan { + +class DeferredCommandBuffer; + +// Vulkan occlusion query pool for ZPD reports. Queries live in VkQueryPool, +// results are copied to a persistent buffer via vkCmdCopyQueryPoolResults. +// vkCmdBeginQuery is only valid inside a render pass, queries get deferred +// when no pass is open and segments split at pass boundaries. +// +// Requires VK_EXT_host_query_reset (1.2 core) so slots can be reset on the +// CPU at release time, no paired vkCmdEndQuery needed, and also allows +// DiscardZPDQuery work outside a pass. +// +// VK_QUERY_RESULT_WAIT_BIT in the copy removes the need for a separate +// availability check. Transfer barrier before InvalidateReadback covers non- +// coherent memory. +class VulkanZPDQueryPool { + public: + VulkanZPDQueryPool() = default; + VulkanZPDQueryPool(const VulkanZPDQueryPool&) = delete; + VulkanZPDQueryPool& operator=(const VulkanZPDQueryPool&) = delete; + ~VulkanZPDQueryPool() { Shutdown(); } + + bool EnsureInitialized(const ui::vulkan::VulkanDevice* vulkan_device, + uint32_t requested_capacity, bool can_recreate, + bool initialize_fsi_counter = false); + void Shutdown(); + + bool fbo_initialized() const { + return query_pool_ != VK_NULL_HANDLE && + readback_buffer_ != VK_NULL_HANDLE && readback_mapping_ != nullptr && + capacity_ != 0; + } + + bool fsi_initialized() const { + return fsi_counter_buffer_ != VK_NULL_HANDLE && + fsi_counter_memory_ != VK_NULL_HANDLE && + fsi_counter_readback_buffer_ != VK_NULL_HANDLE && + fsi_counter_readback_mapping_ != nullptr && capacity_ != 0; + } + + uint32_t capacity() const { return capacity_; } + + bool has_pending_resolve_batch() const { + return !resolve_batch_indices_.empty() || + !fsi_counter_resolve_batch_indices_.empty(); + } + + VkBuffer fsi_counter_buffer() const { return fsi_counter_buffer_; } + + bool has_free_indices() const { return !free_indices_.empty(); } + + bool AcquireQueryIndex(uint32_t& query_index, uint32_t& query_generation); + void ReleaseQueryIndex(uint32_t query_index, uint32_t query_generation); + bool GenerationMatches(uint32_t query_index, uint32_t query_generation) const; + + void BeginQuery(DeferredCommandBuffer& deferred_command_buffer, + uint32_t query_index) const; + void EndQuery(DeferredCommandBuffer& deferred_command_buffer, + uint32_t query_index) const; + void QueueQueryResolve(uint32_t query_index, bool uses_fsi_counter = false); + void ClearFSICounter(DeferredCommandBuffer& deferred_command_buffer, + uint32_t query_index) const; + void RecordResolveBatch(VkCommandBuffer command_buffer); + + void InvalidateReadback(); + + uint64_t GetQueryReadbackValue(uint32_t query_index, + bool uses_fsi_counter = false) const; + + private: + const ui::vulkan::VulkanDevice* vulkan_device_ = nullptr; + + VkQueryPool query_pool_ = VK_NULL_HANDLE; + + VkBuffer readback_buffer_ = VK_NULL_HANDLE; + VkDeviceMemory readback_memory_ = VK_NULL_HANDLE; + uint64_t* readback_mapping_ = nullptr; + // If not HOST_COHERENT, call InvalidateReadback before reading. + bool readback_is_coherent_ = true; + + VkBuffer fsi_counter_buffer_ = VK_NULL_HANDLE; + VkDeviceMemory fsi_counter_memory_ = VK_NULL_HANDLE; + + VkBuffer fsi_counter_readback_buffer_ = VK_NULL_HANDLE; + VkDeviceMemory fsi_counter_readback_memory_ = VK_NULL_HANDLE; + uint32_t* fsi_counter_readback_mapping_ = nullptr; + bool fsi_counter_readback_is_coherent_ = true; + + uint32_t capacity_ = 0; + std::vector free_indices_; + + // Bumped on each acquire so stale copies from recycled slots get dropped. + std::vector index_generations_; + + std::vector resolve_batch_pending_; + // Active indices with resolve_batch_pending_[i] == 1, so flush iterates + // only the active entries instead of scanning the full capacity. + std::vector resolve_batch_indices_; + std::vector fsi_counter_resolve_batch_pending_; + std::vector fsi_counter_resolve_batch_indices_; + // Reusable scratch for coalesced contiguous ranges during flush. + std::vector resolve_batch_ranges_; + // Reusable scratch for FSI counter resolve barriers and copy regions. + std::vector fsi_resolve_barrier_scratch_; + std::vector fsi_resolve_copy_scratch_; +}; + +} // namespace vulkan +} // namespace gpu +} // namespace xe + +#endif // XENIA_GPU_VULKAN_VULKAN_ZPD_QUERY_POOL_H_ diff --git a/src/xenia/gpu/xenos_zpd_report.h b/src/xenia/gpu/xenos_zpd_report.h new file mode 100644 index 000000000..437e10947 --- /dev/null +++ b/src/xenia/gpu/xenos_zpd_report.h @@ -0,0 +1,168 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2026 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_GPU_XENOS_ZPD_REPORT_H_ +#define XENIA_GPU_XENOS_ZPD_REPORT_H_ + +#include +#include +#include + +#include "xenia/gpu/gpu_flags.h" +#include "xenia/gpu/xenos.h" + +namespace xe { +namespace gpu { + +// Guest memory helpers for occlusion query ZPD reports. +struct XenosZPDReport { + static constexpr uint32_t kRecordSizeBytes = 0x20; + static constexpr uint32_t kRecordAlignMask = ~(kRecordSizeBytes - 1); + + // Each slot holds one BEGIN record and one END record. + // END is at the slot base, and BEGIN is +0x20. + static constexpr uint32_t kSlotSizeBytes = 0x40; + static constexpr uint32_t kSlotAlignMask = ~(kSlotSizeBytes - 1); + + static constexpr uint32_t GetRecordBase(uint32_t address) { + return address & kRecordAlignMask; + } + + static constexpr uint32_t GetSlotBase(uint32_t address) { + return address & kSlotAlignMask; + } + + static constexpr uint32_t GetBeginRecordBase(uint32_t address) { + return GetSlotBase(address) + kRecordSizeBytes; + } + + static constexpr uint32_t GetEndRecordBase(uint32_t address) { + return GetSlotBase(address); + } + + static constexpr bool IsBeginRecord(uint32_t address) { + uint32_t record_base = GetRecordBase(address); + return record_base && record_base == GetBeginRecordBase(record_base); + } + + static constexpr bool IsEndRecord(uint32_t address) { + uint32_t record_base = GetRecordBase(address); + return record_base && record_base == GetEndRecordBase(record_base); + } + + // ZPass is where titles almost always test pending boundaries. Some older + // D3D may also check ZFail, so both should be covered. A few titles, like + // 4D5307E8, write distinct B values, but this is rare, and still, there + // isn't any documented case of B lanes mattering for boundary detection. + static bool HasPendingSentinel( + const xenos::xe_gpu_depth_sample_counts* report) { + constexpr uint32_t kSentinelLE = 0xEDFEFFFFu; + constexpr uint32_t kSentinelBE = 0xFFFFFEEDu; + + if (report->ZPass_A == kSentinelLE || report->ZPass_A == kSentinelBE) { + return true; + } + if (report->ZFail_A == kSentinelLE || report->ZFail_A == kSentinelBE) { + return true; + } + return false; + } + + // Xenos has real Total/ZFail/StencilFail counters. Total should technically + // be the sum of all sample counts, not just copied from ZPass. But host + // occlusion queries can only give us the final passing sample count, so + // treat that as ZPass_A and mirror it to Total_A for titles that check it. + // Theoretically, the EDRAM paths could count ZFail/StencilFail since they + // run the emulated depth/stencil test, but that adds more shader work, + // atomics, and resolve challenges for counters that haven't been actually + // proven to be useful yet. That doesn't mean that titles that test those + // counters are unsupported, just that there might be some attenuation + // differences from real hardware in ways we can't confirm yet. + static void WriteSampleCount(xenos::xe_gpu_depth_sample_counts* report, + uint32_t sample_count, bool saturate = true) { + if (saturate) { + sample_count = SaturateSampleCount(sample_count); + } + + report->Total_A = sample_count; + report->Total_B = 0; + report->ZFail_A = 0; + report->ZFail_B = 0; + report->ZPass_A = sample_count; + report->ZPass_B = 0; + report->StencilFail_A = 0; + report->StencilFail_B = 0; + } + + static uint32_t SaturateSampleCount(uint32_t sample_count) { + double saturation = std::clamp( + static_cast(cvars::occlusion_query_sample_count_saturation), + 0.0, 1.0); + + if (sample_count == 0 || saturation >= 1.0) { + return sample_count; + } + if (saturation <= 0.0) { + return 1; + } + + // Preserve lower sample counts often used for visibility testing and + // compress only the higher range used by effects. The knee here is somewhat + // arbitrary but seems to provide a good balance of safety and tunability. + const double knee = 32.0; + if (static_cast(sample_count) <= knee) { + return sample_count; + } + + const double attenuation = 1.0 - saturation; + const double exponent = 1.0 - (1.0 - 0.35) * (attenuation * attenuation * + (3.0 - 2.0 * attenuation)); + double saturated_count = + knee + std::pow(static_cast(sample_count) - knee, exponent); + + return static_cast(saturated_count + 0.5); + } + + // Fake mode for titles (425307EC, 4D5309B1) that use QueryBatch and expect + // the sample count to accumulate across multiple records. + static uint32_t QueryBatchFakeSamples(uint32_t& sample_count) { + int32_t lower = cvars::occlusion_query_fake_lower_threshold; + uint32_t base = lower > 0 ? static_cast(lower) : 0; + uint32_t range = + static_cast(cvars::occlusion_query_querybatch_range); + + if (sample_count - base >= range) { + sample_count = base; + } + + uint32_t current_sample_count = sample_count++; + if (sample_count - base >= range) { + sample_count = base; + } + return current_sample_count; + } + + static void WriteReportDelta(xenos::xe_gpu_depth_sample_counts* begin_report, + xenos::xe_gpu_depth_sample_counts* end_report, + uint32_t begin_value, uint32_t delta_value, + bool write_begin_report) { + delta_value = SaturateSampleCount(delta_value); + uint32_t end_value = begin_value + delta_value; + + if (write_begin_report && begin_report && end_report != begin_report) { + WriteSampleCount(begin_report, begin_value, false); + } + WriteSampleCount(end_report, end_value, false); + } +}; + +} // namespace gpu +} // namespace xe + +#endif // XENIA_GPU_XENOS_ZPD_REPORT_H_ diff --git a/src/xenia/ui/vulkan/functions/device_1_0.inc b/src/xenia/ui/vulkan/functions/device_1_0.inc index 5646e9d1d..c7bb031e9 100644 --- a/src/xenia/ui/vulkan/functions/device_1_0.inc +++ b/src/xenia/ui/vulkan/functions/device_1_0.inc @@ -5,6 +5,7 @@ XE_UI_VULKAN_FUNCTION(vkAllocateMemory) XE_UI_VULKAN_FUNCTION(vkBeginCommandBuffer) XE_UI_VULKAN_FUNCTION(vkBindBufferMemory) XE_UI_VULKAN_FUNCTION(vkBindImageMemory) +XE_UI_VULKAN_FUNCTION(vkCmdBeginQuery) XE_UI_VULKAN_FUNCTION(vkCmdBeginRenderPass) XE_UI_VULKAN_FUNCTION(vkCmdBindDescriptorSets) XE_UI_VULKAN_FUNCTION(vkCmdBindIndexBuffer) @@ -16,12 +17,16 @@ XE_UI_VULKAN_FUNCTION(vkCmdClearColorImage) XE_UI_VULKAN_FUNCTION(vkCmdCopyBuffer) XE_UI_VULKAN_FUNCTION(vkCmdCopyBufferToImage) XE_UI_VULKAN_FUNCTION(vkCmdCopyImageToBuffer) +XE_UI_VULKAN_FUNCTION(vkCmdCopyQueryPoolResults) XE_UI_VULKAN_FUNCTION(vkCmdDispatch) XE_UI_VULKAN_FUNCTION(vkCmdDraw) XE_UI_VULKAN_FUNCTION(vkCmdDrawIndexed) +XE_UI_VULKAN_FUNCTION(vkCmdEndQuery) XE_UI_VULKAN_FUNCTION(vkCmdEndRenderPass) +XE_UI_VULKAN_FUNCTION(vkCmdFillBuffer) XE_UI_VULKAN_FUNCTION(vkCmdPipelineBarrier) XE_UI_VULKAN_FUNCTION(vkCmdPushConstants) +XE_UI_VULKAN_FUNCTION(vkCmdResetQueryPool) XE_UI_VULKAN_FUNCTION(vkCmdSetBlendConstants) XE_UI_VULKAN_FUNCTION(vkCmdSetDepthBias) XE_UI_VULKAN_FUNCTION(vkCmdSetScissor) @@ -42,6 +47,7 @@ XE_UI_VULKAN_FUNCTION(vkCreateImage) XE_UI_VULKAN_FUNCTION(vkCreateImageView) XE_UI_VULKAN_FUNCTION(vkCreatePipelineCache) XE_UI_VULKAN_FUNCTION(vkCreatePipelineLayout) +XE_UI_VULKAN_FUNCTION(vkCreateQueryPool) XE_UI_VULKAN_FUNCTION(vkCreateRenderPass) XE_UI_VULKAN_FUNCTION(vkCreateSampler) XE_UI_VULKAN_FUNCTION(vkCreateSemaphore) @@ -58,6 +64,7 @@ XE_UI_VULKAN_FUNCTION(vkDestroyImageView) XE_UI_VULKAN_FUNCTION(vkDestroyPipeline) XE_UI_VULKAN_FUNCTION(vkDestroyPipelineCache) XE_UI_VULKAN_FUNCTION(vkDestroyPipelineLayout) +XE_UI_VULKAN_FUNCTION(vkDestroyQueryPool) XE_UI_VULKAN_FUNCTION(vkDestroyRenderPass) XE_UI_VULKAN_FUNCTION(vkDestroySampler) XE_UI_VULKAN_FUNCTION(vkDestroySemaphore) diff --git a/src/xenia/ui/vulkan/functions/device_1_2_ext_host_query_reset.inc b/src/xenia/ui/vulkan/functions/device_1_2_ext_host_query_reset.inc new file mode 100644 index 000000000..b3ef98998 --- /dev/null +++ b/src/xenia/ui/vulkan/functions/device_1_2_ext_host_query_reset.inc @@ -0,0 +1,3 @@ +// VK_EXT_host_query_reset functions used in Xenia. +// Promoted to Vulkan 1.2 core. +XE_UI_VULKAN_FUNCTION_PROMOTED(vkResetQueryPoolEXT, vkResetQueryPool) diff --git a/src/xenia/ui/vulkan/vulkan_device.cc b/src/xenia/ui/vulkan/vulkan_device.cc index ab34ce38f..1d279628a 100644 --- a/src/xenia/ui/vulkan/vulkan_device.cc +++ b/src/xenia/ui/vulkan/vulkan_device.cc @@ -175,6 +175,7 @@ std::unique_ptr VulkanDevice::CreateIfSupported( } bool ext_1_2_KHR_sampler_mirror_clamp_to_edge = false; + bool ext_1_2_EXT_host_query_reset = false; bool ext_1_1_KHR_maintenance1 = false; bool ext_1_2_KHR_shader_float_controls = false; bool ext_EXT_fragment_shader_interlock = false; @@ -195,6 +196,7 @@ std::unique_ptr VulkanDevice::CreateIfSupported( XE_UI_VULKAN_STRUCT_PROMOTED_EXTENSION(KHR_sampler_ycbcr_conversion, 1, 1) // #198. Also must be enabled for VK_KHR_spirv_1_4. XE_UI_VULKAN_LOCAL_PROMOTED_EXTENSION(KHR_shader_float_controls, 1, 2) + XE_UI_VULKAN_LOCAL_PROMOTED_EXTENSION(EXT_host_query_reset, 1, 2) // #252. XE_UI_VULKAN_LOCAL_EXTENSION(EXT_fragment_shader_interlock) // #277. @@ -282,6 +284,9 @@ std::unique_ptr VulkanDevice::CreateIfSupported( VulkanFeatures features_1_2; + VulkanFeatures + features_EXT_host_query_reset; VulkanFeatures features_1_3; @@ -310,6 +315,9 @@ std::unique_ptr VulkanDevice::CreateIfSupported( if (get_physical_device_properties2_supported) { if (properties.apiVersion >= VK_MAKE_API_VERSION(0, 1, 2, 0)) { features_1_2.Link(supported_features_2, device_create_info); + } else if (ext_1_2_EXT_host_query_reset) { + features_EXT_host_query_reset.Link(supported_features_2, + device_create_info); } if (properties.apiVersion >= VK_MAKE_API_VERSION(0, 1, 3, 0)) { features_1_3.Link(supported_features_2, device_create_info); @@ -631,12 +639,18 @@ std::unique_ptr VulkanDevice::CreateIfSupported( XE_UI_VULKAN_FEATURE_2(features_1_2, samplerMirrorClampToEdge); XE_UI_VULKAN_FEATURE_2(features_1_2, uniformBufferStandardLayout); XE_UI_VULKAN_FEATURE_2(features_1_2, scalarBlockLayout); + XE_UI_VULKAN_FEATURE_2(features_1_2, hostQueryReset); } } else { if (ext_1_2_KHR_sampler_mirror_clamp_to_edge) { XE_UI_VULKAN_FEATURE_IMPLIED(samplerMirrorClampToEdge) } + if (ext_1_2_EXT_host_query_reset && with_gpu_emulation) { + XE_UI_VULKAN_FEATURE_2(features_EXT_host_query_reset, hostQueryReset); + } } + device->extensions_.ext_1_2_EXT_host_query_reset = + ext_1_2_EXT_host_query_reset; if (properties.apiVersion >= VK_MAKE_API_VERSION(0, 1, 3, 0)) { if (with_gpu_emulation) { @@ -743,6 +757,9 @@ std::unique_ptr VulkanDevice::CreateIfSupported( if (properties.apiVersion >= VK_MAKE_API_VERSION(0, 1, 1, 0)) { #include "xenia/ui/vulkan/functions/device_1_1_khr_bind_memory2.inc" #include "xenia/ui/vulkan/functions/device_1_1_khr_get_memory_requirements2.inc" + } + if (properties.apiVersion >= VK_MAKE_API_VERSION(0, 1, 2, 0)) { +#include "xenia/ui/vulkan/functions/device_1_2_ext_host_query_reset.inc" } if (properties.apiVersion >= VK_MAKE_API_VERSION(0, 1, 3, 0)) { #include "xenia/ui/vulkan/functions/device_1_3_khr_maintenance4.inc" @@ -763,6 +780,11 @@ std::unique_ptr VulkanDevice::CreateIfSupported( #include "xenia/ui/vulkan/functions/device_1_1_khr_bind_memory2.inc" } } + if (properties.apiVersion < VK_MAKE_API_VERSION(0, 1, 2, 0)) { + if (device->extensions_.ext_1_2_EXT_host_query_reset) { +#include "xenia/ui/vulkan/functions/device_1_2_ext_host_query_reset.inc" + } + } if (properties.apiVersion < VK_MAKE_API_VERSION(0, 1, 3, 0)) { if (device->extensions_.ext_1_3_KHR_maintenance4) { #include "xenia/ui/vulkan/functions/device_1_3_khr_maintenance4.inc" diff --git a/src/xenia/ui/vulkan/vulkan_device.h b/src/xenia/ui/vulkan/vulkan_device.h index f655d4f9d..2d03912b4 100644 --- a/src/xenia/ui/vulkan/vulkan_device.h +++ b/src/xenia/ui/vulkan/vulkan_device.h @@ -126,6 +126,10 @@ class VulkanDevice { bool scalarBlockLayout = false; + // VK_EXT_host_query_reset (promoted to 1.2) + + bool hostQueryReset = false; + // VK_KHR_portability_subset (#164) bool constantAlphaColorBlendFactors = false; @@ -181,6 +185,7 @@ class VulkanDevice { bool ext_1_1_KHR_bind_memory2 = false; // #158 bool ext_1_2_KHR_spirv_1_4 = false; // #237 bool ext_EXT_memory_budget = false; // #238 + bool ext_1_2_EXT_host_query_reset = false; // promoted to 1.2 // Has optional features not implied by this being true. bool ext_1_3_KHR_maintenance4 = false; // #414 #if XE_PLATFORM_WIN32 @@ -204,6 +209,8 @@ class VulkanDevice { #include "xenia/ui/vulkan/functions/device_1_1_khr_get_memory_requirements2.inc" // VK_KHR_bind_memory2 (#158, promoted to 1.1) #include "xenia/ui/vulkan/functions/device_1_1_khr_bind_memory2.inc" + // VK_EXT_host_query_reset (promoted to 1.2) +#include "xenia/ui/vulkan/functions/device_1_2_ext_host_query_reset.inc" // VK_KHR_maintenance4 (#414, promoted to 1.3) #include "xenia/ui/vulkan/functions/device_1_3_khr_maintenance4.inc" #undef XE_UI_VULKAN_FUNCTION_PROMOTED