fabi/Xenia-Canary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[D3D12/Vulkan] Simplify host GPU fence management

Browse Source 
Replace the `SubmissionTracker`s with new `GPUCompletionTimeline`s with a
more unified interface (using a base class), and without the internal logic
for queue ownership transfers since that idea was scrapped during the
development of the `Presenter`.

Also use this fence management logic for GPU emulation, though without
architectural reworks for now, just on the bottom level.

Still very messy, but can be cleaned up in further GPU command processor
and presenter reworks.
This commit is contained in:
Triang3l
2025-12-14 21:10:06 +03:00
parent 01ae24e46e
commit fe1fd36137
18 changed files with 894 additions and 974 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
src/xenia/gpu/vulkan/vulkan_command_processor.cc
View File

@@ -69,6 +69,9 @@ const VkDescriptorPoolSize
VulkanCommandProcessor::VulkanCommandProcessor(
VulkanGraphicsSystem* graphics_system, kernel::KernelState* kernel_state)
: CommandProcessor(graphics_system, kernel_state),
completion_timeline_(static_cast<const ui::vulkan::VulkanProvider*>(
graphics_system->provider())
->vulkan_device()),
deferred_command_buffer_(*this),
transient_descriptor_allocator_uniform_buffer_(
static_cast<const ui::vulkan::VulkanProvider*>(
@@ -1150,26 +1153,17 @@ void VulkanCommandProcessor::ShutdownContext() {
dfn.vkDestroySemaphore(device, semaphore.second, nullptr);
}
submissions_in_flight_semaphores_.clear();
for (VkFence& fence : submissions_in_flight_fences_) {
dfn.vkDestroyFence(device, fence, nullptr);
}
submissions_in_flight_fences_.clear();
current_submission_wait_stage_masks_.clear();
for (VkSemaphore semaphore : current_submission_wait_semaphores_) {
dfn.vkDestroySemaphore(device, semaphore, nullptr);
}
current_submission_wait_semaphores_.clear();
submission_completed_ = 0;
submission_open_ = false;
for (VkSemaphore semaphore : semaphores_free_) {
dfn.vkDestroySemaphore(device, semaphore, nullptr);
}
semaphores_free_.clear();
for (VkFence fence : fences_free_) {
dfn.vkDestroyFence(device, fence, nullptr);
}
fences_free_.clear();
device_lost_ = false;
@@ -1418,7 +1412,8 @@ void VulkanCommandProcessor::IssueSwap(uint32_t frontbuffer_ptr,
SwapFramebuffer& new_swap_framebuffer =
swap_framebuffers_[swap_framebuffer_new_index];
if (new_swap_framebuffer.framebuffer != VK_NULL_HANDLE) {
if (submission_completed_ >= new_swap_framebuffer.last_submission) {
if (GetCompletedSubmission() >=
new_swap_framebuffer.last_submission) {
dfn.vkDestroyFramebuffer(device, new_swap_framebuffer.framebuffer,
nullptr);
} else {
@@ -2026,7 +2021,7 @@ VulkanCommandProcessor::AcquireScratchGpuBuffer(
return ScratchBufferAcquisition();
}
if (submission_completed_ >= scratch_buffer_last_usage_submission_) {
if (GetCompletedSubmission() >= scratch_buffer_last_usage_submission_) {
const ui::vulkan::VulkanDevice::Functions& dfn = vulkan_device->functions();
const VkDevice device = vulkan_device->device();
if (scratch_buffer_ != VK_NULL_HANDLE) {
@@ -2341,7 +2336,7 @@ bool VulkanCommandProcessor::IssueDraw(xenos::PrimitiveType prim_type,
texture_cache_->GetSubmissionToAwaitOnSamplerOverflow(
samplers_overflowed_count);
assert_true(sampler_overflow_await_submission <= GetCurrentSubmission());
CheckSubmissionFenceAndDeviceLoss(sampler_overflow_await_submission);
CheckSubmissionCompletionAndDeviceLoss(sampler_overflow_await_submission);
}
// Set up the render targets - this may perform dispatches and draws.
@@ -2646,7 +2641,7 @@ void VulkanCommandProcessor::InitializeTrace() {
}
}
void VulkanCommandProcessor::CheckSubmissionFenceAndDeviceLoss(
void VulkanCommandProcessor::CheckSubmissionCompletionAndDeviceLoss(
uint64_t await_submission) {
// Only report once, no need to retry a wait that won't succeed anyway.
if (device_lost_) {
@@ -2662,69 +2657,23 @@ void VulkanCommandProcessor::CheckSubmissionFenceAndDeviceLoss(
await_submission = GetCurrentSubmission() - 1;
}
const ui::vulkan::VulkanDevice* const vulkan_device = GetVulkanDevice();
const ui::vulkan::VulkanDevice::Functions& dfn = vulkan_device->functions();
const VkDevice device = vulkan_device->device();
completion_timeline_.AwaitSubmissionAndUpdateCompleted(await_submission);
size_t fences_total = submissions_in_flight_fences_.size();
size_t fences_awaited = 0;
if (await_submission > submission_completed_) {
// Await in a blocking way if requested.
// TODO(Triang3l): Await only one fence. "Fence signal operations that are
// defined by vkQueueSubmit additionally include in the first
// synchronization scope all commands that occur earlier in submission
// order."
VkResult wait_result = dfn.vkWaitForFences(
device, uint32_t(await_submission - submission_completed_),
submissions_in_flight_fences_.data(), VK_TRUE, UINT64_MAX);
if (wait_result == VK_SUCCESS) {
fences_awaited += await_submission - submission_completed_;
} else {
XELOGE("Failed to await submission completion Vulkan fences");
if (wait_result == VK_ERROR_DEVICE_LOST) {
device_lost_ = true;
}
}
}
// Check how far into the submissions the GPU currently is, in order because
// submission themselves can be executed out of order, but Xenia serializes
// that for simplicity.
while (fences_awaited < fences_total) {
VkResult fence_status = dfn.vkWaitForFences(
device, 1, &submissions_in_flight_fences_[fences_awaited], VK_TRUE, 0);
if (fence_status != VK_SUCCESS) {
if (fence_status == VK_ERROR_DEVICE_LOST) {
device_lost_ = true;
}
break;
}
++fences_awaited;
}
if (device_lost_) {
const ui::vulkan::VulkanDevice* const vulkan_device = GetVulkanDevice();
if (vulkan_device->IsLost()) {
device_lost_ = true;
graphics_system_->OnHostGpuLossFromAnyThread(true);
return;
}
if (!fences_awaited) {
// Not updated - no need to reclaim or download things.
return;
}
// Reclaim fences.
fences_free_.reserve(fences_free_.size() + fences_awaited);
auto submissions_in_flight_fences_awaited_end =
submissions_in_flight_fences_.cbegin();
std::advance(submissions_in_flight_fences_awaited_end, fences_awaited);
fences_free_.insert(fences_free_.cend(),
submissions_in_flight_fences_.cbegin(),
submissions_in_flight_fences_awaited_end);
submissions_in_flight_fences_.erase(submissions_in_flight_fences_.cbegin(),
submissions_in_flight_fences_awaited_end);
submission_completed_ += fences_awaited;
const uint64_t completed_submission = GetCompletedSubmission();
// Reclaim semaphores.
while (!submissions_in_flight_semaphores_.empty()) {
const auto& semaphore_submission =
submissions_in_flight_semaphores_.front();
if (semaphore_submission.first > submission_completed_) {
if (semaphore_submission.first > completed_submission) {
break;
}
semaphores_free_.push_back(semaphore_submission.second);
@@ -2734,7 +2683,7 @@ void VulkanCommandProcessor::CheckSubmissionFenceAndDeviceLoss(
// Reclaim command pools.
while (!command_buffers_submitted_.empty()) {
const auto& command_buffer_pair = command_buffers_submitted_.front();
if (command_buffer_pair.first > submission_completed_) {
if (command_buffer_pair.first > completed_submission) {
break;
}
command_buffers_writable_.push_back(command_buffer_pair.second);
@@ -2747,12 +2696,14 @@ void VulkanCommandProcessor::CheckSubmissionFenceAndDeviceLoss(
render_target_cache_->CompletedSubmissionUpdated();
texture_cache_->CompletedSubmissionUpdated(submission_completed_);
texture_cache_->CompletedSubmissionUpdated(completed_submission);
// Destroy objects scheduled for destruction.
const ui::vulkan::VulkanDevice::Functions& dfn = vulkan_device->functions();
const VkDevice device = vulkan_device->device();
while (!destroy_framebuffers_.empty()) {
const auto& destroy_pair = destroy_framebuffers_.front();
if (destroy_pair.first > submission_completed_) {
if (destroy_pair.first > completed_submission) {
break;
}
dfn.vkDestroyFramebuffer(device, destroy_pair.second, nullptr);
@@ -2760,7 +2711,7 @@ void VulkanCommandProcessor::CheckSubmissionFenceAndDeviceLoss(
}
while (!destroy_buffers_.empty()) {
const auto& destroy_pair = destroy_buffers_.front();
if (destroy_pair.first > submission_completed_) {
if (destroy_pair.first > completed_submission) {
break;
}
dfn.vkDestroyBuffer(device, destroy_pair.second, nullptr);
@@ -2768,7 +2719,7 @@ void VulkanCommandProcessor::CheckSubmissionFenceAndDeviceLoss(
}
while (!destroy_memory_.empty()) {
const auto& destroy_pair = destroy_memory_.front();
if (destroy_pair.first > submission_completed_) {
if (destroy_pair.first > completed_submission) {
break;
}
dfn.vkFreeMemory(device, destroy_pair.second, nullptr);
@@ -2798,8 +2749,9 @@ bool VulkanCommandProcessor::BeginSubmission(bool is_guest_command) {
is_opening_frame
? closed_frame_submissions_[frame_current_ % kMaxFramesInFlight]
: 0;
CheckSubmissionFenceAndDeviceLoss(await_submission);
if (device_lost_ || submission_completed_ < await_submission) {
CheckSubmissionCompletionAndDeviceLoss(await_submission);
const uint64_t completed_submission = GetCompletedSubmission();
if (device_lost_ || completed_submission < await_submission) {
return false;
}
@@ -2812,7 +2764,7 @@ bool VulkanCommandProcessor::BeginSubmission(bool is_guest_command) {
for (uint64_t frame = frame_completed_ + 1; frame < frame_current_;
++frame) {
if (closed_frame_submissions_[frame % kMaxFramesInFlight] >
submission_completed_) {
completed_submission) {
break;
}
frame_completed_ = frame;
@@ -2925,27 +2877,12 @@ bool VulkanCommandProcessor::BeginSubmission(bool is_guest_command) {
}
bool VulkanCommandProcessor::EndSubmission(bool is_swap) {
const ui::vulkan::VulkanDevice* const vulkan_device = GetVulkanDevice();
ui::vulkan::VulkanDevice* const vulkan_device = GetVulkanDevice();
const ui::vulkan::VulkanDevice::Functions& dfn = vulkan_device->functions();
const VkDevice device = vulkan_device->device();
// Make sure everything needed for submitting exist.
if (submission_open_) {
if (fences_free_.empty()) {
VkFenceCreateInfo fence_create_info;
fence_create_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
fence_create_info.pNext = nullptr;
fence_create_info.flags = 0;
VkFence fence;
if (dfn.vkCreateFence(device, &fence_create_info, nullptr, &fence) !=
VK_SUCCESS) {
XELOGE("Failed to create a Vulkan fence");
// Try to submit later. Completely dropping the submission is not
// permitted because resources would be left in an undefined state.
return false;
}
fences_free_.push_back(fence);
}
if (!sparse_memory_binds_.empty() && semaphores_free_.empty()) {
VkSemaphoreCreateInfo semaphore_create_info;
semaphore_create_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
@@ -3088,58 +3025,37 @@ bool VulkanCommandProcessor::EndSubmission(bool is_swap) {
return false;
}
VkSubmitInfo submit_info;
submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submit_info.pNext = nullptr;
const uint64_t submission_index = GetCurrentSubmission();
VkSubmitInfo submit_info = {VK_STRUCTURE_TYPE_SUBMIT_INFO};
if (!current_submission_wait_semaphores_.empty()) {
submit_info.waitSemaphoreCount =
uint32_t(current_submission_wait_semaphores_.size());
submit_info.pWaitSemaphores = current_submission_wait_semaphores_.data();
submit_info.pWaitDstStageMask =
current_submission_wait_stage_masks_.data();
} else {
submit_info.waitSemaphoreCount = 0;
submit_info.pWaitSemaphores = nullptr;
submit_info.pWaitDstStageMask = nullptr;
}
submit_info.commandBufferCount = 1;
submit_info.pCommandBuffers = &command_buffer.buffer;
submit_info.signalSemaphoreCount = 0;
submit_info.pSignalSemaphores = nullptr;
assert_false(fences_free_.empty());
VkFence fence = fences_free_.back();
if (dfn.vkResetFences(device, 1, &fence) != VK_SUCCESS) {
XELOGE("Failed to reset a Vulkan submission fence");
return false;
}
VkResult submit_result;
{
ui::vulkan::VulkanDevice::Queue::Acquisition queue_acquisition =
vulkan_device->AcquireQueue(
vulkan_device->queue_family_graphics_compute(), 0);
submit_result =
dfn.vkQueueSubmit(queue_acquisition.queue(), 1, &submit_info, fence);
}
const VkResult submit_result = completion_timeline_.AcquireFenceAndSubmit(
vulkan_device->queue_family_graphics_compute(), 0, 1, &submit_info);
if (submit_result != VK_SUCCESS) {
XELOGE("Failed to submit a Vulkan command buffer");
if (submit_result == VK_ERROR_DEVICE_LOST && !device_lost_) {
XELOGE("Failed to submit a GPU emulation Vulkan command buffer: {}",
vk::to_string(vk::Result(submit_result)));
if (vulkan_device->IsLost() && !device_lost_) {
device_lost_ = true;
graphics_system_->OnHostGpuLossFromAnyThread(true);
}
return false;
}
uint64_t submission_current = GetCurrentSubmission();
current_submission_wait_stage_masks_.clear();
for (VkSemaphore semaphore : current_submission_wait_semaphores_) {
submissions_in_flight_semaphores_.emplace_back(submission_current,
submissions_in_flight_semaphores_.emplace_back(submission_index,
semaphore);
}
current_submission_wait_semaphores_.clear();
command_buffers_submitted_.emplace_back(submission_current, command_buffer);
command_buffers_submitted_.emplace_back(submission_index, command_buffer);
command_buffers_writable_.pop_back();
// Increments the current submission number, going to the next submission.
submissions_in_flight_fences_.push_back(fence);
fences_free_.pop_back();
submission_open_ = false;
}

19
src/xenia/gpu/vulkan/vulkan_command_processor.h
View File

@@ -38,6 +38,7 @@
#include "xenia/gpu/xenos.h"
#include "xenia/kernel/kernel_state.h"
#include "xenia/ui/vulkan/linked_type_descriptor_set_allocator.h"
#include "xenia/ui/vulkan/vulkan_gpu_completion_timeline.h"
#include "xenia/ui/vulkan/vulkan_presenter.h"
#include "xenia/ui/vulkan/vulkan_provider.h"
#include "xenia/ui/vulkan/vulkan_upload_buffer_pool.h"
@@ -157,10 +158,11 @@ class VulkanCommandProcessor : public CommandProcessor {
bool submission_open() const { return submission_open_; }
uint64_t GetCurrentSubmission() const {
return submission_completed_ +
uint64_t(submissions_in_flight_fences_.size()) + 1;
return completion_timeline_.GetUpcomingSubmission();
}
uint64_t GetCompletedSubmission() const {
return completion_timeline_.GetCompletedSubmissionFromLastUpdate();
}
uint64_t GetCompletedSubmission() const { return submission_completed_; }
// Sparse binds are:
// - In a single submission, all submitted in one vkQueueBindSparse.
@@ -403,7 +405,7 @@ class VulkanCommandProcessor : public CommandProcessor {
// Rechecks submission number and reclaims per-submission resources. Pass 0 as
// the submission to await to simply check status, or pass
// GetCurrentSubmission() to wait for all queue operations to be completed.
void CheckSubmissionFenceAndDeviceLoss(uint64_t await_submission);
void CheckSubmissionCompletionAndDeviceLoss(uint64_t await_submission);
// If is_guest_command is true, a new full frame - with full cleanup of
// resources and, if needed, starting capturing - is opened if pending (as
// opposed to simply resuming after mid-frame synchronization). Returns
@@ -414,8 +416,9 @@ class VulkanCommandProcessor : public CommandProcessor {
// successfully, if it has failed, leaves it open.
bool EndSubmission(bool is_swap);
bool AwaitAllQueueOperationsCompletion() {
CheckSubmissionFenceAndDeviceLoss(GetCurrentSubmission());
return !submission_open_ && submissions_in_flight_fences_.empty();
CheckSubmissionCompletionAndDeviceLoss(GetCurrentSubmission());
return !submission_open_ &&
GetCompletedSubmission() + 1u >= GetCurrentSubmission();
}
void ClearTransientDescriptorPools();
@@ -460,16 +463,14 @@ class VulkanCommandProcessor : public CommandProcessor {
VkPipelineStageFlags guest_shader_pipeline_stages_ = 0;
VkShaderStageFlags guest_shader_vertex_stages_ = 0;
std::vector<VkFence> fences_free_;
std::vector<VkSemaphore> semaphores_free_;
ui::vulkan::VulkanGPUCompletionTimeline completion_timeline_;
bool submission_open_ = false;
uint64_t submission_completed_ = 0;
// In case vkQueueSubmit fails after something like a successful
// vkQueueBindSparse, to wait correctly on the next attempt.
std::vector<VkSemaphore> current_submission_wait_semaphores_;
std::vector<VkPipelineStageFlags> current_submission_wait_stage_masks_;
std::vector<VkFence> submissions_in_flight_fences_;
std::deque<std::pair<uint64_t, VkSemaphore>>
submissions_in_flight_semaphores_;