Merge commit 'f2fabfdf0' into canary_experimental

Sync changes from master up to before the recent GPU changes, those will be
merged later
This commit is contained in:
Herman S.
2026-02-17 15:46:27 +09:00
294 changed files with 165264 additions and 173899 deletions

View File

@@ -51,22 +51,34 @@ std::unique_ptr<VulkanDevice> VulkanDevice::CreateIfSupported(
VkPhysicalDeviceProperties properties = {};
ifn.vkGetPhysicalDeviceProperties(physical_device, &properties);
// From the VkApplicationInfo specification:
//
// "The Khronos validation layers will treat apiVersion as the highest API
// version the application targets, and will validate API usage against the
// minimum of that version and the implementation version (instance or device,
// depending on context). If an application tries to use functionality from a
// greater version than this, a validation error will be triggered."
//
// "Vulkan 1.0 implementations were required to return
// VK_ERROR_INCOMPATIBLE_DRIVER if apiVersion was larger than 1.0."
//
// Make sure that all usages of the API version in Xenia receive the highest
// minor version that Xenia has been tested on.
// Libraries such as the Vulkan Memory Allocator also may expect a minor
// version that is known to them.
const uint32_t unclamped_api_version = properties.apiVersion;
if (vulkan_instance->api_version() < VK_MAKE_API_VERSION(0, 1, 1, 0)) {
// From the VkApplicationInfo specification:
//
// "The Khronos validation layers will treat apiVersion as the highest API
// version the application targets, and will validate API usage against the
// minimum of that version and the implementation version (instance or
// device, depending on context). If an application tries to use
// functionality from a greater version than this, a validation error will
// be triggered."
//
// "Vulkan 1.0 implementations were required to return
// VK_ERROR_INCOMPATIBLE_DRIVER if apiVersion was larger than 1.0."
properties.apiVersion = VK_MAKE_API_VERSION(
0, 1, 0, VK_API_VERSION_PATCH(properties.apiVersion));
}
const uint32_t clamped_api_minor_version = std::min(
VK_MAKE_API_VERSION(VK_API_VERSION_VARIANT(unclamped_api_version),
VK_API_VERSION_MAJOR(unclamped_api_version),
VK_API_VERSION_MINOR(unclamped_api_version), 0),
vulkan_instance->api_version() >= VK_MAKE_API_VERSION(0, 1, 1, 0)
? kHighestUsedApiMinorVersion
: VK_MAKE_API_VERSION(0, 1, 0, 0));
properties.apiVersion =
VK_MAKE_API_VERSION(VK_API_VERSION_VARIANT(clamped_api_minor_version),
VK_API_VERSION_MAJOR(clamped_api_minor_version),
VK_API_VERSION_MINOR(clamped_api_minor_version),
VK_API_VERSION_PATCH(unclamped_api_version));
VkPhysicalDeviceFeatures supported_features = {};
ifn.vkGetPhysicalDeviceFeatures(physical_device, &supported_features);
@@ -488,20 +500,14 @@ std::unique_ptr<VulkanDevice> VulkanDevice::CreateIfSupported(
std::strcpy(device->properties_.deviceName, properties.deviceName);
XELOGI(
"Vulkan device '{}': API {}.{}.{}, vendor 0x{:04X}, device 0x{:04X}, "
"driver version 0x{:X}",
properties.deviceName, VK_VERSION_MAJOR(properties.apiVersion),
VK_VERSION_MINOR(properties.apiVersion),
VK_VERSION_PATCH(properties.apiVersion), properties.vendorID,
"Vulkan device '{}': API {}.{}.{} ({}.{} used), vendor 0x{:04X}, device "
"0x{:04X}, driver version 0x{:X}",
properties.deviceName, VK_VERSION_MAJOR(unclamped_api_version),
VK_VERSION_MINOR(unclamped_api_version),
VK_VERSION_PATCH(properties.apiVersion),
VK_VERSION_MAJOR(properties.apiVersion),
VK_VERSION_MINOR(properties.apiVersion), properties.vendorID,
properties.deviceID, properties.driverVersion);
if (unclamped_api_version != properties.apiVersion) {
XELOGI(
"Device supports Vulkan API {}.{}.{}, but the used version is limited "
"by the instance",
VK_VERSION_MAJOR(unclamped_api_version),
VK_VERSION_MINOR(unclamped_api_version),
VK_VERSION_PATCH(unclamped_api_version));
}
XELOGI("Enabled Vulkan device extensions:");
for (uint32_t enabled_extension_index = 0;

View File

@@ -10,6 +10,7 @@
#ifndef XENIA_UI_VULKAN_VULKAN_DEVICE_H_
#define XENIA_UI_VULKAN_VULKAN_DEVICE_H_
#include <atomic>
#include <memory>
#include <mutex>
#include <vector>
@@ -269,6 +270,25 @@ class VulkanDevice {
const MemoryTypes& memory_types() const { return memory_types_; }
// Returns whether a device loss has been observed for the first time, so, for
// instance, logging of the device loss can be limited only to the location
// where it was first caught.
bool SetLost() noexcept {
return !lost_.exchange(true, std::memory_order_acq_rel);
}
bool IsLost() const noexcept { return lost_.load(std::memory_order_acquire); }
VkResult SubmitAndUpdateLost(const VkQueue queue, const uint32_t submit_count,
const VkSubmitInfo* const submits,
const VkFence fence) {
const VkResult submit_result =
functions().vkQueueSubmit(queue, submit_count, submits, fence);
if (submit_result == VK_ERROR_DEVICE_LOST) {
SetLost();
}
return submit_result;
}
private:
explicit VulkanDevice(const VulkanInstance* vulkan_instance,
VkPhysicalDevice physical_device);
@@ -288,6 +308,8 @@ class VulkanDevice {
uint32_t queue_family_sparse_binding_ = UINT32_MAX;
MemoryTypes memory_types_;
std::atomic<bool> lost_{false};
};
} // namespace vulkan

View File

@@ -0,0 +1,182 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2025 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include "xenia/ui/vulkan/vulkan_gpu_completion_timeline.h"
#include <algorithm>
#include <iterator>
#include "xenia/base/assert.h"
#include "xenia/base/logging.h"
namespace xe {
namespace ui {
namespace vulkan {
VulkanGPUCompletionTimeline::~VulkanGPUCompletionTimeline() {
#ifndef NDEBUG
assert_zero(fences_acquired_);
#endif
if (!pending_submission_fences_.empty()) {
if (vulkan_device_->functions().vkWaitForFences(
vulkan_device_->device(), 1,
&pending_submission_fences_.back().second, VK_TRUE,
UINT64_MAX) == VK_ERROR_DEVICE_LOST) {
vulkan_device_->SetLost();
}
while (!pending_submission_fences_.empty()) {
vulkan_device_->functions().vkDestroyFence(
vulkan_device_->device(), pending_submission_fences_.back().second,
nullptr);
pending_submission_fences_.pop_back();
}
}
while (!free_fences_.empty()) {
vulkan_device_->functions().vkDestroyFence(vulkan_device_->device(),
free_fences_.back(), nullptr);
free_fences_.pop_back();
}
}
std::optional<VulkanGPUCompletionTimeline::FenceAcquisition>
VulkanGPUCompletionTimeline::AcquireFenceForSubmission(
VkResult* const result_out_opt) {
// Reuse fences if completion was not awaited or updated explicitly.
UpdateAndGetCompletedSubmission();
VkFence fence = VK_NULL_HANDLE;
if (!free_fences_.empty()) {
const VkResult fence_reset_result =
vulkan_device_->functions().vkResetFences(vulkan_device_->device(), 1,
&free_fences_.back());
if (fence_reset_result != VK_SUCCESS) {
XELOGE("Failed to reset a Vulkan fence: {}",
vk::to_string(vk::Result(fence_reset_result)));
} else {
fence = free_fences_.back();
free_fences_.pop_back();
}
}
if (fence == VK_NULL_HANDLE) {
const VkFenceCreateInfo fence_create_info = {
VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
const VkResult fence_create_result =
vulkan_device_->functions().vkCreateFence(
vulkan_device_->device(), &fence_create_info, nullptr, &fence);
if (fence_create_result != VK_SUCCESS) {
XELOGE("Failed to create a Vulkan fence: {}",
vk::to_string(vk::Result(fence_create_result)));
if (result_out_opt != nullptr) {
*result_out_opt = fence_create_result;
}
return std::nullopt;
}
}
#ifndef NDEBUG
++fences_acquired_;
#endif
if (result_out_opt != nullptr) {
*result_out_opt = VK_SUCCESS;
}
return FenceAcquisition(this, fence);
}
VkResult VulkanGPUCompletionTimeline::AcquireFenceAndSubmit(
const uint32_t queue_family_index, const uint32_t queue_index,
const uint32_t submit_count, const VkSubmitInfo* const submits) {
VkResult fence_acquire_result;
std::optional<FenceAcquisition> fence_acquisition =
AcquireFenceForSubmission(&fence_acquire_result);
if (!fence_acquisition.has_value()) {
return fence_acquire_result;
}
VkResult submit_result;
{
const VulkanDevice::Queue::Acquisition queue_acquisition =
vulkan_device_->AcquireQueue(queue_family_index, queue_index);
submit_result = vulkan_device_->SubmitAndUpdateLost(
queue_acquisition.queue(), submit_count, submits,
fence_acquisition->GetFenceForSubmitting());
}
if (submit_result != VK_SUCCESS) {
fence_acquisition->SetSubmissionFailedOrAborted();
}
return submit_result;
}
void VulkanGPUCompletionTimeline::UpdateCompletedSubmission() {
while (!pending_submission_fences_.empty()) {
const VkResult fence_status = vulkan_device_->functions().vkGetFenceStatus(
vulkan_device_->device(), pending_submission_fences_.front().second);
if (fence_status != VK_SUCCESS) {
// Not ready, or an error.
if (fence_status == VK_ERROR_DEVICE_LOST) {
vulkan_device_->SetLost();
}
break;
}
SetCompletedSubmission(pending_submission_fences_.front().first);
free_fences_.push_back(pending_submission_fences_.front().second);
pending_submission_fences_.pop_front();
}
}
void VulkanGPUCompletionTimeline::AwaitSubmissionImpl(
const uint64_t awaited_submission) {
// According to the Vulkan 1.4.335 specification:
// "The first synchronization scope includes every batch submitted in the same
// queue submission command. Fence signal operations that are defined by
// vkQueueSubmit or vkQueueSubmit2 additionally include in the first
// synchronization scope all commands that occur earlier in submission order.
// Fence signal operations that are defined by vkQueueSubmit or vkQueueSubmit2
// or vkQueueBindSparse additionally include in the first synchronization
// scope any semaphore and fence signal operations that occur earlier in
// signal operation order."
auto submission_end_iterator = pending_submission_fences_.cbegin();
while (submission_end_iterator != pending_submission_fences_.cend() &&
submission_end_iterator->first <= awaited_submission) {
submission_end_iterator = std::next(submission_end_iterator);
}
if (submission_end_iterator != pending_submission_fences_.cbegin()) {
const VkResult fence_wait_result =
vulkan_device_->functions().vkWaitForFences(
vulkan_device_->device(), 1,
&std::prev(submission_end_iterator)->second, VK_TRUE, UINT64_MAX);
if (fence_wait_result != VK_SUCCESS) {
XELOGE("Failed to wait for a Vulkan fence: {}",
vk::to_string(vk::Result(fence_wait_result)));
if (fence_wait_result == VK_ERROR_DEVICE_LOST) {
vulkan_device_->SetLost();
}
return;
}
for (auto free_fence_iterator = pending_submission_fences_.cbegin();
free_fence_iterator != submission_end_iterator;
free_fence_iterator = std::next(free_fence_iterator)) {
free_fences_.push_back(free_fence_iterator->second);
}
pending_submission_fences_.erase(pending_submission_fences_.cbegin(),
submission_end_iterator);
}
if (GetCompletedSubmissionFromLastUpdate() < awaited_submission) {
SetCompletedSubmission(awaited_submission);
}
}
} // namespace vulkan
} // namespace ui
} // namespace xe

View File

@@ -0,0 +1,155 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2025 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#ifndef XENIA_UI_VULKAN_VULKAN_GPU_COMPLETION_TIMELINE_H_
#define XENIA_UI_VULKAN_VULKAN_GPU_COMPLETION_TIMELINE_H_
#include <deque>
#include <optional>
#include <utility>
#include <vector>
#include "xenia/base/assert.h"
#include "xenia/ui/gpu_completion_timeline.h"
#include "xenia/ui/vulkan/vulkan_device.h"
namespace xe {
namespace ui {
namespace vulkan {
class VulkanGPUCompletionTimeline : public GPUCompletionTimeline {
public:
explicit VulkanGPUCompletionTimeline(VulkanDevice* const vulkan_device)
: vulkan_device_(vulkan_device) {}
VulkanGPUCompletionTimeline(const VulkanGPUCompletionTimeline&) = delete;
VulkanGPUCompletionTimeline& operator=(const VulkanGPUCompletionTimeline&) =
delete;
VulkanGPUCompletionTimeline(VulkanGPUCompletionTimeline&&) = delete;
VulkanGPUCompletionTimeline& operator=(VulkanGPUCompletionTimeline&&) =
delete;
~VulkanGPUCompletionTimeline();
class FenceAcquisition {
public:
explicit FenceAcquisition(
VulkanGPUCompletionTimeline* const completion_timeline,
const VkFence fence)
: completion_timeline_(completion_timeline), fence_(fence) {
assert_not_null(completion_timeline);
assert_true(fence != VK_NULL_HANDLE);
}
FenceAcquisition(const FenceAcquisition&) = delete;
FenceAcquisition& operator=(const FenceAcquisition&) = delete;
FenceAcquisition(FenceAcquisition&& other)
: completion_timeline_(other.completion_timeline_),
fence_(other.fence_),
submission_successful_(other.submission_successful_) {
other.completion_timeline_ = nullptr;
other.fence_ = VK_NULL_HANDLE;
other.submission_successful_.reset();
}
FenceAcquisition& operator==(FenceAcquisition&& other) {
if (this == &other) {
return *this;
}
completion_timeline_ = other.completion_timeline_;
other.completion_timeline_ = nullptr;
fence_ = other.fence_;
other.fence_ = VK_NULL_HANDLE;
submission_successful_ = other.submission_successful_;
other.submission_successful_.reset();
return *this;
}
~FenceAcquisition() {
if (completion_timeline_ && fence_) {
#ifndef NDEBUG
assert_not_zero(completion_timeline_->fences_acquired_);
--completion_timeline_->fences_acquired_;
#endif
if (submission_successful_.value_or(false)) {
assert_true(
completion_timeline_->pending_submission_fences_.empty() ||
completion_timeline_->pending_submission_fences_.front().first <
completion_timeline_->GetUpcomingSubmission());
completion_timeline_->pending_submission_fences_.emplace_back(
completion_timeline_->GetUpcomingSubmission(), fence_);
completion_timeline_->IncrementUpcomingSubmission();
} else {
completion_timeline_->free_fences_.push_back(fence_);
}
}
}
VkFence GetFenceForSubmitting() {
// Don't mark the fence as used in a submission if already tried to
// submit, but failed.
assert_true(!submission_successful_.has_value() ||
*submission_successful_);
submission_successful_ = true;
return fence_;
}
void SetSubmissionFailedOrAborted() { submission_successful_ = false; }
private:
VulkanGPUCompletionTimeline* completion_timeline_ = nullptr;
VkFence fence_ = VK_NULL_HANDLE;
std::optional<bool> submission_successful_;
};
// If the submission has succeeded (`GetFenceForSubmitting` was called, but
// `SetSubmissionFailedOrAborted` was not), will advance to the next
// submission once the acquisition is released.
//
// It's possible to acquire a fence not right before submitting, but also well
// in advance, such as before recording the command buffer, for instance, to
// skip recording it if fence acquisition has failed.
//
// Acquiring a fence also updates the completed submission in order to reuse
// fences if this completion timeline is used without regular checks or waits
// (for instance, if it's supplementary to another completion timeline, and
// awaited only before destroying something).
[[nodiscard]] std::optional<FenceAcquisition> AcquireFenceForSubmission(
VkResult* result_out_opt = nullptr);
VkResult AcquireFenceAndSubmit(uint32_t queue_family_index,
uint32_t queue_index, uint32_t submit_count,
const VkSubmitInfo* submits);
void UpdateCompletedSubmission() override;
protected:
void AwaitSubmissionImpl(uint64_t awaited_submission) override;
private:
VulkanDevice* const vulkan_device_;
std::vector<VkFence> free_fences_;
// <Submission index, fence>, in submission index order.
std::deque<std::pair<uint64_t, VkFence>> pending_submission_fences_;
#ifndef NDEBUG
size_t fences_acquired_ = 0;
#endif
};
} // namespace vulkan
} // namespace ui
} // namespace xe
#endif // XENIA_UI_VULKAN_VULKAN_GPU_COMPLETION_TIMELINE_H_

View File

@@ -153,8 +153,8 @@ VulkanPresenter::~VulkanPresenter() {
// (paint submission completion already awaited).
// From most likely the latest to most likely the earliest to be signaled, so
// just one sleep will likely be needed.
ui_submission_tracker_.Shutdown();
guest_output_image_refresher_submission_tracker_.Shutdown();
ui_completion_timeline_.AwaitAllSubmissions();
guest_output_image_refresher_completion_timeline_.AwaitAllSubmissions();
const VulkanDevice::Functions& dfn = vulkan_device_->functions();
const VkDevice device = vulkan_device_->device();
@@ -377,51 +377,24 @@ bool VulkanPresenter::CaptureGuestOutput(RawImage& image_out) {
return false;
}
VkSubmitInfo submit_info = {};
submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submit_info.commandBufferCount = 1;
submit_info.pCommandBuffers = &command_buffer;
VulkanSubmissionTracker submission_tracker(vulkan_device_);
{
VulkanSubmissionTracker::FenceAcquisition fence_acqusition(
submission_tracker.AcquireFenceToAdvanceSubmission());
if (!fence_acqusition.fence()) {
XELOGE(
"VulkanPresenter: Failed to acquire a fence for guest output "
"capturing");
fence_acqusition.SubmissionFailedOrDropped();
dfn.vkDestroyCommandPool(device, command_pool, nullptr);
dfn.vkDestroyBuffer(device, buffer, nullptr);
dfn.vkFreeMemory(device, buffer_memory, nullptr);
return false;
}
VkResult submit_result;
{
const VulkanDevice::Queue::Acquisition queue_acquisition =
vulkan_device_->AcquireQueue(
vulkan_device_->queue_family_graphics_compute(), 0);
submit_result =
dfn.vkQueueSubmit(queue_acquisition.queue(), 1, &submit_info,
fence_acqusition.fence());
}
VulkanGPUCompletionTimeline completion_timeline(vulkan_device_);
VkSubmitInfo submit_info = {VK_STRUCTURE_TYPE_SUBMIT_INFO};
submit_info.commandBufferCount = 1;
submit_info.pCommandBuffers = &command_buffer;
const VkResult submit_result = completion_timeline.AcquireFenceAndSubmit(
vulkan_device_->queue_family_graphics_compute(), 0, 1, &submit_info);
if (submit_result != VK_SUCCESS) {
XELOGE(
"VulkanPresenter: Failed to submit the guest output capturing "
"command buffer");
fence_acqusition.SubmissionFailedOrDropped();
"command buffer: {}",
vk::to_string(vk::Result(submit_result)));
dfn.vkDestroyCommandPool(device, command_pool, nullptr);
dfn.vkDestroyBuffer(device, buffer, nullptr);
dfn.vkFreeMemory(device, buffer_memory, nullptr);
return false;
}
}
if (!submission_tracker.AwaitAllSubmissionsCompletion()) {
XELOGE(
"VulkanPresenter: Failed to await the guest output capturing fence");
dfn.vkDestroyCommandPool(device, command_pool, nullptr);
dfn.vkDestroyBuffer(device, buffer, nullptr);
dfn.vkFreeMemory(device, buffer_memory, nullptr);
return false;
// Destroying the completion timeline causes the submission to be awaited.
}
dfn.vkDestroyCommandPool(device, command_pool, nullptr);
@@ -474,8 +447,8 @@ VkCommandBuffer VulkanPresenter::AcquireUISetupCommandBufferFromUIThread() {
// Try to reuse an existing command buffer.
if (!paint_context_.ui_setup_command_buffers.empty()) {
uint64_t submission_index_completed =
ui_submission_tracker_.UpdateAndGetCompletedSubmission();
const uint64_t submission_index_completed =
ui_completion_timeline_.UpdateAndGetCompletedSubmission();
for (size_t i = 0; i < paint_context_.ui_setup_command_buffers.size();
++i) {
PaintContext::UISetupCommandBuffer& ui_setup_command_buffer =
@@ -498,7 +471,7 @@ VkCommandBuffer VulkanPresenter::AcquireUISetupCommandBufferFromUIThread() {
}
paint_context_.ui_setup_command_buffer_current_index = i;
ui_setup_command_buffer.last_usage_submission_index =
ui_submission_tracker_.GetCurrentSubmission();
ui_completion_timeline_.GetUpcomingSubmission();
return ui_setup_command_buffer.command_buffer;
}
}
@@ -541,7 +514,7 @@ VkCommandBuffer VulkanPresenter::AcquireUISetupCommandBufferFromUIThread() {
paint_context_.ui_setup_command_buffers.size();
paint_context_.ui_setup_command_buffers.emplace_back(
new_command_pool, new_command_buffer,
ui_submission_tracker_.GetCurrentSubmission());
ui_completion_timeline_.GetUpcomingSubmission());
return new_command_buffer;
}
@@ -872,8 +845,9 @@ bool VulkanPresenter::RefreshGuestOutputImpl(
if (image_instance.image &&
(image_instance.image->extent().width != frontbuffer_width ||
image_instance.image->extent().height != frontbuffer_height)) {
guest_output_image_refresher_submission_tracker_.AwaitSubmissionCompletion(
image_instance.last_refresher_submission);
guest_output_image_refresher_completion_timeline_
.AwaitSubmissionAndUpdateCompleted(
image_instance.last_refresher_submission);
image_instance.image.reset();
}
if (!image_instance.image) {
@@ -899,26 +873,21 @@ bool VulkanPresenter::RefreshGuestOutputImpl(
// signal and wait slightly longer, for nothing important, while shutting down
// than to destroy the image while it's still in use.
image_instance.last_refresher_submission =
guest_output_image_refresher_submission_tracker_.GetCurrentSubmission();
guest_output_image_refresher_completion_timeline_.GetUpcomingSubmission();
// No need to make the refresher signal the fence by itself - signal it here
// instead to have more control:
// "Fence signal operations that are defined by vkQueueSubmit additionally
// include in the first synchronization scope all commands that occur earlier
// in submission order."
const VulkanDevice::Functions& dfn = vulkan_device_->functions();
{
VulkanSubmissionTracker::FenceAcquisition fence_acqusition(
guest_output_image_refresher_submission_tracker_
.AcquireFenceToAdvanceSubmission());
const VulkanDevice::Queue::Acquisition queue_acquisition =
vulkan_device_->AcquireQueue(
vulkan_device_->queue_family_graphics_compute(), 0);
if (dfn.vkQueueSubmit(queue_acquisition.queue(), 0, nullptr,
fence_acqusition.fence()) != VK_SUCCESS) {
fence_acqusition.SubmissionSucceededSignalFailed();
}
const VkResult submit_result =
guest_output_image_refresher_completion_timeline_.AcquireFenceAndSubmit(
vulkan_device_->queue_family_graphics_compute(), 0, 0, nullptr);
if (submit_result != VK_SUCCESS) {
XELOGE(
"VulkanPresenter: Failed to submit the guest output image refresh "
"fence signal: {}",
vk::to_string(vk::Result(submit_result)));
}
return refresher_succeeded;
}
@@ -1272,7 +1241,7 @@ VkSwapchainKHR VulkanPresenter::PaintContext::CreateSwapchainForVulkanSurface(
VkSwapchainKHR VulkanPresenter::PaintContext::PrepareForSwapchainRetirement() {
if (swapchain != VK_NULL_HANDLE) {
submission_tracker.AwaitAllSubmissionsCompletion();
completion_timeline.AwaitAllSubmissions();
}
const VulkanDevice::Functions& dfn = vulkan_device->functions();
const VkDevice device = vulkan_device->device();
@@ -1380,13 +1349,11 @@ Presenter::PaintResult VulkanPresenter::PaintAndPresentImpl(
bool execute_ui_drawers) {
// Begin the submission in place of the one not currently potentially used on
// the GPU.
uint64_t current_paint_submission_index =
paint_context_.submission_tracker.GetCurrentSubmission();
const uint64_t current_paint_submission_index =
paint_context_.completion_timeline.GetUpcomingSubmission();
uint64_t paint_submission_count = uint64_t(paint_context_.submissions.size());
if (current_paint_submission_index >= paint_submission_count) {
paint_context_.submission_tracker.AwaitSubmissionCompletion(
current_paint_submission_index - paint_submission_count);
}
paint_context_.completion_timeline
.AwaitMaxSubmissionsPendingAndUpdateCompleted(paint_submission_count);
const PaintContext::Submission& paint_submission =
*paint_context_.submissions[current_paint_submission_index %
paint_submission_count];
@@ -1550,7 +1517,7 @@ Presenter::PaintResult VulkanPresenter::PaintAndPresentImpl(
}
// Await the completion of the usage of the old guest output image and
// its descriptors.
paint_context_.submission_tracker.AwaitSubmissionCompletion(
paint_context_.completion_timeline.AwaitSubmissionAndUpdateCompleted(
paint_context_
.guest_output_image_paint_refs
[guest_output_image_paint_ref_new_index]
@@ -1612,9 +1579,10 @@ Presenter::PaintResult VulkanPresenter::PaintAndPresentImpl(
// Need to replace immediately as a new image with the requested
// size is needed.
if (intermediate_image_ptr_ref) {
paint_context_.submission_tracker.AwaitSubmissionCompletion(
paint_context_
.guest_output_intermediate_image_last_submission);
paint_context_.completion_timeline
.AwaitSubmissionAndUpdateCompleted(
paint_context_
.guest_output_intermediate_image_last_submission);
intermediate_image_ptr_ref.reset();
util::DestroyAndNullHandle(
dfn.vkDestroyFramebuffer, device,
@@ -1691,7 +1659,7 @@ Presenter::PaintResult VulkanPresenter::PaintAndPresentImpl(
} else {
// Was previously needed, but not anymore - destroy when possible.
if (intermediate_image_ptr_ref &&
paint_context_.submission_tracker
paint_context_.completion_timeline
.UpdateAndGetCompletedSubmission() >=
paint_context_
.guest_output_intermediate_image_last_submission) {
@@ -1726,7 +1694,7 @@ Presenter::PaintResult VulkanPresenter::PaintAndPresentImpl(
if (swapchain_effect_pipeline.swapchain_pipeline != VK_NULL_HANDLE &&
swapchain_effect_pipeline.swapchain_format !=
paint_context_.swapchain_render_pass_format) {
paint_context_.submission_tracker.AwaitSubmissionCompletion(
paint_context_.completion_timeline.AwaitSubmissionAndUpdateCompleted(
paint_context_.guest_output_image_paint_last_submission);
util::DestroyAndNullHandle(
dfn.vkDestroyPipeline, device,
@@ -1954,10 +1922,10 @@ Presenter::PaintResult VulkanPresenter::PaintAndPresentImpl(
// Release main target guest output image references that aren't needed
// anymore (this is done after various potential guest-output-related main
// target submission tracker waits so the completed submission value is the
// target completion timeline waits so the completed submission index is the
// most actual).
uint64_t completed_paint_submission =
paint_context_.submission_tracker.UpdateAndGetCompletedSubmission();
paint_context_.completion_timeline.UpdateAndGetCompletedSubmission();
for (std::pair<uint64_t, std::shared_ptr<GuestOutputImage>>&
guest_output_image_paint_ref :
paint_context_.guest_output_image_paint_refs) {
@@ -1996,8 +1964,8 @@ Presenter::PaintResult VulkanPresenter::PaintAndPresentImpl(
VulkanUIDrawContext ui_draw_context(
*this, paint_context_.swapchain_extent.width,
paint_context_.swapchain_extent.height, draw_command_buffer,
ui_submission_tracker_.GetCurrentSubmission(),
ui_submission_tracker_.UpdateAndGetCompletedSubmission(),
ui_completion_timeline_.GetUpcomingSubmission(),
ui_completion_timeline_.UpdateAndGetCompletedSubmission(),
paint_context_.swapchain_render_pass,
paint_context_.swapchain_render_pass_format);
ExecuteUIDrawersFromUIThread(ui_draw_context);
@@ -2040,48 +2008,35 @@ Presenter::PaintResult VulkanPresenter::PaintAndPresentImpl(
submit_info.pCommandBuffers = command_buffers;
submit_info.signalSemaphoreCount = 1;
submit_info.pSignalSemaphores = &present_semaphore;
{
VulkanSubmissionTracker::FenceAcquisition fence_acqusition(
paint_context_.submission_tracker.AcquireFenceToAdvanceSubmission());
// Also update the submission tracker giving submission indices to UI draw
// callbacks if submission is successful.
VulkanSubmissionTracker::FenceAcquisition ui_fence_acquisition;
if (execute_ui_drawers) {
ui_fence_acquisition =
ui_submission_tracker_.AcquireFenceToAdvanceSubmission();
const VkResult submit_result =
paint_context_.completion_timeline.AcquireFenceAndSubmit(
vulkan_device_->queue_family_graphics_compute(), 0, 1, &submit_info);
if (submit_result != VK_SUCCESS) {
XELOGE(
"VulkanPresenter: Failed to submit the presentation command buffer: {}",
vk::to_string(vk::Result(submit_result)));
if (ui_setup_command_buffer_index != SIZE_MAX) {
// If failed to submit, make the UI setup command buffer available for
// immediate reuse, as the completed submission index won't be updated to
// the current index, and failing submissions with setup command buffer
// over and over will result in never reusing the setup command buffers.
paint_context_.ui_setup_command_buffers[ui_setup_command_buffer_index]
.last_usage_submission_index = 0;
}
VkResult submit_result;
{
const VulkanDevice::Queue::Acquisition queue_acquisition =
vulkan_device_->AcquireQueue(
vulkan_device_->queue_family_graphics_compute(), 0);
submit_result = dfn.vkQueueSubmit(queue_acquisition.queue(), 1,
&submit_info, fence_acqusition.fence());
if (ui_fence_acquisition.fence() != VK_NULL_HANDLE &&
submit_result == VK_SUCCESS) {
if (dfn.vkQueueSubmit(queue_acquisition.queue(), 0, nullptr,
ui_fence_acquisition.fence()) != VK_SUCCESS) {
ui_fence_acquisition.SubmissionSucceededSignalFailed();
}
}
}
if (submit_result != VK_SUCCESS) {
XELOGE("VulkanPresenter: Failed to submit command buffers");
fence_acqusition.SubmissionFailedOrDropped();
ui_fence_acquisition.SubmissionFailedOrDropped();
if (ui_setup_command_buffer_index != SIZE_MAX) {
// If failed to submit, make the UI setup command buffer available for
// immediate reuse, as the completed submission index won't be updated
// to the current index, and failing submissions with setup command
// buffer over and over will result in never reusing the setup command
// buffers.
paint_context_.ui_setup_command_buffers[ui_setup_command_buffer_index]
.last_usage_submission_index = 0;
}
// The image is in an acquired state - but now, it will be in it forever.
// To avoid that, recreate the swapchain - don't return just
// kNotPresented.
return PaintResult::kNotPresentedConnectionOutdated;
// The image is in an acquired state - but now, it will be in it forever.
// To avoid that, recreate the swapchain - don't return just kNotPresented.
return PaintResult::kNotPresentedConnectionOutdated;
}
if (execute_ui_drawers) {
// Also update the completion timeline providing submission indices to UI
// draw callbacks if submission is successful.
const VkResult ui_signal_submit_result =
ui_completion_timeline_.AcquireFenceAndSubmit(
vulkan_device_->queue_family_graphics_compute(), 0, 0, nullptr);
if (ui_signal_submit_result != VK_SUCCESS) {
XELOGE(
"VulkanPresenter: Failed to submit the UI drawing fence signal: {}",
vk::to_string(vk::Result(ui_signal_submit_result)));
}
}

View File

@@ -23,8 +23,8 @@
#include "xenia/ui/surface.h"
#include "xenia/ui/vulkan/ui_samplers.h"
#include "xenia/ui/vulkan/vulkan_device.h"
#include "xenia/ui/vulkan/vulkan_gpu_completion_timeline.h"
#include "xenia/ui/vulkan/vulkan_instance.h"
#include "xenia/ui/vulkan/vulkan_submission_tracker.h"
namespace xe {
namespace ui {
@@ -124,8 +124,8 @@ class VulkanPresenter final : public Presenter {
};
static std::unique_ptr<VulkanPresenter> Create(
HostGpuLossCallback host_gpu_loss_callback,
const VulkanDevice* vulkan_device, const UISamplers* ui_samplers) {
HostGpuLossCallback host_gpu_loss_callback, VulkanDevice* vulkan_device,
const UISamplers* ui_samplers) {
auto presenter = std::unique_ptr<VulkanPresenter>(new VulkanPresenter(
host_gpu_loss_callback, vulkan_device, ui_samplers));
if (!presenter->InitializeSurfaceIndependent()) {
@@ -136,7 +136,7 @@ class VulkanPresenter final : public Presenter {
~VulkanPresenter();
const VulkanDevice* vulkan_device() const { return vulkan_device_; }
VulkanDevice* vulkan_device() const { return vulkan_device_; }
static Surface::TypeFlags GetSurfaceTypesSupportedByInstance(
const VulkanInstance::Extensions& instance_extensions);
@@ -145,7 +145,7 @@ class VulkanPresenter final : public Presenter {
bool CaptureGuestOutput(RawImage& image_out) override;
void AwaitUISubmissionCompletionFromUIThread(uint64_t submission_index) {
ui_submission_tracker_.AwaitSubmissionCompletion(submission_index);
ui_completion_timeline_.AwaitSubmissionAndUpdateCompleted(submission_index);
}
VkCommandBuffer AcquireUISetupCommandBufferFromUIThread();
@@ -361,8 +361,8 @@ class VulkanPresenter final : public Presenter {
VkFramebuffer framebuffer;
};
explicit PaintContext(const VulkanDevice* const vulkan_device)
: vulkan_device(vulkan_device), submission_tracker(vulkan_device) {}
explicit PaintContext(VulkanDevice* const vulkan_device)
: vulkan_device(vulkan_device), completion_timeline(vulkan_device) {}
PaintContext(const PaintContext& paint_context) = delete;
PaintContext& operator=(const PaintContext& paint_context) = delete;
@@ -386,11 +386,11 @@ class VulkanPresenter final : public Presenter {
// Connection-indepedent.
const VulkanDevice* vulkan_device;
VulkanDevice* vulkan_device;
std::array<std::unique_ptr<PaintContext::Submission>, kSubmissionCount>
submissions;
VulkanSubmissionTracker submission_tracker;
VulkanGPUCompletionTimeline completion_timeline;
std::array<GuestOutputPaintPipeline, size_t(GuestOutputPaintEffect::kCount)>
guest_output_paint_pipelines;
@@ -445,13 +445,13 @@ class VulkanPresenter final : public Presenter {
};
explicit VulkanPresenter(HostGpuLossCallback host_gpu_loss_callback,
const VulkanDevice* vulkan_device,
VulkanDevice* vulkan_device,
const UISamplers* ui_samplers)
: Presenter(host_gpu_loss_callback),
vulkan_device_(vulkan_device),
ui_samplers_(ui_samplers),
guest_output_image_refresher_submission_tracker_(vulkan_device),
ui_submission_tracker_(vulkan_device),
guest_output_image_refresher_completion_timeline_(vulkan_device),
ui_completion_timeline_(vulkan_device),
paint_context_(vulkan_device) {
assert_not_null(vulkan_device);
assert_not_null(ui_samplers);
@@ -462,7 +462,7 @@ class VulkanPresenter final : public Presenter {
[[nodiscard]] VkPipeline CreateGuestOutputPaintPipeline(
GuestOutputPaintEffect effect, VkRenderPass render_pass);
const VulkanDevice* vulkan_device_;
VulkanDevice* vulkan_device_;
const UISamplers* ui_samplers_;
// Static objects for guest output presentation, used only when painting the
@@ -489,11 +489,11 @@ class VulkanPresenter final : public Presenter {
uint64_t guest_output_image_next_version_ = 0;
std::array<GuestOutputImageInstance, kGuestOutputMailboxSize>
guest_output_images_;
VulkanSubmissionTracker guest_output_image_refresher_submission_tracker_;
VulkanGPUCompletionTimeline guest_output_image_refresher_completion_timeline_;
// UI submission tracker with the submission index that can be given to UI
// drawers (accessible from the UI thread only, at any time).
VulkanSubmissionTracker ui_submission_tracker_;
// UI submission completion timeline with the submission index that can be
// given to UI drawers (accessible from the UI thread only, at any time).
VulkanGPUCompletionTimeline ui_completion_timeline_;
// Accessible only by painting and by surface connection lifetime management
// (ConnectOrReconnectPaintingToSurfaceFromUIThread,

View File

@@ -1,174 +0,0 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2021 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include "xenia/ui/vulkan/vulkan_submission_tracker.h"
#include <cstdint>
#include "xenia/base/assert.h"
#include "xenia/ui/vulkan/vulkan_util.h"
namespace xe {
namespace ui {
namespace vulkan {
VulkanSubmissionTracker::FenceAcquisition::~FenceAcquisition() {
if (!submission_tracker_) {
// Dropped submission or left after std::move.
return;
}
assert_true(submission_tracker_->fence_acquired_ == fence_);
if (fence_ != VK_NULL_HANDLE) {
if (signal_failed_) {
// Left in the unsignaled state.
submission_tracker_->fences_reclaimed_.push_back(fence_);
} else {
// Left in the pending state.
submission_tracker_->fences_pending_.emplace_back(
submission_tracker_->submission_current_, fence_);
}
submission_tracker_->fence_acquired_ = VK_NULL_HANDLE;
}
++submission_tracker_->submission_current_;
}
void VulkanSubmissionTracker::Shutdown() {
AwaitAllSubmissionsCompletion();
const VulkanDevice::Functions& dfn = vulkan_device_->functions();
const VkDevice device = vulkan_device_->device();
for (VkFence fence : fences_reclaimed_) {
dfn.vkDestroyFence(device, fence, nullptr);
}
fences_reclaimed_.clear();
for (const std::pair<uint64_t, VkFence>& fence_pair : fences_pending_) {
dfn.vkDestroyFence(device, fence_pair.second, nullptr);
}
fences_pending_.clear();
assert_true(fence_acquired_ == VK_NULL_HANDLE);
util::DestroyAndNullHandle(dfn.vkDestroyFence, device, fence_acquired_);
}
void VulkanSubmissionTracker::FenceAcquisition::SubmissionFailedOrDropped() {
if (!submission_tracker_) {
return;
}
assert_true(submission_tracker_->fence_acquired_ == fence_);
if (fence_ != VK_NULL_HANDLE) {
submission_tracker_->fences_reclaimed_.push_back(fence_);
}
submission_tracker_->fence_acquired_ = VK_NULL_HANDLE;
fence_ = VK_NULL_HANDLE;
// No submission acquisition from now on, don't increment the current
// submission index as well.
submission_tracker_ = VK_NULL_HANDLE;
}
uint64_t VulkanSubmissionTracker::UpdateAndGetCompletedSubmission() {
if (!fences_pending_.empty()) {
const VulkanDevice::Functions& dfn = vulkan_device_->functions();
const VkDevice device = vulkan_device_->device();
while (!fences_pending_.empty()) {
const std::pair<uint64_t, VkFence>& pending_pair =
fences_pending_.front();
assert_true(pending_pair.first > submission_completed_on_gpu_);
if (dfn.vkGetFenceStatus(device, pending_pair.second) != VK_SUCCESS) {
break;
}
fences_reclaimed_.push_back(pending_pair.second);
submission_completed_on_gpu_ = pending_pair.first;
fences_pending_.pop_front();
}
}
return submission_completed_on_gpu_;
}
bool VulkanSubmissionTracker::AwaitSubmissionCompletion(
uint64_t submission_index) {
// The tracker itself can't give a submission index for a submission that
// hasn't even started being recorded yet, the client has provided a
// completely invalid value or has done overly optimistic math if such an
// index has been obtained somehow.
assert_true(submission_index <= submission_current_);
// Waiting for the current submission is fine if there was a failure or a
// refusal to submit, and the submission index wasn't incremented, but still
// need to release objects referenced in the dropped submission (while
// shutting down, for instance - in this case, waiting for the last successful
// submission, which could have also referenced the objects from the new
// submission - we can't know since the client has already overwritten its
// last usage index, would correctly ensure that GPU usage of the objects is
// not pending). Waiting for successful submissions, but failed signals, will
// result in a true race condition, however, but waiting for the closest
// successful signal is the best approximation - also retrying to signal in
// this case.
// Go from the most recent to wait only for one fence, which includes all the
// preceding ones.
// "Fence signal operations that are defined by vkQueueSubmit additionally
// include in the first synchronization scope all commands that occur earlier
// in submission order."
size_t reclaim_end = fences_pending_.size();
if (reclaim_end) {
const VulkanDevice::Functions& dfn = vulkan_device_->functions();
const VkDevice device = vulkan_device_->device();
while (reclaim_end) {
const std::pair<uint64_t, VkFence>& pending_pair =
fences_pending_[reclaim_end - 1];
assert_true(pending_pair.first > submission_completed_on_gpu_);
if (pending_pair.first <= submission_index) {
// Wait if requested.
if (dfn.vkWaitForFences(device, 1, &pending_pair.second, VK_TRUE,
UINT64_MAX) == VK_SUCCESS) {
break;
}
}
// Just refresh the completed submission.
if (dfn.vkGetFenceStatus(device, pending_pair.second) == VK_SUCCESS) {
break;
}
--reclaim_end;
}
if (reclaim_end) {
submission_completed_on_gpu_ = fences_pending_[reclaim_end - 1].first;
for (; reclaim_end; --reclaim_end) {
fences_reclaimed_.push_back(fences_pending_.front().second);
fences_pending_.pop_front();
}
}
}
return submission_completed_on_gpu_ == submission_index;
}
VulkanSubmissionTracker::FenceAcquisition
VulkanSubmissionTracker::AcquireFenceToAdvanceSubmission() {
assert_true(fence_acquired_ == VK_NULL_HANDLE);
// Reclaim fences if the client only gets the completed submission index or
// awaits in special cases such as shutdown.
UpdateAndGetCompletedSubmission();
const VulkanDevice::Functions& dfn = vulkan_device_->functions();
const VkDevice device = vulkan_device_->device();
if (!fences_reclaimed_.empty()) {
VkFence reclaimed_fence = fences_reclaimed_.back();
if (dfn.vkResetFences(device, 1, &reclaimed_fence) == VK_SUCCESS) {
fence_acquired_ = fences_reclaimed_.back();
fences_reclaimed_.pop_back();
}
}
if (fence_acquired_ == VK_NULL_HANDLE) {
VkFenceCreateInfo fence_create_info;
fence_create_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
fence_create_info.pNext = nullptr;
fence_create_info.flags = 0;
// May fail, a null fence is handled in FenceAcquisition.
dfn.vkCreateFence(device, &fence_create_info, nullptr, &fence_acquired_);
}
return FenceAcquisition(*this, fence_acquired_);
}
} // namespace vulkan
} // namespace ui
} // namespace xe

View File

@@ -1,141 +0,0 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2021 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#ifndef XENIA_UI_VULKAN_VULKAN_SUBMISSION_TRACKER_H_
#define XENIA_UI_VULKAN_VULKAN_SUBMISSION_TRACKER_H_
#include <cstdint>
#include <deque>
#include <utility>
#include <vector>
#include "xenia/base/assert.h"
#include "xenia/ui/vulkan/vulkan_device.h"
namespace xe {
namespace ui {
namespace vulkan {
// Fence wrapper, safely handling cases when the fence has not been initialized
// yet or has already been shut down, and failed or dropped submissions.
//
// The current submission index can be associated with the usage of objects to
// release them when the GPU isn't potentially referencing them anymore, and
// should be incremented only
//
// 0 can be used as a "never referenced" submission index.
//
// The submission index timeline survives Shutdown, so submission indices can be
// given to clients that are not aware of the lifetime of the tracker.
//
// To transfer the tracker to another queue (to make sure the first signal on
// the new queue does not happen before the last signal on the old one), call
// AwaitAllSubmissionsCompletion before doing the first submission on the new
// one.
class VulkanSubmissionTracker {
public:
class FenceAcquisition {
public:
FenceAcquisition() : submission_tracker_(nullptr), fence_(VK_NULL_HANDLE) {}
FenceAcquisition(VulkanSubmissionTracker& submission_tracker, VkFence fence)
: submission_tracker_(&submission_tracker), fence_(fence) {}
FenceAcquisition(const FenceAcquisition& fence_acquisition) = delete;
FenceAcquisition& operator=(const FenceAcquisition& fence_acquisition) =
delete;
FenceAcquisition(FenceAcquisition&& fence_acquisition) {
*this = std::move(fence_acquisition);
}
FenceAcquisition& operator=(FenceAcquisition&& fence_acquisition) {
if (this == &fence_acquisition) {
return *this;
}
submission_tracker_ = fence_acquisition.submission_tracker_;
fence_acquisition.submission_tracker_ = nullptr;
fence_ = fence_acquisition.fence_;
fence_acquisition.fence_ = VK_NULL_HANDLE;
return *this;
}
~FenceAcquisition();
// In unsignaled state. May be null if failed to create or to reset a fence.
VkFence fence() { return fence_; }
// Call if vkQueueSubmit has failed (or it was decided not to commit the
// submission), and the submission index shouldn't be incremented by
// releasing this submission (for instance, to retry commands with long-term
// effects like copying or image layout changes later if in the attempt the
// submission index stays the same).
void SubmissionFailedOrDropped();
// Call if for some reason (like signaling multiple fences) the fence
// signaling was done in a separate submission than the command buffer, and
// the command buffer vkQueueSubmit succeeded (so commands with long-term
// effects will be executed), but the fence-only vkQueueSubmit has failed,
// thus the tracker shouldn't attempt to wait for that fence (it will be in
// the unsignaled state).
void SubmissionSucceededSignalFailed() { signal_failed_ = true; }
private:
// If nullptr, has been moved to another FenceAcquisition - not holding a
// fence from now on.
VulkanSubmissionTracker* submission_tracker_;
VkFence fence_;
bool signal_failed_ = false;
};
VulkanSubmissionTracker(const VulkanDevice* vulkan_device)
: vulkan_device_(vulkan_device) {
assert_not_null(vulkan_device);
}
VulkanSubmissionTracker(const VulkanSubmissionTracker& submission_tracker) =
delete;
VulkanSubmissionTracker& operator=(
const VulkanSubmissionTracker& submission_tracker) = delete;
~VulkanSubmissionTracker() { Shutdown(); }
void Shutdown();
uint64_t GetCurrentSubmission() const { return submission_current_; }
uint64_t UpdateAndGetCompletedSubmission();
// Returns whether the expected GPU signal has actually been reached (rather
// than some fallback condition) for cases when stronger completeness
// guarantees as needed (when downloading, as opposed to just destroying).
// If false is returned, it's also not guaranteed that GetCompletedSubmission
// will return a value >= submission_index.
bool AwaitSubmissionCompletion(uint64_t submission_index);
bool AwaitAllSubmissionsCompletion() {
return AwaitSubmissionCompletion(submission_current_ - 1);
}
[[nodiscard]] FenceAcquisition AcquireFenceToAdvanceSubmission();
private:
const VulkanDevice* vulkan_device_;
uint64_t submission_current_ = 1;
// Last submission with a successful fence signal as well as a successful
// fence wait / query.
uint64_t submission_completed_on_gpu_ = 0;
// The flow is:
// Reclaimed (or create if empty) > acquired > pending > reclaimed.
// Or, if dropped the submission while acquired:
// Reclaimed (or create if empty) > acquired > reclaimed.
VkFence fence_acquired_ = VK_NULL_HANDLE;
// Ordered by the submission index (the first pair member).
std::deque<std::pair<uint64_t, VkFence>> fences_pending_;
// Fences are reclaimed when awaiting or when refreshing the completed value.
std::vector<VkFence> fences_reclaimed_;
};
} // namespace vulkan
} // namespace ui
} // namespace xe
#endif // XENIA_UI_VULKAN_VULKAN_SUBMISSION_TRACKER_H_