fabi/Xenia-Canary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Memory, D3D12] Various refactoring from data provider development

Browse Source
This commit is contained in:
Triang3l
2020-02-15 21:35:24 +03:00
parent b59ae30ec3
commit 8ec813de82
17 changed files with 490 additions and 362 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/xenia/gpu/d3d12/d3d12_command_processor.cc
View File

@@ -87,8 +87,8 @@ void D3D12CommandProcessor::RequestFrameTrace(const std::wstring& root_path) {
void D3D12CommandProcessor::TracePlaybackWroteMemory(uint32_t base_ptr,
uint32_t length) {
shared_memory_->MemoryWriteCallback(base_ptr, length, true);
primitive_converter_->MemoryWriteCallback(base_ptr, length, true);
shared_memory_->MemoryInvalidationCallback(base_ptr, length, true);
primitive_converter_->MemoryInvalidationCallback(base_ptr, length, true);
}
void D3D12CommandProcessor::RestoreEDRAMSnapshot(const void* snapshot) {
@@ -866,6 +866,7 @@ bool D3D12CommandProcessor::SetupContext() {
if (FAILED(gamma_ramp_upload_->Map(
0, nullptr, reinterpret_cast<void**>(&gamma_ramp_upload_mapping_)))) {
XELOGE("Failed to map the gamma ramp upload buffer");
gamma_ramp_upload_mapping_ = nullptr;
return false;
}
@@ -1827,42 +1828,24 @@ bool D3D12CommandProcessor::IssueCopy() {
return true;
}
void D3D12CommandProcessor::BeginSubmission(bool is_guest_command) {
#if FINE_GRAINED_DRAW_SCOPES
SCOPE_profile_cpu_f("gpu");
#endif // FINE_GRAINED_DRAW_SCOPES
bool is_opening_frame = is_guest_command && !frame_open_;
if (submission_open_ && !is_opening_frame) {
return;
void D3D12CommandProcessor::CheckSubmissionFence(uint64_t await_submission) {
assert_true(await_submission <= submission_current_);
if (await_submission == submission_current_) {
assert_true(submission_open_);
EndSubmission(false);
}
// Check the fence - needed for all kinds of submissions (to reclaim transient
// resources early) and specifically for frames (not to queue too many).
uint64_t submission_completed_before = submission_completed_;
submission_completed_ = submission_fence_->GetCompletedValue();
if (is_opening_frame) {
// Await the availability of the current frame.
uint64_t frame_current_last_submission =
closed_frame_submissions_[frame_current_ % kQueueFrames];
if (frame_current_last_submission > submission_completed_) {
submission_fence_->SetEventOnCompletion(
frame_current_last_submission, submission_fence_completion_event_);
WaitForSingleObject(submission_fence_completion_event_, INFINITE);
submission_completed_ = submission_fence_->GetCompletedValue();
}
// Update the completed frame index, also obtaining the actual completed
// frame number (since the CPU may be actually less than 3 frames behind)
// before reclaiming resources tracked with the frame number.
frame_completed_ =
std::max(frame_current_, uint64_t(kQueueFrames)) - kQueueFrames;
for (uint64_t frame = frame_completed_ + 1; frame < frame_current_;
++frame) {
if (closed_frame_submissions_[frame % kQueueFrames] >
submission_completed_) {
break;
}
frame_completed_ = frame;
}
if (submission_completed_ < await_submission) {
submission_fence_->SetEventOnCompletion(await_submission,
submission_fence_completion_event_);
WaitForSingleObject(submission_fence_completion_event_, INFINITE);
submission_completed_ = submission_fence_->GetCompletedValue();
}
if (submission_completed_ <= submission_completed_before) {
// Not updated - no need to reclaim or download things.
return;
}
// Reclaim command allocators.
@@ -1898,6 +1881,46 @@ void D3D12CommandProcessor::BeginSubmission(bool is_guest_command) {
}
buffers_for_deletion_.erase(buffers_for_deletion_.begin(), erase_buffers_end);
shared_memory_->CompletedSubmissionUpdated();
render_target_cache_->CompletedSubmissionUpdated();
primitive_converter_->CompletedSubmissionUpdated();
}
void D3D12CommandProcessor::BeginSubmission(bool is_guest_command) {
#if FINE_GRAINED_DRAW_SCOPES
SCOPE_profile_cpu_f("gpu");
#endif // FINE_GRAINED_DRAW_SCOPES
bool is_opening_frame = is_guest_command && !frame_open_;
if (submission_open_ && !is_opening_frame) {
return;
}
// Check the fence - needed for all kinds of submissions (to reclaim transient
// resources early) and specifically for frames (not to queue too many), and
// await the availability of the current frame.
CheckSubmissionFence(
is_opening_frame
? closed_frame_submissions_[frame_current_ % kQueueFrames]
: 0);
if (is_opening_frame) {
// Update the completed frame index, also obtaining the actual completed
// frame number (since the CPU may be actually less than 3 frames behind)
// before reclaiming resources tracked with the frame number.
frame_completed_ =
std::max(frame_current_, uint64_t(kQueueFrames)) - kQueueFrames;
for (uint64_t frame = frame_completed_ + 1; frame < frame_current_;
++frame) {
if (closed_frame_submissions_[frame % kQueueFrames] >
submission_completed_) {
break;
}
frame_completed_ = frame;
}
}
if (!submission_open_) {
submission_open_ = true;
@@ -1920,8 +1943,6 @@ void D3D12CommandProcessor::BeginSubmission(bool is_guest_command) {
current_sampler_heap_ = nullptr;
primitive_topology_ = D3D_PRIMITIVE_TOPOLOGY_UNDEFINED;
shared_memory_->BeginSubmission();
render_target_cache_->BeginSubmission();
primitive_converter_->BeginSubmission();

3
src/xenia/gpu/d3d12/d3d12_command_processor.h
View File

@@ -229,6 +229,9 @@ class D3D12CommandProcessor : public CommandProcessor {
// frame. EndSubmission(true) will close the frame no matter whether the
// submission has already been closed.
// Rechecks submission number and reclaims per-submission resources. Pass 0 as
// the submission to await to simply check status.
void CheckSubmissionFence(uint64_t await_submission);
// If is_guest_command is true, a new full frame - with full cleanup of
// resources and, if needed, starting capturing - is opened if pending (as
// opposed to simply resuming after mid-frame synchronization).

50
src/xenia/gpu/d3d12/primitive_converter.cc
View File

@@ -124,16 +124,18 @@ bool PrimitiveConverter::Initialize() {
static_ib_gpu_address_ = static_ib_->GetGPUVirtualAddress();
memory_regions_invalidated_.store(0ull, std::memory_order_relaxed);
physical_write_watch_handle_ =
memory_->RegisterPhysicalWriteWatch(MemoryWriteCallbackThunk, this);
memory_invalidation_callback_handle_ =
memory_->RegisterPhysicalMemoryInvalidationCallback(
MemoryInvalidationCallbackThunk, this);
return true;
}
void PrimitiveConverter::Shutdown() {
if (physical_write_watch_handle_ != nullptr) {
memory_->UnregisterPhysicalWriteWatch(physical_write_watch_handle_);
physical_write_watch_handle_ = nullptr;
if (memory_invalidation_callback_handle_ != nullptr) {
memory_->UnregisterPhysicalMemoryInvalidationCallback(
memory_invalidation_callback_handle_);
memory_invalidation_callback_handle_ = nullptr;
}
ui::d3d12::util::ReleaseAndNull(static_ib_);
ui::d3d12::util::ReleaseAndNull(static_ib_upload_);
@@ -142,24 +144,25 @@ void PrimitiveConverter::Shutdown() {
void PrimitiveConverter::ClearCache() { buffer_pool_->ClearCache(); }
void PrimitiveConverter::CompletedSubmissionUpdated() {
if (static_ib_upload_ && command_processor_->GetCompletedSubmission() >=
static_ib_upload_submission_) {
// Completely uploaded - release the upload buffer.
static_ib_upload_->Release();
static_ib_upload_ = nullptr;
}
}
void PrimitiveConverter::BeginSubmission() {
// Got a command list now - upload and transition the static index buffer if
// needed.
if (static_ib_upload_) {
if (static_ib_upload_submission_ == UINT64_MAX) {
// Not uploaded yet - upload.
command_processor_->GetDeferredCommandList()->D3DCopyResource(
static_ib_, static_ib_upload_);
command_processor_->PushTransitionBarrier(
static_ib_, D3D12_RESOURCE_STATE_COPY_DEST,
D3D12_RESOURCE_STATE_INDEX_BUFFER);
static_ib_upload_submission_ = command_processor_->GetCurrentSubmission();
} else if (command_processor_->GetCompletedSubmission() >=
static_ib_upload_submission_) {
// Completely uploaded - release the upload buffer.
static_ib_upload_->Release();
static_ib_upload_ = nullptr;
}
if (static_ib_upload_ && static_ib_upload_submission_ == UINT64_MAX) {
command_processor_->GetDeferredCommandList()->D3DCopyResource(
static_ib_, static_ib_upload_);
command_processor_->PushTransitionBarrier(
static_ib_, D3D12_RESOURCE_STATE_COPY_DEST,
D3D12_RESOURCE_STATE_INDEX_BUFFER);
static_ib_upload_submission_ = command_processor_->GetCurrentSubmission();
}
}
@@ -706,7 +709,7 @@ void* PrimitiveConverter::AllocateIndices(
return mapping + simd_offset;
}
std::pair<uint32_t, uint32_t> PrimitiveConverter::MemoryWriteCallback(
std::pair<uint32_t, uint32_t> PrimitiveConverter::MemoryInvalidationCallback(
uint32_t physical_address_start, uint32_t length, bool exact_range) {
// 1 bit = (512 / 64) MB = 8 MB. Invalidate a region of this size.
uint32_t bit_index_first = physical_address_start >> 23;
@@ -719,11 +722,12 @@ std::pair<uint32_t, uint32_t> PrimitiveConverter::MemoryWriteCallback(
return std::make_pair<uint32_t, uint32_t>(0, UINT32_MAX);
}
std::pair<uint32_t, uint32_t> PrimitiveConverter::MemoryWriteCallbackThunk(
std::pair<uint32_t, uint32_t>
PrimitiveConverter::MemoryInvalidationCallbackThunk(
void* context_ptr, uint32_t physical_address_start, uint32_t length,
bool exact_range) {
return reinterpret_cast<PrimitiveConverter*>(context_ptr)
->MemoryWriteCallback(physical_address_start, length, exact_range);
->MemoryInvalidationCallback(physical_address_start, length, exact_range);
}
D3D12_GPU_VIRTUAL_ADDRESS PrimitiveConverter::GetStaticIndexBuffer(

7
src/xenia/gpu/d3d12/primitive_converter.h
View File

@@ -46,6 +46,7 @@ class PrimitiveConverter {
void Shutdown();
void ClearCache();
void CompletedSubmissionUpdated();
void BeginSubmission();
void BeginFrame();
@@ -83,7 +84,7 @@ class PrimitiveConverter {
uint32_t& index_count_out) const;
// Callback for invalidating buffers mid-frame.
std::pair<uint32_t, uint32_t> MemoryWriteCallback(
std::pair<uint32_t, uint32_t> MemoryInvalidationCallback(
uint32_t physical_address_start, uint32_t length, bool exact_range);
void InitializeTrace();
@@ -96,7 +97,7 @@ class PrimitiveConverter {
uint32_t simd_offset,
D3D12_GPU_VIRTUAL_ADDRESS& gpu_address_out);
static std::pair<uint32_t, uint32_t> MemoryWriteCallbackThunk(
static std::pair<uint32_t, uint32_t> MemoryInvalidationCallbackThunk(
void* context_ptr, uint32_t physical_address_start, uint32_t length,
bool exact_range);
@@ -176,7 +177,7 @@ class PrimitiveConverter {
// the cache.
uint64_t memory_regions_used_;
std::atomic<uint64_t> memory_regions_invalidated_ = 0;
void* physical_write_watch_handle_ = nullptr;
void* memory_invalidation_callback_handle_ = nullptr;
uint32_t system_page_size_;
};

8
src/xenia/gpu/d3d12/render_target_cache.cc
View File

@@ -455,12 +455,14 @@ void RenderTargetCache::ClearCache() {
edram_snapshot_restore_pool_.reset();
}
void RenderTargetCache::BeginSubmission() {
void RenderTargetCache::CompletedSubmissionUpdated() {
if (edram_snapshot_restore_pool_) {
edram_snapshot_restore_pool_->Reclaim(
command_processor_->GetCompletedSubmission());
}
}
void RenderTargetCache::BeginSubmission() {
// With the ROV, a submission does not always end in a resolve (for example,
// when memexport readback happens) or something else that would surely submit
// the UAV barrier, so we need to preserve the `current_` variables.
@@ -1417,8 +1419,8 @@ bool RenderTargetCache::ResolveCopy(SharedMemory* shared_memory,
return false;
}
} else {
if (!shared_memory->MakeTilesResident(dest_modified_start,
dest_modified_length)) {
if (!shared_memory->EnsureTilesResident(dest_modified_start,
dest_modified_length)) {
return false;
}
}

1
src/xenia/gpu/d3d12/render_target_cache.h
View File

@@ -257,6 +257,7 @@ class RenderTargetCache {
void Shutdown();
void ClearCache();
void CompletedSubmissionUpdated();
void BeginSubmission();
void EndFrame();
// Called in the beginning of a draw call - may bind pipelines.

106
src/xenia/gpu/d3d12/shared_memory.cc
View File

@@ -11,6 +11,7 @@
#include <algorithm>
#include <cstring>
#include <utility>
#include <vector>
#include "xenia/base/assert.h"
@@ -49,11 +50,6 @@ SharedMemory::SharedMemory(D3D12CommandProcessor* command_processor,
trace_writer_(trace_writer) {
page_size_log2_ = xe::log2_ceil(uint32_t(xe::memory::page_size()));
page_count_ = kBufferSize >> page_size_log2_;
uint32_t page_bitmap_length = page_count_ >> 6;
assert_true(page_bitmap_length != 0);
// Two interleaved bit arrays.
valid_and_gpu_written_pages_.resize(page_bitmap_length << 1);
}
SharedMemory::~SharedMemory() { Shutdown(); }
@@ -125,14 +121,16 @@ bool SharedMemory::Initialize() {
uint32_t(BufferDescriptorIndex::kRawUAV)),
buffer_, kBufferSize);
std::memset(valid_and_gpu_written_pages_.data(), 0,
valid_and_gpu_written_pages_.size() * sizeof(uint64_t));
system_page_flags_.clear();
system_page_flags_.resize((page_count_ + 63) / 64);
upload_buffer_pool_ =
std::make_unique<ui::d3d12::UploadBufferPool>(device, 4 * 1024 * 1024);
upload_buffer_pool_ = std::make_unique<ui::d3d12::UploadBufferPool>(
device,
xe::align(uint32_t(4 * 1024 * 1024), uint32_t(1) << page_size_log2_));
physical_write_watch_handle_ =
memory_->RegisterPhysicalWriteWatch(MemoryWriteCallbackThunk, this);
memory_invalidation_callback_handle_ =
memory_->RegisterPhysicalMemoryInvalidationCallback(
MemoryInvalidationCallbackThunk, this);
ResetTraceGPUWrittenBuffer();
@@ -144,9 +142,10 @@ void SharedMemory::Shutdown() {
// TODO(Triang3l): Do something in case any watches are still registered.
if (physical_write_watch_handle_ != nullptr) {
memory_->UnregisterPhysicalWriteWatch(physical_write_watch_handle_);
physical_write_watch_handle_ = nullptr;
if (memory_invalidation_callback_handle_ != nullptr) {
memory_->UnregisterPhysicalMemoryInvalidationCallback(
memory_invalidation_callback_handle_);
memory_invalidation_callback_handle_ = nullptr;
}
upload_buffer_pool_.reset();
@@ -165,7 +164,7 @@ void SharedMemory::Shutdown() {
}
}
void SharedMemory::BeginSubmission() {
void SharedMemory::CompletedSubmissionUpdated() {
upload_buffer_pool_->Reclaim(command_processor_->GetCompletedSubmission());
}
@@ -273,7 +272,7 @@ void SharedMemory::UnwatchMemoryRange(WatchHandle handle) {
UnlinkWatchRange(reinterpret_cast<WatchRange*>(handle));
}
bool SharedMemory::MakeTilesResident(uint32_t start, uint32_t length) {
bool SharedMemory::EnsureTilesResident(uint32_t start, uint32_t length) {
if (length == 0) {
// Some texture is empty, for example - safe to draw in this case.
return true;
@@ -347,7 +346,7 @@ bool SharedMemory::RequestRange(uint32_t start, uint32_t length) {
#endif // FINE_GRAINED_DRAW_SCOPES
// Ensure all tile heaps are present.
if (!MakeTilesResident(start, length)) {
if (!EnsureTilesResident(start, length)) {
return false;
}
@@ -375,7 +374,8 @@ bool SharedMemory::RequestRange(uint32_t start, uint32_t length) {
return false;
}
uint32_t upload_buffer_pages = upload_buffer_size >> page_size_log2_;
MakeRangeValid(upload_range_start, upload_buffer_pages, false);
MakeRangeValid(upload_range_start << page_size_log2_,
upload_buffer_pages << page_size_log2_, false);
std::memcpy(
upload_buffer_mapping,
memory_->TranslatePhysical(upload_range_start << page_size_log2_),
@@ -439,7 +439,7 @@ void SharedMemory::RangeWrittenByGPU(uint32_t start, uint32_t length) {
// Mark the range as valid (so pages are not reuploaded until modified by the
// CPU) and watch it so the CPU can reuse it and this will be caught.
MakeRangeValid(page_first, page_last - page_first + 1, true);
MakeRangeValid(start, length, true);
}
bool SharedMemory::AreTiledResourcesUsed() const {
@@ -453,14 +453,15 @@ bool SharedMemory::AreTiledResourcesUsed() const {
provider->GetGraphicsAnalysis() == nullptr;
}
void SharedMemory::MakeRangeValid(uint32_t valid_page_first,
uint32_t valid_page_count,
void SharedMemory::MakeRangeValid(uint32_t start, uint32_t length,
bool written_by_gpu) {
if (valid_page_first >= page_count_ || valid_page_count == 0) {
if (length == 0 || start >= kBufferSize) {
return;
}
valid_page_count = std::min(valid_page_count, page_count_ - valid_page_first);
uint32_t valid_page_last = valid_page_first + valid_page_count - 1;
length = std::min(length, kBufferSize - start);
uint32_t last = start + length - 1;
uint32_t valid_page_first = start >> page_size_log2_;
uint32_t valid_page_last = last >> page_size_log2_;
uint32_t valid_block_first = valid_page_first >> 6;
uint32_t valid_block_last = valid_page_last >> 6;
@@ -475,18 +476,21 @@ void SharedMemory::MakeRangeValid(uint32_t valid_page_first,
if (i == valid_block_last && (valid_page_last & 63) != 63) {
valid_bits &= (1ull << ((valid_page_last & 63) + 1)) - 1;
}
valid_and_gpu_written_pages_[i << 1] |= valid_bits;
SystemPageFlagsBlock& block = system_page_flags_[i];
block.valid |= valid_bits;
if (written_by_gpu) {
valid_and_gpu_written_pages_[(i << 1) + 1] |= valid_bits;
block.valid_and_gpu_written |= valid_bits;
} else {
valid_and_gpu_written_pages_[(i << 1) + 1] &= ~valid_bits;
block.valid_and_gpu_written &= ~valid_bits;
}
}
}
if (physical_write_watch_handle_) {
memory_->WatchPhysicalMemoryWrite(valid_page_first << page_size_log2_,
valid_page_count << page_size_log2_);
if (memory_invalidation_callback_handle_) {
memory_->EnablePhysicalMemoryAccessCallbacks(
valid_page_first << page_size_log2_,
(valid_page_last - valid_page_first + 1) << page_size_log2_, true,
false);
}
}
@@ -527,7 +531,7 @@ void SharedMemory::GetRangesToUpload(uint32_t request_page_first,
uint32_t range_start = UINT32_MAX;
for (uint32_t i = request_block_first; i <= request_block_last; ++i) {
uint64_t block_valid = valid_and_gpu_written_pages_[i << 1];
uint64_t block_valid = system_page_flags_[i].valid;
// Consider pages in the block outside the requested range valid.
if (i == request_block_first) {
block_valid |= (1ull << (request_page_first & 63)) - 1;
@@ -569,17 +573,23 @@ void SharedMemory::GetRangesToUpload(uint32_t request_page_first,
}
}
std::pair<uint32_t, uint32_t> SharedMemory::MemoryWriteCallbackThunk(
std::pair<uint32_t, uint32_t> SharedMemory::MemoryInvalidationCallbackThunk(
void* context_ptr, uint32_t physical_address_start, uint32_t length,
bool exact_range) {
return reinterpret_cast<SharedMemory*>(context_ptr)
->MemoryWriteCallback(physical_address_start, length, exact_range);
->MemoryInvalidationCallback(physical_address_start, length, exact_range);
}
std::pair<uint32_t, uint32_t> SharedMemory::MemoryWriteCallback(
std::pair<uint32_t, uint32_t> SharedMemory::MemoryInvalidationCallback(
uint32_t physical_address_start, uint32_t length, bool exact_range) {
if (length == 0 || physical_address_start >= kBufferSize) {
return std::make_pair(uint32_t(0), UINT32_MAX);
}
length = std::min(length, kBufferSize - physical_address_start);
uint32_t physical_address_last = physical_address_start + (length - 1);
uint32_t page_first = physical_address_start >> page_size_log2_;
uint32_t page_last = (physical_address_start + length - 1) >> page_size_log2_;
uint32_t page_last = physical_address_last >> page_size_log2_;
assert_true(page_first < page_count_ && page_last < page_count_);
uint32_t block_first = page_first >> 6;
uint32_t block_last = page_last >> 6;
@@ -596,14 +606,14 @@ std::pair<uint32_t, uint32_t> SharedMemory::MemoryWriteCallback(
// frame, but with 256 KB it's 0.7 ms.
if (page_first & 63) {
uint64_t gpu_written_start =
valid_and_gpu_written_pages_[(block_first << 1) + 1];
system_page_flags_[block_first].valid_and_gpu_written;
gpu_written_start &= (1ull << (page_first & 63)) - 1;
page_first =
(page_first & ~uint32_t(63)) + (64 - xe::lzcnt(gpu_written_start));
}
if ((page_last & 63) != 63) {
uint64_t gpu_written_end =
valid_and_gpu_written_pages_[(block_last << 1) + 1];
system_page_flags_[block_last].valid_and_gpu_written;
gpu_written_end &= ~((1ull << ((page_last & 63) + 1)) - 1);
page_last = (page_last & ~uint32_t(63)) +
(std::max(xe::tzcnt(gpu_written_end), uint8_t(1)) - 1);
@@ -618,8 +628,9 @@ std::pair<uint32_t, uint32_t> SharedMemory::MemoryWriteCallback(
if (i == block_last && (page_last & 63) != 63) {
invalidate_bits &= (1ull << ((page_last & 63) + 1)) - 1;
}
valid_and_gpu_written_pages_[i << 1] &= ~invalidate_bits;
valid_and_gpu_written_pages_[(i << 1) + 1] &= ~invalidate_bits;
SystemPageFlagsBlock& block = system_page_flags_[i];
block.valid &= ~invalidate_bits;
block.valid_and_gpu_written &= ~invalidate_bits;
}
FireWatches(page_first, page_last, false);
@@ -664,10 +675,11 @@ bool SharedMemory::InitializeTraceSubmitDownloads() {
auto global_lock = global_critical_region_.Acquire();
uint32_t fire_watches_range_start = UINT32_MAX;
uint32_t gpu_written_range_start = UINT32_MAX;
for (uint32_t i = 0; i * 2 < valid_and_gpu_written_pages_.size(); ++i) {
uint64_t previously_valid_block = valid_and_gpu_written_pages_[i * 2];
uint64_t gpu_written_block = valid_and_gpu_written_pages_[i * 2 + 1];
valid_and_gpu_written_pages_[i * 2] = gpu_written_block;
for (uint32_t i = 0; i < system_page_flags_.size(); ++i) {
SystemPageFlagsBlock& page_flags_block = system_page_flags_[i];
uint64_t previously_valid_block = page_flags_block.valid;
uint64_t gpu_written_block = page_flags_block.valid_and_gpu_written;
page_flags_block.valid = gpu_written_block;
// Fire watches on the invalidated pages.
uint64_t fire_watches_block = previously_valid_block & ~gpu_written_block;
@@ -748,8 +760,8 @@ bool SharedMemory::InitializeTraceSubmitDownloads() {
&gpu_written_buffer_desc, D3D12_RESOURCE_STATE_COPY_DEST, nullptr,
IID_PPV_ARGS(&trace_gpu_written_buffer_)))) {
XELOGE(
"Failed to create a %u KB GPU-written memory download buffer for frame "
"tracing",
"Shared memory: Failed to create a %u KB GPU-written memory download "
"buffer for frame tracing",
gpu_written_page_count << page_size_log2_ >> 10);
ResetTraceGPUWrittenBuffer();
return false;
@@ -761,8 +773,8 @@ bool SharedMemory::InitializeTraceSubmitDownloads() {
for (auto& gpu_written_submit_range : trace_gpu_written_ranges_) {
// For cases like resolution scale, when the data may not be actually
// written, just marked as valid.
if (!MakeTilesResident(gpu_written_submit_range.first,
gpu_written_submit_range.second)) {
if (!EnsureTilesResident(gpu_written_submit_range.first,
gpu_written_submit_range.second)) {
gpu_written_submit_range.second = 0;
continue;
}

93
src/xenia/gpu/d3d12/shared_memory.h
View File

@@ -11,7 +11,6 @@
#define XENIA_GPU_D3D12_SHARED_MEMORY_H_
#include <memory>
#include <mutex>
#include <utility>
#include <vector>
@@ -44,7 +43,7 @@ class SharedMemory {
return buffer_gpu_address_;
}
void BeginSubmission();
void CompletedSubmissionUpdated();
typedef void (*GlobalWatchCallback)(void* context, uint32_t address_first,
uint32_t address_last,
@@ -57,7 +56,7 @@ class SharedMemory {
// example, if the game changes protection level of a memory range containing
// the watched range.
//
// The callback is called with the mutex locked.
// The callback is called within the global critical region.
GlobalWatchHandle RegisterGlobalWatch(GlobalWatchCallback callback,
void* callback_context);
void UnregisterGlobalWatch(GlobalWatchHandle handle);
@@ -84,15 +83,10 @@ class SharedMemory {
void* callback_data, uint64_t callback_argument);
// Unregisters previously registered watched memory range.
void UnwatchMemoryRange(WatchHandle handle);
// Locks the mutex that gets locked when watch callbacks are invoked - must be
// done when checking variables that may be changed by a watch callback.
inline std::unique_lock<std::recursive_mutex> LockWatchMutex() {
return global_critical_region_.Acquire();
}
// Ensures the buffer tiles backing the range are resident, but doesn't upload
// anything.
bool MakeTilesResident(uint32_t start, uint32_t length);
bool EnsureTilesResident(uint32_t start, uint32_t length);
// Checks if the range has been updated, uploads new data if needed and
// ensures the buffer tiles backing the range are resident. May transition the
@@ -105,7 +99,7 @@ class SharedMemory {
// (to up to the first GPU-written page, as an access violation exception
// count optimization) as modified by the CPU, also invalidating GPU-written
// pages directly in the range.
std::pair<uint32_t, uint32_t> MemoryWriteCallback(
std::pair<uint32_t, uint32_t> MemoryInvalidationCallback(
uint32_t physical_address_start, uint32_t length, bool exact_range);
// Marks the range as containing GPU-generated data (such as resolves),
@@ -141,8 +135,7 @@ class SharedMemory {
bool AreTiledResourcesUsed() const;
// Mark the memory range as updated and protect it.
void MakeRangeValid(uint32_t valid_page_first, uint32_t valid_page_count,
bool written_by_gpu);
void MakeRangeValid(uint32_t start, uint32_t length, bool written_by_gpu);
D3D12CommandProcessor* command_processor_;
Memory* memory_;
@@ -154,6 +147,7 @@ class SharedMemory {
ID3D12Resource* buffer_ = nullptr;
D3D12_GPU_VIRTUAL_ADDRESS buffer_gpu_address_ = 0;
D3D12_RESOURCE_STATES buffer_state_ = D3D12_RESOURCE_STATE_COPY_DEST;
void TransitionBuffer(D3D12_RESOURCE_STATES new_state);
// Heaps are 4 MB, so not too many of them are allocated, but also not to
// waste too much memory for padding (with 16 MB there's too much).
@@ -166,9 +160,11 @@ class SharedMemory {
// Number of the heaps currently resident, for profiling.
uint32_t heap_count_ = 0;
// Log2 of system page size.
// Log2 of invalidation granularity (the system page size, but the dependency
// on it is not hard - the access callback takes a range as an argument, and
// touched pages of the buffer of this size will be invalidated).
uint32_t page_size_log2_;
// Total physical page count.
// Total buffer page count.
uint32_t page_count_;
// Non-shader-visible buffer descriptor heap for faster binding (via copying
@@ -182,24 +178,46 @@ class SharedMemory {
ID3D12DescriptorHeap* buffer_descriptor_heap_ = nullptr;
D3D12_CPU_DESCRIPTOR_HANDLE buffer_descriptor_heap_start_;
// Handle of the physical memory write callback.
void* physical_write_watch_handle_ = nullptr;
// First page and length in pages.
typedef std::pair<uint32_t, uint32_t> UploadRange;
// Ranges that need to be uploaded, generated by GetRangesToUpload (a
// persistently allocated vector).
std::vector<UploadRange> upload_ranges_;
void GetRangesToUpload(uint32_t request_page_first,
uint32_t request_page_last);
std::unique_ptr<ui::d3d12::UploadBufferPool> upload_buffer_pool_ = nullptr;
// Mutex between the exception handler and the command processor, to be locked
// when checking or updating validity of pages/ranges.
// GPU-written memory downloading for traces.
// Start page, length in pages.
std::vector<std::pair<uint32_t, uint32_t>> trace_gpu_written_ranges_;
// Created temporarily, only for downloading.
ID3D12Resource* trace_gpu_written_buffer_ = nullptr;
void ResetTraceGPUWrittenBuffer();
void* memory_invalidation_callback_handle_ = nullptr;
void* memory_data_provider_handle_ = nullptr;
// Mutex between the guest memory subsystem and the command processor, to be
// locked when checking or updating validity of pages/ranges and when firing
// watches.
xe::global_critical_region global_critical_region_;
// ***************************************************************************
// Things below should be protected by global_critical_region.
// Things below should be fully protected by global_critical_region.
// ***************************************************************************
// Bit vector containing:
// - Even block indices - whether physical memory system pages are up to date.
// - Odd block indices - whether phyical memory system pages contain data
// written by the GPU not synchronized with the CPU (subset of valid pages).
std::vector<uint64_t> valid_and_gpu_written_pages_;
struct SystemPageFlagsBlock {
// Whether each page is up to date in the GPU buffer.
uint64_t valid;
// Subset of valid pages - whether each page in the GPU buffer contains data
// that was written on the GPU, thus should not be invalidated spuriously.
uint64_t valid_and_gpu_written;
};
// Flags for each 64 system pages, interleaved as blocks, so bit scan can be
// used to quickly extract ranges.
std::vector<SystemPageFlagsBlock> system_page_flags_;
static std::pair<uint32_t, uint32_t> MemoryWriteCallbackThunk(
static std::pair<uint32_t, uint32_t> MemoryInvalidationCallbackThunk(
void* context_ptr, uint32_t physical_address_start, uint32_t length,
bool exact_range);
@@ -259,30 +277,9 @@ class SharedMemory {
// watches.
void FireWatches(uint32_t page_first, uint32_t page_last,
bool invalidated_by_gpu);
// Unlinks and frees the range and its nodes. Call this with the mutex locked.
// Unlinks and frees the range and its nodes. Call this in the global critical
// region.
void UnlinkWatchRange(WatchRange* range);
// ***************************************************************************
// Things above should be protected by global_critical_region.
// ***************************************************************************
// First page and length in pages.
typedef std::pair<uint32_t, uint32_t> UploadRange;
// Ranges that need to be uploaded, generated by GetRangesToUpload (a
// persistently allocated vector).
std::vector<UploadRange> upload_ranges_;
void GetRangesToUpload(uint32_t request_page_first,
uint32_t request_page_last);
std::unique_ptr<ui::d3d12::UploadBufferPool> upload_buffer_pool_ = nullptr;
void TransitionBuffer(D3D12_RESOURCE_STATES new_state);
// GPU-written memory downloading for traces.
// Start page, length in pages.
std::vector<std::pair<uint32_t, uint32_t>> trace_gpu_written_ranges_;
// Created temporarily, only for downloading.
ID3D12Resource* trace_gpu_written_buffer_ = nullptr;
void ResetTraceGPUWrittenBuffer();
};
} // namespace d3d12

12
src/xenia/gpu/d3d12/texture_cache.cc
View File

@@ -1702,7 +1702,7 @@ void TextureCache::MarkRangeAsResolved(uint32_t start_unscaled,
uint32_t page_last = (start_unscaled + length_unscaled - 1) >> 12;
uint32_t block_first = page_first >> 5;
uint32_t block_last = page_last >> 5;
auto watch_lock = shared_memory_->LockWatchMutex();
auto global_lock = global_critical_region_.Acquire();
for (uint32_t i = block_first; i <= block_last; ++i) {
uint32_t add_bits = UINT32_MAX;
if (i == block_first) {
@@ -1812,8 +1812,8 @@ bool TextureCache::TileResolvedTexture(
return false;
}
} else {
if (!shared_memory_->MakeTilesResident(texture_modified_start,
texture_modified_length)) {
if (!shared_memory_->EnsureTilesResident(texture_modified_start,
texture_modified_length)) {
return false;
}
}
@@ -2404,7 +2404,7 @@ bool TextureCache::LoadTextureData(Texture* texture) {
// See what we need to upload.
bool base_in_sync, mips_in_sync;
{
auto watch_lock = shared_memory_->LockWatchMutex();
auto global_lock = global_critical_region_.Acquire();
base_in_sync = texture->base_in_sync;
mips_in_sync = texture->mips_in_sync;
}
@@ -2672,7 +2672,7 @@ bool TextureCache::LoadTextureData(Texture* texture) {
// regular texture or a vertex buffer, and thus the scaled resolve version is
// not up to date anymore.
{
auto watch_lock = shared_memory_->LockWatchMutex();
auto global_lock = global_critical_region_.Acquire();
texture->base_in_sync = true;
texture->mips_in_sync = true;
if (!base_in_sync) {
@@ -2761,7 +2761,7 @@ bool TextureCache::IsRangeScaledResolved(uint32_t start_unscaled,
uint32_t block_last = page_last >> 5;
uint32_t l2_block_first = block_first >> 6;
uint32_t l2_block_last = block_last >> 6;
auto watch_lock = shared_memory_->LockWatchMutex();
auto global_lock = global_critical_region_.Acquire();
for (uint32_t i = l2_block_first; i <= l2_block_last; ++i) {
uint64_t l2_block = scaled_resolve_pages_l2_[i];
if (i == l2_block_first) {

21
src/xenia/gpu/d3d12/texture_cache.h
View File

@@ -11,9 +11,9 @@
#define XENIA_GPU_D3D12_TEXTURE_CACHE_H_
#include <atomic>
#include <mutex>
#include <unordered_map>
#include "xenia/base/mutex.h"
#include "xenia/gpu/d3d12/d3d12_shader.h"
#include "xenia/gpu/d3d12/shared_memory.h"
#include "xenia/gpu/register_file.h"
@@ -369,15 +369,14 @@ class TextureCache {
static constexpr uint32_t kCachedSRVDescriptorSwizzleMissing = UINT32_MAX;
uint32_t cached_srv_descriptor_swizzle;
// Watch handles for the memory ranges (protected by the shared memory watch
// mutex).
// These are to be accessed within the global critical region to synchronize
// with shared memory.
// Watch handles for the memory ranges.
SharedMemory::WatchHandle base_watch_handle;
SharedMemory::WatchHandle mip_watch_handle;
// Whether the recent base level data has been loaded from the memory
// (protected by the shared memory watch mutex).
// Whether the recent base level data has been loaded from the memory.
bool base_in_sync;
// Whether the recent mip data has been loaded from the memory (protected by
// the shared memory watch mutex).
// Whether the recent mip data has been loaded from the memory.
bool mips_in_sync;
};
@@ -620,16 +619,16 @@ class TextureCache {
kScaledResolveHeapSizeLog2] = {};
// Number of currently resident portions of the tiled buffer, for profiling.
uint32_t scaled_resolve_heap_count_ = 0;
// Global watch for scaled resolve data invalidation.
SharedMemory::GlobalWatchHandle scaled_resolve_global_watch_handle_ = nullptr;
xe::global_critical_region global_critical_region_;
// Bit vector storing whether each 4 KB physical memory page contains scaled
// resolve data. uint32_t rather than uint64_t because parts of it are sent to
// shaders.
// PROTECTED BY THE SHARED MEMORY WATCH MUTEX!
uint32_t* scaled_resolve_pages_ = nullptr;
// Second level of the bit vector for faster rejection of non-scaled textures.
// PROTECTED BY THE SHARED MEMORY WATCH MUTEX!
uint64_t scaled_resolve_pages_l2_[(512 << 20) >> (12 + 5 + 6)];
// Global watch for scaled resolve data invalidation.
SharedMemory::GlobalWatchHandle scaled_resolve_global_watch_handle_ = nullptr;
};
} // namespace d3d12

20
src/xenia/gpu/vulkan/texture_cache.cc
View File

@@ -150,16 +150,18 @@ VkResult TextureCache::Initialize() {
device_queue_ = device_->AcquireQueue(device_->queue_family_index());
physical_write_watch_handle_ =
memory_->RegisterPhysicalWriteWatch(MemoryWriteCallbackThunk, this);
memory_invalidation_callback_handle_ =
memory_->RegisterPhysicalMemoryInvalidationCallback(
MemoryInvalidationCallbackThunk, this);
return VK_SUCCESS;
}
void TextureCache::Shutdown() {
if (physical_write_watch_handle_ != nullptr) {
memory_->UnregisterPhysicalWriteWatch(physical_write_watch_handle_);
physical_write_watch_handle_ = nullptr;
if (memory_invalidation_callback_handle_ != nullptr) {
memory_->UnregisterPhysicalMemoryInvalidationCallback(
memory_invalidation_callback_handle_);
memory_invalidation_callback_handle_ = nullptr;
}
if (device_queue_) {
@@ -411,7 +413,7 @@ void TextureCache::WatchTexture(Texture* texture) {
texture->is_watched = true;
}
memory_->WatchPhysicalMemoryWrite(address, size);
memory_->EnablePhysicalMemoryAccessCallbacks(address, size, true, false);
}
void TextureCache::TextureTouched(Texture* texture) {
@@ -428,7 +430,7 @@ void TextureCache::TextureTouched(Texture* texture) {
texture->pending_invalidation = true;
}
std::pair<uint32_t, uint32_t> TextureCache::MemoryWriteCallback(
std::pair<uint32_t, uint32_t> TextureCache::MemoryInvalidationCallback(
uint32_t physical_address_start, uint32_t length, bool exact_range) {
global_critical_region_.Acquire();
if (watched_textures_.empty()) {
@@ -468,11 +470,11 @@ std::pair<uint32_t, uint32_t> TextureCache::MemoryWriteCallback(
return std::make_pair(previous_end, next_start - previous_end);
}
std::pair<uint32_t, uint32_t> TextureCache::MemoryWriteCallbackThunk(
std::pair<uint32_t, uint32_t> TextureCache::MemoryInvalidationCallbackThunk(
void* context_ptr, uint32_t physical_address_start, uint32_t length,
bool exact_range) {
return reinterpret_cast<TextureCache*>(context_ptr)
->MemoryWriteCallback(physical_address_start, length, exact_range);
->MemoryInvalidationCallback(physical_address_start, length, exact_range);
}
TextureCache::Texture* TextureCache::DemandResolveTexture(

6
src/xenia/gpu/vulkan/texture_cache.h
View File

@@ -147,9 +147,9 @@ class TextureCache {
void WatchTexture(Texture* texture);
void TextureTouched(Texture* texture);
std::pair<uint32_t, uint32_t> MemoryWriteCallback(
std::pair<uint32_t, uint32_t> MemoryInvalidationCallback(
uint32_t physical_address_start, uint32_t length, bool exact_range);
static std::pair<uint32_t, uint32_t> MemoryWriteCallbackThunk(
static std::pair<uint32_t, uint32_t> MemoryInvalidationCallbackThunk(
void* context_ptr, uint32_t physical_address_start, uint32_t length,
bool exact_range);
@@ -220,7 +220,7 @@ class TextureCache {
std::unordered_map<uint64_t, Sampler*> samplers_;
std::list<Texture*> pending_delete_textures_;
void* physical_write_watch_handle_ = nullptr;
void* memory_invalidation_callback_handle_ = nullptr;
xe::global_critical_region global_critical_region_;
std::list<WatchedTexture> watched_textures_;