fabi/Xenia-Canary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[D3D12] Experimental write watch implementation for shared memory

Browse Source
This commit is contained in:
Triang3l
2018-09-24 23:18:16 +03:00
parent 005e590c92
commit 6e36101b42
7 changed files with 424 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

266
src/xenia/cpu/mmio_handler.cc
View File

@@ -9,6 +9,9 @@
#include "xenia/cpu/mmio_handler.h"
#include <algorithm>
#include <cstring>
#include "xenia/base/assert.h"
#include "xenia/base/byte_order.h"
#include "xenia/base/exception_handler.h"
@@ -40,6 +43,20 @@ std::unique_ptr<MMIOHandler> MMIOHandler::Install(uint8_t* virtual_membase,
return handler;
}
MMIOHandler::MMIOHandler(uint8_t* virtual_membase, uint8_t* physical_membase,
uint8_t* membase_end)
: virtual_membase_(virtual_membase),
physical_membase_(physical_membase),
memory_end_(membase_end) {
system_page_size_log2_ = xe::log2_ceil(uint32_t(xe::memory::page_size()));
uint32_t physical_page_count = (512 * 1024 * 1024) >> system_page_size_log2_;
physical_write_watched_pages_.resize(physical_page_count >> 4);
assert_true(physical_write_watched_pages_.size() != 0);
std::memset(physical_write_watched_pages_.data(), 0,
physical_write_watched_pages_.size() * sizeof(uint64_t));
}
MMIOHandler::~MMIOHandler() {
ExceptionHandler::Uninstall(ExceptionCallbackThunk, this);
@@ -214,72 +231,154 @@ void MMIOHandler::CancelAccessWatch(uintptr_t watch_handle) {
delete entry;
}
void MMIOHandler::SetGlobalPhysicalAccessWatch(
GlobalAccessWatchCallback callback, void* callback_context) {
void* MMIOHandler::RegisterPhysicalWriteWatch(
PhysicalWriteWatchCallback callback, void* callback_context) {
PhysicalWriteWatchEntry* entry = new PhysicalWriteWatchEntry;
entry->callback = callback;
entry->callback_context = callback_context;
auto lock = global_critical_region_.Acquire();
global_physical_watch_callback_ = callback;
global_physical_watch_callback_context_ = callback_context;
physical_write_watches_.push_back(entry);
return entry;
}
void MMIOHandler::ProtectPhysicalMemory(uint32_t physical_address,
uint32_t length, WatchType type,
bool protect_host_access) {
uint32_t base_address = physical_address & 0x1FFFFFFF;
// Can only protect sizes matching system page size.
// This means we need to round up, which will cause spurious access
// violations and invalidations.
// TODO(benvanik): only invalidate if actually within the region?
length =
xe::round_up(length + (base_address % uint32_t(xe::memory::page_size())),
uint32_t(xe::memory::page_size()));
base_address = base_address - (base_address % xe::memory::page_size());
auto page_access = memory::PageAccess::kNoAccess;
switch (type) {
case kWatchInvalid:
page_access = memory::PageAccess::kReadWrite;
break;
case kWatchWrite:
page_access = memory::PageAccess::kReadOnly;
break;
case kWatchReadWrite:
page_access = memory::PageAccess::kNoAccess;
break;
default:
assert_unhandled_case(type);
break;
}
// Protect the range under all address spaces.
if (protect_host_access) {
memory::Protect(physical_membase_ + base_address, length, page_access,
nullptr);
}
memory::Protect(virtual_membase_ + 0xA0000000 + base_address, length,
page_access, nullptr);
memory::Protect(virtual_membase_ + 0xC0000000 + base_address, length,
page_access, nullptr);
memory::Protect(virtual_membase_ + 0xE0000000 + base_address, length,
page_access, nullptr);
}
void MMIOHandler::UnprotectPhysicalMemory(uint32_t physical_address,
uint32_t length,
bool unprotect_host_access) {
ProtectPhysicalMemory(physical_address, length, kWatchInvalid,
unprotect_host_access);
}
void MMIOHandler::InvalidateRange(uint32_t physical_address, size_t length) {
void MMIOHandler::UnregisterPhysicalWriteWatch(void* watch_handle) {
auto entry = reinterpret_cast<PhysicalWriteWatchEntry*>(watch_handle);
auto lock = global_critical_region_.Acquire();
auto it = std::find(physical_write_watches_.begin(),
physical_write_watches_.end(), entry);
assert_false(it == physical_write_watches_.end());
if (it != physical_write_watches_.end()) {
physical_write_watches_.erase(it);
}
delete entry;
}
void MMIOHandler::ProtectAndWatchPhysicalMemory(
uint32_t physical_address_and_heap, uint32_t length) {
// Bits to set in 16-bit blocks to mark that the pages are protected.
uint64_t block_heap_mask;
if (physical_address_and_heap >= 0xE0000000) {
block_heap_mask = 0x4444444444444444ull;
} else if (physical_address_and_heap >= 0xC0000000) {
block_heap_mask = 0x2222222222222222ull;
} else if (physical_address_and_heap >= 0xA0000000) {
block_heap_mask = 0x1111111111111111ull;
} else {
assert_always();
return;
}
uint32_t heap_relative_address = physical_address_and_heap & 0x1FFFFFFF;
length = std::min(length, 0x20000000u - heap_relative_address);
if (length == 0) {
return;
}
uint32_t page_first = heap_relative_address >> system_page_size_log2_;
uint32_t page_last =
(heap_relative_address + length - 1) >> system_page_size_log2_;
uint32_t block_first = page_first >> 4;
uint32_t block_last = page_last >> 4;
auto lock = global_critical_region_.Acquire();
// Set the bits indicating that the pages are watched and access violations
// there are intentional.
for (uint32_t i = block_first; i <= block_last; ++i) {
uint64_t block_set_bits = block_heap_mask;
if (i == block_first) {
block_set_bits &= ~((1ull << ((page_first & 15) * 4)) - 1);
}
if (i == block_last && (page_last & 15) != 15) {
block_set_bits &= (1ull << (((page_last & 15) + 1) * 4)) - 1;
}
physical_write_watched_pages_[i] |= block_set_bits;
}
// Protect only in one range (due to difficulties synchronizing protection
// levels between those ranges).
memory::Protect(virtual_membase_ + (physical_address_and_heap & ~0x1FFFFFFF) +
(page_first << system_page_size_log2_),
(page_last - page_first + 1) << system_page_size_log2_,
memory::PageAccess::kReadOnly, nullptr);
}
void MMIOHandler::InvalidateRange(uint32_t physical_address_and_heap,
uint32_t length, bool unprotect) {
uint32_t heap_relative_address = physical_address_and_heap & 0x1FFFFFFF;
length = std::min(length, 0x20000000u - heap_relative_address);
if (length == 0) {
return;
}
auto lock = global_critical_region_.Acquire();
// Trigger the new (per-page) watches and unwatch the pages.
if (physical_address_and_heap >= 0xA0000000) {
uint32_t heap_address = physical_address_and_heap & ~0x1FFFFFFF;
uint64_t heap_bit;
if (heap_address >= 0xE0000000) {
heap_bit = 1 << 2;
} else if (heap_address >= 0xC0000000) {
heap_bit = 1 << 1;
} else {
heap_bit = 1 << 0;
}
uint32_t page_first = heap_relative_address >> system_page_size_log2_;
uint32_t page_last =
(heap_relative_address + length - 1) >> system_page_size_log2_;
uint32_t range_start = UINT32_MAX;
for (uint32_t i = page_first; i <= page_last; ++i) {
uint64_t page_heap_bit = heap_bit << ((i & 15) * 4);
if (physical_write_watched_pages_[i >> 4] & page_heap_bit) {
if (range_start == UINT32_MAX) {
range_start = i;
}
physical_write_watched_pages_[i >> 4] &= ~page_heap_bit;
} else {
if (range_start != UINT32_MAX) {
for (auto it = physical_write_watches_.begin();
it != physical_write_watches_.end(); ++it) {
auto entry = *it;
entry->callback(entry->callback_context, range_start, i - 1);
}
if (unprotect) {
memory::Protect(virtual_membase_ + heap_address +
(range_start << system_page_size_log2_),
(i - range_start) << system_page_size_log2_,
xe::memory::PageAccess::kReadWrite, nullptr);
}
range_start = UINT32_MAX;
}
}
}
if (range_start != UINT32_MAX) {
for (auto it = physical_write_watches_.begin();
it != physical_write_watches_.end(); ++it) {
auto entry = *it;
entry->callback(entry->callback_context, range_start, page_last);
if (unprotect) {
memory::Protect(virtual_membase_ + heap_address +
(range_start << system_page_size_log2_),
(page_last - range_start + 1)
<< system_page_size_log2_,
xe::memory::PageAccess::kReadWrite, nullptr);
}
}
}
}
// Trigger the legacy (per-range) watches.
for (auto it = access_watches_.begin(); it != access_watches_.end();) {
auto entry = *it;
if ((entry->address <= physical_address &&
entry->address + entry->length > physical_address) ||
(entry->address >= physical_address &&
entry->address < physical_address + length)) {
if ((entry->address <= heap_relative_address &&
entry->address + entry->length > heap_relative_address) ||
(entry->address >= heap_relative_address &&
entry->address < heap_relative_address + length)) {
// This watch lies within the range. End it.
FireAccessWatch(entry);
it = access_watches_.erase(it);
@@ -316,17 +415,43 @@ bool MMIOHandler::IsRangeWatched(uint32_t physical_address, size_t length) {
return false;
}
bool MMIOHandler::CheckAccessWatch(uint32_t physical_address) {
auto lock = global_critical_region_.Acquire();
bool MMIOHandler::CheckAccessWatch(uint32_t physical_address,
uint32_t heap_address) {
bool hit = false;
if (global_physical_watch_callback_ != nullptr) {
if (global_physical_watch_callback_(global_physical_watch_callback_context_,
physical_address)) {
return true;
// Trigger new (per-page) access watches.
if (heap_address >= 0xA0000000) {
uint32_t page_index = physical_address >> system_page_size_log2_;
// Check the watch only for the virtual memory mapping it was triggered in,
// because as guest protection levels may be different for different
// mappings of the physical memory, it's difficult to synchronize protection
// between the mappings.
uint64_t heap_bit;
if (heap_address >= 0xE0000000) {
heap_bit = 1 << 2;
} else if (heap_address >= 0xC0000000) {
heap_bit = 1 << 1;
} else {
heap_bit = 1 << 0;
}
heap_bit <<= (page_index & 15) * 4;
if (physical_write_watched_pages_[page_index >> 4] & heap_bit) {
hit = true;
memory::Protect(virtual_membase_ + heap_address +
(page_index << system_page_size_log2_),
size_t(1) << system_page_size_log2_,
xe::memory::PageAccess::kReadWrite, nullptr);
physical_write_watched_pages_[page_index >> 4] &= ~heap_bit;
for (auto it = physical_write_watches_.begin();
it != physical_write_watches_.end(); ++it) {
auto entry = *it;
entry->callback(entry->callback_context, page_index, page_index);
}
}
}
bool hit = false;
// Trigger legacy (per-range) access watches.
auto lock = global_critical_region_.Acquire();
for (auto it = access_watches_.begin(); it != access_watches_.end();) {
auto entry = *it;
if (entry->address <= physical_address &&
@@ -539,14 +664,17 @@ bool MMIOHandler::ExceptionCallback(Exception* ex) {
}
if (!range) {
auto fault_address = reinterpret_cast<uint8_t*>(ex->fault_address());
uint32_t guest_address = 0;
uint32_t guest_address, guest_heap_address;
if (fault_address >= virtual_membase_ &&
fault_address < physical_membase_) {
// Faulting on a virtual address.
guest_address = static_cast<uint32_t>(ex->fault_address()) & 0x1FFFFFFF;
guest_heap_address =
static_cast<uint32_t>(ex->fault_address()) & ~0x1FFFFFFF;
} else {
// Faulting on a physical address.
guest_address = static_cast<uint32_t>(ex->fault_address());
guest_heap_address = 0;
}
// HACK: Recheck if the pages are still protected (race condition - another
@@ -564,7 +692,9 @@ bool MMIOHandler::ExceptionCallback(Exception* ex) {
// Access is not found within any range, so fail and let the caller handle
// it (likely by aborting).
return CheckAccessWatch(guest_address);
// TODO(Triang3l): Don't call for the host physical memory view when legacy
// watches are removed.
return CheckAccessWatch(guest_address, guest_heap_address);
}
auto rip = ex->pc();

90
src/xenia/cpu/mmio_handler.h
View File

@@ -30,7 +30,9 @@ typedef void (*MMIOWriteCallback)(void* ppc_context, void* callback_context,
uint32_t addr, uint32_t value);
typedef void (*AccessWatchCallback)(void* context_ptr, void* data_ptr,
uint32_t address);
typedef bool (*GlobalAccessWatchCallback)(void* context_ptr, uint32_t address);
typedef void (*PhysicalWriteWatchCallback)(void* context_ptr,
uint32_t page_first,
uint32_t page_last);
struct MMIORange {
uint32_t address;
@@ -70,22 +72,69 @@ class MMIOHandler {
// either written to or read from, depending on the watch type. These fire as
// soon as a read/write happens, and only fire once.
// These watches may be spuriously fired if memory is accessed nearby.
// TODO(Triang3l): This is legacy currently used only to support the old
// Vulkan graphics layer. Remove and use WatchPhysicalMemoryWrite instead.
uintptr_t AddPhysicalAccessWatch(uint32_t guest_address, size_t length,
WatchType type, AccessWatchCallback callback,
void* callback_context, void* callback_data);
void CancelAccessWatch(uintptr_t watch_handle);
void SetGlobalPhysicalAccessWatch(GlobalAccessWatchCallback callback,
void* callback_context);
void ProtectPhysicalMemory(uint32_t physical_address, uint32_t length,
WatchType type, bool protect_host_access);
void UnprotectPhysicalMemory(uint32_t physical_address, uint32_t length,
bool unprotect_host_access);
// Physical memory write watching, allowing subsystems to invalidate cached
// data that depends on memory contents.
//
// Placing a watch simply marks the pages (of the system page size) as
// watched, individual watched ranges (or which specific subscribers are
// watching specific pages) are not stored. Because of this, callbacks may be
// triggered multiple times for a single range, and for any watched page every
// registered callbacks is triggered. This is a very simple one-shot method
// for use primarily for cache invalidation - there may be spurious firing,
// for example, if the game only changes the protection level without writing
// anything.
//
// A range of pages can be watched at any time, but pages are only unwatched
// when watches are triggered (since multiple subscribers can depend on the
// same memory, and one subscriber shouldn't interfere with another).
//
// Callbacks can be triggered for one page (if the guest just stores words) or
// for multiple pages (for file reading, protection level changes).
//
// Only guest physical memory mappings are watched - the host-only mapping is
// not protected so it can be used to bypass the write protection (for file
// reads, for example - in this case, watches are triggered manually).
//
// Ranges passed to ProtectAndWatchPhysicalMemory must not contain read-only
// or inaccessible pages - this must be checked externally! Otherwise the MMIO
// handler will make them read-only, but when a read is attempted, it will
// make them read-write!
//
// IMPORTANT NOTE: When a watch is triggered, the watched page is unprotected
// ***ONLY IN THE HEAP WHERE THE ADDRESS IS LOCATED***! Since different
// virtual memory mappings of physical memory can have different protection
// levels for the same pages, and watches must not be placed on read-only or
// totally inaccessible pages, there are significant difficulties with
// synchronizing all the three ranges.
//
// TODO(Triang3l): Allow the callbacks to unwatch regions larger than one page
// (for instance, 64 KB) so there are less access violations. All callbacks
// must agree to unwatch larger ranges because in some cases (like regions
// near the locations that render targets have been resolved to) it is
// necessary to invalidate only a single page and none more.
void* RegisterPhysicalWriteWatch(PhysicalWriteWatchCallback callback,
void* callback_context);
void UnregisterPhysicalWriteWatch(void* watch_handle);
// Force-protects the range in ***ONE SPECIFIC HEAP***, either 0xA0000000,
// 0xC0000000 or 0xE0000000, depending on the higher bits of the address.
void ProtectAndWatchPhysicalMemory(uint32_t physical_address_and_heap,
uint32_t length);
// Fires and clears any access watches that overlap this range.
void InvalidateRange(uint32_t physical_address, size_t length);
// Fires and clears any write watches that overlap this range in one heap.
// Unprotecting can be inhibited if this is called right before applying
// different protection to the same range.
void InvalidateRange(uint32_t physical_address_and_heap, uint32_t length,
bool unprotect = true);
// Returns true if /all/ of this range is watched.
// TODO(Triang3l): Remove when legacy watches are removed.
bool IsRangeWatched(uint32_t physical_address, size_t length);
protected:
@@ -98,18 +147,22 @@ class MMIOHandler {
void* callback_data;
};
struct PhysicalWriteWatchEntry {
PhysicalWriteWatchCallback callback;
void* callback_context;
};
MMIOHandler(uint8_t* virtual_membase, uint8_t* physical_membase,
uint8_t* membase_end)
: virtual_membase_(virtual_membase),
physical_membase_(physical_membase),
memory_end_(membase_end) {}
uint8_t* membase_end);
static bool ExceptionCallbackThunk(Exception* ex, void* data);
bool ExceptionCallback(Exception* ex);
void FireAccessWatch(AccessWatchEntry* entry);
void ClearAccessWatch(AccessWatchEntry* entry);
bool CheckAccessWatch(uint32_t guest_address);
bool CheckAccessWatch(uint32_t guest_address, uint32_t guest_heap_address);
uint32_t system_page_size_log2_;
uint8_t* virtual_membase_;
uint8_t* physical_membase_;
@@ -120,8 +173,13 @@ class MMIOHandler {
xe::global_critical_region global_critical_region_;
// TODO(benvanik): data structure magic.
std::list<AccessWatchEntry*> access_watches_;
GlobalAccessWatchCallback global_physical_watch_callback_ = nullptr;
void* global_physical_watch_callback_context_;
std::vector<PhysicalWriteWatchEntry*> physical_write_watches_;
// For each page, there are 4 bits (16 pages in each word):
// 0 - whether the page is protected in A0000000.
// 1 - whether the page is protected in C0000000.
// 2 - whether the page is protected in E0000000.
// 3 - unused, always zero.
std::vector<uint64_t> physical_write_watched_pages_;
static MMIOHandler* global_handler_;
};