fabi/Xenia-Canary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Memory] Rework guest access resolution and unwatched invalidation

Browse Source 
This is a combination of three different Edge commits.

Guest page access is resolved with system page granularity so that anything deciding on protection now takes permissive access of every guest page a system page covers, which matters when the host is larger than the guest.

Access violations: Write faults on pages with no watch armed are only reported handled if guest mapping allows the write.

Invalidation of unwatched ranges when made writable or freed: Decommit, Release and Protect-to-writable (including write-combine) raise invalidation callbacks even with no watch armed.

Co-authored-by: Herman S. <429230+has207@users.noreply.github.com>
This commit is contained in:
goldislead
2026-08-01 14:30:00 -07:00
committed by Radosław Gliński
parent 59c08cd462
commit aed81ca93a
3 changed files with 240 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/xenia/gpu/shared_memory.cc
View File

@@ -341,6 +341,8 @@ void SharedMemory::MakeRangeValid(uint32_t start, uint32_t length,
}
if (memory_invalidation_callback_handle_) {
// A page that isn't writable here gets no watch. A later guest
// protect-to-writable invalidates it so its writes are still caught.
memory().EnablePhysicalMemoryAccessCallbacks(
valid_page_first << page_size_log2_,
(valid_page_last - valid_page_first + 1) << page_size_log2_, true,

220
src/xenia/memory.cc
View File

@@ -113,20 +113,6 @@ static inline bool ShouldSkipHostCommit(const BaseHeap& heap) {
return false;
}
xe::memory::PageAccess ToPageAccess(uint32_t protect) {
// Write-combine memory is CPU-writable (for GPU uploads)
bool is_writable =
(protect & kMemoryProtectWrite) || (protect & kMemoryProtectWriteCombine);
if ((protect & kMemoryProtectRead) && !is_writable) {
return xe::memory::PageAccess::kReadOnly;
} else if ((protect & kMemoryProtectRead) && is_writable) {
return xe::memory::PageAccess::kReadWrite;
} else {
return xe::memory::PageAccess::kNoAccess;
}
}
void RandomizeMemory(void* range_start, uint32_t size) {
if (!cvars::scribble_heap) {
return;
@@ -670,6 +656,30 @@ void Memory::UnregisterPhysicalMemoryInvalidationCallback(
delete entry;
}
void* Memory::RegisterPhysicalMemoryReadCallback(
PhysicalMemoryReadCallback callback, void* callback_context) {
auto entry = new std::pair<PhysicalMemoryReadCallback, void*>(
callback, callback_context);
auto lock = global_critical_region_.Acquire();
physical_memory_read_callbacks_.push_back(entry);
return entry;
}
void Memory::UnregisterPhysicalMemoryReadCallback(void* callback_handle) {
auto entry = reinterpret_cast<std::pair<PhysicalMemoryReadCallback, void*>*>(
callback_handle);
{
auto lock = global_critical_region_.Acquire();
auto it = std::find(physical_memory_read_callbacks_.begin(),
physical_memory_read_callbacks_.end(), entry);
assert_true(it != physical_memory_read_callbacks_.end());
if (it != physical_memory_read_callbacks_.end()) {
physical_memory_read_callbacks_.erase(it);
}
}
delete entry;
}
void Memory::EnablePhysicalMemoryAccessCallbacks(
uint32_t physical_address, uint32_t length,
bool enable_invalidation_notifications, bool enable_data_providers) {
@@ -1938,7 +1948,8 @@ bool PhysicalHeap::Decommit(uint32_t address, uint32_t size) {
}
// Not caring about the contents anymore.
TriggerCallbacks(std::move(global_lock), address, size, true, true);
TriggerCallbacks(std::move(global_lock), address, size, true, true, true,
true);
return BaseHeap::Decommit(address, size);
}
@@ -1963,7 +1974,7 @@ bool PhysicalHeap::Release(uint32_t base_address, uint32_t* out_region_size) {
uint32_t region_size;
if (QuerySize(base_address, &region_size)) {
TriggerCallbacks(std::move(global_lock), base_address, region_size, true,
true);
true, true, true);
}
return BaseHeap::Release(base_address, out_region_size);
@@ -1974,9 +1985,14 @@ bool PhysicalHeap::Protect(uint32_t address, uint32_t size, uint32_t protect,
auto global_lock = global_critical_region_.Acquire();
// Only invalidate if making writable again, for simplicity - not when simply
// marking some range as immutable, for instance.
if (protect & kMemoryProtectWrite) {
TriggerCallbacks(std::move(global_lock), address, size, true, true, false);
// marking some range as immutable, for instance. The guest is announcing a
// write rather than reacting to a fault, so invalidate even with no watch
// armed: a range that was read-only when it was last uploaded never got one,
// and would otherwise stay stale for as long as the guest keeps it read-only
// outside of its own writes.
if (IsWritableProtect(protect)) {
TriggerCallbacks(std::move(global_lock), address, size, true, true, false,
true);
}
if (!parent_heap_->Protect(GetPhysicalAddress(address), size, protect,
@@ -1992,8 +2008,6 @@ void PhysicalHeap::EnableAccessCallbacks(uint32_t physical_address,
uint32_t length,
bool enable_invalidation_notifications,
bool enable_data_providers) {
// TODO(Triang3l): Implement data providers.
assert_false(enable_data_providers);
if (!enable_invalidation_notifications && !enable_data_providers) {
return;
}
@@ -2031,15 +2045,20 @@ void PhysicalHeap::EnableAccessCallbacks(uint32_t physical_address,
auto global_lock = global_critical_region_.Acquire();
if (enable_invalidation_notifications) {
EnableAccessCallbacksInner<true>(system_page_first, system_page_last,
protect_access);
if (enable_data_providers) {
EnableAccessCallbacksInner<true, true>(system_page_first,
system_page_last, protect_access);
} else {
EnableAccessCallbacksInner<true, false>(system_page_first,
system_page_last, protect_access);
}
} else {
EnableAccessCallbacksInner<false>(system_page_first, system_page_last,
protect_access);
EnableAccessCallbacksInner<false, true>(system_page_first, system_page_last,
protect_access);
}
}
template <bool enable_invalidation_notifications>
template <bool enable_invalidation_notifications, bool enable_data_providers>
XE_NOINLINE void PhysicalHeap::EnableAccessCallbacksInner(
const uint32_t system_page_first, const uint32_t system_page_last,
xe::memory::PageAccess protect_access) XE_RESTRICT {
@@ -2047,19 +2066,11 @@ XE_NOINLINE void PhysicalHeap::EnableAccessCallbacksInner(
uint32_t protect_system_page_first = UINT32_MAX;
SystemPageFlagsBlock* XE_RESTRICT sys_page_flags = system_page_flags_.data();
PageEntry* XE_RESTRICT page_table_ptr = page_table_.data();
// chrispy: a lot of time is spent in this loop, and i think some of the work
// may be avoidable and repetitive profiling shows quite a bit of time spent
// in this loop, but very little spent actually calling Protect
uint32_t i = system_page_first;
uint32_t first_guest_page = SystemPagenumToGuestPagenum(system_page_first);
uint32_t last_guest_page = SystemPagenumToGuestPagenum(system_page_last);
uint32_t guest_one = SystemPagenumToGuestPagenum(1);
uint32_t system_one = GuestPagenumToSystemPagenum(1);
for (; i <= system_page_last; ++i) {
// Check if need to enable callbacks for the page and raise its protection.
//
@@ -2091,24 +2102,30 @@ XE_NOINLINE void PhysicalHeap::EnableAccessCallbacksInner(
uint64_t page_flags_bit = uint64_t(1) << (i & 63);
#endif
uint32_t guest_page_number = SystemPagenumToGuestPagenum(i);
xe::memory::PageAccess current_page_access =
ToPageAccess(page_table_ptr[guest_page_number].current_protect);
xe::memory::PageAccess current_page_access = SystemPageGuestAccess(i);
bool protect_system_page = false;
// Don't do anything with inaccessible pages - don't protect, don't enable
// callbacks - because real access violations are needed there. And don't
// enable invalidation notifications for read-only pages for the same
// reason.
if (current_page_access != xe::memory::PageAccess::kNoAccess) {
// TODO(Triang3l): Enable data providers.
if constexpr (enable_invalidation_notifications) {
if (current_page_access != xe::memory::PageAccess::kReadOnly &&
(page_flags_block.notify_on_invalidation & page_flags_bit) == 0) {
// TODO(Triang3l): Check if data providers are already enabled.
// If data providers are already enabled for the page, it has even
// stricter protection.
protect_system_page = true;
page_flags_block.notify_on_invalidation |= page_flags_bit;
// A read-watched page is already protected no-access, stricter than
// read-only, so don't loosen it here.
if ((page_flags_block.notify_on_read & page_flags_bit) == 0) {
protect_system_page = true;
}
}
}
if constexpr (enable_data_providers) {
// Read watches protect the page no-access, so an accessible page not
// yet read-watched needs protecting.
if ((page_flags_block.notify_on_read & page_flags_bit) == 0) {
protect_system_page = true;
page_flags_block.notify_on_read |= page_flags_bit;
}
}
}
@@ -2137,13 +2154,8 @@ XE_NOINLINE void PhysicalHeap::EnableAccessCallbacksInner(
}
bool PhysicalHeap::TriggerCallbacks(
global_unique_lock_type global_lock_locked_once, uint32_t virtual_address,
uint32_t length, bool is_write, bool unwatch_exact_range, bool unprotect) {
// TODO(Triang3l): Support read watches.
assert_true(is_write);
if (!is_write) {
return false;
}
uint32_t length, bool is_write, bool unwatch_exact_range, bool unprotect,
bool invalidate_unwatched) {
if (virtual_address < heap_base_) {
if (heap_base_ - virtual_address >= length) {
return false;
@@ -2170,10 +2182,90 @@ bool PhysicalHeap::TriggerCallbacks(
uint32_t block_index_first = system_page_first >> 6;
uint32_t block_index_last = system_page_last >> 6;
// Read watches: the first read of a no-access-armed page notifies the read
// callbacks, then the page is downgraded and unwatched so the access
// proceeds. A write to such a page is handled by the write path below, which
// also clears the read watch.
if (!is_write) {
bool any_read_watched = false;
for (uint32_t i = block_index_first; i <= block_index_last; ++i) {
uint64_t block = system_page_flags_[i].notify_on_read;
if (i == block_index_first) {
block &= ~((uint64_t(1) << (system_page_first & 63)) - 1);
}
if (i == block_index_last && (system_page_last & 63) != 63) {
block &= (uint64_t(1) << ((system_page_last & 63) + 1)) - 1;
}
if (block) {
any_read_watched = true;
break;
}
}
if (!any_read_watched) {
// No read watch here. If the guest mapping is accessible this is a race
// with another thread that cleared the watch, so retry. If it is
// no-access it is a genuine access violation, so propagate. Checked via
// the page table to stay signal safe.
return SystemPageGuestAccess(system_page_first) !=
xe::memory::PageAccess::kNoAccess;
}
uint32_t physical_address_offset = GetPhysicalAddress(heap_base_);
uint32_t physical_address_start =
xe::sat_sub(system_page_first << system_page_shift_,
host_address_offset()) +
physical_address_offset;
uint32_t physical_length = std::min(
xe::sat_sub(
(system_page_last << system_page_shift_) + system_page_size_,
host_address_offset()) +
physical_address_offset - physical_address_start,
heap_size_ - (physical_address_start - physical_address_offset));
for (auto read_callback : memory_->physical_memory_read_callbacks_) {
read_callback->first(read_callback->second, physical_address_start,
physical_length);
}
// Downgrade each read-watched page so the access proceeds. Keep it
// read-only if it also has a write watch, otherwise restore the guest
// protection. Then clear the read watch.
if (unprotect) {
uint8_t* protect_base = membase_ + heap_base_;
for (uint32_t i = system_page_first; i <= system_page_last; ++i) {
uint64_t bit = uint64_t(1) << (i & 63);
SystemPageFlagsBlock& flags = system_page_flags_[i >> 6];
if (!(flags.notify_on_read & bit)) {
continue;
}
xe::memory::PageAccess guest_access = SystemPageGuestAccess(i);
xe::memory::PageAccess target;
if (guest_access == xe::memory::PageAccess::kNoAccess) {
target = xe::memory::PageAccess::kNoAccess;
} else if (flags.notify_on_invalidation & bit) {
target = xe::memory::PageAccess::kReadOnly;
} else {
target = guest_access;
}
xe::memory::Protect(protect_base + (i << system_page_shift_),
system_page_size_, target);
}
}
for (uint32_t i = block_index_first; i <= block_index_last; ++i) {
uint64_t mask = 0;
if (i == block_index_first) {
mask |= (uint64_t(1) << (system_page_first & 63)) - 1;
}
if (i == block_index_last && (system_page_last & 63) != 63) {
mask |= ~((uint64_t(1) << ((system_page_last & 63) + 1)) - 1);
}
system_page_flags_[i].notify_on_read &= mask;
}
return true;
}
// Check if watching any page, whether need to call the callback at all.
bool any_watched = false;
for (uint32_t i = block_index_first; i <= block_index_last; ++i) {
uint64_t block = system_page_flags_[i].notify_on_invalidation;
uint64_t block = system_page_flags_[i].notify_on_invalidation |
system_page_flags_[i].notify_on_read;
if (i == block_index_first) {
block &= ~((uint64_t(1) << (system_page_first & 63)) - 1);
}
@@ -2185,14 +2277,17 @@ bool PhysicalHeap::TriggerCallbacks(
break;
}
}
if (!any_watched) {
if (!any_watched && !invalidate_unwatched) {
// No watches on this page — another thread already cleared them (race
// condition between the fault firing and acquiring the lock). Return true
// so the faulting instruction retries; the page is now unprotected and the
// access will succeed. This is the signal-safe equivalent of the
// QueryProtect check in the non-Linux path of
// MMIOHandler::ExceptionCallback.
return true;
// MMIOHandler::ExceptionCallback. If the guest doesn't permit the write
// either, retrying it faults forever, so report it unhandled and let the
// violation surface.
return SystemPageGuestAccess(system_page_first) ==
xe::memory::PageAccess::kReadWrite;
}
// Trigger callbacks.
@@ -2262,15 +2357,14 @@ bool PhysicalHeap::TriggerCallbacks(
uint8_t* protect_base = membase_ + heap_base_;
uint32_t unprotect_system_page_first = UINT32_MAX;
for (uint32_t i = system_page_first; i <= system_page_last; ++i) {
// Check if need to allow writing to this page.
bool unprotect_page = (system_page_flags_[i >> 6].notify_on_invalidation &
(uint64_t(1) << (i & 63))) != 0;
// Check if need to allow writing to this page. Read-watched pages are
// unprotected here too so a write to one doesn't re-fault.
bool unprotect_page =
((system_page_flags_[i >> 6].notify_on_invalidation |
system_page_flags_[i >> 6].notify_on_read) &
(uint64_t(1) << (i & 63))) != 0;
if (unprotect_page) {
uint32_t guest_page_number =
xe::sat_sub(i << system_page_shift_, host_address_offset()) >>
page_size_shift_;
if (ToPageAccess(page_table_[guest_page_number].current_protect) !=
xe::memory::PageAccess::kReadWrite) {
if (SystemPageGuestAccess(i) != xe::memory::PageAccess::kReadWrite) {
unprotect_page = false;
}
}
@@ -2298,7 +2392,8 @@ bool PhysicalHeap::TriggerCallbacks(
}
}
// Mark pages as not write-watched.
// Mark pages as not write-watched. Unprotected pages are readable and
// writable now, so clear the read watch too.
for (uint32_t i = block_index_first; i <= block_index_last; ++i) {
uint64_t mask = 0;
if (i == block_index_first) {
@@ -2308,6 +2403,7 @@ bool PhysicalHeap::TriggerCallbacks(
mask |= ~((uint64_t(1) << ((system_page_last & 63) + 1)) - 1);
}
system_page_flags_[i].notify_on_invalidation &= mask;
system_page_flags_[i].notify_on_read &= mask;
}
return true;

91
src/xenia/memory.h
View File

@@ -58,6 +58,25 @@ enum MemoryProtectFlag : uint32_t {
kMemoryProtectNoAccess = 0,
};
// Write-combine memory is CPU-writable for GPU uploads, so treat it as writable
// alongside the regular write flag.
inline bool IsWritableProtect(uint32_t protect) {
return (protect & kMemoryProtectWrite) ||
(protect & kMemoryProtectWriteCombine);
}
inline xe::memory::PageAccess ToPageAccess(uint32_t protect) {
bool is_writable = IsWritableProtect(protect);
if ((protect & kMemoryProtectRead) && !is_writable) {
return xe::memory::PageAccess::kReadOnly;
} else if ((protect & kMemoryProtectRead) && is_writable) {
return xe::memory::PageAccess::kReadWrite;
} else {
return xe::memory::PageAccess::kNoAccess;
}
}
// Equivalent to the Win32 MEMORY_BASIC_INFORMATION struct.
struct HeapAllocationInfo {
// A pointer to the base address of the region of pages.
@@ -285,16 +304,20 @@ class PhysicalHeap : public BaseHeap {
void EnableAccessCallbacks(uint32_t physical_address, uint32_t length,
bool enable_invalidation_notifications,
bool enable_data_providers);
template <bool enable_invalidation_notifications>
template <bool enable_invalidation_notifications, bool enable_data_providers>
XE_NOINLINE void EnableAccessCallbacksInner(
const uint32_t system_page_first, const uint32_t system_page_last,
xe::memory::PageAccess protect_access) XE_RESTRICT;
// Returns true if any page in the range was watched.
// Returns true if any page in the range was watched. With
// invalidate_unwatched the callbacks are raised even when no watch is armed
// for a caller that knows the range is about to change rather than one
// reacting to a fault.
bool TriggerCallbacks(global_unique_lock_type global_lock_locked_once,
uint32_t virtual_address, uint32_t length,
bool is_write, bool unwatch_exact_range,
bool unprotect = true);
bool unprotect = true,
bool invalidate_unwatched = false);
uint32_t GetPhysicalAddress(uint32_t address) const;
@@ -303,11 +326,41 @@ class PhysicalHeap : public BaseHeap {
page_size_shift_;
}
uint32_t GuestPagenumToSystemPagenum(uint32_t num) {
num <<= page_size_shift_;
num += host_address_offset();
num >>= system_page_shift_;
return num;
// The most permissive guest access of the guest pages a system page covers.
// Protection has system page granularity and BaseHeap::Protect resolves a
// system page the same way, so anything deciding on protection has to agree
// with it - the host page can be larger than the guest page. Inline, called
// per page in the arming loop.
xe::memory::PageAccess SystemPageGuestAccess(
uint32_t system_page_number) const {
uint32_t offset = host_address_offset();
uint32_t system_base = system_page_number << system_page_shift_;
uint32_t system_last = system_base + (system_page_size_ - 1);
if (system_last < offset) {
return xe::memory::PageAccess::kNoAccess;
}
uint32_t guest_page_first =
system_base > offset ? (system_base - offset) >> page_size_shift_ : 0;
uint32_t guest_page_count = uint32_t(page_table_.size());
if (guest_page_first >= guest_page_count) {
return xe::memory::PageAccess::kNoAccess;
}
uint32_t guest_page_last = (system_last - offset) >> page_size_shift_;
if (guest_page_last >= guest_page_count) {
guest_page_last = guest_page_count - 1;
}
xe::memory::PageAccess access = xe::memory::PageAccess::kNoAccess;
for (uint32_t i = guest_page_first; i <= guest_page_last; ++i) {
xe::memory::PageAccess page_access =
ToPageAccess(page_table_[i].current_protect);
if (page_access == xe::memory::PageAccess::kReadWrite) {
return xe::memory::PageAccess::kReadWrite;
}
if (page_access == xe::memory::PageAccess::kReadOnly) {
access = xe::memory::PageAccess::kReadOnly;
}
}
return access;
}
protected:
@@ -321,7 +374,10 @@ class PhysicalHeap : public BaseHeap {
struct SystemPageFlagsBlock {
// Whether writing to each page should result trigger invalidation
// callbacks.
uint64_t notify_on_invalidation;
uint64_t notify_on_invalidation = 0;
// Whether the first access of each page triggers read callbacks. These
// pages are protected no-access. The watch is one-shot, cleared on access.
uint64_t notify_on_read = 0;
};
// Protected by global_critical_region. Flags for each 64 system pages,
// interleaved as blocks, so bit scan can be used to quickly extract ranges.
@@ -501,6 +557,18 @@ class Memory {
// RegisterPhysicalMemoryInvalidationCallback.
void UnregisterPhysicalMemoryInvalidationCallback(void* callback_handle);
// Called on the first CPU access of a page armed as a read watch (via
// EnablePhysicalMemoryAccessCallbacks with data providers). The page is
// downgraded and unwatched right after, so it fires once per arm. Must be
// lightweight and non-blocking. It runs in the fault handler under the global
// critical region.
typedef void (*PhysicalMemoryReadCallback)(void* context_ptr,
uint32_t physical_address_start,
uint32_t length);
void* RegisterPhysicalMemoryReadCallback(PhysicalMemoryReadCallback callback,
void* callback_context);
void UnregisterPhysicalMemoryReadCallback(void* callback_handle);
// Enables physical memory access callbacks for the specified memory range,
// snapped to system page boundaries.
void EnablePhysicalMemoryAccessCallbacks(
@@ -510,8 +578,7 @@ class Memory {
// Forces triggering of watch callbacks for a virtual address range if pages
// are watched there and unwatching them. Returns whether any page was
// watched. Must be called with global critical region locking depth of 1.
// TODO(Triang3l): Implement data providers - this is why locking depth of 1
// will be required in the future.
// The invalidation and read callbacks run under that single lock hold.
bool TriggerPhysicalMemoryCallbacks(
global_unique_lock_type global_lock_locked_once, uint32_t virtual_address,
uint32_t length, bool is_write, bool unwatch_exact_range,
@@ -613,6 +680,8 @@ class Memory {
xe::global_critical_region global_critical_region_;
std::vector<std::pair<PhysicalMemoryInvalidationCallback, void*>*>
physical_memory_invalidation_callbacks_;
std::vector<std::pair<PhysicalMemoryReadCallback, void*>*>
physical_memory_read_callbacks_;
};
} // namespace xe