diff --git a/src/xenia/gpu/shared_memory.cc b/src/xenia/gpu/shared_memory.cc index 162d5b2d6..3afb39120 100644 --- a/src/xenia/gpu/shared_memory.cc +++ b/src/xenia/gpu/shared_memory.cc @@ -341,6 +341,8 @@ void SharedMemory::MakeRangeValid(uint32_t start, uint32_t length, } if (memory_invalidation_callback_handle_) { + // A page that isn't writable here gets no watch. A later guest + // protect-to-writable invalidates it so its writes are still caught. memory().EnablePhysicalMemoryAccessCallbacks( valid_page_first << page_size_log2_, (valid_page_last - valid_page_first + 1) << page_size_log2_, true, diff --git a/src/xenia/memory.cc b/src/xenia/memory.cc index ab79c94b0..aedf47d92 100644 --- a/src/xenia/memory.cc +++ b/src/xenia/memory.cc @@ -113,20 +113,6 @@ static inline bool ShouldSkipHostCommit(const BaseHeap& heap) { return false; } -xe::memory::PageAccess ToPageAccess(uint32_t protect) { - // Write-combine memory is CPU-writable (for GPU uploads) - bool is_writable = - (protect & kMemoryProtectWrite) || (protect & kMemoryProtectWriteCombine); - - if ((protect & kMemoryProtectRead) && !is_writable) { - return xe::memory::PageAccess::kReadOnly; - } else if ((protect & kMemoryProtectRead) && is_writable) { - return xe::memory::PageAccess::kReadWrite; - } else { - return xe::memory::PageAccess::kNoAccess; - } -} - void RandomizeMemory(void* range_start, uint32_t size) { if (!cvars::scribble_heap) { return; @@ -670,6 +656,30 @@ void Memory::UnregisterPhysicalMemoryInvalidationCallback( delete entry; } +void* Memory::RegisterPhysicalMemoryReadCallback( + PhysicalMemoryReadCallback callback, void* callback_context) { + auto entry = new std::pair( + callback, callback_context); + auto lock = global_critical_region_.Acquire(); + physical_memory_read_callbacks_.push_back(entry); + return entry; +} + +void Memory::UnregisterPhysicalMemoryReadCallback(void* callback_handle) { + auto entry = reinterpret_cast*>( + callback_handle); + { + auto lock = global_critical_region_.Acquire(); + auto it = std::find(physical_memory_read_callbacks_.begin(), + physical_memory_read_callbacks_.end(), entry); + assert_true(it != physical_memory_read_callbacks_.end()); + if (it != physical_memory_read_callbacks_.end()) { + physical_memory_read_callbacks_.erase(it); + } + } + delete entry; +} + void Memory::EnablePhysicalMemoryAccessCallbacks( uint32_t physical_address, uint32_t length, bool enable_invalidation_notifications, bool enable_data_providers) { @@ -1938,7 +1948,8 @@ bool PhysicalHeap::Decommit(uint32_t address, uint32_t size) { } // Not caring about the contents anymore. - TriggerCallbacks(std::move(global_lock), address, size, true, true); + TriggerCallbacks(std::move(global_lock), address, size, true, true, true, + true); return BaseHeap::Decommit(address, size); } @@ -1963,7 +1974,7 @@ bool PhysicalHeap::Release(uint32_t base_address, uint32_t* out_region_size) { uint32_t region_size; if (QuerySize(base_address, ®ion_size)) { TriggerCallbacks(std::move(global_lock), base_address, region_size, true, - true); + true, true, true); } return BaseHeap::Release(base_address, out_region_size); @@ -1974,9 +1985,14 @@ bool PhysicalHeap::Protect(uint32_t address, uint32_t size, uint32_t protect, auto global_lock = global_critical_region_.Acquire(); // Only invalidate if making writable again, for simplicity - not when simply - // marking some range as immutable, for instance. - if (protect & kMemoryProtectWrite) { - TriggerCallbacks(std::move(global_lock), address, size, true, true, false); + // marking some range as immutable, for instance. The guest is announcing a + // write rather than reacting to a fault, so invalidate even with no watch + // armed: a range that was read-only when it was last uploaded never got one, + // and would otherwise stay stale for as long as the guest keeps it read-only + // outside of its own writes. + if (IsWritableProtect(protect)) { + TriggerCallbacks(std::move(global_lock), address, size, true, true, false, + true); } if (!parent_heap_->Protect(GetPhysicalAddress(address), size, protect, @@ -1992,8 +2008,6 @@ void PhysicalHeap::EnableAccessCallbacks(uint32_t physical_address, uint32_t length, bool enable_invalidation_notifications, bool enable_data_providers) { - // TODO(Triang3l): Implement data providers. - assert_false(enable_data_providers); if (!enable_invalidation_notifications && !enable_data_providers) { return; } @@ -2031,15 +2045,20 @@ void PhysicalHeap::EnableAccessCallbacks(uint32_t physical_address, auto global_lock = global_critical_region_.Acquire(); if (enable_invalidation_notifications) { - EnableAccessCallbacksInner(system_page_first, system_page_last, - protect_access); + if (enable_data_providers) { + EnableAccessCallbacksInner(system_page_first, + system_page_last, protect_access); + } else { + EnableAccessCallbacksInner(system_page_first, + system_page_last, protect_access); + } } else { - EnableAccessCallbacksInner(system_page_first, system_page_last, - protect_access); + EnableAccessCallbacksInner(system_page_first, system_page_last, + protect_access); } } -template +template XE_NOINLINE void PhysicalHeap::EnableAccessCallbacksInner( const uint32_t system_page_first, const uint32_t system_page_last, xe::memory::PageAccess protect_access) XE_RESTRICT { @@ -2047,19 +2066,11 @@ XE_NOINLINE void PhysicalHeap::EnableAccessCallbacksInner( uint32_t protect_system_page_first = UINT32_MAX; SystemPageFlagsBlock* XE_RESTRICT sys_page_flags = system_page_flags_.data(); - PageEntry* XE_RESTRICT page_table_ptr = page_table_.data(); // chrispy: a lot of time is spent in this loop, and i think some of the work // may be avoidable and repetitive profiling shows quite a bit of time spent // in this loop, but very little spent actually calling Protect uint32_t i = system_page_first; - - uint32_t first_guest_page = SystemPagenumToGuestPagenum(system_page_first); - uint32_t last_guest_page = SystemPagenumToGuestPagenum(system_page_last); - - uint32_t guest_one = SystemPagenumToGuestPagenum(1); - - uint32_t system_one = GuestPagenumToSystemPagenum(1); for (; i <= system_page_last; ++i) { // Check if need to enable callbacks for the page and raise its protection. // @@ -2091,24 +2102,30 @@ XE_NOINLINE void PhysicalHeap::EnableAccessCallbacksInner( uint64_t page_flags_bit = uint64_t(1) << (i & 63); #endif - uint32_t guest_page_number = SystemPagenumToGuestPagenum(i); - xe::memory::PageAccess current_page_access = - ToPageAccess(page_table_ptr[guest_page_number].current_protect); + xe::memory::PageAccess current_page_access = SystemPageGuestAccess(i); bool protect_system_page = false; // Don't do anything with inaccessible pages - don't protect, don't enable // callbacks - because real access violations are needed there. And don't // enable invalidation notifications for read-only pages for the same // reason. if (current_page_access != xe::memory::PageAccess::kNoAccess) { - // TODO(Triang3l): Enable data providers. if constexpr (enable_invalidation_notifications) { if (current_page_access != xe::memory::PageAccess::kReadOnly && (page_flags_block.notify_on_invalidation & page_flags_bit) == 0) { - // TODO(Triang3l): Check if data providers are already enabled. - // If data providers are already enabled for the page, it has even - // stricter protection. - protect_system_page = true; page_flags_block.notify_on_invalidation |= page_flags_bit; + // A read-watched page is already protected no-access, stricter than + // read-only, so don't loosen it here. + if ((page_flags_block.notify_on_read & page_flags_bit) == 0) { + protect_system_page = true; + } + } + } + if constexpr (enable_data_providers) { + // Read watches protect the page no-access, so an accessible page not + // yet read-watched needs protecting. + if ((page_flags_block.notify_on_read & page_flags_bit) == 0) { + protect_system_page = true; + page_flags_block.notify_on_read |= page_flags_bit; } } } @@ -2137,13 +2154,8 @@ XE_NOINLINE void PhysicalHeap::EnableAccessCallbacksInner( } bool PhysicalHeap::TriggerCallbacks( global_unique_lock_type global_lock_locked_once, uint32_t virtual_address, - uint32_t length, bool is_write, bool unwatch_exact_range, bool unprotect) { - // TODO(Triang3l): Support read watches. - assert_true(is_write); - if (!is_write) { - return false; - } - + uint32_t length, bool is_write, bool unwatch_exact_range, bool unprotect, + bool invalidate_unwatched) { if (virtual_address < heap_base_) { if (heap_base_ - virtual_address >= length) { return false; @@ -2170,10 +2182,90 @@ bool PhysicalHeap::TriggerCallbacks( uint32_t block_index_first = system_page_first >> 6; uint32_t block_index_last = system_page_last >> 6; + // Read watches: the first read of a no-access-armed page notifies the read + // callbacks, then the page is downgraded and unwatched so the access + // proceeds. A write to such a page is handled by the write path below, which + // also clears the read watch. + if (!is_write) { + bool any_read_watched = false; + for (uint32_t i = block_index_first; i <= block_index_last; ++i) { + uint64_t block = system_page_flags_[i].notify_on_read; + if (i == block_index_first) { + block &= ~((uint64_t(1) << (system_page_first & 63)) - 1); + } + if (i == block_index_last && (system_page_last & 63) != 63) { + block &= (uint64_t(1) << ((system_page_last & 63) + 1)) - 1; + } + if (block) { + any_read_watched = true; + break; + } + } + if (!any_read_watched) { + // No read watch here. If the guest mapping is accessible this is a race + // with another thread that cleared the watch, so retry. If it is + // no-access it is a genuine access violation, so propagate. Checked via + // the page table to stay signal safe. + return SystemPageGuestAccess(system_page_first) != + xe::memory::PageAccess::kNoAccess; + } + uint32_t physical_address_offset = GetPhysicalAddress(heap_base_); + uint32_t physical_address_start = + xe::sat_sub(system_page_first << system_page_shift_, + host_address_offset()) + + physical_address_offset; + uint32_t physical_length = std::min( + xe::sat_sub( + (system_page_last << system_page_shift_) + system_page_size_, + host_address_offset()) + + physical_address_offset - physical_address_start, + heap_size_ - (physical_address_start - physical_address_offset)); + for (auto read_callback : memory_->physical_memory_read_callbacks_) { + read_callback->first(read_callback->second, physical_address_start, + physical_length); + } + // Downgrade each read-watched page so the access proceeds. Keep it + // read-only if it also has a write watch, otherwise restore the guest + // protection. Then clear the read watch. + if (unprotect) { + uint8_t* protect_base = membase_ + heap_base_; + for (uint32_t i = system_page_first; i <= system_page_last; ++i) { + uint64_t bit = uint64_t(1) << (i & 63); + SystemPageFlagsBlock& flags = system_page_flags_[i >> 6]; + if (!(flags.notify_on_read & bit)) { + continue; + } + xe::memory::PageAccess guest_access = SystemPageGuestAccess(i); + xe::memory::PageAccess target; + if (guest_access == xe::memory::PageAccess::kNoAccess) { + target = xe::memory::PageAccess::kNoAccess; + } else if (flags.notify_on_invalidation & bit) { + target = xe::memory::PageAccess::kReadOnly; + } else { + target = guest_access; + } + xe::memory::Protect(protect_base + (i << system_page_shift_), + system_page_size_, target); + } + } + for (uint32_t i = block_index_first; i <= block_index_last; ++i) { + uint64_t mask = 0; + if (i == block_index_first) { + mask |= (uint64_t(1) << (system_page_first & 63)) - 1; + } + if (i == block_index_last && (system_page_last & 63) != 63) { + mask |= ~((uint64_t(1) << ((system_page_last & 63) + 1)) - 1); + } + system_page_flags_[i].notify_on_read &= mask; + } + return true; + } + // Check if watching any page, whether need to call the callback at all. bool any_watched = false; for (uint32_t i = block_index_first; i <= block_index_last; ++i) { - uint64_t block = system_page_flags_[i].notify_on_invalidation; + uint64_t block = system_page_flags_[i].notify_on_invalidation | + system_page_flags_[i].notify_on_read; if (i == block_index_first) { block &= ~((uint64_t(1) << (system_page_first & 63)) - 1); } @@ -2185,14 +2277,17 @@ bool PhysicalHeap::TriggerCallbacks( break; } } - if (!any_watched) { + if (!any_watched && !invalidate_unwatched) { // No watches on this page — another thread already cleared them (race // condition between the fault firing and acquiring the lock). Return true // so the faulting instruction retries; the page is now unprotected and the // access will succeed. This is the signal-safe equivalent of the // QueryProtect check in the non-Linux path of - // MMIOHandler::ExceptionCallback. - return true; + // MMIOHandler::ExceptionCallback. If the guest doesn't permit the write + // either, retrying it faults forever, so report it unhandled and let the + // violation surface. + return SystemPageGuestAccess(system_page_first) == + xe::memory::PageAccess::kReadWrite; } // Trigger callbacks. @@ -2262,15 +2357,14 @@ bool PhysicalHeap::TriggerCallbacks( uint8_t* protect_base = membase_ + heap_base_; uint32_t unprotect_system_page_first = UINT32_MAX; for (uint32_t i = system_page_first; i <= system_page_last; ++i) { - // Check if need to allow writing to this page. - bool unprotect_page = (system_page_flags_[i >> 6].notify_on_invalidation & - (uint64_t(1) << (i & 63))) != 0; + // Check if need to allow writing to this page. Read-watched pages are + // unprotected here too so a write to one doesn't re-fault. + bool unprotect_page = + ((system_page_flags_[i >> 6].notify_on_invalidation | + system_page_flags_[i >> 6].notify_on_read) & + (uint64_t(1) << (i & 63))) != 0; if (unprotect_page) { - uint32_t guest_page_number = - xe::sat_sub(i << system_page_shift_, host_address_offset()) >> - page_size_shift_; - if (ToPageAccess(page_table_[guest_page_number].current_protect) != - xe::memory::PageAccess::kReadWrite) { + if (SystemPageGuestAccess(i) != xe::memory::PageAccess::kReadWrite) { unprotect_page = false; } } @@ -2298,7 +2392,8 @@ bool PhysicalHeap::TriggerCallbacks( } } - // Mark pages as not write-watched. + // Mark pages as not write-watched. Unprotected pages are readable and + // writable now, so clear the read watch too. for (uint32_t i = block_index_first; i <= block_index_last; ++i) { uint64_t mask = 0; if (i == block_index_first) { @@ -2308,6 +2403,7 @@ bool PhysicalHeap::TriggerCallbacks( mask |= ~((uint64_t(1) << ((system_page_last & 63) + 1)) - 1); } system_page_flags_[i].notify_on_invalidation &= mask; + system_page_flags_[i].notify_on_read &= mask; } return true; diff --git a/src/xenia/memory.h b/src/xenia/memory.h index bd9519a40..6a76078f1 100644 --- a/src/xenia/memory.h +++ b/src/xenia/memory.h @@ -58,6 +58,25 @@ enum MemoryProtectFlag : uint32_t { kMemoryProtectNoAccess = 0, }; +// Write-combine memory is CPU-writable for GPU uploads, so treat it as writable +// alongside the regular write flag. +inline bool IsWritableProtect(uint32_t protect) { + return (protect & kMemoryProtectWrite) || + (protect & kMemoryProtectWriteCombine); +} + +inline xe::memory::PageAccess ToPageAccess(uint32_t protect) { + bool is_writable = IsWritableProtect(protect); + + if ((protect & kMemoryProtectRead) && !is_writable) { + return xe::memory::PageAccess::kReadOnly; + } else if ((protect & kMemoryProtectRead) && is_writable) { + return xe::memory::PageAccess::kReadWrite; + } else { + return xe::memory::PageAccess::kNoAccess; + } +} + // Equivalent to the Win32 MEMORY_BASIC_INFORMATION struct. struct HeapAllocationInfo { // A pointer to the base address of the region of pages. @@ -285,16 +304,20 @@ class PhysicalHeap : public BaseHeap { void EnableAccessCallbacks(uint32_t physical_address, uint32_t length, bool enable_invalidation_notifications, bool enable_data_providers); - template + template XE_NOINLINE void EnableAccessCallbacksInner( const uint32_t system_page_first, const uint32_t system_page_last, xe::memory::PageAccess protect_access) XE_RESTRICT; - // Returns true if any page in the range was watched. + // Returns true if any page in the range was watched. With + // invalidate_unwatched the callbacks are raised even when no watch is armed + // for a caller that knows the range is about to change rather than one + // reacting to a fault. bool TriggerCallbacks(global_unique_lock_type global_lock_locked_once, uint32_t virtual_address, uint32_t length, bool is_write, bool unwatch_exact_range, - bool unprotect = true); + bool unprotect = true, + bool invalidate_unwatched = false); uint32_t GetPhysicalAddress(uint32_t address) const; @@ -303,11 +326,41 @@ class PhysicalHeap : public BaseHeap { page_size_shift_; } - uint32_t GuestPagenumToSystemPagenum(uint32_t num) { - num <<= page_size_shift_; - num += host_address_offset(); - num >>= system_page_shift_; - return num; + // The most permissive guest access of the guest pages a system page covers. + // Protection has system page granularity and BaseHeap::Protect resolves a + // system page the same way, so anything deciding on protection has to agree + // with it - the host page can be larger than the guest page. Inline, called + // per page in the arming loop. + xe::memory::PageAccess SystemPageGuestAccess( + uint32_t system_page_number) const { + uint32_t offset = host_address_offset(); + uint32_t system_base = system_page_number << system_page_shift_; + uint32_t system_last = system_base + (system_page_size_ - 1); + if (system_last < offset) { + return xe::memory::PageAccess::kNoAccess; + } + uint32_t guest_page_first = + system_base > offset ? (system_base - offset) >> page_size_shift_ : 0; + uint32_t guest_page_count = uint32_t(page_table_.size()); + if (guest_page_first >= guest_page_count) { + return xe::memory::PageAccess::kNoAccess; + } + uint32_t guest_page_last = (system_last - offset) >> page_size_shift_; + if (guest_page_last >= guest_page_count) { + guest_page_last = guest_page_count - 1; + } + xe::memory::PageAccess access = xe::memory::PageAccess::kNoAccess; + for (uint32_t i = guest_page_first; i <= guest_page_last; ++i) { + xe::memory::PageAccess page_access = + ToPageAccess(page_table_[i].current_protect); + if (page_access == xe::memory::PageAccess::kReadWrite) { + return xe::memory::PageAccess::kReadWrite; + } + if (page_access == xe::memory::PageAccess::kReadOnly) { + access = xe::memory::PageAccess::kReadOnly; + } + } + return access; } protected: @@ -321,7 +374,10 @@ class PhysicalHeap : public BaseHeap { struct SystemPageFlagsBlock { // Whether writing to each page should result trigger invalidation // callbacks. - uint64_t notify_on_invalidation; + uint64_t notify_on_invalidation = 0; + // Whether the first access of each page triggers read callbacks. These + // pages are protected no-access. The watch is one-shot, cleared on access. + uint64_t notify_on_read = 0; }; // Protected by global_critical_region. Flags for each 64 system pages, // interleaved as blocks, so bit scan can be used to quickly extract ranges. @@ -501,6 +557,18 @@ class Memory { // RegisterPhysicalMemoryInvalidationCallback. void UnregisterPhysicalMemoryInvalidationCallback(void* callback_handle); + // Called on the first CPU access of a page armed as a read watch (via + // EnablePhysicalMemoryAccessCallbacks with data providers). The page is + // downgraded and unwatched right after, so it fires once per arm. Must be + // lightweight and non-blocking. It runs in the fault handler under the global + // critical region. + typedef void (*PhysicalMemoryReadCallback)(void* context_ptr, + uint32_t physical_address_start, + uint32_t length); + void* RegisterPhysicalMemoryReadCallback(PhysicalMemoryReadCallback callback, + void* callback_context); + void UnregisterPhysicalMemoryReadCallback(void* callback_handle); + // Enables physical memory access callbacks for the specified memory range, // snapped to system page boundaries. void EnablePhysicalMemoryAccessCallbacks( @@ -510,8 +578,7 @@ class Memory { // Forces triggering of watch callbacks for a virtual address range if pages // are watched there and unwatching them. Returns whether any page was // watched. Must be called with global critical region locking depth of 1. - // TODO(Triang3l): Implement data providers - this is why locking depth of 1 - // will be required in the future. + // The invalidation and read callbacks run under that single lock hold. bool TriggerPhysicalMemoryCallbacks( global_unique_lock_type global_lock_locked_once, uint32_t virtual_address, uint32_t length, bool is_write, bool unwatch_exact_range, @@ -613,6 +680,8 @@ class Memory { xe::global_critical_region global_critical_region_; std::vector*> physical_memory_invalidation_callbacks_; + std::vector*> + physical_memory_read_callbacks_; }; } // namespace xe