[Memory] Add free block tracker to BaseHeap for O(log n) allocation
Replace linear page_table_ scans in AllocRange with a std::map-based free block index that tracks contiguous free regions. Insertions now coalesce with adjacent blocks on release and AllocFixed uses the targeted tracker for pure reserves and falls back to a full rebuild for mixed-state commits. Also fixing PhysicalHeap leaking parent memory on child allocation failure, Reset() not restoring unreserved_page_count_ and some incorrect method names in PhysicalHeap error messages
This commit is contained in:
Herman S.
388
src/xenia/base/testing/heap_test.cc
Normal file
388
src/xenia/base/testing/heap_test.cc
Normal file
@@ -0,0 +1,388 @@
|
||||
/**
|
||||
******************************************************************************
|
||||
* Xenia : Xbox 360 Emulator Research Project *
|
||||
******************************************************************************
|
||||
* Copyright 2026 Ben Vanik. All rights reserved. *
|
||||
* Released under the BSD license - see LICENSE in the root for more details. *
|
||||
******************************************************************************
|
||||
*/
|
||||
|
||||
#include "third_party/catch/include/catch.hpp"
|
||||
|
||||
#include "xenia/memory.h"
|
||||
|
||||
namespace xe {
|
||||
namespace test {
|
||||
|
||||
// Helper to create a VirtualHeap for testing without a full Memory instance.
|
||||
// Uses reserve-only allocations to avoid needing real host memory mappings.
|
||||
class TestHeap {
|
||||
public:
|
||||
TestHeap(uint32_t heap_base, uint32_t heap_size, uint32_t page_size) {
|
||||
heap_.Initialize(nullptr, nullptr, HeapType::kGuestXex, heap_base,
|
||||
heap_size, page_size);
|
||||
}
|
||||
|
||||
~TestHeap() {
|
||||
// Don't call Dispose — it tries to DeallocFixed on nullptr membase.
|
||||
}
|
||||
|
||||
VirtualHeap& heap() { return heap_; }
|
||||
|
||||
// Reserve-only allocation (skips host memory commit).
|
||||
bool Alloc(uint32_t size, uint32_t alignment, bool top_down,
|
||||
uint32_t* out_address) {
|
||||
return heap_.AllocRange(heap_.heap_base(),
|
||||
heap_.heap_base() + heap_.heap_size() - 1, size,
|
||||
alignment, kMemoryAllocationReserve,
|
||||
kMemoryProtectRead, top_down, out_address);
|
||||
}
|
||||
|
||||
bool AllocRange(uint32_t low, uint32_t high, uint32_t size,
|
||||
uint32_t alignment, bool top_down, uint32_t* out_address) {
|
||||
return heap_.AllocRange(low, high, size, alignment,
|
||||
kMemoryAllocationReserve, kMemoryProtectRead,
|
||||
top_down, out_address);
|
||||
}
|
||||
|
||||
bool AllocFixed(uint32_t base_address, uint32_t size) {
|
||||
return heap_.AllocFixed(base_address, size, heap_.page_size(),
|
||||
kMemoryAllocationReserve, kMemoryProtectRead);
|
||||
}
|
||||
|
||||
bool Release(uint32_t address) { return heap_.Release(address); }
|
||||
|
||||
uint32_t unreserved_page_count() const {
|
||||
return heap_.unreserved_page_count();
|
||||
}
|
||||
|
||||
uint32_t total_page_count() const { return heap_.total_page_count(); }
|
||||
|
||||
private:
|
||||
VirtualHeap heap_;
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Basic allocation and release
|
||||
// ============================================================================
|
||||
|
||||
TEST_CASE("heap_alloc_basic", "[heap]") {
|
||||
// 1MB heap, 4KB pages = 256 pages
|
||||
TestHeap h(0x80000000, 0x100000, 0x1000);
|
||||
REQUIRE(h.total_page_count() == 256);
|
||||
REQUIRE(h.unreserved_page_count() == 256);
|
||||
|
||||
uint32_t addr = 0;
|
||||
REQUIRE(h.Alloc(0x1000, 0x1000, false, &addr));
|
||||
REQUIRE(addr == 0x80000000);
|
||||
REQUIRE(h.unreserved_page_count() == 255);
|
||||
|
||||
REQUIRE(h.Alloc(0x2000, 0x1000, false, &addr));
|
||||
REQUIRE(addr == 0x80001000);
|
||||
REQUIRE(h.unreserved_page_count() == 253);
|
||||
}
|
||||
|
||||
TEST_CASE("heap_alloc_top_down", "[heap]") {
|
||||
TestHeap h(0x80000000, 0x100000, 0x1000);
|
||||
|
||||
uint32_t addr = 0;
|
||||
REQUIRE(h.Alloc(0x1000, 0x1000, true, &addr));
|
||||
// Top-down: should be at the highest aligned address.
|
||||
REQUIRE(addr == 0x800FF000);
|
||||
REQUIRE(h.unreserved_page_count() == 255);
|
||||
|
||||
REQUIRE(h.Alloc(0x2000, 0x1000, true, &addr));
|
||||
REQUIRE(addr == 0x800FD000);
|
||||
REQUIRE(h.unreserved_page_count() == 253);
|
||||
}
|
||||
|
||||
TEST_CASE("heap_alloc_release", "[heap]") {
|
||||
TestHeap h(0x80000000, 0x100000, 0x1000);
|
||||
|
||||
uint32_t addr1 = 0, addr2 = 0;
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &addr1));
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &addr2));
|
||||
REQUIRE(addr1 == 0x80000000);
|
||||
REQUIRE(addr2 == 0x80004000);
|
||||
REQUIRE(h.unreserved_page_count() == 248);
|
||||
|
||||
REQUIRE(h.Release(addr1));
|
||||
REQUIRE(h.unreserved_page_count() == 252);
|
||||
|
||||
REQUIRE(h.Release(addr2));
|
||||
REQUIRE(h.unreserved_page_count() == 256);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Coalescing
|
||||
// ============================================================================
|
||||
|
||||
TEST_CASE("heap_coalesce_adjacent_releases", "[heap]") {
|
||||
TestHeap h(0x80000000, 0x100000, 0x1000);
|
||||
|
||||
// Allocate 3 adjacent 4-page blocks.
|
||||
uint32_t a1 = 0, a2 = 0, a3 = 0;
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &a1));
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &a2));
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &a3));
|
||||
REQUIRE(a1 == 0x80000000);
|
||||
REQUIRE(a2 == 0x80004000);
|
||||
REQUIRE(a3 == 0x80008000);
|
||||
|
||||
// Release middle block, then adjacent blocks — should coalesce.
|
||||
REQUIRE(h.Release(a2));
|
||||
REQUIRE(h.Release(a1));
|
||||
REQUIRE(h.Release(a3));
|
||||
|
||||
// All freed. Now allocate a 12-page block — should succeed in the
|
||||
// coalesced free region.
|
||||
uint32_t big = 0;
|
||||
REQUIRE(h.Alloc(0xC000, 0x1000, false, &big));
|
||||
REQUIRE(big == 0x80000000);
|
||||
}
|
||||
|
||||
TEST_CASE("heap_coalesce_merge_before", "[heap]") {
|
||||
TestHeap h(0x80000000, 0x100000, 0x1000);
|
||||
|
||||
uint32_t a1 = 0, a2 = 0;
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &a1));
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &a2));
|
||||
|
||||
// Release first, then second — second should merge with first.
|
||||
REQUIRE(h.Release(a1));
|
||||
REQUIRE(h.Release(a2));
|
||||
|
||||
uint32_t big = 0;
|
||||
REQUIRE(h.Alloc(0x8000, 0x1000, false, &big));
|
||||
REQUIRE(big == 0x80000000);
|
||||
}
|
||||
|
||||
TEST_CASE("heap_coalesce_merge_after", "[heap]") {
|
||||
TestHeap h(0x80000000, 0x100000, 0x1000);
|
||||
|
||||
uint32_t a1 = 0, a2 = 0;
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &a1));
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &a2));
|
||||
|
||||
// Release second, then first — first should merge with second.
|
||||
REQUIRE(h.Release(a2));
|
||||
REQUIRE(h.Release(a1));
|
||||
|
||||
uint32_t big = 0;
|
||||
REQUIRE(h.Alloc(0x8000, 0x1000, false, &big));
|
||||
REQUIRE(big == 0x80000000);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Fragmentation resistance
|
||||
// ============================================================================
|
||||
|
||||
TEST_CASE("heap_fragmentation_reuse", "[heap]") {
|
||||
// 80KB heap, 4KB pages = 20 pages
|
||||
TestHeap h(0x80000000, 0x14000, 0x1000);
|
||||
|
||||
// Allocate 4 x 4-page blocks (uses 16 of 20 pages).
|
||||
uint32_t a[4];
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &a[i]));
|
||||
}
|
||||
REQUIRE(h.unreserved_page_count() == 4);
|
||||
|
||||
// Release alternating blocks to fragment.
|
||||
REQUIRE(h.Release(a[0])); // free pages 0-3
|
||||
REQUIRE(h.Release(a[2])); // free pages 8-11
|
||||
|
||||
// Can't allocate 5 pages (no single contiguous block of 5 in gaps).
|
||||
uint32_t fail_addr = 0;
|
||||
REQUIRE_FALSE(h.Alloc(0x5000, 0x1000, false, &fail_addr));
|
||||
|
||||
// Can allocate 4 pages (fits in either free gap).
|
||||
uint32_t ok_addr = 0;
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &ok_addr));
|
||||
REQUIRE(ok_addr == 0x80000000); // bottom-up, first fit.
|
||||
|
||||
// Release remaining to defragment.
|
||||
REQUIRE(h.Release(a[1]));
|
||||
REQUIRE(h.Release(a[3]));
|
||||
REQUIRE(h.Release(ok_addr));
|
||||
|
||||
// Now 20 pages should be available as one contiguous block.
|
||||
REQUIRE(h.unreserved_page_count() == 20);
|
||||
uint32_t big = 0;
|
||||
REQUIRE(h.Alloc(0xC000, 0x1000, false, &big));
|
||||
REQUIRE(big == 0x80000000);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Alignment
|
||||
// ============================================================================
|
||||
|
||||
TEST_CASE("heap_alloc_alignment", "[heap]") {
|
||||
// 1MB heap, 4KB pages
|
||||
TestHeap h(0x80000000, 0x100000, 0x1000);
|
||||
|
||||
// Allocate 1 page to offset the next allocation.
|
||||
uint32_t first = 0;
|
||||
REQUIRE(h.Alloc(0x1000, 0x1000, false, &first));
|
||||
REQUIRE(first == 0x80000000);
|
||||
|
||||
// Allocate with 64KB alignment — should skip to 0x80010000.
|
||||
uint32_t aligned = 0;
|
||||
REQUIRE(h.Alloc(0x1000, 0x10000, false, &aligned));
|
||||
REQUIRE((aligned % 0x10000) == 0);
|
||||
REQUIRE(aligned == 0x80010000);
|
||||
}
|
||||
|
||||
TEST_CASE("heap_alloc_alignment_top_down", "[heap]") {
|
||||
// 1MB heap, 4KB pages
|
||||
TestHeap h(0x80000000, 0x100000, 0x1000);
|
||||
|
||||
// Allocate 1 page at the top.
|
||||
uint32_t first = 0;
|
||||
REQUIRE(h.Alloc(0x1000, 0x1000, true, &first));
|
||||
REQUIRE(first == 0x800FF000);
|
||||
|
||||
// Allocate with 64KB alignment top-down — should align down.
|
||||
uint32_t aligned = 0;
|
||||
REQUIRE(h.Alloc(0x1000, 0x10000, true, &aligned));
|
||||
REQUIRE((aligned % 0x10000) == 0);
|
||||
REQUIRE(aligned == 0x800F0000);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// AllocFixed
|
||||
// ============================================================================
|
||||
|
||||
TEST_CASE("heap_alloc_fixed", "[heap]") {
|
||||
TestHeap h(0x80000000, 0x100000, 0x1000);
|
||||
|
||||
REQUIRE(h.AllocFixed(0x80010000, 0x4000));
|
||||
REQUIRE(h.unreserved_page_count() == 252);
|
||||
|
||||
// Allocate bottom-up — should get 0x80000000 (before the fixed alloc).
|
||||
uint32_t addr = 0;
|
||||
REQUIRE(h.Alloc(0x1000, 0x1000, false, &addr));
|
||||
REQUIRE(addr == 0x80000000);
|
||||
|
||||
// Allocating at the same fixed address should fail (already reserved).
|
||||
REQUIRE_FALSE(h.AllocFixed(0x80010000, 0x1000));
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Range allocation
|
||||
// ============================================================================
|
||||
|
||||
TEST_CASE("heap_alloc_range", "[heap]") {
|
||||
TestHeap h(0x80000000, 0x100000, 0x1000);
|
||||
|
||||
// Allocate in a specific sub-range.
|
||||
uint32_t addr = 0;
|
||||
REQUIRE(h.AllocRange(0x80080000, 0x800FFFFF, 0x4000, 0x1000, false, &addr));
|
||||
REQUIRE(addr >= 0x80080000);
|
||||
REQUIRE(addr + 0x4000 <= 0x80100000);
|
||||
}
|
||||
|
||||
TEST_CASE("heap_alloc_range_exhaustion", "[heap]") {
|
||||
// 64KB heap, 4KB pages = 16 pages
|
||||
TestHeap h(0x80000000, 0x10000, 0x1000);
|
||||
|
||||
// Fill the lower half.
|
||||
REQUIRE(h.AllocFixed(0x80000000, 0x8000));
|
||||
|
||||
// Try to allocate in the lower half — should fail.
|
||||
// Use page-aligned high address so xe::align doesn't extend the range.
|
||||
uint32_t addr = 0;
|
||||
REQUIRE_FALSE(
|
||||
h.AllocRange(0x80000000, 0x80007000, 0x1000, 0x1000, false, &addr));
|
||||
|
||||
// Allocate in the upper half — should succeed.
|
||||
REQUIRE(h.AllocRange(0x80008000, 0x8000F000, 0x1000, 0x1000, false, &addr));
|
||||
REQUIRE(addr >= 0x80008000);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Reset
|
||||
// ============================================================================
|
||||
|
||||
TEST_CASE("heap_reset", "[heap]") {
|
||||
TestHeap h(0x80000000, 0x100000, 0x1000);
|
||||
|
||||
// Fill most of the heap.
|
||||
uint32_t addr = 0;
|
||||
while (h.Alloc(0x1000, 0x1000, false, &addr)) {
|
||||
}
|
||||
|
||||
// Reset should restore all pages.
|
||||
h.heap().Reset();
|
||||
REQUIRE(h.unreserved_page_count() == 256);
|
||||
|
||||
// Should be able to allocate a large block again.
|
||||
REQUIRE(h.Alloc(0xF0000, 0x1000, false, &addr));
|
||||
REQUIRE(addr == 0x80000000);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Stress: many alloc/release cycles
|
||||
// ============================================================================
|
||||
|
||||
TEST_CASE("heap_stress_alloc_release", "[heap]") {
|
||||
// 272KB heap, 4KB pages = 68 pages (extra pages avoid off-by-one in range
|
||||
// check for full-heap-sized allocations).
|
||||
TestHeap h(0x80000000, 0x44000, 0x1000);
|
||||
|
||||
// Allocate 16 x 4-page blocks (uses 64 of 68 pages).
|
||||
uint32_t addrs[16];
|
||||
for (int i = 0; i < 16; ++i) {
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &addrs[i]));
|
||||
}
|
||||
REQUIRE(h.unreserved_page_count() == 4);
|
||||
|
||||
// Release all odd-indexed blocks.
|
||||
for (int i = 1; i < 16; i += 2) {
|
||||
REQUIRE(h.Release(addrs[i]));
|
||||
}
|
||||
REQUIRE(h.unreserved_page_count() == 36);
|
||||
|
||||
// Re-allocate 4-page blocks into the gaps.
|
||||
for (int i = 1; i < 16; i += 2) {
|
||||
REQUIRE(h.Alloc(0x4000, 0x1000, false, &addrs[i]));
|
||||
}
|
||||
REQUIRE(h.unreserved_page_count() == 4);
|
||||
|
||||
// Release everything.
|
||||
for (int i = 0; i < 16; ++i) {
|
||||
REQUIRE(h.Release(addrs[i]));
|
||||
}
|
||||
REQUIRE(h.unreserved_page_count() == 68);
|
||||
|
||||
// 64-page allocation should succeed after full release (coalesced).
|
||||
uint32_t full = 0;
|
||||
REQUIRE(h.Alloc(0x40000, 0x1000, false, &full));
|
||||
REQUIRE(full == 0x80000000);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// 64KB page heap (like v40000000)
|
||||
// ============================================================================
|
||||
|
||||
TEST_CASE("heap_64k_pages", "[heap]") {
|
||||
// 4MB heap, 64KB pages = 64 pages
|
||||
TestHeap h(0x40000000, 0x400000, 0x10000);
|
||||
REQUIRE(h.total_page_count() == 64);
|
||||
|
||||
uint32_t addr = 0;
|
||||
REQUIRE(h.Alloc(0x10000, 0x10000, false, &addr));
|
||||
REQUIRE(addr == 0x40000000);
|
||||
REQUIRE(h.unreserved_page_count() == 63);
|
||||
|
||||
REQUIRE(h.Alloc(0x20000, 0x10000, false, &addr));
|
||||
REQUIRE(addr == 0x40010000);
|
||||
REQUIRE(h.unreserved_page_count() == 61);
|
||||
|
||||
REQUIRE(h.Release(0x40000000));
|
||||
REQUIRE(h.Release(0x40010000));
|
||||
REQUIRE(h.unreserved_page_count() == 64);
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
} // namespace xe
|
||||
@@ -803,6 +803,10 @@ void BaseHeap::Initialize(Memory* memory, uint8_t* membase, HeapType heap_type,
|
||||
host_address_offset_ = host_address_offset;
|
||||
page_table_.resize(heap_size / page_size);
|
||||
unreserved_page_count_ = uint32_t(page_table_.size());
|
||||
|
||||
// Initialize free block tracker with a single block covering the entire heap.
|
||||
free_blocks_.clear();
|
||||
free_blocks_[0] = uint32_t(page_table_.size());
|
||||
}
|
||||
|
||||
void BaseHeap::Dispose() {
|
||||
@@ -816,6 +820,7 @@ void BaseHeap::Dispose() {
|
||||
page_number += page_entry.region_page_count;
|
||||
}
|
||||
}
|
||||
free_blocks_.clear();
|
||||
}
|
||||
|
||||
void BaseHeap::DumpMap() {
|
||||
@@ -938,14 +943,98 @@ bool BaseHeap::Restore(ByteStream* stream) {
|
||||
}
|
||||
}
|
||||
|
||||
RebuildFreeBlocks();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void BaseHeap::RebuildFreeBlocks() {
|
||||
free_blocks_.clear();
|
||||
uint32_t run_start = UINT32_MAX;
|
||||
for (uint32_t i = 0; i < uint32_t(page_table_.size()); ++i) {
|
||||
if (page_table_[i].state == 0) {
|
||||
if (run_start == UINT32_MAX) {
|
||||
run_start = i;
|
||||
}
|
||||
} else {
|
||||
if (run_start != UINT32_MAX) {
|
||||
free_blocks_[run_start] = i - run_start;
|
||||
run_start = UINT32_MAX;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (run_start != UINT32_MAX) {
|
||||
free_blocks_[run_start] = uint32_t(page_table_.size()) - run_start;
|
||||
}
|
||||
}
|
||||
|
||||
void BaseHeap::RemoveFreeBlock(uint32_t start_page, uint32_t page_count) {
|
||||
if (free_blocks_.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Find the free block that contains the allocated range.
|
||||
auto it = free_blocks_.upper_bound(start_page);
|
||||
if (it != free_blocks_.begin()) {
|
||||
--it;
|
||||
}
|
||||
|
||||
// Verify the block actually contains our range.
|
||||
uint32_t block_start = it->first;
|
||||
uint32_t block_count = it->second;
|
||||
uint32_t block_end = block_start + block_count;
|
||||
assert_true(start_page >= block_start &&
|
||||
start_page + page_count <= block_end);
|
||||
|
||||
free_blocks_.erase(it);
|
||||
|
||||
// Insert remnant before the allocated range.
|
||||
if (block_start < start_page) {
|
||||
free_blocks_[block_start] = start_page - block_start;
|
||||
}
|
||||
|
||||
// Insert remnant after the allocated range.
|
||||
uint32_t alloc_end = start_page + page_count;
|
||||
if (alloc_end < block_end) {
|
||||
free_blocks_[alloc_end] = block_end - alloc_end;
|
||||
}
|
||||
}
|
||||
|
||||
void BaseHeap::InsertFreeBlock(uint32_t start_page, uint32_t page_count) {
|
||||
uint32_t new_start = start_page;
|
||||
uint32_t new_count = page_count;
|
||||
|
||||
// Try to merge with block immediately after.
|
||||
auto it_after = free_blocks_.find(start_page + page_count);
|
||||
if (it_after != free_blocks_.end()) {
|
||||
new_count += it_after->second;
|
||||
free_blocks_.erase(it_after);
|
||||
}
|
||||
|
||||
// Try to merge with block immediately before.
|
||||
auto it_at = free_blocks_.lower_bound(start_page);
|
||||
if (it_at != free_blocks_.begin()) {
|
||||
auto it_before = std::prev(it_at);
|
||||
if (it_before->first + it_before->second == start_page) {
|
||||
new_start = it_before->first;
|
||||
new_count += it_before->second;
|
||||
free_blocks_.erase(it_before);
|
||||
}
|
||||
}
|
||||
|
||||
free_blocks_[new_start] = new_count;
|
||||
}
|
||||
|
||||
void BaseHeap::Reset() {
|
||||
// TODO(DrChat): protect pages.
|
||||
std::memset(page_table_.data(), 0, sizeof(PageEntry) * page_table_.size());
|
||||
unreserved_page_count_ = uint32_t(page_table_.size());
|
||||
// TODO(Triang3l): Remove access callbacks from pages if this is a physical
|
||||
// memory heap.
|
||||
|
||||
// Re-initialize free block tracker.
|
||||
free_blocks_.clear();
|
||||
free_blocks_[0] = uint32_t(page_table_.size());
|
||||
}
|
||||
|
||||
bool BaseHeap::Alloc(uint32_t size, uint32_t alignment,
|
||||
@@ -992,10 +1081,10 @@ bool BaseHeap::AllocFixed(uint32_t base_address, uint32_t size,
|
||||
|
||||
auto global_lock = global_critical_region_.Acquire();
|
||||
|
||||
// - If we are reserving the entire range requested must not be already
|
||||
// reserved.
|
||||
// - If we are reserving, the entire range must not be already reserved.
|
||||
// - If we are committing it's ok for pages within the range to already be
|
||||
// committed.
|
||||
const bool is_pure_reserve = allocation_type == kMemoryAllocationReserve;
|
||||
for (uint32_t page_number = start_page_number; page_number <= end_page_number;
|
||||
++page_number) {
|
||||
uint32_t state = page_table_[page_number].state;
|
||||
@@ -1042,6 +1131,7 @@ bool BaseHeap::AllocFixed(uint32_t base_address, uint32_t size,
|
||||
}
|
||||
|
||||
// Set page state.
|
||||
bool had_free_pages = false;
|
||||
for (uint32_t page_number = start_page_number; page_number <= end_page_number;
|
||||
++page_number) {
|
||||
auto& page_entry = page_table_[page_number];
|
||||
@@ -1053,11 +1143,25 @@ bool BaseHeap::AllocFixed(uint32_t base_address, uint32_t size,
|
||||
page_entry.allocation_protect = protect;
|
||||
page_entry.current_protect = protect;
|
||||
if (!(page_entry.state & kMemoryAllocationReserve)) {
|
||||
had_free_pages = true;
|
||||
unreserved_page_count_--;
|
||||
}
|
||||
page_entry.state = kMemoryAllocationReserve | allocation_type;
|
||||
}
|
||||
|
||||
// Update free block tracker if any pages transitioned from free.
|
||||
if (had_free_pages) {
|
||||
if (is_pure_reserve) {
|
||||
// Pure reserve: validation confirmed all pages were free, so the range
|
||||
// is within a single coalesced free block.
|
||||
RemoveFreeBlock(start_page_number, page_count);
|
||||
} else {
|
||||
// Mixed state (commit upgraded to reserve+commit): pages may span
|
||||
// multiple free blocks, rebuild from page_table_.
|
||||
RebuildFreeBlocks();
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
template <typename T>
|
||||
@@ -1094,88 +1198,85 @@ bool BaseHeap::AllocRange(uint32_t low_address, uint32_t high_address,
|
||||
|
||||
auto global_lock = global_critical_region_.Acquire();
|
||||
|
||||
// Find a free page range.
|
||||
// The base page must match the requested alignment, so we first scan for
|
||||
// a free aligned page and only then check for continuous free pages.
|
||||
// TODO(benvanik): optimized searching (free list buckets, bitmap, etc).
|
||||
// Find a free page range using the free block tracker.
|
||||
// The base page must match the requested alignment.
|
||||
uint32_t start_page_number = UINT_MAX;
|
||||
uint32_t end_page_number = UINT_MAX;
|
||||
// chrispy:todo, page_scan_stride is probably always a power of two...
|
||||
uint32_t page_scan_stride = alignment >> page_size_shift_;
|
||||
high_page_number =
|
||||
high_page_number - QuickMod(high_page_number, page_scan_stride);
|
||||
|
||||
if (top_down) {
|
||||
for (int64_t base_page_number =
|
||||
high_page_number - xe::round_up(page_count, page_scan_stride);
|
||||
base_page_number >= low_page_number;
|
||||
base_page_number -= page_scan_stride) {
|
||||
if (page_table_[base_page_number].state != 0) {
|
||||
// Base page not free, skip to next usable page.
|
||||
continue;
|
||||
}
|
||||
// Check requested range to ensure free.
|
||||
start_page_number = uint32_t(base_page_number);
|
||||
end_page_number = uint32_t(base_page_number) + page_count - 1;
|
||||
assert_true(end_page_number < page_table_.size());
|
||||
bool any_taken = false;
|
||||
for (uint32_t page_number = uint32_t(base_page_number);
|
||||
!any_taken && page_number <= end_page_number; ++page_number) {
|
||||
bool is_free = page_table_[page_number].state == 0;
|
||||
if (!is_free) {
|
||||
// At least one page in the range is used, skip to next.
|
||||
// We know we'll be starting at least before this page.
|
||||
any_taken = true;
|
||||
if (page_count > page_number) {
|
||||
// Not enough space left to fit entire page range. Breaks outer
|
||||
// loop.
|
||||
base_page_number = -1;
|
||||
} else {
|
||||
base_page_number = page_number - page_count;
|
||||
base_page_number -= QuickMod(base_page_number, page_scan_stride);
|
||||
base_page_number += page_scan_stride; // cancel out loop logic
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!any_taken) {
|
||||
// Found our place.
|
||||
// Search free blocks from high addresses downward.
|
||||
// Find the first block that could overlap our range.
|
||||
auto it = free_blocks_.upper_bound(high_page_number);
|
||||
while (it != free_blocks_.begin()) {
|
||||
--it;
|
||||
uint32_t block_start = it->first;
|
||||
uint32_t block_count = it->second;
|
||||
uint32_t block_end = block_start + block_count;
|
||||
|
||||
// Block is entirely below our search range — stop.
|
||||
if (block_end <= low_page_number) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip blocks too small to possibly fit.
|
||||
if (block_count < page_count) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Compute the highest aligned start within this block and range.
|
||||
uint32_t usable_end = std::min(block_end, high_page_number + 1);
|
||||
if (usable_end < page_count) {
|
||||
continue;
|
||||
}
|
||||
uint32_t latest_start = usable_end - page_count;
|
||||
// Align down to stride.
|
||||
latest_start -= QuickMod(latest_start, page_scan_stride);
|
||||
uint32_t usable_start = std::max(block_start, low_page_number);
|
||||
if (latest_start >= usable_start &&
|
||||
latest_start + page_count <= block_end) {
|
||||
start_page_number = latest_start;
|
||||
end_page_number = latest_start + page_count - 1;
|
||||
break;
|
||||
}
|
||||
// Retry.
|
||||
start_page_number = end_page_number = UINT_MAX;
|
||||
}
|
||||
} else {
|
||||
for (uint32_t base_page_number = low_page_number;
|
||||
base_page_number <= high_page_number - page_count;
|
||||
base_page_number += page_scan_stride) {
|
||||
if (page_table_[base_page_number].state != 0) {
|
||||
// Base page not free, skip to next usable page.
|
||||
continue;
|
||||
// Search free blocks from low addresses upward.
|
||||
auto it = free_blocks_.lower_bound(low_page_number);
|
||||
// Check if the previous block extends into our range.
|
||||
if (it != free_blocks_.begin()) {
|
||||
auto prev = std::prev(it);
|
||||
if (prev->first + prev->second > low_page_number) {
|
||||
it = prev;
|
||||
}
|
||||
// Check requested range to ensure free.
|
||||
start_page_number = base_page_number;
|
||||
end_page_number = base_page_number + page_count - 1;
|
||||
bool any_taken = false;
|
||||
for (uint32_t page_number = base_page_number;
|
||||
!any_taken && page_number <= end_page_number; ++page_number) {
|
||||
bool is_free = page_table_[page_number].state == 0;
|
||||
if (!is_free) {
|
||||
// At least one page in the range is used, skip to next.
|
||||
// We know we'll be starting at least after this page.
|
||||
any_taken = true;
|
||||
base_page_number = xe::round_up(page_number + 1, page_scan_stride);
|
||||
base_page_number -= page_scan_stride; // cancel out loop logic
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!any_taken) {
|
||||
// Found our place.
|
||||
}
|
||||
for (; it != free_blocks_.end(); ++it) {
|
||||
uint32_t block_start = it->first;
|
||||
uint32_t block_count = it->second;
|
||||
uint32_t block_end = block_start + block_count;
|
||||
|
||||
// Block is entirely above our search range — stop.
|
||||
if (block_start > high_page_number) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Skip blocks too small to possibly fit.
|
||||
if (block_count < page_count) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Compute the lowest aligned start within this block and range.
|
||||
uint32_t earliest = std::max(block_start, low_page_number);
|
||||
uint32_t aligned_start = xe::round_up(earliest, page_scan_stride, false);
|
||||
if (aligned_start + page_count <= block_end &&
|
||||
aligned_start + page_count - 1 <= high_page_number) {
|
||||
start_page_number = aligned_start;
|
||||
end_page_number = aligned_start + page_count - 1;
|
||||
break;
|
||||
}
|
||||
// Retry.
|
||||
start_page_number = end_page_number = UINT_MAX;
|
||||
}
|
||||
}
|
||||
|
||||
if (start_page_number == UINT_MAX || end_page_number == UINT_MAX) {
|
||||
// Out of memory.
|
||||
XELOGE("BaseHeap::Alloc failed to find contiguous range");
|
||||
@@ -1183,6 +1284,9 @@ bool BaseHeap::AllocRange(uint32_t low_address, uint32_t high_address,
|
||||
return false;
|
||||
}
|
||||
|
||||
// Update free block tracker.
|
||||
RemoveFreeBlock(start_page_number, page_count);
|
||||
|
||||
// Allocate from host.
|
||||
if (allocation_type == kMemoryAllocationReserve) {
|
||||
// Reserve is not needed, as we are mapped already.
|
||||
@@ -1196,6 +1300,8 @@ bool BaseHeap::AllocRange(uint32_t low_address, uint32_t high_address,
|
||||
page_count << page_size_shift_, alloc_type, ToPageAccess(protect));
|
||||
if (!result) {
|
||||
XELOGE("BaseHeap::Alloc failed to alloc range from host");
|
||||
// Restore the free block since we failed.
|
||||
InsertFreeBlock(start_page_number, page_count);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1329,6 +1435,9 @@ bool BaseHeap::Release(uint32_t base_address, uint32_t* out_region_size) {
|
||||
unreserved_page_count_++;
|
||||
}
|
||||
|
||||
// Insert freed block into tracker with coalescing.
|
||||
InsertFreeBlock(base_page_number, base_page_entry.region_page_count);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1685,7 +1794,10 @@ bool PhysicalHeap::Alloc(uint32_t size, uint32_t alignment,
|
||||
alignment, allocation_type, protect, top_down,
|
||||
&parent_address)) {
|
||||
XELOGE(
|
||||
"PhysicalHeap::Alloc unable to alloc physical memory in parent heap");
|
||||
"PhysicalHeap::Alloc unable to alloc physical memory in parent heap "
|
||||
"(requested {} bytes, parent free {}/{} pages)",
|
||||
size, parent_heap_->unreserved_page_count(),
|
||||
parent_heap_->total_page_count());
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1704,7 +1816,7 @@ bool PhysicalHeap::Alloc(uint32_t size, uint32_t alignment,
|
||||
protect)) {
|
||||
XELOGE(
|
||||
"PhysicalHeap::Alloc unable to pin physical memory in physical heap");
|
||||
// TODO(benvanik): don't leak parent memory.
|
||||
parent_heap_->Release(parent_address);
|
||||
return false;
|
||||
}
|
||||
*out_address = address;
|
||||
@@ -1728,7 +1840,8 @@ bool PhysicalHeap::AllocFixed(uint32_t base_address, uint32_t size,
|
||||
if (!parent_heap_->AllocFixed(parent_base_address, size, alignment,
|
||||
allocation_type, protect)) {
|
||||
XELOGE(
|
||||
"PhysicalHeap::Alloc unable to alloc physical memory in parent heap");
|
||||
"PhysicalHeap::AllocFixed unable to alloc physical memory in parent "
|
||||
"heap");
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1747,8 +1860,9 @@ bool PhysicalHeap::AllocFixed(uint32_t base_address, uint32_t size,
|
||||
if (!BaseHeap::AllocFixed(address, size, alignment, allocation_type,
|
||||
protect)) {
|
||||
XELOGE(
|
||||
"PhysicalHeap::Alloc unable to pin physical memory in physical heap");
|
||||
// TODO(benvanik): don't leak parent memory.
|
||||
"PhysicalHeap::AllocFixed unable to pin physical memory in physical "
|
||||
"heap");
|
||||
parent_heap_->Release(parent_base_address);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1777,7 +1891,10 @@ bool PhysicalHeap::AllocRange(uint32_t low_address, uint32_t high_address,
|
||||
alignment, allocation_type, protect, top_down,
|
||||
&parent_address)) {
|
||||
XELOGE(
|
||||
"PhysicalHeap::Alloc unable to alloc physical memory in parent heap");
|
||||
"PhysicalHeap::AllocRange unable to alloc physical memory in parent "
|
||||
"heap (requested {} bytes, parent free {}/{} pages)",
|
||||
size, parent_heap_->unreserved_page_count(),
|
||||
parent_heap_->total_page_count());
|
||||
return false;
|
||||
}
|
||||
// Given the address we've reserved in the parent heap, pin that here.
|
||||
@@ -1795,8 +1912,9 @@ bool PhysicalHeap::AllocRange(uint32_t low_address, uint32_t high_address,
|
||||
if (!BaseHeap::AllocFixed(address, size, alignment, allocation_type,
|
||||
protect)) {
|
||||
XELOGE(
|
||||
"PhysicalHeap::Alloc unable to pin physical memory in physical heap");
|
||||
// TODO(benvanik): don't leak parent memory.
|
||||
"PhysicalHeap::AllocRange unable to pin physical memory in physical "
|
||||
"heap");
|
||||
parent_heap_->Release(parent_address);
|
||||
return false;
|
||||
}
|
||||
*out_address = address;
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#define XENIA_MEMORY_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
@@ -210,6 +211,15 @@ class BaseHeap {
|
||||
uint32_t heap_base, uint32_t heap_size, uint32_t page_size,
|
||||
uint32_t host_address_offset = 0);
|
||||
|
||||
// Rebuilds free_blocks_ by scanning page_table_. Used after Restore.
|
||||
void RebuildFreeBlocks();
|
||||
|
||||
// Removes (or splits) the free block covering the given page range.
|
||||
void RemoveFreeBlock(uint32_t start_page, uint32_t page_count);
|
||||
|
||||
// Inserts a free block and coalesces with adjacent free blocks.
|
||||
void InsertFreeBlock(uint32_t start_page, uint32_t page_count);
|
||||
|
||||
Memory* memory_;
|
||||
uint8_t* membase_;
|
||||
HeapType heap_type_;
|
||||
@@ -221,6 +231,10 @@ class BaseHeap {
|
||||
uint32_t unreserved_page_count_;
|
||||
xe::global_critical_region global_critical_region_;
|
||||
std::vector<PageEntry> page_table_;
|
||||
|
||||
// Auxiliary free block tracker: maps start_page -> count of contiguous free
|
||||
// pages. Kept in sync with page_table_ mutations. Not serialized.
|
||||
std::map<uint32_t, uint32_t> free_blocks_;
|
||||
};
|
||||
|
||||
// Normal heap allowing allocations from guest virtual address ranges.
|
||||
|
||||
Reference in New Issue
Block a user