Improve error handling in graphics memory init

src/xenia/gpu/d3d12/d3d12_shared_memory.cc

@@ -34,7 +34,9 @@ D3D12SharedMemory::D3D12SharedMemory(D3D12CommandProcessor& command_processor,
 D3D12SharedMemory::~D3D12SharedMemory() { Shutdown(true); }
 
 bool D3D12SharedMemory::Initialize() {
-  InitializeCommon();
+  if (!InitializeCommon()) {
+    return false;
+  }
 
   const ui::d3d12::D3D12Provider& provider =
       command_processor_.GetD3D12Provider();

src/xenia/gpu/shared_memory.cc

@@ -23,7 +23,7 @@ SharedMemory::SharedMemory(Memory& memory) : memory_(memory) {
 
 SharedMemory::~SharedMemory() { ShutdownCommon(); }
 
-void SharedMemory::InitializeCommon() {
+bool SharedMemory::InitializeCommon() {
   size_t num_system_page_flags_entries =
       ((kBufferSize >> page_size_log2_) + 63) / 64;
   num_system_page_flags_ = static_cast<uint32_t>(num_system_page_flags_entries);
@@ -36,6 +36,11 @@ void SharedMemory::InitializeCommon() {
       nullptr, num_system_page_flags_ * 3 * sizeof(uint64_t),
       memory::AllocationType::kReserveCommit, memory::PageAccess::kReadWrite);
 
+  if (!system_page_flags_base) {
+    XELOGE("SharedMemory: Failed to allocate system page flags");
+    return false;
+  }
+
   system_page_flags_valid_ = system_page_flags_base,
   system_page_flags_valid_and_gpu_resolved_ =
       system_page_flags_base + (num_system_page_flags_),
@@ -49,6 +54,7 @@ void SharedMemory::InitializeCommon() {
   memory_invalidation_callback_handle_ =
       memory_.RegisterPhysicalMemoryInvalidationCallback(
           MemoryInvalidationCallbackThunk, this);
+  return true;
 }
 
 void SharedMemory::InitializeSparseHostGpuMemory(uint32_t granularity_log2) {

src/xenia/gpu/shared_memory.h

@@ -105,7 +105,7 @@ class SharedMemory {
  protected:
   SharedMemory(Memory& memory);
   // Call in implementation-specific initialization.
-  void InitializeCommon();
+  bool InitializeCommon();
   void InitializeSparseHostGpuMemory(uint32_t granularity_log2);
   // Call last in implementation-specific shutdown, also callable from the
   // destructor.

src/xenia/gpu/vulkan/vulkan_shared_memory.cc

@@ -44,7 +44,9 @@ VulkanSharedMemory::VulkanSharedMemory(
 VulkanSharedMemory::~VulkanSharedMemory() { Shutdown(true); }
 
 bool VulkanSharedMemory::Initialize() {
-  InitializeCommon();
+  if (!InitializeCommon()) {
+    return false;
+  }
 
   const ui::vulkan::VulkanDevice* const vulkan_device =
       command_processor_.GetVulkanDevice();