[XAM] Limit guest UI to only one instance

This fixes potential crashing in games that spam UI show calls

src/xenia/app/emulator_window.cc

@@ -1587,19 +1587,27 @@ void EmulatorWindow::ToggleDisplayConfigDialog() {
 void EmulatorWindow::ToggleProfilesConfigDialog() {
   if (!profile_config_dialog_) {
     disable_hotkeys_ = true;
-    emulator_->kernel_state()->BroadcastNotification(kXNotificationSystemUI, 1);
+
+    if (emulator_->kernel_state()->xam_state()->IsUIActive()) {
+      return;
+    }
+
+    emulator_->kernel_state()->BroadcastNotification(kXNotificationSystemUI,
+                                                     true);
+    emulator_->kernel_state()->xam_state()->is_xam_dialog_present_.store(true);
+
     profile_config_dialog_ =
         std::make_unique<ProfileConfigDialog>(imgui_drawer_.get(), this);
-    emulator_->kernel_state()->xam_state()->xam_dialogs_shown_++;
   } else {
     disable_hotkeys_ = false;
-    emulator_->kernel_state()->BroadcastNotification(kXNotificationSystemUI, 0);
+    emulator_->kernel_state()->BroadcastNotification(kXNotificationSystemUI,
+                                                     false);
     if (profile_config_dialog_->IsClosing()) {
       profile_config_dialog_.release();
     } else {
       profile_config_dialog_.reset();
     }
-    emulator_->kernel_state()->xam_state()->xam_dialogs_shown_--;
+    emulator_->kernel_state()->xam_state()->is_xam_dialog_present_.store(false);
   }
 }
 
@@ -2227,7 +2235,7 @@ xe::X_STATUS EmulatorWindow::RunTitle(
 
   if (profile_config_dialog_) {
     profile_config_dialog_.reset();
-    emulator_->kernel_state()->xam_state()->xam_dialogs_shown_--;
+    emulator_->kernel_state()->xam_state()->is_xam_dialog_present_.store(false);
   }
 
   if (display_config_dialog_) {
@@ -2376,7 +2384,7 @@ void EmulatorWindow::ClearDialogs() {
   }
 
   imgui_drawer_.get()->ClearDialogs();
-  emulator_->kernel_state()->xam_state()->xam_dialogs_shown_ = 0;
+  emulator_->kernel_state()->xam_state()->is_xam_dialog_present_.store(false);
 }
 
 }  // namespace app

src/xenia/kernel/xam/xam_nui.cc

@@ -182,7 +182,7 @@ dword_result_t XamNuiCameraSetFlags_entry(qword_t unk1, dword_t unk2) {
 DECLARE_XAM_EXPORT1(XamNuiCameraSetFlags, kNone, kStub);
 
 dword_result_t XamIsNuiUIActive_entry() {
-  return kernel_state()->xam_state()->xam_nui_dialogs_shown_ > 0;
+  return kernel_state()->xam_state()->is_xam_dialog_present_.load();
 }
 DECLARE_XAM_EXPORT1(XamIsNuiUIActive, kNone, kImplemented);
 
@@ -369,9 +369,9 @@ dword_result_t XamShowNuiTroubleshooterUI_entry(dword_t user_index,
               "The game has indicated there is a problem with NUI (Kinect).")
               ->Then(&fence);
         })) {
-      kernel_state()->xam_state()->xam_dialogs_shown_++;
+      kernel_state()->xam_state()->is_xam_dialog_present_.store(true);
       fence.Wait();
-      kernel_state()->xam_state()->xam_dialogs_shown_--;
+      kernel_state()->xam_state()->is_xam_dialog_present_.store(false);
     }
   }
 

src/xenia/kernel/xam/xam_state.h

@@ -58,9 +58,7 @@ class XamState {
 
   void SetContentRegisterCallback(uint32_t callback);
 
-  bool IsUIActive() const {
-    return xam_dialogs_shown_ > 0 || xam_nui_dialogs_shown_ > 0;
-  }
+  bool IsUIActive() const { return is_xam_dialog_present_.load(); }
 
   uint32_t GetLanguageFallbackAddress(uint32_t index) const {
     return language_fallback_address_[index];
@@ -75,8 +73,7 @@ class XamState {
   X_DASH_BACKSTACK_DATA dash_backstack_data_[2] = {};
   uint32_t content_register_callback = 0;
 
-  std::atomic<int32_t> xam_dialogs_shown_ = {0};
-  std::atomic<int32_t> xam_nui_dialogs_shown_ = {0};
+  std::atomic<bool> is_xam_dialog_present_ = false;
 
  private:
   void LoadLanguageLocaleFallback();

src/xenia/kernel/xam/xam_ui.cc

@@ -73,9 +73,8 @@ X_RESULT xeXamDispatchDialog(T* dialog,
         kernel_state()->emulator()->display_window()->app_context();
     if (app_context.CallInUIThreadSynchronous(
             [&dialog, &fence]() { dialog->Then(&fence); })) {
-      kernel_state()->xam_state()->xam_dialogs_shown_++;
       fence.Wait();
-      kernel_state()->xam_state()->xam_dialogs_shown_--;
+      kernel_state()->xam_state()->is_xam_dialog_present_.store(false);
     } else {
       delete dialog;
     }
@@ -118,9 +117,8 @@ X_RESULT xeXamDispatchDialogEx(
     xe::threading::Fence fence;
     if (display_window->app_context().CallInUIThreadSynchronous(
             [&dialog, &fence]() { dialog->Then(&fence); })) {
-      kernel_state()->xam_state()->xam_dialogs_shown_++;
       fence.Wait();
-      kernel_state()->xam_state()->xam_dialogs_shown_--;
+      kernel_state()->xam_state()->is_xam_dialog_present_.store(false);
     } else {
       delete dialog;
     }
@@ -198,14 +196,13 @@ template <typename T>
 X_RESULT xeXamDispatchDialogAsync(T* dialog,
                                   std::function<void(T*)> close_callback) {
   kernel_state()->BroadcastNotification(kXNotificationSystemUI, true);
-  kernel_state()->xam_state()->xam_dialogs_shown_++;
   // Important to pass captured vars by value here since we return from this
   // without waiting for the dialog to close so the original local vars will be
   // destroyed.
   dialog->set_close_callback([dialog, close_callback]() {
     close_callback(dialog);
 
-    kernel_state()->xam_state()->xam_dialogs_shown_--;
+    kernel_state()->xam_state()->is_xam_dialog_present_.store(false);
 
     auto run = []() -> void {
       xe::threading::Sleep(kUIDelayMillis);
@@ -221,13 +218,12 @@ X_RESULT xeXamDispatchDialogAsync(T* dialog,
 
 X_RESULT xeXamDispatchHeadlessAsync(std::function<void()> run_callback) {
   kernel_state()->BroadcastNotification(kXNotificationSystemUI, true);
-  kernel_state()->xam_state()->xam_dialogs_shown_++;
 
   auto display_window = kernel_state()->emulator()->display_window();
   display_window->app_context().CallInUIThread([run_callback]() {
     run_callback();
 
-    kernel_state()->xam_state()->xam_dialogs_shown_--;
+    kernel_state()->xam_state()->is_xam_dialog_present_.store(false);
 
     auto run = []() -> void {
       xe::threading::Sleep(kUIDelayMillis);
@@ -373,6 +369,12 @@ static dword_result_t XamShowMessageBoxUi(
       } break;
     }
 
+    if (kernel_state()->xam_state()->IsUIActive()) {
+      return X_ERROR_ACCESS_DENIED;
+    }
+
+    kernel_state()->xam_state()->is_xam_dialog_present_.store(true);
+
     const Emulator* emulator = kernel_state()->emulator();
     xe::ui::ImGuiDrawer* imgui_drawer = emulator->imgui_drawer();
 
@@ -393,6 +395,7 @@ static dword_result_t XamShowMessageBoxUi(
     } else {
       auto close = [result_ptr](MessageBoxDialog* dialog) -> X_RESULT {
         result_ptr->ButtonPressed = dialog->chosen_button();
+        kernel_state()->xam_state()->is_xam_dialog_present_.store(false);
         return X_ERROR_SUCCESS;
       };
 
@@ -504,6 +507,13 @@ dword_result_t XamShowKeyboardUI_entry(
         return X_ERROR_SUCCESS;
       }
     };
+
+    if (kernel_state()->xam_state()->IsUIActive()) {
+      return X_ERROR_ACCESS_DENIED;
+    }
+
+    kernel_state()->xam_state()->is_xam_dialog_present_.store(true);
+
     const Emulator* emulator = kernel_state()->emulator();
     xe::ui::ImGuiDrawer* imgui_drawer = emulator->imgui_drawer();
 
@@ -563,6 +573,12 @@ dword_result_t XamShowDeviceSelectorUI_entry(
     return X_ERROR_SUCCESS;
   };
 
+  if (kernel_state()->xam_state()->IsUIActive()) {
+    return X_ERROR_ACCESS_DENIED;
+  }
+
+  kernel_state()->xam_state()->is_xam_dialog_present_.store(true);
+
   std::string title = "Select storage device";
   std::string desc = "";
 
@@ -657,6 +673,12 @@ dword_result_t XamShowMarketplaceUIEx_entry(dword_t user_index, dword_t ui_type,
     return xeXamDispatchHeadlessAsync([]() {});
   }
 
+  if (kernel_state()->xam_state()->IsUIActive()) {
+    return X_ERROR_ACCESS_DENIED;
+  }
+
+  kernel_state()->xam_state()->is_xam_dialog_present_.store(true);
+
   bool is_xbla_unlock_offer =
       (offer_id == ((uint64_t(kernel_state()->title_id()) << 32) | 1ull));
 
@@ -804,6 +826,12 @@ dword_result_t XamShowMarketplaceDownloadItemsUI_entry(
         overlapped);
   }
 
+  if (kernel_state()->xam_state()->IsUIActive()) {
+    return X_ERROR_ACCESS_DENIED;
+  }
+
+  kernel_state()->xam_state()->is_xam_dialog_present_.store(true);
+
   auto close = [hresult_ptr](MessageBoxDialog* dialog) -> X_RESULT {
     if (hresult_ptr) {
       // TODO
@@ -942,6 +970,12 @@ X_RESULT xeXamShowSigninUI(uint32_t user_index, uint32_t users_needed,
     });
   }
 
+  if (kernel_state()->xam_state()->IsUIActive()) {
+    return X_ERROR_ACCESS_DENIED;
+  }
+
+  kernel_state()->xam_state()->is_xam_dialog_present_.store(true);
+
   auto close = [](ui::SigninUI* dialog) -> void {};
 
   const Emulator* emulator = kernel_state()->emulator();
@@ -962,6 +996,12 @@ X_RESULT xeXamShowCreateProfileUIEx(uint32_t user_index, dword_t flag,
     return X_ERROR_SUCCESS;
   }
 
+  if (kernel_state()->xam_state()->IsUIActive()) {
+    return X_ERROR_ACCESS_DENIED;
+  }
+
+  kernel_state()->xam_state()->is_xam_dialog_present_.store(true);
+
   auto close = [](ui::CreateProfileUI* dialog) -> void {};
 
   return xeXamDispatchDialogAsync<ui::CreateProfileUI>(