total apc rework

2023-10-11 09:47:19 -04:00
parent b5ddd30572
commit d0a6cec024
6 changed files with 430 additions and 282 deletions
--- a/src/xenia/kernel/xboxkrnl/xboxkrnl_threading.cc
+++ b/src/xenia/kernel/xboxkrnl/xboxkrnl_threading.cc
@@ -379,19 +379,35 @@ DECLARE_XBOXKRNL_EXPORT2(KeQueryPerformanceFrequency, kThreading, kImplemented,
                         kHighFrequency);

 uint32_t KeDelayExecutionThread(uint32_t processor_mode, uint32_t alertable,
-                                uint64_t* interval_ptr) {
+                                uint64_t* interval_ptr,
+                                cpu::ppc::PPCContext* ctx) {
  XThread* thread = XThread::GetCurrentThread();
+
+  if (alertable) {
+    X_STATUS stat = xeProcessUserApcs(ctx);
+    if (stat == X_STATUS_USER_APC) {
+      return stat;
+    }
+  }
  X_STATUS result = thread->Delay(processor_mode, alertable, *interval_ptr);

+  if (result == X_STATUS_USER_APC) {
+    result = xeProcessUserApcs(ctx);
+    if (result == X_STATUS_USER_APC) {
+      return result;
+    }
+  }
+
  return result;
 }

 dword_result_t KeDelayExecutionThread_entry(dword_t processor_mode,
                                            dword_t alertable,
-                                            lpqword_t interval_ptr) {
+                                            lpqword_t interval_ptr,
+                                            const ppc_context_t& context) {
  uint64_t interval = interval_ptr ? static_cast<uint64_t>(*interval_ptr) : 0u;
  return KeDelayExecutionThread(processor_mode, alertable,
-                                interval_ptr ? &interval : nullptr);
+                                interval_ptr ? &interval : nullptr, context);
 }
 DECLARE_XBOXKRNL_EXPORT3(KeDelayExecutionThread, kThreading, kImplemented,
                         kBlocking, kHighFrequency);
@@ -889,7 +905,11 @@ uint32_t xeKeWaitForSingleObject(void* object_ptr, uint32_t wait_reason,

  X_STATUS result =
      object->Wait(wait_reason, processor_mode, alertable, timeout_ptr);
-
+  if (alertable) {
+    if (result == X_STATUS_USER_APC) {
+      result = xeProcessUserApcs(nullptr);
+    }
+  }
  return result;
 }

@@ -915,6 +935,11 @@ uint32_t NtWaitForSingleObjectEx(uint32_t object_handle, uint32_t wait_mode,
    uint64_t timeout = timeout_ptr ? static_cast<uint64_t>(*timeout_ptr) : 0u;
    result =
        object->Wait(3, wait_mode, alertable, timeout_ptr ? &timeout : nullptr);
+    if (alertable) {
+      if (result == X_STATUS_USER_APC) {
+        result = xeProcessUserApcs(nullptr);
+      }
+    }
  } else {
    result = X_STATUS_INVALID_HANDLE;
  }
@@ -955,9 +980,15 @@ dword_result_t KeWaitForMultipleObjects_entry(
    }
  }
  uint64_t timeout = timeout_ptr ? static_cast<uint64_t>(*timeout_ptr) : 0u;
-  return XObject::WaitMultiple(
+  X_STATUS result = XObject::WaitMultiple(
      uint32_t(count), reinterpret_cast<XObject**>(&objects[0]), wait_type,
      wait_reason, processor_mode, alertable, timeout_ptr ? &timeout : nullptr);
+  if (alertable) {
+    if (result == X_STATUS_USER_APC) {
+      result = xeProcessUserApcs(nullptr);
+    }
+  }
+  return result;
 }
 DECLARE_XBOXKRNL_EXPORT3(KeWaitForMultipleObjects, kThreading, kImplemented,
                         kBlocking, kHighFrequency);
@@ -993,8 +1024,15 @@ uint32_t xeNtWaitForMultipleObjectsEx(uint32_t count, xe::be<uint32_t>* handles,
    }
  }

-  return XObject::WaitMultiple(count, reinterpret_cast<XObject**>(&objects[0]),
-                               wait_type, 6, wait_mode, alertable, timeout_ptr);
+  auto result =
+      XObject::WaitMultiple(count, reinterpret_cast<XObject**>(&objects[0]),
+                            wait_type, 6, wait_mode, alertable, timeout_ptr);
+  if (alertable) {
+    if (result == X_STATUS_USER_APC) {
+      result = xeProcessUserApcs(nullptr);
+    }
+  }
+  return result;
 }

 dword_result_t NtWaitForMultipleObjectsEx_entry(
@@ -1034,6 +1072,11 @@ dword_result_t NtSignalAndWaitForSingleObjectEx_entry(dword_t signal_handle,
    result = X_STATUS_INVALID_HANDLE;
  }

+  if (alertable) {
+    if (result == X_STATUS_USER_APC) {
+      result = xeProcessUserApcs(nullptr);
+    }
+  }
  return result;
 }
 DECLARE_XBOXKRNL_EXPORT3(NtSignalAndWaitForSingleObjectEx, kThreading,
@@ -1042,8 +1085,7 @@ DECLARE_XBOXKRNL_EXPORT3(NtSignalAndWaitForSingleObjectEx, kThreading,
 static void PrefetchForCAS(const void* value) { swcache::PrefetchW(value); }

 uint32_t xeKeKfAcquireSpinLock(PPCContext* ctx, X_KSPINLOCK* lock,
-                               bool change_irql)
-{
+                               bool change_irql) {
  auto old_irql = change_irql ? xeKfRaiseIrql(ctx, 2) : 0;

  PrefetchForCAS(lock);
@@ -1071,7 +1113,7 @@ void xeKeKfReleaseSpinLock(PPCContext* ctx, X_KSPINLOCK* lock, dword_t old_irql,
  assert_true(lock->prcb_of_owner == static_cast<uint32_t>(ctx->r[13]));
  // Unlock.
  lock->prcb_of_owner.value = 0;
- 
+
  if (change_irql) {
    // Unlock.
    if (old_irql >= 2) {
@@ -1186,23 +1228,169 @@ dword_result_t KfRaiseIrql_entry(dword_t new_irql, const ppc_context_t& ctx) {

 DECLARE_XBOXKRNL_EXPORT2(KfRaiseIrql, kThreading, kImplemented, kHighFrequency);

-void NtQueueApcThread_entry(dword_t thread_handle, lpvoid_t apc_routine,
-                            lpvoid_t apc_routine_context, lpvoid_t arg1,
-                            lpvoid_t arg2) {
+uint32_t xeNtQueueApcThread(uint32_t thread_handle, uint32_t apc_routine,
+                            uint32_t apc_routine_context, uint32_t arg1,
+                            uint32_t arg2, cpu::ppc::PPCContext* context) {
+  auto kernelstate = context->kernel_state;
+  auto memory = kernelstate->memory();
  auto thread =
-      kernel_state()->object_table()->LookupObject<XThread>(thread_handle);
+      kernelstate->object_table()->LookupObject<XThread>(thread_handle);

  if (!thread) {
    XELOGE("NtQueueApcThread: Incorrect thread handle! Might cause crash");
-    return;
+    return X_STATUS_INVALID_HANDLE;
  }

-  if (!apc_routine) {
-    XELOGE("NtQueueApcThread: Incorrect apc routine! Might cause crash");
-    return;
+  uint32_t apc_ptr = memory->SystemHeapAlloc(XAPC::kSize);
+  if (!apc_ptr) {
+    return X_STATUS_NO_MEMORY;
+  }
+  XAPC* apc = context->TranslateVirtual<XAPC*>(apc_ptr);
+  xeKeInitializeApc(apc, thread->guest_object(), XAPC::kDummyKernelRoutine, 0,
+                    apc_routine, 1 /*user apc mode*/, apc_routine_context);
+
+  if (!xeKeInsertQueueApc(apc, arg1, arg2, 0, context)) {
+    memory->SystemHeapFree(apc_ptr);
+    return X_STATUS_UNSUCCESSFUL;
+  }
+  // no-op, just meant to awaken a sleeping alertable thread to process real
+  // apcs
+  thread->thread()->QueueUserCallback([]() {});
+  return X_STATUS_SUCCESS;
+}
+dword_result_t NtQueueApcThread_entry(dword_t thread_handle,
+                                      lpvoid_t apc_routine,
+                                      lpvoid_t apc_routine_context,
+                                      lpvoid_t arg1, lpvoid_t arg2,
+                                      const ppc_context_t& context) {
+  return xeNtQueueApcThread(thread_handle, apc_routine, apc_routine_context,
+                            arg1, arg2, context);
+}
+
+X_STATUS xeProcessUserApcs(PPCContext* ctx) {
+  if (!ctx) {
+    ctx = cpu::ThreadState::Get()->context();
+  }
+  X_STATUS alert_status = X_STATUS_SUCCESS;
+  auto kpcr = ctx->TranslateVirtualGPR<X_KPCR*>(ctx->r[13]);
+
+  auto current_thread = ctx->TranslateVirtual(kpcr->prcb_data.current_thread);
+
+  uint32_t unlocked_irql =
+      xeKeKfAcquireSpinLock(ctx, &current_thread->apc_lock);
+
+  auto& user_apc_queue = current_thread->apc_lists[1];
+
+  // use guest stack for temporaries
+  uint32_t old_stack_pointer = static_cast<uint32_t>(ctx->r[1]);
+
+  uint32_t scratch_address = old_stack_pointer - 16;
+  ctx->r[1] = old_stack_pointer - 32;
+
+  while (!user_apc_queue.empty(ctx)) {
+    uint32_t apc_ptr = user_apc_queue.flink_ptr;
+
+    XAPC* apc = user_apc_queue.ListEntryObject(
+        ctx->TranslateVirtual<X_LIST_ENTRY*>(apc_ptr));
+
+    uint8_t* scratch_ptr = ctx->TranslateVirtual(scratch_address);
+    xe::store_and_swap<uint32_t>(scratch_ptr + 0, apc->normal_routine);
+    xe::store_and_swap<uint32_t>(scratch_ptr + 4, apc->normal_context);
+    xe::store_and_swap<uint32_t>(scratch_ptr + 8, apc->arg1);
+    xe::store_and_swap<uint32_t>(scratch_ptr + 12, apc->arg2);
+    util::XeRemoveEntryList(&apc->list_entry, ctx);
+    apc->enqueued = 0;
+
+    xeKeKfReleaseSpinLock(ctx, &current_thread->apc_lock, unlocked_irql);
+    alert_status = X_STATUS_USER_APC;
+    if (apc->kernel_routine != XAPC::kDummyKernelRoutine) {
+      uint64_t kernel_args[] = {
+          apc_ptr,
+          scratch_address + 0,
+          scratch_address + 4,
+          scratch_address + 8,
+          scratch_address + 12,
+      };
+      ctx->processor->Execute(ctx->thread_state, apc->kernel_routine,
+                              kernel_args, xe::countof(kernel_args));
+    } else {
+      ctx->kernel_state->memory()->SystemHeapFree(apc_ptr);
+    }
+
+    uint32_t normal_routine = xe::load_and_swap<uint32_t>(scratch_ptr + 0);
+    uint32_t normal_context = xe::load_and_swap<uint32_t>(scratch_ptr + 4);
+    uint32_t arg1 = xe::load_and_swap<uint32_t>(scratch_ptr + 8);
+    uint32_t arg2 = xe::load_and_swap<uint32_t>(scratch_ptr + 12);
+
+    if (normal_routine) {
+      uint64_t normal_args[] = {normal_context, arg1, arg2};
+      ctx->processor->Execute(ctx->thread_state, normal_routine, normal_args,
+                              xe::countof(normal_args));
+    }
+
+    unlocked_irql = xeKeKfAcquireSpinLock(ctx, &current_thread->apc_lock);
  }

-  thread->EnqueueApc(apc_routine, apc_routine_context, arg1, arg2);
+  ctx->r[1] = old_stack_pointer;
+
+  xeKeKfReleaseSpinLock(ctx, &current_thread->apc_lock, unlocked_irql);
+  return alert_status;
+}
+
+static void YankApcList(PPCContext* ctx, X_KTHREAD* current_thread, unsigned apc_mode,
+                 bool rundown) {
+  uint32_t unlocked_irql =
+      xeKeKfAcquireSpinLock(ctx, &current_thread->apc_lock);
+
+  XAPC* result = nullptr;
+  auto& user_apc_queue = current_thread->apc_lists[apc_mode];
+
+  if (user_apc_queue.empty(ctx)) {
+    result = nullptr;
+  } else {
+    result = user_apc_queue.HeadObject(ctx);
+    for (auto&& entry : user_apc_queue.IterateForward(ctx)) {
+      entry.enqueued = 0;
+    }
+    util::XeRemoveEntryList(&user_apc_queue, ctx);
+  }
+
+  xeKeKfReleaseSpinLock(ctx, &current_thread->apc_lock, unlocked_irql);
+
+  if (rundown && result) {
+    XAPC* current_entry = result;
+    while (true) {
+      XAPC* this_entry = current_entry;
+      uint32_t next_entry = this_entry->list_entry.flink_ptr;
+
+      if (this_entry->rundown_routine) {
+        uint64_t args[] = {ctx->HostToGuestVirtual(this_entry)};
+        kernel_state()->processor()->Execute(ctx->thread_state,
+                                             this_entry->rundown_routine, args,
+                                             xe::countof(args));
+      } else {
+        ctx->kernel_state->memory()->SystemHeapFree(
+            ctx->HostToGuestVirtual(this_entry));
+      }
+
+      if (next_entry == 0) {
+        break;
+      }
+      current_entry = user_apc_queue.ListEntryObject(
+          ctx->TranslateVirtual<X_LIST_ENTRY*>(next_entry));
+      if (current_entry == result) {
+        break;
+      }
+    }
+  }
+}
+
+void xeRundownApcs(PPCContext* ctx) {
+  auto kpcr = ctx->TranslateVirtualGPR<X_KPCR*>(ctx->r[13]);
+
+  auto current_thread = ctx->TranslateVirtual(kpcr->prcb_data.current_thread);
+  YankApcList(ctx, current_thread, 1, true);
+  YankApcList(ctx, current_thread, 0, false);
 }
 DECLARE_XBOXKRNL_EXPORT1(NtQueueApcThread, kThreading, kImplemented);
 void xeKeInitializeApc(XAPC* apc, uint32_t thread_ptr, uint32_t kernel_routine,
@@ -1231,67 +1419,82 @@ void KeInitializeApc_entry(pointer_t<XAPC> apc, lpvoid_t thread_ptr,
 }
 DECLARE_XBOXKRNL_EXPORT1(KeInitializeApc, kThreading, kImplemented);

+uint32_t xeKeInsertQueueApc(XAPC* apc, uint32_t arg1, uint32_t arg2,
+                            uint32_t priority_increment,
+                            cpu::ppc::PPCContext* context) {
+  uint32_t thread_guest_pointer = apc->thread_ptr;
+  if (!thread_guest_pointer) {
+    return 0;
+  }
+  auto target_thread = context->TranslateVirtual<X_KTHREAD*>(apc->thread_ptr);
+  auto old_irql = xeKeKfAcquireSpinLock(context, &target_thread->apc_lock);
+  uint32_t result;
+  if (!target_thread->apc_related || apc->enqueued) {
+    result = 0;
+  } else {
+    apc->arg1 = arg1;
+    apc->arg2 = arg2;
+
+    auto& which_list = target_thread->apc_lists[apc->apc_mode];
+
+    if (apc->normal_routine) {
+      which_list.InsertTail(apc, context);
+    } else {
+      XAPC* insertion_pos = nullptr;
+      for (auto&& sub_apc : which_list.IterateForward(context)) {
+        insertion_pos = &sub_apc;
+        if (sub_apc.normal_routine) {
+          break;
+        }
+      }
+      if (!insertion_pos) {
+        which_list.InsertHead(apc, context);
+      } else {
+        util::XeInsertHeadList(insertion_pos->list_entry.blink_ptr,
+                               &apc->list_entry, context);
+      }
+    }
+
+    apc->enqueued = 1;
+
+    /*
+        todo: this is incomplete, a ton of other logic happens here, i believe
+       for waking the target thread if its alertable
+    */
+    result = 1;
+  }
+  xeKeKfReleaseSpinLock(context, &target_thread->apc_lock, old_irql);
+  return result;
+}
+
 dword_result_t KeInsertQueueApc_entry(pointer_t<XAPC> apc, lpvoid_t arg1,
-                                      lpvoid_t arg2,
-                                      dword_t priority_increment) {
-  auto thread = XObject::GetNativeObject<XThread>(
-      kernel_state(),
-      kernel_state()->memory()->TranslateVirtual(apc->thread_ptr));
-  if (!thread) {
-    return 0;
-  }
+                                      lpvoid_t arg2, dword_t priority_increment,
+                                      const ppc_context_t& context) {

-  // Lock thread.
-  thread->LockApc();
+  return xeKeInsertQueueApc(apc, arg1, arg2, priority_increment, context);

-  // Fail if already inserted.
-  if (apc->enqueued) {
-    thread->UnlockApc(false);
-    return 0;
-  }
-
-  // Prep APC.
-  apc->arg1 = arg1.guest_address();
-  apc->arg2 = arg2.guest_address();
-  apc->enqueued = 1;
-
-  auto apc_list = thread->apc_list();
-
-  uint32_t list_entry_ptr = apc.guest_address() + 8;
-  apc_list->Insert(list_entry_ptr);
-
-  // Unlock thread.
-  thread->UnlockApc(true);
-
-  return 1;
 }
 DECLARE_XBOXKRNL_EXPORT1(KeInsertQueueApc, kThreading, kImplemented);

-dword_result_t KeRemoveQueueApc_entry(pointer_t<XAPC> apc) {
+dword_result_t KeRemoveQueueApc_entry(pointer_t<XAPC> apc,
+                                      const ppc_context_t& context) {
  bool result = false;

-  auto thread = XObject::GetNativeObject<XThread>(
-      kernel_state(),
-      kernel_state()->memory()->TranslateVirtual(apc->thread_ptr));
-  if (!thread) {
+
+  uint32_t thread_guest_pointer = apc->thread_ptr;
+  if (!thread_guest_pointer) {
    return 0;
  }
+  auto target_thread = context->TranslateVirtual<X_KTHREAD*>(apc->thread_ptr);
+  auto old_irql = xeKeKfAcquireSpinLock(context, &target_thread->apc_lock);

-  thread->LockApc();
-
-  if (!apc->enqueued) {
-    thread->UnlockApc(false);
-    return 0;
-  }
-
-  auto apc_list = thread->apc_list();
-  uint32_t list_entry_ptr = apc.guest_address() + 8;
-  if (apc_list->IsQueued(list_entry_ptr)) {
-    apc_list->Remove(list_entry_ptr);
+  if (apc->enqueued) {
    result = true;
+    apc->enqueued = 0;
+    util::XeRemoveEntryList(&apc->list_entry, context);
+    // todo: this is incomplete, there is more logic here in actual kernel
  }
-
-  thread->UnlockApc(true);
+  xeKeKfReleaseSpinLock(context, &target_thread->apc_lock, old_irql);

  return result ? 1 : 0;
 }
@@ -1358,7 +1561,7 @@ struct X_ERWLOCK {
  be<uint32_t> readers_entry_count;    // 0xC
  X_KEVENT writer_event;               // 0x10
  X_KSEMAPHORE reader_semaphore;       // 0x20
-  X_KSPINLOCK spin_lock;                  // 0x34
+  X_KSPINLOCK spin_lock;               // 0x34
 };
 static_assert_size(X_ERWLOCK, 0x38);

@@ -1393,8 +1596,7 @@ DECLARE_XBOXKRNL_EXPORT2(ExAcquireReadWriteLockExclusive, kThreading,

 dword_result_t ExTryToAcquireReadWriteLockExclusive_entry(
    pointer_t<X_ERWLOCK> lock_ptr, const ppc_context_t& ppc_context) {
-  auto old_irql =
-      xeKeKfAcquireSpinLock(ppc_context, &lock_ptr->spin_lock);
+  auto old_irql = xeKeKfAcquireSpinLock(ppc_context, &lock_ptr->spin_lock);

  uint32_t result;
  if (lock_ptr->lock_count < 0) {
@@ -1412,13 +1614,13 @@ DECLARE_XBOXKRNL_EXPORT1(ExTryToAcquireReadWriteLockExclusive, kThreading,

 void ExAcquireReadWriteLockShared_entry(pointer_t<X_ERWLOCK> lock_ptr,
                                        const ppc_context_t& ppc_context) {
-  auto old_irql = xeKeKfAcquireSpinLock(ppc_context, & lock_ptr->spin_lock);
+  auto old_irql = xeKeKfAcquireSpinLock(ppc_context, &lock_ptr->spin_lock);

  int32_t lock_count = ++lock_ptr->lock_count;
  if (!lock_count ||
      (lock_ptr->readers_entry_count && !lock_ptr->writers_waiting_count)) {
    lock_ptr->readers_entry_count++;
-    xeKeKfReleaseSpinLock(ppc_context, & lock_ptr->spin_lock, old_irql);
+    xeKeKfReleaseSpinLock(ppc_context, &lock_ptr->spin_lock, old_irql);
    return;
  }

@@ -1432,7 +1634,7 @@ DECLARE_XBOXKRNL_EXPORT2(ExAcquireReadWriteLockShared, kThreading, kImplemented,

 dword_result_t ExTryToAcquireReadWriteLockShared_entry(
    pointer_t<X_ERWLOCK> lock_ptr, const ppc_context_t& ppc_context) {
-  auto old_irql = xeKeKfAcquireSpinLock(ppc_context, & lock_ptr->spin_lock);
+  auto old_irql = xeKeKfAcquireSpinLock(ppc_context, &lock_ptr->spin_lock);

  uint32_t result;
  if (lock_ptr->lock_count < 0 ||
@@ -1452,7 +1654,7 @@ DECLARE_XBOXKRNL_EXPORT1(ExTryToAcquireReadWriteLockShared, kThreading,

 void ExReleaseReadWriteLock_entry(pointer_t<X_ERWLOCK> lock_ptr,
                                  const ppc_context_t& ppc_context) {
-  auto old_irql = xeKeKfAcquireSpinLock(ppc_context, & lock_ptr->spin_lock);
+  auto old_irql = xeKeKfAcquireSpinLock(ppc_context, &lock_ptr->spin_lock);

  int32_t lock_count = --lock_ptr->lock_count;

--- a/src/xenia/kernel/xboxkrnl/xboxkrnl_threading.h
+++ b/src/xenia/kernel/xboxkrnl/xboxkrnl_threading.h
@@ -38,7 +38,8 @@ uint32_t NtWaitForSingleObjectEx(uint32_t object_handle, uint32_t wait_mode,
 uint32_t xeKeSetEvent(X_KEVENT* event_ptr, uint32_t increment, uint32_t wait);

 uint32_t KeDelayExecutionThread(uint32_t processor_mode, uint32_t alertable,
-                                uint64_t* interval_ptr);
+                                uint64_t* interval_ptr,
+                                cpu::ppc::PPCContext* ctx);

 uint32_t ExCreateThread(xe::be<uint32_t>* handle_ptr, uint32_t stack_size,
                        xe::be<uint32_t>* thread_id_ptr,
@@ -53,13 +54,21 @@ uint32_t NtClose(uint32_t handle);
 void xeKeInitializeApc(XAPC* apc, uint32_t thread_ptr, uint32_t kernel_routine,
                       uint32_t rundown_routine, uint32_t normal_routine,
                       uint32_t apc_mode, uint32_t normal_context);
-
+uint32_t xeKeInsertQueueApc(XAPC* apc, uint32_t arg1, uint32_t arg2,
+                            uint32_t priority_increment,
+                            cpu::ppc::PPCContext* context);
+uint32_t xeNtQueueApcThread(uint32_t thread_handle, uint32_t apc_routine,
+                            uint32_t apc_routine_context, uint32_t arg1,
+                            uint32_t arg2, cpu::ppc::PPCContext* context);
 void xeKfLowerIrql(PPCContext* ctx, unsigned char new_irql);
 unsigned char xeKfRaiseIrql(PPCContext* ctx, unsigned char new_irql);

 void xeKeKfReleaseSpinLock(PPCContext* ctx, X_KSPINLOCK* lock, dword_t old_irql, bool change_irql=true);
 uint32_t xeKeKfAcquireSpinLock(PPCContext* ctx, X_KSPINLOCK* lock, bool change_irql=true);

+X_STATUS xeProcessUserApcs(PPCContext* ctx);
+
+void xeRundownApcs(PPCContext* ctx);

 }  // namespace xboxkrnl
 }  // namespace kernel