Several changes for timestamp bundle:

Fully defined the structure.
Single copy of it + single timer across all modules, managing it is now the responsibility of KernelState.

add global_critical_region::PrepareToAcquire, which uses Prefetchw on the global crit. We now know we can use Prefetchw on all cpus that have AVX.
add  KeQueryInterruptTime, which is used by some dashboards.

add threading::NanoSleep

src/xenia/kernel/kernel_state.cc

@@ -940,6 +940,57 @@ bool KernelState::Save(ByteStream* stream) {
   return true;
 }
 
+// this only gets triggered once per ms at most, so fields other than tick count
+// will probably not be updated in a timely manner for guest code that uses them
+void KernelState::UpdateKeTimestampBundle() {
+  X_TIME_STAMP_BUNDLE* lpKeTimeStampBundle =
+      memory_->TranslateVirtual<X_TIME_STAMP_BUNDLE*>(ke_timestamp_bundle_ptr_);
+  uint32_t uptime_ms = Clock::QueryGuestUptimeMillis();
+  xe::store_and_swap<uint64_t>(&lpKeTimeStampBundle->interrupt_time,
+                               Clock::QueryGuestInterruptTime());
+  xe::store_and_swap<uint64_t>(&lpKeTimeStampBundle->system_time,
+                               Clock::QueryGuestSystemTime());
+  xe::store_and_swap<uint32_t>(&lpKeTimeStampBundle->tick_count, uptime_ms);
+}
+
+uint32_t KernelState::GetKeTimestampBundle() {
+  XE_LIKELY_IF(ke_timestamp_bundle_ptr_) { 
+	  return ke_timestamp_bundle_ptr_; 
+  }
+  else {
+    global_critical_region::PrepareToAcquire();
+    return CreateKeTimestampBundle();
+  }
+}
+
+XE_NOINLINE
+XE_COLD
+uint32_t KernelState::CreateKeTimestampBundle() {
+  auto crit = global_critical_region::Acquire();
+
+  uint32_t pKeTimeStampBundle =
+      memory_->SystemHeapAlloc(sizeof(X_TIME_STAMP_BUNDLE));
+  X_TIME_STAMP_BUNDLE* lpKeTimeStampBundle =
+      memory_->TranslateVirtual<X_TIME_STAMP_BUNDLE*>(pKeTimeStampBundle);
+
+  xe::store_and_swap<uint64_t>(&lpKeTimeStampBundle->interrupt_time,
+                               Clock::QueryGuestInterruptTime());
+
+  xe::store_and_swap<uint64_t>(&lpKeTimeStampBundle->system_time,
+                               Clock::QueryGuestSystemTime());
+
+  xe::store_and_swap<uint32_t>(&lpKeTimeStampBundle->tick_count,
+                               Clock::QueryGuestUptimeMillis());
+
+  xe::store_and_swap<uint32_t>(&lpKeTimeStampBundle->padding, 0);
+
+  timestamp_timer_ = xe::threading::HighResolutionTimer::CreateRepeating(
+      std::chrono::milliseconds(1),
+      [this]() { this->UpdateKeTimestampBundle(); });
+  ke_timestamp_bundle_ptr_ = pKeTimeStampBundle;
+  return pKeTimeStampBundle;
+}
+
 bool KernelState::Restore(ByteStream* stream) {
   // Check the magic value.
   if (stream->Read<uint32_t>() != kKernelSaveSignature) {

src/xenia/kernel/kernel_state.h

@@ -17,6 +17,7 @@
 #include <memory>
 #include <vector>
 
+#include "achievement_manager.h"
 #include "xenia/base/bit_map.h"
 #include "xenia/base/cvar.h"
 #include "xenia/base/mutex.h"
@@ -30,7 +31,6 @@
 #include "xenia/memory.h"
 #include "xenia/vfs/virtual_file_system.h"
 #include "xenia/xbox.h"
-#include "achievement_manager.h"
 
 namespace xe {
 class ByteStream;
@@ -88,6 +88,17 @@ struct TerminateNotification {
   uint32_t priority;
 };
 
+// structure for KeTimeStampBuindle
+// a bit like the timers on KUSER_SHARED on normal win32
+// https://www.geoffchappell.com/studies/windows/km/ntoskrnl/inc/api/ntexapi_x/kuser_shared_data/index.htm
+struct X_TIME_STAMP_BUNDLE {
+  uint64_t interrupt_time;
+  // i assume system_time is in 100 ns intervals like on win32
+  uint64_t system_time;
+  uint32_t tick_count;
+  uint32_t padding;
+};
+
 class KernelState {
  public:
   explicit KernelState(Emulator* emulator);
@@ -234,6 +245,14 @@ class KernelState {
   bool Restore(ByteStream* stream);
 
   uint32_t notification_position_ = 2;
+
+  uint32_t GetKeTimestampBundle();
+
+  XE_NOINLINE
+  XE_COLD
+  uint32_t CreateKeTimestampBundle();
+  void UpdateKeTimestampBundle();
+
  private:
   void LoadKernelModule(object_ref<KernelModule> kernel_module);
 
@@ -271,7 +290,8 @@ class KernelState {
   std::list<std::function<void()>> dispatch_queue_;
 
   BitMap tls_bitmap_;
-
+  uint32_t ke_timestamp_bundle_ptr_ = 0;
+  std::unique_ptr<xe::threading::HighResolutionTimer> timestamp_timer_;
   friend class XObject;
 };
 

src/xenia/kernel/util/shim_utils.h

@@ -382,6 +382,7 @@ using pointer_t = const shim::TypedPointerParam<T>&;
 
 using int_result_t = shim::ResultBase<int32_t>;
 using dword_result_t = shim::ResultBase<uint32_t>;
+using qword_result_t = shim::ResultBase<uint64_t>;
 using pointer_result_t = shim::ResultBase<uint32_t>;
 using X_HRESULT_result_t = shim::ResultBase<X_HRESULT>;
 using ppc_context_t = shim::ContextParam;

src/xenia/kernel/xboxkrnl/xboxkrnl_misc.cc

@@ -22,7 +22,9 @@ namespace xboxkrnl {
 void KeEnableFpuExceptions_entry(
     const ppc_context_t& ctx) {  // dword_t enabled) {
   // TODO(benvanik): can we do anything about exceptions?
-  // theres a lot more thats supposed to happen here, the floating point state has to be saved to kthread, the irql changes, the machine state register is changed to enable exceptions
+  // theres a lot more thats supposed to happen here, the floating point state
+  // has to be saved to kthread, the irql changes, the machine state register is
+  // changed to enable exceptions
 
   X_KTHREAD* kthread = ctx->TranslateVirtual<X_KTHREAD*>(
       ctx->TranslateVirtualGPR<X_KPCR*>(ctx->r[13])->current_thread);
@@ -117,7 +119,15 @@ void KeSaveFloatingPointState_entry(ppc_context_t& ctx) {
 
 DECLARE_XBOXKRNL_EXPORT1(KeSaveFloatingPointState, kNone, kImplemented);
 #endif
+static qword_result_t KeQueryInterruptTime_entry(const ppc_context_t& ctx) {
+  auto kstate = ctx->kernel_state;
+  uint32_t ts_bundle = kstate->GetKeTimestampBundle();
+  X_TIME_STAMP_BUNDLE* bundle =
+      ctx->TranslateVirtual<X_TIME_STAMP_BUNDLE*>(ts_bundle);
 
+  return xe::load_and_swap<uint64_t>(&bundle->interrupt_time);
+}
+DECLARE_XBOXKRNL_EXPORT1(KeQueryInterruptTime, kNone, kImplemented);
 }  // namespace xboxkrnl
 }  // namespace kernel
 }  // namespace xe

src/xenia/kernel/xboxkrnl/xboxkrnl_module.cc

@@ -78,8 +78,7 @@ bool XboxkrnlModule::SendPIXCommand(const char* cmd) {
 }
 
 XboxkrnlModule::XboxkrnlModule(Emulator* emulator, KernelState* kernel_state)
-    : KernelModule(kernel_state, "xe:\\xboxkrnl.exe"),
-      timestamp_timer_(nullptr) {
+    : KernelModule(kernel_state, "xe:\\xboxkrnl.exe") {
   RegisterExportTable(export_resolver_);
 
   // Register all exported functions.
@@ -216,23 +215,9 @@ XboxkrnlModule::XboxkrnlModule(Emulator* emulator, KernelState* kernel_state)
   xe::store_and_swap<uint8_t>(lpXboxKrnlVersion + 6, 0x80);
   xe::store_and_swap<uint8_t>(lpXboxKrnlVersion + 7, 0x00);
 
-  // KeTimeStampBundle (ad)
-  // This must be updated during execution, at 1ms intevals.
-  // We setup a system timer here to do that.
-  uint32_t pKeTimeStampBundle = memory_->SystemHeapAlloc(24);
-  auto lpKeTimeStampBundle = memory_->TranslateVirtual(pKeTimeStampBundle);
-  export_resolver_->SetVariableMapping(
-      "xboxkrnl.exe", ordinals::KeTimeStampBundle, pKeTimeStampBundle);
-  xe::store_and_swap<uint64_t>(lpKeTimeStampBundle + 0, 0);
-  xe::store_and_swap<uint64_t>(lpKeTimeStampBundle + 8, 0);
-  xe::store_and_swap<uint32_t>(lpKeTimeStampBundle + 16,
-                               Clock::QueryGuestUptimeMillis());
-  xe::store_and_swap<uint32_t>(lpKeTimeStampBundle + 20, 0);
-  timestamp_timer_ = xe::threading::HighResolutionTimer::CreateRepeating(
-      std::chrono::milliseconds(1), [lpKeTimeStampBundle]() {
-        xe::store_and_swap<uint32_t>(lpKeTimeStampBundle + 16,
-                                     Clock::QueryGuestUptimeMillis());
-      });
+  export_resolver_->SetVariableMapping("xboxkrnl.exe",
+                                       ordinals::KeTimeStampBundle,
+                                       kernel_state->GetKeTimestampBundle());
 }
 
 static auto& get_xboxkrnl_exports() {

src/xenia/kernel/xboxkrnl/xboxkrnl_module.h

@@ -40,9 +40,6 @@ class XboxkrnlModule : public KernelModule {
 
  protected:
   uint32_t pix_function_ = 0;
-
- private:
-  std::unique_ptr<xe::threading::HighResolutionTimer> timestamp_timer_;
 };
 
 }  // namespace xboxkrnl

src/xenia/kernel/xboxkrnl/xboxkrnl_threading.cc

@@ -363,9 +363,18 @@ dword_result_t NtYieldExecution_entry() {
 DECLARE_XBOXKRNL_EXPORT2(NtYieldExecution, kThreading, kImplemented,
                          kHighFrequency);
 
-void KeQuerySystemTime_entry(lpqword_t time_ptr) {
-  uint64_t time = Clock::QueryGuestSystemTime();
+void KeQuerySystemTime_entry(lpqword_t time_ptr, const ppc_context_t& ctx) {
   if (time_ptr) {
+    // update the timestamp bundle to the time we queried.
+    // this is a race, but i don't of any sw that requires it, it just seems
+    // like we ought to keep it consistent with ketimestampbundle in case
+    // something uses this function, but also reads it directly
+    uint32_t ts_bundle = ctx->kernel_state->GetKeTimestampBundle();
+    uint64_t time = Clock::QueryGuestSystemTime();
+	//todo: cmpxchg?
+    xe::store_and_swap<uint64_t>(
+        &ctx->TranslateVirtual<X_TIME_STAMP_BUNDLE*>(ts_bundle)->system_time,
+        time);
     *time_ptr = time;
   }
 }