Attempt to emulate reserved load/store more closely. can't do anything for stores of the same value that are done via a non-reserved store to a reserved location

uses a bitmap that splits up the memory space into 65k blocks per bit. Currently is using the guest virtual address but should be using physical addresses instead. Currently if a guest does a reserve on a location and then a reserved store to a totally different location we trigger a breakpoint. This should never happen Also removed the NEGATED_MUL_blah operations. They weren't necessary, nothing special is needed for the negated result variants. Added a log message for when watched physical memory has a race, it just would be nice to know when it happens and in what games.
2023-04-15 16:06:07 -04:00
parent 5e0c67438c
commit 7fb4b4cd41
12 changed files with 353 additions and 187 deletions
--- a/src/xenia/cpu/backend/x64/x64_backend.cc
+++ b/src/xenia/cpu/backend/x64/x64_backend.cc
@@ -70,6 +70,9 @@ class X64HelperEmitter : public X64Emitter {
  void* EmitGuestAndHostSynchronizeStackSizeLoadThunk(
      void* sync_func, unsigned stack_element_size);

+  void* EmitTryAcquireReservationHelper();
+  void* EmitReservedStoreHelper(bool bit64 = false);
+
 private:
  void* EmitCurrentForOffsets(const _code_offsets& offsets,
                              size_t stack_size = 0);
@@ -226,6 +229,10 @@ bool X64Backend::Initialize(Processor* processor) {
        thunk_emitter.EmitGuestAndHostSynchronizeStackSizeLoadThunk(
            synchronize_guest_and_host_stack_helper_, 4);
  }
+  try_acquire_reservation_helper_ =
+      thunk_emitter.EmitTryAcquireReservationHelper();
+  reserved_store_32_helper = thunk_emitter.EmitReservedStoreHelper(false);
+  reserved_store_64_helper = thunk_emitter.EmitReservedStoreHelper(true);

  // Set the code cache to use the ResolveFunction thunk for default
  // indirections.
@@ -799,7 +806,7 @@ void* X64HelperEmitter::EmitGuestAndHostSynchronizeStackHelper() {
  inc(ecx);
  jmp(checkbp, T_NEAR);
  L(we_good);
-  //we're popping this return address, so go down by one
+  // we're popping this return address, so go down by one
  sub(edx, sizeof(X64BackendStackpoint));
  dec(ecx);
  L(checkbp);
@@ -857,6 +864,123 @@ void* X64HelperEmitter::EmitGuestAndHostSynchronizeStackSizeLoadThunk(
  code_offsets.tail = getSize();
  return EmitCurrentForOffsets(code_offsets);
 }
+void* X64HelperEmitter::EmitTryAcquireReservationHelper() {
+  _code_offsets code_offsets = {};
+  code_offsets.prolog = getSize();
+
+  Xbyak::Label already_has_a_reservation;
+  Xbyak::Label acquire_new_reservation;
+
+  btr(GetBackendFlagsPtr(), 1);
+  mov(r8, GetBackendCtxPtr(offsetof(X64BackendContext, reserve_helper_)));
+  jc(already_has_a_reservation);
+
+  shr(ecx, RESERVE_BLOCK_SHIFT);
+  xor_(r9d, r9d);
+  mov(edx, ecx);
+  shr(edx, 6);  // divide by 64
+  lea(rdx, ptr[r8 + rdx * 8]);
+  and_(ecx, 64 - 1);
+
+  lock();
+  bts(qword[rdx], rcx);
+  // DebugBreak();
+  // set flag on local backend context for thread to indicate our previous
+  // attempt to get the reservation succeeded
+  setnc(r9b);  // success = bitmap did not have a set bit at the idx
+  shl(r9b, 1);
+
+  mov(GetBackendCtxPtr(offsetof(X64BackendContext, cached_reserve_offset)),
+      rdx);
+  mov(GetBackendCtxPtr(offsetof(X64BackendContext, cached_reserve_bit)), ecx);
+
+  or_(GetBackendCtxPtr(offsetof(X64BackendContext, flags)), r9d);
+  ret();
+  L(already_has_a_reservation);
+  DebugBreak();
+
+  code_offsets.prolog_stack_alloc = getSize();
+  code_offsets.body = getSize();
+  code_offsets.epilog = getSize();
+  code_offsets.tail = getSize();
+  return EmitCurrentForOffsets(code_offsets);
+}
+// ecx=guest addr
+// r9 = host addr
+// r8 = value
+// if ZF is set and CF is set, we succeeded
+void* X64HelperEmitter::EmitReservedStoreHelper(bool bit64) {
+  _code_offsets code_offsets = {};
+  code_offsets.prolog = getSize();
+  Xbyak::Label done;
+  Xbyak::Label reservation_isnt_for_our_addr;
+  // carry must be set + zero flag must be set
+
+  btr(GetBackendFlagsPtr(), 1);
+
+  jnc(done);
+
+  // mov(edx, i.src1.reg().cvt32());
+  mov(rax, GetBackendCtxPtr(offsetof(X64BackendContext, reserve_helper_)));
+
+  shr(ecx, RESERVE_BLOCK_SHIFT);
+  mov(edx, ecx);
+  shr(edx, 6);  // divide by 64
+  lea(rdx, ptr[rax + rdx * 8]);
+  // begin acquiring exclusive access to cacheline containing our bit
+  prefetchw(ptr[rdx]);
+
+  cmp(GetBackendCtxPtr(offsetof(X64BackendContext, cached_reserve_offset)),
+      rdx);
+  jnz(reservation_isnt_for_our_addr);
+
+  mov(rax,
+      GetBackendCtxPtr(offsetof(X64BackendContext, cached_reserve_value_)));
+
+  // we need modulo bitsize, it turns out bittests' modulus behavior for the
+  // bitoffset only applies for register operands, for memory ones we bug out
+  // todo: actually, the above note may not be true, double check it
+  and_(ecx, 64 - 1);
+  cmp(GetBackendCtxPtr(offsetof(X64BackendContext, cached_reserve_bit)), ecx);
+  jnz(reservation_isnt_for_our_addr);
+
+  // was our memory modified by kernel code or something?
+  lock();
+  if (bit64) {
+    cmpxchg(ptr[r9], r8);
+
+  } else {
+    cmpxchg(ptr[r9], r8d);
+  }
+  // the ZF flag is unaffected by BTR! we exploit this for the retval
+
+  // cancel our lock on the 65k block
+  lock();
+  btr(qword[rdx], rcx);
+
+  // Xbyak::Label check_fucky;
+  jc(done);
+  DebugBreak();
+
+  // L(check_fucky);
+
+  L(done);
+
+  // i don't care that theres a dependency on the prev value of rax atm
+  // sadly theres no CF&ZF condition code
+  setz(al);
+  setc(ah);
+  cmp(ax, 0x0101);
+  ret();
+  L(reservation_isnt_for_our_addr);
+  DebugBreak();
+  code_offsets.prolog_stack_alloc = getSize();
+  code_offsets.body = getSize();
+  code_offsets.epilog = getSize();
+  code_offsets.tail = getSize();
+  return EmitCurrentForOffsets(code_offsets);
+}
+
 void X64HelperEmitter::EmitSaveVolatileRegs() {
  // Save off volatile registers.
  // mov(qword[rsp + offsetof(StackLayout::Thunk, r[0])], rax);
@@ -975,6 +1099,7 @@ void X64Backend::InitializeBackendContext(void* ctx) {
  // https://media.discordapp.net/attachments/440280035056943104/1000765256643125308/unknown.png
  bctx->Ox1000 = 0x1000;
  bctx->guest_tick_count = Clock::GetGuestTickCountPointer();
+  bctx->reserve_helper_ = &reserve_helper_;
 }
 void X64Backend::DeinitializeBackendContext(void* ctx) {
  X64BackendContext* bctx = BackendContextForGuestContext(ctx);
--- a/src/xenia/cpu/backend/x64/x64_backend.h
+++ b/src/xenia/cpu/backend/x64/x64_backend.h
@@ -42,6 +42,17 @@ typedef void* (*HostToGuestThunk)(void* target, void* arg0, void* arg1);
 typedef void* (*GuestToHostThunk)(void* target, void* arg0, void* arg1);
 typedef void (*ResolveFunctionThunk)();

+#define RESERVE_BLOCK_SHIFT 16
+
+#define RESERVE_NUM_ENTRIES \
+  ((1024ULL * 1024ULL * 1024ULL * 4ULL) >> RESERVE_BLOCK_SHIFT)
+// https://codalogic.com/blog/2022/12/06/Exploring-PowerPCs-read-modify-write-operations
+struct ReserveHelper {
+  uint64_t blocks[RESERVE_NUM_ENTRIES / 64];
+
+  ReserveHelper() { memset(blocks, 0, sizeof(blocks)); }
+};
+
 struct X64BackendStackpoint {
  uint64_t host_stack_;
  unsigned guest_stack_;
@@ -55,16 +66,21 @@ struct X64BackendStackpoint {
 // context (somehow placing a global X64BackendCtx prior to membase, so we can
 // negatively index the membase reg)
 struct X64BackendContext {
+  ReserveHelper* reserve_helper_;
+  uint64_t cached_reserve_value_;
  // guest_tick_count is used if inline_loadclock is used
  uint64_t* guest_tick_count;
  // records mapping of host_stack to guest_stack
  X64BackendStackpoint* stackpoints;
-
+  uint64_t cached_reserve_offset;
+  uint32_t cached_reserve_bit;
  unsigned int current_stackpoint_depth;
  unsigned int mxcsr_fpu;  // currently, the way we implement rounding mode
                           // affects both vmx and the fpu
  unsigned int mxcsr_vmx;
-  unsigned int flags;   // bit 0 = 0 if mxcsr is fpu, else it is vmx
+  // bit 0 = 0 if mxcsr is fpu, else it is vmx
+  // bit 1 = got reserve
+  unsigned int flags;
  unsigned int Ox1000;  // constant 0x1000 so we can shrink each tail emitted
                        // add of it by... 2 bytes lol
 };
@@ -152,9 +168,18 @@ class X64Backend : public Backend {
  void* synchronize_guest_and_host_stack_helper_size8_ = nullptr;
  void* synchronize_guest_and_host_stack_helper_size16_ = nullptr;
  void* synchronize_guest_and_host_stack_helper_size32_ = nullptr;
+
+ public:
+  void* try_acquire_reservation_helper_ = nullptr;
+  void* reserved_store_32_helper = nullptr;
+  void* reserved_store_64_helper = nullptr;
+
+ private:
 #if XE_X64_PROFILER_AVAILABLE == 1
  GuestProfilerData profiler_data_;
 #endif
+
+  alignas(64) ReserveHelper reserve_helper_;
 };

 }  // namespace x64
--- a/src/xenia/cpu/backend/x64/x64_seq_memory.cc
+++ b/src/xenia/cpu/backend/x64/x64_seq_memory.cc
@@ -387,7 +387,6 @@ struct LVL_V128 : Sequence<LVL_V128, I<OPCODE_LVL, V128Op, I64Op>> {
 };
 EMITTER_OPCODE_TABLE(OPCODE_LVL, LVL_V128);

-
 struct LVR_V128 : Sequence<LVR_V128, I<OPCODE_LVR, V128Op, I64Op>> {
  static void Emit(X64Emitter& e, const EmitArgType& i) {
    Xbyak::Label endpoint{};
@@ -483,6 +482,84 @@ struct STVR_V128 : Sequence<STVR_V128, I<OPCODE_STVR, VoidOp, I64Op, V128Op>> {
  }
 };
 EMITTER_OPCODE_TABLE(OPCODE_STVR, STVR_V128);
+
+struct RESERVED_LOAD_INT32
+    : Sequence<RESERVED_LOAD_INT32, I<OPCODE_RESERVED_LOAD, I32Op, I64Op>> {
+  static void Emit(X64Emitter& e, const EmitArgType& i) {
+    // should use phys addrs, not virtual addrs!
+
+    // try_acquire_reservation_helper_ doesnt spoil rax
+    e.lea(e.rax, e.ptr[ComputeMemoryAddress(e, i.src1)]);
+    // begin acquiring exclusive access to the location
+    // we will do a load first, but we'll need exclusive access once we do our
+    // atomic op in the store
+    e.prefetchw(e.ptr[e.rax]);
+    e.mov(e.ecx, i.src1.reg().cvt32());
+    e.call(e.backend()->try_acquire_reservation_helper_);
+    e.mov(i.dest, e.dword[e.rax]);
+
+    e.mov(
+        e.GetBackendCtxPtr(offsetof(X64BackendContext, cached_reserve_value_)),
+        i.dest.reg().cvt64());
+  }
+};
+
+struct RESERVED_LOAD_INT64
+    : Sequence<RESERVED_LOAD_INT64, I<OPCODE_RESERVED_LOAD, I64Op, I64Op>> {
+  static void Emit(X64Emitter& e, const EmitArgType& i) {
+    // try_acquire_reservation_helper_ doesnt spoil rax
+    e.lea(e.rax, e.ptr[ComputeMemoryAddress(e, i.src1)]);
+    e.mov(e.ecx, i.src1.reg().cvt32());
+    // begin acquiring exclusive access to the location
+    // we will do a load first, but we'll need exclusive access once we do our
+    // atomic op in the store
+    e.prefetchw(e.ptr[e.rax]);
+
+    e.call(e.backend()->try_acquire_reservation_helper_);
+    e.mov(i.dest, e.qword[ComputeMemoryAddress(e, i.src1)]);
+
+    e.mov(
+        e.GetBackendCtxPtr(offsetof(X64BackendContext, cached_reserve_value_)),
+        i.dest.reg());
+  }
+};
+
+EMITTER_OPCODE_TABLE(OPCODE_RESERVED_LOAD, RESERVED_LOAD_INT32,
+                     RESERVED_LOAD_INT64);
+
+// address, value
+
+struct RESERVED_STORE_INT32
+    : Sequence<RESERVED_STORE_INT32,
+               I<OPCODE_RESERVED_STORE, I8Op, I64Op, I32Op>> {
+  static void Emit(X64Emitter& e, const EmitArgType& i) {
+    // edx=guest addr
+    // r9 = host addr
+    // r8 = value
+    // if ZF is set and CF is set, we succeeded
+    e.mov(e.ecx, i.src1.reg().cvt32());
+    e.lea(e.r9, e.ptr[ComputeMemoryAddress(e, i.src1)]);
+    e.mov(e.r8d, i.src2);
+    e.call(e.backend()->reserved_store_32_helper);
+    e.setz(i.dest);
+  }
+};
+
+struct RESERVED_STORE_INT64
+    : Sequence<RESERVED_STORE_INT64,
+               I<OPCODE_RESERVED_STORE, I8Op, I64Op, I64Op>> {
+  static void Emit(X64Emitter& e, const EmitArgType& i) {
+    e.mov(e.ecx, i.src1.reg().cvt32());
+    e.lea(e.r9, e.ptr[ComputeMemoryAddress(e, i.src1)]);
+    e.mov(e.r8, i.src2);
+    e.call(e.backend()->reserved_store_64_helper);
+    e.setz(i.dest);
+  }
+};
+
+EMITTER_OPCODE_TABLE(OPCODE_RESERVED_STORE, RESERVED_STORE_INT32,
+                     RESERVED_STORE_INT64);
+
 // ============================================================================
 // OPCODE_ATOMIC_COMPARE_EXCHANGE
 // ============================================================================
--- a/src/xenia/cpu/backend/x64/x64_sequences.cc
+++ b/src/xenia/cpu/backend/x64/x64_sequences.cc
@@ -1018,8 +1018,7 @@ struct COMPARE_EQ_F32
    e.ChangeMxcsrMode(MXCSRMode::Fpu);
    if (!HasPrecedingCmpOfSameValues(i.instr)) {
      EmitCommutativeBinaryXmmOp(
-          e, i,
-          [](X64Emitter& e, I8Op dest, const Xmm& src1, const Xmm& src2) {
+          e, i, [](X64Emitter& e, I8Op dest, const Xmm& src1, const Xmm& src2) {
            e.vcomiss(src1, src2);
          });
    }
@@ -1032,8 +1031,7 @@ struct COMPARE_EQ_F64
    e.ChangeMxcsrMode(MXCSRMode::Fpu);
    if (!HasPrecedingCmpOfSameValues(i.instr)) {
      EmitCommutativeBinaryXmmOp(
-          e, i,
-          [](X64Emitter& e, I8Op dest, const Xmm& src1, const Xmm& src2) {
+          e, i, [](X64Emitter& e, I8Op dest, const Xmm& src1, const Xmm& src2) {
            e.vcomisd(src1, src2);
          });
    }
@@ -1935,53 +1933,6 @@ struct MUL_ADD_V128
 };
 EMITTER_OPCODE_TABLE(OPCODE_MUL_ADD, MUL_ADD_F32, MUL_ADD_F64, MUL_ADD_V128);

-struct NEGATED_MUL_ADD_F64
-    : Sequence<NEGATED_MUL_ADD_F64,
-               I<OPCODE_NEGATED_MUL_ADD, F64Op, F64Op, F64Op, F64Op>> {
-  static void Emit(X64Emitter& e, const EmitArgType& i) {
-    e.ChangeMxcsrMode(MXCSRMode::Fpu);
-
-    Xmm src1 = GetInputRegOrConstant(e, i.src1, e.xmm0);
-    Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm1);
-    Xmm src3 = GetInputRegOrConstant(e, i.src3, e.xmm2);
-    if (e.IsFeatureEnabled(kX64EmitFMA)) {
-      // todo: this is garbage
-      e.vmovapd(e.xmm3, src1);
-      e.vfmadd213sd(e.xmm3, src2, src3);
-      e.vxorpd(i.dest, e.xmm3, e.GetXmmConstPtr(XMMSignMaskPD));
-    } else {
-      // todo: might need to use x87 in this case...
-      e.vmulsd(e.xmm3, src1, src2);
-      e.vaddsd(i.dest, e.xmm3, src3);
-      e.vxorpd(i.dest, i.dest, e.GetXmmConstPtr(XMMSignMaskPD));
-    }
-  }
-};
-struct NEGATED_MUL_ADD_V128
-    : Sequence<NEGATED_MUL_ADD_V128,
-               I<OPCODE_NEGATED_MUL_ADD, V128Op, V128Op, V128Op, V128Op>> {
-  static void Emit(X64Emitter& e, const EmitArgType& i) {
-    e.ChangeMxcsrMode(MXCSRMode::Vmx);
-
-    Xmm src1 = GetInputRegOrConstant(e, i.src1, e.xmm0);
-    Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm1);
-    Xmm src3 = GetInputRegOrConstant(e, i.src3, e.xmm2);
-    if (e.IsFeatureEnabled(kX64EmitFMA)) {
-      // todo: this is garbage
-      e.vmovaps(e.xmm3, src1);
-      e.vfmadd213ps(e.xmm3, src2, src3);
-      e.vxorps(i.dest, e.xmm3, e.GetXmmConstPtr(XMMSignMaskPS));
-    } else {
-      // todo: might need to use x87 in this case...
-      e.vmulps(e.xmm3, src1, src2);
-      e.vaddps(i.dest, e.xmm3, src3);
-      e.vxorps(i.dest, i.dest, e.GetXmmConstPtr(XMMSignMaskPS));
-    }
-  }
-};
-EMITTER_OPCODE_TABLE(OPCODE_NEGATED_MUL_ADD, NEGATED_MUL_ADD_F64,
-                     NEGATED_MUL_ADD_V128);
-
 // ============================================================================
 // OPCODE_MUL_SUB
 // ============================================================================
@@ -2038,53 +1989,6 @@ struct MUL_SUB_V128
 };
 EMITTER_OPCODE_TABLE(OPCODE_MUL_SUB, MUL_SUB_F64, MUL_SUB_V128);

-struct NEGATED_MUL_SUB_F64
-    : Sequence<NEGATED_MUL_SUB_F64,
-               I<OPCODE_NEGATED_MUL_SUB, F64Op, F64Op, F64Op, F64Op>> {
-  static void Emit(X64Emitter& e, const EmitArgType& i) {
-    e.ChangeMxcsrMode(MXCSRMode::Fpu);
-
-    Xmm src1 = GetInputRegOrConstant(e, i.src1, e.xmm0);
-    Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm1);
-    Xmm src3 = GetInputRegOrConstant(e, i.src3, e.xmm2);
-    if (e.IsFeatureEnabled(kX64EmitFMA)) {
-      // todo: this is garbage
-      e.vmovapd(e.xmm3, src1);
-      e.vfmsub213sd(e.xmm3, src2, src3);
-      e.vxorpd(i.dest, e.xmm3, e.GetXmmConstPtr(XMMSignMaskPD));
-    } else {
-      // todo: might need to use x87 in this case...
-      e.vmulsd(e.xmm3, src1, src2);
-      e.vsubsd(i.dest, e.xmm3, src3);
-      e.vxorpd(i.dest, i.dest, e.GetXmmConstPtr(XMMSignMaskPD));
-    }
-  }
-};
-struct NEGATED_MUL_SUB_V128
-    : Sequence<NEGATED_MUL_SUB_V128,
-               I<OPCODE_NEGATED_MUL_SUB, V128Op, V128Op, V128Op, V128Op>> {
-  static void Emit(X64Emitter& e, const EmitArgType& i) {
-    e.ChangeMxcsrMode(MXCSRMode::Vmx);
-
-    Xmm src1 = GetInputRegOrConstant(e, i.src1, e.xmm0);
-    Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm1);
-    Xmm src3 = GetInputRegOrConstant(e, i.src3, e.xmm2);
-    if (e.IsFeatureEnabled(kX64EmitFMA)) {
-      // todo: this is garbage
-      e.vmovaps(e.xmm3, src1);
-      e.vfmsub213ps(e.xmm3, src2, src3);
-      e.vxorps(i.dest, e.xmm3, e.GetXmmConstPtr(XMMSignMaskPS));
-    } else {
-      // todo: might need to use x87 in this case...
-      e.vmulps(e.xmm3, src1, src2);
-      e.vsubps(i.dest, e.xmm3, src3);
-      e.vxorps(i.dest, i.dest, e.GetXmmConstPtr(XMMSignMaskPS));
-    }
-  }
-};
-EMITTER_OPCODE_TABLE(OPCODE_NEGATED_MUL_SUB, NEGATED_MUL_SUB_F64,
-                     NEGATED_MUL_SUB_V128);
-
 // ============================================================================
 // OPCODE_NEG
 // ============================================================================
@@ -2641,7 +2545,8 @@ void EmitAndNotXX(X64Emitter& e, const ARGS& i) {
    // src1 constant.
    // `and` instruction only supports up to 32-bit immediate constants
    // 64-bit constants will need a temp register
-	  //only possible with 64 bit inputs, andc is the only instruction that generates this
+    // only possible with 64 bit inputs, andc is the only instruction that
+    // generates this
    auto temp = GetTempReg<typename decltype(i.src1)::reg_type>(e);
    e.mov(temp, i.src1.constant());