diff --git a/src/xenia/cpu/backend/x64/x64_seq_memory.cc b/src/xenia/cpu/backend/x64/x64_seq_memory.cc index ffccd4c31..b3abef37c 100644 --- a/src/xenia/cpu/backend/x64/x64_seq_memory.cc +++ b/src/xenia/cpu/backend/x64/x64_seq_memory.cc @@ -356,29 +356,32 @@ EMITTER_OPCODE_TABLE(OPCODE_LVR, LVR_V128); struct STVL_V128 : Sequence> { static void Emit(X64Emitter& e, const EmitArgType& i) { - e.mov(e.ecx, 15); - e.mov(e.edx, e.ecx); + Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm0); + e.StashXmm(0, src2); + + // Store bytes offset..15 from the source vector. Xenia's host vector byte + // layout is word-swapped from guest byte order, so convert source byte + // indexes with ^ 3 before reading the stashed XMM value. e.lea(e.rax, e.ptr[ComputeMemoryAddress(e, i.src1)]); + e.mov(e.ecx, 15); e.and_(e.ecx, e.eax); - e.vmovd(e.xmm0, e.ecx); + e.mov(e.edx, 15); e.not_(e.rdx); e.and_(e.rax, e.rdx); - e.vmovdqa(e.xmm1, e.GetXmmConstPtr(XMMSTVLShuffle)); - if (e.IsFeatureEnabled(kX64EmitAVX2)) { - e.vpbroadcastb(e.xmm3, e.xmm0); - } else { - e.vpshufb(e.xmm3, e.xmm0, e.GetXmmConstPtr(XMMZero)); - } - e.vpsubb(e.xmm0, e.xmm1, e.xmm3); - e.vpxor(e.xmm1, e.xmm0, - e.GetXmmConstPtr(XMMSwapWordMask)); // xmm1 from now on will be our - // selector for blend/shuffle - Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm0); - - e.vpshufb(e.xmm2, src2, e.xmm1); - e.vpblendvb(e.xmm3, e.xmm2, e.ptr[e.rax], e.xmm1); - e.vmovdqa(e.ptr[e.rax], e.xmm3); + Xbyak::Label loop, done; + e.mov(e.edx, e.ecx); + e.L(loop); + e.cmp(e.edx, 16); + e.jge(done); + e.mov(e.r8d, e.edx); + e.sub(e.r8d, e.ecx); + e.xor_(e.r8d, 3); + e.movzx(e.r9d, e.byte[e.rsp + X64Emitter::kStashOffset + e.r8]); + e.mov(e.byte[e.rax + e.rdx], e.r9b); + e.inc(e.edx); + e.jmp(loop); + e.L(done); } }; EMITTER_OPCODE_TABLE(OPCODE_STVL, STVL_V128); @@ -391,28 +394,26 @@ struct STVR_V128 : Sequence> { e.lea(e.rax, e.ptr[ComputeMemoryAddress(e, i.src1)]); e.and_(e.ecx, e.eax); e.jz(skipper); - e.vmovd(e.xmm0, e.ecx); e.not_(e.rdx); e.and_(e.rax, e.rdx); - e.vmovdqa(e.xmm1, e.GetXmmConstPtr(XMMSTVLShuffle)); - // todo: maybe a table lookup might be a better idea for getting the - // shuffle/blend - - if (e.IsFeatureEnabled(kX64EmitAVX2)) { - e.vpbroadcastb(e.xmm3, e.xmm0); - } else { - e.vpshufb(e.xmm3, e.xmm0, e.GetXmmConstPtr(XMMZero)); - } - e.vpsubb(e.xmm0, e.xmm1, e.xmm3); - e.vpxor(e.xmm1, e.xmm0, - e.GetXmmConstPtr(XMMSTVRSwapMask)); // xmm1 from now on will be our - // selector for blend/shuffle Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm0); + e.StashXmm(0, src2); - e.vpshufb(e.xmm2, src2, e.xmm1); - e.vpblendvb(e.xmm3, e.xmm2, e.ptr[e.rax], e.xmm1); - e.vmovdqa(e.ptr[e.rax], e.xmm3); + // Store bytes 0..offset-1 from the tail of the source vector. + Xbyak::Label loop; + e.xor_(e.edx, e.edx); + e.L(loop); + e.cmp(e.edx, e.ecx); + e.jge(skipper); + e.mov(e.r8d, 16); + e.sub(e.r8d, e.ecx); + e.add(e.r8d, e.edx); + e.xor_(e.r8d, 3); + e.movzx(e.r9d, e.byte[e.rsp + X64Emitter::kStashOffset + e.r8]); + e.mov(e.byte[e.rax + e.rdx], e.r9b); + e.inc(e.edx); + e.jmp(loop); e.L(skipper); } }; diff --git a/src/xenia/cpu/testing/misc_opcode_test.cc b/src/xenia/cpu/testing/misc_opcode_test.cc index c821cacaa..a68f1826b 100644 --- a/src/xenia/cpu/testing/misc_opcode_test.cc +++ b/src/xenia/cpu/testing/misc_opcode_test.cc @@ -13,6 +13,7 @@ #include "xenia/cpu/testing/util.h" +#include #include #include #include @@ -629,3 +630,140 @@ TEST_CASE("LOAD_VECTOR_RIGHT", "[memory]") { test.memory->SystemHeapFree(guest_addr); } + +// ============================================================================ +// STVL / STVR - partial vector store left/right +// These are used by optimized memcpy implementations for unaligned heads/tails. +// ============================================================================ +constexpr std::array kStoreVectorSource = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; +constexpr uint32_t kPartialStoreOffsets[] = {0, 1, 4, 8, 12, 15}; + +TEST_CASE("STORE_VECTOR_LEFT", "[memory]") { + TestFunction test([](HIRBuilder& b) { + b.StoreVectorLeft(LoadGPR(b, 4), LoadVR(b, 5)); + b.Return(); + }); + + for (const uint32_t offset : kPartialStoreOffsets) { + uint32_t guest_addr = test.memory->SystemHeapAlloc(64, 16); + REQUIRE(guest_addr != 0); + const uint32_t aligned_addr = (guest_addr + 15) & ~15u; + auto* host_ptr = + reinterpret_cast(test.memory->TranslateVirtual(aligned_addr)); + + std::array expected; + for (uint32_t i = 0; i < 16; ++i) { + host_ptr[i] = static_cast(0xA0 + i); + expected[i] = host_ptr[i]; + } + + for (uint32_t i = offset; i < 16; ++i) { + expected[i] = kStoreVectorSource[(i - offset) ^ 3]; + } + + test.Run( + [&](PPCContext* ctx) { + ctx->r[4] = aligned_addr + offset; + ctx->v[5] = + vec128b(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + }, + [&](PPCContext* ctx) { + REQUIRE(std::memcmp(host_ptr, expected.data(), expected.size()) == 0); + }); + + test.memory->SystemHeapFree(guest_addr); + } +} + +TEST_CASE("STORE_VECTOR_RIGHT", "[memory]") { + TestFunction test([](HIRBuilder& b) { + b.StoreVectorRight(LoadGPR(b, 4), LoadVR(b, 5)); + b.Return(); + }); + + for (const uint32_t offset : kPartialStoreOffsets) { + uint32_t guest_addr = test.memory->SystemHeapAlloc(64, 16); + REQUIRE(guest_addr != 0); + const uint32_t aligned_addr = (guest_addr + 15) & ~15u; + auto* host_ptr = + reinterpret_cast(test.memory->TranslateVirtual(aligned_addr)); + + std::array expected; + for (uint32_t i = 0; i < 16; ++i) { + host_ptr[i] = static_cast(0xA0 + i); + expected[i] = host_ptr[i]; + } + + for (uint32_t i = 0; i < offset; ++i) { + expected[i] = kStoreVectorSource[(16 - offset + i) ^ 3]; + } + + test.Run( + [&](PPCContext* ctx) { + ctx->r[4] = aligned_addr + offset; + ctx->v[5] = + vec128b(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + }, + [&](PPCContext* ctx) { + REQUIRE(std::memcmp(host_ptr, expected.data(), expected.size()) == 0); + }); + + test.memory->SystemHeapFree(guest_addr); + } +} + +TEST_CASE("STORE_VECTOR_LEFT_MEMCPY_HEAD", "[memory]") { + TestFunction test([](HIRBuilder& b) { + auto src_addr = LoadGPR(b, 4); + auto dest_addr = LoadGPR(b, 5); + auto aligned_src = + b.And(src_addr, b.LoadConstantInt64(static_cast(~0xFULL))); + auto next_src = b.And(b.Add(src_addr, b.LoadConstantInt64(15)), + b.LoadConstantInt64(static_cast(~0xFULL))); + auto source_a = b.ByteSwap(b.Load(aligned_src, VEC128_TYPE)); + auto source_b = b.ByteSwap(b.Load(next_src, VEC128_TYPE)); + auto control = b.LoadVectorShl( + b.Truncate(b.And(src_addr, b.LoadConstantInt64(0xF)), INT8_TYPE)); + auto head = b.Permute(control, source_a, source_b, INT8_TYPE); + b.StoreVectorLeft(dest_addr, head); + b.Return(); + }); + + for (const uint32_t source_offset : {0u, 1u, 4u, 12u}) { + uint32_t guest_addr = test.memory->SystemHeapAlloc(128, 16); + REQUIRE(guest_addr != 0); + const uint32_t aligned_addr = (guest_addr + 15) & ~15u; + auto* host_ptr = + reinterpret_cast(test.memory->TranslateVirtual(aligned_addr)); + auto* src_ptr = host_ptr; + auto* dest_block = host_ptr + 64; + const uint32_t src_addr = aligned_addr + source_offset; + const uint32_t dest_addr = aligned_addr + 64 + 12; + + for (uint32_t i = 0; i < 64; ++i) { + src_ptr[i] = static_cast(i); + } + for (uint32_t i = 0; i < 16; ++i) { + dest_block[i] = static_cast(0xC0 + i); + } + + std::array expected; + std::memcpy(expected.data(), dest_block, expected.size()); + for (uint32_t i = 0; i < 4; ++i) { + expected[12 + i] = src_ptr[source_offset + i]; + } + + test.Run( + [&](PPCContext* ctx) { + ctx->r[4] = src_addr; + ctx->r[5] = dest_addr; + }, + [&](PPCContext* ctx) { + REQUIRE(std::memcmp(dest_block, expected.data(), expected.size()) == + 0); + }); + + test.memory->SystemHeapFree(guest_addr); + } +}