[CPU] fix correct partial vector stores
Replace the x64 STVL/STVR lowering with explicit byte stores that preserve the guest-visible vector byte order. The previous vector shuffle/blend path could write incorrect bytes for partial vector stores, which broke optimized guest memcpy implementations using stvlx/stvrx for unaligned heads and tails. In 4156081C this corrupted skinned model render command data and eventually caused render-thread crashes. Add regression coverage for StoreVectorLeft/StoreVectorRight across common unaligned offsets, plus a memcpy-head style case matching the affected guest instruction pattern.
This commit is contained in:
Michael Oliver
committed by
Radosław Gliński
Radosław Gliński
parent
2027f50b6e
commit
30ac9d7be6
@@ -356,29 +356,32 @@ EMITTER_OPCODE_TABLE(OPCODE_LVR, LVR_V128);
|
||||
|
||||
struct STVL_V128 : Sequence<STVL_V128, I<OPCODE_STVL, VoidOp, I64Op, V128Op>> {
|
||||
static void Emit(X64Emitter& e, const EmitArgType& i) {
|
||||
e.mov(e.ecx, 15);
|
||||
e.mov(e.edx, e.ecx);
|
||||
Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm0);
|
||||
e.StashXmm(0, src2);
|
||||
|
||||
// Store bytes offset..15 from the source vector. Xenia's host vector byte
|
||||
// layout is word-swapped from guest byte order, so convert source byte
|
||||
// indexes with ^ 3 before reading the stashed XMM value.
|
||||
e.lea(e.rax, e.ptr[ComputeMemoryAddress(e, i.src1)]);
|
||||
e.mov(e.ecx, 15);
|
||||
e.and_(e.ecx, e.eax);
|
||||
e.vmovd(e.xmm0, e.ecx);
|
||||
e.mov(e.edx, 15);
|
||||
e.not_(e.rdx);
|
||||
e.and_(e.rax, e.rdx);
|
||||
e.vmovdqa(e.xmm1, e.GetXmmConstPtr(XMMSTVLShuffle));
|
||||
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
|
||||
e.vpbroadcastb(e.xmm3, e.xmm0);
|
||||
} else {
|
||||
e.vpshufb(e.xmm3, e.xmm0, e.GetXmmConstPtr(XMMZero));
|
||||
}
|
||||
e.vpsubb(e.xmm0, e.xmm1, e.xmm3);
|
||||
e.vpxor(e.xmm1, e.xmm0,
|
||||
e.GetXmmConstPtr(XMMSwapWordMask)); // xmm1 from now on will be our
|
||||
// selector for blend/shuffle
|
||||
|
||||
Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm0);
|
||||
|
||||
e.vpshufb(e.xmm2, src2, e.xmm1);
|
||||
e.vpblendvb(e.xmm3, e.xmm2, e.ptr[e.rax], e.xmm1);
|
||||
e.vmovdqa(e.ptr[e.rax], e.xmm3);
|
||||
Xbyak::Label loop, done;
|
||||
e.mov(e.edx, e.ecx);
|
||||
e.L(loop);
|
||||
e.cmp(e.edx, 16);
|
||||
e.jge(done);
|
||||
e.mov(e.r8d, e.edx);
|
||||
e.sub(e.r8d, e.ecx);
|
||||
e.xor_(e.r8d, 3);
|
||||
e.movzx(e.r9d, e.byte[e.rsp + X64Emitter::kStashOffset + e.r8]);
|
||||
e.mov(e.byte[e.rax + e.rdx], e.r9b);
|
||||
e.inc(e.edx);
|
||||
e.jmp(loop);
|
||||
e.L(done);
|
||||
}
|
||||
};
|
||||
EMITTER_OPCODE_TABLE(OPCODE_STVL, STVL_V128);
|
||||
@@ -391,28 +394,26 @@ struct STVR_V128 : Sequence<STVR_V128, I<OPCODE_STVR, VoidOp, I64Op, V128Op>> {
|
||||
e.lea(e.rax, e.ptr[ComputeMemoryAddress(e, i.src1)]);
|
||||
e.and_(e.ecx, e.eax);
|
||||
e.jz(skipper);
|
||||
e.vmovd(e.xmm0, e.ecx);
|
||||
e.not_(e.rdx);
|
||||
e.and_(e.rax, e.rdx);
|
||||
e.vmovdqa(e.xmm1, e.GetXmmConstPtr(XMMSTVLShuffle));
|
||||
// todo: maybe a table lookup might be a better idea for getting the
|
||||
// shuffle/blend
|
||||
|
||||
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
|
||||
e.vpbroadcastb(e.xmm3, e.xmm0);
|
||||
} else {
|
||||
e.vpshufb(e.xmm3, e.xmm0, e.GetXmmConstPtr(XMMZero));
|
||||
}
|
||||
e.vpsubb(e.xmm0, e.xmm1, e.xmm3);
|
||||
e.vpxor(e.xmm1, e.xmm0,
|
||||
e.GetXmmConstPtr(XMMSTVRSwapMask)); // xmm1 from now on will be our
|
||||
// selector for blend/shuffle
|
||||
|
||||
Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm0);
|
||||
e.StashXmm(0, src2);
|
||||
|
||||
e.vpshufb(e.xmm2, src2, e.xmm1);
|
||||
e.vpblendvb(e.xmm3, e.xmm2, e.ptr[e.rax], e.xmm1);
|
||||
e.vmovdqa(e.ptr[e.rax], e.xmm3);
|
||||
// Store bytes 0..offset-1 from the tail of the source vector.
|
||||
Xbyak::Label loop;
|
||||
e.xor_(e.edx, e.edx);
|
||||
e.L(loop);
|
||||
e.cmp(e.edx, e.ecx);
|
||||
e.jge(skipper);
|
||||
e.mov(e.r8d, 16);
|
||||
e.sub(e.r8d, e.ecx);
|
||||
e.add(e.r8d, e.edx);
|
||||
e.xor_(e.r8d, 3);
|
||||
e.movzx(e.r9d, e.byte[e.rsp + X64Emitter::kStashOffset + e.r8]);
|
||||
e.mov(e.byte[e.rax + e.rdx], e.r9b);
|
||||
e.inc(e.edx);
|
||||
e.jmp(loop);
|
||||
e.L(skipper);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
|
||||
#include "xenia/cpu/testing/util.h"
|
||||
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <cmath>
|
||||
#include <cstdlib>
|
||||
@@ -629,3 +630,140 @@ TEST_CASE("LOAD_VECTOR_RIGHT", "[memory]") {
|
||||
|
||||
test.memory->SystemHeapFree(guest_addr);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// STVL / STVR - partial vector store left/right
|
||||
// These are used by optimized memcpy implementations for unaligned heads/tails.
|
||||
// ============================================================================
|
||||
constexpr std::array<uint8_t, 16> kStoreVectorSource = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
|
||||
constexpr uint32_t kPartialStoreOffsets[] = {0, 1, 4, 8, 12, 15};
|
||||
|
||||
TEST_CASE("STORE_VECTOR_LEFT", "[memory]") {
|
||||
TestFunction test([](HIRBuilder& b) {
|
||||
b.StoreVectorLeft(LoadGPR(b, 4), LoadVR(b, 5));
|
||||
b.Return();
|
||||
});
|
||||
|
||||
for (const uint32_t offset : kPartialStoreOffsets) {
|
||||
uint32_t guest_addr = test.memory->SystemHeapAlloc(64, 16);
|
||||
REQUIRE(guest_addr != 0);
|
||||
const uint32_t aligned_addr = (guest_addr + 15) & ~15u;
|
||||
auto* host_ptr =
|
||||
reinterpret_cast<uint8_t*>(test.memory->TranslateVirtual(aligned_addr));
|
||||
|
||||
std::array<uint8_t, 16> expected;
|
||||
for (uint32_t i = 0; i < 16; ++i) {
|
||||
host_ptr[i] = static_cast<uint8_t>(0xA0 + i);
|
||||
expected[i] = host_ptr[i];
|
||||
}
|
||||
|
||||
for (uint32_t i = offset; i < 16; ++i) {
|
||||
expected[i] = kStoreVectorSource[(i - offset) ^ 3];
|
||||
}
|
||||
|
||||
test.Run(
|
||||
[&](PPCContext* ctx) {
|
||||
ctx->r[4] = aligned_addr + offset;
|
||||
ctx->v[5] =
|
||||
vec128b(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
|
||||
},
|
||||
[&](PPCContext* ctx) {
|
||||
REQUIRE(std::memcmp(host_ptr, expected.data(), expected.size()) == 0);
|
||||
});
|
||||
|
||||
test.memory->SystemHeapFree(guest_addr);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("STORE_VECTOR_RIGHT", "[memory]") {
|
||||
TestFunction test([](HIRBuilder& b) {
|
||||
b.StoreVectorRight(LoadGPR(b, 4), LoadVR(b, 5));
|
||||
b.Return();
|
||||
});
|
||||
|
||||
for (const uint32_t offset : kPartialStoreOffsets) {
|
||||
uint32_t guest_addr = test.memory->SystemHeapAlloc(64, 16);
|
||||
REQUIRE(guest_addr != 0);
|
||||
const uint32_t aligned_addr = (guest_addr + 15) & ~15u;
|
||||
auto* host_ptr =
|
||||
reinterpret_cast<uint8_t*>(test.memory->TranslateVirtual(aligned_addr));
|
||||
|
||||
std::array<uint8_t, 16> expected;
|
||||
for (uint32_t i = 0; i < 16; ++i) {
|
||||
host_ptr[i] = static_cast<uint8_t>(0xA0 + i);
|
||||
expected[i] = host_ptr[i];
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < offset; ++i) {
|
||||
expected[i] = kStoreVectorSource[(16 - offset + i) ^ 3];
|
||||
}
|
||||
|
||||
test.Run(
|
||||
[&](PPCContext* ctx) {
|
||||
ctx->r[4] = aligned_addr + offset;
|
||||
ctx->v[5] =
|
||||
vec128b(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
|
||||
},
|
||||
[&](PPCContext* ctx) {
|
||||
REQUIRE(std::memcmp(host_ptr, expected.data(), expected.size()) == 0);
|
||||
});
|
||||
|
||||
test.memory->SystemHeapFree(guest_addr);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("STORE_VECTOR_LEFT_MEMCPY_HEAD", "[memory]") {
|
||||
TestFunction test([](HIRBuilder& b) {
|
||||
auto src_addr = LoadGPR(b, 4);
|
||||
auto dest_addr = LoadGPR(b, 5);
|
||||
auto aligned_src =
|
||||
b.And(src_addr, b.LoadConstantInt64(static_cast<int64_t>(~0xFULL)));
|
||||
auto next_src = b.And(b.Add(src_addr, b.LoadConstantInt64(15)),
|
||||
b.LoadConstantInt64(static_cast<int64_t>(~0xFULL)));
|
||||
auto source_a = b.ByteSwap(b.Load(aligned_src, VEC128_TYPE));
|
||||
auto source_b = b.ByteSwap(b.Load(next_src, VEC128_TYPE));
|
||||
auto control = b.LoadVectorShl(
|
||||
b.Truncate(b.And(src_addr, b.LoadConstantInt64(0xF)), INT8_TYPE));
|
||||
auto head = b.Permute(control, source_a, source_b, INT8_TYPE);
|
||||
b.StoreVectorLeft(dest_addr, head);
|
||||
b.Return();
|
||||
});
|
||||
|
||||
for (const uint32_t source_offset : {0u, 1u, 4u, 12u}) {
|
||||
uint32_t guest_addr = test.memory->SystemHeapAlloc(128, 16);
|
||||
REQUIRE(guest_addr != 0);
|
||||
const uint32_t aligned_addr = (guest_addr + 15) & ~15u;
|
||||
auto* host_ptr =
|
||||
reinterpret_cast<uint8_t*>(test.memory->TranslateVirtual(aligned_addr));
|
||||
auto* src_ptr = host_ptr;
|
||||
auto* dest_block = host_ptr + 64;
|
||||
const uint32_t src_addr = aligned_addr + source_offset;
|
||||
const uint32_t dest_addr = aligned_addr + 64 + 12;
|
||||
|
||||
for (uint32_t i = 0; i < 64; ++i) {
|
||||
src_ptr[i] = static_cast<uint8_t>(i);
|
||||
}
|
||||
for (uint32_t i = 0; i < 16; ++i) {
|
||||
dest_block[i] = static_cast<uint8_t>(0xC0 + i);
|
||||
}
|
||||
|
||||
std::array<uint8_t, 16> expected;
|
||||
std::memcpy(expected.data(), dest_block, expected.size());
|
||||
for (uint32_t i = 0; i < 4; ++i) {
|
||||
expected[12 + i] = src_ptr[source_offset + i];
|
||||
}
|
||||
|
||||
test.Run(
|
||||
[&](PPCContext* ctx) {
|
||||
ctx->r[4] = src_addr;
|
||||
ctx->r[5] = dest_addr;
|
||||
},
|
||||
[&](PPCContext* ctx) {
|
||||
REQUIRE(std::memcmp(dest_block, expected.data(), expected.size()) ==
|
||||
0);
|
||||
});
|
||||
|
||||
test.memory->SystemHeapFree(guest_addr);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user