[CPU] fix correct partial vector stores
Replace the x64 STVL/STVR lowering with explicit byte stores that preserve the guest-visible vector byte order. The previous vector shuffle/blend path could write incorrect bytes for partial vector stores, which broke optimized guest memcpy implementations using stvlx/stvrx for unaligned heads and tails. In 4156081C this corrupted skinned model render command data and eventually caused render-thread crashes. Add regression coverage for StoreVectorLeft/StoreVectorRight across common unaligned offsets, plus a memcpy-head style case matching the affected guest instruction pattern.
This commit is contained in:
committed by
Radosław Gliński
parent
2027f50b6e
commit
30ac9d7be6
@@ -356,29 +356,32 @@ EMITTER_OPCODE_TABLE(OPCODE_LVR, LVR_V128);
|
|||||||
|
|
||||||
struct STVL_V128 : Sequence<STVL_V128, I<OPCODE_STVL, VoidOp, I64Op, V128Op>> {
|
struct STVL_V128 : Sequence<STVL_V128, I<OPCODE_STVL, VoidOp, I64Op, V128Op>> {
|
||||||
static void Emit(X64Emitter& e, const EmitArgType& i) {
|
static void Emit(X64Emitter& e, const EmitArgType& i) {
|
||||||
e.mov(e.ecx, 15);
|
Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm0);
|
||||||
e.mov(e.edx, e.ecx);
|
e.StashXmm(0, src2);
|
||||||
|
|
||||||
|
// Store bytes offset..15 from the source vector. Xenia's host vector byte
|
||||||
|
// layout is word-swapped from guest byte order, so convert source byte
|
||||||
|
// indexes with ^ 3 before reading the stashed XMM value.
|
||||||
e.lea(e.rax, e.ptr[ComputeMemoryAddress(e, i.src1)]);
|
e.lea(e.rax, e.ptr[ComputeMemoryAddress(e, i.src1)]);
|
||||||
|
e.mov(e.ecx, 15);
|
||||||
e.and_(e.ecx, e.eax);
|
e.and_(e.ecx, e.eax);
|
||||||
e.vmovd(e.xmm0, e.ecx);
|
e.mov(e.edx, 15);
|
||||||
e.not_(e.rdx);
|
e.not_(e.rdx);
|
||||||
e.and_(e.rax, e.rdx);
|
e.and_(e.rax, e.rdx);
|
||||||
e.vmovdqa(e.xmm1, e.GetXmmConstPtr(XMMSTVLShuffle));
|
|
||||||
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
|
|
||||||
e.vpbroadcastb(e.xmm3, e.xmm0);
|
|
||||||
} else {
|
|
||||||
e.vpshufb(e.xmm3, e.xmm0, e.GetXmmConstPtr(XMMZero));
|
|
||||||
}
|
|
||||||
e.vpsubb(e.xmm0, e.xmm1, e.xmm3);
|
|
||||||
e.vpxor(e.xmm1, e.xmm0,
|
|
||||||
e.GetXmmConstPtr(XMMSwapWordMask)); // xmm1 from now on will be our
|
|
||||||
// selector for blend/shuffle
|
|
||||||
|
|
||||||
Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm0);
|
Xbyak::Label loop, done;
|
||||||
|
e.mov(e.edx, e.ecx);
|
||||||
e.vpshufb(e.xmm2, src2, e.xmm1);
|
e.L(loop);
|
||||||
e.vpblendvb(e.xmm3, e.xmm2, e.ptr[e.rax], e.xmm1);
|
e.cmp(e.edx, 16);
|
||||||
e.vmovdqa(e.ptr[e.rax], e.xmm3);
|
e.jge(done);
|
||||||
|
e.mov(e.r8d, e.edx);
|
||||||
|
e.sub(e.r8d, e.ecx);
|
||||||
|
e.xor_(e.r8d, 3);
|
||||||
|
e.movzx(e.r9d, e.byte[e.rsp + X64Emitter::kStashOffset + e.r8]);
|
||||||
|
e.mov(e.byte[e.rax + e.rdx], e.r9b);
|
||||||
|
e.inc(e.edx);
|
||||||
|
e.jmp(loop);
|
||||||
|
e.L(done);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
EMITTER_OPCODE_TABLE(OPCODE_STVL, STVL_V128);
|
EMITTER_OPCODE_TABLE(OPCODE_STVL, STVL_V128);
|
||||||
@@ -391,28 +394,26 @@ struct STVR_V128 : Sequence<STVR_V128, I<OPCODE_STVR, VoidOp, I64Op, V128Op>> {
|
|||||||
e.lea(e.rax, e.ptr[ComputeMemoryAddress(e, i.src1)]);
|
e.lea(e.rax, e.ptr[ComputeMemoryAddress(e, i.src1)]);
|
||||||
e.and_(e.ecx, e.eax);
|
e.and_(e.ecx, e.eax);
|
||||||
e.jz(skipper);
|
e.jz(skipper);
|
||||||
e.vmovd(e.xmm0, e.ecx);
|
|
||||||
e.not_(e.rdx);
|
e.not_(e.rdx);
|
||||||
e.and_(e.rax, e.rdx);
|
e.and_(e.rax, e.rdx);
|
||||||
e.vmovdqa(e.xmm1, e.GetXmmConstPtr(XMMSTVLShuffle));
|
|
||||||
// todo: maybe a table lookup might be a better idea for getting the
|
|
||||||
// shuffle/blend
|
|
||||||
|
|
||||||
if (e.IsFeatureEnabled(kX64EmitAVX2)) {
|
|
||||||
e.vpbroadcastb(e.xmm3, e.xmm0);
|
|
||||||
} else {
|
|
||||||
e.vpshufb(e.xmm3, e.xmm0, e.GetXmmConstPtr(XMMZero));
|
|
||||||
}
|
|
||||||
e.vpsubb(e.xmm0, e.xmm1, e.xmm3);
|
|
||||||
e.vpxor(e.xmm1, e.xmm0,
|
|
||||||
e.GetXmmConstPtr(XMMSTVRSwapMask)); // xmm1 from now on will be our
|
|
||||||
// selector for blend/shuffle
|
|
||||||
|
|
||||||
Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm0);
|
Xmm src2 = GetInputRegOrConstant(e, i.src2, e.xmm0);
|
||||||
|
e.StashXmm(0, src2);
|
||||||
|
|
||||||
e.vpshufb(e.xmm2, src2, e.xmm1);
|
// Store bytes 0..offset-1 from the tail of the source vector.
|
||||||
e.vpblendvb(e.xmm3, e.xmm2, e.ptr[e.rax], e.xmm1);
|
Xbyak::Label loop;
|
||||||
e.vmovdqa(e.ptr[e.rax], e.xmm3);
|
e.xor_(e.edx, e.edx);
|
||||||
|
e.L(loop);
|
||||||
|
e.cmp(e.edx, e.ecx);
|
||||||
|
e.jge(skipper);
|
||||||
|
e.mov(e.r8d, 16);
|
||||||
|
e.sub(e.r8d, e.ecx);
|
||||||
|
e.add(e.r8d, e.edx);
|
||||||
|
e.xor_(e.r8d, 3);
|
||||||
|
e.movzx(e.r9d, e.byte[e.rsp + X64Emitter::kStashOffset + e.r8]);
|
||||||
|
e.mov(e.byte[e.rax + e.rdx], e.r9b);
|
||||||
|
e.inc(e.edx);
|
||||||
|
e.jmp(loop);
|
||||||
e.L(skipper);
|
e.L(skipper);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
|
|
||||||
#include "xenia/cpu/testing/util.h"
|
#include "xenia/cpu/testing/util.h"
|
||||||
|
|
||||||
|
#include <array>
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
#include <cmath>
|
#include <cmath>
|
||||||
#include <cstdlib>
|
#include <cstdlib>
|
||||||
@@ -629,3 +630,140 @@ TEST_CASE("LOAD_VECTOR_RIGHT", "[memory]") {
|
|||||||
|
|
||||||
test.memory->SystemHeapFree(guest_addr);
|
test.memory->SystemHeapFree(guest_addr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// STVL / STVR - partial vector store left/right
|
||||||
|
// These are used by optimized memcpy implementations for unaligned heads/tails.
|
||||||
|
// ============================================================================
|
||||||
|
constexpr std::array<uint8_t, 16> kStoreVectorSource = {
|
||||||
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
|
||||||
|
constexpr uint32_t kPartialStoreOffsets[] = {0, 1, 4, 8, 12, 15};
|
||||||
|
|
||||||
|
TEST_CASE("STORE_VECTOR_LEFT", "[memory]") {
|
||||||
|
TestFunction test([](HIRBuilder& b) {
|
||||||
|
b.StoreVectorLeft(LoadGPR(b, 4), LoadVR(b, 5));
|
||||||
|
b.Return();
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const uint32_t offset : kPartialStoreOffsets) {
|
||||||
|
uint32_t guest_addr = test.memory->SystemHeapAlloc(64, 16);
|
||||||
|
REQUIRE(guest_addr != 0);
|
||||||
|
const uint32_t aligned_addr = (guest_addr + 15) & ~15u;
|
||||||
|
auto* host_ptr =
|
||||||
|
reinterpret_cast<uint8_t*>(test.memory->TranslateVirtual(aligned_addr));
|
||||||
|
|
||||||
|
std::array<uint8_t, 16> expected;
|
||||||
|
for (uint32_t i = 0; i < 16; ++i) {
|
||||||
|
host_ptr[i] = static_cast<uint8_t>(0xA0 + i);
|
||||||
|
expected[i] = host_ptr[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
for (uint32_t i = offset; i < 16; ++i) {
|
||||||
|
expected[i] = kStoreVectorSource[(i - offset) ^ 3];
|
||||||
|
}
|
||||||
|
|
||||||
|
test.Run(
|
||||||
|
[&](PPCContext* ctx) {
|
||||||
|
ctx->r[4] = aligned_addr + offset;
|
||||||
|
ctx->v[5] =
|
||||||
|
vec128b(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
|
||||||
|
},
|
||||||
|
[&](PPCContext* ctx) {
|
||||||
|
REQUIRE(std::memcmp(host_ptr, expected.data(), expected.size()) == 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test.memory->SystemHeapFree(guest_addr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_CASE("STORE_VECTOR_RIGHT", "[memory]") {
|
||||||
|
TestFunction test([](HIRBuilder& b) {
|
||||||
|
b.StoreVectorRight(LoadGPR(b, 4), LoadVR(b, 5));
|
||||||
|
b.Return();
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const uint32_t offset : kPartialStoreOffsets) {
|
||||||
|
uint32_t guest_addr = test.memory->SystemHeapAlloc(64, 16);
|
||||||
|
REQUIRE(guest_addr != 0);
|
||||||
|
const uint32_t aligned_addr = (guest_addr + 15) & ~15u;
|
||||||
|
auto* host_ptr =
|
||||||
|
reinterpret_cast<uint8_t*>(test.memory->TranslateVirtual(aligned_addr));
|
||||||
|
|
||||||
|
std::array<uint8_t, 16> expected;
|
||||||
|
for (uint32_t i = 0; i < 16; ++i) {
|
||||||
|
host_ptr[i] = static_cast<uint8_t>(0xA0 + i);
|
||||||
|
expected[i] = host_ptr[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
for (uint32_t i = 0; i < offset; ++i) {
|
||||||
|
expected[i] = kStoreVectorSource[(16 - offset + i) ^ 3];
|
||||||
|
}
|
||||||
|
|
||||||
|
test.Run(
|
||||||
|
[&](PPCContext* ctx) {
|
||||||
|
ctx->r[4] = aligned_addr + offset;
|
||||||
|
ctx->v[5] =
|
||||||
|
vec128b(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
|
||||||
|
},
|
||||||
|
[&](PPCContext* ctx) {
|
||||||
|
REQUIRE(std::memcmp(host_ptr, expected.data(), expected.size()) == 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test.memory->SystemHeapFree(guest_addr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_CASE("STORE_VECTOR_LEFT_MEMCPY_HEAD", "[memory]") {
|
||||||
|
TestFunction test([](HIRBuilder& b) {
|
||||||
|
auto src_addr = LoadGPR(b, 4);
|
||||||
|
auto dest_addr = LoadGPR(b, 5);
|
||||||
|
auto aligned_src =
|
||||||
|
b.And(src_addr, b.LoadConstantInt64(static_cast<int64_t>(~0xFULL)));
|
||||||
|
auto next_src = b.And(b.Add(src_addr, b.LoadConstantInt64(15)),
|
||||||
|
b.LoadConstantInt64(static_cast<int64_t>(~0xFULL)));
|
||||||
|
auto source_a = b.ByteSwap(b.Load(aligned_src, VEC128_TYPE));
|
||||||
|
auto source_b = b.ByteSwap(b.Load(next_src, VEC128_TYPE));
|
||||||
|
auto control = b.LoadVectorShl(
|
||||||
|
b.Truncate(b.And(src_addr, b.LoadConstantInt64(0xF)), INT8_TYPE));
|
||||||
|
auto head = b.Permute(control, source_a, source_b, INT8_TYPE);
|
||||||
|
b.StoreVectorLeft(dest_addr, head);
|
||||||
|
b.Return();
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const uint32_t source_offset : {0u, 1u, 4u, 12u}) {
|
||||||
|
uint32_t guest_addr = test.memory->SystemHeapAlloc(128, 16);
|
||||||
|
REQUIRE(guest_addr != 0);
|
||||||
|
const uint32_t aligned_addr = (guest_addr + 15) & ~15u;
|
||||||
|
auto* host_ptr =
|
||||||
|
reinterpret_cast<uint8_t*>(test.memory->TranslateVirtual(aligned_addr));
|
||||||
|
auto* src_ptr = host_ptr;
|
||||||
|
auto* dest_block = host_ptr + 64;
|
||||||
|
const uint32_t src_addr = aligned_addr + source_offset;
|
||||||
|
const uint32_t dest_addr = aligned_addr + 64 + 12;
|
||||||
|
|
||||||
|
for (uint32_t i = 0; i < 64; ++i) {
|
||||||
|
src_ptr[i] = static_cast<uint8_t>(i);
|
||||||
|
}
|
||||||
|
for (uint32_t i = 0; i < 16; ++i) {
|
||||||
|
dest_block[i] = static_cast<uint8_t>(0xC0 + i);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::array<uint8_t, 16> expected;
|
||||||
|
std::memcpy(expected.data(), dest_block, expected.size());
|
||||||
|
for (uint32_t i = 0; i < 4; ++i) {
|
||||||
|
expected[12 + i] = src_ptr[source_offset + i];
|
||||||
|
}
|
||||||
|
|
||||||
|
test.Run(
|
||||||
|
[&](PPCContext* ctx) {
|
||||||
|
ctx->r[4] = src_addr;
|
||||||
|
ctx->r[5] = dest_addr;
|
||||||
|
},
|
||||||
|
[&](PPCContext* ctx) {
|
||||||
|
REQUIRE(std::memcmp(dest_block, expected.data(), expected.size()) ==
|
||||||
|
0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test.memory->SystemHeapFree(guest_addr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user