fabi/Xenia-Canary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[a64] Optimize OPCODE_MEMSET with zva

Browse Source 
* Add unit tests
* Use `dc zva` to zero out entire cache-lines worth of data, when possible
 * Special instruction that zaps an entire cache-line into being zero-values, typically 64-bytes at a time
* Statically read and initialize the `DCZID_EL0` register value since reading registers with `mrs` can be kinda slow and isn't worth doing redundantly at JIT-time when it never changes.
* Update store-instructions to use inline post-increments to avoid an additional `add` instruction and keep encoded instruction immediates small

Pretty much every arm processor has a cache line size of 64 bytes, so a typical `dcbz128` to clear 128-bytes of data now results in:
```
dc zva, x0
add x0, x0, 64
dc zva, x0
```
rather than a sequence of 8 `stp xzr, xzr, ...` instructions
This commit is contained in:
Wunkolo
2026-03-26 22:03:54 -07:00
committed by Heel
parent 1a53f261f7
commit 664b77f38e
3 changed files with 87 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/xenia/cpu/backend/a64/a64_seq_memory.cc
View File

@@ -818,6 +818,9 @@ EMITTER_OPCODE_TABLE(OPCODE_STORE_OFFSET, STORE_OFFSET_I8, STORE_OFFSET_I16,
// ============================================================================
// OPCODE_MEMSET
// ============================================================================
static const bool zva_enable = (xe_cpu_mrs(DCZID_EL0) & 0b1'0000) == 0;
static const uint64_t zva_length = (4ULL << (xe_cpu_mrs(DCZID_EL0) & 0b0'1111));
struct MEMSET_I64
: Sequence<MEMSET_I64, I<OPCODE_MEMSET, VoidOp, I64Op, I8Op, I64Op>> {
static void Emit(A64Emitter& e, const EmitArgType& i) {
@@ -826,25 +829,37 @@ struct MEMSET_I64
e.add(e.x0, e.GetMembaseReg(), addr);
// Optimize the common case: zeroing a constant-length block (dcbz/dcbz128).
if (i.src2.is_constant && i.src2.constant() == 0 && i.src3.is_constant) {
uint64_t len = i.src3.constant();
// Inline with STP xzr, xzr pairs (16 bytes each).
for (uint64_t off = 0; off + 16 <= len; off += 16) {
e.stp(e.xzr, e.xzr, ptr(e.x0, static_cast<int32_t>(off)));
const uint64_t len = i.src3.constant();
uint64_t off = 0;
// Use `dc zva` if it writes more bytes at time than STP
if (zva_enable && len >= zva_length && zva_length > 16) {
for (; off + zva_length <= len; off += zva_length) {
// dc zva, x0
e.sys(0b011, 0b0111, 0b0100, 0b001, e.x0);
if (off + zva_length < len) {
e.add(e.x0, e.x0, zva_length);
}
}
}
// Handle remaining bytes (0-15).
uint64_t rem = len & 15;
uint64_t base = len & ~15ull;
if (rem >= 8) {
e.str(e.xzr, ptr(e.x0, static_cast<int32_t>(base)));
base += 8;
rem -= 8;
// Inline with STP xzr, xzr pairs (16 bytes each)
for (; off + 16 <= len; off += 16) {
e.stp(e.xzr, e.xzr, AdrPostImm(e.x0, 16));
}
if (rem >= 4) {
e.str(e.wzr, ptr(e.x0, static_cast<int32_t>(base)));
base += 4;
rem -= 4;
// Handle remaining bytes (0-15)
if (off + 8 <= len) {
e.str(e.xzr, AdrPostImm(e.x0, 8));
off += 8;
}
if (off + 4 <= len) {
e.str(e.wzr, AdrPostImm(e.x0, 4));
off += 4;
}
// Byte loop for any remaining 0-3 bytes
for (; off + 1 <= len; off += 1) {
e.strb(e.wzr, AdrPostImm(e.x0, 1));
}
// 1-3 byte remainder unlikely for dcbz/dcbz128, skip for now.
} else {
// General case: splat byte to NEON register, then 16-byte bulk loop
// with a byte loop for the 0-15 byte tail.
@@ -868,8 +883,7 @@ struct MEMSET_I64
e.L(loop16);
e.cmp(e.x2, 16);
e.b(LO, tail);
e.str(QReg(0), ptr(e.x0));
e.add(e.x0, e.x0, 16);
e.str(QReg(0), AdrPostImm(e.x0, 16));
e.sub(e.x2, e.x2, 16);
e.b(loop16);
// Byte loop for remaining 0-15 bytes.
@@ -877,8 +891,7 @@ struct MEMSET_I64
e.cbz(e.x2, done);
auto& byte_loop = e.NewCachedLabel();
e.L(byte_loop);
e.strb(e.w1, ptr(e.x0));
e.add(e.x0, e.x0, 1);
e.strb(e.w1, AdrPostImm(e.x0, 1));
e.subs(e.x2, e.x2, 1);
e.b(Xbyak_aarch64::NE, byte_loop);
e.L(done);

11
src/xenia/cpu/backend/a64/a64_seq_util.h
View File

@@ -19,6 +19,17 @@
#include "xbyak_aarch64.h"
#if XE_COMPILER_MSVC
#include <intrin.h>
constexpr uint32_t DCZID_EL0 = ARM64_SYSREG(0b11, 0b011, 0b0000, 0b0000, 0b111);
#define xe_cpu_mrs(reg) _ReadStatusReg(reg)
#elif XE_COMPILER_CLANG || XE_COMPILER_GNUC
#include <arm_acle.h>
#define xe_cpu_mrs(reg) __arm_rsr64(#reg)
#else
#error "No MRS wrapper available for current compiler implemented."
#endif
namespace xe {
namespace cpu {
namespace backend {

43
src/xenia/cpu/testing/memory_store_load_test.cc
View File

@@ -241,6 +241,49 @@ TEST_CASE("STORE_F64_CONSTANT_BYTE_SWAP", "[instr]") {
});
}
// =============================================================================
// MEMSET — constant size
// =============================================================================
TEST_CASE("MEMSET_32_ZERO", "[instr]") {
TestFunction test([](HIRBuilder& b) {
auto addr = LoadGPR(b, 4);
b.Memset(addr, b.LoadZeroInt8(), b.LoadConstantInt64(32));
b.Return();
});
test.Run(
[&test](PPCContext* ctx) {
uint32_t addr = test.memory->SystemHeapAlloc(32, 32);
ctx->r[4] = addr;
},
[&test](PPCContext* ctx) {
auto* host =
test.memory->TranslateVirtual(static_cast<uint32_t>(ctx->r[4]));
const uint8_t result[32] = {};
REQUIRE(std::memcmp(result, host, 32) == 0);
test.memory->SystemHeapFree(static_cast<uint32_t>(ctx->r[4]));
});
}
TEST_CASE("MEMSET_128_ZERO", "[instr]") {
TestFunction test([](HIRBuilder& b) {
auto addr = LoadGPR(b, 4);
b.Memset(addr, b.LoadZeroInt8(), b.LoadConstantInt64(128));
b.Return();
});
test.Run(
[&test](PPCContext* ctx) {
uint32_t addr = test.memory->SystemHeapAlloc(128, 128);
ctx->r[4] = addr;
},
[&test](PPCContext* ctx) {
auto* host =
test.memory->TranslateVirtual(static_cast<uint32_t>(ctx->r[4]));
const uint8_t result[128] = {};
REQUIRE(std::memcmp(result, host, 128) == 0);
test.memory->SystemHeapFree(static_cast<uint32_t>(ctx->r[4]));
});
}
// =============================================================================
// LOAD_F32 / LOAD_F64 byte-swap (entirely unimplemented on x64)
// =============================================================================