[a64] Optimize more LoadV128Const splat opportunities

Optimize `LoadV128Const` to detect even more opportunities to use `movi`, `mvni`, and `fmov` and such for 16-bit, 32-bit and 64-bit splats. Update some `mov`+`dup` idioms into `LoadV128Const` as well to catch more opportunities to optimize these const-loads when possible.
2026-03-25 21:38:09 -07:00
parent a90f5996de
commit 45cf464f63
2 changed files with 203 additions and 109 deletions
--- a/src/xenia/cpu/backend/a64/a64_seq_util.h
+++ b/src/xenia/cpu/backend/a64/a64_seq_util.h
@@ -43,10 +43,97 @@ inline void EmitWithVmxFpcr(A64Emitter& e, Fn&& emit_op) {
  e.msr(3, 3, 4, 4, 0, e.x13);
 }

+// Try to see if the provided 64-bit value can be compressed into an 8-bit
+// value for the movi instruction.
+// The 8-bit immediate "a:b:c:d:e:f:g:h" maps to the 64-bit value:
+// "aaaaaaaabbbbbbbbccccccccddddddddeeeeeeeeffffffffgggggggghhhhhhhh"
+inline bool TryMovi64Imm(uint64_t value, uint8_t& imm8) {
+  // Common cases
+  if (value == 0) {
+    // 00000...
+    imm8 = 0;
+    return true;
+  } else if (value == ~uint64_t(0)) {
+    // 11111...
+    imm8 = 0xFF;
+    return true;
+  }
+  uint8_t compressed = 0;
+  for (int shift = 0; shift < 8; ++shift) {
+    const uint8_t shift_u8 = static_cast<uint8_t>(value >> (shift * 8));
+    if (shift_u8 == 0xFF || shift_u8 == 0) {
+      compressed |= (shift_u8 == 0xFF) << shift;
+    } else {
+      return false;
+    }
+  }
+  imm8 = compressed;
+  return true;
+}
+
+// Try to see if the provided double value can be compressed into an 8-bit value
+// for the fmov instruction. Returns false if the value cannot be represented
+// abcdefgh
+//    V
+// aBbbbbbc defgh000 00000000 00000000
+// B = NOT(b)
+constexpr bool IsFmov32Imm(float f32) {
+  const uint32_t u32 = std::bit_cast<uint32_t, float>(f32);
+  const uint32_t sign = (u32 >> 31) & 1;
+  int32_t exp = ((u32 >> 23) & 0xff) - 127;
+  int64_t mantissa = u32 & 0x7fffff;
+
+  // Too many mantissa bits
+  if (mantissa & 0x7ffff) {
+    return false;
+  }
+  // Too many exp bits
+  if (exp < -3 || exp > 4) {
+    return false;
+  }
+
+  // mantissa = (16 + e:f:g:h) / 16.
+  mantissa >>= 19;
+  if ((mantissa & 0b1111) != mantissa) {
+    return false;
+  }
+
+  return true;
+}
+
+// Try to see if the provided double value can be compressed into an 8-bit value
+// for the fmov instruction. Returns false if the value cannot be represented
+// abcdefgh
+//    V
+// aBbbbbbb bbcdefgh 00000000 00000000 00000000 00000000 00000000 00000000
+// B = NOT(b)
+constexpr bool IsFmov64Imm(double f64) {
+  const uint64_t u64 = std::bit_cast<uint64_t, double>(f64);
+  int32_t exp = ((u64 >> 52) & 0x7ff) - 1023;
+  int64_t mantissa = u64 & 0xfffffffffffffULL;
+
+  // Too many mantissa bits
+  if (mantissa & 0xffffffffffffULL) {
+    return false;
+  }
+  // Too many exp bits
+  if (exp < -3 || exp > 4) {
+    return false;
+  }
+
+  // mantissa = (16 + e:f:g:h) / 16.
+  mantissa >>= 48;
+  if ((mantissa & 0b1111) != mantissa) {
+    return false;
+  }
+  return true;
+}
+
 // Load a compile-time vec128_t constant into a NEON register.
 // May clobber the provided GPR scratch-register
 inline void LoadV128Const(A64Emitter& e, int vreg_idx, const vec128_t& val,
                          int gpr_scratch_idx = 0) {
+  // Fast common cases
  if (!val.low && !val.high) {
    // 0000...
    e.movi(VReg2D(vreg_idx), 0);
@@ -55,7 +142,9 @@ inline void LoadV128Const(A64Emitter& e, int vreg_idx, const vec128_t& val,
    // 1111...
    e.movi(VReg2D(vreg_idx), ~0ULL);
    return;
-  } else {
+  }
+
+  // Element splats
  bool all_equal_u8 = true;
  const uint8_t splat_u8 = val.u8[0];
  for (unsigned i = 1; i < 16; ++i) {
@@ -79,11 +168,17 @@ inline void LoadV128Const(A64Emitter& e, int vreg_idx, const vec128_t& val,
  }
  if (all_equal_u16) {
    if ((splat_u16 & 0xFF'00) == 0) {
-        e.movi(VReg(vreg_idx).h8, static_cast<uint8_t>(splat_u16));
+      e.movi(VReg(vreg_idx).h8, static_cast<uint8_t>(splat_u16 >> 0), LSL, 0);
    } else if ((splat_u16 & 0x00'FF) == 0) {
      e.movi(VReg(vreg_idx).h8, static_cast<uint8_t>(splat_u16 >> 8), LSL, 8);
+    } else if ((splat_u16 & 0xFF'00) == 0xFF'00) {
+      e.mvni(VReg(vreg_idx).h8, ~static_cast<uint8_t>(splat_u16 >> 0) & 0xFF,
+             LSL, 0);
+    } else if ((splat_u16 & 0x00'FF) == 0x00'FF) {
+      e.mvni(VReg(vreg_idx).h8, ~static_cast<uint8_t>(splat_u16 >> 8) & 0xFF,
+             LSL, 8);
    } else {
-        e.movz(WReg(gpr_scratch_idx), splat_u16);
+      e.movz(WReg(gpr_scratch_idx), splat_u16, 0);
      e.dup(VReg(vreg_idx).h8, WReg(gpr_scratch_idx));
    }
    return;
@@ -91,6 +186,7 @@ inline void LoadV128Const(A64Emitter& e, int vreg_idx, const vec128_t& val,

  bool all_equal_u32 = true;
  const uint32_t splat_u32 = val.u32[0];
+  const float splat_f32 = val.f32[0];
  for (unsigned i = 1; i < 4; ++i) {
    if (val.u32[i] != splat_u32) {
      all_equal_u32 = false;
@@ -99,15 +195,27 @@ inline void LoadV128Const(A64Emitter& e, int vreg_idx, const vec128_t& val,
  }
  if (all_equal_u32) {
    if ((splat_u32 & 0xFF'FF'FF'00) == 0) {
-        e.movi(VReg(vreg_idx).s4, static_cast<uint8_t>(splat_u32));
+      e.movi(VReg(vreg_idx).s4, static_cast<uint8_t>(splat_u32 >> 0), LSL, 0);
    } else if ((splat_u32 & 0xFF'FF'00'FF) == 0) {
      e.movi(VReg(vreg_idx).s4, static_cast<uint8_t>(splat_u32 >> 8), LSL, 8);
    } else if ((splat_u32 & 0xFF'00'FF'FF) == 0) {
-        e.movi(VReg(vreg_idx).s4, static_cast<uint8_t>(splat_u32 >> 16), LSL,
-               16);
+      e.movi(VReg(vreg_idx).s4, static_cast<uint8_t>(splat_u32 >> 16), LSL, 16);
    } else if ((splat_u32 & 0x00'FF'FF'FF) == 0) {
-        e.movi(VReg(vreg_idx).s4, static_cast<uint8_t>(splat_u32 >> 24), LSL,
-               24);
+      e.movi(VReg(vreg_idx).s4, static_cast<uint8_t>(splat_u32 >> 24), LSL, 24);
+    } else if ((splat_u32 & 0xFF'FF'FF'00) == 0xFF'FF'FF'00) {
+      e.mvni(VReg(vreg_idx).s4, ~static_cast<uint8_t>(splat_u32 >> 0) & 0xFF,
+             LSL, 0);
+    } else if ((splat_u32 & 0xFF'FF'00'FF) == 0xFF'FF'00'FF) {
+      e.mvni(VReg(vreg_idx).s4, ~static_cast<uint8_t>(splat_u32 >> 8) & 0xFF,
+             LSL, 8);
+    } else if ((splat_u32 & 0xFF'00'FF'FF) == 0xFF'00'FF'FF) {
+      e.mvni(VReg(vreg_idx).s4, ~static_cast<uint8_t>(splat_u32 >> 16) & 0xFF,
+             LSL, 16);
+    } else if ((splat_u32 & 0x00'FF'FF'FF) == 0x00'FF'FF'FF) {
+      e.mvni(VReg(vreg_idx).s4, ~static_cast<uint8_t>(splat_u32 >> 24) & 0xFF,
+             LSL, 24);
+    } else if (IsFmov32Imm(splat_f32)) {
+      e.fmov(VReg(vreg_idx).s4, splat_f32);
    } else {
      e.mov(WReg(gpr_scratch_idx), splat_u32);
      e.dup(VReg(vreg_idx).s4, WReg(gpr_scratch_idx));
@@ -117,13 +225,20 @@ inline void LoadV128Const(A64Emitter& e, int vreg_idx, const vec128_t& val,

  const bool all_equal_u64 = val.low == val.high;
  const uint64_t splat_u64 = val.u64[0];
+  const double splat_f64 = val.f64[0];
  if (all_equal_u64) {
+    if (uint8_t movi_imm; TryMovi64Imm(val.low, movi_imm)) {
+      e.movi(VReg2D(vreg_idx), movi_imm);
+    } else if (IsFmov64Imm(splat_f64)) {
+      e.fmov(VReg(vreg_idx).d2, splat_f64);
+    } else {
      e.mov(XReg(gpr_scratch_idx), splat_u64);
      e.dup(VReg(vreg_idx).d2, XReg(gpr_scratch_idx));
+    }
    return;
  }
-  }

+  // Fallback
  e.mov(XReg(gpr_scratch_idx), val.low);
  e.fmov(DReg(vreg_idx), XReg(gpr_scratch_idx));
  e.mov(XReg(gpr_scratch_idx), val.high);
--- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc
+++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc
@@ -852,8 +852,7 @@ struct VECTOR_DENORMFLUSH
    // Mask: exponent = bits 30:23 of each float.
    // If (val & 0x7F800000) == 0 then it's zero or denormal.
    // For denormals (mantissa != 0 but exponent == 0), replace with sign | 0.
-    e.mov(e.w0, static_cast<uint64_t>(0x7F800000u));
-    e.dup(VReg(2).s4, e.w0);
+    LoadV128Const(e, 2, vec128i(0x7F800000u), 0);
    e.and_(VReg(0).b16, VReg(s).b16, VReg(2).b16);
    // v0 = exponent bits. Compare with zero.
    e.cmeq(VReg(0).s4, VReg(0).s4, 0);
@@ -1140,8 +1139,7 @@ struct PACK : Sequence<PACK, I<OPCODE_PACK, V128Op, V128Op, V128Op>> {
    // fmaxnm/fminnm: NaN operand returns the non-NaN value (pack NaN as zero).
    e.fmov(VReg(0).s4, 3.0f);
    e.fmaxnm(VReg(d).s4, VReg(s).s4, VReg(0).s4);
-    e.mov(e.w0, 0x404000FFu);  // 3.0f + 255*2^-22
-    e.dup(VReg(0).s4, e.w0);
+    LoadV128Const(e, 0, vec128i(0x404000FFu), 0);  // 3.0f + 255*2^-22
    e.fminnm(VReg(d).s4, VReg(d).s4, VReg(0).s4);
    // TBL: extract low byte from each lane, reorder RGBA->ARGB in lane 3.
    // Control: bytes 0-11=0xFF (->0), bytes 12-15={0x08,0x04,0x00,0x0C}
@@ -1156,11 +1154,9 @@ struct PACK : Sequence<PACK, I<OPCODE_PACK, V128Op, V128Op, V128Op>> {
    int s = SrcVReg(e, i.src1, 2);
    int d = i.dest.reg().getIdx();
    // Clamp to [PackSHORT_Min, PackSHORT_Max].
-    e.mov(e.w0, 0x403F8001u);
-    e.dup(VReg(0).s4, e.w0);
+    LoadV128Const(e, 0, vec128i(0x403F8001u), 0);
    e.fmaxnm(VReg(d).s4, VReg(s).s4, VReg(0).s4);
-    e.mov(e.w0, 0x40407FFFu);
-    e.dup(VReg(0).s4, e.w0);
+    LoadV128Const(e, 0, vec128i(0x40407FFFu), 0);
    e.fminnm(VReg(d).s4, VReg(d).s4, VReg(0).s4);
    // TBL: extract low 2 bytes from lanes 0,1 -> pack into lane 3.
    // TBL ctrl for PACK_SHORT_2: bytes 12-15={0x04,0x05,0x00,0x01}, rest=0xFF
@@ -1174,11 +1170,9 @@ struct PACK : Sequence<PACK, I<OPCODE_PACK, V128Op, V128Op, V128Op>> {
    assert_true(i.src2.value->IsConstantZero());
    int s = SrcVReg(e, i.src1, 2);
    int d = i.dest.reg().getIdx();
-    e.mov(e.w0, 0x403F8001u);
-    e.dup(VReg(0).s4, e.w0);
+    LoadV128Const(e, 0, vec128i(0x403F8001u), 0);
    e.fmaxnm(VReg(d).s4, VReg(s).s4, VReg(0).s4);
-    e.mov(e.w0, 0x40407FFFu);
-    e.dup(VReg(0).s4, e.w0);
+    LoadV128Const(e, 0, vec128i(0x40407FFFu), 0);
    e.fminnm(VReg(d).s4, VReg(d).s4, VReg(0).s4);
    // TBL ctrl for PACK_SHORT_4: bytes 8-11={0x04,0x05,0x00,0x01},
    // 12-15={0x0C,0x0D,0x08,0x09}
@@ -1208,8 +1202,7 @@ struct PACK : Sequence<PACK, I<OPCODE_PACK, V128Op, V128Op, V128Op>> {

    // Normal path: rebias exponent and shift.
    // result = abs + 0xC8000000 (subtract 112 from float exponent, unsigned)
-    e.mov(e.w0, 0xC8000000u);
-    e.dup(VReg(2).s4, e.w0);
+    LoadV128Const(e, 2, vec128i(0xC8000000u), 0);
    e.add(VReg(2).s4, VReg(0).s4, VReg(2).s4);  // v2 = rebiased

    if (round_to_even) {
@@ -1217,28 +1210,24 @@ struct PACK : Sequence<PACK, I<OPCODE_PACK, V128Op, V128Op, V128Op>> {
      e.ushr(VReg(3).s4, VReg(2).s4, 13);
      e.movi(VReg(1).s4, 1);
      e.and_(VReg(3).b16, VReg(3).b16, VReg(1).b16);  // (result>>13)&1
-      e.mov(e.w0, 0xFFFu);
-      e.dup(VReg(1).s4, e.w0);
+      LoadV128Const(e, 1, vec128i(0xFFFu), 0);
      e.add(VReg(3).s4, VReg(3).s4, VReg(1).s4);  // 0xFFF + bit
      e.add(VReg(2).s4, VReg(2).s4, VReg(3).s4);  // rounded
    }

    // Shift and mask to 15 bits.
    e.ushr(VReg(2).s4, VReg(2).s4, 13);
-    e.mov(e.w0, 0x7FFFu);
-    e.dup(VReg(3).s4, e.w0);
+    LoadV128Const(e, 3, vec128i(0x7FFFu), 0);
    e.and_(VReg(2).b16, VReg(2).b16, VReg(3).b16);  // v2 = normal result

    // Saturation: where abs >= 0x47FFE000, force to 0x7FFF.
    // v3 already holds 0x7FFF splat = saturation value.
-    e.mov(e.w0, 0x47FFE000u);
-    e.dup(VReg(1).s4, e.w0);
+    LoadV128Const(e, 1, vec128i(0x47FFE000u), 0);
    e.cmhs(VReg(1).s4, VReg(0).s4, VReg(1).s4);    // v1 = sat mask
    e.bsl(VReg(1).b16, VReg(3).b16, VReg(2).b16);  // v1 = sat?7FFF:normal

    // Flush: where abs < 0x38800000, force to 0.
-    e.mov(e.w0, 0x38800000u);
-    e.dup(VReg(2).s4, e.w0);
+    LoadV128Const(e, 2, vec128i(0x38800000u), 0);
    e.cmhi(VReg(2).s4, VReg(2).s4, VReg(0).s4);    // v2 = small mask
    e.bic(VReg(1).b16, VReg(1).b16, VReg(2).b16);  // zero where small

@@ -1525,14 +1514,12 @@ struct UNPACK : Sequence<UNPACK, I<OPCODE_UNPACK, V128Op, V128Op>> {
  static void EmitXenosHalfToFloat4(A64Emitter& e, int dest) {
    // v0 = zero-extended halfs (16-bit values in 32-bit lanes).
    // abs = v0 & 0x7FFF
-    e.mov(e.w0, 0x7FFFu);
-    e.dup(VReg(1).s4, e.w0);
+    LoadV128Const(e, 1, vec128i(0x7FFFu), 0);
    e.and_(VReg(2).b16, VReg(0).b16, VReg(1).b16);  // v2 = abs

    // value = (abs << 13) + 0x38000000  (bias adjustment: 112 << 23)
    e.shl(VReg(1).s4, VReg(2).s4, 13);
-    e.mov(e.w0, 0x38000000u);
-    e.dup(VReg(3).s4, e.w0);
+    LoadV128Const(e, 3, vec128i(0x38000000u), 0);
    e.add(VReg(1).s4, VReg(1).s4, VReg(3).s4);  // v1 = biased value

    // sign = (v0 >> 15) << 31
@@ -1598,11 +1585,9 @@ struct UNPACK : Sequence<UNPACK, I<OPCODE_UNPACK, V128Op, V128Op>> {
  // encoding), replace it with QNaN 0x7FC00000.  Result in v_dest.
  // Clobbers v0-v2, w0.
  static void EmitMagicFloatOverflowCheck(A64Emitter& e, int dest) {
-    e.mov(e.w0, 0x403F8000u);
-    e.dup(VReg(1).s4, e.w0);
+    LoadV128Const(e, 1, vec128i(0x403F8000u), 0);
    e.cmeq(VReg(1).s4, VReg(0).s4, VReg(1).s4);
-    e.mov(e.w0, 0x7FC00000u);
-    e.dup(VReg(2).s4, e.w0);
+    LoadV128Const(e, 2, vec128i(0x7FC00000u), 0);
    e.bsl(VReg(1).b16, VReg(2).b16, VReg(0).b16);
    if (dest != 1) {
      e.mov(VReg(dest).b16, VReg(1).b16);
@@ -1625,8 +1610,7 @@ struct UNPACK : Sequence<UNPACK, I<OPCODE_UNPACK, V128Op, V128Op>> {
    e.ext(VReg(0).b16, VReg(s).b16, VReg(s).b16, 12);
    e.sxtl(VReg(0).s4, VReg(0).h4);
    // v0 = {sext(h[6])=y, sext(h[7])=x, junk, junk}
-    e.mov(e.w0, 0x40400000u);
-    e.dup(VReg(1).s4, e.w0);
+    LoadV128Const(e, 1, vec128i(0x40400000u), 0);
    e.add(VReg(0).s4, VReg(0).s4, VReg(1).s4);
    // Swap low pair to get {magic+x, magic+y, ...}
    e.rev64(VReg(0).s4, VReg(0).s4);
@@ -1651,8 +1635,7 @@ struct UNPACK : Sequence<UNPACK, I<OPCODE_UNPACK, V128Op, V128Op>> {
    int d = i.dest.reg().getIdx();
    // Sign-extend upper halfs: sxtl2 h[4..7] → s[0..3] = {w, z, y, x}
    e.sxtl2(VReg(0).s4, VReg(s).h8);
-    e.mov(e.w0, 0x40400000u);
-    e.dup(VReg(1).s4, e.w0);
+    LoadV128Const(e, 1, vec128i(0x40400000u), 0);
    e.add(VReg(0).s4, VReg(0).s4, VReg(1).s4);
    // Reorder {w,z,y,x} → {x,y,z,w}: rev64 then swap halves.
    e.rev64(VReg(0).s4, VReg(0).s4);                  // {z,w,x,y}
@@ -1706,11 +1689,9 @@ struct UNPACK : Sequence<UNPACK, I<OPCODE_UNPACK, V128Op, V128Op>> {

    // Overflow check: XYZ == 0x403FFE00 → QNaN (0x7FC00000).
    // W result (0x3F800000..0x3F800003) never matches, so splat is safe.
-    e.mov(e.w0, 0x403FFE00u);
-    e.dup(VReg(1).s4, e.w0);
+    LoadV128Const(e, 1, vec128i(0x403FFE00u), 0);
    e.cmeq(VReg(1).s4, VReg(0).s4, VReg(1).s4);
-    e.mov(e.w0, 0x7FC00000u);
-    e.dup(VReg(2).s4, e.w0);
+    LoadV128Const(e, 2, vec128i(0x7FC00000u), 0);
    e.bsl(VReg(1).b16, VReg(2).b16, VReg(0).b16);
    if (d != 1) {
      e.mov(VReg(d).b16, VReg(1).b16);
@@ -1758,11 +1739,9 @@ struct UNPACK : Sequence<UNPACK, I<OPCODE_UNPACK, V128Op, V128Op>> {
    e.ins(VReg(d).s4[3], e.w4);

    // Overflow: XYZ == 0x40380000 → QNaN.  W can't match.
-    e.mov(e.w0, 0x40380000u);
-    e.dup(VReg(0).s4, e.w0);
+    LoadV128Const(e, 0, vec128i(0x40380000u), 0);
    e.cmeq(VReg(0).s4, VReg(d).s4, VReg(0).s4);
-    e.mov(e.w0, 0x7FC00000u);
-    e.dup(VReg(1).s4, e.w0);
+    LoadV128Const(e, 1, vec128i(0x7FC00000u), 0);
    e.bsl(VReg(0).b16, VReg(1).b16, VReg(d).b16);
    e.mov(VReg(d).b16, VReg(0).b16);
  }