diff --git a/src/xenia/cpu/backend/a64/a64_seq_util.h b/src/xenia/cpu/backend/a64/a64_seq_util.h index f9cf14e6c..8525ce623 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_util.h +++ b/src/xenia/cpu/backend/a64/a64_seq_util.h @@ -43,10 +43,97 @@ inline void EmitWithVmxFpcr(A64Emitter& e, Fn&& emit_op) { e.msr(3, 3, 4, 4, 0, e.x13); } +// Try to see if the provided 64-bit value can be compressed into an 8-bit +// value for the movi instruction. +// The 8-bit immediate "a:b:c:d:e:f:g:h" maps to the 64-bit value: +// "aaaaaaaabbbbbbbbccccccccddddddddeeeeeeeeffffffffgggggggghhhhhhhh" +inline bool TryMovi64Imm(uint64_t value, uint8_t& imm8) { + // Common cases + if (value == 0) { + // 00000... + imm8 = 0; + return true; + } else if (value == ~uint64_t(0)) { + // 11111... + imm8 = 0xFF; + return true; + } + uint8_t compressed = 0; + for (int shift = 0; shift < 8; ++shift) { + const uint8_t shift_u8 = static_cast(value >> (shift * 8)); + if (shift_u8 == 0xFF || shift_u8 == 0) { + compressed |= (shift_u8 == 0xFF) << shift; + } else { + return false; + } + } + imm8 = compressed; + return true; +} + +// Try to see if the provided double value can be compressed into an 8-bit value +// for the fmov instruction. Returns false if the value cannot be represented +// abcdefgh +// V +// aBbbbbbc defgh000 00000000 00000000 +// B = NOT(b) +constexpr bool IsFmov32Imm(float f32) { + const uint32_t u32 = std::bit_cast(f32); + const uint32_t sign = (u32 >> 31) & 1; + int32_t exp = ((u32 >> 23) & 0xff) - 127; + int64_t mantissa = u32 & 0x7fffff; + + // Too many mantissa bits + if (mantissa & 0x7ffff) { + return false; + } + // Too many exp bits + if (exp < -3 || exp > 4) { + return false; + } + + // mantissa = (16 + e:f:g:h) / 16. + mantissa >>= 19; + if ((mantissa & 0b1111) != mantissa) { + return false; + } + + return true; +} + +// Try to see if the provided double value can be compressed into an 8-bit value +// for the fmov instruction. Returns false if the value cannot be represented +// abcdefgh +// V +// aBbbbbbb bbcdefgh 00000000 00000000 00000000 00000000 00000000 00000000 +// B = NOT(b) +constexpr bool IsFmov64Imm(double f64) { + const uint64_t u64 = std::bit_cast(f64); + int32_t exp = ((u64 >> 52) & 0x7ff) - 1023; + int64_t mantissa = u64 & 0xfffffffffffffULL; + + // Too many mantissa bits + if (mantissa & 0xffffffffffffULL) { + return false; + } + // Too many exp bits + if (exp < -3 || exp > 4) { + return false; + } + + // mantissa = (16 + e:f:g:h) / 16. + mantissa >>= 48; + if ((mantissa & 0b1111) != mantissa) { + return false; + } + return true; +} + // Load a compile-time vec128_t constant into a NEON register. // May clobber the provided GPR scratch-register inline void LoadV128Const(A64Emitter& e, int vreg_idx, const vec128_t& val, int gpr_scratch_idx = 0) { + // Fast common cases if (!val.low && !val.high) { // 0000... e.movi(VReg2D(vreg_idx), 0); @@ -55,75 +142,103 @@ inline void LoadV128Const(A64Emitter& e, int vreg_idx, const vec128_t& val, // 1111... e.movi(VReg2D(vreg_idx), ~0ULL); return; - } else { - bool all_equal_u8 = true; - const uint8_t splat_u8 = val.u8[0]; - for (unsigned i = 1; i < 16; ++i) { - if (val.u8[i] != splat_u8) { - all_equal_u8 = false; - break; - } - } - if (all_equal_u8) { - e.movi(VReg(vreg_idx).b16, static_cast(splat_u8)); - return; - } - - bool all_equal_u16 = true; - const uint16_t splat_u16 = val.u16[0]; - for (unsigned i = 1; i < 8; ++i) { - if (val.u16[i] != splat_u16) { - all_equal_u16 = false; - break; - } - } - if (all_equal_u16) { - if ((splat_u16 & 0xFF'00) == 0) { - e.movi(VReg(vreg_idx).h8, static_cast(splat_u16)); - } else if ((splat_u16 & 0x00'FF) == 0) { - e.movi(VReg(vreg_idx).h8, static_cast(splat_u16 >> 8), LSL, 8); - } else { - e.movz(WReg(gpr_scratch_idx), splat_u16); - e.dup(VReg(vreg_idx).h8, WReg(gpr_scratch_idx)); - } - return; - } - - bool all_equal_u32 = true; - const uint32_t splat_u32 = val.u32[0]; - for (unsigned i = 1; i < 4; ++i) { - if (val.u32[i] != splat_u32) { - all_equal_u32 = false; - break; - } - } - if (all_equal_u32) { - if ((splat_u32 & 0xFF'FF'FF'00) == 0) { - e.movi(VReg(vreg_idx).s4, static_cast(splat_u32)); - } else if ((splat_u32 & 0xFF'FF'00'FF) == 0) { - e.movi(VReg(vreg_idx).s4, static_cast(splat_u32 >> 8), LSL, 8); - } else if ((splat_u32 & 0xFF'00'FF'FF) == 0) { - e.movi(VReg(vreg_idx).s4, static_cast(splat_u32 >> 16), LSL, - 16); - } else if ((splat_u32 & 0x00'FF'FF'FF) == 0) { - e.movi(VReg(vreg_idx).s4, static_cast(splat_u32 >> 24), LSL, - 24); - } else { - e.mov(WReg(gpr_scratch_idx), splat_u32); - e.dup(VReg(vreg_idx).s4, WReg(gpr_scratch_idx)); - } - return; - } - - const bool all_equal_u64 = val.low == val.high; - const uint64_t splat_u64 = val.u64[0]; - if (all_equal_u64) { - e.mov(XReg(gpr_scratch_idx), splat_u64); - e.dup(VReg(vreg_idx).d2, XReg(gpr_scratch_idx)); - return; - } } + // Element splats + bool all_equal_u8 = true; + const uint8_t splat_u8 = val.u8[0]; + for (unsigned i = 1; i < 16; ++i) { + if (val.u8[i] != splat_u8) { + all_equal_u8 = false; + break; + } + } + if (all_equal_u8) { + e.movi(VReg(vreg_idx).b16, static_cast(splat_u8)); + return; + } + + bool all_equal_u16 = true; + const uint16_t splat_u16 = val.u16[0]; + for (unsigned i = 1; i < 8; ++i) { + if (val.u16[i] != splat_u16) { + all_equal_u16 = false; + break; + } + } + if (all_equal_u16) { + if ((splat_u16 & 0xFF'00) == 0) { + e.movi(VReg(vreg_idx).h8, static_cast(splat_u16 >> 0), LSL, 0); + } else if ((splat_u16 & 0x00'FF) == 0) { + e.movi(VReg(vreg_idx).h8, static_cast(splat_u16 >> 8), LSL, 8); + } else if ((splat_u16 & 0xFF'00) == 0xFF'00) { + e.mvni(VReg(vreg_idx).h8, ~static_cast(splat_u16 >> 0) & 0xFF, + LSL, 0); + } else if ((splat_u16 & 0x00'FF) == 0x00'FF) { + e.mvni(VReg(vreg_idx).h8, ~static_cast(splat_u16 >> 8) & 0xFF, + LSL, 8); + } else { + e.movz(WReg(gpr_scratch_idx), splat_u16, 0); + e.dup(VReg(vreg_idx).h8, WReg(gpr_scratch_idx)); + } + return; + } + + bool all_equal_u32 = true; + const uint32_t splat_u32 = val.u32[0]; + const float splat_f32 = val.f32[0]; + for (unsigned i = 1; i < 4; ++i) { + if (val.u32[i] != splat_u32) { + all_equal_u32 = false; + break; + } + } + if (all_equal_u32) { + if ((splat_u32 & 0xFF'FF'FF'00) == 0) { + e.movi(VReg(vreg_idx).s4, static_cast(splat_u32 >> 0), LSL, 0); + } else if ((splat_u32 & 0xFF'FF'00'FF) == 0) { + e.movi(VReg(vreg_idx).s4, static_cast(splat_u32 >> 8), LSL, 8); + } else if ((splat_u32 & 0xFF'00'FF'FF) == 0) { + e.movi(VReg(vreg_idx).s4, static_cast(splat_u32 >> 16), LSL, 16); + } else if ((splat_u32 & 0x00'FF'FF'FF) == 0) { + e.movi(VReg(vreg_idx).s4, static_cast(splat_u32 >> 24), LSL, 24); + } else if ((splat_u32 & 0xFF'FF'FF'00) == 0xFF'FF'FF'00) { + e.mvni(VReg(vreg_idx).s4, ~static_cast(splat_u32 >> 0) & 0xFF, + LSL, 0); + } else if ((splat_u32 & 0xFF'FF'00'FF) == 0xFF'FF'00'FF) { + e.mvni(VReg(vreg_idx).s4, ~static_cast(splat_u32 >> 8) & 0xFF, + LSL, 8); + } else if ((splat_u32 & 0xFF'00'FF'FF) == 0xFF'00'FF'FF) { + e.mvni(VReg(vreg_idx).s4, ~static_cast(splat_u32 >> 16) & 0xFF, + LSL, 16); + } else if ((splat_u32 & 0x00'FF'FF'FF) == 0x00'FF'FF'FF) { + e.mvni(VReg(vreg_idx).s4, ~static_cast(splat_u32 >> 24) & 0xFF, + LSL, 24); + } else if (IsFmov32Imm(splat_f32)) { + e.fmov(VReg(vreg_idx).s4, splat_f32); + } else { + e.mov(WReg(gpr_scratch_idx), splat_u32); + e.dup(VReg(vreg_idx).s4, WReg(gpr_scratch_idx)); + } + return; + } + + const bool all_equal_u64 = val.low == val.high; + const uint64_t splat_u64 = val.u64[0]; + const double splat_f64 = val.f64[0]; + if (all_equal_u64) { + if (uint8_t movi_imm; TryMovi64Imm(val.low, movi_imm)) { + e.movi(VReg2D(vreg_idx), movi_imm); + } else if (IsFmov64Imm(splat_f64)) { + e.fmov(VReg(vreg_idx).d2, splat_f64); + } else { + e.mov(XReg(gpr_scratch_idx), splat_u64); + e.dup(VReg(vreg_idx).d2, XReg(gpr_scratch_idx)); + } + return; + } + + // Fallback e.mov(XReg(gpr_scratch_idx), val.low); e.fmov(DReg(vreg_idx), XReg(gpr_scratch_idx)); e.mov(XReg(gpr_scratch_idx), val.high); diff --git a/src/xenia/cpu/backend/a64/a64_seq_vector.cc b/src/xenia/cpu/backend/a64/a64_seq_vector.cc index 8caf9c514..2cedbebd7 100644 --- a/src/xenia/cpu/backend/a64/a64_seq_vector.cc +++ b/src/xenia/cpu/backend/a64/a64_seq_vector.cc @@ -852,8 +852,7 @@ struct VECTOR_DENORMFLUSH // Mask: exponent = bits 30:23 of each float. // If (val & 0x7F800000) == 0 then it's zero or denormal. // For denormals (mantissa != 0 but exponent == 0), replace with sign | 0. - e.mov(e.w0, static_cast(0x7F800000u)); - e.dup(VReg(2).s4, e.w0); + LoadV128Const(e, 2, vec128i(0x7F800000u), 0); e.and_(VReg(0).b16, VReg(s).b16, VReg(2).b16); // v0 = exponent bits. Compare with zero. e.cmeq(VReg(0).s4, VReg(0).s4, 0); @@ -1140,8 +1139,7 @@ struct PACK : Sequence> { // fmaxnm/fminnm: NaN operand returns the non-NaN value (pack NaN as zero). e.fmov(VReg(0).s4, 3.0f); e.fmaxnm(VReg(d).s4, VReg(s).s4, VReg(0).s4); - e.mov(e.w0, 0x404000FFu); // 3.0f + 255*2^-22 - e.dup(VReg(0).s4, e.w0); + LoadV128Const(e, 0, vec128i(0x404000FFu), 0); // 3.0f + 255*2^-22 e.fminnm(VReg(d).s4, VReg(d).s4, VReg(0).s4); // TBL: extract low byte from each lane, reorder RGBA->ARGB in lane 3. // Control: bytes 0-11=0xFF (->0), bytes 12-15={0x08,0x04,0x00,0x0C} @@ -1156,11 +1154,9 @@ struct PACK : Sequence> { int s = SrcVReg(e, i.src1, 2); int d = i.dest.reg().getIdx(); // Clamp to [PackSHORT_Min, PackSHORT_Max]. - e.mov(e.w0, 0x403F8001u); - e.dup(VReg(0).s4, e.w0); + LoadV128Const(e, 0, vec128i(0x403F8001u), 0); e.fmaxnm(VReg(d).s4, VReg(s).s4, VReg(0).s4); - e.mov(e.w0, 0x40407FFFu); - e.dup(VReg(0).s4, e.w0); + LoadV128Const(e, 0, vec128i(0x40407FFFu), 0); e.fminnm(VReg(d).s4, VReg(d).s4, VReg(0).s4); // TBL: extract low 2 bytes from lanes 0,1 -> pack into lane 3. // TBL ctrl for PACK_SHORT_2: bytes 12-15={0x04,0x05,0x00,0x01}, rest=0xFF @@ -1174,11 +1170,9 @@ struct PACK : Sequence> { assert_true(i.src2.value->IsConstantZero()); int s = SrcVReg(e, i.src1, 2); int d = i.dest.reg().getIdx(); - e.mov(e.w0, 0x403F8001u); - e.dup(VReg(0).s4, e.w0); + LoadV128Const(e, 0, vec128i(0x403F8001u), 0); e.fmaxnm(VReg(d).s4, VReg(s).s4, VReg(0).s4); - e.mov(e.w0, 0x40407FFFu); - e.dup(VReg(0).s4, e.w0); + LoadV128Const(e, 0, vec128i(0x40407FFFu), 0); e.fminnm(VReg(d).s4, VReg(d).s4, VReg(0).s4); // TBL ctrl for PACK_SHORT_4: bytes 8-11={0x04,0x05,0x00,0x01}, // 12-15={0x0C,0x0D,0x08,0x09} @@ -1208,8 +1202,7 @@ struct PACK : Sequence> { // Normal path: rebias exponent and shift. // result = abs + 0xC8000000 (subtract 112 from float exponent, unsigned) - e.mov(e.w0, 0xC8000000u); - e.dup(VReg(2).s4, e.w0); + LoadV128Const(e, 2, vec128i(0xC8000000u), 0); e.add(VReg(2).s4, VReg(0).s4, VReg(2).s4); // v2 = rebiased if (round_to_even) { @@ -1217,28 +1210,24 @@ struct PACK : Sequence> { e.ushr(VReg(3).s4, VReg(2).s4, 13); e.movi(VReg(1).s4, 1); e.and_(VReg(3).b16, VReg(3).b16, VReg(1).b16); // (result>>13)&1 - e.mov(e.w0, 0xFFFu); - e.dup(VReg(1).s4, e.w0); + LoadV128Const(e, 1, vec128i(0xFFFu), 0); e.add(VReg(3).s4, VReg(3).s4, VReg(1).s4); // 0xFFF + bit e.add(VReg(2).s4, VReg(2).s4, VReg(3).s4); // rounded } // Shift and mask to 15 bits. e.ushr(VReg(2).s4, VReg(2).s4, 13); - e.mov(e.w0, 0x7FFFu); - e.dup(VReg(3).s4, e.w0); + LoadV128Const(e, 3, vec128i(0x7FFFu), 0); e.and_(VReg(2).b16, VReg(2).b16, VReg(3).b16); // v2 = normal result // Saturation: where abs >= 0x47FFE000, force to 0x7FFF. // v3 already holds 0x7FFF splat = saturation value. - e.mov(e.w0, 0x47FFE000u); - e.dup(VReg(1).s4, e.w0); + LoadV128Const(e, 1, vec128i(0x47FFE000u), 0); e.cmhs(VReg(1).s4, VReg(0).s4, VReg(1).s4); // v1 = sat mask e.bsl(VReg(1).b16, VReg(3).b16, VReg(2).b16); // v1 = sat?7FFF:normal // Flush: where abs < 0x38800000, force to 0. - e.mov(e.w0, 0x38800000u); - e.dup(VReg(2).s4, e.w0); + LoadV128Const(e, 2, vec128i(0x38800000u), 0); e.cmhi(VReg(2).s4, VReg(2).s4, VReg(0).s4); // v2 = small mask e.bic(VReg(1).b16, VReg(1).b16, VReg(2).b16); // zero where small @@ -1525,14 +1514,12 @@ struct UNPACK : Sequence> { static void EmitXenosHalfToFloat4(A64Emitter& e, int dest) { // v0 = zero-extended halfs (16-bit values in 32-bit lanes). // abs = v0 & 0x7FFF - e.mov(e.w0, 0x7FFFu); - e.dup(VReg(1).s4, e.w0); + LoadV128Const(e, 1, vec128i(0x7FFFu), 0); e.and_(VReg(2).b16, VReg(0).b16, VReg(1).b16); // v2 = abs // value = (abs << 13) + 0x38000000 (bias adjustment: 112 << 23) e.shl(VReg(1).s4, VReg(2).s4, 13); - e.mov(e.w0, 0x38000000u); - e.dup(VReg(3).s4, e.w0); + LoadV128Const(e, 3, vec128i(0x38000000u), 0); e.add(VReg(1).s4, VReg(1).s4, VReg(3).s4); // v1 = biased value // sign = (v0 >> 15) << 31 @@ -1598,11 +1585,9 @@ struct UNPACK : Sequence> { // encoding), replace it with QNaN 0x7FC00000. Result in v_dest. // Clobbers v0-v2, w0. static void EmitMagicFloatOverflowCheck(A64Emitter& e, int dest) { - e.mov(e.w0, 0x403F8000u); - e.dup(VReg(1).s4, e.w0); + LoadV128Const(e, 1, vec128i(0x403F8000u), 0); e.cmeq(VReg(1).s4, VReg(0).s4, VReg(1).s4); - e.mov(e.w0, 0x7FC00000u); - e.dup(VReg(2).s4, e.w0); + LoadV128Const(e, 2, vec128i(0x7FC00000u), 0); e.bsl(VReg(1).b16, VReg(2).b16, VReg(0).b16); if (dest != 1) { e.mov(VReg(dest).b16, VReg(1).b16); @@ -1625,8 +1610,7 @@ struct UNPACK : Sequence> { e.ext(VReg(0).b16, VReg(s).b16, VReg(s).b16, 12); e.sxtl(VReg(0).s4, VReg(0).h4); // v0 = {sext(h[6])=y, sext(h[7])=x, junk, junk} - e.mov(e.w0, 0x40400000u); - e.dup(VReg(1).s4, e.w0); + LoadV128Const(e, 1, vec128i(0x40400000u), 0); e.add(VReg(0).s4, VReg(0).s4, VReg(1).s4); // Swap low pair to get {magic+x, magic+y, ...} e.rev64(VReg(0).s4, VReg(0).s4); @@ -1651,8 +1635,7 @@ struct UNPACK : Sequence> { int d = i.dest.reg().getIdx(); // Sign-extend upper halfs: sxtl2 h[4..7] → s[0..3] = {w, z, y, x} e.sxtl2(VReg(0).s4, VReg(s).h8); - e.mov(e.w0, 0x40400000u); - e.dup(VReg(1).s4, e.w0); + LoadV128Const(e, 1, vec128i(0x40400000u), 0); e.add(VReg(0).s4, VReg(0).s4, VReg(1).s4); // Reorder {w,z,y,x} → {x,y,z,w}: rev64 then swap halves. e.rev64(VReg(0).s4, VReg(0).s4); // {z,w,x,y} @@ -1706,11 +1689,9 @@ struct UNPACK : Sequence> { // Overflow check: XYZ == 0x403FFE00 → QNaN (0x7FC00000). // W result (0x3F800000..0x3F800003) never matches, so splat is safe. - e.mov(e.w0, 0x403FFE00u); - e.dup(VReg(1).s4, e.w0); + LoadV128Const(e, 1, vec128i(0x403FFE00u), 0); e.cmeq(VReg(1).s4, VReg(0).s4, VReg(1).s4); - e.mov(e.w0, 0x7FC00000u); - e.dup(VReg(2).s4, e.w0); + LoadV128Const(e, 2, vec128i(0x7FC00000u), 0); e.bsl(VReg(1).b16, VReg(2).b16, VReg(0).b16); if (d != 1) { e.mov(VReg(d).b16, VReg(1).b16); @@ -1758,11 +1739,9 @@ struct UNPACK : Sequence> { e.ins(VReg(d).s4[3], e.w4); // Overflow: XYZ == 0x40380000 → QNaN. W can't match. - e.mov(e.w0, 0x40380000u); - e.dup(VReg(0).s4, e.w0); + LoadV128Const(e, 0, vec128i(0x40380000u), 0); e.cmeq(VReg(0).s4, VReg(d).s4, VReg(0).s4); - e.mov(e.w0, 0x7FC00000u); - e.dup(VReg(1).s4, e.w0); + LoadV128Const(e, 1, vec128i(0x7FC00000u), 0); e.bsl(VReg(0).b16, VReg(1).b16, VReg(d).b16); e.mov(VReg(d).b16, VReg(0).b16); }