[x64] Add AVX512 optimization for VECTOR_DENORMFLUSH

Use the `vptestnmd`-instruction to quickly test if the exponent bits of
each element is zero and put these results into a mask register. This
mask register can then be used to write `+0.0` or `-0.0` values
depending on the original value's sign-bit.

A masked `vrangeps` instruction is used to move zero-values into the effected
elements while preserving the sign-bit without having to touch memory.

src/xenia/cpu/backend/x64/x64_seq_vector.cc

@@ -13,6 +13,7 @@
 #include <cstring>
 
 #include "xenia/cpu/backend/x64/x64_op.h"
+#include "xenia/cpu/backend/x64/x64_util.h"
 
 // For OPCODE_PACK/OPCODE_UNPACK
 #include "third_party/half/include/half.hpp"
@@ -159,6 +160,16 @@ struct VECTOR_DENORMFLUSH
     : Sequence<VECTOR_DENORMFLUSH,
                I<OPCODE_VECTOR_DENORMFLUSH, V128Op, V128Op>> {
   static void Emit(X64Emitter& e, const EmitArgType& i) {
+    if (e.IsFeatureEnabled(kX64EmitAVX512Ortho | kX64EmitAVX512DQ)) {
+      const Xbyak::Opmask denormal_mask = e.k1;
+      e.vptestnmd(denormal_mask, i.src1,
+                  e.GetXmmConstPtr(XMMSingleDenormalMask));
+      e.vxorps(e.xmm1, e.xmm1, e.xmm1);
+      e.vrangeps(i.dest.reg() | denormal_mask, i.src1, e.xmm1,
+                 FpRangeSelect::AbsMin | FpRangeSign::OperandA);
+      return;
+    }
+
     e.ChangeMxcsrMode(MXCSRMode::Vmx);
     e.vxorps(e.xmm1, e.xmm1, e.xmm1);  // 0.25 P0123
 

src/xenia/cpu/backend/x64/x64_util.h

@@ -38,6 +38,21 @@ constexpr uint8_t b = 0b11001100;
 constexpr uint8_t c = 0b10101010;
 }  // namespace TernaryOperand
 
+// Opcodes for use with vrange* instructions
+namespace FpRangeSelect {
+constexpr uint8_t Min = 0b00;
+constexpr uint8_t Max = 0b01;
+constexpr uint8_t AbsMin = 0b10;  // Smaller absolute value
+constexpr uint8_t AbsMax = 0b11;  // Larger absolute value
+};  // namespace FpRangeSelect
+
+namespace FpRangeSign {
+constexpr uint8_t OperandA = 0b00;  // Copy sign of operand A
+constexpr uint8_t Preserve = 0b01;  // Leave sign as is
+constexpr uint8_t Positive = 0b10;  // Set Positive
+constexpr uint8_t Negative = 0b11;  // Set Negative
+};  // namespace FpRangeSign
+
 }  // namespace x64
 }  // namespace backend
 }  // namespace cpu