fabi/Snappy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add RVV support and optmized uncompress speed

Browse Source
This commit is contained in:
anthony-zy
2025-08-28 20:42:13 +08:00
parent 758804a4bc
commit 802ef73aed
4 changed files with 164 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
snappy.cc
View File

@@ -281,6 +281,20 @@ inline char* IncrementalCopySlow(const char* src, char* op,
// 4, 5, 0, 1, 2, 3, 4, 5, 0, 1}. These byte index sequences are generated by
// calling MakePatternMaskBytes(0, 6, index_sequence<16>()) and
// MakePatternMaskBytes(16, 6, index_sequence<16>()) respectively.
// Selects the appropriate vector size based on the current architecture
// vuint8m1_t, RISC-V vector type with fixed 128-bit size
// (sizeof not used due to variable-length vector register in RVV)
#if defined(__SSE2__) || defined(SNAPPY_HAVE_SSSE3)
constexpr size_t kVectorSize = sizeof(V128); // __m128i
#elif defined(__ARM_NEON) || defined(SNAPPY_HAVE_NEON)
constexpr size_t kVectorSize = sizeof(uint8x16_t); // uint8x16_t
#elif defined(SNAPPY_HAVE_RVV) || defined(__riscv_vector)
constexpr size_t kVectorSize = 16; // vuint8m1_t
#else
#error "Unsupported architecture. Please define __SSE2__, __ARM_NEON, or SNAPPY_HAVE_RVV/__riscv_vector."
#endif
template <size_t... indexes>
inline constexpr std::array<char, sizeof...(indexes)> MakePatternMaskBytes(
int index_offset, int pattern_size, index_sequence<indexes...>) {
@@ -290,19 +304,17 @@ inline constexpr std::array<char, sizeof...(indexes)> MakePatternMaskBytes(
// Computes the shuffle control mask bytes array for given pattern-sizes and
// returns an array.
template <size_t... pattern_sizes_minus_one>
inline constexpr std::array<std::array<char, sizeof(V128)>,
inline constexpr std::array<std::array<char, kVectorSize>,
sizeof...(pattern_sizes_minus_one)>
MakePatternMaskBytesTable(int index_offset,
index_sequence<pattern_sizes_minus_one...>) {
return {
MakePatternMaskBytes(index_offset, pattern_sizes_minus_one + 1,
make_index_sequence</*indexes=*/sizeof(V128)>())...};
return {MakePatternMaskBytes(index_offset, pattern_sizes_minus_one + 1,
make_index_sequence<kVectorSize>())...};
}
// This is an array of shuffle control masks that can be used as the source
// operand for PSHUFB to permute the contents of the destination XMM register
// into a repeating byte pattern.
alignas(16) constexpr std::array<std::array<char, sizeof(V128)>,
alignas(16) constexpr std::array<std::array<char, kVectorSize>,
16> pattern_generation_masks =
MakePatternMaskBytesTable(
/*index_offset=*/0,
@@ -313,7 +325,7 @@ alignas(16) constexpr std::array<std::array<char, sizeof(V128)>,
// Basically, pattern_reshuffle_masks is a continuation of
// pattern_generation_masks. It follows that, pattern_reshuffle_masks is same as
// pattern_generation_masks for offsets 1, 2, 4, 8 and 16.
alignas(16) constexpr std::array<std::array<char, sizeof(V128)>,
alignas(16) constexpr std::array<std::array<char, kVectorSize>,
16> pattern_reshuffle_masks =
MakePatternMaskBytesTable(
/*index_offset=*/16,
@@ -329,6 +341,21 @@ static inline V128 LoadPattern(const char* src, const size_t pattern_size) {
return V128_Shuffle(V128_LoadU(reinterpret_cast<const V128*>(src)),
generation_mask);
}
// vuint8m1_t cannot be used as an element of std::pair
#if SNAPPY_HAVE_RVV
#define LoadPatternAndReshuffleMask(src, pattern_size) \
V128 pattern = LoadPattern(src, pattern_size);\
V128 reshuffle_mask = V128_Load(reinterpret_cast<const V128*>(\
pattern_reshuffle_masks[pattern_size - 1].data()));
#else
// Suppress -Wignored-attributes warning for __m128i in x86 SSE2 environment
// warning: ignoring attributes on template argument 'snappy::internal::V128' {aka '__vector(2) long long int'} [-Wignored-attributes]
// This occurs because __m128i has vector attributes (e.g., __attribute__((vector_size(16)))) that are ignored in template parameters.
#ifdef __SSE2__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wignored-attributes"
#endif
SNAPPY_ATTRIBUTE_ALWAYS_INLINE
static inline std::pair<V128 /* pattern */, V128 /* reshuffle_mask */>
@@ -345,7 +372,12 @@ LoadPatternAndReshuffleMask(const char* src, const size_t pattern_size) {
pattern_reshuffle_masks[pattern_size - 1].data()));
return {pattern, reshuffle_mask};
}
// Restore original diagnostic state in x86 SSE2 environment
#ifdef __SSE2__
#pragma GCC diagnostic pop
#endif
#endif
#endif // SNAPPY_HAVE_VECTOR_BYTE_SHUFFLE
// Fallback for when we need to copy while extending the pattern, for example
@@ -379,10 +411,14 @@ static inline bool Copy64BytesWithPatternExtension(char* dst, size_t offset) {
return true;
}
default: {
#if SNAPPY_HAVE_RVV
LoadPatternAndReshuffleMask(dst - offset, offset)
#else
auto pattern_and_reshuffle_mask =
LoadPatternAndReshuffleMask(dst - offset, offset);
V128 pattern = pattern_and_reshuffle_mask.first;
V128 reshuffle_mask = pattern_and_reshuffle_mask.second;
#endif
for (int i = 0; i < 4; i++) {
V128_StoreU(reinterpret_cast<V128*>(dst + 16 * i), pattern);
pattern = V128_Shuffle(pattern, reshuffle_mask);
@@ -490,11 +526,14 @@ inline char* IncrementalCopy(const char* src, char* op, char* const op_limit,
// Typically, the op_limit is the gating factor so try to simplify the loop
// based on that.
if (SNAPPY_PREDICT_TRUE(op_limit <= buf_limit - 15)) {
#if SNAPPY_HAVE_RVV
LoadPatternAndReshuffleMask(src, pattern_size);
#else
auto pattern_and_reshuffle_mask =
LoadPatternAndReshuffleMask(src, pattern_size);
V128 pattern = pattern_and_reshuffle_mask.first;
V128 reshuffle_mask = pattern_and_reshuffle_mask.second;
#endif
// There is at least one, and at most four 16-byte blocks. Writing four
// conditionals instead of a loop allows FDO to layout the code with
// respect to the actual probabilities of each length.
@@ -517,11 +556,14 @@ inline char* IncrementalCopy(const char* src, char* op, char* const op_limit,
}
char* const op_end = buf_limit - 15;
if (SNAPPY_PREDICT_TRUE(op < op_end)) {
#if SNAPPY_HAVE_RVV
LoadPatternAndReshuffleMask(src, pattern_size);
#else
auto pattern_and_reshuffle_mask =
LoadPatternAndReshuffleMask(src, pattern_size);
V128 pattern = pattern_and_reshuffle_mask.first;
V128 reshuffle_mask = pattern_and_reshuffle_mask.second;
#endif
// This code path is relatively cold however so we save code size
// by avoiding unrolling and vectorizing.
//
@@ -1247,13 +1289,41 @@ void MemCopy64(char* dst, const void* src, size_t size) {
_mm256_storeu_si256(reinterpret_cast<__m256i *>(dst) + 1, data);
}
#else
std::memmove(dst, src, kShortMemCopy);
// Profiling shows that nearly all copies are short.
if (SNAPPY_PREDICT_FALSE(size > kShortMemCopy)) {
std::memmove(dst + kShortMemCopy,
static_cast<const uint8_t*>(src) + kShortMemCopy,
64 - kShortMemCopy);
#ifdef SNAPPY_HAVE_RVV
uint8_t* dst_u8 = (uint8_t*)dst;
const uint8_t* src_u8 = (const uint8_t*)src;
if (src_u8 < dst_u8 && dst_u8 < src_u8 + size) { //overlap bwd copy
size_t offset = size;
while (offset > 0) {
size_t vl = VSETVL_E8M1(offset);
offset -= vl;
vuint8m1_t vec = VLE8_V_U8M1(src_u8 + offset, vl);
VSE8_V_U8M1(dst_u8 + offset, vec, vl);
}
} else {
size_t vl = VSETVL_E8M1(size);
if (vl < size) { // if size >vl,use the max_vlen copy
size_t offset = 0;
while (offset < size) {
vl = VSETVL_E8M1(size - offset);
vuint8m1_t vec = VLE8_V_U8M1(src_u8 + offset, vl);
VSE8_V_U8M1(dst_u8 + offset, vec, vl);
offset += vl;
}
} else { // copy the leaft
vuint8m1_t vec = VLE8_V_U8M1(src_u8, vl);
VSE8_V_U8M1(dst_u8, vec, vl);
}
}
#else
std::memmove(dst, src, kShortMemCopy);
//Profiling shows that nearly all copies are short.
if (SNAPPY_PREDICT_FALSE(size > kShortMemCopy)) {
std::memmove(dst + kShortMemCopy,
static_cast<const uint8_t*>(src) + kShortMemCopy,
64 - kShortMemCopy);}
#endif
#endif
}