fabi/Snappy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Internal changes

Browse Source 
PiperOrigin-RevId: 621907514
This commit is contained in:
Richard O'Grady
2024-04-04 17:48:40 +00:00
committed by Danila Kutenin
parent 0212163f6e
commit e1e7329dc7
18 changed files with 83 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
snappy.cc
View File

@@ -74,7 +74,6 @@
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <functional>
#include <memory>
#include <string>
#include <utility>
@@ -281,8 +280,6 @@ inline char* IncrementalCopySlow(const char* src, char* op,
// 4, 5, 0, 1, 2, 3, 4, 5, 0, 1}. These byte index sequences are generated by
// calling MakePatternMaskBytes(0, 6, index_sequence<16>()) and
// MakePatternMaskBytes(16, 6, index_sequence<16>()) respectively.
template <size_t... indexes>
inline constexpr std::array<char, sizeof...(indexes)> MakePatternMaskBytes(
int index_offset, int pattern_size, index_sequence<indexes...>) {
@@ -300,6 +297,7 @@ MakePatternMaskBytesTable(int index_offset,
MakePatternMaskBytes(index_offset, pattern_sizes_minus_one + 1,
make_index_sequence</*indexes=*/sizeof(V128)>())...};
}
// This is an array of shuffle control masks that can be used as the source
// operand for PSHUFB to permute the contents of the destination XMM register
// into a repeating byte pattern.
@@ -330,6 +328,7 @@ static inline V128 LoadPattern(const char* src, const size_t pattern_size) {
return V128_Shuffle(V128_LoadU(reinterpret_cast<const V128*>(src)),
generation_mask);
}
SNAPPY_ATTRIBUTE_ALWAYS_INLINE
static inline std::pair<V128 /* pattern */, V128 /* reshuffle_mask */>
LoadPatternAndReshuffleMask(const char* src, const size_t pattern_size) {
@@ -345,6 +344,7 @@ LoadPatternAndReshuffleMask(const char* src, const size_t pattern_size) {
pattern_reshuffle_masks[pattern_size - 1].data()));
return {pattern, reshuffle_mask};
}
#endif // SNAPPY_HAVE_VECTOR_BYTE_SHUFFLE
// Fallback for when we need to copy while extending the pattern, for example
@@ -493,6 +493,7 @@ inline char* IncrementalCopy(const char* src, char* op, char* const op_limit,
LoadPatternAndReshuffleMask(src, pattern_size);
V128 pattern = pattern_and_reshuffle_mask.first;
V128 reshuffle_mask = pattern_and_reshuffle_mask.second;
// There is at least one, and at most four 16-byte blocks. Writing four
// conditionals instead of a loop allows FDO to layout the code with
// respect to the actual probabilities of each length.
@@ -519,6 +520,7 @@ inline char* IncrementalCopy(const char* src, char* op, char* const op_limit,
LoadPatternAndReshuffleMask(src, pattern_size);
V128 pattern = pattern_and_reshuffle_mask.first;
V128 reshuffle_mask = pattern_and_reshuffle_mask.second;
// This code path is relatively cold however so we save code size
// by avoiding unrolling and vectorizing.
//
@@ -957,8 +959,6 @@ emit_remainder:
char* CompressFragmentDoubleHash(const char* input, size_t input_size, char* op,
uint16_t* table, const int table_size,
uint16_t* table2, const int table_size2) {
(void)table_size2;
assert(table_size == table_size2);
// "ip" is the input pointer, and "op" is the output pointer.
const char* ip = input;
assert(input_size <= kBlockSize);
@@ -1224,7 +1224,7 @@ inline bool Copy64BytesWithPatternExtension(ptrdiff_t dst, size_t offset) {
void MemCopy64(char* dst, const void* src, size_t size) {
// Always copy this many bytes. If that's below size then copy the full 64.
constexpr int kShortMemCopy = 32;
(void)kShortMemCopy;
assert(size <= 64);
assert(std::less_equal<const void*>()(static_cast<const char*>(src) + size,
dst) ||
@@ -1243,27 +1243,6 @@ void MemCopy64(char* dst, const void* src, size_t size) {
data = _mm256_lddqu_si256(static_cast<const __m256i *>(src) + 1);
_mm256_storeu_si256(reinterpret_cast<__m256i *>(dst) + 1, data);
}
// RVV acceleration available on RISC-V when compiled with -march=rv64gcv
#elif defined(__riscv) && SNAPPY_HAVE_RVV
// Cast pointers to the type we will operate on.
unsigned char* dst_ptr = reinterpret_cast<unsigned char*>(dst);
const unsigned char* src_ptr = reinterpret_cast<const unsigned char*>(src);
size_t remaining_bytes = size;
// Loop as long as there are bytes remaining to be copied.
while (remaining_bytes > 0) {
// Set vector configuration: e8 (8-bit elements), m2 (LMUL=2).
// Use e8m2 configuration to maximize throughput.
size_t vl = VSETVL_E8M2(remaining_bytes);
// Load data from the current source pointer.
vuint8m2_t vec = VLE8_V_U8M2(src_ptr, vl);
// Store data to the current destination pointer.
VSE8_V_U8M2(dst_ptr, vec, vl);
// Update pointers and the remaining count.
src_ptr += vl;
dst_ptr += vl;
remaining_bytes -= vl;
}
#else
std::memmove(dst, src, kShortMemCopy);
// Profiling shows that nearly all copies are short.
@@ -1363,32 +1342,6 @@ inline size_t AdvanceToNextTagX86Optimized(const uint8_t** ip_p, size_t* tag) {
return tag_type;
}
SNAPPY_ATTRIBUTE_ALWAYS_INLINE
inline size_t AdvanceToNextTagRVOptimized(const uint8_t** ip_p, size_t* tag) {
const uint8_t*& ip = *ip_p;
// This section is crucial for the throughput of the decompression loop.
// The latency of an iteration is fundamentally constrained by the data chain on ip:
// ip -> c = *tag -> literal_len = c >> 2, tag_type = c & 3
// -> literal_advance = literal_len + 2, copy_advance = tag_type + 1
// -> next_ip = ip + literal_advance OR ip + copy_advance (literal vs copy)
// -> *tag = byte at (next_ip - 1); ip = next_ip
//
// Base RISC-V has no x86-style cmov and no AArch64 csinc on the same shape; this
// computes both candidate advances and both load offsets, then selects with
// (is_literal ? ... : ...). With the Zicond extension (czero.eqz / czero.nez), those
// selections typically lower to branchless conditional-zero ops instead of a
// hard-to-predict literal/copy branch, which is why this form tends to win there.
const size_t literal_len = *tag >> 2;
const size_t tag_type = *tag & 3;
const bool is_literal = (tag_type == 0);
const size_t copy_advance = tag_type + 1;
const size_t literal_advance = literal_len + 2;
const uint8_t* next_ip = is_literal ? (ip + literal_advance) : (ip + copy_advance);
*tag = is_literal ? ip[literal_advance - 1] : ip[copy_advance - 1];
ip = next_ip;
return tag_type;
}
// Extract the offset for copy-1 and copy-2 returns 0 for literals or copy-4.
inline uint32_t ExtractOffset(uint32_t val, size_t tag_type) {
// For x86 non-static storage works better. For ARM static storage is better.
@@ -1401,13 +1354,7 @@ inline uint32_t ExtractOffset(uint32_t val, size_t tag_type) {
reinterpret_cast<const char*>(&kExtractMasksCombined) + 2 * tag_type,
sizeof(result));
return val & result;
// For AArch64 and RISC-V, use a bit-twiddling trick to extract the mask from a
// single combined constant instead of a lookup table. The constant packs multiple
// 16-bit masks based on tag_type (see implementation below). The code calculates
// the shift amount from tag_type, right-shifts the constant to move the desired
// mask to the LSB position, then extracts it with & 0xFFFF. This branchless
// approach is often more performant on modern CPUs.
#elif defined(__aarch64__) || (defined(__riscv) && (__riscv_xlen == 64))
#elif defined(__aarch64__)
constexpr uint64_t kExtractMasksCombined = 0x0000FFFF00FF0000ull;
return val & static_cast<uint32_t>(
(kExtractMasksCombined >> (tag_type * 16)) & 0xFFFF);
@@ -1471,11 +1418,6 @@ std::pair<const uint8_t*, ptrdiff_t> DecompressBranchless(
// We never need more than 16 bits. Doing a Load16 allows the compiler
// to elide the masking operation in ExtractOffset.
next = LittleEndian::Load16(old_ip);
#elif defined(__riscv)
size_t tag_type = AdvanceToNextTagRVOptimized(&ip, &tag);
// We never need more than 16 bits. Doing a Load16 allows the compiler
// to elide the masking operation in ExtractOffset.
next = LittleEndian::Load16(old_ip);
#else
size_t tag_type = AdvanceToNextTagX86Optimized(&ip, &tag);
next = LittleEndian::Load32(old_ip);
@@ -1554,7 +1496,7 @@ class SnappyDecompressor {
// If ip < ip_limit_min_maxtaglen_ it's safe to read kMaxTagLength from
// buffer.
const char* ip_limit_min_maxtaglen_;
uint64_t peeked_; // Bytes peeked from reader (need to skip)
uint32_t peeked_; // Bytes peeked from reader (need to skip)
bool eof_; // Hit end of input without an error?
char scratch_[kMaximumTagLength]; // See RefillTag().
@@ -1745,8 +1687,7 @@ constexpr uint32_t CalculateNeeded(uint8_t tag) {
#if __cplusplus >= 201402L
constexpr bool VerifyCalculateNeeded() {
for (int i = 0; i < 1; i++) {
if (CalculateNeeded(i) != static_cast<uint32_t>((char_table[i] >> 11)) + 1)
return false;
if (CalculateNeeded(i) != (char_table[i] >> 11) + 1) return false;
}
return true;
}
@@ -1782,7 +1723,7 @@ bool SnappyDecompressor::RefillTag() {
assert(needed <= sizeof(scratch_));
// Read more bytes from reader if needed
uint64_t nbuf = ip_limit_ - ip;
uint32_t nbuf = ip_limit_ - ip;
if (nbuf < needed) {
// Stitch together bytes from ip and reader to form the word
// contents. We store the needed bytes in "scratch_". They
@@ -1795,7 +1736,7 @@ bool SnappyDecompressor::RefillTag() {
size_t length;
const char* src = reader_->Peek(&length);
if (length == 0) return false;
uint64_t to_add = std::min<uint64_t>(needed - nbuf, length);
uint32_t to_add = std::min<uint32_t>(needed - nbuf, length);
std::memcpy(scratch_ + nbuf, src, to_add);
nbuf += to_add;
reader_->Skip(to_add);
@@ -1849,16 +1790,11 @@ bool GetUncompressedLength(Source* source, uint32_t* result) {
return decompressor.ReadUncompressedLength(result);
}
size_t Compress(Source* reader, Sink* writer) {
return Compress(reader, writer, CompressionOptions{});
}
size_t Compress(Source* reader, Sink* writer, CompressionOptions options) {
assert(options.level == 1 || options.level == 2);
int token = 0;
size_t written = 0;
size_t N = reader->Available();
assert(N <= 0xFFFFFFFFu);
const size_t uncompressed_size = N;
char ulength[Varint::kMax32];
char* p = Varint::Encode32(ulength, N);
@@ -1950,16 +1886,16 @@ class SnappyIOVecReader : public Source {
if (total_size > 0 && curr_size_remaining_ == 0) Advance();
}
~SnappyIOVecReader() override = default;
~SnappyIOVecReader() = default;
size_t Available() const override { return total_size_remaining_; }
size_t Available() const { return total_size_remaining_; }
const char* Peek(size_t* len) override {
const char* Peek(size_t* len) {
*len = curr_size_remaining_;
return curr_pos_;
}
void Skip(size_t n) override {
void Skip(size_t n) {
while (n >= curr_size_remaining_ && n > 0) {
n -= curr_size_remaining_;
Advance();
@@ -2360,12 +2296,6 @@ bool IsValidCompressed(Source* compressed) {
return InternalUncompress(compressed, &writer);
}
void RawCompress(const char* input, size_t input_length, char* compressed,
size_t* compressed_length) {
RawCompress(input, input_length, compressed, compressed_length,
CompressionOptions{});
}
void RawCompress(const char* input, size_t input_length, char* compressed,
size_t* compressed_length, CompressionOptions options) {
ByteArraySource reader(input, input_length);
@@ -2376,12 +2306,6 @@ void RawCompress(const char* input, size_t input_length, char* compressed,
*compressed_length = (writer.CurrentDestination() - compressed);
}
void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length,
char* compressed, size_t* compressed_length) {
RawCompressFromIOVec(iov, uncompressed_length, compressed, compressed_length,
CompressionOptions{});
}
void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length,
char* compressed, size_t* compressed_length,
CompressionOptions options) {
@@ -2393,11 +2317,6 @@ void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length,
*compressed_length = writer.CurrentDestination() - compressed;
}
size_t Compress(const char* input, size_t input_length,
std::string* compressed) {
return Compress(input, input_length, compressed, CompressionOptions{});
}
size_t Compress(const char* input, size_t input_length, std::string* compressed,
CompressionOptions options) {
// Pre-grow the buffer to the max length of the compressed output
@@ -2410,11 +2329,6 @@ size_t Compress(const char* input, size_t input_length, std::string* compressed,
return compressed_length;
}
size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt,
std::string* compressed) {
return CompressFromIOVec(iov, iov_cnt, compressed, CompressionOptions{});
}
size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt,
std::string* compressed, CompressionOptions options) {
// Compute the number of bytes to be compressed.
@@ -2619,6 +2533,7 @@ bool SnappyScatteredWriter<Allocator>::SlowAppendFromSelf(size_t offset,
class SnappySinkAllocator {
public:
explicit SnappySinkAllocator(Sink* dest) : dest_(dest) {}
~SnappySinkAllocator() {}
char* Allocate(int size) {
Datablock block(new char[size], size);