From f50b387b4280585ddfe03191186ee24848dc630a Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Sat, 9 May 2026 18:33:57 +0000 Subject: [PATCH] Import changes PiperOrigin-RevId: 913030038 --- BUILD.bazel | 50 ++++++++++++++++++------------ CMakeLists.txt | 16 ++++++++-- COPYING.notestdata | 28 +++++++++++++++++ snappy-internal.h | 2 +- snappy-stubs-internal.h | 2 ++ snappy.cc | 65 ++++++++++++++++++++++++++++++++++++++- snappy.h | 28 ++++++++++++++--- snappy_benchmark.cc | 14 ++++----- snappy_compress_fuzzer.cc | 2 +- 9 files changed, 169 insertions(+), 38 deletions(-) create mode 100644 COPYING.notestdata diff --git a/BUILD.bazel b/BUILD.bazel index f8ef4ae..49db08e 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -26,11 +26,19 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +load("@bazel_skylib//rules:expand_template.bzl", "expand_template") +load("@bazel_skylib//rules:write_file.bzl", "write_file") +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) -exports_files(["COPYING"]) +exports_files([ + "COPYING", + "COPYING.notestdata", +]) SNAPPY_VERSION = (1, 2, 2) @@ -54,8 +62,10 @@ cc_library( name = "snappy-stubs-internal", srcs = ["snappy-stubs-internal.cc"], hdrs = ["snappy-stubs-internal.h"], - deps = [ + implementation_deps = [ ":config", + ], + deps = [ ":snappy-stubs-public", ], ) @@ -77,8 +87,10 @@ cc_library( "-Wno-sign-compare", ], }), - deps = [ + implementation_deps = [ ":config", + ], + deps = [ ":snappy-stubs-internal", ":snappy-stubs-public", ], @@ -117,7 +129,7 @@ cc_test( deps = [ ":snappy", ":snappy-test", - "//third_party/benchmark:benchmark_main", + "@com_google_benchmark//:benchmark_main", ], ) @@ -130,15 +142,15 @@ cc_test( deps = [ ":snappy", ":snappy-test", - "//third_party/googletest:gtest_main", + "@com_google_googletest//:gtest_main", ], ) # Generate a config.h similar to what cmake would produce. -genrule( +write_file( name = "config_h", - outs = ["config.h"], - cmd = """cat <$@ + out = "config.h", + content = """\ #define HAVE_STDDEF_H 1 #define HAVE_STDINT_H 1 #ifdef __has_builtin @@ -195,19 +207,17 @@ genrule( # define SNAPPY_IS_BIG_ENDIAN 1 # endif #endif -EOF -""", +""".splitlines(), ) -genrule( +expand_template( name = "snappy_stubs_public_h", - srcs = ["snappy-stubs-public.h.in"], - outs = ["snappy-stubs-public.h"], - # Assume sys/uio.h is available on non-Windows. - # Set the version numbers. - cmd = ("""sed -e 's/$${HAVE_SYS_UIO_H_01}/!_WIN32/g' \ - -e 's/$${PROJECT_VERSION_MAJOR}/%d/g' \ - -e 's/$${PROJECT_VERSION_MINOR}/%d/g' \ - -e 's/$${PROJECT_VERSION_PATCH}/%d/g' \ - $< >$@""" % SNAPPY_VERSION), + out = "snappy-stubs-public.h", + substitutions = { + "${HAVE_SYS_UIO_H_01}": "!_WIN32", + "${PROJECT_VERSION_MAJOR}": str(SNAPPY_VERSION[0]), + "${PROJECT_VERSION_MINOR}": str(SNAPPY_VERSION[1]), + "${PROJECT_VERSION_PATCH}": str(SNAPPY_VERSION[2]), + }, + template = "snappy-stubs-public.h.in", ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 490f5b8..ac43da3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -155,10 +155,20 @@ int main() { return __builtin_expect(0, 1); }" HAVE_BUILTIN_EXPECT) +# Check if the built-in __builtin_ctz (count trailing zeros) is available. +# Require either a non-RISC-V target, or a RISC-V core that implements +# the Zbb bit-manipulation extension where ctz is guaranteed. check_cxx_source_compiles(" -int main() { - return __builtin_ctzll(0); -}" HAVE_BUILTIN_CTZ) +#ifdef __riscv + #ifdef __riscv_zbb + int main() { return __builtin_ctzll(0); } + #else + #error \"ZBB not enabled in current config\" + #endif +#else + int main() { return __builtin_ctzll(0); } +#endif +" HAVE_BUILTIN_CTZ) check_cxx_source_compiles(" int main() { diff --git a/COPYING.notestdata b/COPYING.notestdata new file mode 100644 index 0000000..8d6bd9f --- /dev/null +++ b/COPYING.notestdata @@ -0,0 +1,28 @@ +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/snappy-internal.h b/snappy-internal.h index 7fb88cb..f68ab77 100644 --- a/snappy-internal.h +++ b/snappy-internal.h @@ -194,7 +194,7 @@ char* CompressFragment(const char* input, // riscv and little-endian cpu choose this routinue can be done faster too. #if !SNAPPY_IS_BIG_ENDIAN && \ (defined(__x86_64__) || defined(_M_X64) || defined(ARCH_PPC) || \ - defined(__aarch64__) || defined(__riscv)) + defined(__aarch64__) || (defined(__riscv) && (__riscv_xlen == 64))) static inline std::pair FindMatchLength(const char* s1, const char* s2, const char* s2_limit, diff --git a/snappy-stubs-internal.h b/snappy-stubs-internal.h index 526c38b..1625c67 100644 --- a/snappy-stubs-internal.h +++ b/snappy-stubs-internal.h @@ -304,6 +304,7 @@ class Bits { void operator=(const Bits&); }; +// In RISC-V, CLZ is supported by instructions from the ZBB bit-manipulation extension. #if HAVE_BUILTIN_CTZ inline int Bits::Log2FloorNonZero(uint32_t n) { @@ -393,6 +394,7 @@ inline int Bits::FindLSBSetNonZero(uint32_t n) { #endif // End portable versions. +// In RISC-V, CLZ is supported by instructions from the ZBB bit-manipulation extension. #if HAVE_BUILTIN_CTZ inline int Bits::FindLSBSetNonZero64(uint64_t n) { diff --git a/snappy.cc b/snappy.cc index 846b3b0..715f207 100644 --- a/snappy.cc +++ b/snappy.cc @@ -1364,6 +1364,32 @@ inline size_t AdvanceToNextTagX86Optimized(const uint8_t** ip_p, size_t* tag) { return tag_type; } +SNAPPY_ATTRIBUTE_ALWAYS_INLINE +inline size_t AdvanceToNextTagRVOptimized(const uint8_t** ip_p, size_t* tag) { + const uint8_t*& ip = *ip_p; + // This section is crucial for the throughput of the decompression loop. + // The latency of an iteration is fundamentally constrained by the data chain on ip: + // ip -> c = *tag -> literal_len = c >> 2, tag_type = c & 3 + // -> literal_advance = literal_len + 2, copy_advance = tag_type + 1 + // -> next_ip = ip + literal_advance OR ip + copy_advance (literal vs copy) + // -> *tag = byte at (next_ip - 1); ip = next_ip + // + // Base RISC-V has no x86-style cmov and no AArch64 csinc on the same shape; this + // computes both candidate advances and both load offsets, then selects with + // (is_literal ? ... : ...). With the Zicond extension (czero.eqz / czero.nez), those + // selections typically lower to branchless conditional-zero ops instead of a + // hard-to-predict literal/copy branch, which is why this form tends to win there. + const size_t literal_len = *tag >> 2; + const size_t tag_type = *tag & 3; + const bool is_literal = (tag_type == 0); + const size_t copy_advance = tag_type + 1; + const size_t literal_advance = literal_len + 2; + const uint8_t* next_ip = is_literal ? (ip + literal_advance) : (ip + copy_advance); + *tag = is_literal ? ip[literal_advance - 1] : ip[copy_advance - 1]; + ip = next_ip; + return tag_type; +} + // Extract the offset for copy-1 and copy-2 returns 0 for literals or copy-4. inline uint32_t ExtractOffset(uint32_t val, size_t tag_type) { // For Arm non-static storage works better. For x86 static storage is better. @@ -1376,7 +1402,13 @@ inline uint32_t ExtractOffset(uint32_t val, size_t tag_type) { reinterpret_cast(&kExtractMasksCombined) + 2 * tag_type, sizeof(result)); return val & result; -#elif defined(__aarch64__) + // For AArch64 and RISC-V, use a bit-twiddling trick to extract the mask from a + // single combined constant instead of a lookup table. The constant packs multiple + // 16-bit masks based on tag_type (see implementation below). The code calculates + // the shift amount from tag_type, right-shifts the constant to move the desired + // mask to the LSB position, then extracts it with & 0xFFFF. This branchless + // approach is often more performant on modern CPUs. +#elif defined(__aarch64__) || (defined(__riscv) && (__riscv_xlen == 64)) constexpr uint64_t kExtractMasksCombined = 0x0000FFFF00FF0000ull; return val & static_cast( (kExtractMasksCombined >> (tag_type * 16)) & 0xFFFF); @@ -1440,6 +1472,11 @@ std::pair DecompressBranchless( // We never need more than 16 bits. Doing a Load16 allows the compiler // to elide the masking operation in ExtractOffset. next = LittleEndian::Load16(old_ip); +#elif defined(__riscv) + size_t tag_type = AdvanceToNextTagRVOptimized(&ip, &tag); + // We never need more than 16 bits. Doing a Load16 allows the compiler + // to elide the masking operation in ExtractOffset. + next = LittleEndian::Load16(old_ip); #else size_t tag_type = AdvanceToNextTagX86Optimized(&ip, &tag); next = LittleEndian::Load32(old_ip); @@ -1813,6 +1850,10 @@ bool GetUncompressedLength(Source* source, uint32_t* result) { return decompressor.ReadUncompressedLength(result); } +size_t Compress(Source* reader, Sink* writer) { + return Compress(reader, writer, CompressionOptions{}); +} + size_t Compress(Source* reader, Sink* writer, CompressionOptions options) { assert(options.level == 1 || options.level == 2); int snappy_token = 0; @@ -2319,6 +2360,12 @@ bool IsValidCompressed(Source* compressed) { return InternalUncompress(compressed, &writer); } +void RawCompress(const char* input, size_t input_length, char* compressed, + size_t* compressed_length) { + RawCompress(input, input_length, compressed, compressed_length, + CompressionOptions{}); +} + void RawCompress(const char* input, size_t input_length, char* compressed, size_t* compressed_length, CompressionOptions options) { ByteArraySource reader(input, input_length); @@ -2329,6 +2376,12 @@ void RawCompress(const char* input, size_t input_length, char* compressed, *compressed_length = (writer.CurrentDestination() - compressed); } +void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, + char* compressed, size_t* compressed_length) { + RawCompressFromIOVec(iov, uncompressed_length, compressed, compressed_length, + CompressionOptions{}); +} + void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, char* compressed, size_t* compressed_length, CompressionOptions options) { @@ -2340,6 +2393,11 @@ void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, *compressed_length = writer.CurrentDestination() - compressed; } +size_t Compress(const char* input, size_t input_length, + std::string* compressed) { + return Compress(input, input_length, compressed, CompressionOptions{}); +} + size_t Compress(const char* input, size_t input_length, std::string* compressed, CompressionOptions options) { // Pre-grow the buffer to the max length of the compressed output @@ -2352,6 +2410,11 @@ size_t Compress(const char* input, size_t input_length, std::string* compressed, return compressed_length; } +size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt, + std::string* compressed) { + return CompressFromIOVec(iov, iov_cnt, compressed, CompressionOptions{}); +} + size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt, std::string* compressed, CompressionOptions options) { // Compute the number of bytes to be compressed. diff --git a/snappy.h b/snappy.h index 04b7a96..ecd83d7 100644 --- a/snappy.h +++ b/snappy.h @@ -64,6 +64,10 @@ namespace snappy { // faster decompression speeds than snappy:1 and zstd:-3. int level = DefaultCompressionLevel(); + constexpr CompressionOptions() = default; + constexpr CompressionOptions(int compression_level) + : level(compression_level) {} + static constexpr int MinCompressionLevel() { return 1; } static constexpr int MaxCompressionLevel() { return 2; } static constexpr int DefaultCompressionLevel() { return 1; } @@ -75,8 +79,10 @@ namespace snappy { // Compress the bytes read from "*reader" and append to "*writer". Return the // number of bytes written. + // First version is to preserve ABI. + size_t Compress(Source* reader, Sink* writer); size_t Compress(Source* reader, Sink* writer, - CompressionOptions options = {}); + CompressionOptions options); // Find the uncompressed length of the given stream, as given by the header. // Note that the true length could deviate from this; the stream could e.g. @@ -95,16 +101,22 @@ namespace snappy { // Original contents of *compressed are lost. // // REQUIRES: "input[]" is not an alias of "*compressed". + // First version is to preserve ABI. size_t Compress(const char* input, size_t input_length, - std::string* compressed, CompressionOptions options = {}); + std::string* compressed); + size_t Compress(const char* input, size_t input_length, + std::string* compressed, CompressionOptions options); // Same as `Compress` above but taking an `iovec` array as input. Note that // this function preprocesses the inputs to compute the sum of // `iov[0..iov_cnt-1].iov_len` before reading. To avoid this, use // `RawCompressFromIOVec` below. + // First version is to preserve ABI. + size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt, + std::string* compressed); size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt, std::string* compressed, - CompressionOptions options = {}); + CompressionOptions options); // Decompresses "compressed[0..compressed_length-1]" to "*uncompressed". // Original contents of "*uncompressed" are lost. @@ -147,15 +159,21 @@ namespace snappy { // RawCompress(input, input_length, output, &output_length); // ... Process(output, output_length) ... // delete [] output; + // First version is to preserve ABI. void RawCompress(const char* input, size_t input_length, char* compressed, - size_t* compressed_length, CompressionOptions options = {}); + size_t* compressed_length); + void RawCompress(const char* input, size_t input_length, char* compressed, + size_t* compressed_length, CompressionOptions options); // Same as `RawCompress` above but taking an `iovec` array as input. Note that // `uncompressed_length` is the total number of bytes to be read from the // elements of `iov` (_not_ the number of elements in `iov`). + // First version is to preserve ABI. + void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, + char* compressed, size_t* compressed_length); void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length, char* compressed, size_t* compressed_length, - CompressionOptions options = {}); + CompressionOptions options); // Given data in "compressed[0..compressed_length-1]" generated by // calling the Snappy::Compress routine, this routine diff --git a/snappy_benchmark.cc b/snappy_benchmark.cc index 90243a8..98501ea 100644 --- a/snappy_benchmark.cc +++ b/snappy_benchmark.cc @@ -65,7 +65,7 @@ void BM_UFlat(benchmark::State& state) { std::string zcontents; snappy::Compress(contents.data(), contents.size(), &zcontents, - snappy::CompressionOptions{.level = state.range(1)}); + snappy::CompressionOptions{/*level=*/state.range(1)}); char* dst = new char[contents.size()]; for (auto s : state) { @@ -130,7 +130,7 @@ void BM_UValidate(benchmark::State& state) { std::string zcontents; snappy::Compress(contents.data(), contents.size(), &zcontents, - snappy::CompressionOptions{.level = state.range(1)}); + snappy::CompressionOptions{/*level=*/state.range(1)}); for (auto s : state) { CHECK(snappy::IsValidCompressedBuffer(zcontents.data(), zcontents.size())); @@ -193,7 +193,7 @@ void BM_UIOVecSource(benchmark::State& state) { size_t zsize = 0; for (auto s : state) { snappy::RawCompressFromIOVec(iov, contents.size(), dst, &zsize, - snappy::CompressionOptions{.level = level}); + snappy::CompressionOptions{/*level=*/level}); benchmark::DoNotOptimize(iov); } state.SetBytesProcessed(static_cast(state.iterations()) * @@ -268,7 +268,7 @@ void BM_UFlatSink(benchmark::State& state) { std::string zcontents; snappy::Compress(contents.data(), contents.size(), &zcontents, - snappy::CompressionOptions{.level = state.range(1)}); + snappy::CompressionOptions{/*level=*/state.range(1)}); char* dst = new char[contents.size()]; for (auto s : state) { @@ -304,7 +304,7 @@ void BM_ZFlat(benchmark::State& state) { size_t zsize = 0; for (auto s : state) { snappy::RawCompress(contents.data(), contents.size(), dst, &zsize, - snappy::CompressionOptions{.level = level}); + snappy::CompressionOptions{/*level=*/level}); benchmark::DoNotOptimize(dst); } state.SetBytesProcessed(static_cast(state.iterations()) * @@ -340,7 +340,7 @@ void BM_ZFlatAll(benchmark::State& state) { for (auto s : state) { for (int i = 0; i < num_files; ++i) { snappy::RawCompress(contents[i].data(), contents[i].size(), dst[i], - &zsize, snappy::CompressionOptions{.level = level}); + &zsize, snappy::CompressionOptions{/*level=*/level}); benchmark::DoNotOptimize(dst); } } @@ -377,7 +377,7 @@ void BM_ZFlatIncreasingTableSize(benchmark::State& state) { for (auto s : state) { for (size_t i = 0; i < contents.size(); ++i) { snappy::RawCompress(contents[i].data(), contents[i].size(), dst[i], - &zsize, snappy::CompressionOptions{.level = level}); + &zsize, snappy::CompressionOptions{/*level=*/level}); benchmark::DoNotOptimize(dst); } } diff --git a/snappy_compress_fuzzer.cc b/snappy_compress_fuzzer.cc index 144f438..93254a2 100644 --- a/snappy_compress_fuzzer.cc +++ b/snappy_compress_fuzzer.cc @@ -44,7 +44,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { std::string compressed; size_t compressed_size = snappy::Compress(input.data(), input.size(), &compressed, - snappy::CompressionOptions{.level = level}); + snappy::CompressionOptions{/*level=*/level}); (void)compressed_size; // Variable only used in debug builds. assert(compressed_size == compressed.size());