fabi/Snappy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Internal changes

Browse Source 
PiperOrigin-RevId: 822002697
This commit is contained in:
Danila Kutenin
2025-10-21 08:32:58 +00:00
parent ec20182d26
commit 368c77c96c
10 changed files with 171 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.bazelrc
View File

@@ -1,2 +1,4 @@
# googletest requires C++14 or above
build --cxxopt='-std=c++17'
# Enable Bzlmod for every Bazel command
common --enable_bzlmod

43
.github/workflows/riscv64-qemu-test.yaml vendored Normal file
View File

@@ -0,0 +1,43 @@
name: riscv64-qemu-test
on: [push, pull_request]
jobs:
test:
runs-on: ubuntu-latest
env:
RISCV_CROSSCOMPILE: "ON"
riscv_gnu_toolchain_download_path: https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2025.07.03/riscv64-glibc-ubuntu-24.04-gcc-nightly-2025.07.03-nightly.tar.xz
RISCV_PATH: /opt/riscv
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Install dependencies
run: |
sudo apt update
sudo apt install -y --no-install-recommends \
qemu-user qemu-user-static \
build-essential \
cmake \
git
sudo mkdir -p $RISCV_PATH
wget ${riscv_gnu_toolchain_download_path} -O riscv-toolchain.tar.xz
sudo tar -xvf riscv-toolchain.tar.xz -C $RISCV_PATH --strip-components=1
sudo sed -i "s|libdir='/mnt/riscv/riscv64-unknown-linux-gnu/lib'|libdir='$RISCV_PATH/riscv64-unknown-linux-gnu/lib'|g" $RISCV_PATH/riscv64-unknown-linux-gnu/lib/libatomic.la
- name: Build and Run Unit Tests
run: |
export PATH=$RISCV_PATH/bin:$PATH
export LD_LIBRARY_PATH="/opt/riscv/lib:$LD_LIBRARY_PATH"
export QEMU_LD_PREFIX=$RISCV_PATH/sysroot
mkdir build && cd build
cmake -DCMAKE_BUILD_TYPE=Release ../
make -j$(nproc)
make test
- name: Run Benchmark
run: ./build/snappy_benchmark
working-directory: ./

1
.gitignore vendored
View File

@@ -6,4 +6,5 @@
# Build directory.
build/
/bazel-*
MODULE.bazel.lock
out/

13
BUILD.bazel
View File

@@ -30,7 +30,7 @@ package(default_visibility = ["//visibility:public"])
licenses(["notice"])
SNAPPY_VERSION = (1, 1, 10)
SNAPPY_VERSION = (1, 2, 2)
config_setting(
name = "windows",
@@ -40,7 +40,7 @@ config_setting(
cc_library(
name = "config",
hdrs = ["config.h"],
defines = ["HAVE_CONFIG_H"]
defines = ["HAVE_CONFIG_H"],
)
cc_library(
@@ -70,10 +70,11 @@ cc_library(
"snappy-sinksource.h",
],
copts = select({
":windows": [],
"//conditions:default": [
"-Wno-sign-compare",
]}),
":windows": [],
"//conditions:default": [
"-Wno-sign-compare",
],
}),
deps = [
":config",
":snappy-stubs-internal",

49
CMakeLists.txt
View File

@@ -26,8 +26,8 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cmake_minimum_required(VERSION 3.1)
project(Snappy VERSION 1.1.10 LANGUAGES C CXX)
cmake_minimum_required(VERSION 3.10)
project(Snappy VERSION 1.2.2 LANGUAGES C CXX)
# C++ standard can be overridden when this is used as a sub-project.
if(NOT CMAKE_CXX_STANDARD)
@@ -38,7 +38,7 @@ if(NOT CMAKE_CXX_STANDARD)
endif(NOT CMAKE_CXX_STANDARD)
# https://github.com/izenecloud/cmake/blob/master/SetCompilerWarningAll.cmake
if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
if(MSVC)
# Use the highest warning level for Visual Studio.
set(CMAKE_CXX_WARNING_LEVEL 4)
if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]")
@@ -55,7 +55,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# Disable RTTI.
string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
else(MSVC)
# Use -Wall for clang and gcc.
if(NOT CMAKE_CXX_FLAGS MATCHES "-Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
@@ -85,7 +85,7 @@ else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# Disable RTTI.
string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
endif(MSVC)
# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make
# it prominent in the GUI.
@@ -206,13 +206,41 @@ int main() {
check_cxx_source_compiles("
#include <arm_neon.h>
#include <stdint.h>
int main() {
uint8_t val = 3, dup[8];
uint8x16_t v = vld1q_dup_u8(&val);
vst1q_u8(dup, v);
uint8x16_t v1 = vld1q_dup_u8(&val);
uint8x16_t v2 = vqtbl1q_u8(v1, v1);
vst1q_u8(dup, v1);
vst1q_u8(dup, v2);
return 0;
}" SNAPPY_HAVE_NEON)
#check RVV 1.0 need __riscv_ prefix
check_cxx_source_compiles("
#include <riscv_vector.h>
#include <stdint.h>
#include <stddef.h>
int main() {
uint8_t val = 3, dup[8];
size_t vl = __riscv_vsetvl_e8m1(8);
vuint8m1_t v = __riscv_vmv_v_x_u8m1(val, vl);
return 0;
}" SNAPPY_RVV_1)
#check RVV 0.7.1 not __riscv_ prefix
check_cxx_source_compiles("
#include <riscv_vector.h>
#include <stdint.h>
#include <stddef.h>
int main() {
uint8_t val = 3, dup[8];
size_t vl = vsetvl_e8m1(8);
vuint8m1_t v = vmv_v_x_u8m1(val, vl);
return 0;
}" SNAPPY_RVV_0_7)
include(CheckSymbolExists)
check_symbol_exists("mmap" "sys/mman.h" HAVE_FUNC_MMAP)
check_symbol_exists("sysconf" "unistd.h" HAVE_FUNC_SYSCONF)
@@ -258,9 +286,7 @@ target_sources(snappy
"snappy-stubs-internal.cc"
"snappy.cc"
"${PROJECT_BINARY_DIR}/config.h"
# Only CMake 3.3+ supports PUBLIC sources in targets exported by "install".
$<$<VERSION_GREATER:CMAKE_VERSION,3.2>:PUBLIC>
PUBLIC
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy-c.h>
$<INSTALL_INTERFACE:include/snappy-c.h>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/snappy-sinksource.h>
@@ -297,6 +323,9 @@ if(SNAPPY_BUILD_TESTS OR SNAPPY_BUILD_BENCHMARKS)
# Test files include snappy-test.h, HAVE_CONFIG_H must be defined.
target_compile_definitions(snappy_test_support PUBLIC -DHAVE_CONFIG_H)
if(BUILD_SHARED_LIBS)
set_target_properties(snappy_test_support PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
endif(BUILD_SHARED_LIBS)
target_link_libraries(snappy_test_support snappy)

15
NEWS
View File

@@ -1,3 +1,18 @@
Snappy v1.2.2, Mar 26th 2025:
* We added a new compression level in v1.2.1 which compresses a bit
denser but slower. Decompression speed should be even faster with it.
* We fixed a very old issue of data corruption when compressed size
exceeds 4GB. This can happen when you compress data close to 4GB
and it's incompressible, for example, random data.
* Started to use minimum CMake 3.10 because older ones are not
planned to be supported.
* Various other small fixes and performance improvements (especially
for clang).
Snappy v1.1.10, Mar 8th 2023:
* Performance improvements

8
README.md
View File

@@ -140,10 +140,10 @@ explicitly supports the following:
1. C++11
2. Clang (gcc and MSVC are best-effort).
3. Low level optimizations (e.g. assembly or equivalent intrinsics) for:
1. [x86](https://en.wikipedia.org/wiki/X86)
2. [x86-64](https://en.wikipedia.org/wiki/X86-64)
3. ARMv7 (32-bit)
4. ARMv8 (AArch64)
- [x86](https://en.wikipedia.org/wiki/X86)
- [x86-64](https://en.wikipedia.org/wiki/X86-64)
- ARMv7 (32-bit)
- ARMv8 (AArch64)
4. Supports only the Snappy compression scheme as described in
[format_description.txt](format_description.txt).
5. CMake for building

6
cmake/config.h.in
View File

@@ -58,6 +58,12 @@
/* Define to 1 if you target processors with NEON and have <arm_neon.h>. */
#cmakedefine01 SNAPPY_HAVE_NEON
/* Define to 1 if you target processors with RVV1.0 and have <riscv_vector.h>. */
#cmakedefine01 SNAPPY_RVV_1
/* Define to 1 if you target processors with RVV0.7 and have <riscv_vector.h>. */
#cmakedefine01 SNAPPY_RVV_0_7
/* Define to 1 if you have <arm_neon.h> and <arm_acle.h> and want to optimize
compression speed by using __crc32cw from <arm_acle.h>. */
#cmakedefine01 SNAPPY_HAVE_NEON_CRC32

22
snappy-internal.h
View File

@@ -46,6 +46,23 @@
#include <arm_neon.h>
#endif
#if SNAPPY_RVV_1 || SNAPPY_RVV_0_7
#define SNAPPY_HAVE_RVV 1
#include <riscv_vector.h>
#else
#define SNAPPY_HAVE_RVV 0
#endif
#ifdef SNAPPY_RVV_1
#define VSETVL_E8M2 __riscv_vsetvl_e8m2
#define VLE8_V_U8M2 __riscv_vle8_v_u8m2
#define VSE8_V_U8M2 __riscv_vse8_v_u8m2
#elif SNAPPY_RVV_0_7
#define VSETVL_E8M2 vsetvl_e8m2
#define VLE8_V_U8M2 vle8_v_u8m2
#define VSE8_V_U8M2 vse8_v_u8m2
#endif
#if SNAPPY_HAVE_SSSE3 || SNAPPY_HAVE_NEON
#define SNAPPY_HAVE_VECTOR_BYTE_SHUFFLE 1
#else
@@ -110,6 +127,8 @@ inline V128 V128_Shuffle(V128 input, V128 shuffle_mask) {
}
inline V128 V128_DupChar(char c) { return vdupq_n_u8(c); }
#endif
#endif // SNAPPY_HAVE_VECTOR_BYTE_SHUFFLE
@@ -172,9 +191,10 @@ char* CompressFragment(const char* input,
// loading from s2 + n.
//
// Separate implementation for 64-bit, little-endian cpus.
// riscv and little-endian cpu choose this routinue can be done faster too.
#if !SNAPPY_IS_BIG_ENDIAN && \
(defined(__x86_64__) || defined(_M_X64) || defined(ARCH_PPC) || \
defined(ARCH_ARM))
defined(ARCH_ARM) || defined(__riscv))
static inline std::pair<size_t, bool> FindMatchLength(const char* s1,
const char* s2,
const char* s2_limit,

43
snappy.cc
View File

@@ -280,6 +280,8 @@ inline char* IncrementalCopySlow(const char* src, char* op,
// 4, 5, 0, 1, 2, 3, 4, 5, 0, 1}. These byte index sequences are generated by
// calling MakePatternMaskBytes(0, 6, index_sequence<16>()) and
// MakePatternMaskBytes(16, 6, index_sequence<16>()) respectively.
template <size_t... indexes>
inline constexpr std::array<char, sizeof...(indexes)> MakePatternMaskBytes(
int index_offset, int pattern_size, index_sequence<indexes...>) {
@@ -297,7 +299,6 @@ MakePatternMaskBytesTable(int index_offset,
MakePatternMaskBytes(index_offset, pattern_sizes_minus_one + 1,
make_index_sequence</*indexes=*/sizeof(V128)>())...};
}
// This is an array of shuffle control masks that can be used as the source
// operand for PSHUFB to permute the contents of the destination XMM register
// into a repeating byte pattern.
@@ -493,7 +494,6 @@ inline char* IncrementalCopy(const char* src, char* op, char* const op_limit,
LoadPatternAndReshuffleMask(src, pattern_size);
V128 pattern = pattern_and_reshuffle_mask.first;
V128 reshuffle_mask = pattern_and_reshuffle_mask.second;
// There is at least one, and at most four 16-byte blocks. Writing four
// conditionals instead of a loop allows FDO to layout the code with
// respect to the actual probabilities of each length.
@@ -520,7 +520,6 @@ inline char* IncrementalCopy(const char* src, char* op, char* const op_limit,
LoadPatternAndReshuffleMask(src, pattern_size);
V128 pattern = pattern_and_reshuffle_mask.first;
V128 reshuffle_mask = pattern_and_reshuffle_mask.second;
// This code path is relatively cold however so we save code size
// by avoiding unrolling and vectorizing.
//
@@ -959,6 +958,8 @@ emit_remainder:
char* CompressFragmentDoubleHash(const char* input, size_t input_size, char* op,
uint16_t* table, const int table_size,
uint16_t* table2, const int table_size2) {
(void)table_size2;
assert(table_size == table_size2);
// "ip" is the input pointer, and "op" is the output pointer.
const char* ip = input;
assert(input_size <= kBlockSize);
@@ -1224,7 +1225,7 @@ inline bool Copy64BytesWithPatternExtension(ptrdiff_t dst, size_t offset) {
void MemCopy64(char* dst, const void* src, size_t size) {
// Always copy this many bytes. If that's below size then copy the full 64.
constexpr int kShortMemCopy = 32;
(void)kShortMemCopy;
assert(size <= 64);
assert(std::less_equal<const void*>()(static_cast<const char*>(src) + size,
dst) ||
@@ -1243,6 +1244,27 @@ void MemCopy64(char* dst, const void* src, size_t size) {
data = _mm256_lddqu_si256(static_cast<const __m256i *>(src) + 1);
_mm256_storeu_si256(reinterpret_cast<__m256i *>(dst) + 1, data);
}
// RVV acceleration available on RISC-V when compiled with -march=rv64gcv
#elif defined(__riscv) && SNAPPY_HAVE_RVV
// Cast pointers to the type we will operate on.
unsigned char* dst_ptr = reinterpret_cast<unsigned char*>(dst);
const unsigned char* src_ptr = reinterpret_cast<const unsigned char*>(src);
size_t remaining_bytes = size;
// Loop as long as there are bytes remaining to be copied.
while (remaining_bytes > 0) {
// Set vector configuration: e8 (8-bit elements), m2 (LMUL=2).
// Use e8m2 configuration to maximize throughput.
size_t vl = VSETVL_E8M2(remaining_bytes);
// Load data from the current source pointer.
vuint8m2_t vec = VLE8_V_U8M2(src_ptr, vl);
// Store data to the current destination pointer.
VSE8_V_U8M2(dst_ptr, vec, vl);
// Update pointers and the remaining count.
src_ptr += vl;
dst_ptr += vl;
remaining_bytes -= vl;
}
#else
std::memmove(dst, src, kShortMemCopy);
// Profiling shows that nearly all copies are short.
@@ -1687,7 +1709,8 @@ constexpr uint32_t CalculateNeeded(uint8_t tag) {
#if __cplusplus >= 201402L
constexpr bool VerifyCalculateNeeded() {
for (int i = 0; i < 1; i++) {
if (CalculateNeeded(i) != (char_table[i] >> 11) + 1) return false;
if (CalculateNeeded(i) != static_cast<uint32_t>((char_table[i] >> 11)) + 1)
return false;
}
return true;
}
@@ -1775,6 +1798,7 @@ static bool InternalUncompressAllTags(SnappyDecompressor* decompressor,
Writer* writer, uint32_t compressed_len,
uint32_t uncompressed_len) {
int token = 0;
writer->SetExpectedLength(uncompressed_len);
// Process the entire input
@@ -1885,16 +1909,16 @@ class SnappyIOVecReader : public Source {
if (total_size > 0 && curr_size_remaining_ == 0) Advance();
}
~SnappyIOVecReader() = default;
~SnappyIOVecReader() override = default;
size_t Available() const { return total_size_remaining_; }
size_t Available() const override { return total_size_remaining_; }
const char* Peek(size_t* len) {
const char* Peek(size_t* len) override {
*len = curr_size_remaining_;
return curr_pos_;
}
void Skip(size_t n) {
void Skip(size_t n) override {
while (n >= curr_size_remaining_ && n > 0) {
n -= curr_size_remaining_;
Advance();
@@ -2532,7 +2556,6 @@ bool SnappyScatteredWriter<Allocator>::SlowAppendFromSelf(size_t offset,
class SnappySinkAllocator {
public:
explicit SnappySinkAllocator(Sink* dest) : dest_(dest) {}
~SnappySinkAllocator() {}
char* Allocate(int size) {
Datablock block(new char[size], size);