Add support for Uncompress(source, sink). Various changes to allow
Uncompress(source, sink) to get the same performance as the different variants of Uncompress to Cord/DataBuffer/String/FlatBuffer. Changes to efficiently support Uncompress(source, sink) -------- a) For strings - we add support to StringByteSink to do GetAppendBuffer so we can write to it without copying. b) For flat array buffers, we do GetAppendBuffer and see if we can get a full buffer. With the above changes we get performance with ByteSource/ByteSink that is very close to directly using flat arrays and strings. We add various benchmark cases to demonstrate that. Orthogonal change ------------------ Add support for TryFastAppend() for SnappyScatteredWriter. Benchmark results are below CPU: Intel Core2 dL1:32KB dL2:4096KB Benchmark Time(ns) CPU(ns) Iterations ----------------------------------------------------- BM_UFlat/0 109065 108996 6410 896.0MB/s html BM_UFlat/1 1012175 1012343 691 661.4MB/s urls BM_UFlat/2 26775 26771 26149 4.4GB/s jpg BM_UFlat/3 48947 48940 14363 1.8GB/s pdf BM_UFlat/4 441029 440835 1589 886.1MB/s html4 BM_UFlat/5 39861 39880 17823 588.3MB/s cp BM_UFlat/6 18315 18300 38126 581.1MB/s c BM_UFlat/7 5254 5254 100000 675.4MB/s lsp BM_UFlat/8 1568060 1567376 447 626.6MB/s xls BM_UFlat/9 337512 337734 2073 429.5MB/s txt1 BM_UFlat/10 287269 287054 2434 415.9MB/s txt2 BM_UFlat/11 890098 890219 787 457.2MB/s txt3 BM_UFlat/12 1186593 1186863 590 387.2MB/s txt4 BM_UFlat/13 573927 573318 1000 853.7MB/s bin BM_UFlat/14 64250 64294 10000 567.2MB/s sum BM_UFlat/15 7301 7300 96153 552.2MB/s man BM_UFlat/16 109617 109636 6375 1031.5MB/s pb BM_UFlat/17 364438 364497 1921 482.3MB/s gaviota BM_UFlatSink/0 108518 108465 6450 900.4MB/s html BM_UFlatSink/1 991952 991997 705 675.0MB/s urls BM_UFlatSink/2 26815 26798 26065 4.4GB/s jpg BM_UFlatSink/3 49127 49122 14255 1.8GB/s pdf BM_UFlatSink/4 436674 436731 1604 894.4MB/s html4 BM_UFlatSink/5 39738 39733 17345 590.5MB/s cp BM_UFlatSink/6 18413 18416 37962 577.4MB/s c BM_UFlatSink/7 5677 5676 100000 625.2MB/s lsp BM_UFlatSink/8 1552175 1551026 451 633.2MB/s xls BM_UFlatSink/9 338526 338489 2065 428.5MB/s txt1 BM_UFlatSink/10 289387 289307 2420 412.6MB/s txt2 BM_UFlatSink/11 893803 893706 783 455.4MB/s txt3 BM_UFlatSink/12 1195919 1195459 586 384.4MB/s txt4 BM_UFlatSink/13 559637 559779 1000 874.3MB/s bin BM_UFlatSink/14 65073 65094 10000 560.2MB/s sum BM_UFlatSink/15 7618 7614 92823 529.5MB/s man BM_UFlatSink/16 110085 110121 6352 1027.0MB/s pb BM_UFlatSink/17 369196 368915 1896 476.5MB/s gaviota BM_UValidate/0 46954 46957 14899 2.0GB/s html BM_UValidate/1 500621 500868 1000 1.3GB/s urls BM_UValidate/2 283 283 2481447 417.2GB/s jpg BM_UValidate/3 16230 16228 43137 5.4GB/s pdf BM_UValidate/4 189129 189193 3701 2.0GB/s html4 A=uday R=sanjay
This commit is contained in:
Steinar H. Gunderson
@@ -488,6 +488,23 @@ static int VerifyString(const string& input) {
|
||||
return uncompressed.size();
|
||||
}
|
||||
|
||||
static void VerifyStringSink(const string& input) {
|
||||
string compressed;
|
||||
DataEndingAtUnreadablePage i(input);
|
||||
const size_t written = snappy::Compress(i.data(), i.size(), &compressed);
|
||||
CHECK_EQ(written, compressed.size());
|
||||
CHECK_LE(compressed.size(),
|
||||
snappy::MaxCompressedLength(input.size()));
|
||||
CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
|
||||
|
||||
string uncompressed;
|
||||
uncompressed.resize(input.size());
|
||||
snappy::UncheckedByteArraySink sink(string_as_array(&uncompressed));
|
||||
DataEndingAtUnreadablePage c(compressed);
|
||||
snappy::ByteArraySource source(c.data(), c.size());
|
||||
CHECK(snappy::Uncompress(&source, &sink));
|
||||
CHECK_EQ(uncompressed, input);
|
||||
}
|
||||
|
||||
static void VerifyIOVec(const string& input) {
|
||||
string compressed;
|
||||
@@ -559,6 +576,28 @@ static void VerifyNonBlockedCompression(const string& input) {
|
||||
CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncomp_str));
|
||||
CHECK_EQ(uncomp_str, input);
|
||||
|
||||
// Uncompress using source/sink
|
||||
string uncomp_str2;
|
||||
uncomp_str2.resize(input.size());
|
||||
snappy::UncheckedByteArraySink sink(string_as_array(&uncomp_str2));
|
||||
snappy::ByteArraySource source(compressed.data(), compressed.size());
|
||||
CHECK(snappy::Uncompress(&source, &sink));
|
||||
CHECK_EQ(uncomp_str2, input);
|
||||
|
||||
// Uncompress into iovec
|
||||
{
|
||||
static const int kNumBlocks = 10;
|
||||
struct iovec vec[kNumBlocks];
|
||||
const int block_size = 1 + input.size() / kNumBlocks;
|
||||
string iovec_data(block_size * kNumBlocks, 'x');
|
||||
for (int i = 0; i < kNumBlocks; i++) {
|
||||
vec[i].iov_base = string_as_array(&iovec_data) + i * block_size;
|
||||
vec[i].iov_len = block_size;
|
||||
}
|
||||
CHECK(snappy::RawUncompressToIOVec(compressed.data(), compressed.size(),
|
||||
vec, kNumBlocks));
|
||||
CHECK_EQ(string(iovec_data.data(), input.size()), input);
|
||||
}
|
||||
}
|
||||
|
||||
// Expand the input so that it is at least K times as big as block size
|
||||
@@ -577,6 +616,8 @@ static int Verify(const string& input) {
|
||||
// Compress using string based routines
|
||||
const int result = VerifyString(input);
|
||||
|
||||
// Verify using sink based routines
|
||||
VerifyStringSink(input);
|
||||
|
||||
VerifyNonBlockedCompression(input);
|
||||
VerifyIOVec(input);
|
||||
@@ -1291,6 +1332,37 @@ static void BM_UIOVec(int iters, int arg) {
|
||||
}
|
||||
BENCHMARK(BM_UIOVec)->DenseRange(0, 4);
|
||||
|
||||
static void BM_UFlatSink(int iters, int arg) {
|
||||
StopBenchmarkTiming();
|
||||
|
||||
// Pick file to process based on "arg"
|
||||
CHECK_GE(arg, 0);
|
||||
CHECK_LT(arg, ARRAYSIZE(files));
|
||||
string contents = ReadTestDataFile(files[arg].filename,
|
||||
files[arg].size_limit);
|
||||
|
||||
string zcontents;
|
||||
snappy::Compress(contents.data(), contents.size(), &zcontents);
|
||||
char* dst = new char[contents.size()];
|
||||
|
||||
SetBenchmarkBytesProcessed(static_cast<int64>(iters) *
|
||||
static_cast<int64>(contents.size()));
|
||||
SetBenchmarkLabel(files[arg].label);
|
||||
StartBenchmarkTiming();
|
||||
while (iters-- > 0) {
|
||||
snappy::ByteArraySource source(zcontents.data(), zcontents.size());
|
||||
snappy::UncheckedByteArraySink sink(dst);
|
||||
CHECK(snappy::Uncompress(&source, &sink));
|
||||
}
|
||||
StopBenchmarkTiming();
|
||||
|
||||
string s(dst, contents.size());
|
||||
CHECK_EQ(contents, s);
|
||||
|
||||
delete[] dst;
|
||||
}
|
||||
|
||||
BENCHMARK(BM_UFlatSink)->DenseRange(0, ARRAYSIZE(files) - 1);
|
||||
|
||||
static void BM_ZFlat(int iters, int arg) {
|
||||
StopBenchmarkTiming();
|
||||
|
||||
Reference in New Issue
Block a user