fabi/Snappy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Supports truncated test data in zippy benchmark.

Browse Source 
R=sesse


git-svn-id: https://snappy.googlecode.com/svn/trunk@74 03e5f5b5-db94-4691-08a0-1a8bf15f6143
This commit is contained in:
snappy.mirrorbot@gmail.com
2013-04-09 15:33:30 +00:00
parent bde324c016
commit a3e928d62b
3 changed files with 61 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
snappy-test.cc
View File

@@ -42,21 +42,25 @@ DEFINE_bool(run_microbenchmarks, true,
namespace snappy { namespace snappy {
string ReadTestDataFile(const string& base) { string ReadTestDataFile(const string& base, size_t size_limit) {
string contents; string contents;
const char* srcdir = getenv("srcdir"); // This is set by Automake. const char* srcdir = getenv("srcdir"); // This is set by Automake.
string prefix;
if (srcdir) { if (srcdir) {
file::ReadFileToString(string(srcdir) + "/testdata/" + base, prefix = string(srcdir) + "/";
&contents, }
file::Defaults()).CheckSuccess(); file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults()
} else { ).CheckSuccess();
file::ReadFileToString("testdata/" + base, if (size_limit > 0) {
&contents, contents = contents.substr(0, size_limit);
file::Defaults()).CheckSuccess();
} }
return contents; return contents;
} }
string ReadTestDataFile(const string& base) {
return ReadTestDataFile(base, 0);
}
string StringPrintf(const char* format, ...) { string StringPrintf(const char* format, ...) {
char buf[4096]; char buf[4096];
va_list ap; va_list ap;

20
snappy-test.h
View File

@@ -139,10 +139,10 @@ namespace file {
void CheckSuccess() { } void CheckSuccess() { }
}; };
DummyStatus ReadFileToString(const char* filename, string* data, int unused) { DummyStatus GetContents(const string& filename, string* data, int unused) {
FILE* fp = fopen(filename, "rb"); FILE* fp = fopen(filename.c_str(), "rb");
if (fp == NULL) { if (fp == NULL) {
perror(filename); perror(filename.c_str());
exit(1); exit(1);
} }
@@ -160,15 +160,9 @@ namespace file {
fclose(fp); fclose(fp);
} }
DummyStatus ReadFileToString(const string& filename, DummyStatus SetContents(const string& filename,
string* data, const string& str,
int unused) { int unused) {
ReadFileToString(filename.c_str(), data, unused);
}
DummyStatus WriteStringToFile(const string& str,
const string& filename,
int unused) {
FILE* fp = fopen(filename.c_str(), "wb"); FILE* fp = fopen(filename.c_str(), "wb");
if (fp == NULL) { if (fp == NULL) {
perror(filename.c_str()); perror(filename.c_str());
@@ -203,6 +197,8 @@ void Test_Snappy_ReadPastEndOfBuffer();
void Test_Snappy_FindMatchLength(); void Test_Snappy_FindMatchLength();
void Test_Snappy_FindMatchLengthRandom(); void Test_Snappy_FindMatchLengthRandom();
string ReadTestDataFile(const string& base, size_t size_limit);
string ReadTestDataFile(const string& base); string ReadTestDataFile(const string& base);
// A sprintf() variant that returns a std::string. // A sprintf() variant that returns a std::string.

75
snappy_unittest.cc
View File

@@ -611,7 +611,8 @@ TYPED_TEST(CorruptedTest, VerifyCorrupted) {
// try reading stuff in from a bad file. // try reading stuff in from a bad file.
for (int i = 1; i <= 3; ++i) { for (int i = 1; i <= 3; ++i) {
string data = ReadTestDataFile(StringPrintf("baddata%d.snappy", i).c_str()); string data = ReadTestDataFile(StringPrintf("baddata%d.snappy", i).c_str(),
0);
string uncmp; string uncmp;
// check that we don't return a crazy length // check that we don't return a crazy length
size_t ulen; size_t ulen;
@@ -768,7 +769,8 @@ TEST(Snappy, FourByteOffset) {
string uncompressed; string uncompressed;
CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size())); CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncompressed)); CHECK(snappy::Uncompress(compressed.data(), compressed.size(),
&uncompressed));
CHECK_EQ(uncompressed, src); CHECK_EQ(uncompressed, src);
} }
@@ -971,19 +973,18 @@ TEST(Snappy, FindMatchLengthRandom) {
static void CompressFile(const char* fname) { static void CompressFile(const char* fname) {
string fullinput; string fullinput;
file::ReadFileToString(fname, &fullinput, file::Defaults()).CheckSuccess(); file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
string compressed; string compressed;
Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false); Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false);
file::WriteStringToFile( file::SetContents(string(fname).append(".comp"), compressed, file::Defaults())
string(fname).append(".comp").c_str(), compressed, .CheckSuccess();
file::Defaults()).CheckSuccess();
} }
static void UncompressFile(const char* fname) { static void UncompressFile(const char* fname) {
string fullinput; string fullinput;
file::ReadFileToString(fname, &fullinput, file::Defaults()).CheckSuccess(); file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
size_t uncompLength; size_t uncompLength;
CHECK(CheckUncompressedLength(fullinput, &uncompLength)); CHECK(CheckUncompressedLength(fullinput, &uncompLength));
@@ -992,14 +993,13 @@ static void UncompressFile(const char* fname) {
uncompressed.resize(uncompLength); uncompressed.resize(uncompLength);
CHECK(snappy::Uncompress(fullinput.data(), fullinput.size(), &uncompressed)); CHECK(snappy::Uncompress(fullinput.data(), fullinput.size(), &uncompressed));
file::WriteStringToFile( file::SetContents(string(fname).append(".uncomp"), uncompressed,
string(fname).append(".uncomp").c_str(), uncompressed, file::Defaults()).CheckSuccess();
file::Defaults()).CheckSuccess();
} }
static void MeasureFile(const char* fname) { static void MeasureFile(const char* fname) {
string fullinput; string fullinput;
file::ReadFileToString(fname, &fullinput, file::Defaults()).CheckSuccess(); file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
printf("%-40s :\n", fname); printf("%-40s :\n", fname);
int start_len = (FLAGS_start_len < 0) ? fullinput.size() : FLAGS_start_len; int start_len = (FLAGS_start_len < 0) ? fullinput.size() : FLAGS_start_len;
@@ -1032,25 +1032,29 @@ static void MeasureFile(const char* fname) {
static struct { static struct {
const char* label; const char* label;
const char* filename; const char* filename;
size_t size_limit;
} files[] = { } files[] = {
{ "html", "html" }, { "html", "html", 0 },
{ "urls", "urls.10K" }, { "urls", "urls.10K", 0 },
{ "jpg", "house.jpg" }, { "jpg", "house.jpg", 0 },
{ "pdf", "mapreduce-osdi-1.pdf" }, { "jpg_200", "house.jpg", 200 },
{ "html4", "html_x_4" }, { "pdf", "mapreduce-osdi-1.pdf", 0 },
{ "cp", "cp.html" }, { "html4", "html_x_4", 0 },
{ "c", "fields.c" }, { "cp", "cp.html", 0 },
{ "lsp", "grammar.lsp" }, { "c", "fields.c", 0 },
{ "xls", "kennedy.xls" }, { "lsp", "grammar.lsp", 0 },
{ "txt1", "alice29.txt" }, { "xls", "kennedy.xls", 0 },
{ "txt2", "asyoulik.txt" }, { "xls_200", "kennedy.xls", 200 },
{ "txt3", "lcet10.txt" }, { "txt1", "alice29.txt", 0 },
{ "txt4", "plrabn12.txt" }, { "txt2", "asyoulik.txt", 0 },
{ "bin", "ptt5" }, { "txt3", "lcet10.txt", 0 },
{ "sum", "sum" }, { "txt4", "plrabn12.txt", 0 },
{ "man", "xargs.1" }, { "bin", "ptt5", 0 },
{ "pb", "geo.protodata" }, { "bin_200", "ptt5", 200 },
{ "gaviota", "kppkn.gtb" }, { "sum", "sum", 0 },
{ "man", "xargs.1", 0 },
{ "pb", "geo.protodata", 0 },
{ "gaviota", "kppkn.gtb", 0 },
}; };
static void BM_UFlat(int iters, int arg) { static void BM_UFlat(int iters, int arg) {
@@ -1059,7 +1063,8 @@ static void BM_UFlat(int iters, int arg) {
// Pick file to process based on "arg" // Pick file to process based on "arg"
CHECK_GE(arg, 0); CHECK_GE(arg, 0);
CHECK_LT(arg, ARRAYSIZE(files)); CHECK_LT(arg, ARRAYSIZE(files));
string contents = ReadTestDataFile(files[arg].filename); string contents = ReadTestDataFile(files[arg].filename,
files[arg].size_limit);
string zcontents; string zcontents;
snappy::Compress(contents.data(), contents.size(), &zcontents); snappy::Compress(contents.data(), contents.size(), &zcontents);
@@ -1076,7 +1081,7 @@ static void BM_UFlat(int iters, int arg) {
delete[] dst; delete[] dst;
} }
BENCHMARK(BM_UFlat)->DenseRange(0, 17); BENCHMARK(BM_UFlat)->DenseRange(0, ARRAYSIZE(files) - 1);
static void BM_UValidate(int iters, int arg) { static void BM_UValidate(int iters, int arg) {
StopBenchmarkTiming(); StopBenchmarkTiming();
@@ -1084,7 +1089,8 @@ static void BM_UValidate(int iters, int arg) {
// Pick file to process based on "arg" // Pick file to process based on "arg"
CHECK_GE(arg, 0); CHECK_GE(arg, 0);
CHECK_LT(arg, ARRAYSIZE(files)); CHECK_LT(arg, ARRAYSIZE(files));
string contents = ReadTestDataFile(files[arg].filename); string contents = ReadTestDataFile(files[arg].filename,
files[arg].size_limit);
string zcontents; string zcontents;
snappy::Compress(contents.data(), contents.size(), &zcontents); snappy::Compress(contents.data(), contents.size(), &zcontents);
@@ -1107,7 +1113,8 @@ static void BM_ZFlat(int iters, int arg) {
// Pick file to process based on "arg" // Pick file to process based on "arg"
CHECK_GE(arg, 0); CHECK_GE(arg, 0);
CHECK_LT(arg, ARRAYSIZE(files)); CHECK_LT(arg, ARRAYSIZE(files));
string contents = ReadTestDataFile(files[arg].filename); string contents = ReadTestDataFile(files[arg].filename,
files[arg].size_limit);
char* dst = new char[snappy::MaxCompressedLength(contents.size())]; char* dst = new char[snappy::MaxCompressedLength(contents.size())];
@@ -1128,7 +1135,7 @@ static void BM_ZFlat(int iters, int arg) {
files[arg].label, contents.size(), zsize); files[arg].label, contents.size(), zsize);
delete[] dst; delete[] dst;
} }
BENCHMARK(BM_ZFlat)->DenseRange(0, 17); BENCHMARK(BM_ZFlat)->DenseRange(0, ARRAYSIZE(files) - 1);
} // namespace snappy } // namespace snappy