Use std random number generators in tests.
An earlier CL introduced absl::Uniform, which is not yet open sourced, and therefore unavailable in the open source build. This CL removes absl::Uniform and ACMRandom in favor of equivalent C++11 standard random generators. Abseil promises to be faster than the standard library, but we can afford a speed hit in tests in return for an easier open sourcing story.
This commit is contained in:
@@ -189,56 +189,6 @@ string ReadTestDataFile(const string& base);
|
|||||||
// Not safe for general use due to truncation issues.
|
// Not safe for general use due to truncation issues.
|
||||||
string StringPrintf(const char* format, ...);
|
string StringPrintf(const char* format, ...);
|
||||||
|
|
||||||
// A simple, non-cryptographically-secure random generator.
|
|
||||||
class ACMRandom {
|
|
||||||
public:
|
|
||||||
explicit ACMRandom(uint32 seed) : seed_(seed) {}
|
|
||||||
|
|
||||||
int32 Next();
|
|
||||||
|
|
||||||
int32 Uniform(int32 n) {
|
|
||||||
return Next() % n;
|
|
||||||
}
|
|
||||||
uint8 Rand8() {
|
|
||||||
return static_cast<uint8>((Next() >> 1) & 0x000000ff);
|
|
||||||
}
|
|
||||||
bool OneIn(int X) { return Uniform(X) == 0; }
|
|
||||||
|
|
||||||
// Skewed: pick "base" uniformly from range [0,max_log] and then
|
|
||||||
// return "base" random bits. The effect is to pick a number in the
|
|
||||||
// range [0,2^max_log-1] with bias towards smaller numbers.
|
|
||||||
int32 Skewed(int max_log);
|
|
||||||
|
|
||||||
private:
|
|
||||||
static const uint32 M = 2147483647L; // 2^31-1
|
|
||||||
uint32 seed_;
|
|
||||||
};
|
|
||||||
|
|
||||||
inline int32 ACMRandom::Next() {
|
|
||||||
static const uint64 A = 16807; // bits 14, 8, 7, 5, 2, 1, 0
|
|
||||||
// We are computing
|
|
||||||
// seed_ = (seed_ * A) % M, where M = 2^31-1
|
|
||||||
//
|
|
||||||
// seed_ must not be zero or M, or else all subsequent computed values
|
|
||||||
// will be zero or M respectively. For all other values, seed_ will end
|
|
||||||
// up cycling through every number in [1,M-1]
|
|
||||||
uint64 product = seed_ * A;
|
|
||||||
|
|
||||||
// Compute (product % M) using the fact that ((x << 31) % M) == x.
|
|
||||||
seed_ = (product >> 31) + (product & M);
|
|
||||||
// The first reduction may overflow by 1 bit, so we may need to repeat.
|
|
||||||
// mod == M is not possible; using > allows the faster sign-bit-based test.
|
|
||||||
if (seed_ > M) {
|
|
||||||
seed_ -= M;
|
|
||||||
}
|
|
||||||
return seed_;
|
|
||||||
}
|
|
||||||
|
|
||||||
inline int32 ACMRandom::Skewed(int max_log) {
|
|
||||||
const int32 base = (Next() - 1) % (max_log+1);
|
|
||||||
return (Next() - 1) & ((1u << base)-1);
|
|
||||||
}
|
|
||||||
|
|
||||||
// A wall-time clock. This stub is not super-accurate, nor resistant to the
|
// A wall-time clock. This stub is not super-accurate, nor resistant to the
|
||||||
// system time changing.
|
// system time changing.
|
||||||
class CycleTimer {
|
class CycleTimer {
|
||||||
|
|||||||
@@ -31,6 +31,7 @@
|
|||||||
|
|
||||||
|
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
#include <random>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <utility>
|
#include <utility>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
@@ -405,24 +406,28 @@ static void VerifyIOVec(const string& input) {
|
|||||||
// Try uncompressing into an iovec containing a random number of entries
|
// Try uncompressing into an iovec containing a random number of entries
|
||||||
// ranging from 1 to 10.
|
// ranging from 1 to 10.
|
||||||
char* buf = new char[input.size()];
|
char* buf = new char[input.size()];
|
||||||
ACMRandom rnd(input.size());
|
std::minstd_rand0 rng(input.size());
|
||||||
size_t num = absl::Uniform<uint32_t>(rnd) % 10 + 1;
|
std::uniform_int_distribution<size_t> uniform_1_to_10(1, 10);
|
||||||
|
size_t num = uniform_1_to_10(rng);
|
||||||
if (input.size() < num) {
|
if (input.size() < num) {
|
||||||
num = input.size();
|
num = input.size();
|
||||||
}
|
}
|
||||||
struct iovec* iov = new iovec[num];
|
struct iovec* iov = new iovec[num];
|
||||||
int used_so_far = 0;
|
int used_so_far = 0;
|
||||||
|
std::bernoulli_distribution one_in_five(1.0 / 5);
|
||||||
for (size_t i = 0; i < num; ++i) {
|
for (size_t i = 0; i < num; ++i) {
|
||||||
|
assert(used_so_far < input.size());
|
||||||
iov[i].iov_base = buf + used_so_far;
|
iov[i].iov_base = buf + used_so_far;
|
||||||
if (i == num - 1) {
|
if (i == num - 1) {
|
||||||
iov[i].iov_len = input.size() - used_so_far;
|
iov[i].iov_len = input.size() - used_so_far;
|
||||||
} else {
|
} else {
|
||||||
// Randomly choose to insert a 0 byte entry.
|
// Randomly choose to insert a 0 byte entry.
|
||||||
if (rnd.OneIn(5)) {
|
if (one_in_five(rng)) {
|
||||||
iov[i].iov_len = 0;
|
iov[i].iov_len = 0;
|
||||||
} else {
|
} else {
|
||||||
iov[i].iov_len =
|
std::uniform_int_distribution<size_t> uniform_not_used_so_far(
|
||||||
absl::Uniform<uint32_t>(rnd, 0, input.size() - used_so_far);
|
0, input.size() - used_so_far - 1);
|
||||||
|
iov[i].iov_len = uniform_not_used_so_far(rng);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
used_so_far += iov[i].iov_len;
|
used_so_far += iov[i].iov_len;
|
||||||
@@ -665,42 +670,58 @@ TEST(Snappy, SimpleTests) {
|
|||||||
|
|
||||||
// Verify max blowup (lots of four-byte copies)
|
// Verify max blowup (lots of four-byte copies)
|
||||||
TEST(Snappy, MaxBlowup) {
|
TEST(Snappy, MaxBlowup) {
|
||||||
|
std::mt19937 rng;
|
||||||
|
std::uniform_int_distribution<uint8_t> random_byte;
|
||||||
string input;
|
string input;
|
||||||
for (int i = 0; i < 20000; i++) {
|
for (int i = 0; i < 80000; ++i)
|
||||||
ACMRandom rnd(i);
|
input.push_back(static_cast<char>(random_byte(rng)));
|
||||||
uint32 bytes = static_cast<uint32>(absl::Uniform<uint32_t>(rnd));
|
|
||||||
input.append(reinterpret_cast<char*>(&bytes), sizeof(bytes));
|
for (int i = 0; i < 80000; i += 4) {
|
||||||
}
|
string four_bytes(input.end() - i - 4, input.end() - i);
|
||||||
for (int i = 19999; i >= 0; i--) {
|
input.append(four_bytes);
|
||||||
ACMRandom rnd(i);
|
|
||||||
uint32 bytes = static_cast<uint32>(absl::Uniform<uint32_t>(rnd));
|
|
||||||
input.append(reinterpret_cast<char*>(&bytes), sizeof(bytes));
|
|
||||||
}
|
}
|
||||||
Verify(input);
|
Verify(input);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(Snappy, RandomData) {
|
TEST(Snappy, RandomData) {
|
||||||
ACMRandom rnd(FLAGS_test_random_seed);
|
std::minstd_rand0 rng(FLAGS_test_random_seed);
|
||||||
|
std::uniform_int_distribution<int> uniform_0_to_3(0, 3);
|
||||||
|
std::uniform_int_distribution<int> uniform_0_to_8(0, 8);
|
||||||
|
std::uniform_int_distribution<uint8_t> uniform_byte;
|
||||||
|
std::uniform_int_distribution<size_t> uniform_4k(0, 4095);
|
||||||
|
std::uniform_int_distribution<size_t> uniform_64k(0, 65535);
|
||||||
|
std::bernoulli_distribution one_in_ten(1.0 / 10);
|
||||||
|
|
||||||
const int num_ops = 20000;
|
constexpr int num_ops = 20000;
|
||||||
for (int i = 0; i < num_ops; i++) {
|
for (int i = 0; i < num_ops; i++) {
|
||||||
if ((i % 1000) == 0) {
|
if ((i % 1000) == 0) {
|
||||||
VLOG(0) << "Random op " << i << " of " << num_ops;
|
VLOG(0) << "Random op " << i << " of " << num_ops;
|
||||||
}
|
}
|
||||||
|
|
||||||
string x;
|
string x;
|
||||||
size_t len = absl::Uniform<uint32_t>(rnd, 0, 4096);
|
size_t len = uniform_4k(rng);
|
||||||
if (i < 100) {
|
if (i < 100) {
|
||||||
len = 65536 + absl::Uniform<uint32_t>(rnd, 0, 65536);
|
len = 65536 + uniform_64k(rng);
|
||||||
}
|
}
|
||||||
while (x.size() < len) {
|
while (x.size() < len) {
|
||||||
int run_len = 1;
|
int run_len = 1;
|
||||||
if (rnd.OneIn(10)) {
|
if (one_in_ten(rng)) {
|
||||||
run_len = rnd.Skewed(8);
|
int skewed_bits = uniform_0_to_8(rng);
|
||||||
|
// int is guaranteed to hold at least 16 bits, this uses at most 8 bits.
|
||||||
|
std::uniform_int_distribution<int> skewed_low(0,
|
||||||
|
(1 << skewed_bits) - 1);
|
||||||
|
run_len = skewed_low(rng);
|
||||||
|
}
|
||||||
|
char c = static_cast<char>(uniform_byte(rng));
|
||||||
|
if (i >= 100) {
|
||||||
|
int skewed_bits = uniform_0_to_3(rng);
|
||||||
|
// int is guaranteed to hold at least 16 bits, this uses at most 3 bits.
|
||||||
|
std::uniform_int_distribution<int> skewed_low(0,
|
||||||
|
(1 << skewed_bits) - 1);
|
||||||
|
c = static_cast<char>(skewed_low(rng));
|
||||||
}
|
}
|
||||||
char c = (i < 100) ? absl::Uniform<uint32_t>(rnd, 0, 256) : rnd.Skewed(3);
|
|
||||||
while (run_len-- > 0 && x.size() < len) {
|
while (run_len-- > 0 && x.size() < len) {
|
||||||
x += c;
|
x.push_back(c);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1038,17 +1059,20 @@ TEST(Snappy, FindMatchLength) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
TEST(Snappy, FindMatchLengthRandom) {
|
TEST(Snappy, FindMatchLengthRandom) {
|
||||||
const int kNumTrials = 10000;
|
constexpr int kNumTrials = 10000;
|
||||||
const int kTypicalLength = 10;
|
constexpr int kTypicalLength = 10;
|
||||||
ACMRandom rnd(FLAGS_test_random_seed);
|
std::minstd_rand0 rng(FLAGS_test_random_seed);
|
||||||
|
std::uniform_int_distribution<uint8_t> uniform_byte;
|
||||||
|
std::bernoulli_distribution one_in_two(1.0 / 2);
|
||||||
|
std::bernoulli_distribution one_in_typical_length(1.0 / kTypicalLength);
|
||||||
|
|
||||||
for (int i = 0; i < kNumTrials; i++) {
|
for (int i = 0; i < kNumTrials; i++) {
|
||||||
string s, t;
|
string s, t;
|
||||||
char a = absl::Uniform<uint8_t>(rnd);
|
char a = uniform_byte(rng);
|
||||||
char b = absl::Uniform<uint8_t>(rnd);
|
char b = uniform_byte(rng);
|
||||||
while (!rnd.OneIn(kTypicalLength)) {
|
while (!one_in_typical_length(rng)) {
|
||||||
s.push_back(rnd.OneIn(2) ? a : b);
|
s.push_back(one_in_two(rng) ? a : b);
|
||||||
t.push_back(rnd.OneIn(2) ? a : b);
|
t.push_back(one_in_two(rng) ? a : b);
|
||||||
}
|
}
|
||||||
DataEndingAtUnreadablePage u(s);
|
DataEndingAtUnreadablePage u(s);
|
||||||
DataEndingAtUnreadablePage v(t);
|
DataEndingAtUnreadablePage v(t);
|
||||||
|
|||||||
Reference in New Issue
Block a user