Skip to content

Commit

Permalink
Increase the stress test coverage of GetEntity (facebook#11303)
Browse files Browse the repository at this point in the history 
Summary:
The `GetEntity` API is currently used in the stress tests for verification purposes;
this patch extends the coverage by adding a mode where all point lookups in
the non-batched, batched, and CF consistency stress tests are done using this API.
The PR also includes a bit of refactoring to eliminate some boilerplate code around
the wide-column consistency checks.

Pull Request resolved: facebook#11303

Test Plan: Ran stress tests of the batched, non-batched, and CF consistency varieties.

Reviewed By: akankshamahajan15

Differential Revision: D44148503

Pulled By: ltamasi

fbshipit-source-id: fecdbfd3e65a459bbf16ab7aa7b9173e19240077
  • Loading branch information
@ltamasi @facebook-github-bot
ltamasi authored and facebook-github-bot committed Mar 17, 2023
1 parent 291300e commit a72d55c
Show file tree
Hide file tree
Showing 12 changed files with 473 additions and 102 deletions.
116 changes: 109 additions & 7 deletions db_stress_tool/batched_ops_stress.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,110 @@ class BatchedOpsStressTest : public StressTest {
return ret_status;
}

void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts,
const std::vector<int>& rand_column_families,
const std::vector<int64_t>& rand_keys) override {
assert(thread);

ManagedSnapshot snapshot_guard(db_);

ReadOptions read_opts_copy(read_opts);
read_opts_copy.snapshot = snapshot_guard.snapshot();

assert(!rand_keys.empty());

const std::string key_suffix = Key(rand_keys[0]);

assert(!rand_column_families.empty());
assert(rand_column_families[0] >= 0);
assert(rand_column_families[0] < static_cast<int>(column_families_.size()));

ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]];
assert(cfh);

constexpr size_t num_keys = 10;

std::array<PinnableWideColumns, num_keys> results;

for (size_t i = 0; i < num_keys; ++i) {
const std::string key = std::to_string(i) + key_suffix;

const Status s = db_->GetEntity(read_opts_copy, cfh, key, &results[i]);

if (!s.ok() && !s.IsNotFound()) {
fprintf(stderr, "GetEntity error: %s\n", s.ToString().c_str());
thread->stats.AddErrors(1);
} else if (s.IsNotFound()) {
thread->stats.AddGets(1, 0);
} else {
thread->stats.AddGets(1, 1);
}
}

// Compare columns ignoring the last character of column values
auto compare = [](const WideColumns& lhs, const WideColumns& rhs) {
if (lhs.size() != rhs.size()) {
return false;
}

for (size_t i = 0; i < lhs.size(); ++i) {
if (lhs[i].name() != rhs[i].name()) {
return false;
}

if (lhs[i].value().size() != rhs[i].value().size()) {
return false;
}

if (lhs[i].value().difference_offset(rhs[i].value()) <
lhs[i].value().size() - 1) {
return false;
}
}

return true;
};

for (size_t i = 0; i < num_keys; ++i) {
const WideColumns& columns = results[i].columns();

if (!compare(results[0].columns(), columns)) {
fprintf(stderr,
"GetEntity error: inconsistent entities for key %s: %s, %s\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(results[0].columns()).c_str(),
WideColumnsToHex(columns).c_str());
}

if (!columns.empty()) {
// The last character of each column value should be 'i' as a decimal
// digit
const char expected = static_cast<char>('0' + i);

for (const auto& column : columns) {
const Slice& value = column.value();

if (value.empty() || value[value.size() - 1] != expected) {
fprintf(stderr,
"GetEntity error: incorrect column value for key "
"%s, entity %s, column value %s, expected %c\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(columns).c_str(),
value.ToString(/* hex */ true).c_str(), expected);
}
}

if (!VerifyWideColumns(columns)) {
fprintf(
stderr,
"GetEntity error: inconsistent columns for key %s, entity %s\n",
StringToHex(key_suffix).c_str(),
WideColumnsToHex(columns).c_str());
}
}
}
}

// Given a key, this does prefix scans for "0"+P, "1"+P, ..., "9"+P
// in the same snapshot where P is the first FLAGS_prefix_size - 1 bytes
// of the key. Each of these 10 scans returns a series of values;
Expand Down Expand Up @@ -357,16 +461,14 @@ class BatchedOpsStressTest : public StressTest {
}

// make sure value() and columns() are consistent
const WideColumns expected_columns = GenerateExpectedWideColumns(
GetValueBase(iters[i]->value()), iters[i]->value());
if (iters[i]->columns() != expected_columns) {
if (!VerifyWideColumns(iters[i]->value(), iters[i]->columns())) {
fprintf(stderr,
"prefix scan error : %" ROCKSDB_PRIszt
", value and columns inconsistent for prefix %s: %s\n",
", value and columns inconsistent for prefix %s: value: %s, "
"columns: %s\n",
i, prefix_slices[i].ToString(/* hex */ true).c_str(),
DebugString(iters[i]->value(), iters[i]->columns(),
expected_columns)
.c_str());
iters[i]->value().ToString(/* hex */ true).c_str(),
WideColumnsToHex(iters[i]->columns()).c_str());
}

iters[i]->Next();
Expand Down
159 changes: 147 additions & 12 deletions db_stress_tool/cf_consistency_stress.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,146 @@ class CfConsistencyStressTest : public StressTest {
return statuses;
}

void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts,
const std::vector<int>& rand_column_families,
const std::vector<int64_t>& rand_keys) override {
assert(thread);
assert(!rand_column_families.empty());
assert(!rand_keys.empty());

const std::string key = Key(rand_keys[0]);

Status s;
bool is_consistent = true;

if (thread->rand.OneIn(2)) {
// With a 1/2 chance, do a random read from a random CF
const size_t cf_id = thread->rand.Next() % rand_column_families.size();

assert(rand_column_families[cf_id] >= 0);
assert(rand_column_families[cf_id] <
static_cast<int>(column_families_.size()));

ColumnFamilyHandle* const cfh =
column_families_[rand_column_families[cf_id]];
assert(cfh);

PinnableWideColumns result;
s = db_->GetEntity(read_opts, cfh, key, &result);

if (s.ok()) {
if (!VerifyWideColumns(result.columns())) {
fprintf(
stderr,
"GetEntity error: inconsistent columns for key %s, entity %s\n",
StringToHex(key).c_str(),
WideColumnsToHex(result.columns()).c_str());
is_consistent = false;
}
}
} else {
// With a 1/2 chance, compare one key across all CFs
ManagedSnapshot snapshot_guard(db_);

ReadOptions read_opts_copy = read_opts;
read_opts_copy.snapshot = snapshot_guard.snapshot();

assert(rand_column_families[0] >= 0);
assert(rand_column_families[0] <
static_cast<int>(column_families_.size()));

PinnableWideColumns cmp_result;
s = db_->GetEntity(read_opts_copy,
column_families_[rand_column_families[0]], key,
&cmp_result);

if (s.ok() || s.IsNotFound()) {
const bool cmp_found = s.ok();

if (cmp_found) {
if (!VerifyWideColumns(cmp_result.columns())) {
fprintf(stderr,
"GetEntity error: inconsistent columns for key %s, "
"entity %s\n",
StringToHex(key).c_str(),
WideColumnsToHex(cmp_result.columns()).c_str());
is_consistent = false;
}
}

if (is_consistent) {
for (size_t i = 1; i < rand_column_families.size(); ++i) {
assert(rand_column_families[i] >= 0);
assert(rand_column_families[i] <
static_cast<int>(column_families_.size()));

PinnableWideColumns result;
s = db_->GetEntity(read_opts_copy,
column_families_[rand_column_families[i]], key,
&result);

if (!s.ok() && !s.IsNotFound()) {
break;
}

const bool found = s.ok();

assert(!column_family_names_.empty());
assert(i < column_family_names_.size());

if (!cmp_found && found) {
fprintf(stderr,
"GetEntity returns different results for key %s: CF %s "
"returns not found, CF %s returns entity %s\n",
StringToHex(key).c_str(), column_family_names_[0].c_str(),
column_family_names_[i].c_str(),
WideColumnsToHex(result.columns()).c_str());
is_consistent = false;
break;
}

if (cmp_found && !found) {
fprintf(stderr,
"GetEntity returns different results for key %s: CF %s "
"returns entity %s, CF %s returns not found\n",
StringToHex(key).c_str(), column_family_names_[0].c_str(),
WideColumnsToHex(cmp_result.columns()).c_str(),
column_family_names_[i].c_str());
is_consistent = false;
break;
}

if (found && result != cmp_result) {
fprintf(stderr,
"GetEntity returns different results for key %s: CF %s "
"returns entity %s, CF %s returns entity %s\n",
StringToHex(key).c_str(), column_family_names_[0].c_str(),
WideColumnsToHex(cmp_result.columns()).c_str(),
column_family_names_[i].c_str(),
WideColumnsToHex(result.columns()).c_str());
is_consistent = false;
break;
}
}
}
}
}

if (!is_consistent) {
fprintf(stderr, "TestGetEntity error: results are not consistent\n");
thread->stats.AddErrors(1);
// Fail fast to preserve the DB state.
thread->shared->SetVerificationFailure();
} else if (s.ok()) {
thread->stats.AddGets(1, 1);
} else if (s.IsNotFound()) {
thread->stats.AddGets(1, 0);
} else {
fprintf(stderr, "TestGetEntity error: %s\n", s.ToString().c_str());
thread->stats.AddErrors(1);
}
}

Status TestPrefixScan(ThreadState* thread, const ReadOptions& readoptions,
const std::vector<int>& rand_column_families,
const std::vector<int64_t>& rand_keys) override {
Expand Down Expand Up @@ -290,12 +430,9 @@ class CfConsistencyStressTest : public StressTest {
iter->Next()) {
++count;

const WideColumns expected_columns = GenerateExpectedWideColumns(
GetValueBase(iter->value()), iter->value());
if (iter->columns() != expected_columns) {
s = Status::Corruption(
"Value and columns inconsistent",
DebugString(iter->value(), iter->columns(), expected_columns));
if (!VerifyWideColumns(iter->value(), iter->columns())) {
s = Status::Corruption("Value and columns inconsistent",
DebugString(iter->value(), iter->columns()));
break;
}
}
Expand Down Expand Up @@ -372,12 +509,10 @@ class CfConsistencyStressTest : public StressTest {
assert(iter);

if (iter->Valid()) {
const WideColumns expected_columns = GenerateExpectedWideColumns(
GetValueBase(iter->value()), iter->value());
if (iter->columns() != expected_columns) {
statuses[i] = Status::Corruption(
"Value and columns inconsistent",
DebugString(iter->value(), iter->columns(), expected_columns));
if (!VerifyWideColumns(iter->value(), iter->columns())) {
statuses[i] =
Status::Corruption("Value and columns inconsistent",
DebugString(iter->value(), iter->columns()));
} else {
++valid_cnt;
}
Expand Down
31 changes: 31 additions & 0 deletions db_stress_tool/db_stress_common.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,37 @@ WideColumns GenerateExpectedWideColumns(uint32_t value_base,
return columns;
}

bool VerifyWideColumns(const Slice& value, const WideColumns& columns) {
if (value.size() < sizeof(uint32_t)) {
return false;
}

const uint32_t value_base = GetValueBase(value);

const WideColumns expected_columns =
GenerateExpectedWideColumns(value_base, value);

if (columns != expected_columns) {
return false;
}

return true;
}

bool VerifyWideColumns(const WideColumns& columns) {
if (columns.empty()) {
return false;
}

if (columns.front().name() != kDefaultWideColumnName) {
return false;
}

const Slice& value_of_default = columns.front().value();

return VerifyWideColumns(value_of_default, columns);
}

std::string GetNowNanos() {
uint64_t t = db_stress_env->NowNanos();
std::string ret;
Expand Down
21 changes: 21 additions & 0 deletions db_stress_tool/db_stress_common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ DECLARE_bool(compare_full_db_state_snapshot);
DECLARE_uint64(snapshot_hold_ops);
DECLARE_bool(long_running_snapshots);
DECLARE_bool(use_multiget);
DECLARE_bool(use_get_entity);
DECLARE_int32(readpercent);
DECLARE_int32(prefixpercent);
DECLARE_int32(writepercent);
Expand Down Expand Up @@ -596,6 +597,24 @@ extern inline std::string StringToHex(const std::string& str) {
return result;
}

inline std::string WideColumnsToHex(const WideColumns& columns) {
if (columns.empty()) {
return std::string();
}

std::ostringstream oss;

oss << std::hex;

auto it = columns.begin();
oss << *it;
for (++it; it != columns.end(); ++it) {
oss << ' ' << *it;
}

return oss.str();
}

// Unified output format for double parameters
extern inline std::string FormatDoubleParam(double param) {
return std::to_string(param);
Expand Down Expand Up @@ -626,6 +645,8 @@ extern uint32_t GetValueBase(Slice s);
extern WideColumns GenerateWideColumns(uint32_t value_base, const Slice& slice);
extern WideColumns GenerateExpectedWideColumns(uint32_t value_base,
const Slice& slice);
extern bool VerifyWideColumns(const Slice& value, const WideColumns& columns);
extern bool VerifyWideColumns(const WideColumns& columns);

extern StressTest* CreateCfConsistencyStressTest();
extern StressTest* CreateBatchedOpsStressTest();
Expand Down
Loading

0 comments on commit a72d55c

Please sign in to comment.