Skip to content

Commit

Permalink
KUDU-1993: fixed validation of 'grouped' gflags
Browse files Browse the repository at this point in the history
Added generic implementation for grouped gflags validators.

Use CUSTOM_FLAG_VALIDATOR() macro to register late-phase validator
function for command-line flags. The validation is performed upon call
of HandleCommonFlags() invoked by the top-level ParseCommandLineFlags()
function.

Added unit test to cover the new functionality.

Updated validators for security-related RPC and embedded webserver
flags.

As a workaround for LSAN reports on the leaks in case of ASAN build,
added two scoped leak check disablers into CreateAndStartTimeoutThread.
That does not seem harmful or hiding any potential leaks since it
affects only the timeout thread itself.  Opened separate JIRA to track
the (false positive?) leak warning: KUDU-1995.

Change-Id: I3755d62590cdc63a9d501ba69d980cb15f8069a9
Reviewed-on: http://gerrit.cloudera.org:8080/6795
Reviewed-by: Adar Dembo <[email protected]>
Tested-by: Kudu Jenkins
  • Loading branch information
alexeyserbin committed May 5, 2017
1 parent 7f72105 commit e7334c2
Show file tree
Hide file tree
Showing 8 changed files with 414 additions and 35 deletions.
56 changes: 31 additions & 25 deletions src/kudu/rpc/messenger.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,15 @@
#include <sys/types.h>
#include <unistd.h>

#include <boost/algorithm/string/predicate.hpp>
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <list>
#include <mutex>
#include <set>
#include <string>

#include <boost/algorithm/string/predicate.hpp>
#include <gflags/gflags.h>
#include <glog/logging.h>

#include "kudu/gutil/gscoped_ptr.h"
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/stl_util.h"
Expand All @@ -48,6 +49,7 @@
#include "kudu/security/token_verifier.h"
#include "kudu/util/errno.h"
#include "kudu/util/flag_tags.h"
#include "kudu/util/flag_validators.h"
#include "kudu/util/metrics.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/socket.h"
Expand Down Expand Up @@ -128,56 +130,60 @@ static Status ParseTriState(const char* flag_name, const string& flag_value, T*
return Status::OK();
}

static bool ValidateRpcAuthentication(const char* /*flag_name*/, const string& /*flag_value*/) {
RpcAuthentication authentication;
Status s = ParseTriState("--rpc_authentication", FLAGS_rpc_authentication, &authentication);
static bool ValidateRpcAuthentication(const char* flag_name, const string& flag_value) {
RpcAuthentication result;
Status s = ParseTriState(flag_name, flag_value, &result);
if (!s.ok()) {
LOG(ERROR) << s.message().ToString();
return false;
}
return true;
}
DEFINE_validator(rpc_authentication, &ValidateRpcAuthentication);

RpcEncryption encryption;
s = ParseTriState("--rpc_encryption", FLAGS_rpc_encryption, &encryption);
static bool ValidateRpcEncryption(const char* flag_name, const string& flag_value) {
RpcEncryption result;
Status s = ParseTriState(flag_name, flag_value, &result);
if (!s.ok()) {
// This will be caught by the rpc_encryption validator.
return true;
}

if (encryption == RpcEncryption::DISABLED && authentication != RpcAuthentication::DISABLED) {
LOG(ERROR) << "RPC authentication (--rpc_authentication) must be disabled "
"if RPC encryption (--rpc_encryption) is disabled";
LOG(ERROR) << s.message().ToString();
return false;
}

return true;
}
DEFINE_validator(rpc_authentication, &ValidateRpcAuthentication);
DEFINE_validator(rpc_encryption, &ValidateRpcEncryption);

static bool ValidateRpcAuthnFlags() {
RpcAuthentication authentication;
CHECK_OK(ParseTriState("--rpc_authentication", FLAGS_rpc_authentication, &authentication));

static bool ValidateRpcEncryption(const char* /*flag_name*/, const string& /*flag_value*/) {
RpcEncryption encryption;
Status s = ParseTriState("--rpc_encryption", FLAGS_rpc_encryption, &encryption);
if (!s.ok()) {
LOG(ERROR) << s.message().ToString();
CHECK_OK(ParseTriState("--rpc_encryption", FLAGS_rpc_encryption, &encryption));

if (encryption == RpcEncryption::DISABLED && authentication != RpcAuthentication::DISABLED) {
LOG(ERROR) << "RPC authentication (--rpc_authentication) must be disabled "
"if RPC encryption (--rpc_encryption) is disabled";
return false;
}

return true;
}
DEFINE_validator(rpc_encryption, &ValidateRpcEncryption);
GROUP_FLAG_VALIDATOR(rpc_authn_flags, ValidateRpcAuthnFlags);

static bool ValidatePkiFlags(const char* /*flag_name*/, const string& /*flag_value*/) {
static bool ValidatePkiFlags() {
bool has_cert = !FLAGS_rpc_certificate_file.empty();
bool has_key = !FLAGS_rpc_private_key_file.empty();
bool has_ca = !FLAGS_rpc_ca_certificate_file.empty();

if (has_cert != has_key || has_cert != has_ca) {
LOG(ERROR) << "--rpc_certificate_file, --rpc_private_key_file, and "
"--rpc_ca_certificate_file flags must be set as a group";
"--rpc_ca_certificate_file flags must be set as a group; "
"i.e. either set all or none of them.";
return false;
}

return true;
}
DEFINE_validator(rpc_certificate_file, &ValidatePkiFlags);
GROUP_FLAG_VALIDATOR(pki_flags, ValidatePkiFlags);

MessengerBuilder::MessengerBuilder(std::string name)
: name_(std::move(name)),
Expand Down
19 changes: 11 additions & 8 deletions src/kudu/server/webserver_options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@

#include "kudu/server/webserver_options.h"

#include <cstring>
#include <cstdlib>
#include <string>

#include <gflags/gflags.h>
#include <string.h>
#include <stdlib.h>

#include "kudu/gutil/strings/substitute.h"
#include "kudu/util/flag_tags.h"
#include "kudu/util/flag_validators.h"

using std::string;

Expand Down Expand Up @@ -54,10 +56,11 @@ TAG_FLAG(webserver_enable_doc_root, advanced);
// SSL configuration.
DEFINE_string(webserver_certificate_file, "",
"The location of the debug webserver's SSL certificate file, in PEM format. If "
"empty, webserver SSL support is not enabled");
"empty, webserver SSL support is not enabled. If --webserver_private_key_file "
"is set, this option must be set as well.");
DEFINE_string(webserver_private_key_file, "", "The full path to the private key used as a"
" counterpart to the public key contained in --ssl_server_certificate. If "
"--ssl_server_certificate is set, this option must be set as well.");
" counterpart to the public key contained in --webserver_certificate_file. If "
"--webserver_certificate_file is set, this option must be set as well.");
DEFINE_string(webserver_private_key_password_cmd, "", "A Unix command whose output "
"returns the password used to decrypt the Webserver's certificate private key file "
"specified in --webserver_private_key_file. If the PEM key file is not "
Expand All @@ -84,14 +87,14 @@ TAG_FLAG(webserver_port, stable);

namespace kudu {

static bool ValidateTlsFlags(const char* /*flag_name*/, const string& /*flag_value*/) {
static bool ValidateTlsFlags() {
bool has_cert = !FLAGS_webserver_certificate_file.empty();
bool has_key = !FLAGS_webserver_private_key_file.empty();
bool has_passwd = !FLAGS_webserver_private_key_password_cmd.empty();

if (has_key != has_cert) {
LOG(ERROR) << "--webserver_certificate_file and --webserver_private_key_file "
"must be set as a group";
"must be set as a group; i.e. either set all or none of them.";
return false;
}
if (has_passwd && !has_key) {
Expand All @@ -102,7 +105,7 @@ static bool ValidateTlsFlags(const char* /*flag_name*/, const string& /*flag_val

return true;
}
DEFINE_validator(webserver_private_key_file, &ValidateTlsFlags);
GROUP_FLAG_VALIDATOR(webserver_tls_options, ValidateTlsFlags);

// Returns KUDU_HOME if set, otherwise we won't serve any static files.
static string GetDefaultDocumentRoot() {
Expand Down
2 changes: 2 additions & 0 deletions src/kudu/util/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ set(UTIL_SRCS
file_cache.cc
flags.cc
flag_tags.cc
flag_validators.cc
group_varint.cc
pstack_watcher.cc
hdr_histogram.cc
Expand Down Expand Up @@ -341,6 +342,7 @@ ADD_KUDU_TEST(failure_detector-test)
ADD_KUDU_TEST(file_cache-test)
ADD_KUDU_TEST(file_cache-stress-test RUN_SERIAL true)
ADD_KUDU_TEST(flag_tags-test)
ADD_KUDU_TEST(flag_validators-test)
ADD_KUDU_TEST(flags-test)
ADD_KUDU_TEST(group_varint-test)
ADD_KUDU_TEST(hash_util-test)
Expand Down
172 changes: 172 additions & 0 deletions src/kudu/util/flag_validators-test.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include <string>

#include <gflags/gflags.h>
#include <gtest/gtest.h>

#include "kudu/util/debug/leakcheck_disabler.h"
#include "kudu/util/flags.h"
#include "kudu/util/flag_validators.h"
#include "kudu/util/logging.h"
#include "kudu/util/test_util.h"

DEFINE_string(grouped_0, "", "First flag to set.");
DEFINE_string(grouped_1, "", "Second flag to set.");

namespace kudu {

static bool CheckGroupedFlags() {
const bool is_set_0 = !FLAGS_grouped_0.empty();
const bool is_set_1 = !FLAGS_grouped_1.empty();

if (is_set_0 != is_set_1) {
LOG(ERROR) << "--grouped_0 and --grouped_1 must be set as a group";
return false;
}

return true;
}
GROUP_FLAG_VALIDATOR(test_group_validator, CheckGroupedFlags)

class FlagsValidatorsBasicTest : public KuduTest {
public:
void RunTest(const char** argv, int argc) {
char** casted_argv = const_cast<char**>(argv);
// ParseCommandLineFlags() calls exit(1) if it finds inconsistency in flags.
ASSERT_EQ(1, ParseCommandLineFlags(&argc, &casted_argv, true));
}
};

TEST_F(FlagsValidatorsBasicTest, Grouped) {
const auto& validators = GetFlagValidators();
ASSERT_EQ(1, validators.size());
const auto& validator = validators.begin()->second;
EXPECT_TRUE(validator());
FLAGS_grouped_0 = "0";
EXPECT_FALSE(validator());
FLAGS_grouped_1 = "1";
EXPECT_TRUE(validator());
FLAGS_grouped_0 = "";
EXPECT_FALSE(validator());
FLAGS_grouped_1 = "";
EXPECT_TRUE(validator());
}

class FlagsValidatorsDeathTest : public KuduTest {
public:
void Run(const char** argv, int argc) {
debug::ScopedLeakCheckDisabler disabler;
char** casted_argv = const_cast<char**>(argv);
// ParseCommandLineFlags() calls exit(1) if one of the custom validators
// finds inconsistency in flags.
ParseCommandLineFlags(&argc, &casted_argv, true);
exit(0);
}

void RunSuccess(const char** argv, int argc) {
EXPECT_EXIT(Run(argv, argc), ::testing::ExitedWithCode(0), ".*");
}

void RunFailure(const char** argv, int argc) {
EXPECT_EXIT(Run(argv, argc), ::testing::ExitedWithCode(1),
".* Detected inconsistency in command-line flags; exiting");
}
};

TEST_F(FlagsValidatorsDeathTest, GroupedSuccessNoFlags) {
const char* argv[] = { "argv_set_0" };
NO_FATALS(RunSuccess(argv, ARRAYSIZE(argv)));
}

TEST_F(FlagsValidatorsDeathTest, GroupedSuccessSimple) {
static const size_t kArgvSize = 1 + 2;
const char* argv_sets[][kArgvSize] = {
{
"argv_set_0",
"--grouped_0=first",
"--grouped_1=second",
},
{
"argv_set_1",
"--grouped_0=second",
"--grouped_1=first",
},
{
"argv_set_2",
"--grouped_0=",
"--grouped_1=",
},
{
"argv_set_3",
"--grouped_1=",
"--grouped_0=",
},
};
for (auto argv : argv_sets) {
RunSuccess(argv, kArgvSize);
}
}

TEST_F(FlagsValidatorsDeathTest, GroupedFailureSimple) {
static const size_t kArgvSize = 1 + 1;
const char* argv_sets[][kArgvSize] = {
{
"argv_set_0",
"--grouped_0=a",
},
{
"argv_set_1",
"--grouped_1=b",
},
};
for (auto argv : argv_sets) {
RunFailure(argv, kArgvSize);
}
}

TEST_F(FlagsValidatorsDeathTest, GroupedFailureWithEmptyValues) {
static const size_t kArgvSize = 1 + 2;
const char* argv_sets[][kArgvSize] = {
{
"argv_set_0",
"--grouped_0=a",
"--grouped_1=",
},
{
"argv_set_1",
"--grouped_1=",
"--grouped_0=a",
},
{
"argv_set_2",
"--grouped_0=",
"--grouped_1=b",
},
{
"argv_set_3",
"--grouped_1=b",
"--grouped_0=",
},
};
for (auto argv : argv_sets) {
RunFailure(argv, kArgvSize);
}
}

} // namespace kudu
Loading

0 comments on commit e7334c2

Please sign in to comment.