[master][tool] KUDU-2181 Tool to orchestrate adding a master

This tool orchestrates the mechanism to add a master to an existing
Kudu cluster using the changes made previously including the add master
Raft change config.

This tool must be run from the master being added.

Outline of steps:
- Bring up master from supplied flags
- Invoke add master Raft change config RPC
- Determine whether new master has caught up from WAL
  and ksck is healthy.
- If yes, we are done else continue below
- Delete system catalog on the new master
- Copy system catalog from one of the existing masters
- Verify ksck is healthy.
- Shutdown the new master

The tool now manages bringing up the new master process, copying
system catalog and hence bunch of functionality from the
dynamic_multi_master test is no longer needed and hence deleted.

Using Subprocess we need to shutdown the new master and hence can't
verify certain state which included contacting new master directly
immediately after adding new master. Function
VerifyClusterAfterMasterAddition() covers comprehensive verification
steps so not missing out by removing those steps.

This change also includes some refactoring in ExternalDaemon
to fetch Kerberos related configuration to be used outside
ExternalMiniCluster.

Change-Id: I8f99cf2b3f1738c4127c7e7288beab61daf42e7b
Reviewed-on: http://gerrit.cloudera.org:8080/17267
Reviewed-by: Andrew Wong <[email protected]>
Tested-by: Kudu Jenkins

src/kudu/master/CMakeLists.txt

@@ -108,7 +108,7 @@ SET_KUDU_TEST_LINK_LIBS(
 
 ADD_KUDU_TEST(auto_rebalancer-test)
 ADD_KUDU_TEST(catalog_manager-test)
-ADD_KUDU_TEST(dynamic_multi_master-test)
+ADD_KUDU_TEST(dynamic_multi_master-test NUM_SHARDS 4)
 ADD_KUDU_TEST(hms_notification_log_listener-test)
 ADD_KUDU_TEST(location_cache-test DATA_FILES ../scripts/first_argument.sh)
 ADD_KUDU_TEST(master_options-test)

src/kudu/mini-cluster/external_mini_cluster.cc

@@ -90,6 +90,7 @@ using kudu::tserver::ListTabletsResponsePB;
 using kudu::tserver::TabletServerAdminServiceProxy;
 using kudu::tserver::TabletServerServiceProxy;
 using std::copy;
+using std::map;
 using std::pair;
 using std::string;
 using std::unique_ptr;
@@ -949,20 +950,6 @@ string ExternalMiniCluster::UuidForTS(int ts_idx) const {
   return tablet_server(ts_idx)->uuid();
 }
 
-Status ExternalMiniCluster::AddMaster(const scoped_refptr<ExternalMaster>& new_master) {
-  const auto& new_master_uuid = new_master->instance_id().permanent_uuid();
-  for (const auto& m : masters_) {
-    if (m->instance_id().permanent_uuid() == new_master_uuid) {
-      CHECK(m->bound_rpc_hostport() == new_master->bound_rpc_hostport());
-      return Status::AlreadyPresent(Substitute(
-          "Master $0, uuid: $1 already present in ExternalMiniCluster",
-          m->bound_rpc_hostport().ToString(), new_master_uuid));
-    }
-  }
-  masters_.emplace_back(new_master);
-  return Status::OK();
-}
-
 Status ExternalMiniCluster::RemoveMaster(const HostPort& hp) {
   for (auto it = masters_.begin(); it != masters_.end(); ++it) {
     if ((*it)->bound_rpc_hostport() == hp) {
@@ -1165,25 +1152,35 @@ void ExternalDaemon::SetMetastoreIntegration(const string& hms_uris,
   opts_.extra_flags.emplace_back(Substitute("--hive_metastore_sasl_enabled=$0", enable_kerberos));
 }
 
-Status ExternalDaemon::EnableKerberos(MiniKdc* kdc,
-                                      const string& principal,
-                                      const string& bind_host) {
-  string spn = principal + "/" + bind_host;
+Status ExternalDaemon::CreateKerberosConfig(MiniKdc* kdc,
+                                            const string& principal_base,
+                                            const string& bind_host,
+                                            vector<string>* flags,
+                                            map<string, string>* env_vars) {
+  string spn = principal_base + "/" + bind_host;
   string ktpath;
   RETURN_NOT_OK_PREPEND(kdc->CreateServiceKeytab(spn, &ktpath),
                         "could not create keytab");
-  extra_env_ = kdc->GetEnvVars();
+  *env_vars = kdc->GetEnvVars();
+  *flags =  {
+      Substitute("--keytab_file=$0", ktpath),
+      Substitute("--principal=$0", spn),
+      "--rpc_authentication=required",
+      "--superuser_acl=test-admin",
+      "--user_acl=test-user",
+  };
 
+  return Status::OK();
+}
+
+Status ExternalDaemon::EnableKerberos(MiniKdc* kdc, const string& principal_base,
+                                      const string& bind_host) {
+  vector<string> flags;
+  RETURN_NOT_OK(CreateKerberosConfig(kdc, principal_base, bind_host, &flags, &extra_env_));
   // Insert Kerberos flags at the front of extra_flags, so that user specified
   // flags will override them.
-  opts_.extra_flags.insert(opts_.extra_flags.begin(), {
-    Substitute("--keytab_file=$0", ktpath),
-    Substitute("--principal=$0", spn),
-    "--rpc_authentication=required",
-    "--superuser_acl=test-admin",
-    "--user_acl=test-user",
-  });
-
+  opts_.extra_flags.insert(opts_.extra_flags.begin(), std::make_move_iterator(flags.begin()),
+                           std::make_move_iterator(flags.end()));
   return Status::OK();
 }
 

src/kudu/mini-cluster/external_mini_cluster.h

@@ -480,10 +480,6 @@ class ExternalMiniCluster : public MiniCluster {
   // files that reside in the log dir.
   std::string GetLogPath(const std::string& daemon_id) const;
 
-  // Adds a master to the ExternalMiniCluster when the new master has been added
-  // dynamically after bringing up the ExternalMiniCluster.
-  Status AddMaster(const scoped_refptr<ExternalMaster>& master);
-
   // Removes any bookkeeping of the master specified by 'hp' from the ExternalMiniCluster
   // after already having run through a successful master Raft change config to remove it.
   // This helps keep the state of the actual cluster in sync with the state in ExternalMiniCluster.
@@ -566,6 +562,15 @@ class ExternalDaemon : public RefCountedThreadSafe<ExternalDaemon> {
   void SetMetastoreIntegration(const std::string& hms_uris,
                                bool enable_kerberos);
 
+  // Create a Kerberos principal and keytab using the 'principal_base' and 'bind_host' hostname
+  // of the form <principal_base>/<bind_host>.
+  // Returns the generated keytab file and service principal as 'flags' and the appropriate
+  // environment variables in 'env_vars' output parameters.
+  static Status CreateKerberosConfig(MiniKdc* kdc, const std::string& principal_base,
+                                     const std::string& bind_host,
+                                     std::vector<std::string>* flags,
+                                     std::map<std::string, std::string>* env_vars);
+
   // Enable Kerberos for this daemon. This creates a Kerberos principal
   // and keytab, and sets the appropriate environment variables in the
   // subprocess such that the server will use Kerberos authentication.

src/kudu/tools/kudu-tool-test.cc

@@ -54,11 +54,11 @@
 #include "kudu/common/common.pb.h"
 #include "kudu/common/partial_row.h"
 #include "kudu/common/partition.h"
+#include "kudu/common/row_operations.pb.h"
 #include "kudu/common/schema.h"
 #include "kudu/common/types.h"
 #include "kudu/common/wire_protocol-test-util.h"
 #include "kudu/common/wire_protocol.h"
-#include "kudu/common/wire_protocol.pb.h"
 #include "kudu/consensus/consensus.pb.h"
 #include "kudu/consensus/log.h"
 #include "kudu/consensus/log_util.h"
@@ -1107,8 +1107,8 @@ TEST_F(ToolTest, TestModeHelp) {
         "status.*Get the status",
         "timestamp.*Get the current timestamp",
         "list.*List masters in a Kudu cluster",
-        "add.*Add a master to the Raft configuration",
-        "remove.*Remove a master from the Raft configuration"
+        "add.*Add a master to the Kudu cluster",
+        "remove.*Remove a master from the Kudu cluster"
     };
     NO_FATALS(RunTestHelp(kCmd, kMasterModeRegexes));
     NO_FATALS(RunTestHelpRpcFlags(kCmd,
@@ -1130,9 +1130,10 @@ TEST_F(ToolTest, TestModeHelp) {
   }
   {
     NO_FATALS(RunTestHelp("master add --help",
-                          {"-wait_secs \\(Timeout in seconds to wait for the newly added master"}));
+                          {"-wait_secs.*\\(Timeout in seconds to wait while retrying operations",
+                           "-kudu_abs_path.*\\(Absolute file path of the 'kudu' executable used"}));
     NO_FATALS(RunTestHelp("master remove --help",
-                          {"-master_uuid \\(Permanent UUID of the master"}));
+                          {"-master_uuid.*\\(Permanent UUID of the master"}));
   }
   {
     const vector<string> kPbcModeRegexes = {

src/kudu/tools/tool_action_common.cc

@@ -767,6 +767,20 @@ Status DumpMemTrackers(const string& address, uint16_t default_port) {
   return Status::OK();
 }
 
+Status GetKuduToolAbsolutePathSafe(string* path) {
+  static const char* const kKuduCtlFileName = "kudu";
+  string exe;
+  RETURN_NOT_OK(Env::Default()->GetExecutablePath(&exe));
+  const string binroot = DirName(exe);
+  string tool_abs_path = JoinPathSegments(binroot, kKuduCtlFileName);
+  if (!Env::Default()->FileExists(tool_abs_path)) {
+    return Status::NotFound(Substitute(
+        "$0 binary not found at $1", kKuduCtlFileName, tool_abs_path));
+  }
+  *path = std::move(tool_abs_path);
+  return Status::OK();
+}
+
 namespace {
 
 // Pretty print a table using the psql format. For example:

src/kudu/tools/tool_action_common.h

@@ -216,6 +216,10 @@ Status ParseMasterAddresses(
     const RunnerContext& context,
     std::vector<std::string>* master_addresses);
 
+// Get full path to the top-level 'kudu' tool binary in the output 'path' parameter.
+// Returns appropriate error if the binary is not found.
+Status GetKuduToolAbsolutePathSafe(std::string* path);
+
 // Parses a comma separated list of "host:port" pairs into an unordered set of
 // HostPort objects. If no port is specified for an entry in the comma separated list,
 // the default master port is used for that entry's pair.