KUDU-3328: make rebalancer tool take into account maintenance mode

At present kudu master would not assign new replicas on maintenance tservers
but we could still move replicas to them through the rebalancer tool.
Sometimes we want this kind of operation to be disabled because the maintenance
tservers would be restarted or decommissioned in a short time.

This patch improves the rebalancer tool's behavior. If maintenance tservers are
not specified in 'ignored_tservers' and users don't specify
'-force_rebalance_replicas_on_maintenance_tservers' neither, the rebalancer tool
couldn't get replica moves and return IllegalState.

Change-Id: I843a2521a811ab67c5f98bb43cc84367edb3fe0b
Reviewed-on: http://gerrit.cloudera.org:8080/18074
Tested-by: Kudu Jenkins
Reviewed-by: Andrew Wong <[email protected]>

src/kudu/master/auto_rebalancer.cc

@@ -155,7 +155,8 @@ AutoRebalancerTask::AutoRebalancerTask(CatalogManager* catalog_manager,
       /*run_policy_fixer*/true,
       /*run_cross_location_rebalancing*/true,
       /*run_intra_location_rebalancing*/true,
-      FLAGS_auto_rebalancing_load_imbalance_threshold))),
+      FLAGS_auto_rebalancing_load_imbalance_threshold,
+      /*force_rebalance_replicas_on_maintenance_tservers*/false))),
       random_generator_(random_device_()),
       number_of_loop_iterations_for_test_(0),
       moves_scheduled_this_round_for_test_(0) {

src/kudu/rebalance/rebalancer.cc

@@ -60,20 +60,20 @@ using strings::Substitute;
 namespace kudu {
 namespace rebalance {
 
-Rebalancer::Config::Config(
-    vector<string> ignored_tservers_param,
-    vector<string> master_addresses,
-    vector<string> table_filters,
-    size_t max_moves_per_server,
-    size_t max_staleness_interval_sec,
-    int64_t max_run_time_sec,
-    bool move_replicas_from_ignored_tservers,
-    bool move_rf1_replicas,
-    bool output_replica_distribution_details,
-    bool run_policy_fixer,
-    bool run_cross_location_rebalancing,
-    bool run_intra_location_rebalancing,
-    double load_imbalance_threshold)
+Rebalancer::Config::Config(vector<string> ignored_tservers_param,
+                           vector<string> master_addresses,
+                           vector<string> table_filters,
+                           size_t max_moves_per_server,
+                           size_t max_staleness_interval_sec,
+                           int64_t max_run_time_sec,
+                           bool move_replicas_from_ignored_tservers,
+                           bool move_rf1_replicas,
+                           bool output_replica_distribution_details,
+                           bool run_policy_fixer,
+                           bool run_cross_location_rebalancing,
+                           bool run_intra_location_rebalancing,
+                           double load_imbalance_threshold,
+                           bool force_rebalance_replicas_on_maintenance_tservers)
     : ignored_tservers(ignored_tservers_param.begin(), ignored_tservers_param.end()),
       master_addresses(std::move(master_addresses)),
       table_filters(std::move(table_filters)),
@@ -86,7 +86,9 @@ Rebalancer::Config::Config(
       run_policy_fixer(run_policy_fixer),
       run_cross_location_rebalancing(run_cross_location_rebalancing),
       run_intra_location_rebalancing(run_intra_location_rebalancing),
-      load_imbalance_threshold(load_imbalance_threshold) {
+      load_imbalance_threshold(load_imbalance_threshold),
+      force_rebalance_replicas_on_maintenance_tservers(
+          force_rebalance_replicas_on_maintenance_tservers) {
   DCHECK_GE(max_moves_per_server, 0);
 }
 

src/kudu/rebalance/rebalancer.h

@@ -42,6 +42,7 @@ struct ClusterRawInfo {
   std::vector<cluster_summary::ServerHealthSummary> tserver_summaries;
   std::vector<cluster_summary::TableSummary> table_summaries;
   std::vector<cluster_summary::TabletSummary> tablet_summaries;
+  std::unordered_set<std::string> tservers_in_maintenance_mode;
 };
 
 // A class implementing logic for Kudu cluster rebalancing.
@@ -66,7 +67,8 @@ class Rebalancer {
            bool run_policy_fixer = true,
            bool run_cross_location_rebalancing = true,
            bool run_intra_location_rebalancing = true,
-           double load_imbalance_threshold = kLoadImbalanceThreshold);
+           double load_imbalance_threshold = kLoadImbalanceThreshold,
+           bool force_rebalance_replicas_on_maintenance_tservers = false);
 
     // UUIDs of ignored servers. If empty, run the rebalancing on
     // all tablet servers in the cluster only when all tablet servers
@@ -132,6 +134,8 @@ class Rebalancer {
     // The per-table location load imbalance threshold for the cross-location
     // balancing algorithm.
     double load_imbalance_threshold;
+
+    bool force_rebalance_replicas_on_maintenance_tservers;
   };
 
   // Represents a concrete move of a replica from one tablet server to another.

src/kudu/tools/rebalancer_tool-test.cc

@@ -279,6 +279,71 @@ TEST_P(RebalanceStartCriteriaTest, UnknownIgnoredTServer) {
   }
 }
 
+TEST_P(RebalanceStartCriteriaTest, TabletServerInMaintenanceMode) {
+  const bool is_343_scheme = (GetParam() == Kudu1097::Enable);
+  const vector<string> kMasterFlags = {
+      Substitute("--raft_prepare_replacement_before_eviction=$0", is_343_scheme),
+  };
+  const vector<string> kTserverFlags = {
+      Substitute("--raft_prepare_replacement_before_eviction=$0", is_343_scheme),
+  };
+
+  FLAGS_num_tablet_servers = 5;
+  NO_FATALS(BuildAndStart(kTserverFlags, kMasterFlags));
+
+  auto* ts = cluster_->tablet_server(0);
+  string master_addr = cluster_->master()->bound_rpc_addr().ToString();
+
+  // Set the tserver into maintenance mode.
+  {
+    string out;
+    string err;
+    Status s =
+        RunKuduTool({"tserver", "state", "enter_maintenance", master_addr, ts->uuid()}, &out, &err);
+    ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
+  }
+
+  // Rebalancer doesn't start if any tserver is in maintenance mode.
+  {
+    string out;
+    string err;
+    Status s = RunKuduTool({"cluster", "rebalance", master_addr}, &out, &err);
+    ASSERT_TRUE(s.IsRuntimeError()) << ToolRunInfo(s, out, err);
+    ASSERT_STR_CONTAINS(err, "unacceptable state MAINTENANCE_MODE");
+  }
+
+  // Rebalancer starts when specifying the maintenance tserver in '--ignored_tservers'.
+  {
+    string out;
+    string err;
+    Status s = RunKuduTool(
+        {"cluster", "rebalance", master_addr, "--ignored_tservers=" + ts->uuid()}, &out, &err);
+    ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
+  }
+
+  // Rebalancer starts when specifying '--force_rebalance_replicas_on_maintenance_tservers'.
+  {
+    string out;
+    string err;
+    Status s = RunKuduTool(
+        {"cluster", "rebalance", master_addr, "--force_rebalance_replicas_on_maintenance_tservers"},
+        &out,
+        &err);
+    ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
+  }
+
+  // Now exit maintenance mode and rebalancer could start.
+  {
+    string out;
+    string err;
+    Status s =
+        RunKuduTool({"tserver", "state", "exit_maintenance", master_addr, ts->uuid()}, &out, &err);
+    ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
+    s = RunKuduTool({"cluster", "rebalance", master_addr}, &out, &err);
+    ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
+  }
+}
+
 // Make sure the rebalancer doesn't start if specified too many ignored tservers.
 class RebalanceStartSafetyTest :
     public AdminCliTest,
@@ -874,12 +939,8 @@ TEST_F(IgnoredTserverGoesDownDuringRebalancingTest, TserverDown) {
       "--max_moves_per_server=1",
     }, &out, &err);
     ASSERT_TRUE(s.ok()) << ToolRunInfo(s, out, err);
-
     // The rebalancer tool should not crash.
     ASSERT_STR_NOT_CONTAINS(s.ToString(), kExitOnSignalStr);
-    // The rebalancer tool would log some information of the unhealthy server.
-    ASSERT_STR_CONTAINS(err, Substitute("ignoring unhealthy tablet server $0",
-                                       cluster_->tablet_server(ignored_tserver_idx)->uuid()));
 
     // Restart the ignored tablet server
     ASSERT_OK(cluster_->tablet_server(ignored_tserver_idx)->Restart());