[tools] added insert-generated-rows into kudu tools

The insert-generated-rows tool has been merged into the 'kudu' umbrella toolset. This addresses KUDU-1628. Besides, added ability to run multiple inserter threads and specify additional parameters on batching behavior of the generated write operations. It's possible to run data generating sessions both in MANUAL_FLUSH and AUTO_FLUSH_BACKGROUND modes. Introduced sequential and random modes for the data generator. Overall, these changes allow to use the tool to measure performance of the Kudu C++ client library in simplistic 'push-as-much-as-you-can' scenario: the client generates and sends data as fast as it can. Change-Id: I332927c0b928c9c4fb81a8e26f5c9ed7565299ad Reviewed-on: http://gerrit.cloudera.org:8080/4412 Reviewed-by: Adar Dembo <[email protected]> Tested-by: Kudu Jenkins
cutiechi · Sep 30, 2016 · 5385af8 · 5385af8
1 parent 742d85c
commit 5385af8
Show file tree

Hide file tree

Showing 17 changed files with 751 additions and 183 deletions.
diff --git a/src/kudu/tools/CMakeLists.txt b/src/kudu/tools/CMakeLists.txt
@@ -40,11 +40,6 @@ add_executable(create-demo-table create-demo-table.cc)
 target_link_libraries(create-demo-table
   ${LINK_LIBS})
 
-add_executable(insert-generated-rows insert-generated-rows.cc)
-target_link_libraries(insert-generated-rows
-  kudu_tools_util
-  ${LINK_LIBS})
-
 add_library(ksck
     ksck.cc
     ksck_remote.cc
@@ -70,6 +65,7 @@ add_executable(kudu
   tool_action_remote_replica.cc
   tool_action_table.cc
   tool_action_tablet.cc
+  tool_action_test.cc
   tool_action_tserver.cc
   tool_action_wal.cc
   tool_main.cc
@@ -115,4 +111,3 @@ ADD_KUDU_TEST_DEPENDENCIES(kudu-tool-test
 ADD_KUDU_TEST(kudu-ts-cli-test)
 ADD_KUDU_TEST_DEPENDENCIES(kudu-ts-cli-test
   kudu)
-
diff --git a/src/kudu/tools/insert-generated-rows.cc b/src/kudu/tools/insert-generated-rows.cc
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <algorithm>
+#include <iterator>
 #include <memory>
 #include <sstream>
 #include <string>
@@ -26,6 +28,7 @@
 #include "kudu/cfile/cfile-test-base.h"
 #include "kudu/cfile/cfile_util.h"
 #include "kudu/cfile/cfile_writer.h"
+#include "kudu/client/client-test-util.h"
 #include "kudu/common/partial_row.h"
 #include "kudu/common/partition.h"
 #include "kudu/common/schema.h"
@@ -43,6 +46,7 @@
 #include "kudu/gutil/ref_counted.h"
 #include "kudu/gutil/strings/split.h"
 #include "kudu/gutil/strings/substitute.h"
+#include "kudu/integration-tests/external_mini_cluster.h"
 #include "kudu/tablet/local_tablet_writer.h"
 #include "kudu/tablet/tablet-harness.h"
 #include "kudu/tablet/tablet_metadata.h"
@@ -51,24 +55,29 @@
 #include "kudu/util/async_util.h"
 #include "kudu/util/env.h"
 #include "kudu/util/metrics.h"
+#include "kudu/util/net/sockaddr.h"
 #include "kudu/util/oid_generator.h"
 #include "kudu/util/path_util.h"
 #include "kudu/util/subprocess.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
 
 namespace kudu {
+
 namespace tools {
 
 using cfile::CFileWriter;
 using cfile::StringDataGenerator;
 using cfile::WriterOptions;
+using client::sp::shared_ptr;
 using consensus::OpId;
 using consensus::ReplicateRefPtr;
 using consensus::ReplicateMsg;
 using fs::WritableBlock;
 using log::Log;
 using log::LogOptions;
+using std::back_inserter;
+using std::copy;
 using std::ostringstream;
 using std::string;
 using std::unique_ptr;
@@ -159,10 +168,8 @@ class ToolTest : public KuduTest {
     ASSERT_EQ(0, stderr[usage_idx].find("Usage: "));
 
     // Strip away everything up to the usage string to test for regexes.
-    vector<string> remaining_lines;
-    for (int i = usage_idx + 1; i < stderr.size(); i++) {
-      remaining_lines.push_back(stderr[i]);
-    }
+    const vector<string> remaining_lines(stderr.begin() + usage_idx + 1,
+                                         stderr.end());
     for (const auto& r : regexes) {
       ASSERT_STRINGS_ANY_MATCH(remaining_lines, r);
     }
@@ -182,6 +189,7 @@ TEST_F(ToolTest, TestTopLevelHelp) {
       "remote_replica.*replicas on a Kudu Tablet Server",
       "table.*Kudu tables",
       "tablet.*Kudu tablets",
+      "test.*tests",
       "tserver.*Kudu Tablet Server",
       "wal.*write-ahead log"
   };
@@ -286,6 +294,12 @@ TEST_F(ToolTest, TestModeHelp) {
     };
     NO_FATALS(RunTestHelp("tablet change_config", kChangeConfigModeRegexes));
   }
+  {
+    const vector<string> kTestRegexes = {
+        "loadgen.*Run load generation test with optional scan afterwards",
+    };
+    NO_FATALS(RunTestHelp("test", kTestRegexes));
+  }
   {
     const vector<string> kTServerModeRegexes = {
         "set_flag.*Change a gflag value",
@@ -745,5 +759,111 @@ TEST_F(ToolTest, TestLocalReplicaOps) {
   }
 }
 
+// Create and start Kudu mini cluster, optionally creating a table in the DB,
+// and then run 'kudu test loadgen ...' utility against it.
+void RunLoadgen(size_t num_tservers = 1,
+                const vector<string>& tool_args = {},
+                const string& table_name = "") {
+  kudu::ExternalMiniClusterOptions opts;
+  opts.num_tablet_servers = num_tservers;
+  // fsync causes flakiness on EC2
+  opts.extra_tserver_flags.push_back("--never_fsync");
+
+  unique_ptr<ExternalMiniCluster> cluster(new ExternalMiniCluster(opts));
+  ASSERT_OK(cluster->Start());
+  if (!table_name.empty()) {
+    static const string kKeyColumnName = "key";
+    static const Schema kSchema = Schema(
+      {
+        ColumnSchema(kKeyColumnName, INT64),
+        ColumnSchema("bool_val", BOOL),
+        ColumnSchema("int8_val", INT8),
+        ColumnSchema("int16_val", INT16),
+        ColumnSchema("int32_val", INT32),
+        ColumnSchema("int64_val", INT64),
+        ColumnSchema("float_val", FLOAT),
+        ColumnSchema("double_val", DOUBLE),
+        ColumnSchema("unixtime_micros_val", UNIXTIME_MICROS),
+        ColumnSchema("string_val", STRING),
+        ColumnSchema("binary_val", BINARY),
+      }, 1);
+
+    shared_ptr<client::KuduClient> client;
+    ASSERT_OK(cluster->CreateClient(nullptr, &client));
+    client::KuduSchema client_schema(client::KuduSchemaFromSchema(kSchema));
+    unique_ptr<client::KuduTableCreator> table_creator(
+        client->NewTableCreator());
+    ASSERT_OK(table_creator->table_name(table_name)
+              .schema(&client_schema)
+              .add_hash_partitions({kKeyColumnName}, 2)
+              .num_replicas(cluster->num_tablet_servers())
+              .Create());
+  }
+  vector<string> args = {
+    GetKuduCtlAbsolutePath(),
+    "test",
+    "loadgen",
+    cluster->master()->bound_rpc_addr().ToString(),
+  };
+  if (!table_name.empty()) {
+    args.push_back(Substitute("-table_name=$0", table_name));
+  }
+  copy(tool_args.begin(), tool_args.end(), back_inserter(args));
+  ASSERT_OK(Subprocess::Call(args));
+}
+
+// Run the loadgen benchmark with all optional parameters set to defaults.
+TEST_F(ToolTest, TestLoadgenDefaultParameters) {
+  NO_FATALS(RunLoadgen());
+}
+
+// Run the loadgen benchmark in AUTO_FLUSH_BACKGROUND mode, sequential values.
+TEST_F(ToolTest, TestLoadgenAutoFlushBackgroundSequential) {
+  NO_FATALS(RunLoadgen(3,
+      {
+        "--buffer_flush_watermark_pct=0.125",
+        "--buffer_size_bytes=65536",
+        "--buffers_num=8",
+        "--num_rows_per_thread=2048",
+        "--num_threads=4",
+        "--run_scan",
+        "--string_fixed=0123456789",
+      },
+      "bench_auto_flush_background_sequential"));
+}
+
+// Run loadgen benchmark in AUTO_FLUSH_BACKGROUND mode, randomized values.
+TEST_F(ToolTest, TestLoadgenAutoFlushBackgroundRandom) {
+  NO_FATALS(RunLoadgen(5,
+      {
+        "--buffer_flush_watermark_pct=0.125",
+        "--buffer_size_bytes=65536",
+        "--buffers_num=8",
+        // small number of rows to avoid collisions: it's random generation mode
+        "--num_rows_per_thread=16",
+        "--num_threads=1",
+        "--run_scan",
+        "--string_len=8",
+        "--use_random",
+      },
+      "bench_auto_flush_background_random"));
+}
+
+// Run the loadgen benchmark in MANUAL_FLUSH mode.
+TEST_F(ToolTest, TestLoadgenManualFlush) {
+  NO_FATALS(RunLoadgen(3,
+      {
+        "--buffer_size_bytes=524288",
+        "--buffers_num=2",
+        "--flush_per_n_rows=1024",
+        "--num_rows_per_thread=4096",
+        "--num_threads=3",
+        "--run_scan",
+        "--show_first_n_errors=3",
+        "--string_len=16",
+      },
+      "bench_manual_flush"));
+}
+
 } // namespace tools
 } // namespace kudu
diff --git a/src/kudu/tools/tool_action.cc b/src/kudu/tools/tool_action.cc
@@ -170,9 +170,6 @@ string Mode::BuildHelp(const vector<Mode*>& chain) const {
   return msg;
 }
 
-Mode::Mode() {
-}
-
 ActionBuilder::ActionBuilder(const string& name, const ActionRunner& runner)
     : name_(name),
       runner_(runner) {
@@ -224,9 +221,6 @@ unique_ptr<Action> ActionBuilder::Build() {
   return action;
 }
 
-Action::Action() {
-}
-
 Status Action::Run(const vector<Mode*>& chain,
                    const unordered_map<string, string>& required_args,
                    const vector<string>& variadic_args) const {

diff --git a/src/kudu/tools/tool_action.h b/src/kudu/tools/tool_action.h
@@ -118,7 +118,7 @@ class Mode {
  private:
   friend class ModeBuilder;
 
-  Mode();
+  Mode() = default;
 
   std::string name_;
 
@@ -261,7 +261,7 @@ class Action {
  private:
   friend class ActionBuilder;
 
-  Action();
+  Action() = default;
 
   std::string name_;
 
@@ -283,6 +283,7 @@ std::unique_ptr<Mode> BuildPbcMode();
 std::unique_ptr<Mode> BuildRemoteReplicaMode();
 std::unique_ptr<Mode> BuildTableMode();
 std::unique_ptr<Mode> BuildTabletMode();
+std::unique_ptr<Mode> BuildTestMode();
 std::unique_ptr<Mode> BuildTServerMode();
 std::unique_ptr<Mode> BuildWalMode();