Initial script for the new regression test

Summary:
This diff includes an initial script running a set of benchmarks for
regression test.  The script does the following things:

  checkout the specified rocksdb commit (or origin/master as default)
  make clean && DEBUG_LEVEL=0 make db_bench
  setup test directories
  run set of benchmarks and store results

Currently, the script will run couple benchmarks, store all the benchmark
output, extract micros per op and percentile information for each benchmark
and store them in a single SUMMARY.csv file.  The SUMMARY.csv will make the
follow-up regression detection easier.

In addition, the current script only takes env arguments to set important
attributes of db_bench.  Will follow-up with a patch that allows db_bench
to construct options from an options file.

Test Plan:
NUM_KEYS=100 ./tools/regression_test.sh

  Sample SUMMARY.csv file:

                                     commit id,                      benchmark,  ms-per-op,        p50,        p75,        p99,      p99.9,     p99.99
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                        fillseq,      15.28,      54.66,      77.14,    5000.00,   17900.00,   18483.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                      overwrite,      13.54,      57.69,      86.39,    3000.00,   15600.00,   17013.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                     readrandom,       1.04,       0.80,       1.67,     293.33,     395.00,     504.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,               readwhilewriting,       2.75,       1.01,       1.87,     200.00,     460.00,     485.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                   deleterandom,       3.64,      48.12,      70.09,     200.00,     336.67,     347.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                     seekrandom,      24.31,     391.87,     513.69,     872.73,     990.00,    1048.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,         seekrandomwhilewriting,      14.02,     185.14,     294.15,     700.00,    1440.00,    1527.00

Reviewers: sdong, IslamAbdelRahman, kradhakrishnan, yiwu, andrewkr, gunnarku

Reviewed By: gunnarku

Subscribers: gunnarku, MarkCallaghan, andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D57597

tools/benchmark.sh

@@ -54,7 +54,7 @@ compression_type=${COMPRESSION_TYPE:-snappy}
 duration=${DURATION:-0}
 
 num_keys=${NUM_KEYS:-$((1 * G))}
-key_size=20
+key_size=${KEY_SIZE:-20}
 value_size=${VALUE_SIZE:-400}
 block_size=${BLOCK_SIZE:-8192}
 

tools/db_bench.cc

@@ -19,5 +19,5 @@ int main() {
 }
 #else
 #include <rocksdb/db_bench_tool.h>
-int main(int argc, char** argv) { rocksdb::db_bench_tool(argc, argv); }
+int main(int argc, char** argv) { return rocksdb::db_bench_tool(argc, argv); }
 #endif  // GFLAGS

tools/db_bench_tool.cc

@@ -1915,8 +1915,8 @@ class Benchmark {
     if (!SanityCheck()) {
       exit(1);
     }
-    PrintHeader();
     Open(&open_options_);
+    PrintHeader();
     std::stringstream benchmark_stream(FLAGS_benchmarks);
     std::string name;
     while (std::getline(benchmark_stream, name, ',')) {

tools/regression_test.sh

@@ -0,0 +1,176 @@
+#!/bin/bash
+# REQUIRE: db_bench binary exists in the current directory
+
+function main {
+  commit=${1:-"origin/master"}
+  test_root_dir=${2:-"/tmp/rocksdb/regression_test"}
+
+  init_arguments
+
+  checkout_rocksdb $commit
+  build_db_bench
+  setup_test_directory
+
+  # an additional dot indicates we share same env variables
+  run_db_bench "fillseq" 0
+  run_db_bench "overwrite"
+  run_db_bench "readrandom"
+  run_db_bench "readwhilewriting"
+  run_db_bench "deleterandom"
+  run_db_bench "seekrandom"
+  run_db_bench "seekrandomwhilewriting"
+
+  echo ""
+  echo "Benchmark completed!  Results are available in $RESULT_PATH"
+}
+
+############################################################################
+function init_arguments {
+  K=1024
+  M=$((1024 * K))
+  G=$((1024 * M))
+
+  current_time=$(date +"%F-%H:%M:%S")
+  RESULT_PATH=${4:-"/tmp/rocksdb/regression/results/$current_time"}
+  COMMIT_ID=`git log | head -n1 | cut -c 8-`
+  SUMMARY_FILE="$RESULT_PATH/SUMMARY.csv"
+
+  DB_PATH=${2:-"/tmp/rocksdb/regression/db/"}
+  WAL_PATH=${3:-"/tmp/rocksdb/regression/wal/"}
+  NUM_THREADS=${NUM_THREADS:-16}
+  NUM_KEYS=${NUM_KEYS:-$((1 * G))}
+  KEY_SIZE=${KEY_SIZE:-100}
+  VALUE_SIZE=${VALUE_SIZE:-900}
+  CACHE_SIZE=${CACHE_SIZE:-$((1 * G))}
+  STATISTICS=${STATISTICS:-0}
+  COMPRESSION_RATIO=${COMPRESSION_RATIO:-0.5}
+  HISTOGRAM=${HISTOGRAM:-1}
+  STATS_PER_INTERVAL=${STATS_PER_INTERVAL:-1}
+  STATS_INTERVAL_SECONDS=${STATS_INTERVAL_SECONDS:-60}
+  MAX_BACKGROUND_FLUSHES=${MAX_BACKGROUND_FLUSHES:-4}
+  MAX_BACKGROUND_COMPACTIONS=${MAX_BACKGROUND_COMPACTIONS:-16} 
+  SEEK_NEXTS=${SEEK_NEXTS:-10}
+  SEED=${SEED:-$( date +%s )}
+}
+
+# $1 --- benchmark name
+# $2 --- use_existing_db (optional)
+function run_db_bench {
+  USE_EXISTING_DB=${2:-1}
+  echo ""
+  echo "======================================================================="
+  echo "Benchmark $1"
+  echo "======================================================================="
+  echo ""
+  db_bench_error=0
+  cmd="(./db_bench --benchmarks=$1 --db=$DB_PATH --wal_dir=$WAL_PATH \
+      --use_existing_db=$USE_EXISTING_DB \
+      --threads=$NUM_THREADS \
+      --num=$NUM_KEYS \
+      --key_size=$KEY_SIZE \
+      --value_size=$VALUE_SIZE \
+      --cache_size=$CACHE_SIZE \
+      --statistics=$STATISTICS \
+      --compression_ratio=$COMPRESSION_RATIO \
+      --histogram=$HISTOGRAM \
+      --seek_nexts=$SEEK_NEXTS \
+      --stats_per_interval=$STATS_PER_INTERVAL \
+      --stats_interval_seconds=$STATS_INTERVAL_SECONDS \
+      --max_background_flushes=$MAX_BACKGROUND_FLUSHES \
+      --max_background_compactions=$MAX_BACKGROUND_COMPACTIONS \
+      --seed=$SEED \
+      2>&1 || db_bench_error=1) | tee -a $RESULT_PATH/$1"
+  echo $cmd
+  eval $cmd
+  exit_on_error $db_bench_error
+
+  update_report "$1" "$RESULT_PATH/$1"
+}
+
+# $1 --- name of the benchmark
+# $2 --- the filename of the output log of db_bench
+function update_report {
+  main_result=`cat $2 | grep $1`
+  perc_statement=`cat $2 | grep Percentile`
+
+  # Obtain micros / op
+  main_pattern="$1"'[[:blank:]]+:[[:blank:]]+([0-9\.]+)[[:blank:]]+micros/op'
+  [[ $main_result =~ $main_pattern ]]
+  micros_op=${BASH_REMATCH[1]}
+
+  # Obtain percentile information
+  perc_pattern='Percentiles: P50: ([0-9\.]+) P75: ([0-9\.]+) P99: ([0-9\.]+) P99.9: ([0-9\.]+) P99.99: ([0-9\.]+)'
+  [[ $perc_statement =~ $perc_pattern ]]
+
+  perc[0]=${BASH_REMATCH[1]}  # p50
+  perc[1]=${BASH_REMATCH[2]}  # p75
+  perc[2]=${BASH_REMATCH[3]}  # p99
+  perc[3]=${BASH_REMATCH[4]}  # p99.9
+  perc[4]=${BASH_REMATCH[5]}  # p99.99
+
+  printf "$COMMIT_ID, %30s, %10.2f, %10.2f, %10.2f, %10.2f, %10.2f, %10.2f\n" \
+      $1 $micros_op ${perc[0]} ${perc[1]} ${perc[2]} ${perc[3]} ${perc[4]} \
+      >> $SUMMARY_FILE 
+  exit_on_error $?
+}
+
+function exit_on_error {
+  if [ $1 -ne 0 ]; then
+    echo ""
+    echo "ERROR: Benchmark did not complete successfully. " \
+         "Partial results are output to $RESULT_PATH"
+    echo "ERROR" >> $SUMMARY_FILE
+    exit $1
+  fi
+}
+
+function checkout_rocksdb {
+  echo "Checking out commit $1 ..."
+
+  git fetch --all
+  exit_on_error $?
+
+  git checkout $1
+  exit_on_error $?
+}
+
+function build_db_bench {
+  echo "Building db_bench ..."
+
+  make clean
+  exit_on_error $?
+
+  DEBUG_LEVEL=0 make db_bench -j32
+  exit_on_error $?
+}
+
+function setup_test_directory {
+  echo "Deleting old regression test directories and creating new ones"
+
+  rm -rf "$DB_PATH"
+  exit_on_error $?
+
+  rm -rf "$WAL_PATH"
+  exit_on_error $?
+
+  rm -rf "$RESULT_PATH"
+  exit_on_error $?
+
+  mkdir -p "$DB_PATH"
+  exit_on_error $?
+
+  mkdir -p "$WAL_PATH"
+  exit_on_error $?
+
+  mkdir -p "$RESULT_PATH"
+  exit_on_error $?
+
+  printf "%40s, %30s, %10s, %10s, %10s, %10s, %10s, %10s\n" \
+      "commit id" "benchmark" "ms-per-op" "p50" "p75" "p99" "p99.9" "p99.99" \
+       $micros_op ${perc[0]} ${perc[1]} ${perc[2]} ${perc[3]} ${perc[4]} \
+      >> $SUMMARY_FILE 
+}
+
+############################################################################
+
+main $@