From 13a3310d0ed4540f279b350f95d85bf4cf5caf68 Mon Sep 17 00:00:00 2001
From: Sivakumar Kalva <sivakk@uber.com>
Date: Wed, 8 Mar 2017 16:09:02 -0800
Subject: [PATCH] Script to check stress is clean. (#72)

* Script to check stress is clean.

* Add workflow executions check.

* Upgrade deploy stress script.
---
 scripts/check_stress_run.sh  | 50 ++++++++++++++++++++++++++++++++++++
 scripts/start_stress_test.sh | 33 +++++++++++++-----------
 2 files changed, 68 insertions(+), 15 deletions(-)
 create mode 100644 scripts/check_stress_run.sh

diff --git a/scripts/check_stress_run.sh b/scripts/check_stress_run.sh
new file mode 100644
index 00000000000..08846d59028
--- /dev/null
+++ b/scripts/check_stress_run.sh
@@ -0,0 +1,50 @@
+# the default cqlsh listen port is 9042
+port=9042
+host=compute2487-dca1.prod.uber.internal
+
+## Check workflow executions
+echo "Checking Workflow Executions that are not completed ...."
+
+ROWSLINE=$(cqlsh $host --cqlversion=3.4.0 -e "use cadence; select workflow_id, run_id from executions where type=1 LIMIT 10 allow filtering;" | grep "dcb940ac-0c63-ffa2-ffea-a6c305881d71" -c)
+if [ "$ROWSLINE" != "0" ]; then
+    echo "Warning: Left over workflow executions ..."
+    cqlsh $host --cqlversion=3.4.0 -e "use cadence; select workflow_id, run_id from executions where type=1 LIMIT 10 allow filtering;" | grep "dcb940ac-0c63-ffa2-ffea-a6c305881d71"
+else
+    echo "Workflow executions are clean"
+fi
+
+## Check transfer tasks
+echo "Checking transfer tasks that are not completed ...."
+
+ROWSLINE=$(cqlsh $host --cqlversion=3.4.0 -e "use cadence; select * from executions where type=2 LIMIT 10 allow filtering;" | grep "rows")
+if [ "$ROWSLINE" != "(0 rows)" ]; then
+    echo "Warning: Left over transfer tasks ..."
+    cqlsh $host --cqlversion=3.4.0 -e "use cadence; select count(*) from executions where type=2 LIMIT 10 allow filtering;"
+    cqlsh $host --cqlversion=3.4.0 -e "use cadence; select * from executions where type=2 LIMIT 10 allow filtering;"
+else
+    echo "Transfer tasks are clean"
+fi
+
+## Check timer tasks
+echo "Checking timer tasks that are not completed ...."
+
+ROWSLINE=$(cqlsh $host --cqlversion=3.4.0 -e "use cadence; select * from executions where type=3 LIMIT 10 allow filtering;" | grep "rows")
+if [ "$ROWSLINE" != "(0 rows)" ]; then
+    echo "Warning: Left over timers tasks..."
+    cqlsh $host --cqlversion=3.4.0 -e "use cadence; select count(*) from executions where type=3 LIMIT 10 allow filtering;"
+    cqlsh $host --cqlversion=3.4.0 -e "use cadence; select * from executions where type=3 LIMIT 10 allow filtering;"
+else
+    echo "Timer tasks are clean"
+fi
+
+## Check Task Queue
+echo "Checking task queue that are not completed ...."
+
+ROWSLINE=$(cqlsh $host --cqlversion=3.4.0 -e "use cadence; select * from tasks where type=0 LIMIT 10 allow filtering;" | grep "rows")
+if [ "$ROWSLINE" != "(0 rows)" ]; then
+    echo "Warning: Left over timers tasks..."
+    cqlsh $host --cqlversion=3.4.0 -e "use cadence; select count(*) from tasks where type=0 LIMIT 10 allow filtering;"
+    cqlsh $host --cqlversion=3.4.0 -e "use cadence; select * from tasks where type=0 LIMIT 10 allow filtering;"
+else
+    echo "Task queues are clean"
+fi
diff --git a/scripts/start_stress_test.sh b/scripts/start_stress_test.sh
index 346bd057557..9a9dee60c6e 100644
--- a/scripts/start_stress_test.sh
+++ b/scripts/start_stress_test.sh
@@ -11,24 +11,27 @@
 # the default cqlsh listen port is 9042
 port=9042
 
-# the default keyspace is workflow
+# the default keyspace is cadence
 # TODO: probably allow getting this from command line
-workflow_keyspace="workflow"
+workflow_keyspace="cadence"
 
 DROP_UBER_MINIONS="$HOME/uber-minions"
 DROP_CMD=$DROP_UBER_MINIONS/cmd/stress/
 STRESS_LOG=stress_output.log
 
-# cmd/stress/stress -emitMetric=m3 -host="10.185.19.27,10.184.45.6,10.185.27.8,10.185.17.12,10.185.15.30"
-for host in 10.185.19.27
-do
-  echo Installing schema on cassandra cluster via $host
-  cqlsh --cqlversion=3.4.2 -f ./schema/drop_keyspace_stress_test.cql $host $port
-  cqlsh --cqlversion=3.4.2 -f ./schema/keyspace_prod.cql $host $port
-  cqlsh --cqlversion=3.4.2 -k $workflow_keyspace -f ./schema/workflow_test.cql $host $port
-  cqlsh --cqlversion=3.4.2 -k $workflow_keyspace -f ./schema/stress_schema_setup.cql $host $port
-done
-
-echo "Starting stress on"
-cd $DROP_CMD
-nohup ./stress -emitMetric=m3 -host="10.185.19.27,10.184.45.6,10.185.27.8,10.185.17.12,10.185.15.30" > $STRESS_LOG 2>&1 &
+# This is our managed cassandra cluster.
+host=compute2487-dca1.prod.uber.internal
+
+echo Installing schema on managed cassandra cluster via $host
+
+cqlsh --cqlversion=3.4.0 -f ./schema/drop_keyspace_stress_test.cql $host $port
+cqlsh --cqlversion=3.4.0 -f ./schema/keyspace_prod.cql $host $port
+cqlsh --cqlversion=3.4.0 -k $workflow_keyspace -f ./schema/workflow_test.cql $host $port
+cqlsh --cqlversion=3.4.0 -k $workflow_keyspace -f ./schema/stress_schema_setup.cql $host $port
+
+## We restart the stress by restarting the stress role on the uber-cadence.
+
+# The below steps are for kicking off separately on cherami boxes.
+#echo "Starting stress on"
+#cd $DROP_CMD
+#nohup ./stress -emitMetric=m3 -host="10.185.19.27,10.184.45.6,10.185.27.8,10.185.17.12,10.185.15.30" > $STRESS_LOG 2>&1 &