Add txn manager debug statements

cluster/tools.py

@@ -7,6 +7,7 @@
 import util
 from routing_util import register, deregister
 import subprocess
+import time
 
 # AWS Info
 aws_key_id = util.check_or_get_env_arg('AWS_ACCESS_KEY_ID')
@@ -70,7 +71,16 @@ def restart(pod_ip, kind):
         pod_ips = util.get_pod_ips(client, selector='role='+kind, is_running=True)
 
     # Send config file to the pod
-    sendConfig(pod_ip, None)
+    retry = 0
+    while retry < 5:
+        try:
+            sendConfig(pod_ip, None)
+        except Exception as e:
+            retry += 1
+            print('Caught exception: ' + e)
+            print('Retrying in %d sec' % retry * 5)
+            time.sleep(retry * 5)
+
 
     print('Restarted %s node at %s' % (kind, pod_ip))
 

cmd/manager/main.go

@@ -353,7 +353,7 @@ func (t *TxnManager) collectEndTxnResponses(endRespChan chan *tpb.TransactionTag
 }
 
 func (t *TxnManager) writeToStorage(tid string, endTs int64, entry *WriteBufferEntry, writeChan chan bool) {
-	go t.Monitor.TrackFuncExecTime(tid, "Write to Storage time", time.Now())
+	defer t.Monitor.TrackFuncExecTime(tid, "Write to Storage time", time.Now())
 
 	dbKeys := make([]string, len(entry.buffer)+1)
 	dbVals := make([][]byte, len(entry.buffer)+1)
@@ -372,7 +372,9 @@ func (t *TxnManager) writeToStorage(tid string, endTs int64, entry *WriteBufferE
 	dbKeys[i] = tid
 	dbVals[i] = data
 
+	log.Debugf("Sent storage write for txn %s", tid)
 	t.StorageManager.MultiPut(dbKeys, dbVals)
+	log.Debugf("Storage returned for txn %s", tid)
 	writeChan <- true
 }