[java client] Retry regressing counts in ITClient

There's currently no sure way to read your writes, even with snapshot scans, so we
can either retry counting rows or ignore it. This patch is doing the former, unless
we hit some artificial timeout.

Change-Id: I1e79a6c7aaf069294a6ca40e487947d14d9f2aa7
Reviewed-on: http://gerrit.cloudera.org:8080/4597
Reviewed-by: Adar Dembo <[email protected]>
Tested-by: Kudu Jenkins

java/kudu-client/src/test/java/org/apache/kudu/client/ITClient.java

@@ -380,21 +380,35 @@ private boolean randomGet() {
      * @return true if the full scan was successful, false if there was an error
      */
     private boolean fullScan() {
-      KuduScanner scanner = getScannerBuilder().build();
-      try {
-        int rowCount = countRowsInScan(scanner);
-        if (rowCount < lastRowCount) {
-          reportError("Row count regressed: " + rowCount + " < " + lastRowCount, null);
-          return false;
+      int rowCount;
+      DeadlineTracker deadlineTracker = new DeadlineTracker();
+      deadlineTracker.setDeadline(DEFAULT_SLEEP);
+
+      while (KEEP_RUNNING_LATCH.getCount() > 0 && !deadlineTracker.timedOut()) {
+        KuduScanner scanner = getScannerBuilder().build();
+
+        try {
+          rowCount = countRowsInScan(scanner);
+        } catch (KuduException e) {
+          return checkAndReportError("Got error while row counting", e);
+        }
+
+        if (rowCount >= lastRowCount) {
+          if (rowCount > lastRowCount) {
+            lastRowCount = rowCount;
+            LOG.info("New row count {}", lastRowCount);
+          }
+          return true;
         }
-        if (rowCount > lastRowCount) {
-          lastRowCount = rowCount;
-          LOG.info("New row count {}", lastRowCount);
+
+        // Due to the lack of KUDU-430, we need to loop until the row count stops regressing.
+        try {
+          KEEP_RUNNING_LATCH.await(50, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+          // No need to do anything, we'll exit the loop once we test getCount() in the condition.
         }
-      } catch (KuduException e) {
-        return checkAndReportError("Got error while row counting", e);
       }
-      return true;
+      return !deadlineTracker.timedOut();
     }
 
     private KuduScanner.KuduScannerBuilder getScannerBuilder() {