GEODE-6517: Fix a race by counting down the latch. (apache#3297)

geode-core/src/main/java/org/apache/geode/internal/cache/PRHARedundancyProvider.java

@@ -1707,22 +1707,13 @@ public boolean recoverPersistentBuckets() {
     ArrayList<ProxyBucketRegion> bucketsHostedLocally =
         new ArrayList<ProxyBucketRegion>(proxyBucketArray.length);
 
+
     /*
      * Start the redundancy logger before recovering any proxy buckets.
      */
-    allBucketsRecoveredFromDisk = new CountDownLatch(proxyBucketArray.length);
-    try {
-      if (proxyBucketArray.length > 0) {
-        this.redundancyLogger = new RedundancyLogger(this);
-        Thread loggingThread = new LoggingThread(
-            "RedundancyLogger for region " + this.prRegion.getName(), false, this.redundancyLogger);
-        loggingThread.start();
-      }
-    } catch (RuntimeException e) {
-      allBucketsRecoveredFromDisk = null;
-      throw e;
-    }
+    startRedundancyLogger(proxyBucketArray.length);
 
+    allBucketsRecoveredFromDisk = new CountDownLatch(proxyBucketArray.length);
     /*
      * Spawn a separate thread for bucket that we previously hosted to recover that bucket.
      *
@@ -1793,6 +1784,15 @@ public void run2() {
     // }
   }
 
+  void startRedundancyLogger(int proxyBuckets) {
+    if (proxyBuckets > 0) {
+      redundancyLogger = new RedundancyLogger(this);
+      Thread loggingThread = new LoggingThread(
+          "RedundancyLogger for region " + this.prRegion.getName(), false, this.redundancyLogger);
+      loggingThread.start();
+    }
+  }
+
   /**
    * Check to see if any colocated region of the current region is persistent. It's not enough to
    * check just the leader region, because a child region might be a persistent parallel WAN queue,

geode-core/src/test/java/org/apache/geode/internal/cache/PRHARedundancyProviderTest.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.internal.cache;
+
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class PRHARedundancyProviderTest {
+  private PRHARedundancyProvider provider;
+
+  @Before
+  public void setup() {
+    PartitionedRegion partitionedRegion = mock(PartitionedRegion.class, RETURNS_DEEP_STUBS);
+    provider = spy(new PRHARedundancyProvider(partitionedRegion));
+  }
+
+  @Test
+  public void waitForPersistentBucketRecoveryProceedsWhenAllBucketsRecoveredFromDiskLatchIsNull() {
+    provider.waitForPersistentBucketRecovery();
+  }
+
+  @Test
+  public void waitForPersistentBucketRecoveryProceedsAfterLatchCountDown() throws Exception {
+    provider.allBucketsRecoveredFromDisk = spy(new CountDownLatch(1));
+    provider.allBucketsRecoveredFromDisk.countDown();
+
+    provider.waitForPersistentBucketRecovery();
+
+    verify(provider.allBucketsRecoveredFromDisk).await();
+  }
+}