From 84e911cb22097319aa1130fdd02a57d0354c9cc4 Mon Sep 17 00:00:00 2001 From: Patrick Rhomberg Date: Thu, 12 Jul 2018 12:36:37 -0700 Subject: [PATCH] GEODE-5295: Improve member WaitUntilX methods (#2039) * Improve error reporting in the event of a WaitUntil timeout. * Renamed methods to avoid future developer error similar to GEODE-5299 --- .../JMXMBeanReconnectDUnitTest.java | 6 +- .../internal/JMXMBeanFederationDUnitTest.java | 6 +- .../internal/cli/CliUtilDUnitTest.java | 12 +- .../AlterAsyncEventQueueCommandDUnitTest.java | 2 +- .../commands/AlterCompressorDUnitTest.java | 2 +- ...CreateAsyncEventQueueCommandDUnitTest.java | 2 +- .../commands/CreateIndexCommandDUnitTest.java | 2 +- .../CreateRegionCommandDUnitTest.java | 6 +- .../cli/commands/DescribeRegionDUnitTest.java | 2 +- ...estroyAsyncEventQueueCommandDUnitTest.java | 16 +- .../DestroyRegionCommandDUnitTest.java | 8 +- .../commands/DiskStoreCommandsDUnitTest.java | 6 +- .../ExecuteFunctionCommandDUnitTest.java | 2 +- .../ListAsyncEventQueuesCommandDUnitTest.java | 4 +- .../cli/commands/LocateEntryDUnitTest.java | 6 +- .../cli/commands/RemoveCommandDUnitTest.java | 4 +- .../ClusterConfigImportDUnitTest.java | 2 +- .../geode/test/dunit/rules/MemberVM.java | 23 +-- .../MemberStarterRuleAwaitDUnitTest.java | 141 ++++++++++++++++ ...MemberStarterRuleAwaitIntegrationTest.java | 112 +++++++++++++ .../test/junit/rules/MemberStarterRule.java | 150 +++++++++++++++--- .../DestroyGatewaySenderCommandDUnitTest.java | 8 +- .../CreateRegionCommandDUnitTest.java | 4 +- .../cli/commands/DescribeRegionDUnitTest.java | 4 +- 24 files changed, 446 insertions(+), 84 deletions(-) create mode 100644 geode-core/src/test/java/org/apache/geode/test/dunit/rules/tests/MemberStarterRuleAwaitDUnitTest.java create mode 100644 geode-core/src/test/java/org/apache/geode/test/dunit/rules/tests/MemberStarterRuleAwaitIntegrationTest.java diff --git a/geode-core/src/test/java/org/apache/geode/management/JMXMBeanReconnectDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/JMXMBeanReconnectDUnitTest.java index 6e19a9c2c03d..bc3a6d59bdcc 100644 --- a/geode-core/src/test/java/org/apache/geode/management/JMXMBeanReconnectDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/JMXMBeanReconnectDUnitTest.java @@ -94,7 +94,7 @@ public void before() throws Exception { "create region --type=REPLICATE --name=" + REGION_PATH + " --enable-statistics=true") .statusIsSuccess(); - locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT); + locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT); waitForLocatorsToAgreeOnMembership(); } @@ -129,7 +129,7 @@ public void testLocalBeans_MaintainLocatorAndCrashServer() { .containsExactlyElementsOf(initialLocatorBeans); server1.waitTilServerFullyReconnected(); - locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT); + locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT); List finalLocatorBeans = canonicalBeanNamesFor(locator1); @@ -206,7 +206,7 @@ public void testRemoteBeanKnowledge_MaintainLocatorAndCrashServer() }); server1.waitTilServerFullyReconnected(); - locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT); + locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT); waitForMBeanFederationFrom(numServerMBeans, server1); waitForLocatorsToAgreeOnMembership(); diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/JMXMBeanFederationDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/JMXMBeanFederationDUnitTest.java index e08975d50597..9e078c31c131 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/JMXMBeanFederationDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/JMXMBeanFederationDUnitTest.java @@ -98,7 +98,7 @@ public void before() throws Exception { .statusIsSuccess(); gfsh.disconnect(); - locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT); + locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT); bb = InternalBlackboardImpl.getInstance(); } @@ -109,7 +109,7 @@ public void MBeanFederationAddRemoveServer() throws IOException { server3 = lsRule.startServerVM(SERVER_3_VM_INDEX, locator1.getPort()); SERVER_COUNT++; - locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT); + locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT); List keyset = server3.invoke(() -> { InternalCache cache = ClusterStartupRule.getCache(); DistributedMember member = @@ -131,7 +131,7 @@ public void MBeanFederationAddRemoveServer() throws IOException { lsRule.stopMember(SERVER_3_VM_INDEX); SERVER_COUNT--; - locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT); + locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT); List finalMBeans = getFederatedGemfireBeansFrom(locator1); diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/CliUtilDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/CliUtilDUnitTest.java index 8c1bf9c4ed50..70e7d6db8395 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/CliUtilDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/CliUtilDUnitTest.java @@ -80,9 +80,9 @@ public static void setUpCluster() throws Exception { gfsh.executeAndAssertThat("create region --name=group2Region --group=group2 --type=REPLICATE") .statusIsSuccess(); - locator.waitTillRegionsAreReadyOnServers("/commonRegion", 4); - locator.waitTillRegionsAreReadyOnServers("/group1Region", 2); - locator.waitTillRegionsAreReadyOnServers("/group2Region", 2); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/commonRegion", 4); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/group1Region", 2); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/group2Region", 2); } @Test @@ -202,9 +202,9 @@ public void getMembersWithQueueId() throws Exception { "create async-event-queue --id=queue --listener=" + MyAsyncEventListener.class.getName()) .statusIsSuccess(); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 2); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue2", 2); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue", 4); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 2); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue2", 2); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue", 4); locator.invoke(() -> { members = CliUtil.getMembersWithAsyncEventQueue(ClusterStartupRule.getCache(), "queue1"); diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterAsyncEventQueueCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterAsyncEventQueueCommandDUnitTest.java index 8f8e12c578f3..fcd634575a47 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterAsyncEventQueueCommandDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterAsyncEventQueueCommandDUnitTest.java @@ -57,7 +57,7 @@ public void testAlterAsyncEventQueue() throws Exception { gfsh.executeAndAssertThat("create async-event-queue --id=queue1 --group=group1 --listener=" + MyAsyncEventListener.class.getName()).statusIsSuccess(); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1); // verify that server1's event queue has the default value server1.invoke(() -> { diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterCompressorDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterCompressorDUnitTest.java index f82b0ca71ddc..d116ab7f2d7a 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterCompressorDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterCompressorDUnitTest.java @@ -70,7 +70,7 @@ public static void setupCluster() throws Exception { gfsh.executeAndAssertThat( "create disk-store --name=diskStore --groups=dataStore --dir=diskStore").statusIsSuccess(); - locator.waitTillDiskstoreIsReady("diskStore", 2); + locator.waitUntilDiskStoreIsReadyOnExactlyThisManyServers("diskStore", 2); // create regions gfsh.executeAndAssertThat( "create region --name=testRegion --type=REPLICATE_PERSISTENT --group=dataStore --disk-store=diskStore") diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateAsyncEventQueueCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateAsyncEventQueueCommandDUnitTest.java index dd642df10b9c..dfeb672781b2 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateAsyncEventQueueCommandDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateAsyncEventQueueCommandDUnitTest.java @@ -94,7 +94,7 @@ public void create_sync_event_queue() throws Exception { " java.lang.IllegalStateException: A GatewaySender with id AsyncEventQueue_queue is already defined in this cache."); gfsh.executeAndAssertThat("create disk-store --name=diskStore2 --dir=diskstore"); - locator.waitTillDiskstoreIsReady("diskStore2", 2); + locator.waitUntilDiskStoreIsReadyOnExactlyThisManyServers("diskStore2", 2); // create another queue with different configuration gfsh.executeAndAssertThat(VALID_COMMAND + " --id=queue2 --group=group2 " diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateIndexCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateIndexCommandDUnitTest.java index b8cd1e5d6ad2..d01c66307c42 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateIndexCommandDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateIndexCommandDUnitTest.java @@ -108,7 +108,7 @@ public void regionExistOnServerButNotInClusterConfig() { @Test public void regionExistInClusterConfig() { gfsh.executeAndAssertThat("create region --name=regionB --type=REPLICATE").statusIsSuccess(); - locator.waitTillRegionsAreReadyOnServers("/regionB", 1); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/regionB", 1); locator.invoke(() -> { InternalConfigurationPersistenceService configurationService = ClusterStartupRule.getLocator().getConfigurationPersistenceService(); diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateRegionCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateRegionCommandDUnitTest.java index ef9893eec53e..c7cd66ee08a7 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateRegionCommandDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateRegionCommandDUnitTest.java @@ -358,7 +358,7 @@ public void startWithNonProxyRegion() { "create region --type=REPLICATE_PROXY --group=group2 --name=" + regionName) .statusIsSuccess().tableHasRowWithValues("Member", "server-2"); - locator.waitTillRegionsAreReadyOnServers("/" + regionName, 2); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + regionName, 2); gfsh.executeAndAssertThat( "create region --type=PARTITION_PROXY --group=group2 --name=" + regionName).statusIsError() @@ -374,7 +374,7 @@ public void startWithReplicateProxyRegion() { gfsh.executeAndAssertThat("create region --type=REPLICATE --group=group2 --name=" + regionName) .statusIsSuccess().tableHasRowWithValues("Member", "server-2"); - locator.waitTillRegionsAreReadyOnServers("/" + regionName, 2); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + regionName, 2); // the following two should fail with name check on locator, not on server gfsh.executeAndAssertThat("create region --type=PARTITION --group=group2 --name=" + regionName) .statusIsError().containsOutput("Region /" + regionName + " already exists on the cluster"); @@ -487,7 +487,7 @@ public void startWithPartitionProxyRegion() { gfsh.executeAndAssertThat("create region --type=PARTITION --group=group2 --name=" + regionName) .statusIsSuccess().tableHasRowWithValues("Member", "server-2"); - locator.waitTillRegionsAreReadyOnServers("/" + regionName, 2); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + regionName, 2); gfsh.executeAndAssertThat("create region --type=PARTITION --group=group2 --name=" + regionName) .statusIsError().containsOutput("Region /" + regionName + " already exists on the cluster"); gfsh.executeAndAssertThat( diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DescribeRegionDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DescribeRegionDUnitTest.java index f47cac70f489..ee33ab815ff2 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DescribeRegionDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DescribeRegionDUnitTest.java @@ -143,7 +143,7 @@ public static void setupSystem() throws Exception { gfsh.executeAndAssertThat("create async-event-queue --id=queue1 --group=group1 " + "--listener=org.apache.geode.internal.cache.wan.MyAsyncEventListener").statusIsSuccess(); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1); gfsh.executeAndAssertThat( "create region --name=region4 --type=REPLICATE --async-event-queue-id=queue1") .statusIsSuccess(); diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyAsyncEventQueueCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyAsyncEventQueueCommandDUnitTest.java index b7d2b19124d8..b060c0537fdc 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyAsyncEventQueueCommandDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyAsyncEventQueueCommandDUnitTest.java @@ -59,7 +59,7 @@ public void destroyAeq_returnsSuccess() { "create async-event-queue --id=queue1 --listener=" + MyAsyncEventListener.class.getName()) .statusIsSuccess(); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 2); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 2); gfsh.executeAndAssertThat("list async-event-queues").statusIsSuccess(); locator.invoke(() -> { @@ -88,7 +88,7 @@ public void destroyAeqWhenQueueDoesNotExist_deafultReturnsError() { "create async-event-queue --id=queue1 --listener=" + MyAsyncEventListener.class.getName()) .statusIsSuccess(); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 2); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 2); gfsh.executeAndAssertThat("list async-event-queues").statusIsSuccess(); gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1 ").statusIsSuccess(); @@ -103,7 +103,7 @@ public void destroyAeqWhenQueueDoesNotExist_withIfExistsReturnsSuccess() { "create async-event-queue --id=queue1 --listener=" + MyAsyncEventListener.class.getName()) .statusIsSuccess(); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 2); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 2); gfsh.executeAndAssertThat("list async-event-queues").statusIsSuccess(); gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1 ").statusIsSuccess(); @@ -126,7 +126,7 @@ public void destroyAeqOnGroup_returnsSuccess() { gfsh.executeAndAssertThat("create async-event-queue --id=queue1 --group=group1 --listener=" + MyAsyncEventListener.class.getName()).statusIsSuccess(); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1); gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1 --group=group1") .statusIsSuccess(); @@ -147,7 +147,7 @@ public void destroyAeqOnGroupThatDoesNotExisit_returnsError() { gfsh.executeAndAssertThat("create async-event-queue --id=queue1 --group=group1 --listener=" + MyAsyncEventListener.class.getName()).statusIsSuccess(); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1); gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1 --group=group2") .statusIsError().containsOutput(CliStrings.NO_MEMBERS_FOUND_MESSAGE); @@ -168,7 +168,7 @@ public void destroyAeq_selectsQueuesOnGroup_showsErrorForServersNotInGroup() gfsh.executeAndAssertThat("create async-event-queue --id=queue1 --group=group1 --listener=" + MyAsyncEventListener.class.getName()).statusIsSuccess(); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1); gfsh.executeAndAssertThat("list async-event-queues").statusIsSuccess(); gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1").statusIsSuccess() @@ -193,8 +193,8 @@ public void destroyAeq_selectsQueuesByGroup_returnsSuccess() throws GfJsonExcept gfsh.executeAndAssertThat("create async-event-queue --id=queue3 --group=group3 --listener=" + MyAsyncEventListener.class.getName())/* .statusIsSuccess() */; - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue3", 1); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue3", 1); gfsh.executeAndAssertThat("list async-event-queues").statusIsSuccess(); gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1 --group=group1") diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyRegionCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyRegionCommandDUnitTest.java index c0cb3de5d832..68cac15298d0 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyRegionCommandDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyRegionCommandDUnitTest.java @@ -67,8 +67,8 @@ public void testDestroyDistributedRegion() { gfsh.executeAndAssertThat( "create region --name=Order --type=PARTITION --colocated-with=Customer").statusIsSuccess(); - locator.waitTillRegionsAreReadyOnServers("/Customer", 3); - locator.waitTillRegionsAreReadyOnServers("/Order", 3); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/Customer", 3); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/Order", 3); // Test unable to destroy with co-location gfsh.executeAndAssertThat("destroy region --name=/Customer").statusIsError() @@ -92,7 +92,7 @@ public void testDestroyDistributedRegion() { public void testDestroyLocalRegions() { gfsh.executeAndAssertThat("create region --name=region1 --type=LOCAL").statusIsSuccess(); - locator.waitTillRegionsAreReadyOnServers("/region1", 3); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/region1", 3); gfsh.executeAndAssertThat("destroy region --name=region1").statusIsSuccess() .tableHasRowCount("Member", 3).containsOutput("destroyed successfully"); @@ -110,7 +110,7 @@ public void testDestroyDistributedRegions() { gfsh.executeAndAssertThat("create region --name=region1 --type=REPLICATE --group=group2") .statusIsSuccess(); - locator.waitTillRegionsAreReadyOnServers("/region1", 3); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/region1", 3); locator.invoke(() -> { InternalConfigurationPersistenceService service = diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DiskStoreCommandsDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DiskStoreCommandsDUnitTest.java index 22384207da41..04bd40888eba 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DiskStoreCommandsDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DiskStoreCommandsDUnitTest.java @@ -71,7 +71,7 @@ private void createDiskStoreAndRegion(MemberVM jmxManager, int serverCount) { gfsh.executeAndAssertThat("list disk-stores").statusIsSuccess() .tableHasColumnWithValuesContaining("Disk Store Name", diskStores.toArray(new String[0])); - jmxManager.waitTillDiskstoreIsReady(DISKSTORE, serverCount); + jmxManager.waitUntilDiskStoreIsReadyOnExactlyThisManyServers(DISKSTORE, serverCount); gfsh.executeAndAssertThat(String.format( "create region --name=%s --type=REPLICATE_PERSISTENT --disk-store=%s --group=%s --eviction-action=overflow-to-disk", @@ -119,7 +119,7 @@ public void testMissingDiskStore() throws Exception { serverRule.before(); }); - locator.waitTillDiskstoreIsReady(DISKSTORE, 1); + locator.waitUntilDiskStoreIsReadyOnExactlyThisManyServers(DISKSTORE, 1); gfsh.executeAndAssertThat("show missing-disk-stores").statusIsSuccess() .containsOutput("Missing Disk Stores", "No missing colocated region found"); @@ -130,7 +130,7 @@ public void testMissingDiskStore() throws Exception { gfsh.executeAndAssertThat("revoke missing-disk-store --id=" + diskstoreIDs.get(0)) .statusIsSuccess().containsOutput("Missing disk store successfully revoked"); - locator.waitTillRegionsAreReadyOnServers("/" + REGION_1, 1); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + REGION_1, 1); server1.invoke(() -> { Cache cache = ClusterStartupRule.getCache(); diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExecuteFunctionCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExecuteFunctionCommandDUnitTest.java index ca887051cb99..dc4d4838da00 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExecuteFunctionCommandDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ExecuteFunctionCommandDUnitTest.java @@ -71,7 +71,7 @@ public static void setUpClass() throws Exception { .statusIsSuccess() .tableHasColumnOnlyWithValues("Member", "server-1", "server-2"); - locator.waitTillRegionsAreReadyOnServers("/regionA", 2); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/regionA", 2); server1.invoke(() -> { Region region = ClusterStartupRule.getCache().getRegion("/regionA"); diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ListAsyncEventQueuesCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ListAsyncEventQueuesCommandDUnitTest.java index 7c6844deb050..bd040a5cb989 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ListAsyncEventQueuesCommandDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ListAsyncEventQueuesCommandDUnitTest.java @@ -61,8 +61,8 @@ public void list() { gfsh.executeAndAssertThat("create async-event-queue --id=queue2 --group=group2 --listener=" + MyAsyncEventListener.class.getName()).statusIsSuccess(); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1); - locator.waitTillAsyncEventQueuesAreReadyOnServers("queue2", 1); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue2", 1); gfsh.executeAndAssertThat("list async-event-queue").statusIsSuccess() .tableHasRowCount("Member", 2).tableHasRowWithValues("Member", "ID", "server-1", "queue1") diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/LocateEntryDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/LocateEntryDUnitTest.java index 3feccde9f8f1..3899da1548e9 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/LocateEntryDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/LocateEntryDUnitTest.java @@ -74,9 +74,9 @@ public static void beforeClass() throws Exception { gfsh.executeAndAssertThat("put --region=regionB/regionBB --key=key --value=value") .statusIsSuccess(); - locator.waitTillRegionsAreReadyOnServers("/regionA", 2); - locator.waitTillRegionsAreReadyOnServers("/regionB", 2); - locator.waitTillRegionsAreReadyOnServers("/regionB/regionBB", 2); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/regionA", 2); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/regionB", 2); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/regionB/regionBB", 2); } @Test diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/RemoveCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/RemoveCommandDUnitTest.java index 969746045217..e09771b5416c 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/RemoveCommandDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/RemoveCommandDUnitTest.java @@ -59,8 +59,8 @@ public void setup() throws Exception { gfsh.executeAndAssertThat( "create region --name=" + PARTITIONED_REGION_NAME + " --type=PARTITION").statusIsSuccess(); - locator.waitTillRegionsAreReadyOnServers("/" + REPLICATE_REGION_NAME, 2); - locator.waitTillRegionsAreReadyOnServers("/" + PARTITIONED_REGION_NAME, 2); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + REPLICATE_REGION_NAME, 2); + locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + PARTITIONED_REGION_NAME, 2); VMProvider.invokeInEveryMember(RemoveCommandDUnitTest::populateTestRegions, server1, server2); } diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigImportDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigImportDUnitTest.java index 717b1e93aa82..5074aa4018c3 100644 --- a/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigImportDUnitTest.java +++ b/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigImportDUnitTest.java @@ -108,7 +108,7 @@ public void importFailWithExistingDiskStore() { lsRule.startServerVM(1, locatorVM.getPort()); gfshConnector.executeAndAssertThat("create disk-store --name=diskStore1 --dir=testStore") .statusIsSuccess(); - locatorVM.waitTillDiskstoreIsReady("diskStore1", 1); + locatorVM.waitUntilDiskStoreIsReadyOnExactlyThisManyServers("diskStore1", 1); gfshConnector .executeAndAssertThat( "import cluster-configuration --zip-file-name=" + clusterConfigZipPath) diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/rules/MemberVM.java b/geode-core/src/test/java/org/apache/geode/test/dunit/rules/MemberVM.java index e383be104b50..f1aaf81f283f 100644 --- a/geode-core/src/test/java/org/apache/geode/test/dunit/rules/MemberVM.java +++ b/geode-core/src/test/java/org/apache/geode/test/dunit/rules/MemberVM.java @@ -161,24 +161,27 @@ public void waitTilServerFullyReconnected() { /** * this should called on a locatorVM or a serverVM with jmxManager enabled */ - public void waitTillRegionsAreReadyOnServers(String regionPath, int serverCount) { - vm.invoke(() -> ClusterStartupRule.memberStarter.waitTillRegionIsReadyOnServers(regionPath, - serverCount)); + public void waitUntilRegionIsReadyOnExactlyThisManyServers(String regionPath, int serverCount) { + vm.invoke(() -> ClusterStartupRule.memberStarter + .waitUntilRegionIsReadyOnExactlyThisManyServers(regionPath, serverCount)); } - public void waitTillDiskstoreIsReady(String diskstoreName, int serverCount) { - vm.invoke(() -> ClusterStartupRule.memberStarter.waitTillDiskStoreIsReady(diskstoreName, - serverCount)); + public void waitUntilDiskStoreIsReadyOnExactlyThisManyServers(String diskstoreName, + int serverCount) { + vm.invoke(() -> ClusterStartupRule.memberStarter + .waitUntilDiskStoreIsReadyOnExactlyThisManyServers(diskstoreName, serverCount)); } - public void waitTillAsyncEventQueuesAreReadyOnServers(String queueId, int serverCount) { + public void waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers(String queueId, + int serverCount) { vm.invoke(() -> ClusterStartupRule.memberStarter - .waitTillAsyncEventQueuesAreReadyOnServers(queueId, serverCount)); + .waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers(queueId, serverCount)); } - public void waitTilGatewaySendersAreReady(int expectedGatewayObjectCount) { + public void waitUntilGatewaySendersAreReadyOnExactlyThisManyServers( + int expectedGatewayObjectCount) { vm.invoke(() -> ClusterStartupRule.memberStarter - .waitTilGatewaySendersAreReady(expectedGatewayObjectCount)); + .waitUntilGatewaySendersAreReadyOnExactlyThisManyServers(expectedGatewayObjectCount)); } } diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/rules/tests/MemberStarterRuleAwaitDUnitTest.java b/geode-core/src/test/java/org/apache/geode/test/dunit/rules/tests/MemberStarterRuleAwaitDUnitTest.java new file mode 100644 index 000000000000..ee1458514041 --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/test/dunit/rules/tests/MemberStarterRuleAwaitDUnitTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.geode.test.dunit.rules.tests; + +import static org.assertj.core.api.Java6Assertions.assertThatThrownBy; + +import java.util.Arrays; +import java.util.Collection; + +import org.awaitility.core.ConditionTimeoutException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import org.apache.geode.cache.DiskStore; +import org.apache.geode.cache.RegionShortcut; +import org.apache.geode.test.dunit.rules.ClusterStartupRule; +import org.apache.geode.test.dunit.rules.MemberVM; +import org.apache.geode.test.junit.categories.DistributedTest; +import org.apache.geode.test.junit.rules.MemberStarterRule; +import org.apache.geode.test.junit.runners.CategoryWithParameterizedRunnerFactory; + +@Category(DistributedTest.class) +@RunWith(Parameterized.class) +@Parameterized.UseParametersRunnerFactory(CategoryWithParameterizedRunnerFactory.class) +public class MemberStarterRuleAwaitDUnitTest { + + @ClassRule + public static ClusterStartupRule csRule = new ClusterStartupRule(2); + + // Name snooped in server VM below. At time of writing, should be "DEFAULT" + private static String existingDiskStoreName; + private static String existingRegionName = "existingRegion"; + + private static MemberVM locator, server; + private MemberVM memberToTest; + + @Before + public void before() { + memberToTest = memberTypeToTest.equals("locator") ? locator : server; + } + + @Parameter + public String memberTypeToTest; + + @Parameters(name = "{index}: Using {0} VM") + public static Collection useBothRules() { + return Arrays.asList("locator", "server"); + } + + @BeforeClass + public static void beforeClass() { + locator = csRule.startLocatorVM(0); + int locatorPort = locator.getPort(); + server = csRule.startServerVM(1, member -> member.withJMXManager() + .withConnectionToLocator(locatorPort) + .withRegion(RegionShortcut.PARTITION_PERSISTENT, existingRegionName)); + + existingDiskStoreName = server.invoke(() -> { + DiskStore anExistingDiskStore = + (DiskStore) ClusterStartupRule.getCache().listDiskStores().toArray()[0]; + return anExistingDiskStore.getName(); + }); + + // Override the default 30 second timeout to something lower for the scope of this test + locator.invoke(() -> MemberStarterRule.setWaitUntilTimeout(3)); + server.invoke(() -> MemberStarterRule.setWaitUntilTimeout(3)); + } + + @Test + public void nonexistingRegionTimeout() { + assertThatThrownBy( + () -> memberToTest.waitUntilRegionIsReadyOnExactlyThisManyServers("nonexistentRegion", 3)) + .hasCauseInstanceOf(ConditionTimeoutException.class) + .hasStackTraceContaining("Expecting to find an mbean for region 'nonexistentRegion'"); + } + + @Test + public void existingRegionTimeout() { + assertThatThrownBy( + () -> memberToTest.waitUntilRegionIsReadyOnExactlyThisManyServers(existingRegionName, 3)) + .hasCauseInstanceOf(ConditionTimeoutException.class) + .hasStackTraceContaining( + "Expecting to find an mbean for region '" + existingRegionName); + } + + @Test + public void nonexistingQueueTimeout() { + assertThatThrownBy(() -> memberToTest + .waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("badQueueId", 3)) + .hasCauseInstanceOf(ConditionTimeoutException.class) + .hasStackTraceContaining( + "Expecting exactly 3 servers to have an AEQ with id 'badQueueId'."); + } + + @Test + public void nonexistingDiskStoreTimeout() { + assertThatThrownBy( + () -> memberToTest.waitUntilDiskStoreIsReadyOnExactlyThisManyServers("badDiskName", 3)) + .hasCauseInstanceOf(ConditionTimeoutException.class) + .hasStackTraceContaining( + "Expecting exactly 3 servers to present mbeans for a disk store with name badDiskName"); + } + + @Test + public void existingDiskStoreTimeout() { + assertThatThrownBy(() -> memberToTest + .waitUntilDiskStoreIsReadyOnExactlyThisManyServers(existingDiskStoreName, 3)) + .hasCauseInstanceOf(ConditionTimeoutException.class) + .hasStackTraceContaining( + "Expecting exactly 3 servers to present mbeans for a disk store with name " + + existingDiskStoreName); + } + + @Test + public void nonexistingGatewayTimeout() { + assertThatThrownBy( + () -> memberToTest.waitUntilGatewaySendersAreReadyOnExactlyThisManyServers(3)) + .hasCauseInstanceOf(ConditionTimeoutException.class) + .hasStackTraceContaining("Expecting to find exactly 3 gateway sender beans"); + } +} diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/rules/tests/MemberStarterRuleAwaitIntegrationTest.java b/geode-core/src/test/java/org/apache/geode/test/dunit/rules/tests/MemberStarterRuleAwaitIntegrationTest.java new file mode 100644 index 000000000000..6a9f8a98e800 --- /dev/null +++ b/geode-core/src/test/java/org/apache/geode/test/dunit/rules/tests/MemberStarterRuleAwaitIntegrationTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.geode.test.dunit.rules.tests; + +import static org.assertj.core.api.Java6Assertions.assertThatThrownBy; + +import java.util.Arrays; +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.function.UnaryOperator; + +import com.sun.tools.javac.util.List; +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.awaitility.core.ConditionTimeoutException; +import org.awaitility.core.Predicate; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import org.apache.geode.test.junit.categories.IntegrationTest; +import org.apache.geode.test.junit.rules.LocatorStarterRule; +import org.apache.geode.test.junit.rules.MemberStarterRule; +import org.apache.geode.test.junit.rules.ServerStarterRule; +import org.apache.geode.test.junit.runners.CategoryWithParameterizedRunnerFactory; + +@Category(IntegrationTest.class) +@RunWith(Parameterized.class) +@Parameterized.UseParametersRunnerFactory(CategoryWithParameterizedRunnerFactory.class) +public class MemberStarterRuleAwaitIntegrationTest { + + private static MemberStarterRule locatorStarterRule = new LocatorStarterRule(); + private static MemberStarterRule serverStarterRule = new ServerStarterRule(); + + @Parameter(0) + public MemberStarterRule ruleToUse; + + @Parameter(1) + public String ruleToUseAsString; + + @Parameters(name = "{index}: Testing {1}") + public static Collection useBothRules() { + return Arrays.asList( + new Object[][] { + {locatorStarterRule, locatorStarterRule.getClass().getSimpleName()}, + {serverStarterRule, serverStarterRule.getClass().getSimpleName()}}); + } + + @Test + public void testWithDefaultPresentation() throws Exception { + Supplier alwaysFalseProvider = () -> false; + String description = "Awaiting until boolean becomes true."; + + assertThatThrownBy(printExceptionWrapper(() -> ruleToUse.waitUntilEqual(alwaysFalseProvider, + UnaryOperator.identity(), true, description, 1, TimeUnit.SECONDS))) + .isInstanceOf(ConditionTimeoutException.class) + .hasMessageContaining("false") + .hasMessageContaining(description); + } + + @Test + public void waitCanAcceptNullsIfPredicateAcceptsNulls() throws Exception { + Supplier alwaysNullProvider = () -> null; + Predicate booleanIdentityPredicate = b -> b != null && b.equals(true); + String description = "Awaiting until boolean becomes not null and also true."; + assertThatThrownBy(printExceptionWrapper(() -> ruleToUse.waitUntilEqual(alwaysNullProvider, + UnaryOperator.identity(), true, description, 1, TimeUnit.SECONDS))) + .isInstanceOf(ConditionTimeoutException.class) + .hasMessageContaining("null") + .hasMessageContaining(description); + } + + @Test + public void waitCanPrintMoreComplexResults() throws Exception { + Supplier> abcListProvider = () -> List.of("A", "B", "C"); + Function, Integer> examiner = list -> list.size(); + String description = "Awaiting until list becomes empty."; + assertThatThrownBy(printExceptionWrapper(() -> ruleToUse.waitUntilEqual(abcListProvider, + examiner, 0, description, 1, TimeUnit.SECONDS))) + .isInstanceOf(ConditionTimeoutException.class) + .hasMessageContaining("A,B,C") + .hasMessageContaining(description); + } + + private ThrowingCallable printExceptionWrapper(ThrowingCallable throwingCallable) { + return () -> { + try { + throwingCallable.call(); + } catch (Exception e) { + System.out.println(e); + throw (e); + } + }; + } +} diff --git a/geode-core/src/test/java/org/apache/geode/test/junit/rules/MemberStarterRule.java b/geode-core/src/test/java/org/apache/geode/test/junit/rules/MemberStarterRule.java index 684f94b1298d..f451dd4fbaf2 100644 --- a/geode-core/src/test/java/org/apache/geode/test/junit/rules/MemberStarterRule.java +++ b/geode-core/src/test/java/org/apache/geode/test/junit/rules/MemberStarterRule.java @@ -27,17 +27,24 @@ import static org.apache.geode.distributed.ConfigurationProperties.SECURITY_MANAGER; import static org.awaitility.Awaitility.await; import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertThat; import java.io.File; import java.io.IOException; import java.util.Arrays; -import java.util.Map; +import java.util.List; +import java.util.Objects; import java.util.Properties; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.commons.lang.ArrayUtils; +import org.assertj.core.api.Assertions; +import org.awaitility.core.ConditionTimeoutException; import org.junit.rules.TemporaryFolder; import org.apache.geode.distributed.DistributedSystem; @@ -73,6 +80,12 @@ public abstract class MemberStarterRule extends SerializableExternalResource protected boolean autoStart = false; + public static void setWaitUntilTimeout(int waitUntilTimeout) { + WAIT_UNTIL_TIMEOUT = waitUntilTimeout; + } + + private static int WAIT_UNTIL_TIMEOUT = 30; + public MemberStarterRule() { oldUserDir = System.getProperty("user.dir"); @@ -261,37 +274,130 @@ public ManagementService getManagementService() { public abstract InternalCache getCache(); - public void waitTillRegionIsReadyOnServers(String regionName, int serverCount) { - await().atMost(30, TimeUnit.SECONDS).until(() -> getRegionMBean(regionName) != null); - await().atMost(30, TimeUnit.SECONDS) - .until(() -> getRegionMBean(regionName).getMembers() != null - && getRegionMBean(regionName).getMembers().length == serverCount); + public void waitUntilRegionIsReadyOnExactlyThisManyServers(String regionName, + int exactServerCount) throws Exception { + // First wait until the region mbean is not null... + waitUntilEqual( + () -> getRegionMBean(regionName), + Objects::nonNull, + true, + String.format("Expecting to find an mbean for region '%s'", regionName), + WAIT_UNTIL_TIMEOUT, TimeUnit.SECONDS); + + // Now actually wait for the members to receive the region + String assertionConditionDescription = String.format( + "Expecting region '%s' to be found on exactly %d servers", regionName, exactServerCount); + waitUntilSatisfied( + () -> Arrays.asList(getRegionMBean(regionName).getMembers()), + Function.identity(), + members -> Assertions.assertThat(members).isNotNull().hasSize(exactServerCount), + assertionConditionDescription, + WAIT_UNTIL_TIMEOUT, TimeUnit.SECONDS); } - private long getDiskStoreCount(String diskStoreName) { + public void waitUntilGatewaySendersAreReadyOnExactlyThisManyServers(int exactGatewaySenderCount) + throws Exception { DistributedSystemMXBean dsMXBean = getManagementService().getDistributedSystemMXBean(); - Map diskstores = dsMXBean.listMemberDiskstore(); - long count = - diskstores.values().stream().filter(x -> ArrayUtils.contains(x, diskStoreName)).count(); + String predicateDescription = String.format( + "Expecting to find exactly %d gateway sender beans.", exactGatewaySenderCount); - return count; + waitUntilEqual(() -> dsMXBean.listGatewaySenderObjectNames(), + array -> array.length, exactGatewaySenderCount, predicateDescription, WAIT_UNTIL_TIMEOUT, + TimeUnit.SECONDS); } - public void waitTilGatewaySendersAreReady(int expectedGatewayObjectCount) throws Exception { - DistributedSystemMXBean dsMXBean = getManagementService().getDistributedSystemMXBean(); - await().atMost(30, TimeUnit.SECONDS) - .until(() -> assertThat(dsMXBean.listGatewaySenderObjectNames().length, - is(expectedGatewayObjectCount))); + public void waitUntilDiskStoreIsReadyOnExactlyThisManyServers(String diskStoreName, + int exactServerCount) throws Exception { + final Supplier distributedSystemMXBeanSupplier = + () -> getManagementService().getDistributedSystemMXBean(); + + waitUntilSatisfied(distributedSystemMXBeanSupplier, + Function.identity(), + bean -> assertThat(bean, notNullValue()), + "Distributed System MXBean should not be null", + WAIT_UNTIL_TIMEOUT, TimeUnit.SECONDS); + + DistributedSystemMXBean dsMXBean = distributedSystemMXBeanSupplier.get(); + + String predicateDescription = String.format( + "Expecting exactly %d servers to present mbeans for a disk store with name %s.", + exactServerCount, diskStoreName); + Supplier> diskStoreSupplier = () -> dsMXBean.listMemberDiskstore() + .values().stream().filter(x1 -> ArrayUtils.contains(x1, diskStoreName)) + .collect(Collectors.toList()); + + waitUntilEqual(diskStoreSupplier, + x -> x.size(), + exactServerCount, + predicateDescription, + WAIT_UNTIL_TIMEOUT, TimeUnit.SECONDS); } - public void waitTillDiskStoreIsReady(String diskstoreName, int serverCount) { - await().atMost(30, TimeUnit.SECONDS) - .until(() -> getDiskStoreCount(diskstoreName) == serverCount); + public void waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers(String queueId, + int exactServerCount) + throws Exception { + String examinerDescription = String.format( + "Expecting exactly %d servers to have an AEQ with id '%s'.", exactServerCount, queueId); + waitUntilEqual( + () -> CliUtil.getMembersWithAsyncEventQueue(getCache(), queueId), + membersWithAEQ -> membersWithAEQ.size(), + exactServerCount, + examinerDescription, + WAIT_UNTIL_TIMEOUT, TimeUnit.SECONDS); } - public void waitTillAsyncEventQueuesAreReadyOnServers(String queueId, int serverCount) { - await().atMost(30, TimeUnit.SECONDS).until( - () -> CliUtil.getMembersWithAsyncEventQueue(getCache(), queueId).size() == serverCount); + + /** + * This method wraps an {@link org.awaitility.Awaitility#await} call for more meaningful error + * reporting. + * + * @param supplier Method to retrieve the result to be tested, e.g., + * get a list of visible region mbeans + * @param examiner Method to evaluate the result provided by {@code provider}, e.g., + * get the length of the provided list. + * Use {@link java.util.function.Function#identity()} if {@code assertionConsumer} + * directly tests the value provided by {@code supplier}. + * @param assertionConsumer assertThat styled condition on the output of {@code examiner} against + * which + * the {@code await().until(...)} will be called. E.g., + * {@code beanCount -> assertThat(beanCount, is(5))} + * @param assertionConsumerDescription A description of the {@code assertionConsumer} method, + * for additional failure information should this call time out. + * E.g., "Visible region mbean count should be 5" + * @param timeout With {@code unit}, the maximum time to wait before raising an exception. + * @param unit With {@code timeout}, the maximum time to wait before raising an exception. + * @throws org.awaitility.core.ConditionTimeoutException The timeout has been reached + * @throws Exception Any exception produced by {@code provider.call()} + */ + public void waitUntilSatisfied(Supplier supplier, Function examiner, + Consumer assertionConsumer, String assertionConsumerDescription, long timeout, + TimeUnit unit) + throws Exception { + try { + await(assertionConsumerDescription) + .atMost(timeout, unit) + .until(() -> assertionConsumer.accept(examiner.apply(supplier.get()))); + } catch (ConditionTimeoutException e) { + // There is a very slight race condition here, where the above could conceivably time out, + // and become satisfied before the next supplier.get() + throw new ConditionTimeoutException( + "The observed result '" + String.valueOf(supplier.get()) + + "' does not satisfy the provided assertionConsumer. \n" + e.getMessage()); + } + } + + /** + * Convenience alias for {@link #waitUntilSatisfied}, + * requiring equality rather than a generic assertion. + */ + public void waitUntilEqual(Supplier provider, + Function examiner, + J expectation, + String expectationDesription, + long timeout, TimeUnit unit) + throws Exception { + Consumer assertionConsumer = examined -> assertThat(examined, is(expectation)); + waitUntilSatisfied(provider, examiner, assertionConsumer, expectationDesription, timeout, unit); } abstract void stopMember(); diff --git a/geode-wan/src/test/java/org/apache/geode/internal/cache/wan/wancommand/DestroyGatewaySenderCommandDUnitTest.java b/geode-wan/src/test/java/org/apache/geode/internal/cache/wan/wancommand/DestroyGatewaySenderCommandDUnitTest.java index 0f9ad0f55b08..7ce5e8403d26 100644 --- a/geode-wan/src/test/java/org/apache/geode/internal/cache/wan/wancommand/DestroyGatewaySenderCommandDUnitTest.java +++ b/geode-wan/src/test/java/org/apache/geode/internal/cache/wan/wancommand/DestroyGatewaySenderCommandDUnitTest.java @@ -75,13 +75,13 @@ public void testCreateDestroySerialGatewaySenderWithDefault() throws Exception { gfsh.executeAndAssertThat(CREATE).statusIsSuccess().tableHasColumnWithExactValuesInAnyOrder( "Status", "GatewaySender \"sender\" created on \"happyserver1\""); - locatorSite1.waitTilGatewaySendersAreReady(1); + locatorSite1.waitUntilGatewaySendersAreReadyOnExactlyThisManyServers(1); // destroy gateway sender and verify AEQs cleaned up gfsh.executeAndAssertThat(DESTROY).statusIsSuccess().tableHasColumnWithExactValuesInAnyOrder( "Status", "GatewaySender \"sender\" destroyed on \"happyserver1\""); - locatorSite1.waitTilGatewaySendersAreReady(0); + locatorSite1.waitUntilGatewaySendersAreReadyOnExactlyThisManyServers(0); gfsh.executeAndAssertThat("list gateways").statusIsError() .containsOutput("GatewaySenders or GatewayReceivers are not available in cluster"); @@ -93,13 +93,13 @@ public void testCreateDestroyParallellGatewaySenderWithDefault() throws Exceptio .tableHasColumnWithExactValuesInAnyOrder("Status", "GatewaySender \"sender\" created on \"happyserver1\""); - locatorSite1.waitTilGatewaySendersAreReady(1); + locatorSite1.waitUntilGatewaySendersAreReadyOnExactlyThisManyServers(1); // destroy gateway sender and verify AEQs cleaned up gfsh.executeAndAssertThat(DESTROY).statusIsSuccess().tableHasColumnWithExactValuesInAnyOrder( "Status", "GatewaySender \"sender\" destroyed on \"happyserver1\""); - locatorSite1.waitTilGatewaySendersAreReady(0); + locatorSite1.waitUntilGatewaySendersAreReadyOnExactlyThisManyServers(0); gfsh.executeAndAssertThat("list gateways").statusIsError() .containsOutput("GatewaySenders or GatewayReceivers are not available in cluster"); diff --git a/geode-wan/src/test/java/org/apache/geode/management/internal/cli/commands/CreateRegionCommandDUnitTest.java b/geode-wan/src/test/java/org/apache/geode/management/internal/cli/commands/CreateRegionCommandDUnitTest.java index fb9b90322ed1..a85aa45f5342 100644 --- a/geode-wan/src/test/java/org/apache/geode/management/internal/cli/commands/CreateRegionCommandDUnitTest.java +++ b/geode-wan/src/test/java/org/apache/geode/management/internal/cli/commands/CreateRegionCommandDUnitTest.java @@ -60,7 +60,7 @@ public void createReplicatedRegionWithParallelAsynchronousEventQueueShouldThrowE "create async-event-queue --parallel=true --listener=org.apache.geode.internal.cache.wan.MyAsyncEventListener --id=" + asyncQueueName) .statusIsSuccess(); - locator.waitTillAsyncEventQueuesAreReadyOnServers(asyncQueueName, 2); + locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers(asyncQueueName, 2); gfsh.executeAndAssertThat("create region --type=REPLICATE --name=" + regionName + " --async-event-queue-id=" + asyncQueueName) @@ -86,7 +86,7 @@ public void createReplicatedRegionWithParallelGatewaySenderShouldThrowExceptionA "create gateway-sender --parallel=true --remote-distributed-system-id=2 --id=" + gatewaySenderName) .statusIsSuccess(); - locator.waitTilGatewaySendersAreReady(2); + locator.waitUntilGatewaySendersAreReadyOnExactlyThisManyServers(2); gfsh.executeAndAssertThat("create region --type=REPLICATE --name=" + regionName + " --gateway-sender-id=" + gatewaySenderName) diff --git a/geode-wan/src/test/java/org/apache/geode/management/internal/cli/commands/DescribeRegionDUnitTest.java b/geode-wan/src/test/java/org/apache/geode/management/internal/cli/commands/DescribeRegionDUnitTest.java index 35e70e32d64c..19e672105b79 100644 --- a/geode-wan/src/test/java/org/apache/geode/management/internal/cli/commands/DescribeRegionDUnitTest.java +++ b/geode-wan/src/test/java/org/apache/geode/management/internal/cli/commands/DescribeRegionDUnitTest.java @@ -56,8 +56,8 @@ public void describeRegionWithGatewayAndAsyncEventQueue() throws Exception { + "--listener=org.apache.geode.internal.cache.wan.MyAsyncEventListener").statusIsSuccess(); gfsh.executeAndAssertThat("create gateway-sender --id=sender1 --remote-distributed-system-id=2") .statusIsSuccess(); - sending_locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1); - sending_locator.waitTilGatewaySendersAreReady(2); + sending_locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1); + sending_locator.waitUntilGatewaySendersAreReadyOnExactlyThisManyServers(2); gfsh.executeAndAssertThat( "create region --name=region4 --type=REPLICATE --async-event-queue-id=queue1 --gateway-sender-id=sender1")