Skip to content

Commit

Permalink
GEODE-5295: Improve member WaitUntilX methods (apache#2039)
Browse files Browse the repository at this point in the history
* Improve error reporting in the event of a WaitUntil timeout.
* Renamed methods to avoid future developer error similar to GEODE-5299
  • Loading branch information
PurelyApplied authored Jul 12, 2018
1 parent 2e1673b commit 84e911c
Show file tree
Hide file tree
Showing 24 changed files with 446 additions and 84 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ public void before() throws Exception {
"create region --type=REPLICATE --name=" + REGION_PATH + " --enable-statistics=true")
.statusIsSuccess();

locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT);
locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT);
waitForLocatorsToAgreeOnMembership();
}

Expand Down Expand Up @@ -129,7 +129,7 @@ public void testLocalBeans_MaintainLocatorAndCrashServer() {
.containsExactlyElementsOf(initialLocatorBeans);

server1.waitTilServerFullyReconnected();
locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT);
locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT);

List<String> finalLocatorBeans = canonicalBeanNamesFor(locator1);

Expand Down Expand Up @@ -206,7 +206,7 @@ public void testRemoteBeanKnowledge_MaintainLocatorAndCrashServer()
});

server1.waitTilServerFullyReconnected();
locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT);
locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT);
waitForMBeanFederationFrom(numServerMBeans, server1);
waitForLocatorsToAgreeOnMembership();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ public void before() throws Exception {
.statusIsSuccess();
gfsh.disconnect();

locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT);
locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT);

bb = InternalBlackboardImpl.getInstance();
}
Expand All @@ -109,7 +109,7 @@ public void MBeanFederationAddRemoveServer() throws IOException {

server3 = lsRule.startServerVM(SERVER_3_VM_INDEX, locator1.getPort());
SERVER_COUNT++;
locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT);
locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT);
List keyset = server3.invoke(() -> {
InternalCache cache = ClusterStartupRule.getCache();
DistributedMember member =
Expand All @@ -131,7 +131,7 @@ public void MBeanFederationAddRemoveServer() throws IOException {

lsRule.stopMember(SERVER_3_VM_INDEX);
SERVER_COUNT--;
locator1.waitTillRegionsAreReadyOnServers(REGION_PATH, SERVER_COUNT);
locator1.waitUntilRegionIsReadyOnExactlyThisManyServers(REGION_PATH, SERVER_COUNT);

List<String> finalMBeans = getFederatedGemfireBeansFrom(locator1);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,9 @@ public static void setUpCluster() throws Exception {
gfsh.executeAndAssertThat("create region --name=group2Region --group=group2 --type=REPLICATE")
.statusIsSuccess();

locator.waitTillRegionsAreReadyOnServers("/commonRegion", 4);
locator.waitTillRegionsAreReadyOnServers("/group1Region", 2);
locator.waitTillRegionsAreReadyOnServers("/group2Region", 2);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/commonRegion", 4);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/group1Region", 2);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/group2Region", 2);
}

@Test
Expand Down Expand Up @@ -202,9 +202,9 @@ public void getMembersWithQueueId() throws Exception {
"create async-event-queue --id=queue --listener=" + MyAsyncEventListener.class.getName())
.statusIsSuccess();

locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 2);
locator.waitTillAsyncEventQueuesAreReadyOnServers("queue2", 2);
locator.waitTillAsyncEventQueuesAreReadyOnServers("queue", 4);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 2);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue2", 2);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue", 4);

locator.invoke(() -> {
members = CliUtil.getMembersWithAsyncEventQueue(ClusterStartupRule.getCache(), "queue1");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public void testAlterAsyncEventQueue() throws Exception {
gfsh.executeAndAssertThat("create async-event-queue --id=queue1 --group=group1 --listener="
+ MyAsyncEventListener.class.getName()).statusIsSuccess();

locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1);

// verify that server1's event queue has the default value
server1.invoke(() -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public static void setupCluster() throws Exception {
gfsh.executeAndAssertThat(
"create disk-store --name=diskStore --groups=dataStore --dir=diskStore").statusIsSuccess();

locator.waitTillDiskstoreIsReady("diskStore", 2);
locator.waitUntilDiskStoreIsReadyOnExactlyThisManyServers("diskStore", 2);
// create regions
gfsh.executeAndAssertThat(
"create region --name=testRegion --type=REPLICATE_PERSISTENT --group=dataStore --disk-store=diskStore")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ public void create_sync_event_queue() throws Exception {
" java.lang.IllegalStateException: A GatewaySender with id AsyncEventQueue_queue is already defined in this cache.");

gfsh.executeAndAssertThat("create disk-store --name=diskStore2 --dir=diskstore");
locator.waitTillDiskstoreIsReady("diskStore2", 2);
locator.waitUntilDiskStoreIsReadyOnExactlyThisManyServers("diskStore2", 2);

// create another queue with different configuration
gfsh.executeAndAssertThat(VALID_COMMAND + " --id=queue2 --group=group2 "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ public void regionExistOnServerButNotInClusterConfig() {
@Test
public void regionExistInClusterConfig() {
gfsh.executeAndAssertThat("create region --name=regionB --type=REPLICATE").statusIsSuccess();
locator.waitTillRegionsAreReadyOnServers("/regionB", 1);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/regionB", 1);
locator.invoke(() -> {
InternalConfigurationPersistenceService configurationService =
ClusterStartupRule.getLocator().getConfigurationPersistenceService();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,7 @@ public void startWithNonProxyRegion() {
"create region --type=REPLICATE_PROXY --group=group2 --name=" + regionName)
.statusIsSuccess().tableHasRowWithValues("Member", "server-2");

locator.waitTillRegionsAreReadyOnServers("/" + regionName, 2);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + regionName, 2);

gfsh.executeAndAssertThat(
"create region --type=PARTITION_PROXY --group=group2 --name=" + regionName).statusIsError()
Expand All @@ -374,7 +374,7 @@ public void startWithReplicateProxyRegion() {
gfsh.executeAndAssertThat("create region --type=REPLICATE --group=group2 --name=" + regionName)
.statusIsSuccess().tableHasRowWithValues("Member", "server-2");

locator.waitTillRegionsAreReadyOnServers("/" + regionName, 2);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + regionName, 2);
// the following two should fail with name check on locator, not on server
gfsh.executeAndAssertThat("create region --type=PARTITION --group=group2 --name=" + regionName)
.statusIsError().containsOutput("Region /" + regionName + " already exists on the cluster");
Expand Down Expand Up @@ -487,7 +487,7 @@ public void startWithPartitionProxyRegion() {
gfsh.executeAndAssertThat("create region --type=PARTITION --group=group2 --name=" + regionName)
.statusIsSuccess().tableHasRowWithValues("Member", "server-2");

locator.waitTillRegionsAreReadyOnServers("/" + regionName, 2);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + regionName, 2);
gfsh.executeAndAssertThat("create region --type=PARTITION --group=group2 --name=" + regionName)
.statusIsError().containsOutput("Region /" + regionName + " already exists on the cluster");
gfsh.executeAndAssertThat(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ public static void setupSystem() throws Exception {
gfsh.executeAndAssertThat("create async-event-queue --id=queue1 --group=group1 "
+ "--listener=org.apache.geode.internal.cache.wan.MyAsyncEventListener").statusIsSuccess();

locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1);
gfsh.executeAndAssertThat(
"create region --name=region4 --type=REPLICATE --async-event-queue-id=queue1")
.statusIsSuccess();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public void destroyAeq_returnsSuccess() {
"create async-event-queue --id=queue1 --listener=" + MyAsyncEventListener.class.getName())
.statusIsSuccess();

locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 2);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 2);
gfsh.executeAndAssertThat("list async-event-queues").statusIsSuccess();

locator.invoke(() -> {
Expand Down Expand Up @@ -88,7 +88,7 @@ public void destroyAeqWhenQueueDoesNotExist_deafultReturnsError() {
"create async-event-queue --id=queue1 --listener=" + MyAsyncEventListener.class.getName())
.statusIsSuccess();

locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 2);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 2);
gfsh.executeAndAssertThat("list async-event-queues").statusIsSuccess();

gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1 ").statusIsSuccess();
Expand All @@ -103,7 +103,7 @@ public void destroyAeqWhenQueueDoesNotExist_withIfExistsReturnsSuccess() {
"create async-event-queue --id=queue1 --listener=" + MyAsyncEventListener.class.getName())
.statusIsSuccess();

locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 2);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 2);
gfsh.executeAndAssertThat("list async-event-queues").statusIsSuccess();

gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1 ").statusIsSuccess();
Expand All @@ -126,7 +126,7 @@ public void destroyAeqOnGroup_returnsSuccess() {
gfsh.executeAndAssertThat("create async-event-queue --id=queue1 --group=group1 --listener="
+ MyAsyncEventListener.class.getName()).statusIsSuccess();

locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1);

gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1 --group=group1")
.statusIsSuccess();
Expand All @@ -147,7 +147,7 @@ public void destroyAeqOnGroupThatDoesNotExisit_returnsError() {
gfsh.executeAndAssertThat("create async-event-queue --id=queue1 --group=group1 --listener="
+ MyAsyncEventListener.class.getName()).statusIsSuccess();

locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1);

gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1 --group=group2")
.statusIsError().containsOutput(CliStrings.NO_MEMBERS_FOUND_MESSAGE);
Expand All @@ -168,7 +168,7 @@ public void destroyAeq_selectsQueuesOnGroup_showsErrorForServersNotInGroup()
gfsh.executeAndAssertThat("create async-event-queue --id=queue1 --group=group1 --listener="
+ MyAsyncEventListener.class.getName()).statusIsSuccess();

locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1);
gfsh.executeAndAssertThat("list async-event-queues").statusIsSuccess();

gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1").statusIsSuccess()
Expand All @@ -193,8 +193,8 @@ public void destroyAeq_selectsQueuesByGroup_returnsSuccess() throws GfJsonExcept
gfsh.executeAndAssertThat("create async-event-queue --id=queue3 --group=group3 --listener="
+ MyAsyncEventListener.class.getName())/* .statusIsSuccess() */;

locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1);
locator.waitTillAsyncEventQueuesAreReadyOnServers("queue3", 1);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue3", 1);
gfsh.executeAndAssertThat("list async-event-queues").statusIsSuccess();

gfsh.executeAndAssertThat("destroy async-event-queue --id=queue1 --group=group1")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ public void testDestroyDistributedRegion() {
gfsh.executeAndAssertThat(
"create region --name=Order --type=PARTITION --colocated-with=Customer").statusIsSuccess();

locator.waitTillRegionsAreReadyOnServers("/Customer", 3);
locator.waitTillRegionsAreReadyOnServers("/Order", 3);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/Customer", 3);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/Order", 3);

// Test unable to destroy with co-location
gfsh.executeAndAssertThat("destroy region --name=/Customer").statusIsError()
Expand All @@ -92,7 +92,7 @@ public void testDestroyDistributedRegion() {
public void testDestroyLocalRegions() {
gfsh.executeAndAssertThat("create region --name=region1 --type=LOCAL").statusIsSuccess();

locator.waitTillRegionsAreReadyOnServers("/region1", 3);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/region1", 3);

gfsh.executeAndAssertThat("destroy region --name=region1").statusIsSuccess()
.tableHasRowCount("Member", 3).containsOutput("destroyed successfully");
Expand All @@ -110,7 +110,7 @@ public void testDestroyDistributedRegions() {
gfsh.executeAndAssertThat("create region --name=region1 --type=REPLICATE --group=group2")
.statusIsSuccess();

locator.waitTillRegionsAreReadyOnServers("/region1", 3);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/region1", 3);

locator.invoke(() -> {
InternalConfigurationPersistenceService service =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ private void createDiskStoreAndRegion(MemberVM jmxManager, int serverCount) {
gfsh.executeAndAssertThat("list disk-stores").statusIsSuccess()
.tableHasColumnWithValuesContaining("Disk Store Name", diskStores.toArray(new String[0]));

jmxManager.waitTillDiskstoreIsReady(DISKSTORE, serverCount);
jmxManager.waitUntilDiskStoreIsReadyOnExactlyThisManyServers(DISKSTORE, serverCount);

gfsh.executeAndAssertThat(String.format(
"create region --name=%s --type=REPLICATE_PERSISTENT --disk-store=%s --group=%s --eviction-action=overflow-to-disk",
Expand Down Expand Up @@ -119,7 +119,7 @@ public void testMissingDiskStore() throws Exception {
serverRule.before();
});

locator.waitTillDiskstoreIsReady(DISKSTORE, 1);
locator.waitUntilDiskStoreIsReadyOnExactlyThisManyServers(DISKSTORE, 1);

gfsh.executeAndAssertThat("show missing-disk-stores").statusIsSuccess()
.containsOutput("Missing Disk Stores", "No missing colocated region found");
Expand All @@ -130,7 +130,7 @@ public void testMissingDiskStore() throws Exception {
gfsh.executeAndAssertThat("revoke missing-disk-store --id=" + diskstoreIDs.get(0))
.statusIsSuccess().containsOutput("Missing disk store successfully revoked");

locator.waitTillRegionsAreReadyOnServers("/" + REGION_1, 1);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + REGION_1, 1);

server1.invoke(() -> {
Cache cache = ClusterStartupRule.getCache();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public static void setUpClass() throws Exception {
.statusIsSuccess()
.tableHasColumnOnlyWithValues("Member", "server-1", "server-2");

locator.waitTillRegionsAreReadyOnServers("/regionA", 2);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/regionA", 2);

server1.invoke(() -> {
Region region = ClusterStartupRule.getCache().getRegion("/regionA");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ public void list() {
gfsh.executeAndAssertThat("create async-event-queue --id=queue2 --group=group2 --listener="
+ MyAsyncEventListener.class.getName()).statusIsSuccess();

locator.waitTillAsyncEventQueuesAreReadyOnServers("queue1", 1);
locator.waitTillAsyncEventQueuesAreReadyOnServers("queue2", 1);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue1", 1);
locator.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers("queue2", 1);

gfsh.executeAndAssertThat("list async-event-queue").statusIsSuccess()
.tableHasRowCount("Member", 2).tableHasRowWithValues("Member", "ID", "server-1", "queue1")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,9 @@ public static void beforeClass() throws Exception {
gfsh.executeAndAssertThat("put --region=regionB/regionBB --key=key --value=value")
.statusIsSuccess();

locator.waitTillRegionsAreReadyOnServers("/regionA", 2);
locator.waitTillRegionsAreReadyOnServers("/regionB", 2);
locator.waitTillRegionsAreReadyOnServers("/regionB/regionBB", 2);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/regionA", 2);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/regionB", 2);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/regionB/regionBB", 2);
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ public void setup() throws Exception {
gfsh.executeAndAssertThat(
"create region --name=" + PARTITIONED_REGION_NAME + " --type=PARTITION").statusIsSuccess();

locator.waitTillRegionsAreReadyOnServers("/" + REPLICATE_REGION_NAME, 2);
locator.waitTillRegionsAreReadyOnServers("/" + PARTITIONED_REGION_NAME, 2);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + REPLICATE_REGION_NAME, 2);
locator.waitUntilRegionIsReadyOnExactlyThisManyServers("/" + PARTITIONED_REGION_NAME, 2);

VMProvider.invokeInEveryMember(RemoveCommandDUnitTest::populateTestRegions, server1, server2);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ public void importFailWithExistingDiskStore() {
lsRule.startServerVM(1, locatorVM.getPort());
gfshConnector.executeAndAssertThat("create disk-store --name=diskStore1 --dir=testStore")
.statusIsSuccess();
locatorVM.waitTillDiskstoreIsReady("diskStore1", 1);
locatorVM.waitUntilDiskStoreIsReadyOnExactlyThisManyServers("diskStore1", 1);
gfshConnector
.executeAndAssertThat(
"import cluster-configuration --zip-file-name=" + clusterConfigZipPath)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -161,24 +161,27 @@ public void waitTilServerFullyReconnected() {
/**
* this should called on a locatorVM or a serverVM with jmxManager enabled
*/
public void waitTillRegionsAreReadyOnServers(String regionPath, int serverCount) {
vm.invoke(() -> ClusterStartupRule.memberStarter.waitTillRegionIsReadyOnServers(regionPath,
serverCount));
public void waitUntilRegionIsReadyOnExactlyThisManyServers(String regionPath, int serverCount) {
vm.invoke(() -> ClusterStartupRule.memberStarter
.waitUntilRegionIsReadyOnExactlyThisManyServers(regionPath, serverCount));
}

public void waitTillDiskstoreIsReady(String diskstoreName, int serverCount) {
vm.invoke(() -> ClusterStartupRule.memberStarter.waitTillDiskStoreIsReady(diskstoreName,
serverCount));
public void waitUntilDiskStoreIsReadyOnExactlyThisManyServers(String diskstoreName,
int serverCount) {
vm.invoke(() -> ClusterStartupRule.memberStarter
.waitUntilDiskStoreIsReadyOnExactlyThisManyServers(diskstoreName, serverCount));
}

public void waitTillAsyncEventQueuesAreReadyOnServers(String queueId, int serverCount) {
public void waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers(String queueId,
int serverCount) {
vm.invoke(() -> ClusterStartupRule.memberStarter
.waitTillAsyncEventQueuesAreReadyOnServers(queueId, serverCount));
.waitUntilAsyncEventQueuesAreReadyOnExactlyThisManyServers(queueId, serverCount));
}

public void waitTilGatewaySendersAreReady(int expectedGatewayObjectCount) {
public void waitUntilGatewaySendersAreReadyOnExactlyThisManyServers(
int expectedGatewayObjectCount) {
vm.invoke(() -> ClusterStartupRule.memberStarter
.waitTilGatewaySendersAreReady(expectedGatewayObjectCount));
.waitUntilGatewaySendersAreReadyOnExactlyThisManyServers(expectedGatewayObjectCount));
}

}
Loading

0 comments on commit 84e911c

Please sign in to comment.