GEODE-8609: Check logs for suspicious logs when VM is stopped.

	* When a VM is stopped, the suspect string file is deleted
	* Log checker is called at the end of the test.
	* If a VM is restarted during a test, the logs are deleted and is not checked at the end of the test.
	* With the fix the logs are checked when the VM is stopped.

geode-dunit/src/distributedTest/java/org/apache/geode/test/dunit/internal/SuspiciousLogCheckDUnitTest.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.test.dunit.internal;
+
+import static org.junit.Assert.fail;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.logging.log4j.Logger;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.geode.logging.internal.log4j.api.LogService;
+import org.apache.geode.test.dunit.rules.ClientVM;
+import org.apache.geode.test.dunit.rules.ClusterStartupRule;
+import org.apache.geode.test.dunit.rules.MemberVM;
+import org.apache.geode.test.junit.rules.VMProvider;
+import org.apache.geode.test.junit.runners.CategoryWithParameterizedRunnerFactory;
+
+@RunWith(Parameterized.class)
+@Parameterized.UseParametersRunnerFactory(CategoryWithParameterizedRunnerFactory.class)
+public class SuspiciousLogCheckDUnitTest {
+
+  public SuspiciousLogCheckDUnitTest(String memberType) {
+    this.memberType = memberType;
+  }
+
+  @Parameterized.Parameters(name = "memberType_{0}")
+  public static Collection getMemberType() {
+    return Arrays.asList("locator", "server", "client");
+  }
+
+  public String memberType;
+
+  @Rule
+  public ClusterStartupRule clusterStartupRule = new ClusterStartupRule();
+
+  @Test
+  public void whenLocatorIsStoppedSuspiciousLogsMustBeChecked() throws Exception {
+    MemberVM locator = clusterStartupRule.startLocatorVM(0);
+    int locatorPort = locator.getPort();
+    VMProvider memberToCheck = null;
+    int vmIndex = -1;
+    switch (memberType) {
+      case "locator":
+        memberToCheck = locator;
+        vmIndex = 0;
+        break;
+      case "server":
+        MemberVM server =
+            clusterStartupRule.startServerVM(1, s -> s.withConnectionToLocator(locatorPort));
+        memberToCheck = server;
+        vmIndex = 1;
+        break;
+      case "client":
+        ClientVM client =
+            clusterStartupRule.startClientVM(2, c -> c.withLocatorConnection(locatorPort));
+        memberToCheck = client;
+        vmIndex = 2;
+        break;
+      default:
+        fail("Member type parameter is missing (accepted types are: locator,server,client)");
+    }
+    memberToCheck.invoke(() -> {
+      Logger logger = LogService.getLogger();
+      logger.fatal("Dummy fatal error message");
+    });
+    try {
+      clusterStartupRule.stop(vmIndex);
+      fail();
+    } catch (AssertionError error) {
+      // Assertion error is expected
+    } catch (Exception exception) {
+      fail();
+    }
+  }
+}

geode-dunit/src/main/java/org/apache/geode/test/dunit/internal/DUnitLauncher.java

@@ -370,7 +370,7 @@ private static File createDunitSuspectFile(int vmId, String workingDir) {
     }
 
     File dunitSuspect = new File(getDunitSuspectsDir(),
-        String.format("%s-%s.log", SUSPECT_FILENAME_PREFIX, suffix));
+        getSuspectFileName(suffix));
     dunitSuspect.deleteOnExit();
 
     return dunitSuspect;
@@ -383,19 +383,19 @@ private static String getWorkspaceDir() {
     return workspaceDir;
   }
 
-  public static void closeAndCheckForSuspects() {
-    if (!isLaunched()) {
-      return;
-    }
-
-    List<File> suspectFiles = getDunitSuspectFiles();
+  public static void closeAndCheckForSuspects(int vmIndex) {
+    String suffix = "vm" + vmIndex;
+    String fileName = getSuspectFileName(suffix);
+    File[] suspectFiles = getDunitSuspectsDir()
+        .listFiles((dir, name) -> name.startsWith(fileName));
+    closeAndCheckForSuspects(Arrays.asList(suspectFiles));
+  }
 
-    if (suspectFiles.isEmpty()) {
-      throw new IllegalStateException("No dunit suspect log files found in '"
-          + getDunitSuspectsDir().getAbsolutePath()
-          + "' - perhaps a rule that is cleaning up before suspect processing has already run.");
-    }
+  private static String getSuspectFileName(String suffix) {
+    return String.format("%s-%s.log", SUSPECT_FILENAME_PREFIX, suffix);
+  }
 
+  public static void closeAndCheckForSuspects(List<File> suspectFiles) {
     StringBuilder suspectStringCollector = new StringBuilder();
     for (File suspect : suspectFiles) {
       checkSuspectFile(suspect, suspectStringCollector);
@@ -412,6 +412,19 @@ public static void closeAndCheckForSuspects() {
     }
   }
 
+  public static void closeAndCheckForSuspects() {
+    if (!isLaunched()) {
+      return;
+    }
+    List<File> suspectFiles = getDunitSuspectFiles();
+    if (suspectFiles.isEmpty()) {
+      throw new IllegalStateException("No dunit suspect log files found in '"
+          + getDunitSuspectsDir().getAbsolutePath()
+          + "' - perhaps a rule that is cleaning up before suspect processing has already run.");
+    }
+    closeAndCheckForSuspects(suspectFiles);
+  }
+
   private static void checkSuspectFile(File suspectFile, StringBuilder suspectStringCollector) {
     final List<Pattern> expectedStrings = ExpectedStrings.create("dunit");
     final LogConsumer logConsumer = new LogConsumer(true, expectedStrings,

geode-dunit/src/main/java/org/apache/geode/test/dunit/rules/ClusterStartupRule.java

@@ -19,6 +19,7 @@
 import static org.apache.geode.distributed.ConfigurationProperties.GROUPS;
 import static org.apache.geode.test.dunit.Host.getHost;
 import static org.apache.geode.test.dunit.internal.DUnitLauncher.NUM_VMS;
+import static org.apache.geode.test.dunit.internal.DUnitLauncher.closeAndCheckForSuspects;
 
 import java.io.File;
 import java.util.ArrayList;
@@ -182,7 +183,7 @@ private void after(Description description) throws Throwable {
     // any background thread can fill the dunit_suspect.log
     // after its been truncated if we do it before closing cache
     IgnoredException.removeAllExpectedExceptions();
-    DUnitLauncher.closeAndCheckForSuspects();
+    closeAndCheckForSuspects();
   }
 
 
@@ -351,6 +352,7 @@ public void stop(int index) {
   }
 
   public void stop(int index, boolean cleanWorkingDir) {
+    closeAndCheckForSuspects(index);
     occupiedVMs.get(index).stop(cleanWorkingDir);
   }