GEODE-7852: test ClientHealthMonitor functionality behind a SNI gatew…

…ay (apache#4958)

* GEODE-7852: test ClientHealthMonitor functionality behind a SNI gateway

This ensures that a server sitting behind an SNI gateway detects the
loss of a client and cleans up after it.  In this case the test detects
that the server has closed CQs created by the non-durable client.

Since test code is not accessible in the Docker container that's running
the server I've enhanced the StatArchiveReader to be able to report the
values of a statistic and have enabled statistics recording in the
server.

(cherry picked from commit 376df4c)

* change the docker rule to be a class-rule in case we add more tests to this class

* removed system.out.println per Bill's review

geode-assembly/src/acceptanceTest/java/org/apache/geode/client/sni/ClientSNICQAcceptanceTest.java

@@ -22,6 +22,7 @@
 import static org.apache.geode.distributed.ConfigurationProperties.SSL_REQUIRE_AUTHENTICATION;
 import static org.apache.geode.distributed.ConfigurationProperties.SSL_TRUSTSTORE;
 import static org.apache.geode.distributed.ConfigurationProperties.SSL_TRUSTSTORE_PASSWORD;
+import static org.apache.geode.test.awaitility.GeodeAwaitility.await;
 import static org.apache.geode.test.util.ResourceUtils.createTempFileFromResource;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -31,9 +32,11 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import com.palantir.docker.compose.DockerComposeRule;
+import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.ClassRule;
-import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestRule;
 
@@ -42,6 +45,7 @@
 import org.apache.geode.cache.client.ClientCache;
 import org.apache.geode.cache.client.ClientCacheFactory;
 import org.apache.geode.cache.client.ClientRegionShortcut;
+import org.apache.geode.cache.client.internal.PoolImpl;
 import org.apache.geode.cache.client.proxy.ProxySocketFactories;
 import org.apache.geode.cache.query.CqAttributes;
 import org.apache.geode.cache.query.CqAttributesFactory;
@@ -64,14 +68,17 @@ public class ClientSNICQAcceptanceTest {
   @ClassRule
   public static TestRule ignoreOnWindowsRule = new IgnoreOnWindowsRule();
 
-  @Rule
-  public DockerComposeRule docker = DockerComposeRule.builder()
-      .file(DOCKER_COMPOSE_PATH.getPath())
-      .build();
+  @ClassRule
+  public static NotOnWindowsDockerRule docker =
+      new NotOnWindowsDockerRule(() -> DockerComposeRule.builder()
+          .file(DOCKER_COMPOSE_PATH.getPath()).build());
+
   private CqQuery cqTracker;
 
   AtomicInteger eventCreateCounter = new AtomicInteger(0);
   AtomicInteger eventUpdateCounter = new AtomicInteger(0);
+  private ClientCache cache;
+  private Region<String, Integer> region;
 
   class SNICQListener implements CqListener {
 
@@ -94,22 +101,29 @@ public void onError(CqEvent aCqEvent) {
     }
   }
 
-  private String trustStorePath;
+  private static String trustStorePath;
 
-  @Before
-  public void before() throws IOException, InterruptedException {
+  @BeforeClass
+  public static void beforeClass() throws IOException, InterruptedException {
     trustStorePath =
         createTempFileFromResource(ClientSNICQAcceptanceTest.class,
             "geode-config/truststore.jks")
                 .getAbsolutePath();
-    docker.exec(options("-T"), "geode",
+    docker.get().exec(options("-T"), "geode",
         arguments("gfsh", "run", "--file=/geode/scripts/geode-starter.gfsh"));
 
   }
 
-  @Test
-  public void performSimpleCQOverSNIProxy()
-      throws CqException, CqExistsException, RegionNotFoundException {
+  @AfterClass
+  public static void afterClass() throws Exception {
+    String output =
+        docker.get().exec(options("-T"), "geode",
+            arguments("cat", "server-dolores/server-dolores.log"));
+    System.out.println("Server log file--------------------------------\n" + output);
+  }
+
+  @Before
+  public void before() throws Exception {
     Properties gemFireProps = new Properties();
     gemFireProps.setProperty(SSL_ENABLED_COMPONENTS, "all");
     gemFireProps.setProperty(SSL_KEYSTORE_TYPE, "jks");
@@ -119,22 +133,30 @@ public void performSimpleCQOverSNIProxy()
     gemFireProps.setProperty(SSL_TRUSTSTORE_PASSWORD, "geode");
     gemFireProps.setProperty(SSL_ENDPOINT_IDENTIFICATION_ENABLED, "true");
 
-    int proxyPort = docker.containers()
+    int proxyPort = docker.get().containers()
         .container("haproxy")
         .port(15443)
         .getExternalPort();
-    ClientCache cache = new ClientCacheFactory(gemFireProps)
+    cache = new ClientCacheFactory(gemFireProps)
         .addPoolLocator("locator-maeve", 10334)
         .setPoolSocketFactory(ProxySocketFactories.sni("localhost",
             proxyPort))
         .setPoolSubscriptionEnabled(true)
         .create();
-    Region<String, Integer> region =
-        cache.<String, Integer>createClientRegionFactory(ClientRegionShortcut.PROXY)
-            .create("jellyfish");
+    region = cache.<String, Integer>createClientRegionFactory(ClientRegionShortcut.PROXY)
+        .create("jellyfish");
+  }
 
-    startCQ(region);
+  @After
+  public void after() throws Exception {
+    if (cache != null) {
+      cache.close();
+    }
+  }
 
+  @Test
+  public void performSimpleCQOverSNIProxy() throws Exception {
+    startCQ(region);
     populateRegion(region);
     assertThat(region.get("key0")).isEqualTo(0);
     assertThat(region.get("key1")).isEqualTo(1);
@@ -152,6 +174,20 @@ public void performSimpleCQOverSNIProxy()
 
     assertThat(eventUpdateCounter.get()).isEqualTo(62);
 
+    // verify that the server cleans up when the client connection to the gateway is destroyed
+    ((PoolImpl) cache.getDefaultPool()).killPrimaryEndpoint();
+    // since we can't run code in the server let's grab the CQ statistics and verify that
+    // the CQ has been closed. StatArchiveReader has a main() that we can use to get a printout
+    // of stat values
+    await().untilAsserted(() -> {
+      String stats = docker.get().exec(options("-T"), "geode",
+          arguments("java", "-cp", "/geode/lib/geode-dependencies.jar",
+              "org.apache.geode.internal.statistics.StatArchiveReader",
+              "stat", "server-dolores/statArchive.gfs", "CqServiceStats.numCqsClosed"));
+      // the stat should transition from zero to one at some point
+      assertThat(stats).contains("0.0 1.0");
+    });
+
   }
 
   public void updateRegion(Region<String, Integer> region) {

geode-assembly/src/acceptanceTest/resources/org/apache/geode/client/sni/geode-config/gemfire.properties

@@ -15,4 +15,5 @@
 # limitations under the License.
 #
 
-#empty
+statistic-sampling-enabled=true
+statistic-archive-file=statArchive.gfs

geode-core/src/main/java/org/apache/geode/internal/statistics/StatArchiveReader.java

@@ -210,21 +210,121 @@ protected static double bitsToDouble(int type, long bits) {
     }
   }
 
+  private static class SingleStatRawStatSpec implements StatSpec {
+
+    private final String archive;
+    private final String statType;
+    private final String statName;
+
+    SingleStatRawStatSpec(String archive, String typeAndStat) {
+      this.archive = archive;
+      String[] parts = typeAndStat.split("\\.", 0);
+      this.statType = parts[0];
+      this.statName = parts[1];
+    }
+
+    @Override
+    public boolean archiveMatches(File archive) {
+      return true; // this.archive.equalsIgnoreCase(archive.getName());
+    }
+
+    @Override
+    public boolean typeMatches(String typeName) {
+      return this.statType.equalsIgnoreCase(typeName);
+    }
+
+    @Override
+    public boolean statMatches(String statName) {
+      return this.statName.equalsIgnoreCase(statName);
+    }
+
+    @Override
+    public boolean instanceMatches(String textId, long numericId) {
+      return true;
+    }
+
+    @Override
+    public int getCombineType() {
+      return StatSpec.NONE;
+    }
+  }
+
+  private static void printStatValue(StatArchiveReader.StatValue v, long startTime, long endTime,
+      boolean nofilter, boolean persec, boolean persample, boolean prunezeros, boolean details) {
+    v = v.createTrimmed(startTime, endTime);
+    if (nofilter) {
+      v.setFilter(StatArchiveReader.StatValue.FILTER_NONE);
+    } else if (persec) {
+      v.setFilter(StatArchiveReader.StatValue.FILTER_PERSEC);
+    } else if (persample) {
+      v.setFilter(StatArchiveReader.StatValue.FILTER_PERSAMPLE);
+    }
+    if (prunezeros) {
+      if (v.getSnapshotsMinimum() == 0.0 && v.getSnapshotsMaximum() == 0.0) {
+        return;
+      }
+    }
+    System.out.println("  " + v.toString());
+    if (details) {
+      System.out.print("  values=");
+      double[] snapshots = v.getSnapshots();
+      for (int i = 0; i < snapshots.length; i++) {
+        System.out.print(' ');
+        System.out.print(snapshots[i]);
+      }
+      System.out.println();
+      String desc = v.getDescriptor().getDescription();
+      if (desc != null && desc.length() > 0) {
+        System.out.println("    " + desc);
+      }
+    }
+  }
+
+
   /**
    * Simple utility to read and dump statistic archive.
    */
   public static void main(String args[]) throws IOException {
     String archiveName = null;
+    final StatArchiveReader reader;
     if (args.length > 1) {
-      System.err.println("Usage: [archiveName]");
-      ExitCode.FATAL.doSystemExit();
-    } else if (args.length == 1) {
-      archiveName = args[0];
+      if (!args[0].equals("stat") || args.length > 3) {
+        System.err.println("Usage: stat archiveName statType.statName");
+        ExitCode.FATAL.doSystemExit();
+      }
+      archiveName = args[1];
+      String statSpec = args[2];
+      if (!statSpec.contains(".")) {
+        throw new IllegalArgumentException(
+            "stat spec '" + statSpec + "' is malformed - use StatType.statName");
+      }
+      File archiveFile = new File(archiveName);
+      if (!archiveFile.exists()) {
+        throw new IllegalArgumentException("archive file does not exist: " + archiveName);
+      }
+      if (!archiveFile.canRead()) {
+        throw new IllegalArgumentException("archive file exists but is unreadable: " + archiveName);
+      }
+      File[] archives = new File[] {archiveFile};
+      SingleStatRawStatSpec[] filters =
+          new SingleStatRawStatSpec[] {new SingleStatRawStatSpec(archiveName, args[2])};
+      reader = new StatArchiveReader(archives, filters, false);
+      final StatValue[] statValues = reader.matchSpec(filters[0]);
+      System.out.println(statSpec + " matched " + statValues.length + " stats...");
+      for (StatValue value : statValues) {
+        printStatValue(value, -1, -1, true, false, false, false, true);
+      }
+      System.out.println("");
+      System.out.flush();
     } else {
-      archiveName = "statArchive.gfs";
+      if (args.length == 1) {
+        archiveName = args[0];
+      } else {
+        archiveName = "statArchive.gfs";
+      }
+      reader = new StatArchiveReader(archiveName);
+      System.out.println("DEBUG: memory used = " + reader.getMemoryUsed());
     }
-    StatArchiveReader reader = new StatArchiveReader(archiveName);
-    System.out.println("DEBUG: memory used = " + reader.getMemoryUsed());
     reader.close();
   }