GEODE-6295: Add Micrometer-based metrics system (apache#3277)

- Add MetricsSession interface that represents the lifecycle of a meter registry, and allows connecting "downstream" registries. - Add MetricsPublishingService interface that defines a service that can be implemented to interact with MetricsSessions by connecting "downstream" registries to publish metrics to external monitoring systems. - Add CompositeMeterRegistryFactory whichs handles creating the composite meter registry for the cache and defining its common tags. - Add CacheLifecycleMetricsSession class that loads implementations of MetricsPublishingService and manages the cache meter registry based on the cache lifecycle. - InternalCacheBuilder uses CompositeMeterRegistryFactory to create the cache meter registry and starts a CacheLifecycleMetricsSession. The cache meter registry is passed to GemFireCacheImpl and exposed to internal Geode code by InternalCache.getMeterRegistry(). Co-Authored-By: Dale Emery <[email protected]> Co-Authored-By: Michael Oleske <[email protected]> Co-Authored-By: Mark Hanson <[email protected]> Co-Authored-By: Kirk Lund <[email protected]>
jmelchio · Mar 13, 2019 · 8ee587c · 8ee587c
1 parent cbfa6a8
commit 8ee587c
Show file tree

Hide file tree

Showing 29 changed files with 1,038 additions and 30 deletions.
diff --git a/boms/geode-all-bom/src/test/resources/expected-pom.xml b/boms/geode-all-bom/src/test/resources/expected-pom.xml
@@ -172,6 +172,11 @@
         <artifactId>classgraph</artifactId>
         <version>4.0.6</version>
       </dependency>
+      <dependency>
+        <groupId>io.micrometer</groupId>
+        <artifactId>micrometer-core</artifactId>
+        <version>1.1.3</version>
+      </dependency>
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-all</artifactId>

diff --git a/buildSrc/src/main/groovy/org/apache/geode/gradle/plugins/DependencyConstraints.groovy b/buildSrc/src/main/groovy/org/apache/geode/gradle/plugins/DependencyConstraints.groovy
@@ -41,6 +41,7 @@ class DependencyConstraints implements Plugin<Project> {
     deps.put("javax.transaction-api.version", "1.3")
     deps.put("jgroups.version", "3.6.14.Final")
     deps.put("log4j.version", "2.11.1")
+    deps.put("micrometer.version", "1.1.3")
     deps.put("shiro.version", "1.4.0")
     deps.put("slf4j-api.version", "1.7.25")
 
@@ -111,6 +112,7 @@ class DependencyConstraints implements Plugin<Project> {
         api(group: 'commons-modeler', name: 'commons-modeler', version: '2.0.1')
         api(group: 'commons-validator', name: 'commons-validator', version: get('commons-validator.version'))
         api(group: 'io.github.classgraph', name: 'classgraph', version: '4.0.6')
+        api(group: 'io.micrometer', name: 'micrometer-core', version: get('micrometer.version'))
         api(group: 'io.netty', name: 'netty-all', version: '4.1.31.Final')
         api(group: 'it.unimi.dsi', name: 'fastutil', version: get('fastutil.version'))
         api(group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2')

diff --git a/extensions/geode-modules-assembly/build.gradle b/extensions/geode-modules-assembly/build.gradle
@@ -208,6 +208,7 @@ task distAppServer(type: Zip, dependsOn: [':extensions:geode-modules-session:jar
       filter(ReplaceTokens, tokens:['LOG4J_VERSION': DependencyConstraints.get('log4j.version')])
       filter(ReplaceTokens, tokens:['FASTUTIL_VERSION': DependencyConstraints.get('fastutil.version')])
       filter(ReplaceTokens, tokens:['ANTLR_VERSION': DependencyConstraints.get('antlr.version')])
+      filter(ReplaceTokens, tokens:['MICROMETER_VERSION': DependencyConstraints.get('micrometer.version')])
       filter(ReplaceTokens, tokens:['TX_VERSION': DependencyConstraints.get('javax.transaction-api.version')])
       filter(ReplaceTokens, tokens:['JGROUPS_VERSION': DependencyConstraints.get('jgroups.version')])
       filter(ReplaceTokens, tokens:['JETTY_VERSION': DependencyConstraints.get('jetty.version')])

diff --git a/extensions/geode-modules-assembly/release/session/bin/modify_war b/extensions/geode-modules-assembly/release/session/bin/modify_war
@@ -97,6 +97,7 @@ WHERE <args>:
                         fastutil.jar
                         javax.transactions-api.jar
                         jgroups.jar
+                        micrometer-core.jar
                         slf4j-api.jar
                         slf4j-jdk14.jar (not for WebLogic)
                         geode-modules-slf4j-weblogic.jar (WebLogic only)
@@ -280,6 +281,7 @@ OTHER_JARS=(${GEODE}/lib/geode-core-${VERSION}.jar \
     ${GEODE}/lib/commons-lang3-@[email protected] \
     ${GEODE}/lib/shiro-core-@[email protected] \
     ${GEODE}/lib/commons-validator-@[email protected] \
+    ${GEODE}/lib/micrometer-core-@[email protected] \
     ${LIB_DIR}/geode-modules-${VERSION}.jar \
     ${LIB_DIR}/geode-modules-session-internal-${VERSION}.jar \
     ${LIB_DIR}/slf4j-api-@[email protected] \

diff --git a/...embly/geode-assembly-test/src/main/java/org/apache/geode/session/tests/TomcatInstall.java b/...embly/geode-assembly-test/src/main/java/org/apache/geode/session/tests/TomcatInstall.java
@@ -92,8 +92,8 @@ public String jarSkipPropertyName() {
   private static final String[] tomcatRequiredJars =
       {"antlr", "commons-io", "commons-lang", "commons-validator", "fastutil", "geode-common",
           "geode-core", "geode-management", "javax.transaction-api", "jgroups", "log4j-api",
-          "log4j-core", "log4j-jul", "shiro-core", "jetty-server", "jetty-util", "jetty-http",
-          "jetty-io"};
+          "log4j-core", "log4j-jul", "micrometer", "shiro-core", "jetty-server", "jetty-util",
+          "jetty-http", "jetty-io"};
 
   private final TomcatVersion version;
 

diff --git a/geode-assembly/src/integrationTest/resources/assembly_content.txt b/geode-assembly/src/integrationTest/resources/assembly_content.txt
@@ -724,6 +724,11 @@ javadoc/org/apache/geode/memcached/GemFireMemcachedServer.html
 javadoc/org/apache/geode/memcached/package-frame.html
 javadoc/org/apache/geode/memcached/package-summary.html
 javadoc/org/apache/geode/memcached/package-tree.html
+javadoc/org/apache/geode/metrics/MetricsPublishingService.html
+javadoc/org/apache/geode/metrics/MetricsSession.html
+javadoc/org/apache/geode/metrics/package-frame.html
+javadoc/org/apache/geode/metrics/package-summary.html
+javadoc/org/apache/geode/metrics/package-tree.html
 javadoc/org/apache/geode/modules/gatewaydelta/AbstractGatewayDeltaEvent.html
 javadoc/org/apache/geode/modules/gatewaydelta/GatewayDelta.html
 javadoc/org/apache/geode/modules/gatewaydelta/GatewayDeltaCreateEvent.html
@@ -874,7 +879,9 @@ javadoc/package-list
 javadoc/script.js
 javadoc/serialized-form.html
 javadoc/stylesheet.css
+lib/HdrHistogram-2.1.9.jar
 lib/HikariCP-3.2.0.jar
+lib/LatencyUtils-2.0.3.jar
 lib/antlr-2.7.7.jar
 lib/classgraph-4.0.6.jar
 lib/commons-beanutils-1.9.3.jar
@@ -945,6 +952,7 @@ lib/lucene-analyzers-phonetic-6.6.2.jar
 lib/lucene-core-6.6.2.jar
 lib/lucene-queries-6.6.2.jar
 lib/lucene-queryparser-6.6.2.jar
+lib/micrometer-core-1.1.3.jar
 lib/mx4j-3.0.2.jar
 lib/mx4j-remote-3.0.2.jar
 lib/mx4j-tools-3.0.1.jar

diff --git a/geode-assembly/src/integrationTest/resources/dependency_classpath.txt b/geode-assembly/src/integrationTest/resources/dependency_classpath.txt
@@ -1,4 +1,6 @@
+HdrHistogram-2.1.9.jar
 HikariCP-3.2.0.jar
+LatencyUtils-2.0.3.jar
 antlr-2.7.7.jar
 classgraph-4.0.6.jar
 commons-beanutils-1.9.3.jar
@@ -61,6 +63,7 @@ lucene-analyzers-phonetic-6.6.2.jar
 lucene-core-6.6.2.jar
 lucene-queries-6.6.2.jar
 lucene-queryparser-6.6.2.jar
+micrometer-core-1.1.3.jar
 netty-all-4.1.31.Final.jar
 protobuf-java-3.6.1.jar
 rmiio-2.1.2.jar

diff --git a/geode-assembly/src/integrationTest/resources/expected_jars.txt b/geode-assembly/src/integrationTest/resources/expected_jars.txt
@@ -1,4 +1,6 @@
+HdrHistogram
 HikariCP
+LatencyUtils
 animal-sniffer-annotations
 antlr
 aopalliance
@@ -68,6 +70,7 @@ lucene-core
 lucene-queries
 lucene-queryparser
 mapstruct
+micrometer-core
 mx4j
 mx4j-remote
 mx4j-tools

diff --git a/geode-assembly/src/main/dist/LICENSE b/geode-assembly/src/main/dist/LICENSE
@@ -725,4 +725,6 @@ domain:
   - AOP Alliance v1.0 (http://aopalliance.sourceforge.net)
   - CompactConcurrentHashSet2, derived from JSR-166 ConcurrentHashMap v1.43
     (http://gee.cs.oswego.edu/dl/concurrency-interest).
+  - HdrHistogram (https://github.com/HdrHistogram/HdrHistogram)
+  - LatencyUtils (https://github.com/LatencyUtils/LatencyUtils)
   - tooltip.js v1.2.6 (https://github.com/jquerytools/jquerytools)
diff --git a/geode-core/build.gradle b/geode-core/build.gradle
@@ -194,6 +194,8 @@ dependencies {
     exclude module: 'xml-apis'
     ext.optional = true
   }
+  compile('io.micrometer:micrometer-core')
+
   compile('io.netty:netty-all') {
     ext.optional = true
   }

diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/GemFireCacheImpl.java b/geode-core/src/main/java/org/apache/geode/internal/cache/GemFireCacheImpl.java
@@ -77,6 +77,7 @@
 
 import com.sun.jna.Native;
 import com.sun.jna.Platform;
+import io.micrometer.core.instrument.MeterRegistry;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 
@@ -281,7 +282,8 @@ public class GemFireCacheImpl implements InternalCache, InternalClientCache, Has
    * The {@code CacheLifecycleListener} s that have been registered in this VM
    */
   @MakeNotStatic
-  private static final Set<CacheLifecycleListener> cacheLifecycleListeners = new HashSet<>();
+  private static final Set<CacheLifecycleListener> cacheLifecycleListeners =
+      new CopyOnWriteArraySet<>();
 
   /**
    * Define gemfire.Cache.ASYNC_EVENT_LISTENERS=true to invoke event listeners in the background
@@ -613,6 +615,8 @@ public class GemFireCacheImpl implements InternalCache, InternalClientCache, Has
 
   private Optional<HttpService> httpService = Optional.ofNullable(null);
 
+  private final MeterRegistry meterRegistry;
+
   static {
     // this works around jdk bug 6427854, reported in ticket #44434
     String propertyName = "sun.nio.ch.bugLevel";
@@ -763,6 +767,7 @@ public static GemFireCacheImpl getExisting(String reason) {
    * @deprecated Rather than fishing for a cache with this static method, use a cache that is passed
    *             in to your method.
    */
+  @Deprecated
   public static GemFireCacheImpl getForPdx(String reason) {
 
     InternalDistributedSystem system = getAnyInstance();
@@ -784,12 +789,13 @@ public static GemFireCacheImpl getForPdx(String reason) {
    * Currently only unit tests set the typeRegistry parameter to a non-null value
    */
   GemFireCacheImpl(boolean isClient, PoolFactory poolFactory,
-      InternalDistributedSystem internalDistributedSystem,
-      CacheConfig cacheConfig, boolean useAsyncEventListeners, TypeRegistry typeRegistry) {
+      InternalDistributedSystem internalDistributedSystem, CacheConfig cacheConfig,
+      boolean useAsyncEventListeners, TypeRegistry typeRegistry, MeterRegistry meterRegistry) {
     this.isClient = isClient;
     this.poolFactory = poolFactory;
     this.cacheConfig = cacheConfig; // do early for bug 43213
     this.pdxRegistry = typeRegistry;
+    this.meterRegistry = meterRegistry;
 
     // Synchronized to prevent a new cache from being created
     // before an old one has finished closing
@@ -952,6 +958,11 @@ public void throwCacheExistsException() {
         creationStack);
   }
 
+  @Override
+  public MeterRegistry getMeterRegistry() {
+    return meterRegistry;
+  }
+
   @Override
   public Optional<HttpService> getHttpService() {
     return httpService;

diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/InternalCache.java b/geode-core/src/main/java/org/apache/geode/internal/cache/InternalCache.java
@@ -27,6 +27,8 @@
 
 import javax.transaction.TransactionManager;
 
+import io.micrometer.core.instrument.MeterRegistry;
+
 import org.apache.geode.cache.Cache;
 import org.apache.geode.cache.CacheClosedException;
 import org.apache.geode.cache.Declarable;
@@ -377,4 +379,6 @@ void invokeRegionEntrySynchronizationListenersAfterSynchronization(
   void initialize();
 
   void throwCacheExistsException();
+
+  MeterRegistry getMeterRegistry();
 }
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/InternalCacheBuilder.java b/geode-core/src/main/java/org/apache/geode/internal/cache/InternalCacheBuilder.java
@@ -20,8 +20,11 @@
 
 import java.util.Optional;
 import java.util.Properties;
+import java.util.function.Consumer;
 import java.util.function.Supplier;
 
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.composite.CompositeMeterRegistry;
 import org.apache.logging.log4j.Logger;
 
 import org.apache.geode.annotations.VisibleForTesting;
@@ -39,6 +42,8 @@
 import org.apache.geode.distributed.internal.InternalDistributedSystem;
 import org.apache.geode.distributed.internal.SecurityConfig;
 import org.apache.geode.internal.logging.LogService;
+import org.apache.geode.internal.metrics.CacheLifecycleMetricsSession;
+import org.apache.geode.internal.metrics.CompositeMeterRegistryFactory;
 import org.apache.geode.pdx.PdxSerializer;
 import org.apache.geode.pdx.internal.TypeRegistry;
 import org.apache.geode.security.AuthenticationFailedException;
@@ -58,6 +63,8 @@ public class InternalCacheBuilder {
 
   private final Properties configProperties;
   private final CacheConfig cacheConfig;
+  private final CompositeMeterRegistryFactory compositeMeterRegistryFactory;
+  private final Consumer<CompositeMeterRegistry> metricsSessionInitializer;
   private final Supplier<InternalDistributedSystem> singletonSystemSupplier;
   private final Supplier<InternalCache> singletonCacheSupplier;
   private final InternalDistributedSystemConstructor internalDistributedSystemConstructor;
@@ -102,20 +109,29 @@ public InternalCacheBuilder(CacheConfig cacheConfig) {
   }
 
   private InternalCacheBuilder(Properties configProperties, CacheConfig cacheConfig) {
-    this(configProperties, cacheConfig, InternalDistributedSystem::getConnectedInstance,
+    this(configProperties,
+        cacheConfig,
+        new CompositeMeterRegistryFactory() {},
+        CacheLifecycleMetricsSession.builder()::build,
+        InternalDistributedSystem::getConnectedInstance,
         InternalDistributedSystem::connectInternal,
-        GemFireCacheImpl::getInstance, GemFireCacheImpl::new);
+        GemFireCacheImpl::getInstance,
+        GemFireCacheImpl::new);
   }
 
   @VisibleForTesting
   InternalCacheBuilder(Properties configProperties,
       CacheConfig cacheConfig,
+      CompositeMeterRegistryFactory compositeMeterRegistryFactory,
+      Consumer<CompositeMeterRegistry> metricsSessionInitializer,
       Supplier<InternalDistributedSystem> singletonSystemSupplier,
       InternalDistributedSystemConstructor internalDistributedSystemConstructor,
       Supplier<InternalCache> singletonCacheSupplier,
       InternalCacheConstructor internalCacheConstructor) {
     this.configProperties = configProperties;
     this.cacheConfig = cacheConfig;
+    this.compositeMeterRegistryFactory = compositeMeterRegistryFactory;
+    this.metricsSessionInitializer = metricsSessionInitializer;
     this.singletonSystemSupplier = singletonSystemSupplier;
     this.internalDistributedSystemConstructor = internalDistributedSystemConstructor;
     this.internalCacheConstructor = internalCacheConstructor;
@@ -173,9 +189,18 @@ public InternalCache create(InternalDistributedSystem internalDistributedSystem)
               existingCache(internalDistributedSystem::getCache, singletonCacheSupplier);
           if (cache == null) {
 
+            int systemId = internalDistributedSystem.getConfig().getDistributedSystemId();
+            String memberName = internalDistributedSystem.getName();
+            String hostName = internalDistributedSystem.getDistributedMember().getHost();
+
+            CompositeMeterRegistry compositeMeterRegistry = compositeMeterRegistryFactory
+                .create(systemId, memberName, hostName);
+
+            metricsSessionInitializer.accept(compositeMeterRegistry);
+
             cache =
                 internalCacheConstructor.construct(isClient, poolFactory, internalDistributedSystem,
-                    cacheConfig, useAsyncEventListeners, typeRegistry);
+                    cacheConfig, useAsyncEventListeners, typeRegistry, compositeMeterRegistry);
 
             internalDistributedSystem.setCache(cache);
             cache.initialize();
@@ -384,7 +409,7 @@ private static void validateUsabilityOfSecurityCallbacks(
   interface InternalCacheConstructor {
     InternalCache construct(boolean isClient, PoolFactory poolFactory,
         InternalDistributedSystem internalDistributedSystem, CacheConfig cacheConfig,
-        boolean useAsyncEventListeners, TypeRegistry typeRegistry);
+        boolean useAsyncEventListeners, TypeRegistry typeRegistry, MeterRegistry meterRegistry);
   }
 
   @VisibleForTesting

diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/InternalCacheForClientAccess.java b/geode-core/src/main/java/org/apache/geode/internal/cache/InternalCacheForClientAccess.java
@@ -32,6 +32,8 @@
 import javax.naming.Context;
 import javax.transaction.TransactionManager;
 
+import io.micrometer.core.instrument.MeterRegistry;
+
 import org.apache.geode.CancelCriterion;
 import org.apache.geode.LogWriter;
 import org.apache.geode.cache.Cache;
@@ -1234,4 +1236,9 @@ public void initialize() {
   public void throwCacheExistsException() {
     delegate.throwCacheExistsException();
   }
+
+  @Override
+  public MeterRegistry getMeterRegistry() {
+    return delegate.getMeterRegistry();
+  }
 }
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/xmlcache/CacheCreation.java b/geode-core/src/main/java/org/apache/geode/internal/cache/xmlcache/CacheCreation.java
@@ -40,6 +40,8 @@
 import javax.naming.Context;
 import javax.transaction.TransactionManager;
 
+import io.micrometer.core.instrument.MeterRegistry;
+
 import org.apache.geode.CancelCriterion;
 import org.apache.geode.GemFireIOException;
 import org.apache.geode.LogWriter;
@@ -2441,4 +2443,9 @@ public void throwCacheExistsException() {
   public Optional<HttpService> getHttpService() {
     throw new UnsupportedOperationException("Should not be invoked");
   }
+
+  @Override
+  public MeterRegistry getMeterRegistry() {
+    throw new UnsupportedOperationException("Should not be invoked");
+  }
 }