Extend micrometer configuration for SCDF

- enable HTTP requests percentiles-histogram. - enable JPA/Hibernate statistics collection. - add Micrometer common tags: application and application.version populated with the SCDF artifactId and version. - drop the spring.cloud.dataflow.x prefix for the http.server.request metrics (Has Wavefront implications). - create a custom spring.cloud.dataflo.server metrics of type LontTaskTimer to be used a Wavfront activation. - add new scdf-servers Grafana dashboard that can be used monitorint SCDF and Skipper servers. - update the docker-compose prometheus configurations to enable the SCDF and Skipper server metrics publishing. (TODO: update K8s manifests & documentation). - update the Grafana and Prometheus image versions. Resolves spring-cloud#4070
Feixu2015 · Aug 13, 2020 · 4b1e4d0 · 4b1e4d0
1 parent 1b8c84b
commit 4b1e4d0
Show file tree

Hide file tree

Showing 9 changed files with 5,398 additions and 8 deletions.
diff --git a/pom.xml b/pom.xml
@@ -97,7 +97,7 @@
 		<findbugs.version>3.0.2</findbugs.version>
 		<aws-java-sdk-ecr.version>1.11.731</aws-java-sdk-ecr.version>
 		<commons-text.version>1.8</commons-text.version>
-		<prometheus-rsocket-spring.version>0.11.0</prometheus-rsocket-spring.version>
+		<prometheus-rsocket-spring.version>1.0.0</prometheus-rsocket-spring.version>
 		<joda-time.version>2.10.6</joda-time.version>
 	</properties>
 	<modules>

diff --git a/.../src/main/java/org/springframework/cloud/dataflow/server/config/web/WebConfiguration.java b/.../src/main/java/org/springframework/cloud/dataflow/server/config/web/WebConfiguration.java
@@ -25,6 +25,9 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import io.micrometer.core.instrument.LongTaskTimer;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Tags;
 import org.h2.tools.Server;
 import org.slf4j.LoggerFactory;
 
@@ -75,6 +78,7 @@ public class WebConfiguration implements ServletContextInitializer, ApplicationL
 	private String dataSourceUrl;
 
 	private Server server = null;
+	private LongTaskTimer.Sample longTaskSample;
 
 	public Server initH2TCPServer() {
 		logger.info("Starting H2 Server with URL: " + dataSourceUrl);
@@ -97,6 +101,11 @@ private String getH2Port(String url) {
 
 	@Override
 	public void onStartup(ServletContext servletContext) {
+		LongTaskTimer longTaskTimer = LongTaskTimer
+				.builder("spring.cloud.dataflow.server").description("Spring Cloud Data Flow duration timer")
+				.tags(Tags.empty()).register(Metrics.globalRegistry);
+		this.longTaskSample = longTaskTimer.start();
+
 		if (StringUtils.hasText(dataSourceUrl) && dataSourceUrl.startsWith("jdbc:h2:tcp://localhost:")) {
 			logger.info("Start Embedded H2");
 			initH2TCPServer();
@@ -161,6 +170,10 @@ public Object postProcessAfterInitialization(Object bean, String s) throws Beans
 
 	@Override
 	public void onApplicationEvent(ContextClosedEvent event) {
+		if (this.longTaskSample != null) {
+			this.longTaskSample.stop();
+			this.longTaskSample = null;
+		}
 		if (this.server != null) {
 			this.server.stop();
 			logger.info("Embedded H2 server stopped!");

diff --git a/spring-cloud-dataflow-server-core/src/main/resources/META-INF/dataflow-server-defaults.yml b/spring-cloud-dataflow-server-core/src/main/resources/META-INF/dataflow-server-defaults.yml
@@ -1,11 +1,14 @@
 management:
   metrics:
+    tags:
+      application: ${info.app.name}
+      application.version: ${info.app.version}
     web:
       server:
         request:
           autotime:
-            enabled: true
-          metric-name: 'spring.cloud.dataflow.http.server.requests'
+            enabled: true # true is default to Boot 2.3.2 at least.
+            percentiles-histogram: true
     export:
       influx:
         enabled: false
@@ -41,7 +44,10 @@ spring:
     initialize-schema: never
   jpa:
     properties:
-      hibernate.id.new_generator_mappings: true
+      hibernate:
+        id.new_generator_mappings: true
+        # Statistics generation is required for publishing JPA micrometer metrics.
+        # generate_statistics: true
     hibernate:
         naming:
           physical-strategy: org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy

diff --git a/spring-cloud-dataflow-server/docker-compose-prometheus.yml b/spring-cloud-dataflow-server/docker-compose-prometheus.yml
@@ -9,6 +9,12 @@ services:
       - |
         SPRING_APPLICATION_JSON=
         {
+          "management.metrics.export.prometheus":{
+             "enabled":true,
+             "rsocket.enabled":true,
+             "rsocket.host":"prometheus-rsocket-proxy",
+             "rsocket.port":7001
+          },
           "spring.cloud.dataflow.applicationProperties":{
              "task.management.metrics.export.prometheus":{
                 "enabled":true,
@@ -23,9 +29,18 @@ services:
                 "rsocket.port":7001
              }
           },
-          "spring.cloud.dataflow.grafana-info.url":"http://localhost:3000"
+          "spring.cloud.dataflow.grafana-info.url":"http://localhost:3000",
+          "spring.jpa.properties.hibernate.generate_statistics":true
         }
 
+  skipper-server:
+    environment:
+      - management.metrics.export.prometheus.enabled=true
+      - management.metrics.export.prometheus.rsocket.enabled=true
+      - management.metrics.export.prometheus.rsocket.host=prometheus-rsocket-proxy
+      - management.metrics.export.prometheus.rsocket.port=7001
+      - spring.jpa.properties.hibernate.generate_statistics=true
+
   prometheus-rsocket-proxy:
     image: micrometermetrics/prometheus-rsocket-proxy:1.0.0
     container_name: prometheus-rsocket-proxy

diff --git a/spring-cloud-dataflow-server/docker-compose-wavefront.yml b/spring-cloud-dataflow-server/docker-compose-wavefront.yml
@@ -14,6 +14,7 @@ services:
       - management.metrics.export.wavefront.api-token=${WAVEFRONT_KEY:?WAVEFRONT_KEY is not set!}
       - management.metrics.export.wavefront.uri=${WAVEFRONT_URI:-https://vmware.wavefront.com}
       - management.metrics.export.wavefront.source=${WAVEFRONT_SOURCE:-scdf-docker-compose}
+      - spring.jpa.properties.hibernate.generate_statistics=true
 
       - spring.cloud.dataflow.applicationProperties.stream.management.metrics.export.wavefront.enabled=true
       - spring.cloud.dataflow.applicationProperties.stream.management.metrics.export.wavefront.api-token=${WAVEFRONT_KEY:?WAVEFRONT_KEY is not set!}
@@ -23,3 +24,11 @@ services:
       - spring.cloud.dataflow.applicationProperties.task.management.metrics.export.wavefront.api-token=${WAVEFRONT_KEY:?WAVEFRONT_KEY is not set!}
       - spring.cloud.dataflow.applicationProperties.task.management.metrics.export.wavefront.uri=${WAVEFRONT_URI:-https://vmware.wavefront.com}
       - spring.cloud.dataflow.applicationProperties.task.management.metrics.export.wavefront.source=${WAVEFRONT_SOURCE:-scdf-docker-compose}
+
+  skipper-server:
+    environment:
+      - management.metrics.export.wavefront.enabled=true
+      - management.metrics.export.wavefront.api-token=${WAVEFRONT_KEY:?WAVEFRONT_KEY is not set!}
+      - management.metrics.export.wavefront.uri=${WAVEFRONT_URI:-https://vmware.wavefront.com}
+      - management.metrics.export.wavefront.source=${WAVEFRONT_SOURCE:-scdf-docker-compose}
+      - spring.jpa.properties.hibernate.generate_statistics=true
diff --git a/src/grafana/influxdb/docker/grafana/Dockerfile b/src/grafana/influxdb/docker/grafana/Dockerfile
@@ -1,4 +1,4 @@
-FROM grafana/grafana:6.3.5
+FROM grafana/grafana:7.1.3
 ADD ./provisioning /etc/grafana/provisioning
 ADD ./config.ini /etc/grafana/config.ini
 ADD ./dashboards /var/lib/grafana/dashboards

diff --git a/src/grafana/prometheus/docker/grafana/Dockerfile b/src/grafana/prometheus/docker/grafana/Dockerfile
@@ -1,4 +1,4 @@
-FROM grafana/grafana:6.3.5
+FROM grafana/grafana:7.1.3
 ADD ./provisioning /etc/grafana/provisioning
 ADD ./config.ini /etc/grafana/config.ini
 ADD ./dashboards /var/lib/grafana/dashboards