Add pulsar SQL woker metrics to repo (apache#7031)

* Add pulsar SQL woker metrics to repo Signed-off-by: xiaolong.ran <[email protected]> * fix comments Signed-off-by: xiaolong.ran <[email protected]> * fix comments Signed-off-by: xiaolong.ran <[email protected]>
wanghhao · May 25, 2020 · d4d380a · d4d380a
1 parent 7880425
commit d4d380a
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 11 deletions.
diff --git a/conf/presto/catalog/pulsar.properties b/conf/presto/catalog/pulsar.properties
@@ -101,3 +101,8 @@ pulsar.managed-ledger-cache-size-MB = 0
 # Number of threads to be used for managed ledger scheduled tasks,
 # default is Runtime.getRuntime().availableProcessors().
 # pulsar.managed-ledger-num-scheduler-threads =
+
+####### PROMETHEUS CONFIGS #######
+
+# pulsar.stats-provider=org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
+# pulsar.stats-provider-configs={"httpServerEnabled":"false", "prometheusStatsHttpPort":"9092", "prometheusStatsHttpEnable":"true"}
diff --git a/...esto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java b/...esto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorMetricsTracker.java
@@ -36,28 +36,28 @@ public class PulsarConnectorMetricsTracker implements AutoCloseable{
 
     // metric names
 
-    // time spend waiting to get entry from entry queue because it is empty
+    // time spend on waiting to get entry from entry queue because it is empty
     private static final String ENTRY_QUEUE_DEQUEUE_WAIT_TIME = "entry-queue-dequeue-wait-time";
 
-    // total time spend waiting to get entry from entry queue per query
+    // total time spent on waiting to get entry from entry queue per query
     private static final String ENTRY_QUEUE_DEQUEUE_WAIT_TIME_PER_QUERY = "entry-queue-dequeue-wait-time-per-query";
 
-    // number of bytes read from bookkeeper
+    // number of bytes read from BookKeeper
     private static final String BYTES_READ = "bytes-read";
 
     // total number of bytes read per query
     private static final String BYTES_READ_PER_QUERY = "bytes-read-per-query";
 
-    // time spent derserializing entries
+    // time spent on derserializing entries
     private static final String ENTRY_DESERIALIZE_TIME = "entry-deserialize-time";
 
-    // time spent derserializing entries per query
+    // time spent on derserializing entries per query
     private static final String ENTRY_DESERIALIZE_TIME_PER_QUERY = "entry-deserialize-time_per_query";
 
-    // time spent waiting for message queue enqueue because message queue is full
+    // time spent on waiting for message queue enqueue because the message queue is full
     private static final String MESSAGE_QUEUE_ENQUEUE_WAIT_TIME = "message-queue-enqueue-wait-time";
 
-    // time spent waiting for message queue enqueue because message queue is full per query
+    // time spent on waiting for message queue enqueue because message queue is full per query
     private static final String MESSAGE_QUEUE_ENQUEUE_WAIT_TIME_PER_QUERY = "message-queue-enqueue-wait-time-per-query";
 
     private static final String NUM_MESSAGES_DERSERIALIZED = "num-messages-deserialized";
@@ -68,7 +68,7 @@ public class PulsarConnectorMetricsTracker implements AutoCloseable{
     // number of messages deserialized per query
     public static final String NUM_MESSAGES_DERSERIALIZED_PER_QUERY = "num-messages-deserialized-per-query";
 
-    // number of read attempts.  Will fail if queues are full
+    // number of read attempts (fail if queues are full)
     public static final String READ_ATTEMPTS = "read-attempts";
 
     // number of read attempts per query
@@ -86,15 +86,16 @@ public class PulsarConnectorMetricsTracker implements AutoCloseable{
     // number of entries per query
     public static final String NUM_ENTRIES_PER_QUERY = "num-entries-per-query";
 
-    // time spent waiting to dequeue from message queue because its empty per query
+    // time spent on waiting to dequeue from message queue because it is empty per query
     public static final String MESSAGE_QUEUE_DEQUEUE_WAIT_TIME_PER_QUERY = "message-queue-dequeue-wait-time-per-query";
 
-    // time spent deserializing message to record e.g. avro, json, etc
+    // time spent on deserializing message to record. For example, Avro, JSON, and so on
     public static final String RECORD_DESERIALIZE_TIME = "record-deserialize-time";
 
-    // time spent deserializing message to record per query
+    // time spent on deserializing message to record per query
     private static final String RECORD_DESERIALIZE_TIME_PER_QUERY = "record-deserialize-time-per-query";
 
+    // Number of records deserialized
     private static final String NUM_RECORD_DESERIALIZED = "num-record-deserialized";
 
     private static final String TOTAL_EXECUTION_TIME = "total-execution-time";

diff --git a/site2/docs/reference-metrics.md b/site2/docs/reference-metrics.md
@@ -17,6 +17,7 @@ Pulsar exposes metrics in Prometheus format that can be collected and used for m
 * [Broker](#broker)
 * [Pulsar Functions](#pulsar functions)
 * [Proxy](#proxy)
+* [Pulsar SQL Worker](#Pulsar SQL Worker)
 
 ## Overview
 
@@ -359,6 +360,34 @@ All the proxy metrics are labelled with the following labels:
 | pulsar_proxy_binary_ops | Counter | Counter of proxy operations. |
 | pulsar_proxy_binary_bytes | Counter | Counter of proxy bytes. |
 
+# Pulsar SQL Worker
+
+| Name | Type | Description |
+|---|---|---|
+| split_bytes_read | Counter | Number of bytes read from BookKeeper. |
+| split_num_messages_deserialized | Counter | Number of messages deserialized. |
+| split_num_record_deserialized | Counter | Number of records deserialized. |
+| split_bytes_read_per_query | Summary | Total number of bytes read per query. |
+| split_entry_deserialize_time | Summary | Time spent on derserializing entries. |
+| split_entry_deserialize_time_per_query | Summary | Time spent on derserializing entries per query. |
+| split_entry_queue_dequeue_wait_time | Summary | Time spend on waiting to get entry from entry queue because it is empty. |
+| split_entry_queue_dequeue_wait_time_per_query | Summary | Total time spent on waiting to get entry from entry queue per query. |
+| split_message_queue_dequeue_wait_time_per_query | Summary | Time spent on waiting to dequeue from message queue because is is empty per query. |
+| split_message_queue_enqueue_wait_time | Summary | Time spent on waiting for message queue enqueue because the message queue is full. |
+| split_message_queue_enqueue_wait_time_per_query | Summary | Time spent on waiting for message queue enqueue because the message queue is full per query. |
+| split_num_entries_per_batch | Summary | Number of entries per batch. |
+| split_num_entries_per_query | Summary | Number of entries per query. |
+| split_num_messages_deserialized_per_entry | Summary | Number of messages deserialized per entry. |
+| split_num_messages_deserialized_per_query | Summary | Number of messages deserialized per query. |
+| split_read_attempts | Summary | Number of read attempts (fail if queues are full). |
+| split_read_attempts_per_query | Summary | Number of read attempts per query. |
+| split_read_latency_per_batch | Summary | Latency of reads per batch. |
+| split_read_latency_per_query | Summary | Total read latency per query. |
+| split_record_deserialize_time | Summary | Time spent on deserializing message to record. For example, Avro, JSON, and so on. |
+| split_record_deserialize_time_per_query | Summary | Time spent on deserializing message to record per query. |
+| split_total_execution_time | Summary | Total execution time . |
+
+
 ## Monitor
 
 You can [set up a Prometheus instance](https://prometheus.io/) to collect all the metrics exposed at Pulsar components and set up