Remove unused config managedLedgerNumWorkerThreads and related metrics (

apache#14506)
teaey · Mar 15, 2022 · 491eb69 · 491eb69
1 parent bcba273
commit 491eb69
Show file tree

Hide file tree

Showing 21 changed files with 19 additions and 88 deletions.
diff --git a/conf/broker.conf b/conf/broker.conf
@@ -943,9 +943,6 @@ managedLedgerStatsPeriodSeconds=60
 # Other possible options are "CRC32", "MAC" or "DUMMY" (no checksum).
 managedLedgerDigestType=CRC32C
 
-# Number of threads to be used for managed ledger tasks dispatching
-managedLedgerNumWorkerThreads=
-
 # Number of threads to be used for managed ledger scheduled tasks
 managedLedgerNumSchedulerThreads=
 

diff --git a/conf/standalone.conf b/conf/standalone.conf
@@ -649,9 +649,6 @@ managedLedgerCursorPositionFlushSeconds=60
 # Other possible options are "CRC32", "MAC" or "DUMMY" (no checksum).
 managedLedgerDigestType=CRC32C
 
-# Number of threads to be used for managed ledger tasks dispatching
-managedLedgerNumWorkerThreads=4
-
 # Number of threads to be used for managed ledger scheduled tasks
 managedLedgerNumSchedulerThreads=4
 

diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf
@@ -758,9 +758,6 @@ managedLedgerDefaultAckQuorum=2
 # Other possible options are "CRC32", "MAC" or "DUMMY" (no checksum).
 managedLedgerDigestType=CRC32C
 
-# Number of threads to be used for managed ledger tasks dispatching
-managedLedgerNumWorkerThreads=8
-
 # Number of threads to be used for managed ledger scheduled tasks
 managedLedgerNumSchedulerThreads=8
 

diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -1736,11 +1736,6 @@ public class ServiceConfiguration implements PulsarConfiguration {
         doc = "The class of the managed ledger storage"
     )
     private String managedLedgerStorageClassName = "org.apache.pulsar.broker.ManagedLedgerClientFactory";
-    @FieldContext(
-        category = CATEGORY_STORAGE_ML,
-        doc = "Number of threads to be used for managed ledger tasks dispatching"
-    )
-    private int managedLedgerNumWorkerThreads = Runtime.getRuntime().availableProcessors();
     @FieldContext(
         category = CATEGORY_STORAGE_ML,
         doc = "Number of threads to be used for managed ledger scheduled tasks"

diff --git a/...main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java b/...main/java/org/apache/pulsar/broker/stats/prometheus/metrics/PrometheusTextFormatUtil.java
@@ -59,39 +59,39 @@ static void writeCounter(Writer w, String name, String cluster, Counter counter)
 
     static void writeOpStat(Writer w, String name, String cluster, DataSketchesOpStatsLogger opStat) {
         // Example:
-        // # TYPE pulsar_bookie_client_bookkeeper_ml_workers_task_queued summary
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="false",
+        // # TYPE pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued summary
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="false",
         // quantile="0.5"} NaN
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="false",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="false",
         // quantile="0.75"} NaN
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="false",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="false",
         // quantile="0.95"} NaN
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="false",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="false",
         // quantile="0.99"} NaN
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="false",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="false",
         // quantile="0.999"} NaN
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="false",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="false",
         // quantile="0.9999"} NaN
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="false",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="false",
         // quantile="1.0"} -Infinity
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued_count{cluster="pulsar", success="false"} 0
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued_sum{cluster="pulsar", success="false"} 0.0
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="true",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued_count{cluster="pulsar", success="false"} 0
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued_sum{cluster="pulsar", success="false"} 0.0
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="true",
         // quantile="0.5"} 0.031
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="true",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="true",
         // quantile="0.75"} 0.043
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="true",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="true",
         // quantile="0.95"} 0.061
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="true",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="true",
         // quantile="0.99"} 0.064
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="true",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="true",
         // quantile="0.999"} 0.073
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="true",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="true",
         // quantile="0.9999"} 0.073
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued{cluster="pulsar", success="true",
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued{cluster="pulsar", success="true",
         // quantile="1.0"} 0.552
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued_count{cluster="pulsar", success="true"} 40911432
-        // pulsar_bookie_client_bookkeeper_ml_workers_task_queued_sum{cluster="pulsar", success="true"} 527.0
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued_count{cluster="pulsar", success="true"} 40911432
+        // pulsar_bookie_client_bookkeeper_ml_scheduler_task_queued_sum{cluster="pulsar", success="true"} 527.0
         try {
             w.append("# TYPE ").append(name).append(" summary\n");
             writeQuantile(w, opStat, name, cluster, false, 0.5);

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/EmbeddedPulsarCluster.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/EmbeddedPulsarCluster.java
@@ -110,7 +110,6 @@ private ServiceConfiguration getConf() {
         conf.setBookkeeperClientNumWorkerThreads(1);
         conf.setBookkeeperNumberOfChannelsPerBookie(1);
         conf.setManagedLedgerNumSchedulerThreads(1);
-        conf.setManagedLedgerNumWorkerThreads(1);
         conf.setWebSocketNumIoThreads(1);
         conf.setNumTransactionReplayThreadPoolSize(1);
         conf.setNumHttpServerThreads(4);

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java
@@ -999,12 +999,6 @@ public void testManagedLedgerBookieClientStats() throws Exception {
         cm = (List<Metric>) metrics.get("pulsar_managedLedger_client_bookkeeper_ml_scheduler_total_tasks_0");
         assertEquals(cm.size(), 1);
         assertEquals(cm.get(0).tags.get("cluster"), "test");
-
-        cm = (List<Metric>) metrics.get("pulsar_managedLedger_client_bookkeeper_ml_workers_completed_tasks_0");
-        assertEquals(cm.size(), 0);
-
-        cm = (List<Metric>) metrics.get("pulsar_managedLedger_client_bookkeeper_ml_workers_task_execution_count");
-        assertEquals(cm.size(), 0);
     }
 
     @Test

diff --git a/...r-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorConfig.java b/...r-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorConfig.java
@@ -80,7 +80,6 @@ public class PulsarConnectorConfig implements AutoCloseable {
 
     // --- ManagedLedger
     private long managedLedgerCacheSizeMB = 0L;
-    private int managedLedgerNumWorkerThreads = Runtime.getRuntime().availableProcessors();
     private int managedLedgerNumSchedulerThreads = Runtime.getRuntime().availableProcessors();
 
     // --- Nar extraction
@@ -393,16 +392,6 @@ public PulsarConnectorConfig setManagedLedgerCacheSizeMB(int managedLedgerCacheS
         return this;
     }
 
-    public int getManagedLedgerNumWorkerThreads() {
-        return managedLedgerNumWorkerThreads;
-    }
-
-    @Config("pulsar.managed-ledger-num-worker-threads")
-    public PulsarConnectorConfig setManagedLedgerNumWorkerThreads(int managedLedgerNumWorkerThreads) {
-        this.managedLedgerNumWorkerThreads = managedLedgerNumWorkerThreads;
-        return this;
-    }
-
     public int getManagedLedgerNumSchedulerThreads() {
         return managedLedgerNumSchedulerThreads;
     }

diff --git a/...l/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java b/...l/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java
@@ -65,7 +65,6 @@ public void testDefaultManagedLedgerConfig() {
         int availableProcessors = Runtime.getRuntime().availableProcessors();
         PulsarConnectorConfig connectorConfig = new PulsarConnectorConfig();
         Assert.assertEquals(0L, connectorConfig.getManagedLedgerCacheSizeMB());
-        Assert.assertEquals(availableProcessors, connectorConfig.getManagedLedgerNumWorkerThreads());
         Assert.assertEquals(availableProcessors, connectorConfig.getManagedLedgerNumSchedulerThreads());
         Assert.assertEquals(connectorConfig.getMaxSplitQueueSizeBytes(), -1);
     }

diff --git a/site2/docs/reference-configuration.md b/site2/docs/reference-configuration.md
@@ -638,7 +638,6 @@ You can set the log level and configuration in the  [log4j2.yaml](https://github
 |managedLedgerDefaultWriteQuorum|   |1|
 |managedLedgerDefaultAckQuorum|   |1|
 | managedLedgerDigestType | Default type of checksum to use when writing to BookKeeper. | CRC32C |
-| managedLedgerNumWorkerThreads | Number of threads to be used for managed ledger tasks dispatching. | Runtime.getRuntime().availableProcessors() |
 | managedLedgerNumSchedulerThreads | Number of threads to be used for managed ledger scheduled tasks. | Runtime.getRuntime().availableProcessors() |
 |managedLedgerCacheSizeMB|    |N/A|
 |managedLedgerCacheCopyEntries| Whether to copy the entry payloads when inserting in cache.| false|

diff --git a/site2/docs/reference-metrics.md b/site2/docs/reference-metrics.md
@@ -420,13 +420,8 @@ All the managed ledger bookie client metrics are labelled with the following lab
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_completed_tasks_* | Gauge |  The number of tasks the scheduler executor execute completed. <br>The number of metrics determined by the scheduler executor thread number configured by `managedLedgerNumSchedulerThreads` in `broker.conf`. <br> |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_queue_* | Gauge | The number of tasks queued in the scheduler executor's queue. <br>The number of metrics determined by scheduler executor's thread number configured by `managedLedgerNumSchedulerThreads` in `broker.conf`. <br> |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_total_tasks_* | Gauge | The total number of tasks the scheduler executor received. <br>The number of metrics determined by scheduler executor's thread number configured by `managedLedgerNumSchedulerThreads` in `broker.conf`. <br> |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_completed_tasks_* | Gauge | The number of tasks the worker executor execute completed. <br>The number of metrics determined by the number of worker task thread number configured by `managedLedgerNumWorkerThreads` in `broker.conf` <br> |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_queue_* | Gauge | The number of tasks queued in the worker executor's queue. <br>The number of metrics determined by scheduler executor's thread number configured by `managedLedgerNumWorkerThreads` in `broker.conf`. <br> |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_total_tasks_* | Gauge | The total number of tasks the worker executor received. <br>The number of metrics determined by worker executor's thread number configured by `managedLedgerNumWorkerThreads` in `broker.conf`. <br> |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_task_execution | Summary | The scheduler task execution latency calculated in milliseconds. |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_task_queued | Summary | The scheduler task queued latency calculated in milliseconds. |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_task_execution | Summary | The worker task execution latency calculated in milliseconds. |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_task_queued | Summary | The worker task queued latency calculated in milliseconds. |
 
 ### Token metrics
 

diff --git a/site2/website/versioned_docs/version-2.8.0/reference-configuration.md b/site2/website/versioned_docs/version-2.8.0/reference-configuration.md
@@ -610,7 +610,6 @@ You can set the log level and configuration in the  [log4j2.yaml](https://github
 |managedLedgerDefaultWriteQuorum|   |1|
 |managedLedgerDefaultAckQuorum|   |1|
 | managedLedgerDigestType | Default type of checksum to use when writing to BookKeeper. | CRC32C |
-| managedLedgerNumWorkerThreads | Number of threads to be used for managed ledger tasks dispatching. | 8 |
 | managedLedgerNumSchedulerThreads | Number of threads to be used for managed ledger scheduled tasks. | 8 |
 |managedLedgerCacheSizeMB|    |N/A|
 |managedLedgerCacheCopyEntries| Whether to copy the entry payloads when inserting in cache.| false|

diff --git a/site2/website/versioned_docs/version-2.8.0/reference-metrics.md b/site2/website/versioned_docs/version-2.8.0/reference-metrics.md
@@ -356,13 +356,8 @@ All the managed ledger bookie client metrics are labelled with the following lab
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_completed_tasks_* | Gauge |  The number of tasks the scheduler executor execute completed. <br>The number of metrics determined by the scheduler executor thread number configured by `managedLedgerNumSchedulerThreads` in `broker.conf`. <br> |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_queue_* | Gauge | The number of tasks queued in the scheduler executor's queue. <br>The number of metrics determined by scheduler executor's thread number configured by `managedLedgerNumSchedulerThreads` in `broker.conf`. <br> |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_total_tasks_* | Gauge | The total number of tasks the scheduler executor received. <br>The number of metrics determined by scheduler executor's thread number configured by `managedLedgerNumSchedulerThreads` in `broker.conf`. <br> |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_completed_tasks_* | Gauge | The number of tasks the worker executor execute completed. <br>The number of metrics determined by the number of worker task thread number configured by `managedLedgerNumWorkerThreads` in `broker.conf` <br> |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_queue_* | Gauge | The number of tasks queued in the worker executor's queue. <br>The number of metrics determined by scheduler executor's thread number configured by `managedLedgerNumWorkerThreads` in `broker.conf`. <br> |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_total_tasks_* | Gauge | The total number of tasks the worker executor received. <br>The number of metrics determined by worker executor's thread number configured by `managedLedgerNumWorkerThreads` in `broker.conf`. <br> |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_task_execution | Summary | The scheduler task execution latency calculated in milliseconds. |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_task_queued | Summary | The scheduler task queued latency calculated in milliseconds. |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_task_execution | Summary | The worker task execution latency calculated in milliseconds. |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_task_queued | Summary | The worker task queued latency calculated in milliseconds. |
 
 ### Token metrics
 

diff --git a/site2/website/versioned_docs/version-2.8.1/reference-configuration.md b/site2/website/versioned_docs/version-2.8.1/reference-configuration.md
@@ -611,7 +611,6 @@ You can set the log level and configuration in the  [log4j2.yaml](https://github
 |managedLedgerDefaultWriteQuorum|   |1|
 |managedLedgerDefaultAckQuorum|   |1|
 | managedLedgerDigestType | Default type of checksum to use when writing to BookKeeper. | CRC32C |
-| managedLedgerNumWorkerThreads | Number of threads to be used for managed ledger tasks dispatching. | 8 |
 | managedLedgerNumSchedulerThreads | Number of threads to be used for managed ledger scheduled tasks. | 8 |
 |managedLedgerCacheSizeMB|    |N/A|
 |managedLedgerCacheCopyEntries| Whether to copy the entry payloads when inserting in cache.| false|

diff --git a/site2/website/versioned_docs/version-2.8.1/reference-metrics.md b/site2/website/versioned_docs/version-2.8.1/reference-metrics.md
@@ -359,13 +359,8 @@ All the managed ledger bookie client metrics are labelled with the following lab
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_completed_tasks_* | Gauge |  The number of tasks the scheduler executor execute completed. <br>The number of metrics determined by the scheduler executor thread number configured by `managedLedgerNumSchedulerThreads` in `broker.conf`. <br> |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_queue_* | Gauge | The number of tasks queued in the scheduler executor's queue. <br>The number of metrics determined by scheduler executor's thread number configured by `managedLedgerNumSchedulerThreads` in `broker.conf`. <br> |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_total_tasks_* | Gauge | The total number of tasks the scheduler executor received. <br>The number of metrics determined by scheduler executor's thread number configured by `managedLedgerNumSchedulerThreads` in `broker.conf`. <br> |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_completed_tasks_* | Gauge | The number of tasks the worker executor execute completed. <br>The number of metrics determined by the number of worker task thread number configured by `managedLedgerNumWorkerThreads` in `broker.conf` <br> |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_queue_* | Gauge | The number of tasks queued in the worker executor's queue. <br>The number of metrics determined by scheduler executor's thread number configured by `managedLedgerNumWorkerThreads` in `broker.conf`. <br> |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_total_tasks_* | Gauge | The total number of tasks the worker executor received. <br>The number of metrics determined by worker executor's thread number configured by `managedLedgerNumWorkerThreads` in `broker.conf`. <br> |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_task_execution | Summary | The scheduler task execution latency calculated in milliseconds. |
 | pulsar_managedLedger_client_bookkeeper_ml_scheduler_task_queued | Summary | The scheduler task queued latency calculated in milliseconds. |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_task_execution | Summary | The worker task execution latency calculated in milliseconds. |
-| pulsar_managedLedger_client_bookkeeper_ml_workers_task_queued | Summary | The worker task queued latency calculated in milliseconds. |
 
 ### Token metrics
 

diff --git a/site2/website/versioned_docs/version-2.8.2/reference-configuration.md b/site2/website/versioned_docs/version-2.8.2/reference-configuration.md
@@ -613,7 +613,6 @@ You can set the log level and configuration in the  [log4j2.yaml](https://github
 |managedLedgerDefaultWriteQuorum|   |1|
 |managedLedgerDefaultAckQuorum|   |1|
 | managedLedgerDigestType | Default type of checksum to use when writing to BookKeeper. | CRC32C |
-| managedLedgerNumWorkerThreads | Number of threads to be used for managed ledger tasks dispatching. | 8 |
 | managedLedgerNumSchedulerThreads | Number of threads to be used for managed ledger scheduled tasks. | 8 |
 |managedLedgerCacheSizeMB|    |N/A|
 |managedLedgerCacheCopyEntries| Whether to copy the entry payloads when inserting in cache.| false|