Introduce metrics servlet timeout setting (apache#10886)

*Motivation*

Generating metrics is an expensive task and it can take more than 30 seconds
if you have close to or more than a million topics.

*Modification*

This change provides a setting to adjust the async context timeout.

conf/broker.conf

@@ -1142,6 +1142,11 @@ exposeManagedCursorMetricsInPrometheus=false
 # Classname of Pluggable JVM GC metrics logger that can log GC specific metrics
 # jvmGCMetricsLoggerClassName=
 
+# Time in milliseconds that metrics endpoint would time out. Default is 30s.
+# Increase it if there are a lot of topics to expose topic-level metrics.
+# Set it to 0 to disable timeout.
+metricsServletTimeoutMs=30000
+
 ### --- Functions --- ###
 
 # Enable Functions Worker Service in Broker

conf/standalone.conf

@@ -831,6 +831,11 @@ webSocketMaxTextFrameSize=1048576
 # Enable topic level metrics
 exposeTopicLevelMetricsInPrometheus=true
 
+# Time in milliseconds that metrics endpoint would time out. Default is 30s.
+# Increase it if there are a lot of topics to expose topic-level metrics.
+# Set it to 0 to disable timeout.
+metricsServletTimeoutMs=30000
+
 # Classname of Pluggable JVM GC metrics logger that can log GC specific metrics
 # jvmGCMetricsLoggerClassName=
 

pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java

@@ -1961,6 +1961,14 @@ public class ServiceConfiguration implements PulsarConfiguration {
     )
     private boolean exposePreciseBacklogInPrometheus = false;
 
+    @FieldContext(
+        category = CATEGORY_METRICS,
+        doc = "Time in milliseconds that metrics endpoint would time out. Default is 30s.\n" +
+            " Increase it if there are a lot of topics to expose topic-level metrics.\n" +
+            " Set it to 0 to disable timeout."
+    )
+    private long metricsServletTimeoutMs = 30000;
+
     @FieldContext(
             category = CATEGORY_METRICS,
             doc = "Enable expose the backlog size for each subscription when generating stats.\n" +

pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsServlet.java

@@ -43,6 +43,7 @@ public class PrometheusMetricsServlet extends HttpServlet {
     private final boolean shouldExportTopicMetrics;
     private final boolean shouldExportConsumerMetrics;
     private final boolean shouldExportProducerMetrics;
+    private final long metricsServletTimeoutMs;
     private List<PrometheusRawMetricsProvider> metricsProviders;
 
     private ExecutorService executor = null;
@@ -53,6 +54,7 @@ public PrometheusMetricsServlet(PulsarService pulsar, boolean includeTopicMetric
         this.shouldExportTopicMetrics = includeTopicMetrics;
         this.shouldExportConsumerMetrics = includeConsumerMetrics;
         this.shouldExportProducerMetrics = shouldExportProducerMetrics;
+        this.metricsServletTimeoutMs = pulsar.getConfiguration().getMetricsServletTimeoutMs();
     }
 
     @Override
@@ -64,6 +66,7 @@ public void init() throws ServletException {
     protected void doGet(HttpServletRequest request, HttpServletResponse response)
             throws ServletException, IOException {
         AsyncContext context = request.startAsync();
+        context.setTimeout(metricsServletTimeoutMs);
         executor.execute(safeRun(() -> {
             HttpServletResponse res = (HttpServletResponse) context.getResponse();
             try {