diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java
index 2581e5ae6dc05..8fc5570987878 100644
--- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java
+++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyPrometheusMetricsTest.java
@@ -25,6 +25,8 @@
 import com.google.common.base.Splitter;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Multimap;
+import io.prometheus.client.Collector;
+import io.prometheus.client.CollectorRegistry;
 import io.prometheus.client.Counter;
 import java.util.List;
 import java.util.Map;
@@ -87,8 +89,12 @@ protected void setup() throws Exception {
     @AfterClass(alwaysRun = true)
     protected void cleanup() throws Exception {
         internalCleanup();
-        proxyWebServer.stop();
-        proxyService.close();
+        if (proxyWebServer != null) {
+            proxyWebServer.stop();
+        }
+        if (proxyService != null) {
+            proxyService.close();
+        }
     }
 
     /**
@@ -96,30 +102,34 @@ protected void cleanup() throws Exception {
      */
     @Test
     public void testMetrics() {
-        Counter.build("test_counter", "a test counter").create().register();
-
-        Client httpClient = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
-        Response r = httpClient.target(proxyWebServer.getServiceUri()).path("/metrics").request()
-                .get();
-        Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
-        String response = r.readEntity(String.class).trim();
-
-        Multimap<String, Metric> metrics = parseMetrics(response);
-
-        // Check that ProxyService metrics are present
-        List<Metric> cm = (List<Metric>) metrics.get("pulsar_proxy_binary_bytes_total");
-        assertEquals(cm.size(), 1);
-        assertEquals(cm.get(0).tags.get("cluster"), TEST_CLUSTER);
-
-        // Check that any Prometheus metric registered in the default CollectorRegistry is present
-        List<Metric> cm2 = (List<Metric>) metrics.get("test_metrics");
-        assertEquals(cm2.size(), 1);
-        assertEquals(cm2.get(0).tags.get("label1"), "xyz");
-
-        // Check that PrometheusRawMetricsProvider metrics are present
-        List<Metric> cm3 = (List<Metric>) metrics.get("test_counter_total");
-        assertEquals(cm3.size(), 1);
-        assertEquals(cm3.get(0).tags.get("cluster"), TEST_CLUSTER);
+        Counter counter = Counter.build("test_counter", "a test counter").create();
+        Collector collector = counter.register();
+        try {
+            Client httpClient = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
+            Response r = httpClient.target(proxyWebServer.getServiceUri()).path("/metrics").request()
+                    .get();
+            Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
+            String response = r.readEntity(String.class).trim();
+
+            Multimap<String, Metric> metrics = parseMetrics(response);
+
+            // Check that ProxyService metrics are present
+            List<Metric> cm = (List<Metric>) metrics.get("pulsar_proxy_binary_bytes_total");
+            assertEquals(cm.size(), 1);
+            assertEquals(cm.get(0).tags.get("cluster"), TEST_CLUSTER);
+
+            // Check that any Prometheus metric registered in the default CollectorRegistry is present
+            List<Metric> cm2 = (List<Metric>) metrics.get("test_metrics");
+            assertEquals(cm2.size(), 1);
+            assertEquals(cm2.get(0).tags.get("label1"), "xyz");
+
+            // Check that PrometheusRawMetricsProvider metrics are present
+            List<Metric> cm3 = (List<Metric>) metrics.get("test_counter_total");
+            assertEquals(cm3.size(), 1);
+            assertEquals(cm3.get(0).tags.get("cluster"), TEST_CLUSTER);
+        } finally {
+            CollectorRegistry.defaultRegistry.unregister(collector);
+        }
     }
 
     /**