From 47a213c1b682565974c48a43dea139c521797436 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 15 Sep 2016 10:57:29 -0700 Subject: [PATCH] Expose broker URLs in loadbalancer registration node (#11) --- .../pulsar/broker/admin/BrokerStats.java | 2 +- .../broker/loadbalance/LoadManager.java | 2 +- .../loadbalance/ResourceDescription.java | 2 +- .../impl/PulsarLoadReportImpl.java | 6 +-- .../impl/PulsarResourceDescription.java | 2 +- .../impl/ResourceAvailabilityRanker.java | 2 +- .../impl/SimpleLoadManagerImpl.java | 20 +++++----- .../pulsar/broker/service/BrokerService.java | 2 +- .../pulsar/broker/service/PulsarStats.java | 2 +- .../service/persistent/PersistentTopic.java | 2 +- .../yahoo/pulsar/broker/admin/AdminTest.java | 2 +- .../broker/loadbalance/LoadBalancerTest.java | 9 ++--- .../SimpleLoadManagerImplTest.java | 24 ++++++------ .../service/PersistentTopicE2ETest.java | 2 +- .../data/loadbalancer}/BrokerUsage.java | 2 +- .../policies/data/loadbalancer}/JvmUsage.java | 2 +- .../data/loadbalancer}/LoadReport.java | 39 +++++++++++++++++-- .../loadbalancer}/NamespaceBundleStats.java | 2 +- .../NamespaceBundleStatsComparator.java | 4 +- .../data/loadbalancer}/NamespaceUsage.java | 2 +- .../loadbalancer}/ResourceUnitRanking.java | 8 ++-- .../data/loadbalancer}/ResourceUsage.java | 2 +- .../loadbalancer}/SystemResourceUsage.java | 4 +- 23 files changed, 87 insertions(+), 57 deletions(-) rename {pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data => pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer}/BrokerUsage.java (97%) rename {pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data => pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer}/JvmUsage.java (96%) rename {pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data => pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer}/LoadReport.java (85%) rename {pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data => pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer}/NamespaceBundleStats.java (98%) rename {pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data => pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer}/NamespaceBundleStatsComparator.java (92%) rename {pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data => pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer}/NamespaceUsage.java (98%) rename {pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data => pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer}/ResourceUnitRanking.java (98%) rename {pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data => pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer}/ResourceUsage.java (96%) rename {pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data => pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer}/SystemResourceUsage.java (95%) diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/BrokerStats.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/BrokerStats.java index 071f72dcc3004..10bb11c0e264d 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/BrokerStats.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/BrokerStats.java @@ -38,9 +38,9 @@ import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; import com.yahoo.pulsar.common.naming.NamespaceName; +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; import com.yahoo.pulsar.common.stats.AllocatorStats; import com.yahoo.pulsar.broker.loadbalance.ResourceUnit; -import com.yahoo.pulsar.broker.loadbalance.data.LoadReport; import com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; import com.yahoo.pulsar.broker.stats.AllocatorStatsGenerator; import com.yahoo.pulsar.broker.stats.BookieClientStatsGenerator; diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/LoadManager.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/LoadManager.java index 11abdeab8a494..8052d066be82d 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/LoadManager.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/LoadManager.java @@ -19,8 +19,8 @@ import com.google.common.collect.Lists; import com.yahoo.pulsar.common.naming.ServiceUnitId; +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; import com.yahoo.pulsar.broker.PulsarServerException; -import com.yahoo.pulsar.broker.loadbalance.data.LoadReport; import com.yahoo.pulsar.broker.stats.Metrics; /** diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/ResourceDescription.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/ResourceDescription.java index 0251794aa72ea..6b63b5923a68f 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/ResourceDescription.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/ResourceDescription.java @@ -17,7 +17,7 @@ import java.util.Map; -import com.yahoo.pulsar.broker.loadbalance.data.ResourceUsage; +import com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUsage; /* ResourceDescription is an abstraction to represent resources like memory, cpu, network and io combined; diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/PulsarLoadReportImpl.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/PulsarLoadReportImpl.java index 9bbfca59df23b..5a1e4d17596bc 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/PulsarLoadReportImpl.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/PulsarLoadReportImpl.java @@ -27,7 +27,7 @@ import com.yahoo.pulsar.broker.loadbalance.ResourceUnit; import com.yahoo.pulsar.broker.loadbalance.ServiceRequest; import com.yahoo.pulsar.broker.loadbalance.ServiceUnit; -import com.yahoo.pulsar.broker.loadbalance.data.SystemResourceUsage; +import com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage; import com.yahoo.pulsar.common.util.ObjectMapperFactory; public class PulsarLoadReportImpl implements LoadReport { @@ -50,8 +50,8 @@ public static LoadReport parse(String loadReportJson) { PulsarLoadReportImpl pulsarLoadReport = new PulsarLoadReportImpl(); ObjectMapper mapper = ObjectMapperFactory.create(); try { - com.yahoo.pulsar.broker.loadbalance.data.LoadReport report = mapper.readValue(loadReportJson, - com.yahoo.pulsar.broker.loadbalance.data.LoadReport.class); + com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport report = mapper.readValue(loadReportJson, + com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport.class); SystemResourceUsage sru = report.getSystemResourceUsage(); String resourceUnitName = report.getName(); pulsarLoadReport.resourceDescription = new PulsarResourceDescription(); diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/PulsarResourceDescription.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/PulsarResourceDescription.java index c9596ff43d498..099fcc1740b6d 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/PulsarResourceDescription.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/PulsarResourceDescription.java @@ -19,7 +19,7 @@ import java.util.Map; import com.yahoo.pulsar.broker.loadbalance.ResourceDescription; -import com.yahoo.pulsar.broker.loadbalance.data.ResourceUsage; +import com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUsage; public class PulsarResourceDescription extends ResourceDescription { diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/ResourceAvailabilityRanker.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/ResourceAvailabilityRanker.java index c69cfe9106ad7..8dc39ac9ce323 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/ResourceAvailabilityRanker.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/ResourceAvailabilityRanker.java @@ -20,7 +20,7 @@ import com.yahoo.pulsar.broker.loadbalance.LoadRanker; import com.yahoo.pulsar.broker.loadbalance.ResourceDescription; -import com.yahoo.pulsar.broker.loadbalance.data.ResourceUsage; +import com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUsage; import java.util.Comparator; import java.util.Map; diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index 90be7156cc55a..50842ded5d516 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -58,6 +58,11 @@ import com.yahoo.pulsar.common.naming.NamespaceName; import com.yahoo.pulsar.common.naming.ServiceUnitId; import com.yahoo.pulsar.common.policies.data.ResourceQuota; +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; +import com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats; +import com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUnitRanking; +import com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage; +import com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage.ResourceType; import com.yahoo.pulsar.common.util.ObjectMapperFactory; import com.yahoo.pulsar.broker.PulsarService; import com.yahoo.pulsar.broker.PulsarServerException; @@ -68,11 +73,6 @@ import com.yahoo.pulsar.broker.loadbalance.LoadRanker; import com.yahoo.pulsar.broker.loadbalance.PlacementStrategy; import com.yahoo.pulsar.broker.loadbalance.ResourceUnit; -import com.yahoo.pulsar.broker.loadbalance.data.LoadReport; -import com.yahoo.pulsar.broker.loadbalance.data.NamespaceBundleStats; -import com.yahoo.pulsar.broker.loadbalance.data.ResourceUnitRanking; -import com.yahoo.pulsar.broker.loadbalance.data.SystemResourceUsage; -import com.yahoo.pulsar.broker.loadbalance.data.SystemResourceUsage.ResourceType; import com.yahoo.pulsar.broker.stats.Metrics; import com.yahoo.pulsar.client.admin.PulsarAdmin; import com.yahoo.pulsar.client.api.Authentication; @@ -177,7 +177,8 @@ public SimpleLoadManagerImpl(PulsarService pulsar) { this.realtimeResourceQuotas.set(new HashMap<>()); this.realtimeAvgResourceQuota = new ResourceQuota(); placementStrategy = new WRRPlacementStrategy(); - lastLoadReport = new LoadReport(); + lastLoadReport = new LoadReport(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), + pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls()); brokerHostUsage = new BrokerHostUsage(pulsar); loadReportCacheZk = new ZooKeeperDataCache(pulsar.getLocalZkCache()) { @Override @@ -279,7 +280,7 @@ public void start() throws PulsarServerException { throw new PulsarServerException(e); } } - + @Override public void disableBroker() throws Exception { if (isNotEmpty(brokerZnodePath)) { @@ -625,7 +626,7 @@ public void writeResourceQuotasToZooKeeper() throws Exception { * - Available capacity for weighted random selection (weightedRandomSelection): ranks ResourceUnits units based on * estimation of their capacity which is basically how many bundles each ResourceUnit is able can handle with its * available resources (CPU, memory, network, etc); - * + * * - Load percentage for least loaded server (leastLoadedServer): ranks ResourceUnits units based on estimation of * their load percentage which is basically how many percent of resource is allocated which is * max(resource_actually_used, resource_quota) @@ -1091,7 +1092,8 @@ public LoadReport generateLoadReport() throws Exception { } try { - LoadReport loadReport = new LoadReport(); + LoadReport loadReport = new LoadReport(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), + pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls()); loadReport.setName(String.format("%s:%s", pulsar.getHost(), pulsar.getConfiguration().getWebServicePort())); SystemResourceUsage systemResourceUsage = this.getSystemResourceUsage(); loadReport.setOverLoaded( diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java index 13ae08728bb79..8df6784437046 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java @@ -55,7 +55,6 @@ import com.yahoo.pulsar.broker.admin.AdminResource; import com.yahoo.pulsar.broker.authentication.AuthenticationService; import com.yahoo.pulsar.broker.authorization.AuthorizationManager; -import com.yahoo.pulsar.broker.loadbalance.data.NamespaceBundleStats; import com.yahoo.pulsar.broker.service.BrokerServiceException.PersistenceException; import com.yahoo.pulsar.broker.service.BrokerServiceException.ServerMetadataException; import com.yahoo.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException; @@ -81,6 +80,7 @@ import com.yahoo.pulsar.common.policies.data.PersistentTopicStats; import com.yahoo.pulsar.common.policies.data.Policies; import com.yahoo.pulsar.common.policies.data.RetentionPolicies; +import com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats; import com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap; import com.yahoo.pulsar.zookeeper.ZooKeeperCacheListener; diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/PulsarStats.java index 66067f01c6e9b..350b1c7954ef7 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/PulsarStats.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/PulsarStats.java @@ -25,13 +25,13 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.yahoo.pulsar.broker.PulsarService; -import com.yahoo.pulsar.broker.loadbalance.data.NamespaceBundleStats; import com.yahoo.pulsar.broker.service.persistent.PersistentTopic; import com.yahoo.pulsar.broker.stats.BrokerOperabilityMetrics; import com.yahoo.pulsar.broker.stats.ClusterReplicationMetrics; import com.yahoo.pulsar.broker.stats.Metrics; import com.yahoo.pulsar.broker.stats.NamespaceStats; import com.yahoo.pulsar.common.naming.NamespaceBundle; +import com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats; import com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap; import com.yahoo.pulsar.utils.StatsOutputStream; diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/persistent/PersistentTopic.java index 97b533f5b010d..ba2a84e2c9de4 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/persistent/PersistentTopic.java @@ -59,6 +59,7 @@ import com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats; import com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats.CursorStats; import com.yahoo.pulsar.common.policies.data.PersistentTopicInternalStats.LedgerInfo; +import com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats; import com.yahoo.pulsar.common.policies.data.PersistentTopicStats; import com.yahoo.pulsar.common.policies.data.PublisherStats; import com.yahoo.pulsar.common.policies.data.Policies; @@ -67,7 +68,6 @@ import com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashMap; import com.yahoo.pulsar.common.util.collections.ConcurrentOpenHashSet; import com.yahoo.pulsar.broker.admin.AdminResource; -import com.yahoo.pulsar.broker.loadbalance.data.NamespaceBundleStats; import com.yahoo.pulsar.broker.service.Consumer; import com.yahoo.pulsar.broker.service.BrokerService; import com.yahoo.pulsar.broker.service.BrokerServiceException; diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminTest.java index 0edb7e6a60b7e..b376db79b92a5 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminTest.java @@ -60,7 +60,6 @@ import com.yahoo.pulsar.broker.auth.MockedPulsarServiceBaseTest; import com.yahoo.pulsar.broker.cache.ConfigurationCacheService; import com.yahoo.pulsar.broker.loadbalance.ResourceUnit; -import com.yahoo.pulsar.broker.loadbalance.data.LoadReport; import com.yahoo.pulsar.broker.stats.Metrics; import com.yahoo.pulsar.broker.web.PulsarWebResource; import com.yahoo.pulsar.broker.web.RestException; @@ -83,6 +82,7 @@ import com.yahoo.pulsar.broker.web.PulsarWebResource; import com.yahoo.pulsar.broker.web.RestException; import com.yahoo.pulsar.common.policies.data.ResourceQuota; +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; import com.yahoo.pulsar.common.util.ObjectMapperFactory; @Test diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/LoadBalancerTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/LoadBalancerTest.java index 7b2b71809c597..2ab40ecf181b3 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/LoadBalancerTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/LoadBalancerTest.java @@ -61,17 +61,12 @@ import com.yahoo.pulsar.broker.PulsarService; import com.yahoo.pulsar.broker.ServiceConfiguration; import com.yahoo.pulsar.broker.admin.AdminResource; -import com.yahoo.pulsar.broker.loadbalance.data.LoadReport; -import com.yahoo.pulsar.broker.loadbalance.data.NamespaceBundleStats; -import com.yahoo.pulsar.broker.loadbalance.data.ResourceUsage; -import com.yahoo.pulsar.broker.loadbalance.data.SystemResourceUsage; import com.yahoo.pulsar.broker.loadbalance.impl.PulsarResourceDescription; import com.yahoo.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker; import com.yahoo.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; import com.yahoo.pulsar.broker.loadbalance.impl.SimpleResourceUnit; import com.yahoo.pulsar.client.admin.PulsarAdmin; import com.yahoo.pulsar.client.admin.internal.NamespacesImpl; -import com.yahoo.pulsar.client.admin.internal.BrokerStatsImpl; import com.yahoo.pulsar.client.api.Authentication; import com.yahoo.pulsar.common.naming.DestinationName; import com.yahoo.pulsar.common.naming.NamespaceName; @@ -81,6 +76,10 @@ import com.yahoo.pulsar.common.policies.data.NamespaceIsolationData; import com.yahoo.pulsar.common.policies.data.Policies; import com.yahoo.pulsar.common.policies.data.ResourceQuota; +import com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport; +import com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats; +import com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUsage; +import com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage; import com.yahoo.pulsar.common.policies.impl.NamespaceIsolationPolicies; import com.yahoo.pulsar.common.util.ObjectMapperFactory; import com.yahoo.pulsar.zookeeper.LocalBookkeeperEnsemble; diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java index a87a9b8688146..ca8dc358f7f22 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java @@ -57,12 +57,6 @@ import com.yahoo.pulsar.broker.PulsarService; import com.yahoo.pulsar.broker.ServiceConfiguration; import com.yahoo.pulsar.broker.admin.AdminResource; -import com.yahoo.pulsar.broker.loadbalance.data.BrokerUsage; -import com.yahoo.pulsar.broker.loadbalance.data.JvmUsage; -import com.yahoo.pulsar.broker.loadbalance.data.NamespaceBundleStats; -import com.yahoo.pulsar.broker.loadbalance.data.ResourceUnitRanking; -import com.yahoo.pulsar.broker.loadbalance.data.ResourceUsage; -import com.yahoo.pulsar.broker.loadbalance.data.SystemResourceUsage; import com.yahoo.pulsar.broker.loadbalance.impl.PulsarLoadReportImpl; import com.yahoo.pulsar.broker.loadbalance.impl.PulsarResourceDescription; import com.yahoo.pulsar.broker.loadbalance.impl.ResourceAvailabilityRanker; @@ -77,6 +71,12 @@ import com.yahoo.pulsar.common.policies.data.AutoFailoverPolicyType; import com.yahoo.pulsar.common.policies.data.NamespaceIsolationData; import com.yahoo.pulsar.common.policies.data.ResourceQuota; +import com.yahoo.pulsar.common.policies.data.loadbalancer.BrokerUsage; +import com.yahoo.pulsar.common.policies.data.loadbalancer.JvmUsage; +import com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats; +import com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUnitRanking; +import com.yahoo.pulsar.common.policies.data.loadbalancer.ResourceUsage; +import com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage; import com.yahoo.pulsar.common.policies.impl.NamespaceIsolationPolicies; import com.yahoo.pulsar.common.util.ObjectMapperFactory; import com.yahoo.pulsar.zookeeper.LocalBookkeeperEnsemble; @@ -243,8 +243,8 @@ public void testPrimary() throws Exception { LoadRanker lr = new ResourceAvailabilityRanker(); // inject the load report and rankings - Map loadReports = new HashMap<>(); - com.yahoo.pulsar.broker.loadbalance.data.LoadReport loadReport = new com.yahoo.pulsar.broker.loadbalance.data.LoadReport(); + Map loadReports = new HashMap<>(); + com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport loadReport = new com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport(); loadReport.setSystemResourceUsage(new SystemResourceUsage()); loadReports.put(ru1, loadReport); setObjectField(SimpleLoadManagerImpl.class, loadManager, "currentLoadReports", loadReports); @@ -349,7 +349,7 @@ public void testResourceDescription() { public void testLoadReportParsing() throws Exception { ObjectMapper mapper = ObjectMapperFactory.create(); - com.yahoo.pulsar.broker.loadbalance.data.LoadReport reportData = new com.yahoo.pulsar.broker.loadbalance.data.LoadReport(); + com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport reportData = new com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport(); reportData.setName("b1"); SystemResourceUsage resource = new SystemResourceUsage(); ResourceUsage resourceUsage = new ResourceUsage(); @@ -398,11 +398,11 @@ public void testDoLoadShedding() throws Exception { stats.put("property/cluster/namespace1/0x00000000_0xFFFFFFFF", nsb1); stats.put("property/cluster/namespace2/0x00000000_0xFFFFFFFF", nsb2); - Map loadReports = new HashMap<>(); - com.yahoo.pulsar.broker.loadbalance.data.LoadReport loadReport1 = new com.yahoo.pulsar.broker.loadbalance.data.LoadReport(); + Map loadReports = new HashMap<>(); + com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport loadReport1 = new com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport(); loadReport1.setSystemResourceUsage(systemResource); loadReport1.setBundleStats(stats); - com.yahoo.pulsar.broker.loadbalance.data.LoadReport loadReport2 = new com.yahoo.pulsar.broker.loadbalance.data.LoadReport(); + com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport loadReport2 = new com.yahoo.pulsar.common.policies.data.loadbalancer.LoadReport(); loadReport2.setSystemResourceUsage(new SystemResourceUsage()); loadReport2.setBundleStats(stats); loadReports.put(ru1, loadReport1); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/PersistentTopicE2ETest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/PersistentTopicE2ETest.java index 08f0db76e252d..b504ccc277f25 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/PersistentTopicE2ETest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/PersistentTopicE2ETest.java @@ -44,7 +44,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.yahoo.pulsar.broker.loadbalance.data.NamespaceBundleStats; import com.yahoo.pulsar.broker.service.persistent.PersistentSubscription; import com.yahoo.pulsar.broker.service.persistent.PersistentTopic; import com.yahoo.pulsar.broker.stats.Metrics; @@ -65,6 +64,7 @@ import com.yahoo.pulsar.client.impl.ProducerImpl; import com.yahoo.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType; import com.yahoo.pulsar.common.naming.DestinationName; +import com.yahoo.pulsar.common.policies.data.loadbalancer.NamespaceBundleStats; /** */ diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/BrokerUsage.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/BrokerUsage.java similarity index 97% rename from pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/BrokerUsage.java rename to pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/BrokerUsage.java index ab63d95585580..5cc94510966ed 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/BrokerUsage.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/BrokerUsage.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.yahoo.pulsar.broker.loadbalance.data; +package com.yahoo.pulsar.common.policies.data.loadbalancer; import java.util.Map; diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/JvmUsage.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/JvmUsage.java similarity index 96% rename from pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/JvmUsage.java rename to pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/JvmUsage.java index 834389290fb68..82e0f6464465c 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/JvmUsage.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/JvmUsage.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.yahoo.pulsar.broker.loadbalance.data; +package com.yahoo.pulsar.common.policies.data.loadbalancer; import java.util.Map; diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/LoadReport.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/LoadReport.java similarity index 85% rename from pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/LoadReport.java rename to pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/LoadReport.java index ae964746f2a06..2cc4a7cc8104d 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/LoadReport.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/LoadReport.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.yahoo.pulsar.broker.loadbalance.data; +package com.yahoo.pulsar.common.policies.data.loadbalancer; import java.util.HashMap; import java.util.HashSet; @@ -23,16 +23,24 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.Maps; -import com.yahoo.pulsar.broker.loadbalance.data.SystemResourceUsage.ResourceType; +import com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage.ResourceType; + /** * This class represents the overall load of the broker - it includes overall {@link SystemResourceUsage} and * {@link NamespaceUsage} for all the namespaces hosted by this broker. */ public class LoadReport { + private String name; + + private final String webServiceUrl; + private final String webServiceUrlTls; + + private final String pulsarServiceUrl; + private final String pulsarServieUrlTls; + private boolean isUnderLoaded; private boolean isOverLoaded; - private String name; private long timestamp; private double msgRateIn; private double msgRateOut; @@ -42,6 +50,15 @@ public class LoadReport { private long numBundles; public LoadReport() { + this(null, null, null, null); + } + + public LoadReport(String webServiceUrl, String webServiceUrlTls, String pulsarServiceUrl, String pulsarServieUrlTls) { + this.webServiceUrl = webServiceUrl; + this.webServiceUrlTls = webServiceUrlTls; + this.pulsarServiceUrl = pulsarServiceUrl; + this.pulsarServieUrlTls = pulsarServieUrlTls; + isUnderLoaded = false; isOverLoaded = false; timestamp = 0; @@ -208,4 +225,20 @@ public TreeMap getSortedBundleStats(ResourceType r sortedBundleStats.putAll(bundleStats); return sortedBundleStats; } + + public String getWebServiceUrl() { + return webServiceUrl; + } + + public String getWebServiceUrlTls() { + return webServiceUrlTls; + } + + public String getPulsarServiceUrl() { + return pulsarServiceUrl; + } + + public String getPulsarServieUrlTls() { + return pulsarServieUrlTls; + } } diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/NamespaceBundleStats.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/NamespaceBundleStats.java similarity index 98% rename from pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/NamespaceBundleStats.java rename to pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/NamespaceBundleStats.java index 7f7df8cc5157a..5cc5e0fae1e9d 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/NamespaceBundleStats.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/NamespaceBundleStats.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.yahoo.pulsar.broker.loadbalance.data; +package com.yahoo.pulsar.common.policies.data.loadbalancer; /** */ diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/NamespaceBundleStatsComparator.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/NamespaceBundleStatsComparator.java similarity index 92% rename from pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/NamespaceBundleStatsComparator.java rename to pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/NamespaceBundleStatsComparator.java index 233c5356187d1..d77c8d9357f81 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/NamespaceBundleStatsComparator.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/NamespaceBundleStatsComparator.java @@ -13,12 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.yahoo.pulsar.broker.loadbalance.data; +package com.yahoo.pulsar.common.policies.data.loadbalancer; import java.util.Comparator; import java.util.Map; -import com.yahoo.pulsar.broker.loadbalance.data.SystemResourceUsage.ResourceType; +import com.yahoo.pulsar.common.policies.data.loadbalancer.SystemResourceUsage.ResourceType; /** */ diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/NamespaceUsage.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/NamespaceUsage.java similarity index 98% rename from pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/NamespaceUsage.java rename to pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/NamespaceUsage.java index e5c14c6fba8ba..6a804973f252a 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/NamespaceUsage.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/NamespaceUsage.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.yahoo.pulsar.broker.loadbalance.data; +package com.yahoo.pulsar.common.policies.data.loadbalancer; /** * this class represents usages of resources used by a namespace, this is an aggregate of all the topics and queues on diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/ResourceUnitRanking.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/ResourceUnitRanking.java similarity index 98% rename from pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/ResourceUnitRanking.java rename to pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/ResourceUnitRanking.java index 47827e050b57d..d397e8378fb15 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/ResourceUnitRanking.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/ResourceUnitRanking.java @@ -13,16 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.yahoo.pulsar.broker.loadbalance.data; +package com.yahoo.pulsar.common.policies.data.loadbalancer; -import java.util.Map; import java.util.Set; import com.yahoo.pulsar.common.policies.data.ResourceQuota; public class ResourceUnitRanking implements Comparable { - private static final long MBytes = 1024 * 1024; private static final long KBITS_TO_BYTES = 1024 / 8; private static final double PERCENTAGE_DIFFERENCE_THRESHOLD = 5.0; private static double cpuUsageByMsgRate = 0.05; @@ -71,9 +69,9 @@ public static void setCpuUsageByMsgRate(double cpuUsageByMsgRate) { */ private void estimateLoadPercentage() { double cpuUsed = this.systemResourceUsage.cpu.usage; - double cpuAllocated = this.cpuUsageByMsgRate + double cpuAllocated = cpuUsageByMsgRate * (this.allocatedQuota.getMsgRateIn() + this.allocatedQuota.getMsgRateOut()); - double cpuPreAllocated = this.cpuUsageByMsgRate + double cpuPreAllocated = cpuUsageByMsgRate * (this.preAllocatedQuota.getMsgRateIn() + this.preAllocatedQuota.getMsgRateOut()); this.allocatedLoadPercentageCPU = (this.systemResourceUsage.cpu.limit <= 0) ? 0 : Math.min(100, 100 * cpuAllocated / this.systemResourceUsage.cpu.limit); diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/ResourceUsage.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/ResourceUsage.java similarity index 96% rename from pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/ResourceUsage.java rename to pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/ResourceUsage.java index 3949f2dea9e19..d5ad985773144 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/ResourceUsage.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/ResourceUsage.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.yahoo.pulsar.broker.loadbalance.data; +package com.yahoo.pulsar.common.policies.data.loadbalancer; /** * POJO used to represents any system specific resource usage this is the format that load manager expects it in. diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/SystemResourceUsage.java b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/SystemResourceUsage.java similarity index 95% rename from pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/SystemResourceUsage.java rename to pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/SystemResourceUsage.java index 1af165ecbd041..ba04571fc5f96 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/data/SystemResourceUsage.java +++ b/pulsar-common/src/main/java/com/yahoo/pulsar/common/policies/data/loadbalancer/SystemResourceUsage.java @@ -13,9 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.yahoo.pulsar.broker.loadbalance.data; - -import com.yahoo.pulsar.broker.ServiceConfiguration; +package com.yahoo.pulsar.common.policies.data.loadbalancer; /** * This class represents a object which reflects system resource usage per resource and the upper limit on the resource.