From 595a125f97f1618a60901706a06589350477dcf2 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Thu, 2 Feb 2023 23:22:59 +0100 Subject: [PATCH] [cleanup][broker] Remove references to blacklist/whitelist (#19407) --- .../src/main/resources/pulsar/checkstyle.xml | 2 +- ...IsolatedBookieEnsemblePlacementPolicy.java | 56 +++++++++---------- ...atedBookieEnsemblePlacementPolicyTest.java | 4 +- .../pulsar/PulsarClusterMetadataSetup.java | 2 +- .../broker/loadbalance/BrokerFilter.java | 2 +- 5 files changed, 30 insertions(+), 36 deletions(-) diff --git a/buildtools/src/main/resources/pulsar/checkstyle.xml b/buildtools/src/main/resources/pulsar/checkstyle.xml index b5141cc5eb51a..b3812ca8cccd7 100644 --- a/buildtools/src/main/resources/pulsar/checkstyle.xml +++ b/buildtools/src/main/resources/pulsar/checkstyle.xml @@ -148,7 +148,7 @@ page at http://checkstyle.sourceforge.net/config.html --> diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java index 9e416985ec553..2594798485a20 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java @@ -82,9 +82,7 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf, String isolationGroupsString = ConfigurationStringUtil .castToString(conf.getProperty(ISOLATION_BOOKIE_GROUPS)); if (!isolationGroupsString.isEmpty()) { - for (String isolationGroup : isolationGroupsString.split(",")) { - primaryIsolationGroups.add(isolationGroup); - } + Collections.addAll(primaryIsolationGroups, isolationGroupsString.split(",")); } // Only add the bookieMappingCache if we have defined an isolation group bookieMappingCache = store.getMetadataCache(BookiesRackConfiguration.class); @@ -94,9 +92,7 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf, String secondaryIsolationGroupsString = ConfigurationStringUtil .castToString(conf.getProperty(SECONDARY_ISOLATION_BOOKIE_GROUPS)); if (!secondaryIsolationGroupsString.isEmpty()) { - for (String isolationGroup : secondaryIsolationGroupsString.split(",")) { - secondaryIsolationGroups.add(isolationGroup); - } + Collections.addAll(secondaryIsolationGroups, secondaryIsolationGroupsString.split(",")); } } defaultIsolationGroups = ImmutablePair.of(primaryIsolationGroups, secondaryIsolationGroups); @@ -107,11 +103,10 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf, public PlacementResult> newEnsemble(int ensembleSize, int writeQuorumSize, int ackQuorumSize, Map customMetadata, Set excludeBookies) throws BKNotEnoughBookiesException { - Set blacklistedBookies = getBlacklistedBookies(ensembleSize, customMetadata); if (excludeBookies == null) { - excludeBookies = new HashSet(); + excludeBookies = new HashSet<>(); } - excludeBookies.addAll(blacklistedBookies); + excludeBookies.addAll(getExcludedBookies(ensembleSize, customMetadata)); return super.newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, customMetadata, excludeBookies); } @@ -120,29 +115,28 @@ public PlacementResult replaceBookie(int ensembleSize, int writeQuorum Map customMetadata, List currentEnsemble, BookieId bookieToReplace, Set excludeBookies) throws BKNotEnoughBookiesException { - Set blacklistedBookies = getBlacklistedBookies(ensembleSize, customMetadata); if (excludeBookies == null) { - excludeBookies = new HashSet(); + excludeBookies = new HashSet<>(); } - excludeBookies.addAll(blacklistedBookies); + excludeBookies.addAll(getExcludedBookies(ensembleSize, customMetadata)); return super.replaceBookie(ensembleSize, writeQuorumSize, ackQuorumSize, customMetadata, currentEnsemble, bookieToReplace, excludeBookies); } - private Set getBlacklistedBookies(int ensembleSize, Map customMetadata){ + private Set getExcludedBookies(int ensembleSize, Map customMetadata){ // parse the ensemble placement policy from the custom metadata, if it is present, we will apply it to // the isolation groups for filtering the bookies. Optional ensemblePlacementPolicyConfig = getEnsemblePlacementPolicyConfig(customMetadata); - Set blacklistedBookies; + Set excludedBookies; if (ensemblePlacementPolicyConfig.isPresent()) { EnsemblePlacementPolicyConfig config = ensemblePlacementPolicyConfig.get(); Pair, Set> groups = getIsolationGroup(config); - blacklistedBookies = getBlacklistedBookiesWithIsolationGroups(ensembleSize, groups); + excludedBookies = getExcludedBookiesWithIsolationGroups(ensembleSize, groups); } else { - blacklistedBookies = getBlacklistedBookiesWithIsolationGroups(ensembleSize, defaultIsolationGroups); + excludedBookies = getExcludedBookiesWithIsolationGroups(ensembleSize, defaultIsolationGroups); } - return blacklistedBookies; + return excludedBookies; } private static Optional getEnsemblePlacementPolicyConfig( @@ -172,12 +166,12 @@ private static Pair, Set> getIsolationGroup( String secondaryIsolationGroupString = ConfigurationStringUtil .castToString(properties.getOrDefault(SECONDARY_ISOLATION_BOOKIE_GROUPS, "")); if (!primaryIsolationGroupString.isEmpty()) { - pair.setLeft(new HashSet(Arrays.asList(primaryIsolationGroupString.split(",")))); + pair.setLeft(new HashSet<>(Arrays.asList(primaryIsolationGroupString.split(",")))); } else { pair.setLeft(Collections.emptySet()); } if (!secondaryIsolationGroupString.isEmpty()) { - pair.setRight(new HashSet(Arrays.asList(secondaryIsolationGroupString.split(",")))); + pair.setRight(new HashSet<>(Arrays.asList(secondaryIsolationGroupString.split(",")))); } else { pair.setRight(Collections.emptySet()); } @@ -185,11 +179,11 @@ private static Pair, Set> getIsolationGroup( return pair; } - private Set getBlacklistedBookiesWithIsolationGroups(int ensembleSize, + private Set getExcludedBookiesWithIsolationGroups(int ensembleSize, Pair, Set> isolationGroups) { - Set blacklistedBookies = new HashSet<>(); + Set excludedBookies = new HashSet<>(); if (isolationGroups != null && isolationGroups.getLeft().contains(PULSAR_SYSTEM_TOPIC_ISOLATION_GROUP)) { - return blacklistedBookies; + return excludedBookies; } try { if (bookieMappingCache != null) { @@ -199,8 +193,8 @@ private Set getBlacklistedBookiesWithIsolationGroups(int ensembleSize, Optional optRes = (future.isDone() && !future.isCompletedExceptionally()) ? future.join() : Optional.empty(); - if (!optRes.isPresent()) { - return blacklistedBookies; + if (optRes.isEmpty()) { + return excludedBookies; } BookiesRackConfiguration allGroupsBookieMapping = optRes.get(); @@ -217,7 +211,7 @@ private Set getBlacklistedBookiesWithIsolationGroups(int ensembleSize, Set bookiesInGroup = allGroupsBookieMapping.get(group).keySet(); if (!primaryIsolationGroup.contains(group)) { for (String bookieAddress : bookiesInGroup) { - blacklistedBookies.add(BookieId.parse(bookieAddress)); + excludedBookies.add(BookieId.parse(bookieAddress)); } } else { for (String groupBookie : bookiesInGroup) { @@ -228,10 +222,10 @@ private Set getBlacklistedBookiesWithIsolationGroups(int ensembleSize, } } - Set otherGroupBookies = new HashSet<>(blacklistedBookies); + Set otherGroupBookies = new HashSet<>(excludedBookies); Set nonRegionBookies = new HashSet<>(knownBookies.keySet()); nonRegionBookies.removeAll(primaryGroupBookies); - blacklistedBookies.addAll(nonRegionBookies); + excludedBookies.addAll(nonRegionBookies); // sometime while doing isolation, user might not want to remove isolated bookies from other default // groups. so, same set of bookies could be overlapped into isolated-group and other default groups. so, @@ -241,7 +235,7 @@ private Set getBlacklistedBookiesWithIsolationGroups(int ensembleSize, Map bookieGroup = allGroupsBookieMapping.get(group); if (bookieGroup != null && !bookieGroup.isEmpty()) { for (String bookieAddress : bookieGroup.keySet()) { - blacklistedBookies.remove(BookieId.parse(bookieAddress)); + excludedBookies.remove(BookieId.parse(bookieAddress)); } } } @@ -255,7 +249,7 @@ private Set getBlacklistedBookiesWithIsolationGroups(int ensembleSize, Map bookieGroup = allGroupsBookieMapping.get(group); if (bookieGroup != null && !bookieGroup.isEmpty()) { for (String bookieAddress : bookieGroup.keySet()) { - blacklistedBookies.remove(BookieId.parse(bookieAddress)); + excludedBookies.remove(BookieId.parse(bookieAddress)); totalAvailableBookiesFromPrimaryAndSecondary += 1; } } @@ -268,13 +262,13 @@ private Set getBlacklistedBookiesWithIsolationGroups(int ensembleSize, primaryIsolationGroup, secondaryIsolationGroup); nonRegionBookies.removeAll(otherGroupBookies); for (BookieId bookie: nonRegionBookies) { - blacklistedBookies.remove(bookie); + excludedBookies.remove(bookie); } } } } catch (Exception e) { log.warn("Error getting bookie isolation info from metadata store: {}", e.getMessage()); } - return blacklistedBookies; + return excludedBookies; } } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java index 5ca946bbbb501..f535ced08f731 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java @@ -339,7 +339,7 @@ public void testNoIsolationGroup() throws Exception { * 
      * a. default-group has all 5 bookies.
      * b. 3 of the default-group bookies have been added to isolated-group without being removed from default-group.
-     * c. isolated-policy-placement should be identify those 3 overlapped bookies and exclude them from blacklisted bookies.
+     * c. isolated-policy-placement should be identify those 3 overlapped bookies and remove them from excluded bookies.
      *
* * @throws Exception @@ -494,7 +494,7 @@ public void testTheIsolationPolicyUsingCustomMetadata() throws Exception { Optional.empty()).join(); // prepare a custom placement policy and put it into the custom metadata. The isolation policy should decode - // from the custom metadata and apply it to the get black list method. + // from the custom metadata and apply it to the get excluded list method. Map placementPolicyProperties = new HashMap<>(); placementPolicyProperties.put( IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, primaryGroupName); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java index 6b0b4ed8d730c..1a2ca1ec4fb53 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java @@ -333,7 +333,7 @@ private static void initializeCluster(Arguments arguments, int bundleNumberForDe resources.getClusterResources().createCluster("global", globalClusterData); } - // Create public tenant, whitelisted to use the this same cluster, along with other clusters + // Create public tenant, allowed to use this same cluster, along with other clusters createTenantIfAbsent(resources, TopicName.PUBLIC_TENANT, arguments.cluster); // Create system tenant diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/BrokerFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/BrokerFilter.java index 1e5e41488d623..3313e138cd68e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/BrokerFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/BrokerFilter.java @@ -26,7 +26,7 @@ * Load management component which determines what brokers should not be considered for topic placement by the placement * strategy. For example, the placement strategy may determine that the broker with the least msg/s should get the * bundle assignment, but we may not want to consider brokers whose CPU usage is very high. Thus, we could use a filter - * to blacklist brokers with high CPU usage. + * to exclude brokers with high CPU usage. */ public interface BrokerFilter {