Skip to content

Commit

Permalink
[cleanup][broker] Remove references to blacklist/whitelist (apache#19407
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@cbornet
cbornet authored Feb 2, 2023
1 parent 3a069ed commit 595a125
Show file tree
Hide file tree
Showing 5 changed files with 30 additions and 36 deletions.
2 changes: 1 addition & 1 deletion buildtools/src/main/resources/pulsar/checkstyle.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ page at http://checkstyle.sourceforge.net/config.html -->
</module>

<!--
IllegalImport cannot blacklist classes, and c.g.api.client.util is used for some shaded
IllegalImport cannot exclude classes, and c.g.api.client.util is used for some shaded
code and some useful code. So we need to fall back to Regexp.
-->
<module name="RegexpSinglelineJava">
Expand Down
56 changes: 25 additions & 31 deletions ...in/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,7 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf,
String isolationGroupsString = ConfigurationStringUtil
.castToString(conf.getProperty(ISOLATION_BOOKIE_GROUPS));
if (!isolationGroupsString.isEmpty()) {
for (String isolationGroup : isolationGroupsString.split(",")) {
primaryIsolationGroups.add(isolationGroup);
}
Collections.addAll(primaryIsolationGroups, isolationGroupsString.split(","));
}
// Only add the bookieMappingCache if we have defined an isolation group
bookieMappingCache = store.getMetadataCache(BookiesRackConfiguration.class);
Expand All @@ -94,9 +92,7 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf,
String secondaryIsolationGroupsString = ConfigurationStringUtil
.castToString(conf.getProperty(SECONDARY_ISOLATION_BOOKIE_GROUPS));
if (!secondaryIsolationGroupsString.isEmpty()) {
for (String isolationGroup : secondaryIsolationGroupsString.split(",")) {
secondaryIsolationGroups.add(isolationGroup);
}
Collections.addAll(secondaryIsolationGroups, secondaryIsolationGroupsString.split(","));
}
}
defaultIsolationGroups = ImmutablePair.of(primaryIsolationGroups, secondaryIsolationGroups);
Expand All @@ -107,11 +103,10 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf,
public PlacementResult<List<BookieId>> newEnsemble(int ensembleSize, int writeQuorumSize, int ackQuorumSize,
Map<String, byte[]> customMetadata, Set<BookieId> excludeBookies)
throws BKNotEnoughBookiesException {
Set<BookieId> blacklistedBookies = getBlacklistedBookies(ensembleSize, customMetadata);
if (excludeBookies == null) {
excludeBookies = new HashSet<BookieId>();
excludeBookies = new HashSet<>();
}
excludeBookies.addAll(blacklistedBookies);
excludeBookies.addAll(getExcludedBookies(ensembleSize, customMetadata));
return super.newEnsemble(ensembleSize, writeQuorumSize, ackQuorumSize, customMetadata, excludeBookies);
}

Expand All @@ -120,29 +115,28 @@ public PlacementResult<BookieId> replaceBookie(int ensembleSize, int writeQuorum
Map<String, byte[]> customMetadata, List<BookieId> currentEnsemble,
BookieId bookieToReplace, Set<BookieId> excludeBookies)
throws BKNotEnoughBookiesException {
Set<BookieId> blacklistedBookies = getBlacklistedBookies(ensembleSize, customMetadata);
if (excludeBookies == null) {
excludeBookies = new HashSet<BookieId>();
excludeBookies = new HashSet<>();
}
excludeBookies.addAll(blacklistedBookies);
excludeBookies.addAll(getExcludedBookies(ensembleSize, customMetadata));
return super.replaceBookie(ensembleSize, writeQuorumSize, ackQuorumSize, customMetadata, currentEnsemble,
bookieToReplace, excludeBookies);
}

private Set<BookieId> getBlacklistedBookies(int ensembleSize, Map<String, byte[]> customMetadata){
private Set<BookieId> getExcludedBookies(int ensembleSize, Map<String, byte[]> customMetadata){
// parse the ensemble placement policy from the custom metadata, if it is present, we will apply it to
// the isolation groups for filtering the bookies.
Optional<EnsemblePlacementPolicyConfig> ensemblePlacementPolicyConfig =
getEnsemblePlacementPolicyConfig(customMetadata);
Set<BookieId> blacklistedBookies;
Set<BookieId> excludedBookies;
if (ensemblePlacementPolicyConfig.isPresent()) {
EnsemblePlacementPolicyConfig config = ensemblePlacementPolicyConfig.get();
Pair<Set<String>, Set<String>> groups = getIsolationGroup(config);
blacklistedBookies = getBlacklistedBookiesWithIsolationGroups(ensembleSize, groups);
excludedBookies = getExcludedBookiesWithIsolationGroups(ensembleSize, groups);
} else {
blacklistedBookies = getBlacklistedBookiesWithIsolationGroups(ensembleSize, defaultIsolationGroups);
excludedBookies = getExcludedBookiesWithIsolationGroups(ensembleSize, defaultIsolationGroups);
}
return blacklistedBookies;
return excludedBookies;
}

private static Optional<EnsemblePlacementPolicyConfig> getEnsemblePlacementPolicyConfig(
Expand Down Expand Up @@ -172,24 +166,24 @@ private static Pair<Set<String>, Set<String>> getIsolationGroup(
String secondaryIsolationGroupString = ConfigurationStringUtil
.castToString(properties.getOrDefault(SECONDARY_ISOLATION_BOOKIE_GROUPS, ""));
if (!primaryIsolationGroupString.isEmpty()) {
pair.setLeft(new HashSet(Arrays.asList(primaryIsolationGroupString.split(","))));
pair.setLeft(new HashSet<>(Arrays.asList(primaryIsolationGroupString.split(","))));
} else {
pair.setLeft(Collections.emptySet());
}
if (!secondaryIsolationGroupString.isEmpty()) {
pair.setRight(new HashSet(Arrays.asList(secondaryIsolationGroupString.split(","))));
pair.setRight(new HashSet<>(Arrays.asList(secondaryIsolationGroupString.split(","))));
} else {
pair.setRight(Collections.emptySet());
}
}
return pair;
}

private Set<BookieId> getBlacklistedBookiesWithIsolationGroups(int ensembleSize,
private Set<BookieId> getExcludedBookiesWithIsolationGroups(int ensembleSize,
Pair<Set<String>, Set<String>> isolationGroups) {
Set<BookieId> blacklistedBookies = new HashSet<>();
Set<BookieId> excludedBookies = new HashSet<>();
if (isolationGroups != null && isolationGroups.getLeft().contains(PULSAR_SYSTEM_TOPIC_ISOLATION_GROUP)) {
return blacklistedBookies;
return excludedBookies;
}
try {
if (bookieMappingCache != null) {
Expand All @@ -199,8 +193,8 @@ private Set<BookieId> getBlacklistedBookiesWithIsolationGroups(int ensembleSize,
Optional<BookiesRackConfiguration> optRes = (future.isDone() && !future.isCompletedExceptionally())
? future.join() : Optional.empty();

if (!optRes.isPresent()) {
return blacklistedBookies;
if (optRes.isEmpty()) {
return excludedBookies;
}

BookiesRackConfiguration allGroupsBookieMapping = optRes.get();
Expand All @@ -217,7 +211,7 @@ private Set<BookieId> getBlacklistedBookiesWithIsolationGroups(int ensembleSize,
Set<String> bookiesInGroup = allGroupsBookieMapping.get(group).keySet();
if (!primaryIsolationGroup.contains(group)) {
for (String bookieAddress : bookiesInGroup) {
blacklistedBookies.add(BookieId.parse(bookieAddress));
excludedBookies.add(BookieId.parse(bookieAddress));
}
} else {
for (String groupBookie : bookiesInGroup) {
Expand All @@ -228,10 +222,10 @@ private Set<BookieId> getBlacklistedBookiesWithIsolationGroups(int ensembleSize,
}
}

Set<BookieId> otherGroupBookies = new HashSet<>(blacklistedBookies);
Set<BookieId> otherGroupBookies = new HashSet<>(excludedBookies);
Set<BookieId> nonRegionBookies = new HashSet<>(knownBookies.keySet());
nonRegionBookies.removeAll(primaryGroupBookies);
blacklistedBookies.addAll(nonRegionBookies);
excludedBookies.addAll(nonRegionBookies);

// sometime while doing isolation, user might not want to remove isolated bookies from other default
// groups. so, same set of bookies could be overlapped into isolated-group and other default groups. so,
Expand All @@ -241,7 +235,7 @@ private Set<BookieId> getBlacklistedBookiesWithIsolationGroups(int ensembleSize,
Map<String, BookieInfo> bookieGroup = allGroupsBookieMapping.get(group);
if (bookieGroup != null && !bookieGroup.isEmpty()) {
for (String bookieAddress : bookieGroup.keySet()) {
blacklistedBookies.remove(BookieId.parse(bookieAddress));
excludedBookies.remove(BookieId.parse(bookieAddress));
}
}
}
Expand All @@ -255,7 +249,7 @@ private Set<BookieId> getBlacklistedBookiesWithIsolationGroups(int ensembleSize,
Map<String, BookieInfo> bookieGroup = allGroupsBookieMapping.get(group);
if (bookieGroup != null && !bookieGroup.isEmpty()) {
for (String bookieAddress : bookieGroup.keySet()) {
blacklistedBookies.remove(BookieId.parse(bookieAddress));
excludedBookies.remove(BookieId.parse(bookieAddress));
totalAvailableBookiesFromPrimaryAndSecondary += 1;
}
}
Expand All @@ -268,13 +262,13 @@ private Set<BookieId> getBlacklistedBookiesWithIsolationGroups(int ensembleSize,
primaryIsolationGroup, secondaryIsolationGroup);
nonRegionBookies.removeAll(otherGroupBookies);
for (BookieId bookie: nonRegionBookies) {
blacklistedBookies.remove(bookie);
excludedBookies.remove(bookie);
}
}
}
} catch (Exception e) {
log.warn("Error getting bookie isolation info from metadata store: {}", e.getMessage());
}
return blacklistedBookies;
return excludedBookies;
}
}
4 changes: 2 additions & 2 deletions ...ava/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ public void testNoIsolationGroup() throws Exception {
* <pre>
* a. default-group has all 5 bookies.
* b. 3 of the default-group bookies have been added to isolated-group without being removed from default-group.
* c. isolated-policy-placement should be identify those 3 overlapped bookies and exclude them from blacklisted bookies.
* c. isolated-policy-placement should be identify those 3 overlapped bookies and remove them from excluded bookies.
* </pre>
*
* @throws Exception
Expand Down Expand Up @@ -494,7 +494,7 @@ public void testTheIsolationPolicyUsingCustomMetadata() throws Exception {
Optional.empty()).join();

// prepare a custom placement policy and put it into the custom metadata. The isolation policy should decode
// from the custom metadata and apply it to the get black list method.
// from the custom metadata and apply it to the get excluded list method.
Map<String, Object> placementPolicyProperties = new HashMap<>();
placementPolicyProperties.put(
IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, primaryGroupName);
Expand Down
2 changes: 1 addition & 1 deletion pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ private static void initializeCluster(Arguments arguments, int bundleNumberForDe
resources.getClusterResources().createCluster("global", globalClusterData);
}

// Create public tenant, whitelisted to use the this same cluster, along with other clusters
// Create public tenant, allowed to use this same cluster, along with other clusters
createTenantIfAbsent(resources, TopicName.PUBLIC_TENANT, arguments.cluster);

// Create system tenant
Expand Down
2 changes: 1 addition & 1 deletion pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/BrokerFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
* Load management component which determines what brokers should not be considered for topic placement by the placement
* strategy. For example, the placement strategy may determine that the broker with the least msg/s should get the
* bundle assignment, but we may not want to consider brokers whose CPU usage is very high. Thus, we could use a filter
* to blacklist brokers with high CPU usage.
* to exclude brokers with high CPU usage.
*/
public interface BrokerFilter {

Expand Down

0 comments on commit 595a125

Please sign in to comment.