Upgrade bk version to 4.10.0 (apache#5607)

* Upgrade bk version to 4.10.0 Signed-off-by: xiaolong.ran <[email protected]> * Handle NoSuchLedgerExistsOnMetadataServer error
wanghhao · Nov 14, 2019 · b866787 · b866787
1 parent 74e210e
commit b866787
Show file tree

Hide file tree

Showing 12 changed files with 123 additions and 59 deletions.
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -381,32 +381,32 @@ The Apache Software License, Version 2.0
     - org.apache.logging.log4j-log4j-web-2.10.0.jar
  * Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar
  * BookKeeper
-    - org.apache.bookkeeper-bookkeeper-common-4.9.2.jar
-    - org.apache.bookkeeper-bookkeeper-common-allocator-4.9.2.jar
-    - org.apache.bookkeeper-bookkeeper-proto-4.9.2.jar
-    - org.apache.bookkeeper-bookkeeper-server-4.9.2.jar
-    - org.apache.bookkeeper-bookkeeper-tools-framework-4.9.2.jar
-    - org.apache.bookkeeper-circe-checksum-4.9.2.jar
-    - org.apache.bookkeeper-cpu-affinity-4.9.2.jar
-    - org.apache.bookkeeper-statelib-4.9.2.jar
-    - org.apache.bookkeeper-stream-storage-api-4.9.2.jar
-    - org.apache.bookkeeper-stream-storage-common-4.9.2.jar
-    - org.apache.bookkeeper-stream-storage-java-client-4.9.2.jar
-    - org.apache.bookkeeper-stream-storage-java-client-base-4.9.2.jar
-    - org.apache.bookkeeper-stream-storage-proto-4.9.2.jar
-    - org.apache.bookkeeper-stream-storage-server-4.9.2.jar
-    - org.apache.bookkeeper-stream-storage-service-api-4.9.2.jar
-    - org.apache.bookkeeper-stream-storage-service-impl-4.9.2.jar
-    - org.apache.bookkeeper.http-http-server-4.9.2.jar
-    - org.apache.bookkeeper.http-vertx-http-server-4.9.2.jar
-    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.9.2.jar
-    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.9.2.jar
-    - org.apache.bookkeeper.tests-stream-storage-tests-common-4.9.2.jar
-    - org.apache.distributedlog-distributedlog-common-4.9.2.jar
-    - org.apache.distributedlog-distributedlog-core-4.9.2-tests.jar
-    - org.apache.distributedlog-distributedlog-core-4.9.2.jar
-    - org.apache.distributedlog-distributedlog-protocol-4.9.2.jar
-    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.9.2.jar
+    - org.apache.bookkeeper-bookkeeper-common-4.10.0.jar
+    - org.apache.bookkeeper-bookkeeper-common-allocator-4.10.0.jar
+    - org.apache.bookkeeper-bookkeeper-proto-4.10.0.jar
+    - org.apache.bookkeeper-bookkeeper-server-4.10.0.jar
+    - org.apache.bookkeeper-bookkeeper-tools-framework-4.10.0.jar
+    - org.apache.bookkeeper-circe-checksum-4.10.0.jar
+    - org.apache.bookkeeper-cpu-affinity-4.10.0.jar
+    - org.apache.bookkeeper-statelib-4.10.0.jar
+    - org.apache.bookkeeper-stream-storage-api-4.10.0.jar
+    - org.apache.bookkeeper-stream-storage-common-4.10.0.jar
+    - org.apache.bookkeeper-stream-storage-java-client-4.10.0.jar
+    - org.apache.bookkeeper-stream-storage-java-client-base-4.10.0.jar
+    - org.apache.bookkeeper-stream-storage-proto-4.10.0.jar
+    - org.apache.bookkeeper-stream-storage-server-4.10.0.jar
+    - org.apache.bookkeeper-stream-storage-service-api-4.10.0.jar
+    - org.apache.bookkeeper-stream-storage-service-impl-4.10.0.jar
+    - org.apache.bookkeeper.http-http-server-4.10.0.jar
+    - org.apache.bookkeeper.http-vertx-http-server-4.10.0.jar
+    - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.10.0.jar
+    - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.10.0.jar
+    - org.apache.bookkeeper.tests-stream-storage-tests-common-4.10.0.jar
+    - org.apache.distributedlog-distributedlog-common-4.10.0.jar
+    - org.apache.distributedlog-distributedlog-core-4.10.0-tests.jar
+    - org.apache.distributedlog-distributedlog-core-4.10.0.jar
+    - org.apache.distributedlog-distributedlog-protocol-4.10.0.jar
+    - org.apache.bookkeeper.stats-codahale-metrics-provider-4.10.0.jar
  * LZ4 -- org.lz4-lz4-java-1.5.0.jar
  * AsyncHttpClient
     - org.asynchttpclient-async-http-client-2.7.0.jar
@@ -449,7 +449,7 @@ The Apache Software License, Version 2.0
     - io.grpc-grpc-protobuf-lite-1.18.0.jar
     - io.grpc-grpc-protobuf-nano-1.18.0.jar
     - io.grpc-grpc-stub-1.18.0.jar
-    - io.grpc-grpc-testing-1.12.0.jar
+    - io.grpc-grpc-testing-1.18.0.jar
   * OpenCensus
     - io.opencensus-opencensus-api-0.18.0.jar
     - io.opencensus-opencensus-contrib-grpc-metrics-0.18.0.jar

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -24,6 +24,7 @@
 import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.DEFAULT_LEDGER_DELETE_BACKOFF_TIME_SEC;
 import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.DEFAULT_LEDGER_DELETE_RETRIES;
 import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.createManagedLedgerException;
+import static org.apache.bookkeeper.mledger.util.Errors.isNoSuchLedgerExistsException;
 import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -63,6 +64,7 @@
 import org.apache.bookkeeper.client.BookKeeper;
 import org.apache.bookkeeper.client.LedgerEntry;
 import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.client.api.BKException.Code;
 import org.apache.bookkeeper.mledger.AsyncCallbacks;
 import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback;
 import org.apache.bookkeeper.mledger.AsyncCallbacks.FindEntryCallback;
@@ -2429,7 +2431,7 @@ private void asyncDeleteLedger(final LedgerHandle lh, int retry) {
             if (rc != BKException.Code.OK) {
                 log.warn("[{}] Failed to delete ledger {}: {}", ledger.getName(), lh.getId(),
                         BKException.getMessage(rc));
-                if (rc != BKException.Code.NoSuchLedgerExistsException) {
+                if (!isNoSuchLedgerExistsException(rc)) {
                     ledger.getScheduledExecutor().schedule(safeRun(() -> {
                         asyncDeleteLedger(lh, retry - 1);
                     }), DEFAULT_LEDGER_DELETE_BACKOFF_TIME_SEC, TimeUnit.SECONDS);
@@ -2465,7 +2467,7 @@ private void asyncDeleteCursorLedger(int retry) {
             } else {
                 log.warn("[{}][{}] Failed to delete ledger {}: {}", ledger.getName(), name, cursorLedger.getId(),
                         BKException.getMessage(rc));
-                if (rc != BKException.Code.NoSuchLedgerExistsException) {
+                if (!isNoSuchLedgerExistsException(rc)) {
                     ledger.getScheduledExecutor().schedule(safeRun(() -> {
                         asyncDeleteCursorLedger(retry - 1);
                     }), DEFAULT_LEDGER_DELETE_BACKOFF_TIME_SEC, TimeUnit.SECONDS);
@@ -2477,12 +2479,13 @@ private void asyncDeleteCursorLedger(int retry) {
     /**
      * return BK error codes that are considered not likely to be recoverable.
      */
-    private static boolean isBkErrorNotRecoverable(int rc) {
+    public static boolean isBkErrorNotRecoverable(int rc) {
         switch (rc) {
-        case BKException.Code.NoSuchLedgerExistsException:
-        case BKException.Code.ReadException:
-        case BKException.Code.LedgerRecoveryException:
-        case BKException.Code.NoSuchEntryException:
+        case Code.NoSuchLedgerExistsException:
+        case Code.NoSuchLedgerExistsOnMetadataServerException:
+        case Code.ReadException:
+        case Code.LedgerRecoveryException:
+        case Code.NoSuchEntryException:
             return true;
 
         default:

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -21,6 +21,7 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static java.lang.Math.min;
+import static org.apache.bookkeeper.mledger.util.Errors.isNoSuchLedgerExistsException;
 import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
 
 import com.google.common.collect.BoundType;
@@ -68,6 +69,7 @@
 import org.apache.bookkeeper.client.AsyncCallback.CreateCallback;
 import org.apache.bookkeeper.client.AsyncCallback.OpenCallback;
 import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.client.BKException.Code;
 import org.apache.bookkeeper.client.BookKeeper;
 import org.apache.bookkeeper.client.BookKeeper.DigestType;
 import org.apache.bookkeeper.client.LedgerHandle;
@@ -306,7 +308,7 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) {
                                         .setTimestamp(clock.millis()).build();
                                 ledgers.put(id, info);
                                 initializeBookKeeper(callback);
-                            } else if (rc == BKException.Code.NoSuchLedgerExistsException) {
+                            } else if (isNoSuchLedgerExistsException(rc)) {
                                 log.warn("[{}] Ledger not found: {}", name, ledgers.lastKey());
                                 ledgers.remove(ledgers.lastKey());
                                 initializeBookKeeper(callback);
@@ -2159,7 +2161,7 @@ private void asyncDeleteLedger(long ledgerId, long retry) {
             return;
         }
         bookKeeper.asyncDeleteLedger(ledgerId, (rc, ctx) -> {
-            if (rc == BKException.Code.NoSuchLedgerExistsException) {
+            if (isNoSuchLedgerExistsException(rc)) {
                 log.warn("[{}] Ledger was already deleted {}", name, ledgerId);
             } else if (rc != BKException.Code.OK) {
                 log.error("[{}] Error deleting ledger {}", name, ledgerId, BKException.getMessage(rc));
@@ -2190,7 +2192,8 @@ private void deleteAllLedgers(DeleteLedgerCallback callback, Object ctx) {
             }
             bookKeeper.asyncDeleteLedger(ls.getLedgerId(), (rc, ctx1) -> {
                 switch (rc) {
-                case BKException.Code.NoSuchLedgerExistsException:
+                case Code.NoSuchLedgerExistsException:
+                case Code.NoSuchLedgerExistsOnMetadataServerException:
                     log.warn("[{}] Ledger {} not found when deleting it", name, ls.getLedgerId());
                     // Continue anyway
 
@@ -3010,8 +3013,9 @@ protected boolean isReadOnly() {
      */
     private static boolean isBkErrorNotRecoverable(int rc) {
         switch (rc) {
-        case BKException.Code.NoSuchLedgerExistsException:
-        case BKException.Code.NoSuchEntryException:
+        case Code.NoSuchLedgerExistsException:
+        case Code.NoSuchLedgerExistsOnMetadataServerException:
+        case Code.NoSuchEntryException:
             return true;
 
         default:

diff --git a/...-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java b/...-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerOfflineBacklog.java
@@ -37,6 +37,7 @@
 import org.apache.bookkeeper.client.api.DigestType;
 import org.apache.bookkeeper.mledger.ManagedLedgerException;
 import org.apache.bookkeeper.mledger.proto.MLDataFormats;
+import org.apache.bookkeeper.mledger.util.Errors;
 import org.apache.pulsar.common.naming.TopicName;
 import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats;
 import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
@@ -165,7 +166,7 @@ public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, MetaStore.
                                             .setSize(lh.getLength()).setTimestamp(System.currentTimeMillis()).build();
                                     ledgers.put(id, info);
                                     mlMetaCounter.countDown();
-                                } else if (rc == BKException.Code.NoSuchLedgerExistsException) {
+                                } else if (Errors.isNoSuchLedgerExistsException(rc)) {
                                     log.warn("[{}] Ledger not found: {}", managedLedgerName, ledgers.lastKey());
                                     ledgers.remove(ledgers.lastKey());
                                     mlMetaCounter.countDown();

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Errors.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/Errors.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.util;
+
+import org.apache.bookkeeper.client.api.BKException.Code;
+
+/**
+ * Utility class for bookkeeper related error codes.
+ */
+public final class Errors {
+
+    public static boolean isNoSuchLedgerExistsException(int rc) {
+        switch (rc) {
+            case Code.NoSuchLedgerExistsOnMetadataServerException:
+            case Code.NoSuchLedgerExistsException:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    private Errors() {}
+
+}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java
@@ -268,6 +268,11 @@ static class MockMetadata implements LedgerMetadata {
         @Override
         public int getMetadataFormatVersion() { return metadataFormatVersion; }
 
+        @Override
+        public long getCToken() {
+            return 0;
+        }
+
         @Override
         public int getEnsembleSize() { return ensembleSize; }
 

diff --git a/pom.xml b/pom.xml
@@ -148,7 +148,7 @@ flexible messaging model and an intuitive client API.</description>
     <!-- apache commons -->
     <commons-compress.version>1.19</commons-compress.version>
 
-    <bookkeeper.version>4.9.2</bookkeeper.version>
+    <bookkeeper.version>4.10.0</bookkeeper.version>
     <zookeeper.version>3.5.6</zookeeper.version>
     <netty.version>4.1.43.Final</netty.version>
     <netty-tc-native.version>2.0.26.Final</netty-tc-native.version>

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java
@@ -225,7 +225,8 @@ public void testCleanupOldCompactedTopicLedger() throws Exception {
                           Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE,
                           Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD).close();
             Assert.fail("Should have failed to open old ledger");
-        } catch (BKException.BKNoSuchLedgerExistsException e) {
+        } catch (BKException.BKNoSuchLedgerExistsException
+            | BKException.BKNoSuchLedgerExistsOnMetadataServerException e) {
             // correct, expected behaviour
         }
         bk.openLedger(newCompactedLedger.getId(),

diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
@@ -407,17 +407,17 @@ The Apache Software License, Version 2.0
     - async-http-client-2.7.0.jar
     - async-http-client-netty-utils-2.7.0.jar
   * Apache Bookkeeper
-    - bookkeeper-common-4.9.2.jar
-    - bookkeeper-common-allocator-4.9.2.jar
-    - bookkeeper-proto-4.9.2.jar
-    - bookkeeper-server-4.9.2.jar
-    - bookkeeper-stats-api-4.9.2.jar
-    - bookkeeper-tools-framework-4.9.2.jar
-    - circe-checksum-4.9.2.jar
-    - codahale-metrics-provider-4.9.2.jar
-    - cpu-affinity-4.9.2.jar
-    - http-server-4.9.2.jar
-    - prometheus-metrics-provider-4.9.2.jar
+    - bookkeeper-common-4.10.0.jar
+    - bookkeeper-common-allocator-4.10.0.jar
+    - bookkeeper-proto-4.10.0.jar
+    - bookkeeper-server-4.10.0.jar
+    - bookkeeper-stats-api-4.10.0.jar
+    - bookkeeper-tools-framework-4.10.0.jar
+    - circe-checksum-4.10.0.jar
+    - codahale-metrics-provider-4.10.0.jar
+    - cpu-affinity-4.10.0.jar
+    - http-server-4.10.0.jar
+    - prometheus-metrics-provider-4.10.0.jar
   * Apache Commons
     - commons-cli-1.2.jar
     - commons-codec-1.10.jar

diff --git a/...ookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMapping.java b/...ookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/ZkBookieRackAffinityMapping.java
@@ -27,12 +27,12 @@
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.bookkeeper.client.ITopologyAwareEnsemblePlacementPolicy;
 import org.apache.bookkeeper.client.RackChangeNotifier;
-import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl;
 import org.apache.bookkeeper.conf.ClientConfiguration;
 import org.apache.bookkeeper.net.AbstractDNSToSwitchMapping;
+import org.apache.bookkeeper.net.BookieNode;
 import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.net.NetworkTopology;
 import org.apache.bookkeeper.zookeeper.ZooKeeperClient;
 import org.apache.commons.configuration.Configuration;
 import org.apache.pulsar.common.policies.data.BookieInfo;
@@ -53,7 +53,7 @@ public class ZkBookieRackAffinityMapping extends AbstractDNSToSwitchMapping
     public static final String BOOKIE_INFO_ROOT_PATH = "/bookies";
 
     private ZooKeeperDataCache<BookiesRackConfiguration> bookieMappingCache = null;
-    private RackawareEnsemblePlacementPolicyImpl rackawarePolicy = null;
+    private ITopologyAwareEnsemblePlacementPolicy<BookieNode> rackawarePolicy = null;
 
     private static final ObjectMapper jsonMapper = ObjectMapperFactory.create();
 
@@ -187,6 +187,11 @@ public void reloadCachedMappings() {
         // no-op
     }
 
+    @Override
+    public boolean useHostName() {
+        return false;
+    }
+
     @Override
     public void onUpdate(String path, BookiesRackConfiguration data, Stat stat) {
         if (rackawarePolicy != null) {
@@ -206,8 +211,7 @@ public void onUpdate(String path, BookiesRackConfiguration data, Stat stat) {
     }
 
     @Override
-    public void registerRackChangeListener(RackawareEnsemblePlacementPolicyImpl rackawarePolicy) {
+    public void registerRackChangeListener(ITopologyAwareEnsemblePlacementPolicy<BookieNode> rackawarePolicy) {
         this.rackawarePolicy = rackawarePolicy;
-
     }
 }
diff --git a/...ntegration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java b/...ntegration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java
@@ -234,7 +234,8 @@ public boolean ledgerExistsInBookKeeper(long ledgerId) throws Exception {
             try {
                 bk.openLedger(ledgerId).close();
                 return true;
-            } catch (BKException.BKNoSuchLedgerExistsException e) {
+            } catch (BKException.BKNoSuchLedgerExistsException
+                | BKException.BKNoSuchLedgerExistsOnMetadataServerException e) {
                 return false;
             }
         }

diff --git a/...rc/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadIndexBlockImpl.java b/...rc/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/OffloadIndexBlockImpl.java
@@ -259,6 +259,11 @@ static private class InternalLedgerMetadata implements LedgerMetadata {
         @Override
         public int getMetadataFormatVersion() { return 2; }
 
+        @Override
+        public long getCToken() {
+            return 0;
+        }
+
         @Override
         public int getEnsembleSize() {
             return this.ensembleSize;