Skip to content

Commit

Permalink
[bookkeeper] bump bookkeeper version to 4.9.0 (apache#2906)
Browse files Browse the repository at this point in the history
*Motivation*

This change was part of apache#2714. Moving it out as a separate change to make reviews easier.

*Changes*

Bump bookkeeper version to 4.9.0
sijie authored Feb 10, 2019

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 00c71e2 commit 1b44aaa
Showing 25 changed files with 274 additions and 120 deletions.
83 changes: 45 additions & 38 deletions distribution/server/src/assemble/LICENSE.bin.txt
Original file line number Diff line number Diff line change
@@ -323,7 +323,7 @@ The Apache Software License, Version 2.0
- com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.9.7.jar
- com.fasterxml.jackson.module-jackson-module-jsonSchema-2.9.7.jar
* Caffeine -- com.github.ben-manes.caffeine-caffeine-2.6.2.jar
* Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-1.0.0.jar
* Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-1.12.0.jar
* Gson -- com.google.code.gson-gson-2.8.2.jar
* Guava -- com.google.guava-guava-21.0.jar
* Netty Reactive Streams -- com.typesafe.netty-netty-reactive-streams-2.0.0.jar
@@ -350,8 +350,8 @@ The Apache Software License, Version 2.0
- org.apache.commons-commons-lang3-3.4.jar
* Netty
- io.netty-netty-3.10.1.Final.jar
- io.netty-netty-all-4.1.22.Final.jar
- io.netty-netty-tcnative-boringssl-static-2.0.7.Final.jar
- io.netty-netty-all-4.1.32.Final.jar
- io.netty-netty-tcnative-boringssl-static-2.0.20.Final.jar
* Prometheus client
- io.prometheus-simpleclient-0.5.0.jar
- io.prometheus-simpleclient_common-0.5.0.jar
@@ -368,29 +368,32 @@ The Apache Software License, Version 2.0
- org.apache.logging.log4j-log4j-web-2.10.0.jar
* Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar
* BookKeeper
- org.apache.bookkeeper-bookkeeper-common-4.7.3.jar
- org.apache.bookkeeper-bookkeeper-proto-4.7.3.jar
- org.apache.bookkeeper-bookkeeper-server-4.7.3.jar
- org.apache.bookkeeper-circe-checksum-4.7.3.jar
- org.apache.bookkeeper-statelib-4.7.3.jar
- org.apache.bookkeeper-stream-storage-api-4.7.3.jar
- org.apache.bookkeeper-stream-storage-common-4.7.3.jar
- org.apache.bookkeeper-stream-storage-java-client-4.7.3.jar
- org.apache.bookkeeper-stream-storage-java-client-base-4.7.3.jar
- org.apache.bookkeeper-stream-storage-proto-4.7.3.jar
- org.apache.bookkeeper-stream-storage-server-4.7.3.jar
- org.apache.bookkeeper-stream-storage-service-api-4.7.3.jar
- org.apache.bookkeeper-stream-storage-service-impl-4.7.3.jar
- org.apache.bookkeeper.http-http-server-4.7.3.jar
- org.apache.bookkeeper.http-vertx-http-server-4.7.3.jar
- org.apache.bookkeeper.stats-bookkeeper-stats-api-4.7.3.jar
- org.apache.bookkeeper.stats-prometheus-metrics-provider-4.7.3.jar
- org.apache.bookkeeper.tests-stream-storage-tests-common-4.7.3.jar
- org.apache.distributedlog-distributedlog-common-4.7.3.jar
- org.apache.distributedlog-distributedlog-core-4.7.3-tests.jar
- org.apache.distributedlog-distributedlog-core-4.7.3.jar
- org.apache.distributedlog-distributedlog-protocol-4.7.3.jar
- org.apache.bookkeeper.stats-codahale-metrics-provider-4.7.3.jar
- org.apache.bookkeeper-bookkeeper-common-4.9.0.jar
- org.apache.bookkeeper-bookkeeper-common-allocator-4.9.0.jar
- org.apache.bookkeeper-bookkeeper-proto-4.9.0.jar
- org.apache.bookkeeper-bookkeeper-server-4.9.0.jar
- org.apache.bookkeeper-bookkeeper-tools-framework-4.9.0.jar
- org.apache.bookkeeper-circe-checksum-4.9.0.jar
- org.apache.bookkeeper-cpu-affinity-4.9.0.jar
- org.apache.bookkeeper-statelib-4.9.0.jar
- org.apache.bookkeeper-stream-storage-api-4.9.0.jar
- org.apache.bookkeeper-stream-storage-common-4.9.0.jar
- org.apache.bookkeeper-stream-storage-java-client-4.9.0.jar
- org.apache.bookkeeper-stream-storage-java-client-base-4.9.0.jar
- org.apache.bookkeeper-stream-storage-proto-4.9.0.jar
- org.apache.bookkeeper-stream-storage-server-4.9.0.jar
- org.apache.bookkeeper-stream-storage-service-api-4.9.0.jar
- org.apache.bookkeeper-stream-storage-service-impl-4.9.0.jar
- org.apache.bookkeeper.http-http-server-4.9.0.jar
- org.apache.bookkeeper.http-vertx-http-server-4.9.0.jar
- org.apache.bookkeeper.stats-bookkeeper-stats-api-4.9.0.jar
- org.apache.bookkeeper.stats-prometheus-metrics-provider-4.9.0.jar
- org.apache.bookkeeper.tests-stream-storage-tests-common-4.9.0.jar
- org.apache.distributedlog-distributedlog-common-4.9.0.jar
- org.apache.distributedlog-distributedlog-core-4.9.0-tests.jar
- org.apache.distributedlog-distributedlog-core-4.9.0.jar
- org.apache.distributedlog-distributedlog-protocol-4.9.0.jar
- org.apache.bookkeeper.stats-codahale-metrics-provider-4.9.0.jar
* LZ4 -- org.lz4-lz4-java-1.5.0.jar
* AsyncHttpClient
- org.asynchttpclient-async-http-client-2.7.0.jar
@@ -423,20 +426,20 @@ The Apache Software License, Version 2.0
* Okio - com.squareup.okio-okio-1.13.0.jar
* Javassist -- org.javassist-javassist-3.21.0-GA.jar
* gRPC
- io.grpc-grpc-all-1.16.1.jar
- io.grpc-grpc-auth-1.16.1.jar
- io.grpc-grpc-context-1.16.1.jar
- io.grpc-grpc-core-1.16.1.jar
- io.grpc-grpc-netty-1.16.1.jar
- io.grpc-grpc-okhttp-1.16.1.jar
- io.grpc-grpc-protobuf-1.16.1.jar
- io.grpc-grpc-protobuf-lite-1.16.1.jar
- io.grpc-grpc-protobuf-nano-1.16.1.jar
- io.grpc-grpc-stub-1.16.1.jar
- io.grpc-grpc-all-1.18.0.jar
- io.grpc-grpc-auth-1.18.0.jar
- io.grpc-grpc-context-1.18.0.jar
- io.grpc-grpc-core-1.18.0.jar
- io.grpc-grpc-netty-1.18.0.jar
- io.grpc-grpc-okhttp-1.18.0.jar
- io.grpc-grpc-protobuf-1.18.0.jar
- io.grpc-grpc-protobuf-lite-1.18.0.jar
- io.grpc-grpc-protobuf-nano-1.18.0.jar
- io.grpc-grpc-stub-1.18.0.jar
- io.grpc-grpc-testing-1.12.0.jar
* OpenCensus
- io.opencensus-opencensus-api-0.12.3.jar
- io.opencensus-opencensus-contrib-grpc-metrics-0.12.3.jar
- io.opencensus-opencensus-api-0.18.0.jar
- io.opencensus-opencensus-contrib-grpc-metrics-0.18.0.jar
* Paranamer
- com.thoughtworks.paranamer-paranamer-2.7.jar
* Jodah
@@ -477,6 +480,8 @@ The Apache Software License, Version 2.0
- io.jsonwebtoken-jjwt-jackson-0.10.5.jar
* JavaX Injection
- javax.inject-javax.inject-1.jar
* JCTools - Java Concurrency Tools for the JVM
- org.jctools-jctools-core-2.1.2.jar
* Vertx
- io.vertx-vertx-auth-common-3.4.1.jar
- io.vertx-vertx-core-3.4.1.jar
@@ -501,6 +506,8 @@ MIT License
- org.slf4j-jcl-over-slf4j-1.7.25.jar
* Animal Sniffer Annotations
- org.codehaus.mojo-animal-sniffer-annotations-1.17.jar
* The Checker Framework
- org.checkerframework-checker-compat-qual-2.5.2.jar

Protocol Buffers License
* Protocol Buffers
15 changes: 15 additions & 0 deletions distribution/server/src/assemble/NOTICE.bin.txt
Original file line number Diff line number Diff line change
@@ -200,3 +200,18 @@ http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
* Providing benchmark suite
* Alec Wysoker
* Performance and memory usage improvement

------------------------------------------------------------------------------------
- org.checkerframework-checker-compat-qual-2.5.2.jar

A few parts of the Checker Framework have more permissive licenses.

* The annotations are licensed under the MIT License. (The text of this
license appears below.) More specifically, all the parts of the Checker
Framework that you might want to include with your own program use the
MIT License. This is the checker-qual.jar file and all the files that
appear in it: every file in a qual/ directory, plus utility files such
as NullnessUtil.java, RegexUtil.java, SignednessUtil.java, etc.
In addition, the cleanroom implementations of third-party annotations,
which the Checker Framework recognizes as aliases for its own
annotations, are licensed under the MIT License.
Original file line number Diff line number Diff line change
@@ -95,15 +95,10 @@ public void setCloseWhenDone(boolean closeWhenDone) {

public void initiate() {
ByteBuf duplicateBuffer = data.retainedDuplicate();
// duplicatedBuffer has refCnt=1 at this point

// internally asyncAddEntry() will take the ownership of the buffer and release it at the end
lastInitTime = System.nanoTime();
ledger.asyncAddEntry(duplicateBuffer, this, ctx);

// Internally, asyncAddEntry() is refCnt neutral to respect to the passed buffer and it will keep a ref on it
// until is done using it. We need to release this buffer here to balance the 1 refCnt added at the creation
// time.
duplicateBuffer.release();
}

public void failed(ManagedLedgerException e) {
Original file line number Diff line number Diff line change
@@ -33,11 +33,13 @@
import org.apache.bookkeeper.client.AsyncCallback.CreateCallback;
import org.apache.bookkeeper.client.AsyncCallback.DeleteCallback;
import org.apache.bookkeeper.client.AsyncCallback.OpenCallback;
import org.apache.bookkeeper.client.BKException.Code;
import org.apache.bookkeeper.client.api.OpenBuilder;
import org.apache.bookkeeper.client.api.ReadHandle;
import org.apache.bookkeeper.client.impl.OpenBuilderBase;
import org.apache.bookkeeper.common.concurrent.FutureUtils;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.net.BookieSocketAddress;
import org.apache.zookeeper.ZooKeeper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -198,8 +200,9 @@ public OpenBuilder newOpenLedgerOp() {
public CompletableFuture<ReadHandle> execute() {
return getProgrammedFailure().thenCompose(
(res) -> {
if (!validate()) {
return FutureUtils.exception(new BKException.BKNoSuchLedgerExistsException());
int rc = validate();
if (rc != BKException.Code.OK) {
return FutureUtils.exception(BKException.create(rc));
}

PulsarMockLedgerHandle lh = ledgers.get(ledgerId);
Original file line number Diff line number Diff line change
@@ -29,6 +29,7 @@
import java.util.Arrays;
import java.util.EnumSet;
import java.util.Enumeration;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.CompletableFuture;

@@ -38,10 +39,14 @@
import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.client.api.LastConfirmedAndEntry;
import org.apache.bookkeeper.client.api.LedgerEntries;
import org.apache.bookkeeper.client.api.LedgerMetadata;
import org.apache.bookkeeper.client.api.ReadHandle;
import org.apache.bookkeeper.client.api.WriteFlag;
import org.apache.bookkeeper.client.impl.LedgerEntryImpl;
import org.apache.bookkeeper.common.concurrent.FutureUtils;
import org.apache.bookkeeper.net.BookieSocketAddress;
import org.apache.bookkeeper.versioning.LongVersion;
import org.apache.bookkeeper.versioning.Versioned;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@@ -61,8 +66,8 @@ public class PulsarMockLedgerHandle extends LedgerHandle {

public PulsarMockLedgerHandle(PulsarMockBookKeeper bk, long id,
DigestType digest, byte[] passwd) throws GeneralSecurityException {
super(bk, id, new LedgerMetadata(3, 3, 2, DigestType.MAC, "".getBytes()), DigestType.MAC, "".getBytes(),
EnumSet.noneOf(WriteFlag.class));
super(bk.getClientCtx(), id, new Versioned<>(createMetadata(digest, passwd), new LongVersion(0L)),
digest, passwd, WriteFlag.NONE);
this.bk = bk;
this.id = id;
this.digest = digest;
@@ -75,6 +80,14 @@ public PulsarMockLedgerHandle(PulsarMockBookKeeper bk, long id,
public void asyncClose(CloseCallback cb, Object ctx) {
bk.getProgrammedFailure().thenComposeAsync((res) -> {
fenced = true;

Versioned<LedgerMetadata> current = getVersionedLedgerMetadata();
Versioned<LedgerMetadata> newMetadata = new Versioned<>(
LedgerMetadataBuilder.from(current.getValue())
.withClosedState().withLastEntryId(getLastAddConfirmed())
.withLength(getLength()).build(),
new LongVersion(((LongVersion)current.getVersion()).getLongVersion() + 1));
setLedgerMetadata(current, newMetadata);
return FutureUtils.value(null);
}, bk.executor).whenCompleteAsync((res, exception) -> {
if (exception != null) {
@@ -242,6 +255,18 @@ public CompletableFuture<LastConfirmedAndEntry> readLastAddConfirmedAndEntryAsyn
return readHandle.readLastAddConfirmedAndEntryAsync(entryId, timeOutInMillis, parallel);
}

private static LedgerMetadata createMetadata(DigestType digest, byte[] passwd) {
List<BookieSocketAddress> ensemble = Lists.newArrayList(
new BookieSocketAddress("192.0.2.1", 1234),
new BookieSocketAddress("192.0.2.2", 1234),
new BookieSocketAddress("192.0.2.3", 1234));
return LedgerMetadataBuilder.create()
.withDigestType(digest.toApiDigestType())
.withPassword(passwd)
.newEnsembleEntry(0L, ensemble)
.build();
}

private static final Logger log = LoggerFactory.getLogger(PulsarMockLedgerHandle.class);

}
Original file line number Diff line number Diff line change
@@ -223,7 +223,7 @@ public void verifyConcurrentUsage() throws Exception {
public void testSimple() throws Exception {
ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
ManagedLedgerConfig mlConfig = new ManagedLedgerConfig();
mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1);
mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setWriteQuorumSize(1);
// set the data ledger size
mlConfig.setMaxEntriesPerLedger(100);
// set the metadata ledger size to 1 to kick off many ledger switching cases
@@ -238,7 +238,9 @@ public void testSimple() throws Exception {
public void testConcurrentMarkDelete() throws Exception {
ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
ManagedLedgerConfig mlConfig = new ManagedLedgerConfig();
mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataAckQuorumSize(1);
mlConfig.setEnsembleSize(1).setWriteQuorumSize(1)
.setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1)
.setMetadataAckQuorumSize(1);
// set the data ledger size
mlConfig.setMaxEntriesPerLedger(100);
// set the metadata ledger size to 1 to kick off many ledger switching cases
Original file line number Diff line number Diff line change
@@ -62,7 +62,6 @@
import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.client.api.LedgerEntries;
import org.apache.bookkeeper.client.api.ReadHandle;
import org.apache.bookkeeper.client.LedgerMetadata;
import org.apache.bookkeeper.client.PulsarMockBookKeeper;
import org.apache.bookkeeper.client.PulsarMockLedgerHandle;
import org.apache.bookkeeper.client.api.LedgerEntries;
Original file line number Diff line number Diff line change
@@ -31,6 +31,7 @@

import io.netty.buffer.ByteBuf;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -245,6 +246,9 @@ static class MockMetadata implements LedgerMetadata {
private final DigestType digestType;
private final long ctime;
private final boolean isClosed;
private final int metadataFormatVersion;
private final State state;
private final byte[] password;
private final Map<String, byte[]> customMetadata;

MockMetadata(LedgerMetadata toCopy) {
@@ -256,10 +260,21 @@ static class MockMetadata implements LedgerMetadata {
digestType = toCopy.getDigestType();
ctime = toCopy.getCtime();
isClosed = toCopy.isClosed();

metadataFormatVersion = toCopy.getMetadataFormatVersion();
state = toCopy.getState();
password = Arrays.copyOf(toCopy.getPassword(), toCopy.getPassword().length);
customMetadata = ImmutableMap.copyOf(toCopy.getCustomMetadata());
}

@Override
public boolean hasPassword() { return true; }

@Override
public State getState() { return state; }

@Override
public int getMetadataFormatVersion() { return metadataFormatVersion; }

@Override
public int getEnsembleSize() { return ensembleSize; }

@@ -278,6 +293,9 @@ static class MockMetadata implements LedgerMetadata {
@Override
public DigestType getDigestType() { return digestType; }

@Override
public byte[] getPassword() { return password; }

@Override
public long getCtime() { return ctime; }

@@ -296,5 +314,10 @@ public List<BookieSocketAddress> getEnsembleAt(long entryId) {
public NavigableMap<Long, ? extends List<BookieSocketAddress>> getAllEnsembles() {
throw new UnsupportedOperationException("Pulsar shouldn't look at this");
}

@Override
public String toSafeString() {
return toString();
}
}
}
Original file line number Diff line number Diff line change
@@ -23,6 +23,9 @@

package org.apache.bookkeeper.test;

import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;

import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
@@ -38,7 +41,9 @@
import java.util.concurrent.Executors;
import org.apache.bookkeeper.bookie.Bookie;
import org.apache.bookkeeper.bookie.BookieException;
import org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage;
import org.apache.bookkeeper.client.BookKeeperTestClient;
import org.apache.bookkeeper.common.allocator.PoolingPolicy;
import org.apache.bookkeeper.conf.AbstractConfiguration;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.conf.ServerConfiguration;
@@ -202,12 +207,15 @@ protected ServerConfiguration newServerConfiguration(int port, String zkServers,
conf.setAllowLoopback(true);
conf.setFlushInterval(60 * 1000);
conf.setGcWaitTime(60 * 1000);
conf.setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap);
String[] ledgerDirNames = new String[ledgerDirs.length];
for (int i = 0; i < ledgerDirs.length; i++) {
ledgerDirNames[i] = ledgerDirs[i].getPath();
}
conf.setLedgerDirNames(ledgerDirNames);
conf.setLedgerStorageClass("org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage");
conf.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 4);
conf.setProperty(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB, 4);
return conf;
}

@@ -434,7 +442,7 @@ protected BookieServer startBookie(ServerConfiguration conf) throws Exception {
protected BookieServer startBookie(ServerConfiguration conf, final Bookie b) throws Exception {
BookieServer server = new BookieServer(conf) {
@Override
protected Bookie newBookie(ServerConfiguration conf)
protected Bookie newBookie(ServerConfiguration conf, ByteBufAllocator allocator)
throws IOException, KeeperException, InterruptedException, BookieException {
return b;
}
Loading

0 comments on commit 1b44aaa

Please sign in to comment.