Skip to content

Commit

Permalink
Pulsar with Bk 4.15.0 (apache#15142)
Browse files Browse the repository at this point in the history
* Pulsar with bk 4.15.0 rc1

* license check

* licensecheck 2

* use 4.15.0 RC2

* Build with 4.15.0 release
  • Loading branch information
dlg99 authored May 10, 2022
1 parent f0b7efa commit 7a7b5fa
Show file tree
Hide file tree
Showing 39 changed files with 2,791 additions and 819 deletions.
2 changes: 1 addition & 1 deletion buildtools/src/main/resources/pulsar/checkstyle.xml
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ page at http://checkstyle.sourceforge.net/config.html -->

<module name="IllegalImport">
<property name="illegalPkgs"
value="autovalue.shaded, avro.shaded, bk-shade, com.google.api.client.repackaged, com.google.appengine.repackaged, io.netty.util.internal"/>
value="autovalue.shaded, avro.shaded, bk-shade, com.google.api.client.repackaged, com.google.appengine.repackaged" />
</module>

<module name="RedundantModifier">
Expand Down
30 changes: 30 additions & 0 deletions conf/default_rocksdb.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

[DBOptions]
# set by jni: options.setCreateIfMissing
create_if_missing=true
# set by jni: options.setInfoLogLevel
info_log_level=INFO_LEVEL
# set by jni: options.setKeepLogFileNum
keep_log_file_num=30

[CFOptions "default"]
# set by jni: options.setLogFileTimeToRoll
log_file_time_to_roll=86400
70 changes: 70 additions & 0 deletions conf/entry_location_rocksdb.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

[DBOptions]
# set by jni: options.setCreateIfMissing
create_if_missing=true
# set by jni: options.setInfoLogLevel
info_log_level=INFO_LEVEL
# set by jni: options.setKeepLogFileNum
keep_log_file_num=30
# set by jni: options.setLogFileTimeToRoll
log_file_time_to_roll=86400
# set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism
max_background_jobs=2
# set by jni: options.setMaxSubcompactions
max_subcompactions=1
# set by jni: options.setMaxTotalWalSize
max_total_wal_size=536870912
# set by jni: options.setMaxOpenFiles
max_open_files=-1
# set by jni: options.setDeleteObsoleteFilesPeriodMicros
delete_obsolete_files_period_micros=3600000000

[CFOptions "default"]
# set by jni: options.setCompressionType
compression=kLZ4Compression
# set by jni: options.setWriteBufferSize
write_buffer_size=67108864
# set by jni: options.setMaxWriteBufferNumber
max_write_buffer_number=4
# set by jni: options.setNumLevels
num_levels=7
# set by jni: options.setLevelZeroFileNumCompactionTrigger
level0_file_num_compaction_trigger=4
# set by jni: options.setMaxBytesForLevelBase
max_bytes_for_level_base=268435456
# set by jni: options.setTargetFileSizeBase
target_file_size_base=67108864

[TableOptions/BlockBasedTable "default"]
# set by jni: tableOptions.setBlockSize
block_size=65536
# set by jni: tableOptions.setBlockCache
block_cache=206150041
# set by jni: tableOptions.setFormatVersion
format_version=2
# set by jni: tableOptions.setChecksumType
checksum=kxxHash
# set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder]
filter_policy=rocksdb.BloomFilter:10:false
# set by jni: tableOptions.setCacheIndexAndFilterBlocks
cache_index_and_filter_blocks=true
# set by jni: options.setLevelCompactionDynamicLevelBytes
level_compaction_dynamic_level_bytes=true
30 changes: 30 additions & 0 deletions conf/ledger_metadata_rocksdb.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

[DBOptions]
# set by jni: options.setCreateIfMissing
create_if_missing=true
# set by jni: options.setInfoLogLevel
info_log_level=INFO_LEVEL
# set by jni: options.setKeepLogFileNum
keep_log_file_num=30

[CFOptions "default"]
# set by jni: options.setLogFileTimeToRoll
log_file_time_to_roll=86400
64 changes: 34 additions & 30 deletions distribution/server/src/assemble/LICENSE.bin.txt
Original file line number Diff line number Diff line change
Expand Up @@ -397,31 +397,31 @@ The Apache Software License, Version 2.0
- org.apache.logging.log4j-log4j-1.2-api-2.17.1.jar
* Java Native Access JNA -- net.java.dev.jna-jna-4.2.0.jar
* BookKeeper
- org.apache.bookkeeper-bookkeeper-common-4.14.4.jar
- org.apache.bookkeeper-bookkeeper-common-allocator-4.14.4.jar
- org.apache.bookkeeper-bookkeeper-proto-4.14.4.jar
- org.apache.bookkeeper-bookkeeper-server-4.14.4.jar
- org.apache.bookkeeper-bookkeeper-tools-framework-4.14.4.jar
- org.apache.bookkeeper-circe-checksum-4.14.4.jar
- org.apache.bookkeeper-cpu-affinity-4.14.4.jar
- org.apache.bookkeeper-statelib-4.14.4.jar
- org.apache.bookkeeper-stream-storage-api-4.14.4.jar
- org.apache.bookkeeper-stream-storage-common-4.14.4.jar
- org.apache.bookkeeper-stream-storage-java-client-4.14.4.jar
- org.apache.bookkeeper-stream-storage-java-client-base-4.14.4.jar
- org.apache.bookkeeper-stream-storage-proto-4.14.4.jar
- org.apache.bookkeeper-stream-storage-server-4.14.4.jar
- org.apache.bookkeeper-stream-storage-service-api-4.14.4.jar
- org.apache.bookkeeper-stream-storage-service-impl-4.14.4.jar
- org.apache.bookkeeper.http-http-server-4.14.4.jar
- org.apache.bookkeeper.http-vertx-http-server-4.14.4.jar
- org.apache.bookkeeper.stats-bookkeeper-stats-api-4.14.4.jar
- org.apache.bookkeeper.stats-prometheus-metrics-provider-4.14.4.jar
- org.apache.distributedlog-distributedlog-common-4.14.4.jar
- org.apache.distributedlog-distributedlog-core-4.14.4-tests.jar
- org.apache.distributedlog-distributedlog-core-4.14.4.jar
- org.apache.distributedlog-distributedlog-protocol-4.14.4.jar
- org.apache.bookkeeper.stats-codahale-metrics-provider-4.14.4.jar
- org.apache.bookkeeper-bookkeeper-common-4.15.0.jar
- org.apache.bookkeeper-bookkeeper-common-allocator-4.15.0.jar
- org.apache.bookkeeper-bookkeeper-proto-4.15.0.jar
- org.apache.bookkeeper-bookkeeper-server-4.15.0.jar
- org.apache.bookkeeper-bookkeeper-tools-framework-4.15.0.jar
- org.apache.bookkeeper-circe-checksum-4.15.0.jar
- org.apache.bookkeeper-cpu-affinity-4.15.0.jar
- org.apache.bookkeeper-statelib-4.15.0.jar
- org.apache.bookkeeper-stream-storage-api-4.15.0.jar
- org.apache.bookkeeper-stream-storage-common-4.15.0.jar
- org.apache.bookkeeper-stream-storage-java-client-4.15.0.jar
- org.apache.bookkeeper-stream-storage-java-client-base-4.15.0.jar
- org.apache.bookkeeper-stream-storage-proto-4.15.0.jar
- org.apache.bookkeeper-stream-storage-server-4.15.0.jar
- org.apache.bookkeeper-stream-storage-service-api-4.15.0.jar
- org.apache.bookkeeper-stream-storage-service-impl-4.15.0.jar
- org.apache.bookkeeper.http-http-server-4.15.0.jar
- org.apache.bookkeeper.http-vertx-http-server-4.15.0.jar
- org.apache.bookkeeper.stats-bookkeeper-stats-api-4.15.0.jar
- org.apache.bookkeeper.stats-prometheus-metrics-provider-4.15.0.jar
- org.apache.distributedlog-distributedlog-common-4.15.0.jar
- org.apache.distributedlog-distributedlog-core-4.15.0-tests.jar
- org.apache.distributedlog-distributedlog-core-4.15.0.jar
- org.apache.distributedlog-distributedlog-protocol-4.15.0.jar
- org.apache.bookkeeper.stats-codahale-metrics-provider-4.15.0.jar
* Apache HTTP Client
- org.apache.httpcomponents-httpclient-4.5.13.jar
- org.apache.httpcomponents-httpcore-4.4.15.jar
Expand Down Expand Up @@ -451,7 +451,7 @@ The Apache Software License, Version 2.0
- org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.44.v20210927.jar
- org.eclipse.jetty-jetty-alpn-server-9.4.44.v20210927.jar
* SnakeYaml -- org.yaml-snakeyaml-1.30.jar
* RocksDB - org.rocksdb-rocksdbjni-6.10.2.jar
* RocksDB - org.rocksdb-rocksdbjni-6.29.4.1.jar
* Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar
* Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar
* OkHttp3
Expand Down Expand Up @@ -505,9 +505,10 @@ The Apache Software License, Version 2.0
- io.kubernetes-client-java-api-12.0.1.jar
- io.kubernetes-client-java-proto-12.0.1.jar
* Dropwizard
- io.dropwizard.metrics-metrics-core-3.2.5.jar
- io.dropwizard.metrics-metrics-graphite-3.2.5.jar
- io.dropwizard.metrics-metrics-jvm-3.2.5.jar
- io.dropwizard.metrics-metrics-core-4.1.12.1.jar
- io.dropwizard.metrics-metrics-graphite-4.1.12.1.jar
- io.dropwizard.metrics-metrics-jvm-4.1.12.1.jar
- io.dropwizard.metrics-metrics-jmx-4.1.12.1.jar
* Prometheus
- io.prometheus-simpleclient_httpserver-0.5.0.jar
* Java JSON WebTokens
Expand All @@ -529,7 +530,6 @@ The Apache Software License, Version 2.0
* Snappy Java
- org.xerial.snappy-snappy-java-1.1.7.jar
* Google HTTP Client
- com.google.http-client-google-http-client-jackson2-1.41.0.jar
- com.google.http-client-google-http-client-gson-1.41.0.jar
- com.google.http-client-google-http-client-1.41.0.jar
- com.google.auto.value-auto-value-annotations-1.9.jar
Expand All @@ -541,6 +541,10 @@ The Apache Software License, Version 2.0
- com.github.seancfoley-ipaddress-5.3.3.jar
* LMAX Disruptor
- com.lmax-disruptor-3.4.3.jar
* RxJava
- io.reactivex.rxjava3-rxjava-3.0.1.jar
* RabbitMQ Java Client
- com.rabbitmq-amqp-client-5.5.3.jar

BSD 3-clause "New" or "Revised" License
* Google auth library
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public void testSimpleRead() throws Exception {
ManagedLedgerConfig config = new ManagedLedgerConfig();
config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1)
.setMetadataAckQuorumSize(1);
ManagedLedger ledger = factory.open("my-ledger", config);
ManagedLedger ledger = factory.open("my-ledger" + testName, config);
ManagedCursor cursor = ledger.openCursor("c1");

int N = 1;
Expand All @@ -88,7 +88,7 @@ public void testBookieFailure() throws Exception {
ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(metadataStore, bkc);
ManagedLedgerConfig config = new ManagedLedgerConfig();
config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
ManagedLedger ledger = factory.open("my-ledger", config);
ManagedLedger ledger = factory.open("my-ledger" + testName, config);
ManagedCursor cursor = ledger.openCursor("my-cursor");
ledger.addEntry("entry-0".getBytes());

Expand Down Expand Up @@ -121,7 +121,7 @@ public void testBookieFailure() throws Exception {
factory.shutdown();

factory = new ManagedLedgerFactoryImpl(metadataStore, bkc);
ledger = factory.open("my-ledger", config);
ledger = factory.open("my-ledger" + testName, config);
cursor = ledger.openCursor("my-cursor");

// Next add should succeed
Expand Down Expand Up @@ -159,7 +159,7 @@ public void verifyConcurrentUsage() throws Exception {
EntryCacheManager cacheManager = factory.getEntryCacheManager();
ManagedLedgerConfig conf = new ManagedLedgerConfig();
conf.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
final ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my-ledger", conf);
final ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my-ledger" + testName, conf);

int NumProducers = 1;
int NumConsumers = 1;
Expand Down Expand Up @@ -299,7 +299,7 @@ public void asyncMarkDeleteAndClose() throws Exception {
ManagedLedgerConfig config = new ManagedLedgerConfig().setEnsembleSize(1).setWriteQuorumSize(1)
.setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1)
.setMetadataAckQuorumSize(1);
ManagedLedger ledger = factory.open("my_test_ledger", config);
ManagedLedger ledger = factory.open("my_test_ledger" + testName, config);
ManagedCursor cursor = ledger.openCursor("c1");

List<Position> positions = Lists.newArrayList();
Expand Down Expand Up @@ -348,7 +348,7 @@ public void ledgerFencedByAutoReplication() throws Exception {
ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(metadataStore, bkc);
ManagedLedgerConfig config = new ManagedLedgerConfig();
config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", config);
ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger" + testName, config);
ManagedCursor c1 = ledger.openCursor("c1");

PositionImpl p1 = (PositionImpl) ledger.addEntry("entry-1".getBytes());
Expand Down Expand Up @@ -378,15 +378,15 @@ public void ledgerFencedByFailover() throws Exception {
ManagedLedgerFactoryImpl factory1 = new ManagedLedgerFactoryImpl(metadataStore, bkc);
ManagedLedgerConfig config = new ManagedLedgerConfig();
config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
ManagedLedgerImpl ledger1 = (ManagedLedgerImpl) factory1.open("my_test_ledger", config);
ManagedLedgerImpl ledger1 = (ManagedLedgerImpl) factory1.open("my_test_ledger" + testName, config);
ledger1.openCursor("c");

ledger1.addEntry("entry-1".getBytes());

// Open the ML from another factory
@Cleanup("shutdown")
ManagedLedgerFactoryImpl factory2 = new ManagedLedgerFactoryImpl(metadataStore, bkc);
ManagedLedgerImpl ledger2 = (ManagedLedgerImpl) factory2.open("my_test_ledger", config);
ManagedLedgerImpl ledger2 = (ManagedLedgerImpl) factory2.open("my_test_ledger" + testName, config);
ManagedCursor c2 = ledger2.openCursor("c");

try {
Expand Down Expand Up @@ -477,7 +477,7 @@ public void managedLedgerClosed() throws Exception {
ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(metadataStore, bkc);
ManagedLedgerConfig config = new ManagedLedgerConfig();
config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
ManagedLedgerImpl ledger1 = (ManagedLedgerImpl) factory.open("my_test_ledger", config);
ManagedLedgerImpl ledger1 = (ManagedLedgerImpl) factory.open("my_test_ledger" + testName, config);

int N = 100;

Expand Down Expand Up @@ -517,7 +517,7 @@ public void testChangeCrcType() throws Exception {
ManagedLedgerConfig config = new ManagedLedgerConfig();
config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
config.setDigestType(DigestType.CRC32);
ManagedLedger ledger = factory.open("my_test_ledger", config);
ManagedLedger ledger = factory.open("my_test_ledger" + testName, config);
ManagedCursor c1 = ledger.openCursor("c1");

ledger.addEntry("entry-0".getBytes());
Expand All @@ -527,7 +527,7 @@ public void testChangeCrcType() throws Exception {
ledger.close();

config.setDigestType(DigestType.CRC32C);
ledger = factory.open("my_test_ledger", config);
ledger = factory.open("my_test_ledger" + testName, config);
c1 = ledger.openCursor("c1");

ledger.addEntry("entry-3".getBytes());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ protected String changeLedgerPath() {
return "/test";
}

@Test()
@Test(timeOut = 60000)
public void testChangeZKPath() throws Exception {
ClientConfiguration configuration = new ClientConfiguration();
String zkConnectString = zkUtil.getZooKeeperConnectString() + "/test";
Expand All @@ -61,8 +61,8 @@ public void testChangeZKPath() throws Exception {
.setAckQuorumSize(1)
.setMetadataAckQuorumSize(1)
.setMetadataAckQuorumSize(1);
ManagedLedger ledger = factory.open("test-ledger", config);
ManagedCursor cursor = ledger.openCursor("test-c1");
ManagedLedger ledger = factory.open("test-ledger" + testName, config);
ManagedCursor cursor = ledger.openCursor("test-c1" + testName);

for (int i = 0; i < 10; i++) {
String entry = "entry" + i;
Expand All @@ -77,7 +77,7 @@ public void testChangeZKPath() throws Exception {
Assert.assertEquals(("entry" + i).getBytes("UTF8"), entry.getData());
}
}
@Test()
@Test(timeOut = 60000)
public void testChangeZKPath2() throws Exception {
ClientConfiguration configuration = new ClientConfiguration();
String zkConnectString = zkUtil.getZooKeeperConnectString() + "/test";
Expand All @@ -98,8 +98,8 @@ public void testChangeZKPath2() throws Exception {
.setAckQuorumSize(1)
.setMetadataAckQuorumSize(1)
.setMetadataAckQuorumSize(1);
ManagedLedger ledger = factory.open("test-ledger", config);
ManagedCursor cursor = ledger.openCursor("test-c1");
ManagedLedger ledger = factory.open("test-ledger" + testName, config);
ManagedCursor cursor = ledger.openCursor("test-c1" + testName);

for (int i = 0; i < 10; i++) {
String entry = "entry" + i;
Expand Down
Loading

0 comments on commit 7a7b5fa

Please sign in to comment.