diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 775b0a6420986..2796a32a10b2b 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -69,15 +69,10 @@ - org.bouncycastle - bc-fips - ${bouncycastlefips.version} - - - - org.bouncycastle - bcpkix-fips - ${bouncycastlefips.version} + ${project.groupId} + bouncy-castle-bcfips + ${project.version} + pkg diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 71c0e278c47f2..d180083b1c616 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -40,14 +40,6 @@ ${project.version} - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - true - - ${project.groupId} pulsar-discovery-service diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index de0a216244f74..9bfb9e43b6360 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -40,13 +40,6 @@ pulsar-broker ${project.parent.version} - - ${project.groupId} - bouncy-castle-bc - ${project.parent.version} - pkg - true - @@ -111,8 +104,6 @@ com.thoughtworks.paranamer:paranamer org.apache.commons:commons-compress org.tukaani:xz - - org.bouncycastle:* @@ -121,6 +112,10 @@ ** + + + org/bouncycastle/** + diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 4310d526d7ee2..635143dcc4209 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -76,14 +76,6 @@ ${project.version} - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - true - - ${project.groupId} pulsar-discovery-service diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index b461e29e2a91b..651a6eb39d79e 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -69,6 +69,10 @@ ** + + + org/bouncycastle/** + diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 52db2278ef44e..ce33a41d6e36a 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -92,8 +92,6 @@ org.yaml:snakeyaml io.swagger:* org.apache.bookkeeper:bookkeeper-common-allocator - - org.bouncycastle:* @@ -102,12 +100,20 @@ ** + + + org/bouncycastle/** + org.apache.pulsar:pulsar-client-admin-original ** + + + org/bouncycastle/** + diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 83a259791b349..6c849d84edd45 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -49,19 +49,11 @@ ${project.parent.version} true - - ${project.groupId} - bouncy-castle-bc - ${project.parent.version} - pkg - true - - org.apache.maven.plugins maven-dependency-plugin @@ -85,8 +77,6 @@ ${project.build.directory}/classes - - @@ -162,8 +152,7 @@ org.apache.commons:commons-compress org.tukaani:xz org.apache.bookkeeper:bookkeeper-common-allocator - - org.bouncycastle:* + org.apache.pulsar:pulsar-client-messagecrypto-bc @@ -173,6 +162,10 @@ ** + + + org/bouncycastle/** + diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 1095d60e0408c..8662b131119fa 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -42,14 +42,6 @@ ${project.groupId} pulsar-client-messagecrypto-bc ${project.version} - true - - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - true @@ -89,8 +81,6 @@ ${project.build.directory}/classes/org/asynchttpclient/config - - @@ -149,27 +139,20 @@ com.thoughtworks.paranamer:paranamer org.apache.commons:commons-compress org.tukaani:xz - - org.bouncycastle:* + org.apache.pulsar:pulsar-client-messagecrypto-bc - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - org.apache.pulsar:pulsar-client-original ** + + + org/bouncycastle/** + diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index e81fbec9047c3..198e12ae75117 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -57,13 +57,6 @@ ${project.version} test - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - test - diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 8909964022d0b..3bb93827f0f98 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -56,11 +56,10 @@ bouncy-castle-bc ${project.parent.version} pkg - - true + ${project.groupId} pulsar-client-messagecrypto-bc ${project.parent.version} diff --git a/pulsar-discovery-service/pom.xml b/pulsar-discovery-service/pom.xml index 7196c2341b86c..f991585d5a15a 100644 --- a/pulsar-discovery-service/pom.xml +++ b/pulsar-discovery-service/pom.xml @@ -67,7 +67,6 @@ bouncy-castle-bc ${project.version} pkg - true diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 680c6e40923a3..35b8b386c7f05 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -60,13 +60,13 @@ - - *:* + org.apache.pulsar:pulsar-client-original + + ** + - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA + + org/bouncycastle/** @@ -250,10 +250,6 @@ jline org.apache.pulsar.functions.runtime.shaded.jline - - org.bouncycastle - org.apache.pulsar.functions.runtime.shaded.org.bouncycastle - org.xerial.snappy org.apache.pulsar.functions.runtime.shaded.org.xerial.snappy @@ -353,4 +349,4 @@ - \ No newline at end of file + diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 2c4f1485b421d..a8cb15aaf517a 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -52,13 +52,6 @@ ${project.version} - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - - ${project.groupId} pulsar-client-admin-original diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index fb4dcc8e7722f..3747756871740 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -68,13 +68,6 @@ ${project.version} - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - - ${project.groupId} pulsar-broker diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 974ef1f57ba8f..1f8b0c7f7c2ae 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -68,13 +68,6 @@ ${project.version} - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - - ${project.groupId} kafka-connect-avro-converter-shaded diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index ebdb32fdf9ab4..c5f4ecb216669 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -171,12 +171,5 @@ com.beust jcommander - - - ${project.groupId} - bouncy-castle-bc - ${project.version} - test - diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index c2caef58fc715..019ea2545d210 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -63,14 +63,6 @@ ${project.version} - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - true - - ${project.groupId} managed-ledger @@ -148,6 +140,10 @@ ** + + + org/bouncycastle/** + diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index 97c55b1a7b3e7..66859b3ed65d3 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -66,14 +66,6 @@ true - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - true - - ${project.groupId} pulsar-broker diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index e39c4e225c8e7..163a79a5415d6 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -45,14 +45,6 @@ ${project.version} - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - true - - ${project.groupId} managed-ledger diff --git a/site2/docs/security-bouncy-castle.md b/site2/docs/security-bouncy-castle.md index 9805fdc4bea26..9b22b37d97292 100644 --- a/site2/docs/security-bouncy-castle.md +++ b/site2/docs/security-bouncy-castle.md @@ -7,7 +7,7 @@ sidebar_label: Bouncy Castle Providers ## BouncyCastle Introduce `Bouncy Castle` is a Java library that complements the default Java Cryptographic Extension (JCE), -and it many more cipher suites and algorithms than the default JCE provided by Sun. +and it provides more cipher suites and algorithms than the default JCE provided by Sun. In addition to that, `Bouncy Castle` has lots of utilities for reading arcane formats like PEM and ASN.1 that no sane person would want to rewrite themselves. @@ -17,57 +17,84 @@ In Pulsar, security and crypto have dependencies on BouncyCastle Jars. For the d In Pulsar, the security and crypto methods also depends on `Bouncy Castle`, especially in [TLS Authentication](security-tls-authentication.md) and [Transport Encryption](security-encryption.md). This document contains the configuration between BouncyCastle FIPS(BC-FIPS) and non-FIPS(BC-non-FIPS) version while using Pulsar. -## Include dependencies of BC-non-FIPS +## How BouncyCastle modules packaged in Pulsar -By default, BouncyCastle non-FIPS version is build along with Pulsar's Broker and Java client. +In Pulsar's `bouncy-castle` module, We provide 2 sub modules: `bouncy-castle-bc`(for non-FIPS version) and `bouncy-castle-bcfips`(for FIPS version), to package BC jars together to make the include and exclude of `Bouncy Castle` easier. -Pulsar module `bouncy-castle-bc`, which defined by `bouncy-castle/bc/pom.xml` contains the needed non-FIPS jars for Pulsar. +To achieve this goal, we will need to package several `bouncy-castle` jars together into `bouncy-castle-bc` or `bouncy-castle-bcfips` jar. +Each of the original bouncy-castle jar is related with security, so BouncyCastle dutifully supplies signed of each JAR. +But when we do the re-package, Maven shade explodes the BouncyCastle jar file which puts the signatures into META-INF, +these signatures aren't valid for this new, uber-jar (signatures are only for the original BC jar). +Usually, You will meet error like `java.lang.SecurityException: Invalid signature file digest for Manifest main attributes`. -```xml - - org.bouncycastle - bcpkix-jdk15on - ${bouncycastle.version} - - - - org.bouncycastle - bcprov-ext-jdk15on - ${bouncycastle.version} - +You could exclude these signatures in mvn pom file to avoid above error, by +```access transformers +META-INF/*.SF +META-INF/*.DSA +META-INF/*.RSA ``` +But it can also lead to new, cryptic errors, e.g. `java.security.NoSuchAlgorithmException: PBEWithSHA256And256BitAES-CBC-BC SecretKeyFactory not available` +By explicitly specifying where to find the algorithm like this: `SecretKeyFactory.getInstance("PBEWithSHA256And256BitAES-CBC-BC","BC")` +It will get the real error: `java.security.NoSuchProviderException: JCE cannot authenticate the provider BC` -By using this `bouncy-castle-bc` module, you can easily include and exclude BouncyCastle non-FIPS jars. +So, we used a [executable packer plugin](https://github.com/nthuemmel/executable-packer-maven-plugin) that uses a jar-in-jar approach to preserve the BouncyCastle signature in a single, executable jar. -### Pulsar Client and Broker dependencies on BC-non-FIPS +### Include dependencies of BC-non-FIPS -Pulsar Client(`pulsar-client-original`) module include BouncyCastle non-FIPS jars by add dependency like this: +Pulsar module `bouncy-castle-bc`, which defined by `bouncy-castle/bc/pom.xml` contains the needed non-FIPS jars for Pulsar, and packaged as a jar-in-jar(need to provide `pkg`). ```xml - - org.apache.pulsar - bouncy-castle-bc - ${project.parent.version} - pkg - + + org.bouncycastle + bcpkix-jdk15on + ${bouncycastle.version} + + + + org.bouncycastle + bcprov-ext-jdk15on + ${bouncycastle.version} + ``` -And Pulsar Broker (`pulsar-broker`) module include BouncyCastle non-FIPS jars by indirectly include Pulsar Client(`pulsar-client-original`) module. +By using this `bouncy-castle-bc` module, you can easily include and exclude BouncyCastle non-FIPS jars. + +### Modules that include BC-non-FIPS module (`bouncy-castle-bc`) + +For Pulsar client, user need the bouncy-castle module, so `pulsar-client-original` will include the `bouncy-castle-bc` module, and have `pkg` set to reference the `jar-in-jar` package. +It is included as following example: ```xml - - org.apache.pulsar - pulsar-client-original - ${project.version} - + + org.apache.pulsar + bouncy-castle-bc + ${pulsar.version} + pkg + ``` -## Exclude BC-non-FIPS and include BC-FIPS +By default `bouncy-castle-bc` already included in `pulsar-client-original`, And `pulsar-client-original` has been included in a lot of other modules like `pulsar-client-admin`, `pulsar-broker`. +But for the above shaded jar and signatures reason, we should not package Pulsar's `bouncy-castle` module into `pulsar-client-all` other shaded modules directly, such as `pulsar-client-shaded`, `pulsar-client-admin-shaded` and `pulsar-broker-shaded`. +So in the shaded modules, we will exclude the `bouncy-castle` modules. +```xml + + + org.apache.pulsar:pulsar-client-original + + ** + + + org/bouncycastle/** + + + +``` -After understanding the above dependencies, user can easily exclude non-FIPS version and include FIPS version. +That means, `bouncy-castle` related jars are not shaded in these fat jars. -### BC-FIPS +### Module BC-FIPS (`bouncy-castle-bcfips`) -Pulsar module `bouncy-castle-bcfips`, which defined by `bouncy-castle/bcfips/pom.xml` contains the needed FIPS jars for Pulsar. +Pulsar module `bouncy-castle-bcfips`, which defined by `bouncy-castle/bcfips/pom.xml` contains the needed FIPS jars for Pulsar. +Similar to `bouncy-castle-bc`, `bouncy-castle-bcfips` also packaged as a `jar-in-jar` package for easy include/exclude. ```xml @@ -83,38 +110,29 @@ Pulsar module `bouncy-castle-bcfips`, which defined by `bouncy-castle/bcfips/pom ``` -User can choose include module `bouncy-castle-bcfips` module directly, or include original BC-FIPS jars. - -For example: +### Exclude BC-non-FIPS and include BC-FIPS +If you want to switch from BC-non-FIPS to BC-FIPS version, Here is an example for `pulsar-broker` module: ```xml - - ${project.groupId} - pulsar-broker - ${project.version} - - - ${project.groupId} - bouncy-castle-bc - - - - - - - org.bouncycastle - bc-fips - ${bouncycastlefips.version} - - - - org.bouncycastle - bcpkix-fips - ${bouncycastlefips.version} - -``` + + org.apache.pulsar + pulsar-broker + ${pulsar.version} + + + org.apache.pulsar + bouncy-castle-bc + + + + + + org.apache.pulsar + bouncy-castle-bcfips + ${pulsar.version} + pkg + +``` -Besides this, module `bouncy-castle-bcfips` builds contain an output with format NAR, you can set java environment by `-DBcPath='nar/file/path'`, Pulsar will auto load it. - -For more example, you can reference module `bcfips-include-test` and `bcfips-nar-test`. +For more example, you can reference module `bcfips-include-test`. diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 55729a5a822fb..18d610259154c 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -66,13 +66,6 @@ ${project.version} test - - org.apache.pulsar - bouncy-castle-bc - ${project.version} - pkg - test - org.apache.pulsar pulsar-client-admin diff --git a/tests/pulsar-kafka-compat-client-test/pom.xml b/tests/pulsar-kafka-compat-client-test/pom.xml index d0adddc2a8626..e77cb8df93fd5 100644 --- a/tests/pulsar-kafka-compat-client-test/pom.xml +++ b/tests/pulsar-kafka-compat-client-test/pom.xml @@ -62,13 +62,6 @@ pulsar-common ${project.version} - - org.apache.pulsar - bouncy-castle-bc - ${project.version} - pkg - test - org.apache.pulsar pulsar-client-kafka diff --git a/tests/pulsar-storm-test/pom.xml b/tests/pulsar-storm-test/pom.xml index 0a0d524c860b4..3cf562d59cc36 100644 --- a/tests/pulsar-storm-test/pom.xml +++ b/tests/pulsar-storm-test/pom.xml @@ -65,14 +65,6 @@ - - org.apache.pulsar - bouncy-castle-bc - ${project.version} - pkg - test - - org.apache.pulsar pulsar-broker