Fixed shading of Netty (apache#2783)

lujp · Oct 12, 2018 · f077f9d · f077f9d
1 parent 9bf3357
commit f077f9d
Show file tree

Hide file tree

Showing 9 changed files with 46 additions and 33 deletions.
diff --git a/managed-ledger-shaded/pom.xml b/managed-ledger-shaded/pom.xml
@@ -66,13 +66,17 @@
                                     <include>commons-codec:commons-codec</include>
                                     <include>commons-collections:commons-collections</include>
                                     <include>org.asynchttpclient:*</include>
+                                    <include>io.netty:netty-codec-http</include>
+                                    <include>io.netty:netty-transport-native-epoll</include>
                                     <include>org.reactivestreams:reactive-streams</include>
                                     <include>com.typesafe.netty:netty-reactive-streams</include>
                                     <include>org.javassist:javassist</include>
                                     <include>com.google.protobuf:protobuf-java</include>
                                     <include>com.google.guava:guava</include>
                                     <include>com.google.code.gson:gson</include>
                                     <include>com.fasterxml.jackson.core</include>
+                                    <include>io.netty:netty</include>
+                                    <include>io.netty:netty-all</include>
                                     <include>org.apache.pulsar:pulsar-common</include>
                                     <include>com.yahoo.datasketches:sketches-core</include>
                                     <include>org.glassfish.jersey*:*</include>
@@ -82,6 +86,7 @@
                                     <include>com.fasterxml.jackson.*:*</include>
                                     <include>io.grpc:*</include>
                                     <include>com.yahoo.datasketches:*</include>
+                                    <include>io.netty:*</include>
                                     <include>com.squareup.*:*</include>
                                     <include>com.google.*:*</include>
                                     <include>commons-*:*</include>
@@ -137,6 +142,10 @@
                                     <pattern>com.fasterxml.jackson</pattern>
                                     <shadedPattern>org.apache.pulsar.shade.com.fasterxml.jackson</shadedPattern>
                                 </relocation>
+                                <relocation>
+                                    <pattern>io.netty</pattern>
+                                    <shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern>
+                                </relocation>
                                 <relocation>
                                     <pattern>org.apache.pulsar.policies</pattern>
                                     <shadedPattern>org.apache.pulsar.shade.org.apache.pulsar.policies</shadedPattern>
@@ -262,6 +271,30 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+              <!-- This plugin is used to run a script after the package phase in order to rename
+                  libnetty_transport_native_epoll_x86_64.so from Netty into
+                  liborg_apache_pulsar_shade_netty_transport_native_epoll_x86_64.so
+                  to reflect the shade that is being applied.
+               -->
+              <artifactId>exec-maven-plugin</artifactId>
+              <groupId>org.codehaus.mojo</groupId>
+              <executions>
+                <execution>
+                  <id>rename-epoll-library</id>
+                  <phase>package</phase>
+                  <goals>
+                    <goal>exec</goal>
+                  </goals>
+                  <configuration>
+                    <executable>${project.parent.basedir}/src/rename-netty-native-libs.sh</executable>
+                    <arguments>
+                      <argument>${project.artifactId}</argument>
+                    </arguments>
+                  </configuration>
+                </execution>
+              </executions>
+            </plugin>
         </plugins>
     </build>
 </project>
diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml
@@ -76,6 +76,7 @@
                   <include>io.netty:netty</include>
                   <include>io.netty:netty-all</include>
                   <include>org.apache.pulsar:pulsar-common</include>
+                  <include>org.apache.bookkeeper:circe-checksum</include>
                   <include>com.yahoo.datasketches:sketches-core</include>
 
                   <include>javax.ws.rs:*</include>
@@ -148,9 +149,6 @@
                 <relocation>
                   <pattern>io.netty</pattern>
                   <shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern>
-                  <excludes>
-                    <exclude>io.netty.buffer.ByteBuf</exclude>
-                  </excludes>
                 </relocation>
                 <relocation>
                   <pattern>org.apache.pulsar.common</pattern>

diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml
@@ -77,6 +77,7 @@
                   <include>io.netty:netty</include>
                   <include>io.netty:netty-all</include>
                   <include>org.apache.pulsar:pulsar-common</include>
+                  <include>org.apache.bookkeeper:circe-checksum</include>
                   <include>com.yahoo.datasketches:sketches-core</include>
                   <include>org.glassfish.jersey*:*</include>
                   <include>javax.ws.rs:*</include>
@@ -132,9 +133,6 @@
                 <relocation>
                   <pattern>io.netty</pattern>
                   <shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern>
-                  <excludes>
-                    <exclude>io.netty.buffer.ByteBuf</exclude>
-                  </excludes>
                 </relocation>
                 <relocation>
                   <pattern>org.apache.pulsar.policies</pattern>

diff --git a/pulsar-client-kafka-compat/pulsar-client-kafka-shaded/pom.xml b/pulsar-client-kafka-compat/pulsar-client-kafka-shaded/pom.xml
@@ -87,6 +87,7 @@
                   <include>io.netty:netty</include>
                   <include>io.netty:netty-*</include>
                   <include>org.apache.pulsar:pulsar-common</include>
+                  <include>org.apache.bookkeeper:circe-checksum</include>
                   <include>com.yahoo.datasketches:sketches-core</include>
                   <include>org.apache.httpcomponents:httpclient</include>
                   <include>commons-logging:commons-logging</include>

diff --git a/pulsar-client-schema/pom.xml b/pulsar-client-schema/pom.xml
@@ -141,6 +141,7 @@
                                     <include>com.fasterxml.jackson.core</include>
                                     <include>io.netty:netty</include>
                                     <include>io.netty:netty-all</include>
+                                    <include>org.apache.bookkeeper:circe-checksum</include>
                                     <include>com.yahoo.datasketches:sketches-core</include>
                                     <include>org.glassfish.jersey*:*</include>
                                     <include>javax.ws.rs:*</include>
@@ -212,9 +213,6 @@
                                 <relocation>
                                     <pattern>io.netty</pattern>
                                     <shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern>
-                                    <excludes>
-                                        <exclude>io.netty.buffer.ByteBuf</exclude>
-                                    </excludes>
                                 </relocation>
                                 <relocation>
                                     <pattern>org.apache.pulsar.policies</pattern>

diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml
@@ -88,6 +88,7 @@
                   <include>commons-*:*</include>
 
                   <include>org.apache.pulsar:pulsar-common</include>
+                  <include>org.apache.bookkeeper:circe-checksum</include>
                   <include>com.yahoo.datasketches:sketches-core</include>
                   <include>org.apache.httpcomponents:httpclient</include>
                   <include>commons-logging:commons-logging</include>
@@ -137,9 +138,6 @@
                 <relocation>
                   <pattern>io.netty</pattern>
                   <shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern>
-                  <excludes>
-                    <exclude>io.netty.buffer.ByteBuf</exclude>
-                  </excludes>
                 </relocation>
                 <relocation>
                   <pattern>org.apache.pulsar.policies</pattern>

diff --git a/pulsar-spark/pom.xml b/pulsar-spark/pom.xml
@@ -115,9 +115,6 @@
                 <relocation>
                   <pattern>io.netty</pattern>
                   <shadedPattern>org.apache.pulsar.shade.io.netty</shadedPattern>
-                  <excludes>
-                    <exclude>io.netty.buffer.ByteBuf</exclude>
-                  </excludes>
                 </relocation>
               </relocations>
             </configuration>

diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
@@ -238,15 +238,7 @@ The Apache Software License, Version 2.0
     - commons-compress-1.15.jar
     - commons-lang3-3.3.2.jar
     - commons-lang3-3.4.jar
-    - commons-beanutils-1.7.0.jar
-    - commons-collections-3.2.1.jar
-    - commons-configuration-1.6.jar
-    - commons-digester-1.8.jar
-    - commons-lang-2.4.jar
-    - commons-logging-1.1.1.jar
  * Netty
-    - netty-all-4.1.22.Final.jar
-    - netty-tcnative-boringssl-static-2.0.7.Final.jar
     - netty-3.6.2.Final.jar
  * Joda Time
      - joda-time-2.9.9.jar
@@ -399,8 +391,6 @@ The Apache Software License, Version 2.0
     - simpleclient_common-0.0.23.jar
     - simpleclient_hotspot-0.0.23.jar
     - simpleclient_servlet-0.0.23.jar
-  * BookKeeper
-    - circe-checksum-4.7.2.jar
   * LZ4
     - lz4-java-1.5.0.jar
 

diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnector.java
@@ -50,7 +50,6 @@
 import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
 import org.apache.pulsar.common.schema.SchemaInfo;
 import org.apache.pulsar.common.schema.SchemaType;
-import io.netty.buffer.ByteBuf;
 import org.apache.pulsar.shade.javax.ws.rs.ClientErrorException;
 import org.apache.pulsar.shade.javax.ws.rs.core.Response;
 import org.apache.pulsar.shade.org.apache.bookkeeper.stats.NullStatsProvider;
@@ -603,10 +602,11 @@ private static List<Entry> getTopicEntries(String topicSchemaName) {
 
             Schema schema = topicsToSchemas.get(topicSchemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class) : JSONSchema.of(Foo.class);
 
-            ByteBuf payload
-                    = org.apache.pulsar.shade.io.netty.buffer.Unpooled.copiedBuffer(schema.encode(foo));
+            org.apache.pulsar.shade.io.netty.buffer.ByteBuf payload = org.apache.pulsar.shade.io.netty.buffer.Unpooled
+                    .copiedBuffer(schema.encode(foo));
 
-            ByteBuf byteBuf = serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, messageMetadata, payload);
+            org.apache.pulsar.shade.io.netty.buffer.ByteBuf byteBuf = serializeMetadataAndPayload(
+                    Commands.ChecksumType.Crc32c, messageMetadata, payload);
 
             Entry entry = EntryImpl.create(0, i, byteBuf);
             log.info("create entry: %s", entry.getEntryId());
@@ -851,11 +851,11 @@ public void run() {
 
                                     Schema schema = topicsToSchemas.get(schemaName).getType() == SchemaType.AVRO ? AvroSchema.of(Foo.class) : JSONSchema.of(Foo.class);
 
-                                    ByteBuf payload
-                                            = org.apache.pulsar.shade.io.netty.buffer.Unpooled.copiedBuffer(schema.encode(foo));
+                                    org.apache.pulsar.shade.io.netty.buffer.ByteBuf payload = org.apache.pulsar.shade.io.netty.buffer.Unpooled
+                                            .copiedBuffer(schema.encode(foo));
 
-                                    ByteBuf byteBuf = serializeMetadataAndPayload
-                                            (Commands.ChecksumType.Crc32c, messageMetadata, payload);
+                                    org.apache.pulsar.shade.io.netty.buffer.ByteBuf byteBuf = serializeMetadataAndPayload(
+                                            Commands.ChecksumType.Crc32c, messageMetadata, payload);
 
                                     completedBytes += byteBuf.readableBytes();