Distribute Pulsar IO connectors individually (apache#3601)

tahirh · Feb 14, 2019 · 0a6fa4b · 0a6fa4b
1 parent 6fba72c
commit 0a6fa4b
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 92 deletions.
diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml
@@ -59,8 +59,7 @@
               <goal>single</goal>
             </goals>
             <configuration>
-              <attach>true</attach>
-              <tarLongFileMode>posix</tarLongFileMode>
+              <attach>false</attach>
               <finalName>apache-pulsar-io-connectors-${project.version}</finalName>
               <descriptors>
                 <descriptor>src/assemble/io.xml</descriptor>

diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml
@@ -24,15 +24,9 @@
   xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
   <id>bin</id>
   <formats>
-    <format>tar.gz</format>
+    <format>dir</format>
   </formats>
-  <includeBaseDirectory>true</includeBaseDirectory>
-  <fileSets>
-    <fileSet>
-      <directory>${basedir}/../../distribution/io/target/conf</directory>
-      <outputDirectory>conf</outputDirectory>
-    </fileSet>
-  </fileSets>
+  <includeBaseDirectory>false</includeBaseDirectory>
   <files>
     <file>
       <source>${basedir}/../../LICENSE</source>
@@ -46,70 +40,30 @@
       <fileMode>644</fileMode>
     </file>
 
-    <file>
-      <source>${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar</source>
-      <outputDirectory>connectors</outputDirectory>
-      <fileMode>644</fileMode>
-    </file>
-
-    <file>
-      <source>${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar</source>
-      <outputDirectory>connectors</outputDirectory>
-      <fileMode>644</fileMode>
-    </file>
-
-    <file>
-      <source>${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar</source>
-      <outputDirectory>connectors</outputDirectory>
-      <fileMode>644</fileMode>
-    </file>
-
-    <file>
-      <source>${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar</source>
-      <outputDirectory>connectors</outputDirectory>
-      <fileMode>644</fileMode>
-    </file>
-
-    <file>
-      <source>${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar</source>
-      <outputDirectory>connectors</outputDirectory>
-      <fileMode>644</fileMode>
-    </file>
-
-    <file>
-      <source>${basedir}/../../pulsar-io/jdbc/target/pulsar-io-jdbc-${project.version}.nar</source>
-      <outputDirectory>connectors</outputDirectory>
-      <fileMode>644</fileMode>
-    </file>
-
-    <file>
-      <source>${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar</source>
-      <outputDirectory>connectors</outputDirectory>
-      <fileMode>644</fileMode>
-    </file>
+    <!-- Include all connectors.
+         There doesn't seem to be a straight way to automatically
+         include all nars and place them in a single directory.
+     -->
 
-    <file>
-      <source>${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar</source>
-      <outputDirectory>connectors</outputDirectory>
-      <fileMode>644</fileMode>
-    </file>
-
-    <file>
-      <source>${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar</source>
-      <outputDirectory>connectors</outputDirectory>
-      <fileMode>644</fileMode>
-    </file>
-
-    <file>
-      <source>${basedir}/../../pulsar-io/kafka-connect-adaptor/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar</source>
-      <outputDirectory>connectors</outputDirectory>
-      <fileMode>644</fileMode>
-    </file>
-
-    <file>
-      <source>${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar</source>
-      <outputDirectory>connectors</outputDirectory>
-      <fileMode>644</fileMode>
-    </file>
+    <file><source>${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/jdbc/target/pulsar-io-jdbc-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/kafka-connect-adaptor/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/hdfs2/target/pulsar-io-hdfs2-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/hdfs3/target/pulsar-io-hdfs3-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/file/target/pulsar-io-file-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/canal/target/pulsar-io-canal-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/netty/target/pulsar-io-netty-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/debezium/target/pulsar-io-debezium-${project.version}.nar</source></file>
+    <file><source>${basedir}/../../pulsar-io/mongo/target/pulsar-io-mongo-${project.version}.nar</source></file>
   </files>
 </assembly>
diff --git a/docker/pulsar-all/Dockerfile b/docker/pulsar-all/Dockerfile
@@ -19,11 +19,10 @@
 
 FROM apachepulsar/pulsar:latest
 
-ARG PULSAR_IO_TARBALL
+ARG PULSAR_IO_DIR
 ARG PULSAR_OFFLOADER_TARBALL
 
-ADD ${PULSAR_IO_TARBALL} /
-RUN mv /apache-pulsar-io-connectors-*/connectors /pulsar/connectors
+ADD ${PULSAR_IO_DIR} /pulsar/connectors
 
 ADD ${PULSAR_OFFLOADER_TARBALL} /
 RUN mv /apache-pulsar-offloaders-*/offloaders /pulsar/offloaders
diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
@@ -26,7 +26,6 @@
     <version>2.3.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
-  <groupId>org.apache.pulsar</groupId>
   <artifactId>pulsar-all-docker-image</artifactId>
   <name>Apache Pulsar :: Docker Images :: Pulsar Latest Version (Include All Components)</name>
   <packaging>pom</packaging>
@@ -36,8 +35,7 @@
       <groupId>org.apache.pulsar</groupId>
       <artifactId>pulsar-io-distribution</artifactId>
       <version>${project.parent.version}</version>
-      <classifier>bin</classifier>
-      <type>tar.gz</type>
+      <type>pom</type>
       <scope>provided</scope>
     </dependency>
     <dependency>
@@ -65,21 +63,30 @@
       <build>
         <plugins>
           <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-dependency-plugin</artifactId>
+            <artifactId>maven-resources-plugin</artifactId>
             <executions>
               <execution>
-                <id>copy-io-tarball</id>
+                <id>copy-resources</id>
+                <phase>generate-resources</phase>
                 <goals>
-                  <goal>copy-dependencies</goal>
+                  <goal>copy-resources</goal>
                 </goals>
-                <phase>generate-resources</phase>
                 <configuration>
-                  <outputDirectory>${project.build.directory}/</outputDirectory>
-                  <includeArtifactIds>pulsar-io-distribution</includeArtifactIds>
-                  <excludeTransitive>true</excludeTransitive>
+                  <outputDirectory>${basedir}/target/apache-pulsar-io-connectors-${project.version}-bin</outputDirectory>
+                  <resources>
+                    <resource>
+                      <directory>${basedir}/../../distribution/io/target/apache-pulsar-io-connectors-${project.version}-bin</directory>
+                      <filtering>false</filtering>
+                    </resource>
+                  </resources>
                 </configuration>
               </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-dependency-plugin</artifactId>
+            <executions>
               <execution>
                 <id>copy-offloader-tarball</id>
                 <goals>
@@ -122,7 +129,7 @@
               <pullNewerImage>false</pullNewerImage>
               <tag>${project.version}</tag>
               <buildArgs>
-                <PULSAR_IO_TARBALL>target/pulsar-io-distribution-${project.version}-bin.tar.gz</PULSAR_IO_TARBALL>
+                <PULSAR_IO_DIR>target/apache-pulsar-io-connectors-${project.version}-bin</PULSAR_IO_DIR>
                 <PULSAR_OFFLOADER_TARBALL>target/pulsar-offloader-distribution-${project.version}-bin.tar.gz</PULSAR_OFFLOADER_TARBALL>
               </buildArgs>
             </configuration>

diff --git a/src/stage-release.sh b/src/stage-release.sh
@@ -27,16 +27,17 @@ fi
 
 DEST_PATH=$1
 
-pushd $(dirname "$0")
+pushd pushd $(dirname "$0")
 PULSAR_PATH=$(git rev-parse --show-toplevel)
 VERSION=`./get-project-version.py`
 popd
 
 cp $PULSAR_PATH/distribution/server/target/apache-pulsar-$VERSION-src.tar.gz $DEST_PATH
 cp $PULSAR_PATH/distribution/server/target/apache-pulsar-$VERSION-bin.tar.gz $DEST_PATH
-cp $PULSAR_PATH/distribution/io/target/apache-pulsar-io-connectors-$VERSION-bin.tar.gz $DEST_PATH
 cp $PULSAR_PATH/distribution/offloaders/target/apache-pulsar-offloaders-$VERSION-bin.tar.gz $DEST_PATH
 
+cp -r $PULSAR_PATH/distribution/io/target/apache-pulsar-io-connectors-$VERSION-bin $DEST_PATH/connectors
+
 mkdir $DEST_PATH/RPMS
 cp -r $PULSAR_PATH/pulsar-client-cpp/pkg/rpm/RPMS/x86_64/* $DEST_PATH/RPMS
 
@@ -45,4 +46,4 @@ cp -r $PULSAR_PATH/pulsar-client-cpp/pkg/deb/BUILD/DEB/* $DEST_PATH/DEB
 
 # Sign all files
 cd $DEST_PATH
-find . -type f | xargs $PULSAR_PATH/src/sign-release.sh
+find . -type f | grep -v LICENSE | grep -v README | xargs $PULSAR_PATH/src/sign-release.sh