Package Pulsar IO connectors into separate tgz in form of nar archives (

apache#2054) ### Motivation We need to package Pulsar IO connectors as part of the release. We provide a new file `apache-pulsar-io-connectors-$VERSION-bin.tar.gz` that contains [NAR](https://medium.com/hashmapinc/nifi-nar-files-explained-14113f7796fd). In subsequent PR, I'll add the logic to load and use these archives.
campolake · Jul 2, 2018 · b22e883 · b22e883
1 parent 4ade1bd
commit b22e883
Show file tree

Hide file tree

Showing 52 changed files with 306 additions and 78 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -75,7 +75,7 @@ deploy:
     api_key:
       secure: cmjsJ43FwlwWUVh+4TJ/V+yKu/bd78ov0Olm0BFao0tco5ZYNfbRB5jxWD9X4AxpRN4Rfn+u/17oRavUSfv1M/CKMj6mTzXl8gQ6gieIWrgGYfdZrpBxEY0mccaEbaXSGNEWFj/e8oJixdeBeFCp3AkUSTO9DS5f+yoKeF1XibeJIhDLY0xWgeLMkUimzizsiplQBhrPmHExM8DRwSEojs4np56QgcJHdpU9snxkKzjCW5kKcQ6vPXzWpRLBRxLNU0MzYf6HRSqsGE5M3oG8PXNi+WuMS+4pKfxhcw7vS7642/8dWV28Flvet9E+rXutaM7I+jd5ZQG+/jTo2IOTUJ164ZaxYl6rjsf94d8u51AxDPLer+/C19DfrjiYGOLX8Cad+dLWT+otVISie76oNTeThqyG/5W+PpX9cTP/yBeZs5j/mgYJI4mVU4z0fACgyh+Gc3SyBwPvc3eePsFoVS4CvksgoMJJzgtN6to5hh8Pl4dM3FtpWnhjS3Zm2ieu8CUiC3HKCB1DzeNXXRkgcL+D8CXrxcfeOtBs34MEwua7C3fmSagyt1e1/PeYPE4F9Oi6Tiu59jGA5mnx27TDiPCEsf0a5UBHIY95ducB/BBKsXrACmnzAgPSP321fDNKBWMveTw3h2dKLC6PIf78LJ3Rln/i4okCQ8PkgEOz1A0=
     file_glob: true
-    file: "all/target/pulsar-*.tar.gz"
+    file: "distribution/server/target/pulsar-*.tar.gz"
     on:
       repo: apache/incubator-pulsar
       tags: true

diff --git a/bin/bookkeeper b/bin/bookkeeper
@@ -119,7 +119,7 @@ add_maven_deps_to_classpath() {
     # Need to generate classpath from maven pom. This is costly so generate it
     # and cache it. Save the file into our target dir so a mvn clean will get
     # clean it up and force us create a new one.
-    f="${BK_HOME}/all/target/classpath.txt"
+    f="${BK_HOME}/distribution/server/target/classpath.txt"
     if [ ! -f "${f}" ]
     then
 	${MVN} -f "${BK_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null

diff --git a/bin/pulsar b/bin/pulsar
@@ -151,7 +151,7 @@ add_maven_deps_to_classpath() {
     # Need to generate classpath from maven pom. This is costly so generate it
     # and cache it. Save the file into our target dir so a mvn clean will get
     # clean it up and force us create a new one.
-    f="${PULSAR_HOME}/all/target/classpath.txt"
+    f="${PULSAR_HOME}/distribution/server/target/classpath.txt"
     if [ ! -f "${f}" ]
     then
 	    ${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null

diff --git a/bin/pulsar-admin b/bin/pulsar-admin
@@ -72,7 +72,7 @@ add_maven_deps_to_classpath() {
     # Need to generate classpath from maven pom. This is costly so generate it
     # and cache it. Save the file into our target dir so a mvn clean will get
     # clean it up and force us create a new one.
-    f="${PULSAR_HOME}/all/target/classpath.txt"
+    f="${PULSAR_HOME}/distribution/server/target/classpath.txt"
     if [ ! -f "${f}" ]
     then
 	${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null

diff --git a/bin/pulsar-client b/bin/pulsar-client
@@ -65,7 +65,7 @@ add_maven_deps_to_classpath() {
     # Need to generate classpath from maven pom. This is costly so generate it
     # and cache it. Save the file into our target dir so a mvn clean will get
     # clean it up and force us create a new one.
-    f="${PULSAR_HOME}/all/target/classpath.txt"
+    f="${PULSAR_HOME}/distribution/server/target/classpath.txt"
     if [ ! -f "${f}" ]
     then
 	${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null

diff --git a/bin/pulsar-perf b/bin/pulsar-perf
@@ -65,7 +65,7 @@ add_maven_deps_to_classpath() {
     # Need to generate classpath from maven pom. This is costly so generate it
     # and cache it. Save the file into our target dir so a mvn clean will get
     # clean it up and force us create a new one.
-    f="${PULSAR_HOME}/all/target/classpath.txt"
+    f="${PULSAR_HOME}/distribution/server/target/classpath.txt"
     if [ ! -f "${f}" ]
     then
 	${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null

diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml
@@ -0,0 +1,68 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache.pulsar</groupId>
+    <artifactId>distribution</artifactId>
+    <version>2.2.0-incubating-SNAPSHOT</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <artifactId>pulsar-io-distribution</artifactId>
+  <packaging>pom</packaging>
+  <name>Pulsar :: Distribution :: IO</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.pulsar</groupId>
+      <artifactId>pulsar-io-core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>distro-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+            <configuration>
+              <attach>false</attach>
+              <tarLongFileMode>posix</tarLongFileMode>
+              <finalName>apache-pulsar-io-connectors-${project.version}</finalName>
+              <descriptors>
+                <descriptor>src/assemble/io.xml</descriptor>
+              </descriptors>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/distribution/io/src/assemble/README b/distribution/io/src/assemble/README
@@ -0,0 +1,10 @@
+
+Please refer to http://pulsar.incubator.apache.org/ for access to documentation.
+
+This package contains Pulsar IO connectors archives. Each archive
+contains:
+
+ * the connector code plus all the dependencies
+
+ * META-INF/DEPEDENCIES file with licensing information for all transitive
+   dependencies
diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml
@@ -0,0 +1,84 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<assembly
+  xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+  <id>bin</id>
+  <formats>
+    <format>tar.gz</format>
+  </formats>
+  <includeBaseDirectory>true</includeBaseDirectory>
+  <files>
+    <file>
+      <source>${basedir}/../../DISCLAIMER</source>
+      <outputDirectory>.</outputDirectory>
+      <fileMode>644</fileMode>
+    </file>
+    <file>
+      <source>${basedir}/../../LICENSE</source>
+      <outputDirectory>.</outputDirectory>
+      <fileMode>644</fileMode>
+    </file>
+    <file>
+      <source>${basedir}/src/assemble/README</source>
+      <destName>README</destName>
+      <outputDirectory>.</outputDirectory>
+      <fileMode>644</fileMode>
+    </file>
+
+    <file>
+      <source>${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar</source>
+      <outputDirectory>connectors</outputDirectory>
+      <fileMode>644</fileMode>
+    </file>
+
+    <file>
+      <source>${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar</source>
+      <outputDirectory>connectors</outputDirectory>
+      <fileMode>644</fileMode>
+    </file>
+
+    <file>
+      <source>${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar</source>
+      <outputDirectory>connectors</outputDirectory>
+      <fileMode>644</fileMode>
+    </file>
+
+    <file>
+      <source>${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar</source>
+      <outputDirectory>connectors</outputDirectory>
+      <fileMode>644</fileMode>
+    </file>
+
+    <file>
+      <source>${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar</source>
+      <outputDirectory>connectors</outputDirectory>
+      <fileMode>644</fileMode>
+    </file>
+
+    <file>
+      <source>${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar</source>
+      <outputDirectory>connectors</outputDirectory>
+      <fileMode>644</fileMode>
+    </file>
+  </files>
+</assembly>
diff --git a/distribution/pom.xml b/distribution/pom.xml
@@ -0,0 +1,40 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache.pulsar</groupId>
+    <artifactId>pulsar</artifactId>
+    <version>2.2.0-incubating-SNAPSHOT</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <artifactId>distribution</artifactId>
+  <packaging>pom</packaging>
+  <name>Pulsar :: Distribution</name>
+
+  <modules>
+    <module>server</module>
+    <module>io</module>
+  </modules>
+</project>
diff --git a/all/licenses/LICENSE-AspectJ.txt → ...ution/server/licenses/LICENSE-AspectJ.txt b/all/licenses/LICENSE-AspectJ.txt → ...ution/server/licenses/LICENSE-AspectJ.txt
diff --git a/all/licenses/LICENSE-CC0.txt → distribution/server/licenses/LICENSE-CC0.txt b/all/licenses/LICENSE-CC0.txt → distribution/server/licenses/LICENSE-CC0.txt
diff --git a/all/licenses/LICENSE-CDDL-1.1.txt → ...tion/server/licenses/LICENSE-CDDL-1.1.txt b/all/licenses/LICENSE-CDDL-1.1.txt → ...tion/server/licenses/LICENSE-CDDL-1.1.txt
diff --git a/all/licenses/LICENSE-EA-Agent-Loader.txt → ...rver/licenses/LICENSE-EA-Agent-Loader.txt b/all/licenses/LICENSE-EA-Agent-Loader.txt → ...rver/licenses/LICENSE-EA-Agent-Loader.txt
diff --git a/all/licenses/LICENSE-EPL-1.0.txt → ...ution/server/licenses/LICENSE-EPL-1.0.txt b/all/licenses/LICENSE-EPL-1.0.txt → ...ution/server/licenses/LICENSE-EPL-1.0.txt
diff --git a/all/licenses/LICENSE-Hamcrest.txt → ...tion/server/licenses/LICENSE-Hamcrest.txt b/all/licenses/LICENSE-Hamcrest.txt → ...tion/server/licenses/LICENSE-Hamcrest.txt
diff --git a/all/licenses/LICENSE-HdrHistogram.txt → .../server/licenses/LICENSE-HdrHistogram.txt b/all/licenses/LICENSE-HdrHistogram.txt → .../server/licenses/LICENSE-HdrHistogram.txt
diff --git a/all/licenses/LICENSE-HdrHistogram0.txt → ...server/licenses/LICENSE-HdrHistogram0.txt b/all/licenses/LICENSE-HdrHistogram0.txt → ...server/licenses/LICENSE-HdrHistogram0.txt
diff --git a/all/licenses/LICENSE-JSR305.txt → ...bution/server/licenses/LICENSE-JSR305.txt b/all/licenses/LICENSE-JSR305.txt → ...bution/server/licenses/LICENSE-JSR305.txt
diff --git a/all/licenses/LICENSE-LevelDB.txt → ...ution/server/licenses/LICENSE-LevelDB.txt b/all/licenses/LICENSE-LevelDB.txt → ...ution/server/licenses/LICENSE-LevelDB.txt
diff --git a/all/licenses/LICENSE-Lombok.txt → ...bution/server/licenses/LICENSE-Lombok.txt b/all/licenses/LICENSE-Lombok.txt → ...bution/server/licenses/LICENSE-Lombok.txt
diff --git a/all/licenses/LICENSE-SLF4J.txt → ...ibution/server/licenses/LICENSE-SLF4J.txt b/all/licenses/LICENSE-SLF4J.txt → ...ibution/server/licenses/LICENSE-SLF4J.txt
diff --git a/all/licenses/LICENSE-SemVer.txt → ...bution/server/licenses/LICENSE-SemVer.txt b/all/licenses/LICENSE-SemVer.txt → ...bution/server/licenses/LICENSE-SemVer.txt
diff --git a/all/licenses/LICENSE-Webbit.txt → ...bution/server/licenses/LICENSE-Webbit.txt b/all/licenses/LICENSE-Webbit.txt → ...bution/server/licenses/LICENSE-Webbit.txt
diff --git a/all/licenses/LICENSE-bouncycastle.txt → .../server/licenses/LICENSE-bouncycastle.txt b/all/licenses/LICENSE-bouncycastle.txt → .../server/licenses/LICENSE-bouncycastle.txt
diff --git a/all/licenses/LICENSE-google-auth-library.txt → .../licenses/LICENSE-google-auth-library.txt b/all/licenses/LICENSE-google-auth-library.txt → .../licenses/LICENSE-google-auth-library.txt
diff --git a/all/licenses/LICENSE-jbzip2.txt → ...bution/server/licenses/LICENSE-jbzip2.txt b/all/licenses/LICENSE-jbzip2.txt → ...bution/server/licenses/LICENSE-jbzip2.txt
diff --git a/all/licenses/LICENSE-jfastlz.txt → ...ution/server/licenses/LICENSE-jfastlz.txt b/all/licenses/LICENSE-jfastlz.txt → ...ution/server/licenses/LICENSE-jfastlz.txt
diff --git a/all/licenses/LICENSE-libdivsufsort.txt → ...server/licenses/LICENSE-libdivsufsort.txt b/all/licenses/LICENSE-libdivsufsort.txt → ...server/licenses/LICENSE-libdivsufsort.txt
diff --git a/all/licenses/LICENSE-protobuf.txt → ...tion/server/licenses/LICENSE-protobuf.txt b/all/licenses/LICENSE-protobuf.txt → ...tion/server/licenses/LICENSE-protobuf.txt
diff --git a/all/licenses/LICENSE-xz.txt → distribution/server/licenses/LICENSE-xz.txt b/all/licenses/LICENSE-xz.txt → distribution/server/licenses/LICENSE-xz.txt
diff --git a/all/pom.xml → distribution/server/pom.xml b/all/pom.xml → distribution/server/pom.xml
@@ -24,14 +24,14 @@
 
   <parent>
     <groupId>org.apache.pulsar</groupId>
-    <artifactId>pulsar</artifactId>
+    <artifactId>distribution</artifactId>
     <version>2.2.0-incubating-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 
-  <artifactId>distribution</artifactId>
+  <artifactId>pulsar-server-distribution</artifactId>
   <packaging>pom</packaging>
-  <name>distribution</name>
+  <name>Pulsar :: Distribution :: Server</name>
 
   <dependencies>
     <dependency>

diff --git a/all/src/assemble/LICENSE.bin.txt → ...ution/server/src/assemble/LICENSE.bin.txt b/all/src/assemble/LICENSE.bin.txt → ...ution/server/src/assemble/LICENSE.bin.txt
diff --git a/all/src/assemble/NOTICE.bin.txt → ...bution/server/src/assemble/NOTICE.bin.txt b/all/src/assemble/NOTICE.bin.txt → ...bution/server/src/assemble/NOTICE.bin.txt
diff --git a/all/src/assemble/README.bin.txt → ...bution/server/src/assemble/README.bin.txt b/all/src/assemble/README.bin.txt → ...bution/server/src/assemble/README.bin.txt
diff --git a/all/src/assemble/bin.xml → distribution/server/src/assemble/bin.xml b/all/src/assemble/bin.xml → distribution/server/src/assemble/bin.xml
@@ -34,23 +34,23 @@
   </moduleSets>
   <fileSets>
     <fileSet>
-      <directory>${basedir}/../conf</directory>
+      <directory>${basedir}/../../conf</directory>
     </fileSet>
     <fileSet>
-      <directory>${basedir}/../bin</directory>
+      <directory>${basedir}/../../bin</directory>
       <fileMode>755</fileMode>
     </fileSet>
     <fileSet>
       <directory>${basedir}/licenses</directory>
     </fileSet>
     <fileSet>
-      <directory>${basedir}/../pulsar-functions/runtime/target/python-instance</directory>
+      <directory>${basedir}/../../pulsar-functions/runtime/target/python-instance</directory>
       <outputDirectory>instances/python-instance</outputDirectory>
     </fileSet>
   </fileSets>
   <files>
     <file>
-      <source>${basedir}/../DISCLAIMER</source>
+      <source>${basedir}/../../DISCLAIMER</source>
       <outputDirectory>.</outputDirectory>
       <fileMode>644</fileMode>
     </file>
@@ -73,22 +73,22 @@
       <fileMode>644</fileMode>
     </file>
     <file>
-      <source>${basedir}/../pulsar-functions/runtime-all/target/java-instance.jar</source>
+      <source>${basedir}/../../pulsar-functions/runtime-all/target/java-instance.jar</source>
       <destName>java-instance.jar</destName>
       <outputDirectory>instances</outputDirectory>
     </file>
     <file>
-      <source>${basedir}/../pulsar-functions/java-examples/target/pulsar-functions-api-examples.jar</source>
+      <source>${basedir}/../../pulsar-functions/java-examples/target/pulsar-functions-api-examples.jar</source>
       <destName>api-examples.jar</destName>
       <outputDirectory>examples</outputDirectory>
     </file>
     <file>
-      <source>${basedir}/../pulsar-functions/java-examples/src/main/resources/example-function-config.yaml</source>
+      <source>${basedir}/../../pulsar-functions/java-examples/src/main/resources/example-function-config.yaml</source>
       <destName>example-function-config.yaml</destName>
       <outputDirectory>examples</outputDirectory>
     </file>
     <file>
-      <source>${basedir}/../pulsar-functions/java-examples/src/main/resources/example-window-function-config.yaml</source>
+      <source>${basedir}/../../pulsar-functions/java-examples/src/main/resources/example-window-function-config.yaml</source>
       <destName>example-window-function-config.yaml</destName>
       <outputDirectory>examples</outputDirectory>
     </file>

diff --git a/all/src/assemble/src.xml → distribution/server/src/assemble/src.xml b/all/src/assemble/src.xml → distribution/server/src/assemble/src.xml
@@ -29,7 +29,7 @@
   <includeBaseDirectory>true</includeBaseDirectory>
   <fileSets>
     <fileSet>
-      <directory>..</directory>
+      <directory>../..</directory>
       <useDefaultExcludes>true</useDefaultExcludes>
       <includes>
         <include>**/README.md</include>

diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
@@ -34,7 +34,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.pulsar</groupId>
-      <artifactId>distribution</artifactId>
+      <artifactId>pulsar-server-distribution</artifactId>
       <version>${project.parent.version}</version>
       <classifier>bin</classifier>
       <type>tar.gz</type>
@@ -95,7 +95,7 @@
               <pullNewerImage>false</pullNewerImage>
               <tag>${project.version}</tag>
               <buildArgs>
-                <PULSAR_TARBALL>target/distribution-${project.version}-bin.tar.gz</PULSAR_TARBALL>
+                <PULSAR_TARBALL>target/pulsar-server-distribution-${project.version}-bin.tar.gz</PULSAR_TARBALL>
               </buildArgs>
             </configuration>
           </plugin>
@@ -111,7 +111,7 @@
                 <phase>generate-resources</phase>
                 <configuration>
                   <outputDirectory>${project.build.directory}/</outputDirectory>
-                  <includeArtifactIds>distribution</includeArtifactIds>
+                  <includeArtifactIds>pulsar-server-distribution</includeArtifactIds>
                   <excludeTransitive>true</excludeTransitive>
                 </configuration>
               </execution>