[Build] Improve maven module build order (apache#9882)

- add dependencies in distribution builds so that the modules get built in the correct order - allow building the distribution without pulsar-presto-distribution when core-modules is active - modify src/check-binary-license script to allow skipping Pulsar SQL checks - this is preparation for a build without Presto / Pulsar SQL - consistently exclude transitive dependencies from the provided dependencies that have been added to impact build order. - this has mainly an impact with maven-assembly-plugin which will include the transitive dependencies of a tar.gz dependency unless the dependencies are excluded explicitly. - add "skipDocker" and "skipIntegrationTests" profiles which enable processing all other modules except docker or integration test modules - this is preparation for CI build to ensure that we don't miss running some unit tests in some module. - fix running the backwards compatibility tests that are part of tests/bc_* modules - it's unclear how these are run currently. Perhaps they run in all integration test runs. This removes the possible duplication. - Improve the docker profile so that it's possible to build tests/docker-images/* modules without activating integration tests
mahak · Mar 12, 2021 · 1881459 · 1881459
1 parent 8370817
commit 1881459
Show file tree

Hide file tree

Showing 19 changed files with 283 additions and 69 deletions.
diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh
@@ -57,6 +57,7 @@ test_group_shade() {
 
 test_group_backwards_compat() {
   mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-backwards-compatibility.xml -DintegrationTests
+  mvn_run_integration_test "$@" -DBackwardsCompatTests
 }
 
 test_group_cli() {

diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml
@@ -39,6 +39,32 @@
       <artifactId>managed-ledger</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>tiered-storage-file-system</artifactId>
+      <version>${project.version}</version>
+      <type>pom</type>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>tiered-storage-jcloud</artifactId>
+      <version>${project.version}</version>
+      <type>pom</type>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
 
   <build>

diff --git a/distribution/pom.xml b/distribution/pom.xml
@@ -41,9 +41,9 @@
         <activeByDefault>true</activeByDefault>
       </activation>
       <modules>
-        <module>server</module>
         <module>io</module>
         <module>offloaders</module>
+        <module>server</module>
       </modules>
     </profile>
 

diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
@@ -113,22 +113,22 @@
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-web</artifactId>
     </dependency>
-    
+
     <dependency>
        <groupId>io.dropwizard.metrics</groupId>
        <artifactId>metrics-core</artifactId>
     </dependency>
-    
+
     <dependency>
        <groupId>io.dropwizard.metrics</groupId>
        <artifactId>metrics-graphite</artifactId>
     </dependency>
-    
+
     <dependency>
        <groupId>io.dropwizard.metrics</groupId>
        <artifactId>metrics-jvm</artifactId>
     </dependency>
-    
+
     <dependency>
        <groupId>org.xerial.snappy</groupId>
        <artifactId>snappy-java</artifactId>
@@ -167,6 +167,12 @@
       <version>${project.version}</version>
       <!-- make sure the api examples are compiled before assembly -->
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -193,6 +199,12 @@
       <version>${project.version}</version>
       <!-- make sure the api examples are compiled before assembly -->
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <!-- local-runner -->
@@ -337,4 +349,32 @@
       </plugins>
     </pluginManagement>
   </build>
+  <profiles>
+    <profile>
+      <id>core-modules</id>
+      <!-- allow building the distribution without pulsar-presto-distribution when core-modules is active -->
+    </profile>
+    <profile>
+      <id>main</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <!-- depend on pulsar-presto-distribution by default, unless core-modules profile is active -->
+      <dependencies>
+        <dependency>
+          <groupId>${project.groupId}</groupId>
+          <artifactId>pulsar-presto-distribution</artifactId>
+          <version>${project.version}</version>
+          <type>tar.gz</type>
+          <scope>provided</scope>
+          <exclusions>
+            <exclusion>
+              <groupId>*</groupId>
+              <artifactId>*</artifactId>
+            </exclusion>
+          </exclusions>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
 </project>
diff --git a/distribution/server/src/assemble/bin.xml b/distribution/server/src/assemble/bin.xml
@@ -127,6 +127,11 @@
         <exclude>junit:junit</exclude>
 
         <exclude>org.projectlombok:lombok</exclude>
+
+        <!-- prevent adding pulsar-functions-api-examples in lib -->
+        <exclude>org.apache.pulsar:pulsar-functions-api-examples</exclude>
+        <!-- prevent adding any distribution .tar.gz files in lib -->
+        <exclude>*:tar.gz</exclude>
       </excludes>
     </dependencySet>
   </dependencySets>

diff --git a/docker/pom.xml b/docker/pom.xml
@@ -30,12 +30,6 @@
   </parent>
   <artifactId>docker-images</artifactId>
   <name>Apache Pulsar :: Docker Images</name>
-  <modules>
-    <module>pulsar</module>
-    <module>grafana</module>
-    <module>pulsar-all</module>
-    <module>pulsar-standalone</module>
-  </modules>
   <build>
     <plugins>
       <plugin>
@@ -47,4 +41,27 @@
       </plugin>
     </plugins>
   </build>
+  <profiles>
+    <profile>
+      <id>skipDocker</id>
+      <activation>
+        <property>
+          <name>skipDocker</name>
+        </property>
+      </activation>
+      <modules/>
+    </profile>
+    <profile>
+      <id>docker</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <modules>
+        <module>pulsar</module>
+        <module>grafana</module>
+        <module>pulsar-all</module>
+        <module>pulsar-standalone</module>
+      </modules>
+    </profile>
+  </profiles>
 </project>
diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
@@ -37,6 +37,12 @@
       <version>${project.parent.version}</version>
       <type>pom</type>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>io.perfmark</groupId>
@@ -50,6 +56,12 @@
       <classifier>bin</classifier>
       <type>tar.gz</type>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
   </dependencies>
 

diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
@@ -38,6 +38,12 @@
       <classifier>bin</classifier>
       <type>tar.gz</type>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
   </dependencies>
 

diff --git a/pom.xml b/pom.xml
@@ -1689,6 +1689,10 @@ flexible messaging model and an intuitive client API.</description>
     </profile>
     <profile>
       <id>docker</id>
+      <modules>
+        <module>docker</module>
+        <module>tests</module>
+      </modules>
     </profile>
 
     <profile>
@@ -1804,16 +1808,17 @@ flexible messaging model and an intuitive client API.</description>
 
         <module>pulsar-client-messagecrypto-bc</module>
 
-        <!-- all these 3 modules should be put at the end in this exact sequence -->
-        <module>distribution</module>
-        <module>docker</module>
-        <module>tests</module>
         <module>pulsar-metadata</module>
         <module>jclouds-shaded</module>
 
         <!-- package management releated modules (begin) -->
         <module>pulsar-package-management</module>
         <!-- package management releated modules (end) -->
+
+        <!-- all these 3 modules should be put at the end in this exact sequence -->
+        <module>distribution</module>
+        <module>docker</module>
+        <module>tests</module>
       </modules>
     </profile>
 

diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
@@ -60,7 +60,7 @@
         <executions>
           <execution>
             <id>unpack</id>
-            <phase>generate-sources</phase>
+            <phase>prepare-package</phase>
             <goals>
               <goal>unpack</goal>
             </goals>

diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml
@@ -55,7 +55,7 @@
         <executions>
           <execution>
             <id>unpack</id>
-            <phase>generate-sources</phase>
+            <phase>prepare-package</phase>
             <goals>
               <goal>unpack</goal>
             </goals>

diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml
@@ -149,7 +149,7 @@
     </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
-      <artifactId>pulsar-io-kafka-connect-adaptor</artifactId>
+      <artifactId>pulsar-io-kafka-connect-adaptor-nar</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>

diff --git a/src/check-binary-license b/src/check-binary-license
@@ -27,10 +27,18 @@
 # all error fatal
 set -e
 
+# skip checks for Presto licenses if 1. argument is "--no-presto"/"no-pulsar-sql"
+# this is to allow building the server distribution without Pulsar SQL
+NO_PRESTO=0
+if [[ "$1" == "--no-presto" || "$1" == "--no-pulsar-sql" ]]; then
+  NO_PRESTO=1
+  shift
+fi
+
 TARBALL="$1"
 if [ -z $TARBALL ]; then
     echo "Usage: $0 <binary-tarball>"
-    exit -1
+    exit 1
 fi
 
 JARS=$(tar -tf $TARBALL | grep '\.jar' | grep -v 'lib/presto/' | grep -v '/examples/' | grep -v '/instances/' | sed 's!.*/!!' | sort)
@@ -86,34 +94,36 @@ for J in $NOTICEJARS; do
     fi
 done
 
-# check pulsar sql jars
-JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'lib/presto/' | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v  'pulsar-client-admin' | grep -v  'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-transaction-common' | grep -v 'pulsar-package-core' | sed 's!.*/!!' | sort)
-LICENSEPATH=$(tar -tf $TARBALL  | awk '/^[^\/]*\/lib\/presto\/LICENSE/')
-LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH")
-LICENSEJARS=$(echo "$LICENSE" | sed -nE 's!.* (.*\.jar).*!\1!gp')
-
-
-for J in $JARS; do
-    echo $J | grep -q "org.apache.pulsar"
-    if [ $? == 0 ]; then
-        continue
-    fi
-
-    echo "$LICENSE" | grep -q $J
-    if [ $? != 0 ]; then
-        echo $J unaccounted for in lib/presto/LICENSE
-        EXIT=1
-    fi
-done
-
-# Check all jars mentioned in LICENSE are bundled
-for J in $LICENSEJARS; do
-    echo "$JARS" | grep -q $J
-    if [ $? != 0 ]; then
-        echo $J mentioned in lib/presto/LICENSE, but not bundled
-        EXIT=2
-    fi
-done
+if [ "$NO_PRESTO" -ne 1 ]; then
+  # check pulsar sql jars
+  JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'lib/presto/' | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v  'pulsar-client-admin' | grep -v  'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-transaction-common' | grep -v 'pulsar-package-core' | sed 's!.*/!!' | sort)
+  LICENSEPATH=$(tar -tf $TARBALL  | awk '/^[^\/]*\/lib\/presto\/LICENSE/')
+  LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH")
+  LICENSEJARS=$(echo "$LICENSE" | sed -nE 's!.* (.*\.jar).*!\1!gp')
+
+
+  for J in $JARS; do
+      echo $J | grep -q "org.apache.pulsar"
+      if [ $? == 0 ]; then
+          continue
+      fi
+
+      echo "$LICENSE" | grep -q $J
+      if [ $? != 0 ]; then
+          echo $J unaccounted for in lib/presto/LICENSE
+          EXIT=1
+      fi
+  done
+
+  # Check all jars mentioned in LICENSE are bundled
+  for J in $LICENSEJARS; do
+      echo "$JARS" | grep -q $J
+      if [ $? != 0 ]; then
+          echo $J mentioned in lib/presto/LICENSE, but not bundled
+          EXIT=2
+      fi
+  done
+fi
 
 if [ $EXIT != 0 ]; then
     echo

diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml
@@ -79,10 +79,10 @@
 
     <profiles>
         <profile>
-            <id>integrationTests</id>
+            <id>BackwardsCompatTests</id>
             <activation>
                 <property>
-                    <name>integrationTests</name>
+                    <name>BackwardsCompatTests</name>
                 </property>
             </activation>
             <build>