Use core-module build profile to avoid downloading snapshot jars (apa…

…che#7640) * Use core-module build profile to avoid downloading snapshot jars * Revert cpp test workflow to use vm build * Mark BlobStoreManagedLedgerOffloaderTest as flaky
instructure · Jul 24, 2020 · 257a2c8 · 257a2c8
1 parent 83e853b
commit 257a2c8
Show file tree

Hide file tree

Showing 11 changed files with 115 additions and 35 deletions.
diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml
@@ -31,19 +31,40 @@ jobs:
   cpp-tests:
     name:
     runs-on: ubuntu-latest
-    container:
-      image: apachepulsar/pulsar-build:latest
     timeout-minutes: 120
 
     steps:
+
       - name: checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
 
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        if: steps.docs.outputs.changed_only == 'no'
+        with:
+          java-version: 1.8
+
+      - name: clean disk
+        run: |
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          sudo apt clean
+          docker rmi $(docker images -q) -f
+          df -h
+
       - name: build package
-        run: mvn -B -ntp -q package -pl distribution/server -am -DskipTests
+        run: mvn -B -ntp -q install -Pcore-modules -DskipTests
 
       - name: build cpp artifacts
         run: |

diff --git a/.github/workflows/ci-unit-broker-broker-gp1.yaml b/.github/workflows/ci-unit-broker-broker-gp1.yaml
@@ -43,13 +43,13 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: build modules
-        run: mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests
-
-      - name: run unit tests for pulsar-broker
-        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/**/*.java' -Dexclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java,**/AdminApiOffloadTest.java'
+        run: mvn -B -ntp -q clean install -Pcore-modules -DskipTests
 
       - name: run flaky test
-        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/AdminApiOffloadTest.java' -DtestForkCount=1 -DtestReuseFork=true
+        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/AdminApiOffloadTest.java' -DtestForkCount=1 -DtestReuseFork=true
+
+      - name: run unit tests for pulsar-broker
+        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/**/*.java' -Dexclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java,**/AdminApiOffloadTest.java'
 
       - name: package surefire artifacts
         if: failure()

diff --git a/.github/workflows/ci-unit-broker-broker-gp2.yaml b/.github/workflows/ci-unit-broker-broker-gp2.yaml
@@ -43,23 +43,22 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: build modules
-        run: mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests
+        run: mvn -B -ntp -q clean install -Pcore-modules -DskipTests
 
       - name: run flaky test
-        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/MessagePublishBufferThrottleTest.java' -DtestForkCount=1 -DtestReuseFork=true
+        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/MessagePublishBufferThrottleTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
       - name: run flaky test
-        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/ReplicatorTest.java' -DtestForkCount=1 -DtestReuseFork=true
+        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/ReplicatorTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
       - name: run flaky test
-        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/TopicOwnerTest.java' -DtestForkCount=1 -DtestReuseFork=true
+        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/TopicOwnerTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
       - name: run flaky test
-        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/AntiAffinityNamespaceGroupTest.java' -DtestForkCount=1 -DtestReuseFork=true
+        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/AntiAffinityNamespaceGroupTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
       - name: run unit tests for pulsar-broker "org/apache/pulsar/broker/"
-        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java' -Dexclude='**/ReplicatorTest.java,**/MessagePublishBufferThrottleTest.java,**/TopicOwnerTest.java,**/AntiAffinityNamespaceGroupTest.java'
-
+        run: ./build/retry.sh mvn ${MAVEN_CLI_OPTS} test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java' -Dexclude='**/ReplicatorTest.java,**/MessagePublishBufferThrottleTest.java,**/TopicOwnerTest.java,**/AntiAffinityNamespaceGroupTest.java'
 
       - name: package surefire artifacts
         if: failure()

diff --git a/.github/workflows/ci-unit-broker-client-api.yaml b/.github/workflows/ci-unit-broker-client-api.yaml
@@ -43,12 +43,12 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: build modules
-        run:  mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests
+        run: mvn -B -ntp -q install -Pcore-modules -DskipTests
 
       - name: run unit tests for pulsar-broker "org/apache/pulsar/client/impl"
         run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/client/api/**/*.java' -Dexclude='**/DispatcherBlockConsumerTest.java'
 
-      - name: run flay test
+      - name: run flaky test
         run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/DispatcherBlockConsumerTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
       - name: package surefire artifacts

diff --git a/.github/workflows/ci-unit-broker-client-impl.yaml b/.github/workflows/ci-unit-broker-client-impl.yaml
@@ -43,7 +43,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: build modules
-        run:  mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests
+        run: mvn -B -ntp -q install -Pcore-modules -DskipTests
 
       - name: run unit tests for pulsar-broker "org/apache/pulsar/client/impl"
         run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/client/impl/**/*.java'

diff --git a/.github/workflows/ci-unit-broker-other.yaml b/.github/workflows/ci-unit-broker-other.yaml
@@ -43,7 +43,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: build modules
-        run: mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests
+        run: mvn -B -ntp -q clean install -Pcore-modules -DskipTests
 
       - name: run unit tests for pulsar-broker exlcude "org/apache/pulsar/broker", "org/apache/pulsar/client"
         run: mvn -B -ntp test -pl pulsar-broker -Dexclude='org/apache/pulsar/broker/**/*.java,org/apache/pulsar/client/**/*.java'

diff --git a/.github/workflows/ci-unit-proxy.yaml b/.github/workflows/ci-unit-proxy.yaml
@@ -43,22 +43,22 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: build modules pulsar-proxy
-        run:  mvn -B -q -ntp install -pl pulsar-proxy -am -DskipTests
+        run: mvn -B -ntp -q install -Pcore-modules -DskipTests
 
       - name: run flaky test
-        run:  mvn -B -ntp install -pl pulsar-proxy -Dexclude="**/ProxyRolesEnforcementTest.java,**/ProxyAuthenticationTest.java,**/ProxyTest.java,**/MessagePublishBufferThrottleTest.java"
+        run: mvn -B -ntp install -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dexclude="**/ProxyRolesEnforcementTest.java,**/ProxyAuthenticationTest.java,**/ProxyTest.java,**/MessagePublishBufferThrottleTest.java"
 
       - name: run flaky test
-        run:  mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyRolesEnforcementTest.java" -DtestForkCount=1 -DtestReuseFork=true
+        run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyRolesEnforcementTest.java" -DtestForkCount=1 -DtestReuseFork=true
 
       - name: run flaky test
-        run:  mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyAuthenticationTest.java" -DtestForkCount=1 -DtestReuseFork=true
+        run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyAuthenticationTest.java" -DtestForkCount=1 -DtestReuseFork=true
 
       - name: run flaky test
-        run:  mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyTest.java" -DtestForkCount=1 -DtestReuseFork=true
+        run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/ProxyTest.java" -DtestForkCount=1 -DtestReuseFork=true
 
       - name: run flaky test
-        run:  mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/MessagePublishBufferThrottleTest.java" -DtestForkCount=1 -DtestReuseFork=true
+        run: mvn -B -ntp test -pl pulsar-proxy -DtestForkCount=1 -DtestReuseFork=true -Dinclude="**/MessagePublishBufferThrottleTest.java" -DtestForkCount=1 -DtestReuseFork=true
 
       - name: package surefire artifacts
         if: failure()

diff --git a/.github/workflows/ci-unit.yaml b/.github/workflows/ci-unit.yaml
@@ -42,20 +42,23 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: run flaky test
-        run:  mvn -B -ntp install -pl managed-ledger -Dinclude="**/ManagedLedgerTest.java" -DtestForkCount=1 -DtestReuseFork=true
+      - name: build and run unit tests exclude pulsar-broker and pulsar-proxy
+        run: ./build/retry.sh mvn -B -ntp install -PbrokerSkipTest -Dexclude="org/apache/pulsar/proxy/**/*.java,**/KafkaProducerSimpleConsumerTest.java,**/ManagedLedgerTest.java,**/TestPulsarKeyValueSchemaHandler.java,**/PrimitiveSchemaTest.java,**/BlobStoreManagedLedgerOffloaderTest.java"
 
-      - name: run flaky test
-        run:  mvn -B -ntp install -pl pulsar-client-kafka-compat/pulsar-client-kafka_0_8 -Dinclude="**/KafkaProducerSimpleConsumerTest.java" -DtestForkCount=1 -DtestReuseFork=true
+      - name: run flaky test "**/ManagedLedgerTest.java"
+        run: ./build/retry.sh mvn -B -ntp test -pl managed-ledger -Dinclude="**/ManagedLedgerTest.java" -DtestForkCount=1 -DtestReuseFork=true
 
-      - name: run flaky test
-        run:  mvn -B -ntp install -pl pulsar-sql/presto-pulsar-plugin -Dinclude="**/TestPulsarKeyValueSchemaHandler.java" -DtestForkCount=1 -DtestReuseFork=true
+      - name: run flaky test "**/KafkaProducerSimpleConsumerTest.java"
+        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-client-kafka-compat/pulsar-client-kafka_0_8 -Dinclude="**/KafkaProducerSimpleConsumerTest.java" -DtestForkCount=1 -DtestReuseFork=true
 
-      - name: run flaky test
-        run:  mvn -B -ntp install -pl pulsar-client -Dinclude="**/PrimitiveSchemaTest.java" -DtestForkCount=1 -DtestReuseFork=true
+      - name: run flaky test "**/TestPulsarKeyValueSchemaHandler.java"
+        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-sql/presto-pulsar-plugin -Dinclude="**/TestPulsarKeyValueSchemaHandler.java" -DtestForkCount=1 -DtestReuseFork=true
 
-      - name: run unit tests exclude pulsar-broker and pulsar-proxy
-        run:  mvn -B -ntp install -PbrokerSkipTest -Dexclude="org/apache/pulsar/proxy/**/*.java,**/KafkaProducerSimpleConsumerTest.java,**/ManagedLedgerTest.java,**/TestPulsarKeyValueSchemaHandler.java,**/PrimitiveSchemaTest.java"
+      - name: run flaky test "**/PrimitiveSchemaTest.java"
+        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-client -Dinclude="**/PrimitiveSchemaTest.java" -DtestForkCount=1 -DtestReuseFork=true
+
+      - name: run flaky test "**/BlobStoreManagedLedgerOffloaderTest.java"
+        run: ./build/retry.sh mvn -B -ntp test -pl tiered-storage/jcloud -Dinclude="**/BlobStoreManagedLedgerOffloaderTest.java" -DtestForkCount=1 -DtestReuseFork=true
 
       - name: package surefire artifacts
         if: failure()

diff --git a/build/retry.sh b/build/retry.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+function fail {
+  echo $1 >&2
+  exit 1
+}
+
+function retry {
+  local n=1
+  local max=3
+  local delay=10
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempt $n/$max:"
+        sleep $delay;
+      else
+        fail "The command has failed after $n attempts."
+      fi
+    }
+  done
+}
+
+retry $@
diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
@@ -333,6 +333,13 @@
 
     <!-- transaction related dependencies (end) -->
 
+     <dependency>
+       <groupId>org.bouncycastle</groupId>
+       <artifactId>bc-fips</artifactId>
+       <version>${bouncycastlefips.version}</version>
+       <scope>test</scope>
+     </dependency>
+
   </dependencies>
 
   <build>

diff --git a/pulsar-discovery-service/pom.xml b/pulsar-discovery-service/pom.xml
@@ -133,6 +133,13 @@
       <artifactId>javax.activation</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bc-fips</artifactId>
+      <version>${bouncycastlefips.version}</version>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>testmocks</artifactId>