Limit cpu count for proxy unit test cases (apache#7845)

* Limit cpu count for proxy unit test cases * Reduce logging in github workflows Co-authored-by: Ali Ahmed <[email protected]>
vzhikserg · Aug 19, 2020 · 809c6be · 809c6be
1 parent 23d795c
commit 809c6be
Show file tree

Hide file tree

Showing 16 changed files with 30 additions and 19 deletions.
diff --git a/.github/workflows/ci-integration-backwards-compatibility.yaml b/.github/workflows/ci-integration-backwards-compatibility.yaml
@@ -71,7 +71,7 @@ jobs:
 
       - name: run install by skip tests
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+        run: mvn -q -B -ntp clean install -DskipTests
 
       - name: build artifacts and docker image
         if: steps.docs.outputs.changed_only == 'no'

diff --git a/.github/workflows/ci-integration-cli.yaml b/.github/workflows/ci-integration-cli.yaml
@@ -71,7 +71,7 @@ jobs:
 
       - name: run install by skip tests
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+        run: mvn -q -B -ntp clean install -DskipTests
 
       - name: build artifacts and docker image
         if: steps.docs.outputs.changed_only == 'no'

diff --git a/.github/workflows/ci-integration-function-state.yaml b/.github/workflows/ci-integration-function-state.yaml
@@ -71,7 +71,7 @@ jobs:
 
       - name: run install by skip tests
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+        run: mvn -q -B -ntp clean install -DskipTests
 
       - name: build artifacts and docker image
         if: steps.docs.outputs.changed_only == 'no'

diff --git a/.github/workflows/ci-integration-messaging.yaml b/.github/workflows/ci-integration-messaging.yaml
@@ -71,7 +71,7 @@ jobs:
 
       - name: run install by skip tests
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+        run: mvn -q -B -ntp clean install -DskipTests
 
       - name: build artifacts and docker image
         if: steps.docs.outputs.changed_only == 'no'

diff --git a/.github/workflows/ci-integration-process.yaml b/.github/workflows/ci-integration-process.yaml
@@ -71,7 +71,7 @@ jobs:
 
       - name: run install by skip tests
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+        run: mvn -q -B -ntp clean install -DskipTests
 
       - name: build pulsar image
         if: steps.docs.outputs.changed_only == 'no'

diff --git a/.github/workflows/ci-integration-schema.yaml b/.github/workflows/ci-integration-schema.yaml
@@ -71,7 +71,7 @@ jobs:
 
       - name: run install by skip tests
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+        run: mvn -q -B -ntp clean install -DskipTests
 
       - name: build artifacts and docker image
         if: steps.docs.outputs.changed_only == 'no'

diff --git a/.github/workflows/ci-integration-sql.yaml b/.github/workflows/ci-integration-sql.yaml
@@ -71,7 +71,7 @@ jobs:
 
       - name: run install by skip tests
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+        run: mvn -q -B -ntp clean install -DskipTests
 
 #      Flaky Test: https://github.com/apache/pulsar/issues/7750
 #      - name: build pulsar image

diff --git a/.github/workflows/ci-integration-standalone.yaml b/.github/workflows/ci-integration-standalone.yaml
@@ -71,7 +71,7 @@ jobs:
 
       - name: run install by skip tests
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+        run: mvn -q -B -ntp clean install -DskipTests
 
       - name: build artifacts and docker image
         if: steps.docs.outputs.changed_only == 'no'

diff --git a/.github/workflows/ci-integration-thread.yaml b/.github/workflows/ci-integration-thread.yaml
@@ -71,7 +71,7 @@ jobs:
 
       - name: run install by skip tests
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+        run: mvn -q -B -ntp clean install -DskipTests
 
       - name: build pulsar image
         if: steps.docs.outputs.changed_only == 'no'

diff --git a/.github/workflows/ci-integration-tiered-filesystem.yaml b/.github/workflows/ci-integration-tiered-filesystem.yaml
@@ -71,7 +71,7 @@ jobs:
 
       - name: run install by skip tests
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+        run: mvn -q -B -ntp clean install -DskipTests
 
       - name: build pulsar image
         if: steps.docs.outputs.changed_only == 'no'

diff --git a/.github/workflows/ci-integration-tiered-jcloud.yaml b/.github/workflows/ci-integration-tiered-jcloud.yaml
@@ -71,7 +71,7 @@ jobs:
 
       - name: run install by skip tests
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+        run: mvn -q -B -ntp clean install -DskipTests
 
       - name: build pulsar image
         if: steps.docs.outputs.changed_only == 'no'

diff --git a/.github/workflows/ci-license.yaml b/.github/workflows/ci-license.yaml
@@ -70,7 +70,7 @@ jobs:
 
       - name: build and check license
         if: steps.docs.outputs.changed_only == 'no'
-        run: mvn -B -ntp -DskipTests license:check install
+        run: mvn -q -B -ntp -DskipTests license:check install
 
       - name: license check
         if: steps.docs.outputs.changed_only == 'no'

diff --git a/.github/workflows/ci-unit-broker-broker-gp1.yaml b/.github/workflows/ci-unit-broker-broker-gp1.yaml
@@ -65,9 +65,13 @@ jobs:
         if: steps.docs.outputs.changed_only == 'no'
         run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/AdminApiOffloadTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
+      - name: run flaky test '**/TransactionProduceTest.java'
+        if: steps.docs.outputs.changed_only == 'no'
+        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='**/TransactionProduceTest.java' -DtestForkCount=1 -DtestReuseFork=true
+
       - name: run unit tests for pulsar-broker
         if: steps.docs.outputs.changed_only == 'no'
-        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/**/*.java' -Dexclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java,**/AdminApiOffloadTest.java'
+        run: ./build/retry.sh mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/**/*.java' -Dexclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java,**/AdminApiOffloadTest.java,**/TransactionProduceTest.java'
 
       - name: package surefire artifacts
         if: failure()

diff --git a/.github/workflows/ci-unit-broker-client-api.yaml b/.github/workflows/ci-unit-broker-client-api.yaml
@@ -61,14 +61,18 @@ jobs:
         if: steps.docs.outputs.changed_only == 'no'
         run: mvn -B -ntp -q install -Pcore-modules -DskipTests
 
-      - name: run unit tests for pulsar-broker "org/apache/pulsar/client/impl"
-        if: steps.docs.outputs.changed_only == 'no'
-        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/client/api/**/*.java' -Dexclude='**/DispatcherBlockConsumerTest.java'
-
       - name: run flaky test '**/DispatcherBlockConsumerTest.java'
         if: steps.docs.outputs.changed_only == 'no'
         run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/DispatcherBlockConsumerTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
+      - name: run flaky test '**/SimpleProducerConsumerTest.java'
+        if: steps.docs.outputs.changed_only == 'no'
+        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/SimpleProducerConsumerTest.java' -DtestForkCount=1 -DtestReuseFork=true
+
+      - name: run unit tests for pulsar-broker "org/apache/pulsar/client/impl"
+        if: steps.docs.outputs.changed_only == 'no'
+        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/client/api/**/*.java' -Dexclude='**/DispatcherBlockConsumerTest.java,**/SimpleProducerConsumerTest.java'
+
       - name: package surefire artifacts
         if: failure()
         run: |

diff --git a/pulsar-client-cpp/python/setup.py b/pulsar-client-cpp/python/setup.py
@@ -38,7 +38,7 @@ def get_version():
     root = ET.XML(open(POM_PATH).read())
     version = root.find('{http://maven.apache.org/POM/4.0.0}version').text.strip()
 
-    if use_full_pom_name:    
+    if use_full_pom_name:
         return version
     else:
         # Strip the '-incubating' suffix, since it prevents the packages
@@ -78,7 +78,7 @@ def build_extension(self, ext):
 
 
 dependencies = [
-    'fastavro',
+    'fastavro==0.24.0',
     'grpcio',
     'protobuf>=3.6.1',
     'six',

diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyIsAHttpProxyTest.java
@@ -47,6 +47,7 @@
 import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.ProcessorUtils;
 import org.glassfish.jersey.client.ClientConfig;
 import org.glassfish.jersey.logging.LoggingFeature;
 
@@ -69,6 +70,8 @@ public class ProxyIsAHttpProxyTest extends MockedPulsarServiceBaseTest {
     @BeforeClass
     protected void setup() throws Exception {
         internalSetup();
+        // Set number of CPU's to two for unit tests for running in resource constrained env.
+        ProcessorUtils.setAvailableProcessors(2);
 
         backingServer1 = new Server(0);
         backingServer1.setHandler(newHandler("server1"));