Enable test retries for integration tests, shade tests and backwards …

…compat tests (apache#10191) ### Motivation Some integration tests fail often and retrying all integration tests is slow. ### Modifications Enable the test retries at TestNG level also for integration tests. When a Pulsar broker url is passed as a parameter to a test method, it's necessary to use a `Supplier<String>` parameter so that the value gets evaluated again for test retries.
sarvex · Apr 13, 2021 · d1305c1 · d1305c1
1 parent c3f8b77
commit d1305c1
Show file tree

Hide file tree

Showing 70 changed files with 453 additions and 511 deletions.
diff --git a/build/retry_integration.sh b/build/retry_integration.sh
diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh
@@ -37,7 +37,7 @@ mvn_run_integration_test() {
   RETRY=""
   # wrap with retry.sh script if next parameter is "--retry"
   if [[ "$1" == "--retry" ]]; then
-    RETRY="./build/retry_integration.sh"
+    RETRY="./build/retry.sh"
     shift
   fi
   # skip wrapping with retry.sh script if next parameter is "--no-retry"
@@ -53,12 +53,12 @@ mvn_run_integration_test() {
 }
 
 test_group_shade() {
-  mvn_run_integration_test "$@" -DShadeTests
+  mvn_run_integration_test "$@" -DShadeTests -DtestForkCount=1 -DtestReuseFork=false
 }
 
 test_group_backwards_compat() {
   mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-backwards-compatibility.xml -DintegrationTests
-  mvn_run_integration_test "$@" -DBackwardsCompatTests
+  mvn_run_integration_test --retry "$@" -DBackwardsCompatTests -DtestForkCount=1 -DtestReuseFork=false
 }
 
 test_group_cli() {
@@ -77,9 +77,9 @@ test_group_messaging() {
   # run integration messaging tests
   mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-messaging.xml -DintegrationTests
   # run integration proxy tests
-  mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-proxy.xml -DintegrationTests
+  mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-proxy.xml -DintegrationTests
   # run integration proxy with WebSocket tests
-  mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-proxy-websocket.xml -DintegrationTests
+  mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-proxy-websocket.xml -DintegrationTests
 }
 
 test_group_schema() {
@@ -125,7 +125,7 @@ test_group_pulsar_connectors_process() {
 }
 
 test_group_sql() {
-  mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-sql.xml -DintegrationTests -DtestForkCount=1
+  mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-sql.xml -DintegrationTests -DtestForkCount=1 -DtestReuseFork=false
 }
 
 echo "Test Group : $TEST_GROUP"

diff --git a/pom.xml b/pom.xml
@@ -1178,15 +1178,11 @@ flexible messaging model and an intuitive client API.</description>
           <shutdown>kill</shutdown>
           <redirectTestOutputToFile>${redirectTestOutputToFile}</redirectTestOutputToFile>
           <trimStackTrace>false</trimStackTrace>
+          <systemPropertyVariables>
+            <testRealAWS>${testRealAWS}</testRealAWS>
+            <testRetryCount>${testRetryCount}</testRetryCount>
+          </systemPropertyVariables>
           <properties>
-            <property>
-              <name>testRealAWS</name>
-              <value>${testRealAWS}</value>
-            </property>
-            <property>
-              <name>testRetryCount</name>
-              <value>${testRetryCount}</value>
-            </property>
             <property>
               <name>listener</name>
               <value>org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener</value>

diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml
@@ -91,16 +91,6 @@
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-surefire-plugin</artifactId>
                         <configuration>
-                            <properties>
-                                <property>
-                                    <name>testRetryCount</name>
-                                    <value>0</value>
-                                </property>
-                                <property>
-                                    <name>listener</name>
-                                    <value>org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener</value>
-                                </property>
-                            </properties>
                             <argLine>-Xmx2G -XX:MaxDirectMemorySize=8G
                                 -Dio.netty.leakDetectionLevel=advanced
                             </argLine>

diff --git a/tests/bc_2_0_0/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java b/tests/bc_2_0_0/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java
@@ -26,19 +26,21 @@
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.tests.TestRetrySupport;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import java.util.concurrent.TimeUnit;
 
-public class SmokeTest {
+public class SmokeTest extends TestRetrySupport {
 
     private PulsarContainer pulsarContainer;
 
-    @BeforeClass
-    public void setup(){
+    @Override
+    @BeforeClass(alwaysRun = true)
+    public final void setup(){
         pulsarContainer = new PulsarContainer();
         pulsarContainer.start();
     }
@@ -72,8 +74,9 @@ public void checkMessages() throws PulsarClientException {
 
     }
 
+    @Override
     @AfterClass(alwaysRun = true)
-    public void cleanup(){
+    public final void cleanup(){
         pulsarContainer.stop();
     }
 

diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml
@@ -91,16 +91,6 @@
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-surefire-plugin</artifactId>
                         <configuration>
-                            <properties>
-                                <property>
-                                    <name>testRetryCount</name>
-                                    <value>0</value>
-                                </property>
-                                <property>
-                                    <name>listener</name>
-                                    <value>org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener</value>
-                                </property>
-                            </properties>
                             <argLine>-Xmx2G -XX:MaxDirectMemorySize=8G
                                 -Dio.netty.leakDetectionLevel=advanced
                             </argLine>

diff --git a/tests/bc_2_0_1/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java b/tests/bc_2_0_1/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java
@@ -26,19 +26,21 @@
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.tests.TestRetrySupport;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import java.util.concurrent.TimeUnit;
 
-public class SmokeTest {
+public class SmokeTest extends TestRetrySupport {
 
     private PulsarContainer pulsarContainer;
 
-    @BeforeClass
-    public void setup(){
+    @Override
+    @BeforeClass(alwaysRun = true)
+    public final void setup(){
         pulsarContainer = new PulsarContainer();
         pulsarContainer.start();
     }
@@ -72,8 +74,9 @@ public void checkMessages() throws PulsarClientException {
 
     }
 
+    @Override
     @AfterClass(alwaysRun = true)
-    public void cleanup(){
+    public final void cleanup(){
         pulsarContainer.stop();
     }
 

diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml
@@ -98,16 +98,6 @@
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-surefire-plugin</artifactId>
                         <configuration>
-                            <properties>
-                                <property>
-                                    <name>testRetryCount</name>
-                                    <value>0</value>
-                                </property>
-                                <property>
-                                    <name>listener</name>
-                                    <value>org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener</value>
-                                </property>
-                            </properties>
                             <argLine>-Xmx2G -XX:MaxDirectMemorySize=8G
                                 -Dio.netty.leakDetectionLevel=advanced
                             </argLine>

diff --git a/tests/bc_2_6_0/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java b/tests/bc_2_6_0/src/test/java/org/apache/pulsar/tests/integration/SmokeTest.java
@@ -26,19 +26,21 @@
 import org.apache.pulsar.client.api.PulsarClientException;
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.SubscriptionType;
+import org.apache.pulsar.tests.TestRetrySupport;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import java.util.concurrent.TimeUnit;
 
-public class SmokeTest {
+public class SmokeTest extends TestRetrySupport {
 
     private PulsarContainer pulsarContainer;
 
-    @BeforeClass
-    public void setup(){
+    @Override
+    @BeforeClass(alwaysRun = true)
+    public final void setup(){
         pulsarContainer = new PulsarContainer();
         pulsarContainer.start();
     }
@@ -72,8 +74,9 @@ public void checkMessages() throws PulsarClientException {
 
     }
 
+    @Override
     @AfterClass(alwaysRun = true)
-    public void cleanup(){
+    public final void cleanup(){
         pulsarContainer.stop();
     }
 

diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml
@@ -225,16 +225,6 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
-              <properties>
-                <property>
-                  <name>testRetryCount</name>
-                  <value>0</value>
-                </property>
-                <property>
-                  <name>listener</name>
-                  <value>org.apache.pulsar.tests.PulsarTestListener,org.apache.pulsar.tests.AnnotationListener</value>
-                </property>
-              </properties>
               <argLine>-Xmx2G -XX:MaxDirectMemorySize=8G
               -Dio.netty.leakDetectionLevel=advanced
               </argLine>

diff --git a/...src/test/java/org/apache/pulsar/tests/integration/auth/admin/PackagesOpsWithAuthTest.java b/...src/test/java/org/apache/pulsar/tests/integration/auth/admin/PackagesOpsWithAuthTest.java
@@ -27,6 +27,7 @@
 import org.apache.pulsar.client.api.AuthenticationFactory;
 import org.apache.pulsar.client.impl.auth.AuthenticationToken;
 import org.apache.pulsar.common.policies.data.AuthAction;
+import org.apache.pulsar.tests.TestRetrySupport;
 import org.apache.pulsar.tests.integration.containers.PulsarContainer;
 import org.apache.pulsar.tests.integration.containers.ZKContainer;
 import org.apache.pulsar.tests.integration.topologies.PulsarCluster;
@@ -53,7 +54,7 @@
  * PackagesOpsWithAuthTest will test all package operations with and without the proper permission.
  */
 @Slf4j
-public class PackagesOpsWithAuthTest {
+public class PackagesOpsWithAuthTest extends TestRetrySupport {
 
     private static final String CLUSTER_PREFIX = "package-auth";
     private static final String PRIVATE_KEY_PATH_INSIDE_CONTAINER = "/tmp/private.key";
@@ -70,8 +71,10 @@ public class PackagesOpsWithAuthTest {
     private PulsarCluster pulsarCluster;
     private PulsarContainer cmdContainer;
 
-    @BeforeClass
+    @Override
+    @BeforeClass(alwaysRun = true)
     public void setup() throws Exception {
+        incrementSetupNumber();
         // Before starting the cluster, generate the secret key and the token
         // Use Zk container to have 1 container available before starting the cluster
         final String clusterName = String.format("%s-%s", CLUSTER_PREFIX, RandomStringUtils.randomAlphabetic(6));
@@ -100,8 +103,10 @@ public void setup() throws Exception {
         pulsarCluster.start();
     }
 
+    @Override
     @AfterClass(alwaysRun = true)
-    public void teardown() {
+    public void cleanup() {
+        markCurrentSetupNumberCleaned();
         if (cmdContainer != null) {
             cmdContainer.stop();
         }

diff --git a/...va/org/apache/pulsar/tests/integration/auth/token/PulsarTokenAuthenticationBaseSuite.java b/...va/org/apache/pulsar/tests/integration/auth/token/PulsarTokenAuthenticationBaseSuite.java
@@ -70,9 +70,10 @@ public abstract class PulsarTokenAuthenticationBaseSuite extends PulsarClusterTe
 
     protected ZKContainer<?> cmdContainer;
 
-    @BeforeClass
+    @BeforeClass(alwaysRun = true)
     @Override
-    public void setupCluster() throws Exception {
+    public final void setupCluster() throws Exception {
+        incrementSetupNumber();
         // Before starting the cluster, generate the secret key and the token
         // Use Zk container to have 1 container available before starting the cluster
         this.cmdContainer = new ZKContainer<>("cli-setup");
@@ -130,7 +131,7 @@ public void setupCluster() throws Exception {
 
     @AfterClass(alwaysRun = true)
     @Override
-    public void tearDownCluster() {
+    public final void tearDownCluster() throws Exception {
         super.tearDownCluster();
         cmdContainer.close();
     }

diff --git a/...c/test/java/org/apache/pulsar/tests/integration/backwardscompatibility/ClientTest2_2.java b/...c/test/java/org/apache/pulsar/tests/integration/backwardscompatibility/ClientTest2_2.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.tests.integration.backwardscompatibility;
 
 
+import java.util.function.Supplier;
 import org.apache.pulsar.tests.integration.topologies.ClientTestBase;
 import org.testng.annotations.Test;
 
@@ -27,9 +28,9 @@ public class ClientTest2_2 extends PulsarStandaloneTestSuite2_2 {
     private final ClientTestBase clientTestBase = new ClientTestBase();
 
     @Test(dataProvider = "StandaloneServiceUrlAndHttpUrl")
-    public void testResetCursorCompatibility(String serviceUrl, String httpServiceUrl) throws Exception {
+    public void testResetCursorCompatibility(Supplier<String> serviceUrl, Supplier<String> httpServiceUrl) throws Exception {
         String topicName = generateTopicName("test-reset-cursor-compatibility", true);
-        clientTestBase.resetCursorCompatibility(serviceUrl, httpServiceUrl, topicName);
+        clientTestBase.resetCursorCompatibility(serviceUrl.get(), httpServiceUrl.get(), topicName);
     }
 
 }
diff --git a/...c/test/java/org/apache/pulsar/tests/integration/backwardscompatibility/ClientTest2_3.java b/...c/test/java/org/apache/pulsar/tests/integration/backwardscompatibility/ClientTest2_3.java
@@ -19,6 +19,7 @@
 package org.apache.pulsar.tests.integration.backwardscompatibility;
 
 
+import java.util.function.Supplier;
 import org.apache.pulsar.tests.integration.topologies.ClientTestBase;
 import org.testng.annotations.Test;
 
@@ -27,9 +28,9 @@ public class ClientTest2_3 extends PulsarStandaloneTestSuite2_3 {
     private final ClientTestBase clientTestBase = new ClientTestBase();
 
     @Test(dataProvider = "StandaloneServiceUrlAndHttpUrl")
-    public void testResetCursorCompatibility(String serviceUrl, String httpServiceUrl) throws Exception {
+    public void testResetCursorCompatibility(Supplier<String> serviceUrl, Supplier<String> httpServiceUrl) throws Exception {
         String topicName = generateTopicName("test-reset-cursor-compatibility", true);
-        clientTestBase.resetCursorCompatibility(serviceUrl, httpServiceUrl, topicName);
+        clientTestBase.resetCursorCompatibility(serviceUrl.get(), httpServiceUrl.get(), topicName);
     }
 
 }