[Tests] Disable test retries for new or modified tests (apache#14607)

### Motivation There's a [discussion on the mailing about improving unit test stability](https://lists.apache.org/thread/d4tq105wmsmm3s84lwy19d0b6hbxn7vz). One of the methods is to disable test retries for new or modified test classes. This will reduce the likelihood of introducing more flaky tests into the code base. ### Modifications - Use the paths-filter in GitHub Actions to list test class changes. Use GitHub Actions to place the CSV list to a environment variable called `CHANGED_TESTS`. - parse `CHANGED_TESTS` in RetryAnalyzer which determines whether a failed test method should be retried or not. Disable test retries for new or modified tests. - parse `CHANGED_TESTS` in `build/retry.sh` script. Skip retries if there's a test failure in one of the changed tests.
drianpauL · Mar 9, 2022 · 98b4492 · 98b4492
1 parent 519dd6e
commit 98b4492
Show file tree

Hide file tree

Showing 33 changed files with 101 additions and 27 deletions.
diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml
@@ -8,3 +8,5 @@ docs:
   - '.asf.yaml'
   - '*.md'
   - '**/*.md'
+tests:
+  - added|modified: '**/src/test/java/**/*.java'
diff --git a/.github/workflows/ci-build-macos.yaml b/.github/workflows/ci-build-macos.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes

diff --git a/.github/workflows/ci-cpp-build-centos7.yaml b/.github/workflows/ci-cpp-build-centos7.yaml
@@ -50,6 +50,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -59,4 +60,4 @@ jobs:
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |
           echo "Build C++ client library on CentOS 7"
-          pulsar-client-cpp/docker-build-centos7.sh
+          pulsar-client-cpp/docker-build-centos7.sh
diff --git a/.github/workflows/ci-cpp-build-windows.yaml b/.github/workflows/ci-cpp-build-windows.yaml
@@ -71,6 +71,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes

diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes

diff --git a/.github/workflows/ci-go-functions-style.yaml b/.github/workflows/ci-go-functions-style.yaml
@@ -56,6 +56,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes

diff --git a/.github/workflows/ci-go-functions-test.yaml b/.github/workflows/ci-go-functions-test.yaml
@@ -58,6 +58,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes

diff --git a/.github/workflows/ci-integration-backwards-compatibility.yaml b/.github/workflows/ci-integration-backwards-compatibility.yaml
@@ -49,6 +49,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -99,7 +100,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh BACKWARDS_COMPAT
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh BACKWARDS_COMPAT
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-integration-cli.yaml b/.github/workflows/ci-integration-cli.yaml
@@ -49,6 +49,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -99,7 +100,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh CLI
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh CLI
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-integration-function.yaml b/.github/workflows/ci-integration-function.yaml
@@ -49,6 +49,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -99,7 +100,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh FUNCTION
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh FUNCTION
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-integration-messaging.yaml b/.github/workflows/ci-integration-messaging.yaml
@@ -49,6 +49,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -99,7 +100,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh MESSAGING
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh MESSAGING
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-integration-process.yaml b/.github/workflows/ci-integration-process.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -98,7 +99,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh PULSAR_CONNECTORS_PROCESS
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_CONNECTORS_PROCESS
 
       - name: Log dmesg when failed
         if: ${{ failure() }}

diff --git a/.github/workflows/ci-integration-pulsar-io-ora.yaml b/.github/workflows/ci-integration-pulsar-io-ora.yaml
@@ -49,6 +49,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -100,7 +101,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh PULSAR_IO_ORA
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_IO_ORA
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-integration-pulsar-io.yaml b/.github/workflows/ci-integration-pulsar-io.yaml
@@ -49,6 +49,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -100,7 +101,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh PULSAR_IO
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_IO
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-integration-schema.yaml b/.github/workflows/ci-integration-schema.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -95,7 +96,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh SCHEMA
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh SCHEMA
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-integration-sql.yaml b/.github/workflows/ci-integration-sql.yaml
@@ -49,6 +49,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -99,7 +100,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh SQL
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh SQL
 
       - name: Log dmesg when failed
         if: ${{ failure() }}

diff --git a/.github/workflows/ci-integration-standalone.yaml b/.github/workflows/ci-integration-standalone.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -98,7 +99,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh STANDALONE
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh STANDALONE
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-integration-thread.yaml b/.github/workflows/ci-integration-thread.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -98,7 +99,7 @@ jobs:
 
       - name: run integration function
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh PULSAR_CONNECTORS_THREAD
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_CONNECTORS_THREAD
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-integration-tiered-filesystem.yaml b/.github/workflows/ci-integration-tiered-filesystem.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -98,7 +99,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh TIERED_FILESYSTEM
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh TIERED_FILESYSTEM
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-integration-tiered-jcloud.yaml b/.github/workflows/ci-integration-tiered-jcloud.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -98,7 +99,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh TIERED_JCLOUD
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh TIERED_JCLOUD
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-integration-transaction.yaml b/.github/workflows/ci-integration-transaction.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -95,7 +96,7 @@ jobs:
 
       - name: run integration tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh TRANSACTION
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh TRANSACTION
 
       - name: Upload container logs
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/ci-license.yaml b/.github/workflows/ci-license.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes

diff --git a/.github/workflows/ci-python-build-3.9-client.yaml b/.github/workflows/ci-python-build-3.9-client.yaml
@@ -50,6 +50,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -59,4 +60,4 @@ jobs:
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |
           echo "Build Python3.9 client library"
-          pulsar-client-cpp/docker-build-python3.9.sh
+          pulsar-client-cpp/docker-build-python3.9.sh
diff --git a/.github/workflows/ci-shade-test.yaml b/.github/workflows/ci-shade-test.yaml
@@ -49,6 +49,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -99,4 +100,4 @@ jobs:
 
       - name: run shade tests
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_integration_group.sh SHADE
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh SHADE
diff --git a/.github/workflows/ci-unit-broker-broker-gp.yaml b/.github/workflows/ci-unit-broker-broker-gp.yaml
@@ -52,6 +52,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -81,15 +82,15 @@ jobs:
 
       - name: run unit test 'BROKER_GROUP_1'
         if: ${{ steps.check_changes.outputs.docs_only != 'true' && matrix.group == 'group1' }}
-        run: ./build/run_unit_group.sh BROKER_GROUP_1
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_GROUP_1
 
       - name: run unit test 'BROKER_GROUP_2'
         if: ${{ steps.check_changes.outputs.docs_only != 'true' && matrix.group == 'group2' }}
-        run: ./build/run_unit_group.sh BROKER_GROUP_2
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_GROUP_2
 
       - name: run unit test 'BROKER_GROUP_3'
         if: ${{ steps.check_changes.outputs.docs_only != 'true' && matrix.group == 'group3' }}
-        run: ./build/run_unit_group.sh BROKER_GROUP_3
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_GROUP_3
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()

diff --git a/.github/workflows/ci-unit-broker-client-api.yaml b/.github/workflows/ci-unit-broker-client-api.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -77,7 +78,7 @@ jobs:
 
       - name: run unit test 'BROKER_CLIENT_API'
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_unit_group.sh BROKER_CLIENT_API
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_CLIENT_API
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()

diff --git a/.github/workflows/ci-unit-broker-client-impl.yaml b/.github/workflows/ci-unit-broker-client-impl.yaml
@@ -48,6 +48,7 @@ jobs:
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
+          list-files: csv
 
       - name: Check changed files
         id: check_changes
@@ -77,7 +78,7 @@ jobs:
 
       - name: run unit test 'BROKER_CLIENT_IMPL'
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
-        run: ./build/run_unit_group.sh BROKER_CLIENT_IMPL
+        run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_CLIENT_IMPL
 
       - name: print JVM thread dumps when cancelled
         if: cancelled()