[Build] Fix skipping tests when there are only documentation changes …

…and run centos7 & windows cpp build only when there are cpp changes (apache#10376) Fixes apache#10373 ### Motivation Tests should be skipped for PR builds that only contain documentation changes. This is currently broken. A new build has been introduced for building the CPP client on Windows. Make the changes in `ci-cpp-build-centos7.yaml` and `ci-cpp-build-windows.yaml` so that the builds run only when there are changes in CPP client files (or the workflow definitions). ### Modifications Refactor logic to be less error prone. Use [`fromJSON`](https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#fromjson) function to parse a number in a string as a number. ### Additional Context apache#10276
sarvex · Apr 28, 2021 · 0984022 · 0984022
1 parent 212ad48
commit 0984022
Show file tree

Hide file tree

Showing 29 changed files with 373 additions and 452 deletions.
diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml
@@ -0,0 +1,10 @@
+# contains pattern definitions used in workflows "changes" step
+# pattern syntax: https://github.com/micromatch/picomatch
+all:
+  - '**'
+docs:
+  - 'site2/**'
+  - 'deployment/**'
+  - '.asf.yaml'
+  - '*.md'
+  - '**/*.md'
diff --git a/.github/workflows/ci-build-macos.yaml b/.github/workflows/ci-build-macos.yaml
@@ -22,11 +22,9 @@ on:
   pull_request:
     branches:
       - master
-    paths-ignore: ['site2/**', 'deployment/**']
   push:
     branches:
       - branch-*
-    paths-ignore: ['site2/**', 'deployment/**']
 
 env:
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
@@ -45,7 +43,18 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
+      - name: Detect changed files
+        id:   changes
+        uses: apache/pulsar-test-infra/paths-filter@master
+        with:
+          filters: .github/changes-filter.yaml
+
+      - name: Check changed files
+        id: check_changes
+        run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+
       - name: Cache Maven dependencies
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         uses: actions/cache@v2
         with:
           path: |
@@ -56,12 +65,15 @@ jobs:
             ${{ runner.os }}-m2-dependencies-all-
 
       - name: Set up JDK 1.8
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
 
       - name: Replace maven's wagon-http version
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: sudo ./build/replace_maven-wagon-http-version.sh
 
       - name: build package
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: mvn -B clean install -DskipTests
diff --git a/.github/workflows/ci-cpp-build-centos7.yaml b/.github/workflows/ci-cpp-build-centos7.yaml
@@ -22,10 +22,15 @@ on:
   pull_request:
     branches:
       - master
+    paths:
+      - '.github/workflows/**'
+      - 'pulsar-client-cpp/**'
   push:
     branches:
       - branch-*
-
+    paths:
+      - '.github/workflows/**'
+      - 'pulsar-client-cpp/**'
 jobs:
 
   cpp-build-centos7:
@@ -40,23 +45,18 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Changed files check
+      - name: Detect changed files
         id:   changes
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
-          filters: |
-            # pattern syntax: https://github.com/micromatch/picomatch
-            all:
-              - 'pulsar-client-cpp/**'
-            docs:
-              - 'site2/**'
-              - 'deployment/**'
-              - '.asf.yaml'
-              - '*.md'
-              - '**/*.md'
+          filters: .github/changes-filter.yaml
+
+      - name: Check changed files
+        id: check_changes
+        run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
 
       - name: build cpp client on centos 7
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |
           echo "Build C++ client library on CentOS 7"
-          pulsar-client-cpp/docker-build-centos7.sh
+          pulsar-client-cpp/docker-build-centos7.sh
diff --git a/.github/workflows/ci-cpp-build-windows.yaml b/.github/workflows/ci-cpp-build-windows.yaml
@@ -22,9 +22,15 @@ on:
   pull_request:
     branches:
       - master
+    paths:
+      - '.github/workflows/**'
+      - 'pulsar-client-cpp/**'
   push:
     branches:
       - branch-*
+    paths:
+      - '.github/workflows/**'
+      - 'pulsar-client-cpp/**'
 
 env:
   VCPKG_FEATURE_FLAGS: manifests
@@ -51,12 +57,24 @@ jobs:
       - name: checkout
         uses: actions/checkout@v2
 
+      - name: Detect changed files
+        id:   changes
+        uses: apache/pulsar-test-infra/paths-filter@master
+        with:
+          filters: .github/changes-filter.yaml
+
+      - name: Check changed files
+        id: check_changes
+        run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+
       - name: Install vcpkg packages
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         shell: bash
         run: |
           cd pulsar-client-cpp && vcpkg install --triplet ${{ matrix.triplet }}
 
       - name: Configure
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         shell: bash
         run: |
           if [ "$RUNNER_OS" == "Windows" ]; then
@@ -71,9 +89,10 @@ jobs:
           fi
 
       - name: Compile
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         shell: bash
         run: |
           if [ "$RUNNER_OS" == "Windows" ]; then
             cd pulsar-client-cpp && \
             cmake --build ./build
-          fi
+          fi
diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml
@@ -43,23 +43,18 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Changed files check
+      - name: Detect changed files
         id:   changes
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
-          filters: |
-            # pattern syntax: https://github.com/micromatch/picomatch
-            all:
-              - '**'
-            docs:
-              - 'site2/**'
-              - 'deployment/**'
-              - '.asf.yaml'
-              - '*.md'
-              - '**/*.md'
+          filters: .github/changes-filter.yaml
+
+      - name: Check changed files
+        id: check_changes
+        run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
 
       - name: Cache local Maven repository
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         uses: actions/cache@v2
         with:
           path: |
@@ -71,12 +66,12 @@ jobs:
 
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         with:
           java-version: 1.8
 
       - name: clean disk
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |
           sudo swapoff -a
           sudo rm -f /swapfile
@@ -85,22 +80,22 @@ jobs:
           df -h
 
       - name: Replace maven's wagon-http version
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: sudo ./build/replace_maven-wagon-http-version.sh
 
       - name: build package
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: mvn -B -ntp -q install -Pcore-modules,-main -DskipTests
 
       - name: build cpp artifacts
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |
           echo "Build C++ client library"
           export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug -DBUILD_DYNAMIC_LIB=OFF -DPYTHON_INCLUDE_DIR=/usr/include/python2.7 -DPYTHON_LIBRARY=/usr/lib/x86_64-linux-gnu/libpython2.7.so"
           pulsar-client-cpp/docker-build.sh
 
       - name: run c++ tests
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: pulsar-client-cpp/docker-tests.sh
 
       - name: Upload test-logs

diff --git a/.github/workflows/ci-go-functions-style.yaml b/.github/workflows/ci-go-functions-style.yaml
@@ -51,43 +51,38 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Changed files check
+      - name: Detect changed files
         id:   changes
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
-          filters: |
-            # pattern syntax: https://github.com/micromatch/picomatch
-            all:
-              - '**'
-            docs:
-              - 'site2/**'
-              - 'deployment/**'
-              - '.asf.yaml'
-              - '*.md'
-              - '**/*.md'
+          filters: .github/changes-filter.yaml
+
+      - name: Check changed files
+        id: check_changes
+        run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
 
       - name: Set up Go
         uses: actions/setup-go@v2
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         with:
           go-version: ${{ matrix.go-version }}
         id: go
 
       - name: InstallTool
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |
           cd pulsar-function-go
           wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.18.0
           ./bin/golangci-lint --version
 
       - name: Build
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |
           cd pulsar-function-go
           go build ./...
 
       - name: CheckStyle
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |
           cd pulsar-function-go
           ./bin/golangci-lint run -c ./golangci.yml ./pf
diff --git a/.github/workflows/ci-go-functions-test.yaml b/.github/workflows/ci-go-functions-test.yaml
@@ -53,30 +53,25 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Changed files check
+      - name: Detect changed files
         id:   changes
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
-          filters: |
-            # pattern syntax: https://github.com/micromatch/picomatch
-            all:
-              - '**'
-            docs:
-              - 'site2/**'
-              - 'deployment/**'
-              - '.asf.yaml'
-              - '*.md'
-              - '**/*.md'
+          filters: .github/changes-filter.yaml
+
+      - name: Check changed files
+        id: check_changes
+        run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
 
       - name: Set up Go
         uses: actions/setup-go@v2
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         with:
           go-version: ${{ matrix.go-version }}
         id: go
 
       - name: Run tests
-        if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |
           cd pulsar-function-go
           go test -v $(go list ./... | grep -v examples)