[improve][ci] Improve code coverage reporting (apache#19296)

.github/actions/upload-coverage/action.yml

@@ -83,4 +83,18 @@ runs:
       with:
         flags: ${{ inputs.flags }}
         fail_ci_if_error: true
-        verbose: true
+        verbose: true
+    - name: "Show link to Codecov report"
+      shell: bash
+      run: |
+        if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+          head_sha=$(jq -r '.pull_request.head.sha' "${GITHUB_EVENT_PATH}")
+        else
+          head_sha=$(git rev-parse HEAD)
+        fi
+        tee -a "$GITHUB_STEP_SUMMARY" <<EOF
+        ## Code Coverage report in Codecov
+        
+        [Code coverage report](https://app.codecov.io/github/$GITHUB_REPOSITORY/commit/${head_sha}/tree)
+        
+        EOF

build/pulsar_ci_tool.sh

@@ -363,14 +363,8 @@ _ci_upload_coverage_files() {
     for execFile in $execFiles; do
       local project="${execFile/%"/target/jacoco.exec"}"
       local artifactId=$(xmlstarlet sel -t -m _:project -v _:artifactId -n $project/pom.xml)
-      local scope=runtime
-      # for integration tests, there's no dependencies in the runtime scope
-      # detect a plain test project based on missing src/main/java
-      if [ ! -d $project/src/main/java ]; then
-        scope=test
-      fi
-      # find the runtime classpath for the project to ensure that only production classes get covered
-      mvn -f $project/pom.xml -DincludeScope=$scope -Dscan=false dependency:build-classpath  -B | { grep 'Dependencies classpath:' -A1 || true; } | tail -1 \
+      # find the test scope classpath for the project
+      mvn -f $project/pom.xml -DincludeScope=test -Dscan=false dependency:build-classpath  -B | { grep 'Dependencies classpath:' -A1 || true; } | tail -1 \
                                     | sed 's/:/\n/g' | { grep 'org/apache/pulsar' || true; } \
                                     | { tee -a $completeClasspathFile || true; } > target/classpath_$artifactId || true
     done
@@ -468,7 +462,8 @@ ci_create_test_coverage_report() {
   else
     cd "$SCRIPT_DIR/.."
   fi
-  local execFiles=$(find . '(' -path "*/target/jacoco.exec" -or -path "*/target/jacoco_*.exec" ')' -printf "%P\n")
+
+  local execFiles=$(find . '(' -path "*/target/jacoco.exec" -or -path "*/target/jacoco_*.exec" ')' -printf "%P\n" )
   if [[ -n "$execFiles" ]]; then
     mkdir -p /tmp/jacocoDir
     if [ ! -f /tmp/jacocoDir/jacococli.jar ]; then
@@ -492,41 +487,42 @@ ci_create_test_coverage_report() {
       done
     } | sort | uniq)
 
+    # projects that aren't considered as production code and their own src/main/java source code shouldn't be analysed
+    local excludeProjectsPattern="testmocks|testclient|buildtools"
+
     # iterate projects
     for project in $projects; do
       local artifactId="$(printf "%s" "$projectToArtifactIdMapping" | grep -F "$project " | cut -d' ' -f2)"
-      if [ -d "$project/target/classes" ]; then
+      if [[ -d "$project/target/classes" && -d "$project/src/main/java" ]]; then
         mkdir -p "$classesDir/$project"
         cp -Rl "$project/target/classes" "$classesDir/$project"
-        echo "/$artifactId/" >> $filterArtifactsFile
-      fi
-      local scope=runtime
-      if [ -d $project/src/main/java ]; then
         echo "$project/src/main/java" >> $sourcefilesFile
-      else
-        # for integration tests, there's no dependencies in the runtime scope
-        scope=test
       fi
-      if [ -f "target/classpath_$artifactId" ]; then
-        echo "Found cached classpath for $artifactId."
-        cat "target/classpath_$artifactId" >> $completeClasspathFile
+      echo "/$artifactId/" >> $filterArtifactsFile
+      if [[ -n "$(echo "$project" | grep -v -E "$excludeProjectsPattern")" ]]; then
+        if [ -f "target/classpath_$artifactId" ]; then
+          echo "Found cached classpath for $artifactId."
+          cat "target/classpath_$artifactId" >> $completeClasspathFile
+        else
+          echo "Resolving classpath for $project..."
+          # find the test scope classpath for the project
+          mvn -f $project/pom.xml -DincludeScope=test -Dscan=false dependency:build-classpath  -B | { grep 'Dependencies classpath:' -A1 || true; } | tail -1 \
+                                        | sed 's/:/\n/g' | { grep 'org/apache/pulsar' || true; } \
+                                        >> $completeClasspathFile || true
+        fi
       else
-        echo "Resolving classpath for $project..."
-        # find the runtime classpath for the project to ensure that only production classes get covered
-        mvn -f $project/pom.xml -DincludeScope=$scope -Dscan=false dependency:build-classpath  -B | { grep 'Dependencies classpath:' -A1 || true; } | tail -1 \
-                                      | sed 's/:/\n/g' | { grep 'org/apache/pulsar' || true; } \
-                                      >> $completeClasspathFile || true
+        echo "Skipping analysing of $project"
       fi
     done
 
     # delete any possible embedded jar files in the classes directory
     find "$classesDir" -name "*.jar" -print -delete
 
-    filterJarsPattern="bouncy-castle-bc|tests|/buildtools/"
+    local excludeJarsPattern="bouncy-castle-bc|tests|$excludeProjectsPattern"
 
     local classfilesArgs="--classfiles $({
       {
-        for classpathEntry in $(cat $completeClasspathFile | { grep -v -f $filterArtifactsFile || true; } | sort | uniq | { grep -v -E $filterJarsPattern || true; }); do
+        for classpathEntry in $(cat $completeClasspathFile | { grep -v -f $filterArtifactsFile || true; } | sort | uniq | { grep -v -E "$excludeJarsPattern" || true; }); do
             if [[ -f $classpathEntry && -n "$(unzip -Z1C $classpathEntry 'META-INF/bundled-dependencies/*' 2>/dev/null)" ]]; then
               # file must be processed by removing META-INF/bundled-dependencies
               local jartempfile=$(mktemp -t jarfile.XXXX --suffix=.jar)
@@ -543,7 +539,7 @@ ci_create_test_coverage_report() {
 
     local sourcefilesArgs="--sourcefiles $({
       # find the source file folders for the pulsar .jar files that are on the classpath
-      for artifactId in $(cat $completeClasspathFile  | sort | uniq | { grep -v -E $filterJarsPattern || true; } | perl -p -e 's|.*/org/apache/pulsar/([^/]*)/.*|$1|'); do
+      for artifactId in $(cat $completeClasspathFile  | sort | uniq | { grep -v -E "$excludeJarsPattern" || true; } | perl -p -e 's|.*/org/apache/pulsar/([^/]*)/.*|$1|'); do
         local project="$(printf "%s" "$projectToArtifactIdMapping" | { grep $artifactId || true; } | cut -d' ' -f1)"
         if [[ -n "$project" && -d "$project/src/main/java" ]]; then
           echo "$project/src/main/java"
@@ -595,10 +591,12 @@ ci_create_inttest_coverage_report() {
       # remove any bundled dependencies as part of .jar/.nar files
       find /tmp/jacocoDir/pulsar_lib '(' -name "*.jar" -or -name "*.nar" ')' -exec echo "Processing {}" \; -exec zip -q -d {} 'META-INF/bundled-dependencies/*' \; |grep -E -v "Nothing to do|^$" || true
     fi
+    # projects that aren't considered as production code and their own src/main/java source code shouldn't be analysed
+    local excludeProjectsPattern="testmocks|testclient|buildtools"
     # produce jacoco XML coverage report from the exec files and using the extracted jar files
     java -jar /tmp/jacocoDir/jacococli.jar report /tmp/jacocoDir/*.exec \
       --classfiles /tmp/jacocoDir/pulsar_lib --encoding UTF-8 --name "Pulsar Integration Tests - coverage in containers" \
-      $(find -path "*/src/main/java" -printf "--sourcefiles %P ") \
+      $(find -path "*/src/main/java" -printf "--sourcefiles %P " | grep -v -E "$excludeProjectsPattern") \
       --xml target/jacoco_inttest_coverage_report/jacoco.xml \
       --html target/jacoco_inttest_coverage_report/html \
       --csv target/jacoco_inttest_coverage_report/jacoco.csv

codecov.yml

@@ -18,12 +18,15 @@
 #
 
 codecov:
-  require_ci_to_pass: yes
+  # the state in Codecov for a PR status in GitHub is often wrong. Just ignore this condition and
+  # rely on after_n_builds for notifying about the coverage
+  require_ci_to_pass: no
   notify:
     # should match the number of coverage report uploads
     # pulsar-ci.yaml contains 3 uploads (unittests, inttests, systests)
     # pulsar-ci-flaky.yaml contains 1 upload
     after_n_builds: 4
+    wait_for_ci: no
 
 comment:
   # should match the number of builds sending coverage reports
@@ -44,3 +47,12 @@ coverage:
       default:
         target: auto
         informational: true
+
+# ignore test code
+ignore:
+  - tests
+  - testmocks
+  - buildtools
+  - pulsar-testclient
+  - pulsar-client-tools-customcommand-example
+  - pulsar-functions/java-examples

pom.xml

@@ -1970,6 +1970,11 @@ flexible messaging model and an intuitive client API.</description>
                     <include>org.apache.pulsar.*</include>
                     <include>org.apache.bookkeeper.mledger.*</include>
                   </includes>
+                  <excludes>
+                    <exclude>*.proto.*</exclude>
+                    <exclude>*.shade.*</exclude>
+                    <exclude>*.shaded.*</exclude>
+                  </excludes>
                 </configuration>
               </execution>
               <execution>

tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java

@@ -281,7 +281,8 @@ protected void configureCodeCoverage() {
             }
             withEnv("OPTS", "-javaagent:/jacocoDir/" + jacocoAgentJar.getName()
                     + "=destfile=/jacocoDir/jacoco_" + getContainerName() + "_" + System.currentTimeMillis() + ".exec"
-                    + ",includes=org.apache.pulsar.*:org.apache.bookkeeper.mledger.*");
+                    + ",includes=org.apache.pulsar.*:org.apache.bookkeeper.mledger.*"
+                    + ",excludes=*.proto.*:*.shade.*:*.shaded.*");
         } else {
             log.error("Cannot find jacoco agent jar from '" + jacocoAgentJar.getAbsolutePath() + "'");
         }