[Tests] Fix cpp build flakiness by enabling gtest retries, also impro…

…ve logging in CI (apache#10309)

* Reduce the number of parallel workers for running cpp tests

* Store logs for gtests

* Copy logs from the docker container to test-logs directory

* disable color output by default

* Upload test-logs directory if job fails

* Remove "-t" parameter

.github/workflows/ci-cpp.yaml

@@ -102,3 +102,11 @@ jobs:
       - name: run c++ tests
         if: ${{ steps.changes.outputs.all_count }} > ${{ steps.changes.outputs.docs_count }}
         run: pulsar-client-cpp/docker-tests.sh
+
+      - name: Upload test-logs
+        uses: actions/upload-artifact@v2
+        if: failure()
+        continue-on-error: true
+        with:
+          name: test-logs
+          path: test-logs

pulsar-client-cpp/docker-tests.sh

@@ -42,7 +42,9 @@ echo "---- Testing Pulsar C++ client using image $IMAGE (type --help for more op
 
 docker pull $IMAGE
 
-DOCKER_CMD="docker run -i -v $ROOT_DIR:/pulsar $IMAGE"
+CONTAINER_LABEL="pulsartests=$$"
+export GTEST_COLOR=${GTEST_COLOR:-no}
+DOCKER_CMD="docker run -e GTEST_COLOR -i -l $CONTAINER_LABEL -v $ROOT_DIR:/pulsar $IMAGE"
 
 
 for args in "$@"
@@ -58,4 +60,26 @@ done
 
 # Start 2 Pulsar standalone instances (one with TLS and one without)
 # and execute the tests
-$DOCKER_CMD bash -c "cd /pulsar/pulsar-client-cpp && ./run-unit-tests.sh ${tests}"
+set +e
+DISABLE_COLOR_OUTPUT=""
+if [ "$GTEST_COLOR" = "no" ]; then
+  DISABLE_COLOR_OUTPUT="| cat"
+fi
+$DOCKER_CMD bash -c "cd /pulsar/pulsar-client-cpp && ./run-unit-tests.sh ${tests} $DISABLE_COLOR_OUTPUT"
+RES=$?
+if [ $RES -ne 0 ]; then
+  (
+  cd "$ROOT_DIR"
+  mkdir -p test-logs
+  cd test-logs
+  container_id=$(docker ps -a -q --filter "label=$CONTAINER_LABEL")
+  if [ -n "$container_id" ]; then
+    # copy logs from the container that ran the tests
+    docker commit $container_id pulsartests/$container_id
+    docker run -i --rm pulsartests/$container_id \
+      bash -c "cd /tmp; tar zcf - gtest-parallel-logs gtest_parallel_results.json pulsar-test-dist/logs" \
+      | tar zxvf -
+  fi
+  )
+fi
+exit $RES

pulsar-client-cpp/run-unit-tests.sh

@@ -27,14 +27,25 @@ cd $ROOT_DIR/pulsar-client-cpp
 
 pushd tests
 
+export RETRY_FAILED="${RETRY_FAILED:-1}"
+
 if [ -f /gtest-parallel/gtest-parallel ]; then
-    echo "---- Run unit tests in parallel"
+    gtest_workers=10
+    # use nproc to set workers to 2 x the number of available cores if nproc is available
+    if [ -x "$(command -v nproc)" ]; then
+      gtest_workers=$(( $(nproc) * 2 ))
+    fi
+    # set maximum workers to 10
+    gtest_workers=$(( gtest_workers > 10 ? 10 : gtest_workers ))
+    echo "---- Run unit tests in parallel (workers=$gtest_workers) (retry_failed=${RETRY_FAILED})"
     tests=""
     if [ $# -eq 1 ]; then
         tests="--gtest_filter=$1"
         echo "Running tests: $1"
     fi
-    /gtest-parallel/gtest-parallel ./main $tests --workers=10
+    /gtest-parallel/gtest-parallel $tests --dump_json_test_results=/tmp/gtest_parallel_results.json \
+      --workers=$gtest_workers --retry_failed=$RETRY_FAILED -d /tmp \
+      ./main
     RES=$?
 else
     ./main