JUnit XML for Unit Tests and Bug Fix (istio#3709)

Automatic merge from submit-queue.

JUnit XML for Unit Tests and Bug Fix

The bug was that when a Prow job fails, no xml results are uploaded because the test script exit on nonzero status immediately and the xml file was move to artifacts / upload directory as the last step. By creating the files in `_artifacts` dir directly, it is always uploaded by Prow. 

Also git ignores the junit results so developers running tests locally will not pollute his/her workspace. Passing junit xml file names through environment variables provides more flexibility and prevents overwriting results if executing multiple commands that all generate junit xmls.

Makefile

@@ -401,10 +401,21 @@ istioctl-install:
 # Target: test
 #-----------------------------------------------------------------------------
 
-.PHONY: test localTestEnv test-bins
+.PHONY: junit-parser test localTestEnv test-bins
+
+JUNIT_REPORT := $(shell which go-junit-report 2> /dev/null || echo "${ISTIO_BIN}/go-junit-report")
+
+${ISTIO_BIN}/go-junit-report:
+	@echo "go-junit-report not found. Installing it now..."
+	unset GOOS && CGO_ENABLED=1 go get -u github.com/jstemmer/go-junit-report
 
 # Run coverage tests
-test: pilot-test mixer-test security-test broker-test galley-test common-test
+JUNIT_UNIT_TEST_XML ?= $(ISTIO_OUT)/junit_unit_tests.xml
+test: | $(JUNIT_REPORT)
+	mkdir -p $(dir $(JUNIT_UNIT_TEST_XML))
+	set -o pipefail; \
+	$(MAKE) pilot-test mixer-test security-test broker-test galley-test common-test \
+	|& tee >($(JUNIT_REPORT) > $(JUNIT_UNIT_TEST_XML))
 
 GOTEST_PARALLEL ?= '-test.parallel=4'
 GOTEST_P ?= -p 1

prow/e2e-suite.sh

@@ -91,7 +91,9 @@ if ${SINGLE_MODE}; then
     for T in ${TEST_TARGETS[@]}; do
         if [ "${T}" == "${SINGLE_TEST}" ]; then
             VALID_TEST=true
-            time ISTIO_DOCKER_HUB=$HUB E2E_ARGS="${E2E_ARGS[@]}" make "${SINGLE_TEST}"
+            time ISTIO_DOCKER_HUB=$HUB \
+              E2E_ARGS="${E2E_ARGS[@]}" \
+              make "${SINGLE_TEST}"
         fi
     done
     if [ "${VALID_TEST}" == "false" ]; then
@@ -102,10 +104,8 @@ if ${SINGLE_MODE}; then
 
 else
     echo "Executing e2e test suite"
-    time ISTIO_DOCKER_HUB=$HUB E2E_ARGS="${E2E_ARGS[@]}" make e2e_all
-fi
-
-if [ "${CI:-}" == 'bootstrap' ] && [ -f junit.xml ]; then
-  # allow bootsrap to upload junit results
-  mv junit.xml ${ARTIFACTS_DIR}
+    time ISTIO_DOCKER_HUB=$HUB \
+      E2E_ARGS="${E2E_ARGS[@]}" \
+      JUNIT_E2E_XML="${ARTIFACTS_DIR}/junit_e2e_all.xml" \
+      make e2e_all
 fi

prow/istio-postsubmit.sh

@@ -36,7 +36,9 @@ cd $ROOT
 make init
 
 echo 'Running Unit Tests'
-time make localTestEnv test
+time JUNIT_UNIT_TEST_XML="${ARTIFACTS_DIR}/junit_unit_tests.xml" \
+T="-v" \
+make localTestEnv test
 
 HUB="gcr.io/istio-testing"
 TAG="${GIT_SHA}"

prow/istio-unit-tests.sh

@@ -27,9 +27,10 @@ set -e
 source ${ROOT}/prow/lib.sh
 setup_and_export_git_sha
 
-echo 'Build'
-(cd ${ROOT}; make build)
+cd ${ROOT}
 
 # Unit tests are run against a local apiserver and etcd.
 # Integration/e2e tests in the other scripts are run against GKE or real clusters.
-(cd ${ROOT}; make localTestEnv test)
+JUNIT_UNIT_TEST_XML="${ARTIFACTS_DIR}/junit_unit_tests.xml" \
+T="-v" \
+make build localTestEnv test

tests/istio.mk

@@ -73,8 +73,12 @@ e2e_bookinfo: istioctl generate_yaml
 e2e_upgrade: istioctl generate_yaml
 	go test -v -timeout 20m ./tests/e2e/tests/upgrade -args ${E2E_ARGS} ${EXTRA_E2E_ARGS}
 
-e2e_all:
-	set -o pipefail; $(MAKE) e2e_simple e2e_mixer e2e_bookinfo |& tee >(go-junit-report > junit.xml)
+JUNIT_E2E_XML ?= $(ISTIO_OUT)/junit_e2e_all.xml
+e2e_all: | $(JUNIT_REPORT)
+	mkdir -p $(dir $(JUNIT_E2E_XML))
+	set -o pipefail; \
+	$(MAKE) e2e_simple e2e_mixer e2e_bookinfo \
+	|& tee >($(JUNIT_REPORT) > $(JUNIT_E2E_XML))
 
 e2e_pilot: istioctl generate_yaml
 	go test -v -timeout 20m ./tests/e2e/tests/pilot ${TESTOPTS} -hub ${HUB} -tag ${TAG}