Add support for running tests with gotestsum

The current go-junit-reporter does not include the log/failure
message in the XML, which makes it harder to see the error in
CircleCI.

gotestsum JUnit XML support does include the error message, but it
runs the tests different. This adds support for gotestsum while trying
to keep support for go-junit-reporter so we can switch back quickly if
we need to.

Makefile

@@ -213,6 +213,10 @@ bin/soda: .check_go_version.stamp .check_gopath.stamp pkg/tools/tools.go
 bin/go-junit-report: .check_go_version.stamp .check_gopath.stamp pkg/tools/tools.go
 	go build -o bin/go-junit-report github.com/jstemmer/go-junit-report
 
+# No static linking / $(LDFLAGS) because gotestsum is only used for building the CirlceCi test report
+bin/gotestsum: .check_go_version.stamp .check_gopath.stamp pkg/tools/tools.go
+	go build -o bin/gotestsum gotest.tools/gotestsum
+
 # No static linking / $(LDFLAGS) because mockery is only used for testing
 bin/mockery: .check_go_version.stamp .check_gopath.stamp pkg/tools/tools.go
 	go build -o bin/mockery github.com/vektra/mockery/v2

go.mod

@@ -103,6 +103,7 @@ require (
 	go.opentelemetry.io/otel/sdk/export/metric v0.25.0
 	go.opentelemetry.io/otel/sdk/metric v0.25.0
 	go.opentelemetry.io/otel/trace v1.3.0
+	gotest.tools/gotestsum v1.7.0
 )
 
 require (
@@ -115,6 +116,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/danieljoos/wincred v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dnephin/pflag v1.0.7 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b // indirect
 	github.com/fatih/color v1.13.0 // indirect
@@ -142,6 +144,7 @@ require (
 	github.com/google/btree v1.0.0 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gookit/color v1.4.2 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect
@@ -161,6 +164,7 @@ require (
 	github.com/jackc/pgx/v4 v4.13.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/joho/godotenv v1.4.0 // indirect
+	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/karrick/godirwalk v1.16.1 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect

go.sum

@@ -194,6 +194,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
+github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk=
+github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -220,6 +222,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
@@ -545,6 +548,8 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
@@ -721,6 +726,8 @@ github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqx
 github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
 github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
+github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -810,6 +817,7 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
@@ -1491,6 +1499,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -1760,6 +1769,10 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools/gotestsum v1.7.0 h1:RwpqwwFKBAa2h+F6pMEGpE707Edld0etUD3GhqqhDNc=
+gotest.tools/gotestsum v1.7.0/go.mod h1:V1m4Jw3eBerhI/A6qCxUE07RnCg7ACkKj9BYcAm09V8=
+gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/tools/tools.go

@@ -27,4 +27,7 @@ import (
 
 	// Install go-junit-report for CirclCI test result report generation
 	_ "github.com/jstemmer/go-junit-report"
+
+	// Possible replacement for go-junit-report
+	_ "gotest.tools/gotestsum"
 )

scripts/run-server-test

@@ -9,6 +9,7 @@
 # - COVERAGE: '1' will enable test coverage flags
 # - DRY_RUN: '1' will build the tests but not run them
 # - SERVER_REPORT: '1' will run go-junit-report and go coverage on output
+# - GO_JUNIT_REPORT: Use go-junit-report to generate the junit report
 # - LONG_TEST: '1' will remove the '-short' flag and run extended tests
 # - NO_DB: Will disable the db reset and migration steps
 #
@@ -17,50 +18,54 @@
 
 set -eu -o pipefail
 
+# default to gotestsum if GO_JUNIT is not defined
+if [[ "${SERVER_REPORT:-}" == "1" && -z "${GO_JUNIT_REPORT:-}" ]]; then
+  GOTESTSUM=1
+fi
+
 # colors
 RED='\033[0;31m'
 NC='\033[0m' # No Color
 
 if [ "${APPLICATION}" == "app" ]; then
-  package_list=$(go list ./... | grep -E -v '(ordersapi|/pkg/gen/|/pkg/assets/|/cmd|mocks|/pkg/testdatagen)')
+  readarray -t package_list < <(go list ./... | grep -E -v '(ordersapi|/pkg/gen/|/pkg/assets/|/cmd|mocks|/pkg/testdatagen)')
 elif [ "${APPLICATION}" == "orders" ]; then
-  package_list=$(go list ./pkg/handlers/ordersapi)
+  readarray -t package_list < <(go list ./pkg/handlers/ordersapi)
 else
   echo "Must provider the environment variable APPLICATION and set to 'app' or 'orders'"
   exit 1
 fi
 
-verbose_flag=""
-parallel=8
-failfast_flag="-failfast"
+gotest_args=("-count" "1")
 if [ -n "${CIRCLECI+x}" ]; then
 	# Limit the maximum number of tests to run in parallel for CircleCI due to memory constraints.
-  parallel=4
-	# Add verbose (-v) so go-junit-report can parse it for CircleCI results
+  gotest_args+=("-parallel" "4")
+	# Add verbose (-v) so go-junit-report can parse it for CircleCI
+  # results
   verbose_flag="-v"
   # Don't fail fast in Circle CI so we can see all the tests that failed
-  failfast_flag=""
+else
+  verbose_flag=""
+  gotest_args+=("-parallel" "8")
+  gotest_args+=("-failfast")
 fi
 
 # Try to compile tests, but don't run them.
 if [[ "${DRY_RUN:-}" == "1" ]]; then
   echo "Compiling tests only, not running"
-  # shellcheck disable=SC2086
-	go test -run=nope -parallel ${parallel} -count 1 ${package_list}
+  go test -run=nope "${gotest_args[@]}" "${package_list[@]}"
   exit 0
 fi
 
 # Like server_test but runs extended tests that may hit external services.
-short_flag="-short"
-if [[ "${LONG_TEST:-}" == "1" ]]; then
-  short_flag=""
+if [[ "${LONG_TEST:-}" != "1" ]]; then
+  gotest_args+=("-short")
 fi
 
 # Add coverage tracker via go cover
-coverage_flag=""
 if [[ "${COVERAGE:-}" == "1" ]]; then
-  coverpkg=$(echo "${package_list}" | paste -s -d , - | tr '[:blank:]' ',')
-  coverage_flag="-coverprofile=coverage.out -covermode=count -coverpkg=${coverpkg}"
+  coverpkg=$(echo "${package_list[@]}" | paste -s -d , - | tr '[:blank:]' ',')
+  gotest_args+=(-coverprofile=coverage.out -covermode=count -coverpkg="${coverpkg}")
 fi
 
 # Set up the test DB instance and wipe it
@@ -77,8 +82,10 @@ mkdir -p "${test_dir}"
 
 function server_report_cleanup()
 {
-  # generate the junit report
-  bin/go-junit-report < "${test_output_file}" > "${test_report_file}"
+  if [[ "${GOTESTSUM:-}" != "1" ]]; then
+    # generate the junit report
+    bin/go-junit-report < "${test_output_file}" > "${test_report_file}"
+  fi
   if [[ "${COVERAGE:-}" == "1" ]]; then
     go tool cover -func=coverage.out -o "${test_dir}/go-coverage.txt"
     go tool cover -html=coverage.out -o "${test_dir}/go-coverage.html"
@@ -103,12 +110,38 @@ function server_report_cleanup()
 
 # Set up junit report and ensure its run on exit
 if [[ "${SERVER_REPORT:-}" == "1" ]]; then
-  if [ ! -f bin/go-junit-report ]; then
-    make bin/go-junit-report
+  if [[ "${GOTESTSUM:-}" != "1" ]]; then
+    if [ ! -f bin/go-junit-report ]; then
+      make bin/go-junit-report
+    fi
   fi
   # setup a trap incase the tests fail, we still want to generate the report
   trap server_report_cleanup EXIT
 fi
 
-# shellcheck disable=SC2086
-DB_NAME="${DB_NAME_TEST}" DB_PORT="${DB_PORT_TEST}" go test ${failfast_flag} -vet=off -parallel "${parallel}" -count 1 ${verbose_flag} ${coverage_flag} ${short_flag} ${package_list} | tee "${test_output_file}"
+gotest_args+=("-vet=off")
+if [[ "${GOTESTSUM:-}" == "1" ]]; then
+  if [[ ! -f bin/gotestsum ]]; then
+    make bin/gotestsum
+  fi
+  gotestsum_args=(--format=standard-quiet)
+  if [[ -n "${verbose_flag}" ]]; then
+    gotestsum_args=(--format=standard-verbose)
+  fi
+  if [[ "${SERVER_REPORT:-}" == "1" ]]; then
+    gotestsum_args+=("--junitfile=${test_report_file}")
+  fi
+
+  DB_NAME="${DB_NAME_TEST}" DB_PORT="${DB_PORT_TEST}" \
+         gotestsum "${gotestsum_args[@]}" -- \
+         "${gotest_args[@]}" \
+         "${package_list[@]}"
+else
+  if [[ -n "${verbose_flag}" ]]; then
+    gotest_args+=("-v")
+  fi
+  DB_NAME="${DB_NAME_TEST}" DB_PORT="${DB_PORT_TEST}" \
+         go test \
+         "${gotest_args[@]}" \
+         "${package_list[@]}" | tee "${test_output_file}"
+fi