Merge pull request moby#34000 from dnephin/test-integration-api

Introduce `test-integration` target

.gitignore

@@ -4,7 +4,7 @@
 *.exe
 *.exe~
 *.orig
-*.test
+test.main
 .*.swp
 .DS_Store
 # a .bashrc may be added to customize the build environment

CONTRIBUTING.md

@@ -160,6 +160,10 @@ it! Take a look at existing tests for inspiration. [Run the full test
 suite](https://docs.docker.com/opensource/project/test-and-docs/) on your branch before
 submitting a pull request.
 
+If your changes need integration tests, write them against the API. The `cli`
+integration tests are slowly either migrated to API tests or moved away as unit
+tests in `docker/cli` and end-to-end tests for docker.
+
 Update the documentation when creating or modifying features. Test your
 documentation changes for clarity, concision, and correctness, as well as a
 clean documentation build. See our contributors guide for [our style

Dockerfile

@@ -9,7 +9,7 @@
 # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
 #
 # # Run the test suite:
-# docker run -e DOCKER_GITCOMMIT=foo --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py
+# docker run -e DOCKER_GITCOMMIT=foo --privileged docker hack/make.sh test-unit test-integration test-docker-py
 #
 # # Publish a release:
 # docker run --privileged \

Dockerfile.aarch64

@@ -9,7 +9,7 @@
 # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
 #
 # # Run the test suite:
-# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py
+# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py
 #
 # Note: AppArmor used to mess with privileged mode, but this is no longer
 # the case. Therefore, you don't have to disable it anymore.

Dockerfile.armhf

@@ -9,7 +9,7 @@
 # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
 #
 # # Run the test suite:
-# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py
+# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py
 #
 # Note: AppArmor used to mess with privileged mode, but this is no longer
 # the case. Therefore, you don't have to disable it anymore.

Dockerfile.ppc64le

@@ -9,7 +9,7 @@
 # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
 #
 # # Run the test suite:
-# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py
+# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py
 #
 # Note: AppArmor used to mess with privileged mode, but this is no longer
 # the case. Therefore, you don't have to disable it anymore.

Dockerfile.s390x

@@ -9,7 +9,7 @@
 # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash
 #
 # # Run the test suite:
-# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py
+# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py
 #
 # Note: AppArmor used to mess with privileged mode, but this is no longer
 # the case. Therefore, you don't have to disable it anymore.

Dockerfile.simple

@@ -1,7 +1,7 @@
 # docker build -t docker:simple -f Dockerfile.simple .
 # docker run --rm docker:simple hack/make.sh dynbinary
 # docker run --rm --privileged docker:simple hack/dind hack/make.sh test-unit
-# docker run --rm --privileged -v /var/lib/docker docker:simple hack/dind hack/make.sh dynbinary test-integration-cli
+# docker run --rm --privileged -v /var/lib/docker docker:simple hack/dind hack/make.sh dynbinary test-integration
 
 # This represents the bare minimum required to build and test Docker.
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all binary dynbinary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration-cli test-unit tgz validate win
+.PHONY: all binary dynbinary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration test-unit tgz validate win
 
 # set the graph driver as the current graphdriver if not set
 DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //'))
@@ -149,13 +149,15 @@ shell: build ## start a shell inside the build env
 	$(DOCKER_RUN_DOCKER) bash
 
 test: build ## run the unit, integration and docker-py tests
-	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration-cli test-docker-py
+	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration test-docker-py
 
 test-docker-py: build ## run the docker-py tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-docker-py
 
-test-integration-cli: build ## run the integration tests
-	$(DOCKER_RUN_DOCKER) hack/make.sh build-integration-test-binary dynbinary test-integration-cli
+test-integration-cli: test-integration ## (DEPRECATED) use test-integration
+
+test-integration: build ## run the integration tests
+	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration
 
 test-unit: build ## run the unit tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh test-unit

hack/README.md

@@ -37,14 +37,14 @@ More information is found within `make.ps1` by the author, @jhowardmsft
 - Referenced via `make test` when running tests on a local machine,
 or directly referenced when running tests inside a Docker development container.  
 - When running on a local machine, `make test` to run all tests found in
-`test`, `test-unit`, `test-integration-cli`, and `test-docker-py` on
+`test`, `test-unit`, `test-integration`, and `test-docker-py` on
 your local machine. The default timeout is set in `make.sh` to 60 minutes
 (`${TIMEOUT:=60m}`), since it currently takes up to an hour to run
 all of the tests.
 - When running inside a Docker development container, `hack/make.sh` does
 not have a single target that runs all the tests. You need to provide a
 single command line with multiple targets that performs the same thing.
-An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration-cli test-docker-py`
+An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration test-docker-py`
 - For more information related to testing outside the scope of this README,
 refer to
 [Run tests and test documentation](https://docs.docker.com/opensource/project/test-and-docs/)

hack/make.ps1

@@ -318,7 +318,7 @@ Function Run-UnitTests() {
     $pkgList = $pkgList | Select-String -Pattern "github.com/docker/docker"
     $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/vendor"
     $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man"
-    $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration-cli"
+    $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration"
     $pkgList = $pkgList -replace "`r`n", " "
     $goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList"
     Invoke-Expression $goTestCommand

hack/make.sh

@@ -60,7 +60,7 @@ DEFAULT_BUNDLES=(
 	dynbinary
 
 	test-unit
-	test-integration-cli
+	test-integration
 	test-docker-py
 
 	cross

hack/make/.ensure-emptyfs

@@ -1,23 +1,23 @@
 #!/usr/bin/env bash
 set -e
 
-if ! docker inspect -t image emptyfs &> /dev/null; then
-	# let's build a "docker save" tarball for "emptyfs"
+if ! docker image inspect emptyfs > /dev/null; then
+	# build a "docker save" tarball for "emptyfs"
 	# see https://github.com/docker/docker/pull/5262
 	# and also https://github.com/docker/docker/issues/4242
 	dir="$DEST/emptyfs"
-	mkdir -p "$dir"
+	uuid=511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158
+	mkdir -p "$dir/$uuid"
 	(
-		cd "$dir"
-		echo '{"emptyfs":{"latest":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158"}}' > repositories
-		mkdir -p 511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158
-		(
-			cd 511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158
-			echo '{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}' > json
-			echo '1.0' > VERSION
-			tar -cf layer.tar --files-from /dev/null
-		)
+		echo '{"emptyfs":{"latest":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158"}}' > "$dir/repositories"
+		cd "$dir/$uuid"
+		echo '{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}' > json
+		echo '1.0' > VERSION
+		tar -cf layer.tar --files-from /dev/null
+	)
+	(
+		[ -n "$TESTDEBUG" ] && set -x
+		tar -cC "$dir" . | docker load
 	)
-	( set -x; tar -cC "$dir" . | docker load )
 	rm -rf "$dir"
 fi

hack/make/.integration-daemon-setup

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-bundle .detect-daemon-osarch
+source "$MAKEDIR/.detect-daemon-osarch"
 if [ "$DOCKER_ENGINE_GOOS" != "windows" ]; then
 	bundle .ensure-emptyfs
 fi

hack/make/.integration-daemon-start

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# see test-integration-cli for example usage of this script
+# see test-integration for example usage of this script
 
 base="$ABS_DEST/.."
 export PATH="$base/binary-daemon:$base/dynbinary-daemon:$PATH"
@@ -76,24 +76,26 @@ if [ -z "$DOCKER_TEST_HOST" ]; then
 		# see https://github.com/docker/libcontainer/blob/master/apparmor/apparmor.go#L16
 		export container=""
 		(
-			set -x
+			[ -n "$TESTDEBUG" ] && set -x
 			/etc/init.d/apparmor start
 		)
 	fi
 
-	export DOCKER_HOST="unix://$(cd "$DEST" && pwd)/docker.sock" # "pwd" tricks to make sure $DEST is an absolute path, not a relative one
-	( set -x; exec \
-		dockerd --debug \
-		--host "$DOCKER_HOST" \
-		--storage-driver "$DOCKER_GRAPHDRIVER" \
-		--pidfile "$DEST/docker.pid" \
-		--userland-proxy="$DOCKER_USERLANDPROXY" \
-		$storage_params \
-		$extra_params \
-			&> "$DEST/docker.log"
+	# "pwd" tricks to make sure $DEST is an absolute path, not a relative one
+	export DOCKER_HOST="unix://$(cd "$DEST" && pwd)/docker.sock"
+	(
+		echo "Starting dockerd"
+		[ -n "$TESTDEBUG" ] && set -x
+		exec \
+			dockerd --debug \
+			--host "$DOCKER_HOST" \
+			--storage-driver "$DOCKER_GRAPHDRIVER" \
+			--pidfile "$DEST/docker.pid" \
+			--userland-proxy="$DOCKER_USERLANDPROXY" \
+			$storage_params \
+			$extra_params \
+				&> "$DEST/docker.log"
 	) &
-	# make sure that if the script exits unexpectedly, we stop this daemon we just started
-	trap 'bundle .integration-daemon-stop' EXIT
 else
 	export DOCKER_HOST="$DOCKER_TEST_HOST"
 fi

hack/make/.integration-daemon-stop

@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 
 if [ ! "$(go env GOOS)" = 'windows' ]; then
-	trap - EXIT # reset EXIT trap applied in .integration-daemon-start
-
 	for pidFile in $(find "$DEST" -name docker.pid); do
-		pid=$(set -x; cat "$pidFile")
-		( set -x; kill "$pid" )
+		pid=$([ -n "$TESTDEBUG" ] && set -x; cat "$pidFile")
+		(
+			[ -n "$TESTDEBUG" ] && set -x
+			kill "$pid"
+		)
 		if ! wait "$pid"; then
 			echo >&2 "warning: PID $pid from $pidFile had a nonzero exit code"
 		fi
@@ -15,7 +16,7 @@ if [ ! "$(go env GOOS)" = 'windows' ]; then
 		# Stop apparmor if it is enabled
 		if [ -e "/sys/module/apparmor/parameters/enabled" ] && [ "$(cat /sys/module/apparmor/parameters/enabled)" == "Y" ]; then
 			(
-				set -x
+				[ -n "$TESTDEBUG" ] && set -x
 				/etc/init.d/apparmor stop
 			)
 		fi

hack/make/.integration-test-helpers

@@ -1,65 +1,72 @@
 #!/usr/bin/env bash
-
-: ${TEST_REPEAT:=0}
-
-bundle_test_integration_cli() {
-	TESTFLAGS="$TESTFLAGS -check.v -check.timeout=${TIMEOUT} -test.timeout=360m"
-	go_test_dir integration-cli $DOCKER_INTEGRATION_TESTS_VERIFIED
-}
-
-# If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'.
-# You can use this to select certain tests to run, e.g.
-#
-#     TESTFLAGS='-test.run ^TestBuild$' ./hack/make.sh test-unit
 #
 # For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want
 # to run certain tests on your local host, you should run with command:
 #
-#     TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration-cli
+#     TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration
 #
-go_test_dir() {
-	dir=$1
-	precompiled=$2
-	testbinary="$ABS_DEST/test.main"
-	testcover=()
-	testcoverprofile=()
+
+source "$SCRIPTDIR/make/.go-autogen"
+
+: ${TEST_REPEAT:=1}
+
+integration_api_dirs=("$(
+	find ./integration -type d |
+	grep -vE '^(./integration$|./integration/util)')")
+
+run_test_integration() {
+	local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS"
+	for dir in $integration_api_dirs; do
+		(
+			cd $dir
+			echo "Running $PWD"
+			test_env ./test.main $flags
+		)
+	done
+
 	(
-		set -e
-		mkdir -p "$DEST/coverprofiles"
-		export DEST="$ABS_DEST" # in a subshell this is safe -- our integration-cli tests need DEST, and "cd" screws it up
-		if [ -z $precompiled ]; then
-			ensure_test_dir $1 $testbinary
-		fi
-		cd "$dir"
-		i=0
-		while ((++i)); do
-			test_env "$testbinary" $TESTFLAGS
-			if [ $i -gt "$TEST_REPEAT" ]; then
-				break
-			fi
-			echo "Repeating test ($i)"
-		done
+		flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS"
+		cd integration-cli
+		echo "Running $PWD"
+		test_env ./test.main $flags
 	)
 }
 
-ensure_test_dir() {
-	(
-		# make sure a test dir will compile
-		dir="$1"
-		out="$2"
-		echo Building test dir: "$dir"
-		set -xe
-		cd "$dir"
-		go test -c -o "$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}"
-	)
+build_test_suite_binaries() {
+	build_test_suite_binary ./integration-cli "test.main"
+	for dir in $integration_api_dirs; do
+		build_test_suite_binary "$dir" "test.main"
+	done
+}
+
+# Build a binary for a test suite package
+build_test_suite_binary() {
+	local dir="$1"
+	local out="$2"
+	echo Building test suite binary "$dir/$out"
+	go test -c -o "$dir/$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" "$dir"
 }
 
+cleanup_test_suite_binaries() {
+	[ -n "$TESTDEBUG" ] && return
+	echo "Removing test suite binaries"
+	find integration* -name test.main | xargs -r rm
+}
+
+repeat() {
+	for i in $(seq 1 $TEST_REPEAT); do
+		echo "Running integration-test (iteration $i)"
+		$@
+	done
+}
+
+# use "env -i" to tightly control the environment variables that bleed into the tests
 test_env() {
 	(
-		set -xe
-		# use "env -i" to tightly control the environment variables that bleed into the tests
+		set -e
+		[ -n "$TESTDEBUG" ] && set -x
 		env -i \
-			DEST="$DEST" \
+			DEST="$ABS_DEST" \
 			DOCKER_CLI_VERSION="$DOCKER_CLI_VERSION" \
 			DOCKER_API_VERSION="$DOCKER_API_VERSION" \
 			DOCKER_INTEGRATION_DAEMON_DEST="$DOCKER_INTEGRATION_DAEMON_DEST" \
@@ -82,3 +89,19 @@ test_env() {
 			"$@"
 	)
 }
+
+
+error_on_leaked_containerd_shims() {
+	if [ "$(go env GOOS)" == 'windows' ]; then
+		return
+	fi
+
+	leftovers=$(ps -ax -o pid,cmd |
+	            awk '$2 == "docker-containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration/ { print $1 }')
+	if [ -n "$leftovers" ]; then
+		ps aux
+		kill -9 $leftovers 2> /dev/null
+		echo "!!!! WARNING you have left over shim(s), Cleanup your test !!!!"
+		exit 1
+	fi
+}