CI/build adjustments (istio#3599)

Automatic merge from submit-queue. CI/build adjustments - pull master - go 1.10 was released - pilot tests capturing junit-style test result (thanks @nmittler for converting to std tests) - pilot test also using the recommended release style ( istio-system ) and fixed ns so it's easy to re-run and test after (only used in circleci, new target). The new target also works manually. - adjusted the pilot labels to allow upgrade tests ( will install istio.yaml first ). Currently pilot tests are not compatible with normal istio configs.
jinglinxu · Feb 21, 2018 · 3cd34d8 · 3cd34d8
1 parent 4c6004d
commit 3cd34d8
Show file tree

Hide file tree

Showing 9 changed files with 132 additions and 43 deletions.
diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile
@@ -1,4 +1,4 @@
-FROM    circleci/golang:1.9
+FROM    circleci/golang:1.10
 
 # The base circleci image runs as user 'circleci'(3434), with sudo capabilities.
 # Based on Debian9. Go installed in /usr/local/go

diff --git a/.circleci/Makefile b/.circleci/Makefile
@@ -1,5 +1,5 @@
 # Builder version
-CI_VERSION ?= go1.9-k8s1.9-helm2.7.2-minikube0.25
+CI_VERSION ?= go1.10-k8s1.9-helm2.7.2-minikube0.25
 CI_HUB ?= istio
 
 ci.image:

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -4,7 +4,7 @@ version: 2
 defaults: &defaults
   working_directory: /go/src/istio.io/istio
   docker:
-    - image: istio/ci:go1.9-k8s1.9-helm2.7.2-minikube0.25
+    - image: istio/ci:go1.10-k8s1.9-helm2.7.2-minikube0.25
   environment:
     GOPATH: /go
 
@@ -156,6 +156,7 @@ jobs:
           sudo chown -R circleci /go
           mkdir -p /home/circleci/logs
       - checkout
+      - run: make git.pullmaster
       - run: make submodule-sync
       - attach_workspace:
           at:  /go
@@ -169,9 +170,31 @@ jobs:
             make docker.tag generate_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
-      - run: make e2e_pilot HUB="${HUB}" TAG="${TAG}" TESTOPTS="--logtostderr --skip-cleanup -mixer=true -auth=enable -errorlogsdir=/home/circleci/logs -use-sidecar-injector=false --core-files-dir=/home/circleci/logs"
+      - run:
+          command: |
+            mkdir -p /go/out/tests
+            trap "/go/bin/go-junit-report </go/out/tests/go-test-report.out > /go/out/tests/go-test-report.xml" EXIT
+            # Run the tests in env similar with recommended setup.
+            make test/minikube/auth/e2e_pilot HUB="${HUB}" TAG="${TAG}" \
+               | tee -a /go/out/tests/go-test-report.out
+      - run:
+          name: dumpsys
+          when: always
+          command: |
+            # TODO: move to a make target 'dumpsys'.
+            kubectl get all -o wide --all-namespaces
+            kubectl cluster-info dump > /go/out/logs/cluster-info.dump.txt
+            kubectl describe pods -n istio-system > /go/out/logs/pods-system.txt
+            kubectl describe pods -n istio-test > /go/out/logs/pods-test.txt
       - store_artifacts:
-          path: /home/circleci/logs
+          path: /go/out/tests
+      - store_artifacts:
+          path: /go/out/logs
+      - store_artifacts:
+          path: /tmp
+      - store_test_results:
+          path: /go/out/tests
+
 
   e2e-pilot-noauth:
     <<: *integrationDefaults
@@ -265,12 +288,25 @@ jobs:
           path: vendor/dep.dot
       # TODO: auto-commit Gopkg.lock if test is successful
 
+  # Runs first, will download any dependencies - currently envoy, and do any work
+  # that can be shared between build/test/lint.
   dependencies:
     <<: *defaults
     steps:
       - checkout
+      - run: make git.pullmaster
       - run: make submodule-sync
       - run: make init
+      - run:
+            command: |
+              date '+%Y-%W-%D' > /go/out/cache.stamp
+
+      - persist_to_workspace:
+          root: /go
+          paths:
+            - src
+            - bin
+            - out
 
   codecov:
     <<: *defaults
@@ -309,8 +345,14 @@ jobs:
       KUBECONFIG: /go/src/istio.io/istio/.circleci/config
     resource_class: xlarge
     steps:
-      - checkout
-      - run: make submodule-sync
+      - attach_workspace:
+          at:  /go
+      - restore_cache:
+          keys:
+            - test_cache_{{ checksum "/go/out/cache.stamp" }}
+      #- checkout
+      #- run: make git.pullmaster
+      #- run: make submodule-sync
       - run:
           command: |
             mkdir -p /go/out/tests
@@ -319,6 +361,11 @@ jobs:
             make test T=-v | tee -a /go/out/tests/go-test-report.out
       - store_test_results:
           path: /go/out/tests
+      - save_cache:
+          key: test_cache_{{ checksum "/go/out/cache.stamp" }}
+          paths:
+            - /go/pkg
+
 
   racetest:
     <<: *defaults
@@ -356,8 +403,14 @@ jobs:
     <<: *defaults
     resource_class: xlarge
     steps:
-      - checkout
-      - run: make submodule-sync
+      - attach_workspace:
+          at:  /go
+      - restore_cache:
+          keys:
+            - build_cache_{{ checksum "/go/out/cache.stamp" }}
+      #- checkout
+      #- run: make git.pullmaster
+      #- run: make submodule-sync
       - run:
           command: |
             cd /go/src/istio.io/istio
@@ -370,6 +423,10 @@ jobs:
             - bin
             - out
       - run:  make sidecar.deb
+      - save_cache:
+          key: build_cache_{{ checksum "/go/out/cache.stamp" }}
+          paths:
+            - /go/pkg
       - store_artifacts:
           path: /go/out/istio-sidecar.deb
 
@@ -379,28 +436,19 @@ jobs:
       KUBECONFIG: /go/src/istio.io/istio/.circleci/config
     resource_class: xlarge
     steps:
-      - checkout
-      - run: make submodule-sync
-      - run: make lint
+      - attach_workspace:
+          at:  /go
+      - restore_cache:
+          keys:
+            - lint_cache_{{ checksum "/go/out/cache.stamp" }}
 
-  ## TODO DISBALE ME and use the container based lint
-  # lint:
-  #   <<: *integrationDefaults
-  #   steps:
-  #     - type: shell
-  #       name: Initialize Working Directory
-  #       pwd: /
-  #       command: |
-  #         sudo mkdir -p /go/src/istio.io/istio
-  #         sudo chown -R circleci /go
-  #     - checkout
-  #     - restore_cache:
-  #         keys:
-  #           - dep-cache-{{ checksum "Gopkg.lock" }}
-  #     - run:
-  #         no_output_timeout: 15m
-  #         command: |
-  #           make lint
+      #- checkout
+      #- run: make submodule-sync
+      - run: make lint
+      - save_cache:
+          key: lint_cache_{{ checksum "/go/out/cache.stamp" }}
+          paths:
+            - /go/pkg
 
   docker-push:
     <<: *defaults

diff --git a/.gitignore b/.gitignore
@@ -48,3 +48,4 @@ install/kubernetes/istio-sidecar-injector.yaml
 install/kubernetes/istio.yaml
 samples/bookinfo/consul/bookinfo.sidecars.yaml
 samples/bookinfo/eureka/bookinfo.sidecars.yaml
+istio.VERSION
diff --git a/Makefile b/Makefile
@@ -100,6 +100,7 @@ GO_FILES_CMD := find . -name '*.go' | grep -v -E '$(GO_EXCLUDE)'
 
 export ISTIO_BIN=$(GO_TOP)/bin
 # Using same package structure as pkg/
+export OUT_DIR=$(GO_TOP)/out
 export ISTIO_OUT:=$(GO_TOP)/out/$(GOOS)_$(GOARCH)/$(BUILDTYPE_DIR)
 
 # scratch dir: this shouldn't be simply 'docker' since that's used for docker.save to store tar.gz files
@@ -187,7 +188,14 @@ pull:
 	git pull
 	$(MAKE) submodule-sync
 
-.PHONY: submodule pull submodule-sync
+# Merge master. To be used in CI or by developers, assumes the
+# remote is called 'origin' (git default). Will fail on conflicts
+# Note: in a branch, this will get the latest from master. In master it has no effect.
+# This should be run after a 'git fetch' (typically done in the checkout step in CI)
+git.pullmaster:
+	git merge master
+
+.PHONY: submodule pull submodule-sync git.pullmaster
 
 # I tried to make this dependent on what I thought was the appropriate
 # lock file, but it caused the rule for that file to get run (which
@@ -304,7 +312,7 @@ $(MIXER_GO_BINS):
 	bin/gobuild.sh $@ istio.io/istio/pkg/version ./mixer/cmd/$(@F)
 
 servicegraph:
-	bin/gobuild.sh $@ istio.io/istio/pkg/version ./addons/servicegraph/cmd/server
+	bin/gobuild.sh ${ISTIO_OUT}/$@ istio.io/istio/pkg/version ./addons/servicegraph/cmd/server
 
 ${ISTIO_OUT}/servicegraph:
 	bin/gobuild.sh $@ istio.io/istio/pkg/version ./addons/$(@F)/cmd/server
@@ -338,7 +346,7 @@ pilot: pilot-discovery
 
 .PHONY: multicluster_ca node_agent istio_ca
 multicluster_ca node_agent istio_ca:
-	bin/gobuild.sh $@ istio.io/istio/pkg/version ./security/cmd/$(@F)
+	bin/gobuild.sh ${ISTIO_OUT}/$@ istio.io/istio/pkg/version ./security/cmd/$(@F)
 
 # istioctl-all makes all of the non-static istioctl executables for each supported OS
 .PHONY: istioctl-all

diff --git a/bin/init.sh b/bin/init.sh
@@ -68,16 +68,16 @@ fi
 PROXYVERSION=$(grep envoy-debug pilot/docker/Dockerfile.proxy_debug  |cut -d: -f2)
 PROXY=debug-$PROXYVERSION
 
-# Save envoy in vendor, which is cached
-if [ ! -f vendor/envoy-$PROXYVERSION ] ; then
+# Save envoy in $out
+if [ ! -f $OUT/envoy-$PROXYVERSION ] ; then
     mkdir -p $OUT
     pushd $OUT
     # New version of envoy downloaded. Save it to cache, and clean any old version.
 
     DOWNLOAD_COMMAND=""
     if command -v curl > /dev/null; then
-       if curl --version | grep Protocols  | grep https; then
-	   DOWNLOAD_COMMAND='curl -Lo -'
+       if curl --version | grep Protocols  | grep https > /dev/null; then
+	   DOWNLOAD_COMMAND='curl -Lso -'
        else
            echo curl does not support https, will try wget for downloading files.
        fi
@@ -99,21 +99,21 @@ if [ ! -f vendor/envoy-$PROXYVERSION ] ; then
         exit 1
     fi
 
-    echo "Downloading envoy $PROXY $PROXYVERSION using $DOWNLOAD_COMMAND"
-    ${DOWNLOAD_COMMAND} https://storage.googleapis.com/istio-build/proxy/envoy-$PROXY.tar.gz | tar xz
-    cp usr/local/bin/envoy $ISTIO_GO/vendor/envoy-$PROXYVERSION
+    echo "Downloading envoy $PROXY using $DOWNLOAD_COMMAND"
+    time ${DOWNLOAD_COMMAND} https://storage.googleapis.com/istio-build/proxy/envoy-$PROXY.tar.gz | tar xz
+    cp usr/local/bin/envoy $OUT/envoy-$PROXYVERSION
     rm -f ${ISTIO_OUT}/envoy ${ROOT}/pilot/pkg/proxy/envoy/envoy ${ISTIO_BIN}/envoy
     popd
 fi
 
 if [ ! -f ${ISTIO_OUT}/envoy ] ; then
     mkdir -p ${ISTIO_OUT}
     # Make sure the envoy binary exists.
-    cp $ISTIO_GO/vendor/envoy-$PROXYVERSION ${ISTIO_OUT}/envoy
+    cp $OUT/envoy-$PROXYVERSION ${ISTIO_OUT}/envoy
 fi
 
 # circleCI expects this in the bin directory
 if [ ! -f ${ISTIO_BIN}/envoy ] ; then
     mkdir -p ${ISTIO_BIN}
-    cp $ISTIO_GO/vendor/envoy-$PROXYVERSION ${ISTIO_BIN}/envoy
+    cp $OUT/envoy-$PROXYVERSION ${ISTIO_BIN}/envoy
 fi
diff --git a/pilot/test/util/kubernetes.go b/pilot/test/util/kubernetes.go
@@ -16,6 +16,7 @@ package util
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 	// TODO(nmittler): Remove this
@@ -123,6 +124,9 @@ func CopyPodFiles(container, pod, ns, source, dest string) {
 // GetAppPods awaits till all pods are running in a namespace, and returns a map
 // from "app" label value to the pod names.
 func GetAppPods(cl kubernetes.Interface, kubeconfig string, nslist []string) (map[string][]string, error) {
+	// TODO: clean and move this method to top level, 'AwaitPods' or something similar,
+	// merged with the similar method used by the other tests. Eventually make it part of
+	// istioctl or a similar helper.
 	pods := make(map[string][]string)
 	var items []v1.Pod
 
@@ -138,6 +142,14 @@ func GetAppPods(cl kubernetes.Interface, kubeconfig string, nslist []string) (ma
 			ready := true
 
 			for _, pod := range items {
+				// Exclude pods that may be in non-running state when helm is used to
+				// initialize.
+				if strings.HasPrefix(pod.Name, "istio-mixer-create") {
+					continue
+				}
+				if strings.HasPrefix(pod.Name, "istio-sidecar-injector") {
+					continue
+				}
 				if pod.Status.Phase != "Running" {
 					log.Infof("Pod %s.%s has status %s", pod.Name, ns, pod.Status.Phase)
 					ready = false
@@ -161,6 +173,9 @@ func GetAppPods(cl kubernetes.Interface, kubeconfig string, nslist []string) (ma
 					if app, exists := pod.Labels["app"]; exists {
 						pods[app] = append(pods[app], pod.Name)
 					}
+					if app, exists := pod.Labels["istio"]; exists {
+						pods[app] = append(pods[app], pod.Name)
+					}
 				}
 
 				break
@@ -174,6 +189,7 @@ func GetAppPods(cl kubernetes.Interface, kubeconfig string, nslist []string) (ma
 		}
 	}
 
+	log.Infof("Found pods: %v", pods)
 	return pods, nil
 }
 

diff --git a/tests/e2e/tests/pilot/util/infra.go b/tests/e2e/tests/pilot/util/infra.go
@@ -344,7 +344,7 @@ func (infra *Infra) Setup() error {
 			},
 		})
 		if err != nil {
-			return err
+			log.Warn("Secret already exists")
 		}
 	}
 	if infra.Zipkin {

diff --git a/tests/istio.mk b/tests/istio.mk
@@ -88,3 +88,19 @@ e2e_all: e2e_simple e2e_mixer e2e_bookinfo
 
 e2e_pilot: istioctl generate_yaml
 	go test -v -timeout 20m ./tests/e2e/tests/pilot ${TESTOPTS} -hub ${HUB} -tag ${TAG}
+
+# Target for running e2e pilot in a minikube env. Used by CI
+test/minikube/auth/e2e_pilot: istioctl generate_yaml
+	mkdir -p ${OUT_DIR}/logs
+	kubectl create ns istio-system || true
+	kubectl create ns istio-test || true
+	go test -test.v -timeout 20m ./tests/e2e/tests/pilot -args \
+		-hub ${HUB} -tag ${TAG} \
+		--skip-cleanup --mixer=true --auth=enable \
+		-errorlogsdir=${OUT_DIR}/logs \
+		--use-sidecar-injector=false \
+		--core-files-dir=${OUT_DIR}/logs \
+         --ns istio-system \
+          --logtostderr \
+        -n istio-test \
+           ${TESTOPTS}