Final optimisations (argoproj#2486)

tkasuz · Oct 15, 2019 · 549503c · 549503c
1 parent ae959ec
commit 549503c
Show file tree

Hide file tree

Showing 11 changed files with 155 additions and 211 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,14 +1,5 @@
 version: 2.1
 commands:
-  before:
-    steps:
-      - restore_go_cache
-      - install_golang
-      - clean_checkout
-      - install_tools
-      - configure_git
-      - install_go_deps
-      - dep_ensure
   configure_git:
     steps:
       - run:
@@ -20,10 +11,6 @@ commands:
             git config --global user.name "Your Name"
             echo "export PATH=/home/circleci/.go_workspace/src/github.com/argoproj/argo-cd/hack:\$PATH" | tee -a $BASH_ENV
             echo "export GIT_ASKPASS=git-ask-pass.sh" | tee -a $BASH_ENV
-  clean_checkout:
-    steps:
-      - run: rm -Rf /home/circleci/.go_workspace/src/github.com/argoproj/argo-cd
-      - checkout
   dep_ensure:
     steps:
       - restore_cache:
@@ -45,71 +32,10 @@ commands:
             [ -e /home/circleci/sdk/go1.12.6 ] || go1.12.6 download
             echo "export GOPATH=/home/circleci/.go_workspace" | tee -a $BASH_ENV
             echo "export PATH=/home/circleci/sdk/go1.12.6/bin:\$PATH" | tee -a $BASH_ENV
-      - run:
-          name: Golang diagnostics
-          command: |
-            env
-            which go
-            go version
-            go env
-  install_go_deps:
-    steps:
-      - run:
-          name: Install Go deps
-          command: go get github.com/jstemmer/go-junit-report
-  install_tools:
-    steps:
-      - run:
-          name: Create downloads dir
-          command: mkdir -p /tmp/dl
-      - restore_cache:
-          keys:
-            - dl-v7
-      - run:
-          name: Install Kubectl v1.14.0
-          command: |
-            set -x
-            [ -e /tmp/dl/kubectl ] || curl -sLf -C - -o /tmp/dl/kubectl https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl
-            sudo cp /tmp/dl/kubectl /usr/local/bin/kubectl
-            sudo chmod +x /usr/local/bin/kubectl
-      - run:
-          name: Install Kubectx v0.6.3
-          command: |
-            set -x
-            [ -e /tmp/dl/kubectx.zip ] || curl -sLf -C - -o /tmp/dl/kubectx.zip https://github.com/ahmetb/kubectx/archive/v0.6.3.zip
-            sudo unzip /tmp/dl/kubectx.zip kubectx-0.6.3/kubectx
-            sudo unzip /tmp/dl/kubectx.zip kubectx-0.6.3/kubens
-            sudo mv kubectx-0.6.3/kubectx /usr/local/bin/
-            sudo mv kubectx-0.6.3/kubens /usr/local/bin/
-            sudo chmod +x /usr/local/bin/kubectx
-            sudo chmod +x /usr/local/bin/kubens
-      - run:
-          name: Install Dep v0.5.3
-          command: |
-            set -x
-            [ -e /tmp/dl/dep ] || curl -sLf -C - -o /tmp/dl/dep https://github.com/golang/dep/releases/download/v0.5.3/dep-linux-amd64
-            sudo cp /tmp/dl/dep /usr/local/go/bin/dep
-            sudo chmod +x /usr/local/go/bin/dep
-            dep version
-      - run:
-          name: Install Ksonnet v0.13.1
-          command: |
-            set -x
-            [ -e /tmp/dl/ks.tar.gz ] || curl -sLf -C - -o /tmp/dl/ks.tar.gz https://github.com/ksonnet/ksonnet/releases/download/v0.13.1/ks_0.13.1_linux_amd64.tar.gz
-            tar -C /tmp -xf /tmp/dl/ks.tar.gz
-            sudo cp /tmp/ks_0.13.1_linux_amd64/ks /usr/local/go/bin/ks
-            sudo chmod +x /usr/local/go/bin/ks
-            ks version
-      - run: sudo ./hack/install.sh helm-linux && helm init --client-only
-      - run: sudo ./hack/install.sh kustomize-linux
-      - save_cache:
-          key: dl-v7
-          paths:
-            - /tmp/dl
   save_go_cache:
     steps:
       - save_cache:
-          key: go-v17-{{ .Branch }}
+          key: go-v18-{{ .Branch }}
           paths:
             - /home/circleci/.go_workspace
             - /home/circleci/.cache/go-build
@@ -118,34 +44,26 @@ commands:
     steps:
       - restore_cache:
           keys:
+            - go-v18-{{ .Branch }}
+            - go-v18-master
             - go-v17-{{ .Branch }}
             - go-v17-master
-            - go-v16-{{ .Branch }}
-            - go-v16-master
 jobs:
-  build:
-    working_directory: /home/circleci/.go_workspace/src/github.com/argoproj/argo-cd
-    machine:
-      image: circleci/classic:201808-01
+  codegen:
+    docker:
+      - image: circleci/golang:1.12
+    working_directory: /go/src/github.com/argoproj/argo-cd
     steps:
-      - before
-      - run:
-          name: Run unit tests
-          command: |
-            set -x
-            mkdir -p /tmp/test-results
-            trap "go-junit-report </tmp/test-results/go-test.out > /tmp/test-results/go-test-report.xml" EXIT
-            make test | tee /tmp/test-results/go-test.out
-      - save_go_cache
-      - run:
-          name: Uploading code coverage
-          command: bash <(curl -s https://codecov.io/bash) -f coverage.out
-          # This takes 2m, lets background it.
-          background: true
-      - store_test_results:
-          path: /tmp/test-results
+      - checkout
+      - restore_cache:
+          keys: [codegen-v2]
       - run: ./hack/install.sh codegen-go-tools
       - run: sudo ./hack/install.sh codegen-tools
+      - run: dep ensure
+      - save_cache:
+          key: codegen-v2
+          paths: [vendor, /tmp/dl, /go/pkg]
+      - run: helm init --client-only
       - run: make codegen-local
       - run:
           name: Check nothing has changed
@@ -156,13 +74,41 @@ jobs:
             # When this fails, it will, create a patch file you can apply locally to fix it.
             # To troubleshoot builds: https://argoproj.github.io/argo-cd/developer-guide/ci/
             git diff --exit-code -- . ':!Gopkg.lock'  ':!assets/swagger.json' | tee codegen.patch
+  test:
+    working_directory: /home/circleci/.go_workspace/src/github.com/argoproj/argo-cd
+    machine:
+      image: circleci/classic:201808-01
+    steps:
+      - restore_go_cache
+      - install_golang
+      - checkout
+      - restore_cache:
+          key: test-dl-v1
+      - run: sudo ./hack/install.sh kubectl-linux kubectx-linux dep-linux ksonnet-linux helm-linux kustomize-linux
+      - save_cache:
+          key: test-dl-v1
+          paths: [/tmp/dl]
+      - configure_git
+      - run: go get github.com/jstemmer/go-junit-report
+      - dep_ensure
+      - save_go_cache
+      - run: make test
+      - run:
+          name: Uploading code coverage
+          command: bash <(curl -s https://codecov.io/bash) -f coverage.out
+      - store_test_results:
+          path: test-results
       - store_artifacts:
-          path: codegen.patch
-          when: always
+          path: test-results
+          destination: .
   e2e:
     working_directory: /home/circleci/.go_workspace/src/github.com/argoproj/argo-cd
     machine:
       image: circleci/classic:201808-01
+    environment:
+      ARGOCD_FAKE_IN_CLUSTER: "true"
+      ARGOCD_SSH_DATA_PATH: "/tmp/argo-e2e/app/config/ssh"
+      ARGOCD_TLS_DATA_PATH: "/tmp/argo-e2e/app/config/tls"
     steps:
       - run:
           name: Install and start K3S v0.5.0
@@ -174,26 +120,31 @@ jobs:
           environment:
             INSTALL_K3S_EXEC: --docker
             INSTALL_K3S_VERSION: v0.5.0
-      - before
-      - run:
-          # do this before we build everything else in the background, as they tend to explode
-          name: Make CLI
-          command: |
-            set -x
-            make cli
-            # must be added to path for tests
-            echo export PATH="`pwd`/dist:\$PATH" | tee -a $BASH_ENV
+      - restore_go_cache
+      - install_golang
+      - checkout
+      - restore_cache:
+          keys: [e2e-dl-v1]
+      - run: sudo ./hack/install.sh kubectx-linux dep-linux ksonnet-linux helm-linux kustomize-linux
+      - run: go get github.com/jstemmer/go-junit-report
+      - save_cache:
+          key: e2e-dl-v10
+          paths: [/tmp/dl]
+      - dep_ensure
+      - configure_git
+      - run: make cli
       - run:
           name: Create namespace
           command: |
             set -x
-            cat /etc/rancher/k3s/k3s.yaml | sed "s/localhost/`hostname`/" | tee ~/.kube/config
-            echo "127.0.0.1 `hostname`" | sudo tee -a /etc/hosts
             kubectl create ns argocd-e2e
             kubens argocd-e2e
             # install the certificates (not 100% sure we need this)
             sudo cp /var/lib/rancher/k3s/server/tls/token-ca.crt /usr/local/share/ca-certificates/k3s.crt
             sudo update-ca-certificates
+            # create the kubecfg, again - not sure we need this
+            cat /etc/rancher/k3s/k3s.yaml | sed "s/localhost/`hostname`/" | tee ~/.kube/config
+            echo "127.0.0.1 `hostname`" | sudo tee -a /etc/hosts
       - run:
           name: Apply manifests
           command: kustomize build test/manifests/base | kubectl apply -f -
@@ -205,98 +156,57 @@ jobs:
           name: Start repo server
           command: go run ./cmd/argocd-repo-server/main.go --loglevel debug --redis localhost:6379
           background: true
-          environment:
-            # pft. if you do not quote "true", CircleCI turns it into "1", stoopid
-            ARGOCD_FAKE_IN_CLUSTER: "true"
-            ARGOCD_SSH_DATA_PATH: "/tmp/argo-e2e/app/config/ssh"
-            ARGOCD_TLS_DATA_PATH: "/tmp/argo-e2e/app/config/tls"
       - run:
           name: Start API server
           command: go run ./cmd/argocd-server/main.go --loglevel debug --redis localhost:6379 --insecure --dex-server http://localhost:5556 --repo-server localhost:8081 --staticassets ../argo-cd-ui/dist/app
           background: true
-          environment:
-            ARGOCD_FAKE_IN_CLUSTER: "true"
-            ARGOCD_SSH_DATA_PATH: "/tmp/argo-e2e/app/config/ssh"
-            ARGOCD_TLS_DATA_PATH: "/tmp/argo-e2e/app/config/tls"
-            ARGOCD_ZJWT_FEATURE_FLAG: "always"
       - run:
           name: Start Test Git
           command: |
             test/fixture/testrepos/start-git.sh
           background: true
-      - run:
-          name: Wait for API server
-          command: |
-            set -x
-            until curl -v http://localhost:8080/healthz; do sleep 3; done
+      - run: until curl -v http://localhost:8080/healthz; do sleep 10; done
       - run:
           name: Start controller
           command: go run ./cmd/argocd-application-controller/main.go --loglevel debug --redis localhost:6379 --repo-server localhost:8081 --kubeconfig ~/.kube/config
           background: true
-          environment:
-            ARGOCD_FAKE_IN_CLUSTER: "true"
-      - run:
-          name: Smoke test
-          command: |
-            set -x
-            argocd login localhost:8080 --plaintext  --username admin --password password
-            argocd app create guestbook --dest-namespace default --dest-server https://kubernetes.default.svc --repo https://github.com/argoproj/argocd-example-apps.git --path guestbook
-            argocd app sync guestbook
-            argocd app delete guestbook
       - run:
-          name: Run e2e tests
-          command: |
-            set -x
-            mkdir -p /tmp/test-results
-            trap "go-junit-report </tmp/test-results/go-e2e.out > /tmp/test-results/go-e2e-report.xml" EXIT
-            make test-e2e | tee /tmp/test-results/go-e2e.out
+          command: PATH=dist:$PATH make test-e2e
           environment:
             ARGOCD_OPTS: "--server localhost:8080 --plaintext"
             ARGOCD_E2E_EXPECT_TIMEOUT: "30"
             ARGOCD_E2E_K3S: "true"
       - store_test_results:
-          path: /tmp/test-results
+          path: test-results
+      - store_artifacts:
+          path: test-results
+          destination: .
   ui:
-    # note that we checkout the code in ~/argo-cd/, but then work in ~/argo-cd/ui
-    working_directory: ~/argo-cd/ui
     docker:
       - image: node:11.15.0
+    working_directory: ~/argo-cd/ui
     steps:
       - checkout:
           path: ~/argo-cd/
       - restore_cache:
-          name: Restore Yarn Package Cache
           keys:
-            - yarn-packages-v3-{{ checksum "yarn.lock" }}
-      - run:
-          name: Install
-          command:
-            yarn install --frozen-lockfile --ignore-optional --non-interactive
+            - yarn-packages-v4-{{ checksum "yarn.lock" }}
+      - run: yarn install --frozen-lockfile --ignore-optional --non-interactive
       - save_cache:
-          name: Save Yarn Package Cache
-          key: yarn-packages-v3-{{ checksum "yarn.lock" }}
-          paths:
-            - ~/.cache/yarn
-            - node_modules
-      - run:
-          name: Test
-          command: yarn test
-      # This does not appear to work, and I don't want to spend time on it.
-      - store_test_results:
-          path: junit.xml
-      - run:
-          name: Build
-          command: yarn build
-      - run:
-          name: Lint
-          command: yarn lint
+          key: yarn-packages-v4-{{ checksum "yarn.lock" }}
+          paths: [~/.cache/yarn, node_modules]
+      - run: yarn test
+      - run: yarn build
+      - run: yarn lint
 workflows:
   version: 2
   workflow:
     jobs:
-      - build
-      - e2e
+      - test
+      - codegen:
+          requires:
+            - test
       - ui:
-          # this isn't strictly true, we just put in here so that we 2/4 executors rather than 3/4
           requires:
-            - build
+            - codegen
+      - e2e
diff --git a/.codecov.yml b/.codecov.yml
@@ -6,13 +6,11 @@ ignore:
 - "**/*_test.go"
 - "pkg/apis/client/.*"
 - "pkg/client/.*"
-- "test/.*"
+- "vendor/.*"
 coverage:
   status:
-    patch:
-      default:
-        # allow test coverage to drop by 10%, assume that it's typically due to CI problems
-        threshold: 10
+    # we've found this not to be useful
+    patch: off
     project:
       default:
         # allow test coverage to drop by 1%, assume that it's typically due to CI problems

diff --git a/.gitignore b/.gitignore
@@ -9,3 +9,4 @@ site/
 cmd/**/debug
 debug.test
 coverage.out
+test-results
diff --git a/Makefile b/Makefile
@@ -170,15 +170,11 @@ build:
 
 .PHONY: test
 test:
-	go test -v -covermode=count -coverprofile=coverage.out `go list ./... | grep -v "test/e2e"`
-
-.PHONY: cover
-cover:
-	go tool cover -html=coverage.out
+	 ./hack/test.sh -coverprofile=coverage.out `go list ./... | grep -v 'test/e2e'`
 
 .PHONY: test-e2e
-test-e2e: cli
-	go test -v -timeout 15m ./test/e2e
+test-e2e:
+	./hack/test.sh -timeout 15m ./test/e2e
 
 .PHONY: start-e2e
 start-e2e: cli