CircleCI updates (2600hz#5551)

checkout base branch in circlec script

use CHANGED env var if not empty

fix and update apps cicrleci config

battle of caches

add dash

invalidating deps cache

update app's ci config

copy pasta failed

workflow for apps circleci which runs eunit tests too

.circleci/config.yml

@@ -46,13 +46,20 @@ jobs:
           at: .
       - restore_cache:
           keys:
-            - deps-v2-{{ checksum "make/deps.mk" }}
+            - deps-v3-{{ checksum "make/deps.mk" }}
       - run: make fmt
+      - run: |
+          deps_hash=$(md5sum make/deps.mk | cut -d" " -f1)
+          if [ x"$(cat .git/.kz_deps_hash)" = x"$deps_hash" ]; then
+            touch "make/.deps.mk.$deps_hash"
+          fi
       - run: JOBS="2" make
+      - run: echo -n $(md5sum make/deps.mk | cut -d" " -f1) > .git/.kz_deps_hash
       - save_cache:
-          key: deps-v2-{{ checksum "make/deps.mk" }}
+          key: deps-v3-{{ checksum "make/deps.mk" }}
           paths:
             - deps/
+            - .git/.kz_deps_hash
       - persist_to_workspace:
           root: .
           paths:

.circleci/sequential.yml

@@ -19,15 +19,20 @@ jobs:
       # build sequence
       - restore_cache:
           keys:
-            - v2-dep-{{ .Branch }}-
-            - v2-dep-master-
-            - v2-dep-
+            - deps-v3-{{ checksum "make/deps.mk" }}
       - run: make fmt
+      - run: |
+          deps_hash=$(md5sum make/deps.mk | cut -d" " -f1)
+          if [ x"$(cat .git/.kz_deps_hash)" = x"$deps_hash" ]; then
+            touch "make/.deps.mk.$deps_hash"
+          fi
       - run: JOBS="2" make
+      - run: echo -n $(md5sum make/deps.mk | cut -d" " -f1) > .git/.kz_deps_hash
       - save_cache:
-          key: v2-dep-{{ .Branch }}-{{ epoch }}
+          key: deps-v3-{{ checksum "make/deps.mk" }}
           paths:
-            - .deps
+            - deps/
+            - .git/.kz_deps_hash
 
       # code check sequence
       - run: make code_checks

.circleci/workflows.yml

@@ -46,13 +46,20 @@ jobs:
           at: .
       - restore_cache:
           keys:
-            - deps-v2-{{ checksum "make/deps.mk" }}
+            - deps-v3-{{ checksum "make/deps.mk" }}
       - run: make fmt
+      - run: |
+          deps_hash=$(md5sum make/deps.mk | cut -d" " -f1)
+          if [ x"$(cat .git/.kz_deps_hash)" = x"$deps_hash" ]; then
+            touch "make/.deps.mk.$deps_hash"
+          fi
       - run: JOBS="2" make
+      - run: echo -n $(md5sum make/deps.mk | cut -d" " -f1) > .git/.kz_deps_hash
       - save_cache:
-          key: deps-v2-{{ checksum "make/deps.mk" }}
+          key: deps-v3-{{ checksum "make/deps.mk" }}
           paths:
             - deps/
+            - .git/.kz_deps_hash
       - persist_to_workspace:
           root: .
           paths:

Makefile

@@ -7,7 +7,11 @@ ERLANG_MK_COMMIT = d30dda39b08e6ed9e12b44533889eaf90aba86de
 BASE_BRANCH := $(shell cat $(ROOT)/.base_branch)
 
 ## list files changed for more focused checks
-CHANGED := $(shell git --no-pager diff --name-only HEAD $(BASE_BRANCH) -- applications core scripts)
+ifeq ($(strip $(CHANGED)),)
+	CHANGED := $(shell git --no-pager diff --name-only HEAD $(BASE_BRANCH) -- applications core scripts)
+else
+	CHANGED := $(CHANGED)
+endif
 CHANGED_SWAGGER := $(shell git --no-pager diff --name-only HEAD $(BASE_BRANCH) -- applications/crossbar/priv/api/swagger.json)
 
 # You can override this when calling make, e.g. make JOBS=1
@@ -40,10 +44,10 @@ JOBS ?= 1
 all: compile
 
 changed:
-	@git --no-pager diff --name-only HEAD $(BASE_BRANCH) -- applications core scripts
+	@echo "$(CHANGED)"
 
 changed_swagger:
-	@git --no-pager diff --name-only HEAD $(BASE_BRANCH) -- applications/crossbar/priv/api/swagger.json
+	@echo "$(CHANGED_SWAGGER)"
 
 compile: ACTION = all
 compile: deps kazoo

applications/acdc/.circleci/config.yml

@@ -1,73 +1,198 @@
 version: 2
+
+workflows:
+  version: 2
+  build-analyze-run:
+    jobs:
+      - setup
+      - build_deps:
+          requires:
+            - setup
+      - build:
+          requires:
+            - build_deps
+      - eunit_tests:
+          requires:
+            - build_deps
+      - checks:
+          requires:
+            - build
+      - docs:
+          requires:
+            - build
+      - analyze:
+          requires:
+            - build
+      - release:
+          requires:
+            - build
+
+defaults: &defaults
+  docker:
+    - image: offical2600hz/circleci:19.3.6.13
+  shell: /bin/bash --login
+  working_directory: ~/2600hz/the_app
+  environment:
+    CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+    CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
+    BASH_ENV: "/home/circleci/2600hz/.bashrc"
+
 jobs:
-  build:
-    shell: /bin/bash --login
-    environment:
-      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
-      CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
-      TZ: "/usr/share/zoneinfo/UTC"
-    docker:
-      - image: offical2600hz/circleci:19.3.6.13
-      - image: couchdb:2.1.1
-      - image: rabbitmq:3.7
+
+  # setup build environment sequence
+  setup:
+    <<: *defaults
     steps:
+      - checkout
+      # setup enviroment variable
+      - run: |
+          BAZE="$(cat "${HOME}/2600hz/the_app/.base_branch")"
+          if [ -n "$BAZE" ]; then
+            echo -e "\n\nexport BASE_BRANCH=$BAZE" >> $BASH_ENV
+          else
+            echo "add base branch name of main Kazoo repo (like origin/master) to '.base_branch' in your application root directory"
+            exit 1
+          fi
       - run: |
           APP_DASH=${CIRCLE_PROJECT_REPONAME#kazoo-}
           APP=${APP_DASH/-/_}
           echo -e "export KAZOO_APP=${APP}\n" >> $BASH_ENV
-      - run: echo -e "export KAZOO_ROOT=${HOME}/${CIRCLE_PROJECT_USERNAME}/kazoo\nexport APP_PATH=applications/${KAZOO_APP}\n\n" >> $BASH_ENV
+          echo -e "export KAZOO_ROOT=${HOME}/2600hz/kazoo" >> $BASH_ENV
+          echo -e "export APP_PATH=applications/${APP}\n\n" >> $BASH_ENV
+      - run: |
+          echo ":: behold, running ci tests for application: $KAZOO_APP"
+          echo "KAZOO_ROOT: $KAZOO_ROOT"
+          echo "APP_PATH: $APP_PATH"
+          echo "BASE_BRANCH: $BASE_BRANCH"
+      - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
 
-      # setup build environment sequence
-      - checkout
+      # setup kazoo source code
       - run: |
           if [ ! -d ${KAZOO_ROOT} ]; then
             git clone https://github.com/2600hz/kazoo $KAZOO_ROOT
           fi
-      - run: mkdir -p $CIRCLE_ARTIFACTS
       - run: ${KAZOO_ROOT}/scripts/circleci.bash
-      - run: echo -e 'export CHANGED="$(for file in `find ${KAZOO_ROOT}/applications/${KAZOO_APP} -name "*.[e|h]rl"`; do readlink -e $file; done)"' >> $BASH_ENV
+      - run: echo -e 'export CHANGED="$(for file in `find ${KAZOO_ROOT}/applications/${KAZOO_APP} -name "*.[e|h]rl"`; do readlink -e $file; done | xargs echo)"' >> $BASH_ENV
       - run: echo $CHANGED
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - 2600hz
 
-      # build sequence
+  # build deps sequence
+  build_deps:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/
       - restore_cache:
           keys:
-            - v2-dep-{{ .Branch }}-
-            - v2-dep-master-
-            - v2-dep-
-      - run: TO_FMT="${CHANGED}" make -C ${KAZOO_ROOT} fmt
-      - run: JOBS="2" make -C ${KAZOO_ROOT}
+            - deps-v3.1-{{ checksum "~/2600hz/kazoo/make/deps.mk" }}
+      - run: |
+          deps_hash=$(md5sum $KAZOO_ROOT/make/deps.mk | cut -d" " -f1)
+          if [ x"$(cat $KAZOO_ROOT/.git/.kz_deps_hash)" = x"$deps_hash" ]; then
+            touch "$KAZOO_ROOT/make/.deps.mk.$deps_hash"
+          fi
+      - run: echo -n $(md5sum $KAZOO_ROOT/make/deps.mk | cut -d" " -f1) > "$KAZOO_ROOT/.git/.kz_deps_hash"
+      - run: |
+          md5sum $KAZOO_ROOT/make/deps.mk || true
+          ls -al $KAZOO_ROOT/make || true
+      - run: make -C ${KAZOO_ROOT} deps
       - save_cache:
-          key: v2-dep-{{ .Branch }}-{{ epoch }}
-            paths:
-              - ${KAZOO_ROOT}/.deps
+          key: deps-v3.1-{{ checksum "~/2600hz/kazoo/make/deps.mk" }}
+          paths:
+            - ~/2600hz/kazoo/deps/
+            - ~/2600hz/kazoo/.git/.kz_deps_hash
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - 2600hz
+
+  # build sequence
+  build:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/
+      - restore_cache:
+          keys:
+            - deps-v3.1-{{ checksum "~/2600hz/kazoo/make/deps.mk" }}
+      - run: |
+          md5sum $KAZOO_ROOT/make/deps.mk || true
+          ls -al $KAZOO_ROOT/make || true
+      - run: make -C ${KAZOO_ROOT} fmt
+      - run: JOBS="2" make -C ${KAZOO_ROOT}
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - 2600hz
+
+  # eunit_tests sequence
+  eunit_tests:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/
+      - restore_cache:
+          keys:
+            - deps-v3.1-{{ checksum "~/2600hz/kazoo/make/deps.mk" }}
+      - run: |
+          md5sum $KAZOO_ROOT/make/deps.mk || true
+          ls -al $KAZOO_ROOT/make || true
+      - run: ERLC_OPTS='-DPROPER' make -C ${KAZOO_ROOT}/core compile-test
+      - run: make -C "$KAZOO_ROOT/$APP_PATH" eunit
 
-      # code checks sequence
+  # code checks sequence
+  checks:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/
       - run: make -C ${KAZOO_ROOT} code_checks
       - run: ${KAZOO_ROOT}/scripts/code_checks.bash $CHANGED
       - run: make -C ${KAZOO_ROOT} app_applications
       - run: make -C ${KAZOO_ROOT} sup_completion
       - run: make -C ${KAZOO_ROOT} xref
       - run: make -C ${KAZOO_ROOT} elvis
 
-      # docs sequence
+  # docs sequence
+  docs:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/
       - run: make -C ${KAZOO_ROOT} apis
       - run: make -C ${KAZOO_ROOT} validate-js
       - run: make -C ${KAZOO_ROOT} validate-schemas
       - run: make -C ${KAZOO_ROOT} docs || true
       - run: ${KAZOO_ROOT}/scripts/state-of-docs.sh || true
       - run: ${KAZOO_ROOT}/scripts/state-of-edoc.escript
 
-      # analyze sequence
+  # analyze sequence
+  analyze:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/
       - restore_cache:
           keys:
-            - plt-v2-{{ checksum "~/2600hz/kazoo/make/deps.mk" }}
+            - plt-v3-{{ checksum "~/2600hz/kazoo/make/deps.mk" }}
       - run: TO_DIALYZE="$(echo $CHANGED)" make -C ${KAZOO_ROOT} build-plt dialyze
       - save_cache:
-          key: plt-v2-{{ checksum "~/2600hz/kazoo/make/deps.mk" }}
+          key: plt-v3-{{ checksum "~/2600hz/kazoo/make/deps.mk" }}
           paths:
-            - ${KAZOO_ROOT}/.kazoo.plt
+            - ~/2600hz/kazoo/.kazoo.plt
 
-      # release sequence
+  # release sequence
+  release:
+    <<: *defaults
+    docker:
+      - image: offical2600hz/circleci:19.3.6.13
+      - image: couchdb:2.1.1
+      - image: rabbitmq:3.7
+    steps:
+      - attach_workspace:
+          at: ~/
       - run: make -C ${KAZOO_ROOT} build-ci-release
       - run: ${KAZOO_ROOT}/scripts/check-unstaged.bash
       - run: KAZOO_CONFIG=${KAZOO_ROOT}/rel/ci.config.ini REL="kazoo_apps" ACT="console" NODE_NAME_TYPE="-sname" make -C ${KAZOO_ROOT} release