Refactor CI config, require lockfile to be up to date (palantir#4667)

.circleci/config.yml

@@ -1,167 +1,184 @@
 version: 2
+
+aliases:
+    - &docker-node-latest
+        - image: circleci/node:latest
+
+    - &docker-node-lts
+        - image: circleci/node:lts
+
+    - &docker-node-8
+        - image: circleci/node:8
+
+    - &docker-node-6
+        - image: circleci/node:6
+
+    - &restore-node-modules-cache
+        name: Restore node_modules cache
+        key: v1-yarn-deps-{{ checksum "yarn.lock" }}
+
+    - &restore-yarn-cache
+        name: Restore yarnpkg cache
+        key: v1-yarn-cache
+
+    - &save-node-modules-cache
+        name: Save node_modules cache
+        paths:
+            - node_modules
+        key: v1-yarn-deps-{{ checksum "yarn.lock" }}
+
+    - &save-yarn-cache
+        name: Save yarnpkg cache
+        paths:
+            - ~/.cache/yarn
+        key: v1-yarn-cache
+
 jobs:
-    build:
-        docker:
-            - image: circleci/node:latest
+    checkout-code:
+        docker: *docker-node-lts
         steps:
             - checkout
-            - restore_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
+            - restore_cache: *restore-yarn-cache
+            - restore_cache: *restore-node-modules-cache
+            - run: yarn install --non-interactive --cache-folder ~/.cache/yarn
             - run:
-                  name: Install dependencies
-                  command: yarn
-            - save_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
-                  paths:
-                      - node_modules
+                name: Check if yarn.lock changed during install
+                command: git diff --exit-code
+            - save_cache: *save-node-modules-cache
+            - save_cache: *save-yarn-cache
+
+    clean-lockfile:
+        docker: *docker-node-lts
+        steps:
+            - checkout
+            - restore_cache: *restore-node-modules-cache
+            - run: ./scripts/verifyCleanLockfile.sh
+
+    build:
+        docker: *docker-node-lts
+        steps:
+            - checkout
+            - restore_cache: *restore-node-modules-cache
+            - attach_workspace: { at: '.' }
             - run: yarn compile
             - persist_to_workspace:
-                  root: "."
-                  paths:
-                      - lib
-                      - build
-                      - yarn.lock
+                root: "."
+                paths: [lib, build]
+
     lint:
-        docker:
-            - image: circleci/node:latest
+        docker: *docker-node-lts
         steps:
             - checkout
-            - attach_workspace:
-                  at: "."
-            - restore_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
+            - restore_cache: *restore-node-modules-cache
+            - attach_workspace: { at: '.' }
             - run: yarn lint
+
     test:
-        docker:
-            - image: circleci/node:8
+        docker: *docker-node-8
         steps:
             - checkout
-            - attach_workspace:
-                  at: "."
-            - restore_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
+            - restore_cache: *restore-node-modules-cache
+            - attach_workspace: { at: '.' }
             - run: yarn test
+
     test2.1:
-        docker:
-            - image: circleci/node:6
+        docker: *docker-node-6
         steps:
             - checkout
-            - attach_workspace:
-                  at: "."
-            - restore_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
+            - restore_cache: *restore-node-modules-cache
+            - attach_workspace: { at: '.' }
             - run: yarn add [email protected]
             - run: yarn test
+
     test2.4:
-        docker:
-            - image: circleci/node:6
+        docker: *docker-node-6
         steps:
             - checkout
-            - attach_workspace:
-                  at: "."
-            - restore_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
+            - restore_cache: *restore-node-modules-cache
+            - attach_workspace: { at: '.' }
             - run: yarn add [email protected]
             - run: yarn test
+
     test2.7:
-        docker:
-            - image: circleci/node:6
+        docker: *docker-node-6
         steps:
             - checkout
-            - attach_workspace:
-                  at: "."
-            - restore_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
+            - restore_cache: *restore-node-modules-cache
+            - attach_workspace: { at: '.' }
             - run: yarn add [email protected]
             - run: yarn test
+
     test2.8:
-        docker:
-            - image: circleci/node:6
+        docker: *docker-node-6
         steps:
             - checkout
-            - attach_workspace:
-                  at: "."
-            - restore_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
+            - restore_cache: *restore-node-modules-cache
+            - attach_workspace: { at: '.' }
             - run: yarn add [email protected]
             - run: yarn test
+
     test2.9:
-        docker:
-            - image: circleci/node:6
+        docker: *docker-node-6
         steps:
             - checkout
-            - attach_workspace:
-                  at: "."
-            - restore_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
+            - restore_cache: *restore-node-modules-cache
+            - attach_workspace: { at: '.' }
             - run: yarn add [email protected]
             - run: yarn test
+
     test3.0:
-        docker:
-            - image: circleci/node:6
+        docker: *docker-node-8
         steps:
             - checkout
-            - attach_workspace:
-                  at: "."
-            - restore_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
+            - restore_cache: *restore-node-modules-cache
+            - attach_workspace: { at: '.' }
             - run: yarn add [email protected]
             - run: yarn test
+
     testRc:
-        docker:
-            - image: circleci/node:6
+        docker: *docker-node-lts
         steps:
             - checkout
-            - attach_workspace:
-                  at: "."
-            - restore_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
+            - restore_cache: *restore-node-modules-cache
+            - attach_workspace: { at: '.' }
             - run: yarn add typescript@rc
             - run: yarn test
+
     testNext:
-        docker:
-            - image: circleci/node:latest
+        docker: *docker-node-latest
         steps:
             - checkout
-            - attach_workspace:
-                  at: "."
-            - restore_cache:
-                  key: dependency-cache-{{ checksum "yarn.lock" }}
+            - restore_cache: *restore-node-modules-cache
+            - attach_workspace: { at: '.' }
             - run: yarn add typescript@next
             - run: yarn test
 
 workflows:
     version: 2
     build_lint_test:
         jobs:
-            - build
+            - checkout-code
+            - clean-lockfile:
+                requires: [checkout-code]
+            - build:
+                requires: [checkout-code]
             - lint:
-                  requires:
-                      - build
+                  requires: [build]
             - test:
-                  requires:
-                      - build
+                  requires: [build]
             - test2.1:
-                  requires:
-                      - build
+                  requires: [build]
             - test2.4:
-                  requires:
-                      - build
+                  requires: [build]
             - test2.7:
-                  requires:
-                      - build
+                  requires: [build]
             - test2.8:
-                  requires:
-                      - build
+                  requires: [build]
             - test2.9:
-                  requires:
-                      - build
+                  requires: [build]
             - test3.0:
-                  requires:
-                      - build
+                  requires: [build]
             - testRc:
-                  requires:
-                      - build
+                  requires: [build]
             - testNext:
-                  requires:
-                      - build
+                  requires: [build]

package.json

@@ -73,7 +73,8 @@
         "tslint-config-prettier": "^1.18.0",
         "tslint-plugin-prettier": "^2.0.1",
         "tslint-test-config-non-relative": "file:test/external/tslint-test-config-non-relative",
-        "typescript": "~3.1.6"
+        "typescript": "~3.1.6",
+        "yarn-deduplicate": "^1.1.1"
     },
     "engines": {
         "node": ">=4.8.0"

scripts/verifyCleanLockfile.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Verifies that yarn.lock is in its cleanest possible state
+
+set -e
+set -o pipefail
+
+scripts_dir="$( dirname "$(readlink -f "$0")" )"
+bin="$scripts_dir/../node_modules/.bin"
+duplicates="$("$bin/yarn-deduplicate" "$scripts_dir/../yarn.lock" --list)"
+
+if [[ $duplicates ]]; then
+    echo "Found duplicate blocks in yarn.lock which can be cleaned up. Please run 'yarn-deduplicate yarn.lock --strategy fewer'"
+    echo ""
+    echo "$duplicates"
+    exit 1
+else
+    exit 0
+fi