From f3ac89715575de9208c9626526bbafd6ef1a0af4 Mon Sep 17 00:00:00 2001
From: Howard Wu <9260812+howardwu@users.noreply.github.com>
Date: Sat, 22 Oct 2022 22:43:48 -0700
Subject: [PATCH] Update CI

---
 .circleci/config.yml | 401 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 401 insertions(+)
 create mode 100644 .circleci/config.yml

diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 0000000000..843c84abad
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,401 @@
+version: 2.1
+commands:
+  check_windows:
+    description: "Runs cargo check on Windows"
+    parameters:
+      workspace_member:
+        type: string
+    steps:
+      - checkout
+      - restore_cache:
+          key: cargo-cache-{{ arch }}-{{ checksum "Cargo.lock" }}
+      - run:
+          name: "Install Rust and run cargo check"
+          command: |
+            $ProgressPreference = "SilentlyContinue"
+            Invoke-WebRequest -Uri "https://win.rustup.rs/" -OutFile "C:\rustup-init.exe"
+            & C:\rustup-init.exe -y --default-toolchain "stable-x86_64-pc-windows-msvc" --no-modify-path --profile minimal
+            $env:Path += ";C:\Users\circleci\.cargo\bin"
+            rustc -Vv
+            cargo --version
+            rustc --version | Out-File -FilePath "rust-version"
+            if (!(Test-Path "Cargo.lock" -PathType Leaf)) {
+                cargo generate-lockfile
+            }
+            cd << parameters.workspace_member >>
+            cargo check --examples --benches --tests
+      - save_cache:
+          paths:
+            - C:\Users\circleci\.cargo\registry
+            - C:\Users\circleci\.cargo\git
+            - target
+          key: cargo-cache-{{ arch }}-{{ checksum "rust-version" }}-{{ checksum "Cargo.lock" }}
+
+  setup_environment:
+    description: "Setup testing environment"
+    parameters:
+      cache_key:
+        type: string
+        default: snarkos-stable-cache
+    steps:
+      - run: set -e
+      - setup_remote_docker
+      - run:
+          name: Prepare environment and install dependencies
+          command: |
+            export SCCACHE_CACHE_SIZE=200M
+            export WORK_DIR="$CIRCLE_WORKING_DIRECTORY/.cache/sccache"
+            export SCCACHE_DIR="$CIRCLE_WORKING_DIRECTORY/.cache/sccache"
+            mkdir -p "$CIRCLE_WORKING_DIRECTORY/.bin"
+            wget https://github.com/mozilla/sccache/releases/download/v0.3.0/sccache-v0.3.0-x86_64-unknown-linux-musl.tar.gz
+            tar -C "$CIRCLE_WORKING_DIRECTORY/.bin" -xvf sccache-v0.3.0-x86_64-unknown-linux-musl.tar.gz
+            mv $CIRCLE_WORKING_DIRECTORY/.bin/sccache-v0.3.0-x86_64-unknown-linux-musl/sccache $CIRCLE_WORKING_DIRECTORY/.bin/sccache
+            export PATH="$PATH:$CIRCLE_WORKING_DIRECTORY/.bin"
+            export RUSTC_WRAPPER="sccache"
+            rm -rf "$CIRCLE_WORKING_DIRECTORY/.cargo/registry"
+            DEBIAN_FRONTEND=noninteractive sudo apt-get update
+            DEBIAN_FRONTEND=noninteractive sudo apt-get dist-upgrade -y -o DPkg::Options::=--force-confold
+            DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends clang llvm-dev llvm pkg-config xz-utils make libssl-dev libssl-dev
+      - restore_cache:
+          keys:
+            - << parameters.cache_key >>
+
+  clear_environment:
+    description: "Clear environment"
+    parameters:
+      cache_key:
+        type: string
+        default: snarkos-stable-cache
+    steps:
+      - run: (sccache -s||true)
+      - run: set +e
+      - save_cache:
+          key: << parameters.cache_key >>
+          paths:
+            - .cache/sccache
+            - .cargo
+
+  run_serial:
+    description: "Build and run tests"
+    parameters:
+      workspace_member:
+        type: string
+      cache_key:
+        type: string
+      flags:
+        type: string
+        default: ""
+    steps:
+      - checkout
+      - setup_environment:
+          cache_key: << parameters.cache_key >>
+      - run:
+          no_output_timeout: 30m
+          command: cd << parameters.workspace_member >> && RUST_MIN_STACK=67108864 cargo test << parameters.flags >>
+      - clear_environment:
+          cache_key: << parameters.cache_key >>
+
+  run_parallel:
+    description: "Build and run tests (in parallel)"
+    parameters:
+      workspace_member:
+        type: string
+      cache_key:
+        type: string
+      flags:
+        type: string
+        default: ""
+    steps:
+      - checkout
+      - setup_environment:
+          cache_key: << parameters.cache_key >>
+      - run:
+          no_output_timeout: 45m
+          command: |
+            cd << parameters.workspace_member >>
+            cargo test -- --list --format terse | sed 's/: test//' > test_names.txt
+            TEST_NAMES=$(circleci tests split test_names.txt)
+            for i in $(echo $TEST_NAMES | sed "s/ / /g")
+            do
+                RUST_MIN_STACK=67108864 cargo test $i << parameters.flags >>
+            done
+      - clear_environment:
+          cache_key: << parameters.cache_key >>
+
+  install_rust_nightly:
+    description: "Install Rust nightly toolchain"
+    steps:
+      - run: rustup toolchain install nightly-x86_64-unknown-linux-gnu
+
+jobs:
+  snarkos:
+    docker:
+      - image: cimg/rust:1.64
+    resource_class: xlarge
+    steps:
+      - run_serial:
+          workspace_member: .
+          cache_key: snarkos-stable-cache
+
+  account:
+    docker:
+      - image: cimg/rust:1.64
+    resource_class: xlarge
+    steps:
+      - run_serial:
+          workspace_member: account
+          cache_key: snarkos-account-cache
+
+  cli:
+    docker:
+      - image: cimg/rust:1.64
+    resource_class: xlarge
+    steps:
+      - run_serial:
+          workspace_member: cli
+          cache_key: snarkos-cli-cache
+
+  display:
+    docker:
+      - image: cimg/rust:1.64
+    resource_class: xlarge
+    steps:
+      - run_serial:
+          workspace_member: display
+          cache_key: snarkos-display-cache
+
+  node:
+    docker:
+      - image: cimg/rust:1.64
+    resource_class: xlarge
+    steps:
+      - run_serial:
+          workspace_member: node
+          cache_key: snarkos-node-cache
+
+  node-executor:
+    docker:
+      - image: cimg/rust:1.64
+    resource_class: xlarge
+    steps:
+      - run_serial:
+          workspace_member: node/executor
+          cache_key: snarkos-node-executor-cache
+
+  node-ledger:
+    docker:
+      - image: cimg/rust:1.64
+    resource_class: xlarge
+    steps:
+      - run_serial:
+          workspace_member: node/ledger
+          cache_key: snarkos-node-ledger-cache
+
+  node-router:
+    docker:
+      - image: cimg/rust:1.64
+    resource_class: xlarge
+    steps:
+      - run_serial:
+          workspace_member: node/router
+          cache_key: snarkos-node-router-cache
+
+  node-store:
+    docker:
+      - image: cimg/rust:1.64
+    resource_class: xlarge
+    steps:
+      - run_serial:
+          workspace_member: node/store
+          cache_key: snarkos-node-store-cache
+
+  check-fmt:
+    docker:
+      - image: cimg/rust:1.64
+    resource_class: xlarge
+    steps:
+      - checkout
+      - install_rust_nightly
+      - setup_environment:
+          cache_key: snarkos-fmt-cache
+      - run:
+          name: Check style
+          no_output_timeout: 35m
+          command: cargo +nightly fmt --all -- --check
+      - clear_environment:
+          cache_key: snarkos-fmt-cache
+
+  check-clippy:
+    docker:
+      - image: cimg/rust:1.64
+    resource_class: 2xlarge
+    steps:
+      - checkout
+      - setup_environment:
+          cache_key: snarkos-clippy-cache
+      - run:
+          name: Check lint
+          no_output_timeout: 35m
+          command: |
+            cargo clippy --workspace --all-targets -- -D warnings
+            cargo clippy --workspace --all-targets --all-features -- -D warnings
+      - clear_environment:
+          cache_key: snarkos-clippy-cache
+
+  verify-windows:
+    executor:
+      name: windows/default
+      size: 2xlarge
+    environment:
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+    parameters:
+      workspace_member:
+        type: string
+    steps:
+      - check_windows:
+          workspace_member: << parameters.workspace_member >>
+
+  build-and-publish-docker-arm:
+    machine:
+      image: ubuntu-2204:current
+      docker_layer_caching: true
+    resource_class: arm.2xlarge
+    steps:
+      - checkout
+      - run: mkdir -p my_workspace
+      - run:
+          name: "Build snarkOS Docker image arm-v8"
+          no_output_timeout: 2h
+          command: |
+            VERSION=$(git rev-parse --short HEAD)
+            docker build -f Dockerfile -t $DOCKER_REPO:$CIRCLE_BRANCH-$VERSION-arm64 .
+      - run:
+          name: "Push snarkOS Docker image arm-v8"
+          command: |
+            VERSION=$(git rev-parse --short HEAD)
+            echo $DOCKERHUB_TOKEN | docker login -u $DOCKERHUB_USERNAME --password-stdin
+            # CREATE THE SHELL FILE WITH IMAGE NAME AND TAG
+            docker push $DOCKER_REPO:$CIRCLE_BRANCH-$VERSION-arm64
+            echo "Pushed $DOCKER_REPO:$CIRCLE_BRANCH-$VERSION-arm64"
+      - run:
+          name: "Save arm-v8 image tag"
+          command: |
+            VERSION=$(git rev-parse --short HEAD)
+            echo "$CIRCLE_BRANCH-$VERSION-arm64" > my_workspace/docker_tag_arm
+      - persist_to_workspace:
+          root: my_workspace
+          paths:
+            - docker_tag_arm
+
+  build-and-publish-docker-amd:
+    machine:
+      image: ubuntu-2204:current
+      docker_layer_caching: true
+    resource_class: 2xlarge
+    steps:
+      - checkout
+      - run: mkdir -p my_workspace
+      - run:
+          name: "Build snarkOS Docker image amd64"
+          no_output_timeout: 2h
+          command: |
+            VERSION=$(git rev-parse --short HEAD)
+            docker build -f Dockerfile -t $DOCKER_REPO:$CIRCLE_BRANCH-$VERSION-amd64 .
+      - run:
+          name: "Push snarkOS Docker image amd64"
+          command: |
+            VERSION=$(git rev-parse --short HEAD)
+            echo $DOCKERHUB_TOKEN | docker login -u $DOCKERHUB_USERNAME --password-stdin
+            docker push $DOCKER_REPO:$CIRCLE_BRANCH-$VERSION-amd64
+            echo "Pushed $DOCKER_REPO:$CIRCLE_BRANCH-$VERSION-amd64"
+      - run:
+          name: "Save amd64 image tag"
+          command: |
+            VERSION=$(git rev-parse --short HEAD)
+            echo "$CIRCLE_BRANCH-$VERSION-amd64" > my_workspace/docker_tag_amd
+      - persist_to_workspace:
+          root: my_workspace
+          paths:
+            - docker_tag_amd
+
+  publish_snarkos_manifest:
+    machine:
+      image: ubuntu-2204:current
+      docker_layer_caching: true
+    resource_class: arm.2xlarge
+    steps:
+      - checkout
+      - attach_workspace:
+          at: my_workspace
+      - run:
+          name: "Pull ARM Docker image"
+          command: |
+            ARM_TAG=$(cat my_workspace/docker_tag_arm)
+            echo $ARM_TAG
+            echo $DOCKERHUB_TOKEN | docker login -u $DOCKERHUB_USERNAME --password-stdin
+            docker pull $DOCKER_REPO:$ARM_TAG
+      - run:
+          name: "Pull AMD Docker image"
+          command: |
+            AMD_TAG=$(cat my_workspace/docker_tag_amd)
+            echo $AMD_TAG
+            docker pull $DOCKER_REPO:$AMD_TAG
+      - run:
+          name: "Create and push Docker multi-arch manifests"
+          command: |
+            ARM_TAG=$(cat my_workspace/docker_tag_arm)
+            AMD_TAG=$(cat my_workspace/docker_tag_amd)
+            echo $DOCKERHUB_TOKEN | docker login -u $DOCKERHUB_USERNAME --password-stdin
+            echo $DOCKER_REPO
+            echo $CIRCLE_BRANCH
+            echo $ARM_TAG
+            echo $AMD_TAG
+            docker manifest create $DOCKER_REPO:${CIRCLE_BRANCH}-latest --amend $DOCKER_REPO:${ARM_TAG} --amend $DOCKER_REPO:${AMD_TAG}
+            docker manifest inspect $DOCKER_REPO:${CIRCLE_BRANCH}-latest
+            docker manifest push $DOCKER_REPO:${CIRCLE_BRANCH}-latest
+            echo "Pushed $DOCKER_REPO:${CIRCLE_BRANCH}-latest"
+            docker manifest create $DOCKER_REPO:latest --amend $DOCKER_REPO:${ARM_TAG} --amend $DOCKER_REPO:${AMD_TAG}
+            docker manifest inspect $DOCKER_REPO:latest
+            docker manifest push $DOCKER_REPO:latest
+            echo "Pushed $DOCKER_REPO:latest"
+
+workflows:
+  version: 2
+
+  main-workflow:
+    jobs:
+      - snarkos
+      - account
+      - cli
+      - display
+      - node
+      - node-executor
+      - node-ledger
+      - node-router
+      - node-store
+      - check-fmt
+      - check-clippy
+
+  windows-workflow:
+    jobs:
+      - verify-windows:
+          matrix:
+            parameters:
+              workspace_member: [
+                account,
+                cli,
+                display
+                node
+              ]
+
+  build-snarkos-docker-images:
+    when:
+      or:
+        - equal: [ testnet3, << pipeline.git.branch >> ]
+    jobs:
+      - build-and-publish-docker-arm
+      - build-and-publish-docker-amd
+      - publish_snarkos_manifest:
+          requires:
+            - build-and-publish-docker-arm
+            - build-and-publish-docker-amd