config.yml

version: 2.1
# Validate using `circleci config validate`

# TODO: Push relevant packages to crates.io when a version of master is tagged.

executors:
  minimal:
    docker:
      - image: circleci/buildpack-deps
  # TODO: Add a container with fixed versions for reproducible builds from master.
  docker-builder:
    docker:
      - image: docker:stable-git
    environment:
      # Necessary for `docker manifest` subcommand
      DOCKER_CLI_EXPERIMENTAL: enabled
      # Bump the tag version whenever changes are made.
      IMAGE: &build_image "0xchain/rust-build-env:11"
  docker-rust:
    docker:
      - image: *build_image
    environment:
      # See also environment variables declared in rust-build-env.Dockerfile
      RUST_BACKTRACE: "1"
      SCCACHE_CACHE_SIZE: 1G
      NO_STD_PACKAGES: |
        utils/macros-decl
        algebra/u256
        algebra/primefield
        algebra/elliptic-curve
        crypto/hash
        crypto/merkle-tree
        crypto/elliptic-curve-crypto
        crypto/stark
  docker-npm:
    docker:
      - image: node:latest
commands:
  stop-if-image-exists:
    steps:
      - run:
          name: Stop if image exists
          command: |
            docker manifest inspect $IMAGE && (
              # See https://support.circleci.com/hc/en-us/articles/360015562253-Conditionally-end-a-running-job-gracefully
              circleci-agent step halt
            ) || true
  publish-image:
    steps:
      - run:
          name: Build and publish docker image
          command: |
            # Switch to CI Docker ignore file.
            rm .dockerignore
            cp .dockerignore-ci .dockerignore
            echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
            docker build -t $IMAGE . -f .circleci/images/rust-build-env.Dockerfile
            docker push $IMAGE
  enable-sccache:
    steps:
      - restore_cache:
          name: Restore sccache cache
          keys:
            - sccache-cache-v4-{{ arch }}-{{ .Environment.CIRCLE_JOB }}
      - run:
          name: Enable sccache
          command: |
            echo 'export RUSTC_WRAPPER=sccache' >> $BASH_ENV
            source $BASH_ENV
  save-sccache-cache:
    steps:
      - run:
          name: Show sccache statistics
          command: |
            sccache --show-stats
      - save_cache:
          name: Save sccache cache
          # We use {{ epoch }} to always upload a fresh cache:
          # Of course, restore_cache will not find this exact key,
          # but it will fall back to the closest key (aka the most recent).
          # See https://discuss.circleci.com/t/add-mechanism-to-update-existing-cache-key/9014/13
          key: sccache-cache-v4-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ epoch }}
          paths:
            - "~/.cache/sccache"

jobs:
  rebuild-docker:
    executor: docker-builder
    # We need more resources to build the image
    resource_class: xlarge
    steps:
      - checkout
      - setup_remote_docker:
          docker_layer_caching: true
      - publish-image
  build-docker:
    executor: docker-builder
    # We need more resources to build the image
    resource_class: xlarge
    steps:
      - stop-if-image-exists
      - setup_remote_docker:
          docker_layer_caching: true
      - checkout
      - publish-image
  tests:
    # See https://github.com/mozilla/grcov#grcov-with-travis
    # Requires nightly because of: https://github.com/rust-lang/rust/issues/42524
    # Requires non-incremental build.
    executor: docker-rust
    # We need more resources to build the tests
    resource_class: xlarge
    environment:
      GRCOVFLAGS: -s . --llvm --branch --ignore-not-existing --ignore "/*"
    steps:
      - checkout
      - enable-sccache
      - run:
          name: Build tests with coverage
          command: |
            # Build with coverage settings from Dockerfile
            # TODO: Build and run tests from all targets
            CARGO_INCREMENTAL=0 RUSTFLAGS="$COVFLAGS" cargo +$NIGHTLY t --no-run
      - save-sccache-cache
      - run:
          name: Run tests
          command: |
            # TODO: Store test results in json `--format json`
            # TODO: Fix coverage on zkp-macros-lib
            # TODO: Enable --nocapture
            CARGO_INCREMENTAL=0 RUSTFLAGS="$COVFLAGS" cargo +$NIGHTLY t --workspace --exclude zkp-macros-lib
            cargo +$NIGHTLY t --package zkp-macros-lib
      - run:
          name: Combine coverage
          command: |
            zip -0 target/ccov.zip `find . \( -name "*.gc*" \) -print`
            grcov target/ccov.zip $GRCOVFLAGS -t lcov -o target/lcov.info
      - run:
          name: Submit to codecov
          command: |
            bash <(curl -s https://codecov.io/bash)
      - store_artifacts:
          path: target/lcov.info
          destination: lcov.info
  benchmarks:
    executor: docker-rust
    steps:
      - checkout
      - enable-sccache
      - run:
          name: Build benchmarks in release mode on stable
          command: |
            # Clean out old builds (if any)
            rm target/release/deps/benchmark-* || true
            cargo perf_all --no-run
      - save-sccache-cache
      - run:
          name: Test benchmarks
          command: |
            # Actually running the benchmarks on CI is too noisy. The results
            # would be meaningless. Instead we just run it to make sure they work.
            # Exclude benchmarks that compare against external crates (pulls in a lot of deps)
            cargo perf_all -- --test
      - run:
          # The artifact glob would have picked these up
          name: Cleanup target folder for wokspace glob
          command: rm target/release/deps/benchmark-*.d
      - persist_to_workspace:
          root: target/release/deps
          paths:
            - benchmark-*
  benchmark-ec2:
    # Run benchmarks on AWS EC2
    # See https://engineering.mongodb.com/post/reducing-variability-in-performance-tests-on-ec2-setup-and-key-results
    executor: minimal
    environment:
      SSH_OPT: -o StrictHostKeyChecking=no
      HOST: ec2-user@ec2-18-208-129-239.compute-1.amazonaws.com
    steps:
      - restore_cache:
          name: Restore master benchmark results
          keys:
            - benchmark-criterion-v1
      - attach_workspace:
          # Restoring benchmark executables.
          at: /tmp/workspace
      - run:
          name: Execute benchmarks on ECS
          command: |
            # TODO: Fix concurrency. We don't want multiple benchmarks in parallel.
            mkdir $CIRCLE_BUILD_NUM
            cp /tmp/workspace/benchmark-* ./$CIRCLE_BUILD_NUM
            scp $SSH_OPT -r ./$CIRCLE_BUILD_NUM $HOST:~/
            scp $SSH_OPT -r ./criterion.tgz $HOST:~/ &&
              ssh $SSH_OPT $HOST tar xzf ./criterion.tgz ||
              echo "No previous results cached"
            for bench in ./$CIRCLE_BUILD_NUM/*; do
              # Note: works better if gnuplot is installed on the machine
              ssh $SSH_OPT $HOST $bench --bench --color always
            done
            ssh $SSH_OPT $HOST tar czf ./criterion.tgz ./target/criterion
            scp $SSH_OPT -r $HOST:~/criterion.tgz .
            ssh $SSH_OPT $HOST rm -r ./$CIRCLE_BUILD_NUM ./target ./criterion.tgz
      - persist_to_workspace:
          root: .
          paths:
            - criterion.tgz
      - store_artifacts:
          path: criterion.tgz
          destination: criterion.tgz
  benchmark-store:
    # Stores criterion results as baseline.
    executor: minimal
    steps:
      - attach_workspace:
          # Restoring benchmark criterion result folder
          at: .
      - save_cache:
          name: Save sccache cache
          key: benchmark-criterion-v1-{{ epoch }}
          paths:
            - ./criterion.tgz
  doc:
    executor: docker-rust
    steps:
      - checkout
      - run:
          name: Build docs
          command: cargo d
      - store_artifacts:
          path: ./target/doc
          destination: doc
      - run:
          name: Test examples
          command: cargo d --test
  book:
    executor: docker-rust
    steps:
      - checkout
      - run:
          # TODO: Move to docker image
          name: Install mdbook
          command: cargo install mdbook
      - run:
          name: Build book
          working_directory: ./book
          command: mdbook build
      - store_artifacts:
          path: ./book/book
          destination: book
      - run:
          name: Compile examples
          working_directory: ./book
          command: mdbook test
  rustformat:
    executor: docker-rust
    steps:
      - checkout
      - run:
          name: Lint with rustformat nightly
          command: cargo +$NIGHTLY fmt --all -- --check
          # TODO: Run over wasm targets (which are not part of the workspace)
  build-no-std:
    executor: docker-rust
    steps:
      - checkout
      - enable-sccache
      - run:
          # HACK: We try to build for Cortex-M3, not because we want to create
          # IoT devices, but because there is no `std` available for this target.
          # This forces the build to fail if an accidental dependency on `std`
          # exists. Wasm has a working std, so is not a suitable target here.
          # See: https://forge.rust-lang.org/platform-support.html
          name: Check no std build
          command: |
            cargo +$NIGHTLY nostd_all
      - save-sccache-cache
  clippy:
    executor: docker-rust
    steps:
      - checkout
      - enable-sccache
      - run:
          name: Lint with clippy
          command: |
            cargo lint
      - save-sccache-cache
  analysis:
    executor: docker-rust
    steps:
      - checkout
      - run:
          name: Check lints.rs copies
          command: |
            ./lints_check.sh
      - run:
          name: Check for vulnerabilities in workspace
          command: |
            # TODO: Add --deny-warnings
            # Pending <https://github.com/rust-lang-nursery/lazy-static.rs/issues/163>
            cargo generate-lockfile
            cargo audit
      - run:
          name: Check for use of unsafe in workspace
          command: |
            cargo hack geiger
      - run:
          name: Check for outdated dependencies
          command: |
            cargo outdated
  accept:
    executor: minimal
    steps:
      - run:
          name: Stop
          command: circleci-agent step halt
  update-issues:
    executor: minimal
    steps:
      - run:
          name: Install python tools
          command: |
            # TODO: Have a prepared executor
            sudo apt-get install python3 python3-pip
            pip3 install PyGithub
            pip3 install numpy
      - checkout
      - run:
          name: Update GitHub issues
          command: |
            # This will dry run except on master
            if [ "$CIRCLE_BRANCH" == "master" ]
            then
              DRY_RUN=false python3 .circleci/issue_tracker.py
            else
              DRY_RUN=true python3 .circleci/issue_tracker.py
            fi
  solidity-checking:
    executor: docker-npm
    steps:
      - checkout
      - run:
          # TODO: Move to docker image
          name: Install the npm packages
          working_directory: ./crypto/stark-verifier-ethereum
          command: npm i
      - run:
          name: Compile the contracts
          working_directory: ./crypto/stark-verifier-ethereum
          command: npm run-script build
      - run:
          name: Run the tests
          working_directory: ./crypto/stark-verifier-ethereum
          command: npm run-script test
      - run:
          name: Run the linter
          working_directory: ./crypto/stark-verifier-ethereum
          command: npm run-script lint

workflows:
  version: 2
  weekly:
    triggers:
      - schedule:
          # Weekly on Sunday 0am.
          cron: "0 0 * * 0"
          filters:
            branches:
              only:
                - master
    jobs:
      # Rebuild docker build environment to get up-to-date dependencies
      # and build cache.
      - rebuild-docker
  commit:
    jobs:
      - build-docker
      - tests:
          requires:
            - build-docker
      - benchmarks:
          requires:
            - build-docker
      - doc:
          requires:
            - build-docker
      - book:
          requires:
            - build-docker
      - rustformat:
          requires:
            - build-docker
      - build-no-std:
          requires:
            - build-docker
      - clippy:
          requires:
            - build-docker
      - analysis:
          requires:
            - build-docker
      - benchmark-ec2:
          requires:
            - benchmarks
      - benchmark-store:
          requires:
            - benchmark-ec2
      - update-issues:
          filters:
            branches:
              only: master
      - solidity-checking
      # `accept` is a phony job which requires every check for a commit.
      # This allows us to set a single 'required status check' in GitHub.
      - accept:
          requires:
            - rustformat
            - clippy
            # TODO: Fix analysis
            # - analysis
            - tests
            - benchmarks
            - doc
            - book
            - build-no-std
            - update-issues
            - solidity-checking