[Dockerfile] build speed improvements (aptos-labs#6619)

Uses forked buildx setup action

.dockerignore

@@ -7,6 +7,7 @@
 # explicitly include stuff we actually need via negation
 
 !docker/build-rust-all.sh
+!docker/experimental/*.sh
 !docker/tools/boto.cfg
 
 !.cargo/

.github/actions/docker-setup/action.yaml

@@ -39,10 +39,12 @@ runs:
       run: docker context create builders
 
     - name: setup docker buildx
-      uses: docker/setup-buildx-action@15c905b16b06416d2086efa066dd8e3a35cc7f98 # pin v2.4.0
+      uses: aptos-labs/setup-buildx-action@7952e9cf0debaf1f3f3e5dc7d9c5ea6ececb127e # pin v2.4.0
       with:
         endpoint: builders
         version: v0.10.1
+        custom-name: "core-builder"
+        keep-state: true
 
     - uses: imjasonh/setup-crane@5146f708a817ea23476677995bf2133943b9be0b # [email protected]
 

.github/workflows/experimental-docker-build-test.yaml

@@ -0,0 +1,127 @@
+## IMPORTANT NOTE TO EDITORS OF THIS FILE ##
+
+## Note that when you create a PR the jobs in this file are triggered off the
+## `pull_request_target` event instead of `pull_request` event. This is because
+## the `pull_request` event makes secrets only available to PRs from branches,
+## not from forks, and some of these jobs require secrets. So with `pull_request_target`
+## we're making secrets available to fork-based PRs too. Using `pull_request_target"
+## has a side effect, which is that the workflow execution will be driven by the
+## state of the <workflow>.yaml on the `main` (=target) branch, even if you edited
+## the <workflow>.yaml in your PR. So when you for example add a new job here, you
+## won't see that job appear in the PR itself. It will only become effective once
+## you merge the PR to main. Therefore, if you want to add a new job here and want
+## to test it's functionality prior to a merge to main, you have to to _temporarily_
+## change the trigger event from `pull_request_target` to `pull_request`.
+
+## Additionally, because `pull_request_target` gets secrets injected for forked PRs
+## we use `https://github.com/sushichop/action-repository-permission` to ensure these
+## jobs are only executed when a repo member with "write" permission has triggered
+## the workflow (directly through a push or indirectly by applying a label or enabling
+## auto_merge).
+
+name: "[Experimental] Build+Test Docker Images"
+on: # build on main branch OR when a PR is labeled with `CICD:build-images`
+  # Allow us to run this specific workflow without a PR
+  workflow_dispatch:
+  pull_request_target:
+    types: [labeled, opened, synchronize, reopened, auto_merge_enabled]
+  # For PR that modify .github, run from that PR
+  # This will fail to get secrets if you are not from aptos
+  pull_request:
+    types: [labeled, opened, synchronize, reopened, auto_merge_enabled]
+    paths:
+      - ".github/workflows/experimental-docker-build-test.yaml"
+      - ".github/workflows/experimental-docker-rust-build.yaml"
+      - ".github/workflows/docker-build-test.yaml"
+      - ".github/workflows/workflow-run-forge.yaml"
+      - ".github/workflows/docker-rust-build.yaml"
+      - ".github/workflows/sdk-release.yaml"
+      - ".github/workflows/lint-test.yaml"
+
+# cancel redundant builds
+concurrency:
+  # for push and workflow_dispatch events we use `github.sha` in the concurrency group and don't really cancel each other out/limit concurrency
+  # for pull_request events newer jobs cancel earlier jobs to save on CI etc.
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha || github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+env:
+  GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
+  AWS_ECR_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
+  # In case of pull_request events by default github actions merges main into the PR branch and then runs the tests etc
+  # on the prospective merge result instead of only on the tip of the PR.
+  # For more info also see https://github.com/actions/checkout#checkout-pull-request-head-commit-instead-of-merge-commit
+  GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+
+  # TARGET_CACHE_ID is used as part of the docker tag / cache key inside our bake.hcl docker bake files.
+  # The goal here is to have a branch or PR-local cache such that consecutive pushes to a shared branch or a specific PR can
+  # reuse layers from a previous docker build/commit.
+  # We use `pr-<pr_number>` as cache-id for PRs and simply <branch_name> otherwise.
+  TARGET_CACHE_ID: ${{ github.event.number && format('pr-{0}', github.event.number) || github.ref_name }}
+
+permissions:
+  contents: read
+  id-token: write #required for GCP Workload Identity federation which we use to login into Google Artifact Registry
+  issues: write
+  pull-requests: write
+
+# Note on the job-level `if` conditions:
+# This workflow is designed such that:
+# 1. Run ALL jobs when a 'push', 'workflow_dispatch' triggered the workflow or on 'pull_request's which have set auto_merge=true or have the label "CICD:run-e2e-tests".
+# 2. Run ONLY the docker image building jobs on PRs with the "CICD:build-images" label.
+# 3. Run NOTHING when neither 1. or 2.'s conditions are satisfied.
+jobs:
+  permission-check:
+    if: |
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      contains(github.event.pull_request.labels.*.name, 'CICD:experimental-build') ||
+      contains(github.event.pull_request.labels.*.name, 'CICD:experimental-forge')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check repository permission for user which triggered workflow
+        uses: sushichop/action-repository-permission@13d208f5ae7a6a3fc0e5a7c2502c214983f0241c
+        with:
+          required-permission: write
+          comment-not-permitted: Sorry, you don't have permission to trigger this workflow.
+
+  # Because the docker build happens in a reusable workflow, have a separate job that collects the right metadata
+  # for the subsequent docker builds. Reusable workflows do not currently have the "env" context: https://github.com/orgs/community/discussions/26671
+  determine-docker-build-metadata:
+    runs-on: ubuntu-latest
+    steps:
+      - name: collect metadata
+        run: |
+          echo "GIT_SHA: ${{ env.GIT_SHA }}"
+          echo "TARGET_CACHE_ID: ${{ env.TARGET_CACHE_ID }}"
+    outputs:
+      gitSha: ${{ env.GIT_SHA }}
+      targetCacheId: ${{ env.TARGET_CACHE_ID }}
+
+  rust-images:
+    needs: [permission-check, determine-docker-build-metadata]
+    uses: ./.github/workflows/experimental-docker-rust-build.yaml
+    secrets: inherit
+    with:
+      GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
+      TARGET_CACHE_ID: ${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}
+      PROFILE: release
+      BUILD_ADDL_TESTING_IMAGES: true
+
+  forge-e2e-test:
+    needs: [rust-images, determine-docker-build-metadata]
+    if: |
+      !contains(github.event.pull_request.labels.*.name, 'CICD:skip-forge-e2e-test') && (
+        (github.event_name == 'push' && github.ref_name != 'main') ||
+        github.event_name == 'workflow_dispatch' ||
+        contains(github.event.pull_request.labels.*.name, 'CICD:experimental-forge')
+      )
+    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
+    secrets: inherit
+    with:
+      GIT_SHA: ${{ needs.determine-docker-build-metadata.outputs.gitSha }}
+      COMMENT_HEADER: forge-e2e
+      # Use the cache ID as the Forge namespace so we can limit Forge test concurrency on k8s, since Forge
+      # test lifecycle is separate from that of GHA. This protects us from the case where many Forge tests are triggered
+      # by this GHA. If there is a Forge namespace collision, Forge will pre-empt the existing test running in the namespace.
+      FORGE_NAMESPACE: forge-e2e-${{ needs.determine-docker-build-metadata.outputs.targetCacheId }}

.github/workflows/experimental-docker-rust-build.yaml

@@ -0,0 +1,62 @@
+name: "Build+Push Rust Docker Images"
+
+on:
+  workflow_call:
+    inputs:
+      GIT_SHA:
+        required: true
+        type: string
+        description: The git SHA1 to build. If not specified, the latest commit on the triggering branch will be built
+      TARGET_CACHE_ID:
+        required: true
+        type: string
+        description: ID of the docker cache to use for the build
+      FEATURES:
+        required: false
+        type: string
+        description: The cargo features to build. If not specified, none will be built other than those specified in cargo config
+      PROFILE:
+        default: release
+        required: false
+        type: string
+        description: The cargo profile to build. If not specified, the default release profile will be used
+      BUILD_ADDL_TESTING_IMAGES:
+        default: false
+        required: false
+        type: boolean
+        description: Whether to build additional testing images. If not specified, only the base release images will be built
+
+env:
+  GIT_SHA: ${{ inputs.GIT_SHA }}
+  TARGET_CACHE_ID: ${{ inputs.TARGET_CACHE_ID }}
+  PROFILE: ${{ inputs.PROFILE }}
+  FEATURES: ${{ inputs.FEATURES }}
+  BUILD_ADDL_TESTING_IMAGES: ${{ inputs.BUILD_ADDL_TESTING_IMAGES }}
+  GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
+  AWS_ECR_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
+
+jobs:
+  rust-all:
+    runs-on: experimental-docker
+    steps:
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3
+        with:
+          ref: ${{ env.GIT_SHA }}
+
+      - uses: ./.github/actions/docker-setup/
+        with:
+          GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          GCP_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GCP_SERVICE_ACCOUNT_EMAIL }}
+          GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DOCKER_ARTIFACT_REPO: ${{ secrets.AWS_DOCKER_ARTIFACT_REPO }}
+          GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
+
+      - name: Build and Push Rust images
+        run: docker/experimental/docker-bake-rust-all.sh
+        env:
+          PROFILE: ${{ env.PROFILE }}
+          FEATURES: ${{ env.FEATURES }}
+          BUILD_ADDL_TESTING_IMAGES: ${{ env.BUILD_ADDL_TESTING_IMAGES }}
+          GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}

.gitignore

@@ -107,5 +107,5 @@ aptos-move/aptos-vm-profiling/**/*.txt
 
 # replay-verify and module-verify script outputs
 metadata-cache
-local
+/local/
 run_*

docker/experimental/build-node.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Copyright (c) Aptos
+# SPDX-License-Identifier: Apache-2.0
+set -e
+
+PROFILE=${PROFILE:-release}
+FEATURES=${FEATURES:-""}
+
+echo "Building aptos-node"
+echo "PROFILE: $PROFILE"
+echo "FEATURES: $FEATURES"
+
+# Build and overwrite the aptos-node binary with features if specified
+if [ -n "$FEATURES" ]; then
+    echo "Building aptos-node with features ${FEATURES}"
+    (cd aptos-node && cargo build --profile=$PROFILE --features=$FEATURES "$@")
+else 
+    # Build aptos-node separately
+    cargo build --locked --profile=$PROFILE \
+        -p aptos-node \
+        "$@"
+fi
+
+# After building, copy the binaries we need to `dist` since the `target` directory is used as docker cache mount and only available during the RUN step
+BINS=(
+    aptos-node
+)
+
+mkdir dist
+
+for BIN in "${BINS[@]}"; do
+    cp target/$PROFILE/$BIN dist/$BIN
+done

docker/experimental/build-tools.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Copyright (c) Aptos
+# SPDX-License-Identifier: Apache-2.0
+set -e
+
+PROFILE=${PROFILE:-release}
+
+echo "Building all rust-based docker images"
+echo "PROFILE: $PROFILE"
+
+# Build all the rust binaries
+cargo build --locked --profile=$PROFILE \
+    -p aptos \
+    -p aptos-backup-cli \
+    -p aptos-faucet \
+    -p aptos-forge-cli \
+    -p aptos-fn-check-client \
+    -p aptos-node-checker \
+    -p aptos-openapi-spec-generator \
+    -p aptos-telemetry-service \
+    -p aptos-db-bootstrapper \
+    -p aptos-transaction-emitter \
+    "$@"
+
+# After building, copy the binaries we need to `dist` since the `target` directory is used as docker cache mount and only available during the RUN step
+BINS=(
+    aptos
+    aptos-faucet
+    aptos-node-checker
+    aptos-openapi-spec-generator
+    aptos-telemetry-service
+    aptos-fn-check-client
+    db-backup
+    db-backup-verify
+    aptos-db-bootstrapper
+    db-restore
+    forge
+    aptos-transaction-emitter
+)
+
+mkdir dist
+
+for BIN in "${BINS[@]}"; do
+    cp target/$PROFILE/$BIN dist/$BIN
+done
+
+# Build the Aptos Move framework and place it in dist. It can be found afterwards in the current directory.
+(cd dist && cargo run --locked --profile=$PROFILE --package aptos-framework -- release)

docker/experimental/builder.Dockerfile

@@ -0,0 +1,78 @@
+#syntax=docker/dockerfile:1.4
+
+FROM rust as rust-base
+WORKDIR /aptos
+
+RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update && apt-get --no-install-recommends install -y \
+        cmake \
+        curl \
+        clang \
+        git \
+        pkg-config \
+        libssl-dev \
+        libpq-dev \
+        binutils \
+        lld
+
+# install cargo chef to cache dependencies
+RUN --mount=type=cache,target=/root/.cargo <<EOT
+    cargo install [email protected]
+    cargo install cargo-cache --no-default-features --features ci-autoclean
+EOT
+
+### Build Rust dependencies ###
+FROM rust-base as planner
+
+COPY --link . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+### Build Rust code ###
+FROM rust-base as builder-base
+
+# Confirm that this Dockerfile is being invoked from an appropriate builder.
+# See https://github.com/aptos-labs/aptos-core/pull/2471
+# See https://github.com/aptos-labs/aptos-core/pull/2472
+ARG BUILT_VIA_BUILDKIT
+ENV BUILT_VIA_BUILDKIT $BUILT_VIA_BUILDKIT
+RUN test -n "$BUILT_VIA_BUILDKIT" || (printf "===\nREAD ME\n===\n\nYou likely just tried run a docker build using this Dockerfile using\nthe standard docker builder (e.g. docker build). The standard docker\nbuild command uses a builder that does not respect our .dockerignore\nfile, which will lead to a build failure. To build, you should instead\nrun a command like one of these:\n\ndocker/docker-bake-rust-all.sh\ndocker/docker-bake-rust-all.sh indexer\n\nIf you are 100 percent sure you know what you're doing, you can add this flag:\n--build-arg BUILT_VIA_BUILDKIT=true\n\nFor more information, see https://github.com/aptos-labs/aptos-core/pull/2472\n\nThanks!" && false)
+
+# cargo profile and features
+ARG PROFILE
+ENV PROFILE ${PROFILE}
+ARG FEATURES
+ENV FEATURES ${FEATURES}
+
+RUN ARCHITECTURE=$(uname -m | sed -e "s/arm64/arm_64/g" | sed -e "s/aarch64/aarch_64/g") \
+    && curl -LOs "https://github.com/protocolbuffers/protobuf/releases/download/v21.5/protoc-21.5-linux-$ARCHITECTURE.zip" \
+    && unzip -o "protoc-21.5-linux-$ARCHITECTURE.zip" -d /usr/local bin/protoc \
+    && unzip -o "protoc-21.5-linux-$ARCHITECTURE.zip" -d /usr/local 'include/*' \
+    && chmod +x "/usr/local/bin/protoc" \
+    && rm "protoc-21.5-linux-$ARCHITECTURE.zip"
+
+# Use cargo chef
+COPY  --link --from=planner /aptos/recipe.json recipe.json
+
+# Build dependencies - this is the caching Docker layer!
+RUN <<EOT
+    cargo chef cook --profile ${PROFILE} --workspace --recipe-path recipe.json
+    cargo cache
+EOT
+
+COPY --link . /aptos/
+
+FROM builder-base as aptos-node-builder
+
+RUN --mount=type=secret,id=git-credentials,target=/root/.git-credentials \
+    --mount=type=cache,target=/root/.cargo,id=node-cargo-cache \
+    --mount=type=cache,target=/aptos/target,id=node-target-cache \
+        docker/experimental/build-node.sh
+
+FROM builder-base as tools-builder
+
+RUN --mount=type=secret,id=git-credentials,target=/root/.git-credentials \
+    --mount=type=cache,target=/root/.cargo,id=tools-cargo-cache \
+    --mount=type=cache,target=/aptos/target,id=tools-target-cache \
+        docker/experimental/build-tools.sh

docker/experimental/debian-base.Dockerfile

@@ -0,0 +1,10 @@
+#syntax=docker/dockerfile:1.4
+
+FROM debian AS debian-base
+
+RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
+
+# Add Tini to make sure the binaries receive proper SIGTERM signals when Docker is shut down
+ADD https://github.com/krallin/tini/releases/download/v0.19.0/tini /tini
+RUN chmod +x /tini
+ENTRYPOINT ["/tini", "--"]