From 3a4393e33eab5afdd84a24438fe1b7a873dd10bc Mon Sep 17 00:00:00 2001 From: jk jensen Date: Thu, 9 Mar 2023 12:17:04 -0800 Subject: [PATCH] Separate indexer into its own docker image (#8931) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description We moved fast by putting indexer into sui-tools. Separate it out as its own image. ## Test Plan Builds successfully: ``` docker/sui-indexer [jkj/indexer-image-split] ยป ./build.sh Building sui-indexer docker image Dockerfile: /Users/jordankylejensen/mysten/sui/docker/sui-indexer/Dockerfile docker context: /Users/jordankylejensen/mysten/sui build date: 2023-03-08 git revision: 9e9e10544 [+] Building 1159.6s (26/26) FINISHED => [internal] load build definition from Dockerfile 0.0s => => transferring dockerfile: 1.87kB 0.0s => [internal] load .dockerignore 0.0s => => transferring context: 34B 0.0s => [internal] load metadata for docker.io/library/debian:bullseye-slim 2.7s => [internal] load metadata for docker.io/library/rust:1.65.0 2.4s => [auth] library/rust:pull token for registry-1.docker.io 0.0s => [auth] library/debian:pull token for registry-1.docker.io 0.0s => [chef 1/3] FROM docker.io/library/rust:1.65.0@sha256:891bc3b252c43a1c2667083e3861f26e6f571dcc3bc98dcc151d6ff6edc62cb9 0.0s => [runtime 1/4] FROM docker.io/library/debian:bullseye-slim@sha256:77f46c1cf862290e750e913defffb2828c889d291a93bdd10a7a0597720948fc 0.0s => [internal] load build context 0.2s => => transferring context: 10.80MB 0.2s => CACHED [chef 2/3] WORKDIR sui 0.0s => [chef 3/3] RUN apt-get update && apt-get install -y cmake clang 14.5s => [planner 1/3] COPY Cargo.toml Cargo.lock ./ 0.0s => [planner 2/3] COPY crates/workspace-hack crates/workspace-hack 0.0s => [planner 3/3] RUN sed -i '/crates\/workspace-hack/b; /crates/d; /narwhal/d' Cargo.toml && cargo metadata -q >/dev/null 95.9s => [builder 1/8] COPY --from=planner /sui/Cargo.toml Cargo.toml 0.0s => [builder 2/8] COPY --from=planner /sui/Cargo.lock Cargo.lock 0.0s => [builder 3/8] COPY --from=planner /sui/crates/workspace-hack crates/workspace-hack 0.0s => [builder 4/8] RUN cargo build --release 799.7s => [builder 5/8] COPY Cargo.toml Cargo.lock ./ 0.0s => [builder 6/8] COPY crates crates 0.5s => [builder 7/8] COPY narwhal narwhal 0.1s => [builder 8/8] RUN cargo build --release --bin sui-indexer 240.5s => CACHED [runtime 2/4] WORKDIR sui 0.0s => [runtime 3/4] COPY --from=builder /sui/target/release/sui-indexer /usr/local/bin 0.0s => [runtime 4/4] RUN apt update && apt install -y libpq5 ca-certificates 5.0s => exporting to image 0.1s => => exporting layers 0.1s => => writing image sha256:06950ad3dd97d13a22e4459728730f2ccf3d4b092de929359bdc31243956cafd 0.0s ``` Built and published successfully to [dockerhub](https://hub.docker.com/repository/docker/mysten/sui-indexer/general) --- If your changes are not user-facing and not a breaking change, you can skip the following section. Otherwise, please indicate what changed, and then add to the Release Notes section as highlighted during the release process. ### Type of Change (Check all that apply) - [ ] user-visible impact - [ ] breaking change for a client SDKs - [ ] breaking change for FNs (FN binary must upgrade) - [ ] breaking change for validators or node operators (must upgrade binaries) - [ ] breaking change for on-chain data layout - [ ] necessitate either a data wipe or data migration ### Release notes - Separate the new sui-indexer into a dedicated image. --- docker/sui-indexer/Dockerfile | 53 +++++++++++++++++++++++++++++++++++ docker/sui-indexer/build.sh | 25 +++++++++++++++++ docker/sui-tools/Dockerfile | 4 --- docker/sui-tools/build.sh | 1 - 4 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 docker/sui-indexer/Dockerfile create mode 100755 docker/sui-indexer/build.sh diff --git a/docker/sui-indexer/Dockerfile b/docker/sui-indexer/Dockerfile new file mode 100644 index 0000000000000..2093ba63ed2c2 --- /dev/null +++ b/docker/sui-indexer/Dockerfile @@ -0,0 +1,53 @@ +FROM rust:1.65.0 AS chef +WORKDIR sui +ARG GIT_REVISION +ENV GIT_REVISION=$GIT_REVISION +RUN apt-get update && apt-get install -y cmake clang + +# Plan out the 3rd-party dependencies that need to be built. +# +# This is done by: +# 1. Copy in Cargo.toml, Cargo.lock, and the workspace-hack crate +# 2. Removing all workspace crates, other than the workpsace-hack +# crate, from the workspace Cargo.toml file. +# 3. Update the lockfile in order to reflect the changes to the +# root Cargo.toml file. +FROM chef AS planner +COPY Cargo.toml Cargo.lock ./ +COPY crates/workspace-hack crates/workspace-hack +RUN sed -i '/crates\/workspace-hack/b; /crates/d; /narwhal/d' Cargo.toml \ + && cargo metadata -q >/dev/null + +# Build and cache all dependencies. +# +# In a fresh layer, copy in the "plan" generated by the planner +# and run `cargo build` in order to create a caching Docker layer +# with all dependencies built. +FROM chef AS builder +COPY --from=planner /sui/Cargo.toml Cargo.toml +COPY --from=planner /sui/Cargo.lock Cargo.lock +COPY --from=planner /sui/crates/workspace-hack crates/workspace-hack +RUN cargo build --release + +# Build application +# +# Copy in the rest of the crates (and an unmodified Cargo.toml and Cargo.lock) +# and build the application. At this point no dependencies should need to be +# built as they were built and cached by the previous layer. +COPY Cargo.toml Cargo.lock ./ +COPY crates crates +COPY narwhal narwhal +RUN cargo build --release \ + --bin sui-indexer + +# Production Image +FROM debian:bullseye-slim AS runtime +WORKDIR sui +COPY --from=builder /sui/target/release/sui-indexer /usr/local/bin +# sui-indexer needs postgres libpq5 and ca-certificates +RUN apt update && apt install -y libpq5 ca-certificates + +ARG BUILD_DATE +ARG GIT_REVISION +LABEL build-date=$BUILD_DATE +LABEL git-revision=$GIT_REVISION diff --git a/docker/sui-indexer/build.sh b/docker/sui-indexer/build.sh new file mode 100755 index 0000000000000..aab9aa3281187 --- /dev/null +++ b/docker/sui-indexer/build.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# fast fail. +set -e + +DIR="$( cd "$( dirname "$0" )" && pwd )" +REPO_ROOT="$(git rev-parse --show-toplevel)" +DOCKERFILE="$DIR/Dockerfile" +GIT_REVISION="$(git describe --always --dirty --exclude '*')" +BUILD_DATE="$(date -u +'%Y-%m-%d')" + +echo +echo "Building sui-indexer docker image" +echo "Dockerfile: \t$DOCKERFILE" +echo "docker context: $REPO_ROOT" +echo "build date: \t$BUILD_DATE" +echo "git revision: \t$GIT_REVISION" +echo + +docker build -f "$DOCKERFILE" "$REPO_ROOT" \ + --build-arg GIT_REVISION="$GIT_REVISION" \ + --build-arg BUILD_DATE="$BUILD_DATE" \ + "$@" diff --git a/docker/sui-tools/Dockerfile b/docker/sui-tools/Dockerfile index d10e34d6f4382..2fd9aef26b16f 100644 --- a/docker/sui-tools/Dockerfile +++ b/docker/sui-tools/Dockerfile @@ -42,7 +42,6 @@ RUN cargo build --release \ --bin sui \ --bin sui-faucet \ --bin stress \ - --bin sui-indexer \ --bin sui-cluster-test # Production Image @@ -52,9 +51,6 @@ COPY --from=builder /sui/target/release/sui-node /usr/local/bin COPY --from=builder /sui/target/release/sui /usr/local/bin COPY --from=builder /sui/target/release/sui-faucet /usr/local/bin COPY --from=builder /sui/target/release/stress /usr/local/bin -COPY --from=builder /sui/target/release/sui-indexer /usr/local/bin -# sui-indexer needs postgres libpq5 and ca-certificates -RUN apt update && apt install -y libpq5 ca-certificates COPY --from=builder /sui/target/release/sui-cluster-test /usr/local/bin ARG BUILD_DATE diff --git a/docker/sui-tools/build.sh b/docker/sui-tools/build.sh index c4c389fbe9124..ffb040b3258ae 100755 --- a/docker/sui-tools/build.sh +++ b/docker/sui-tools/build.sh @@ -12,7 +12,6 @@ GIT_REVISION="$(git describe --always --dirty --exclude '*')" BUILD_DATE="$(date -u +'%Y-%m-%d')" echo -echo "Building sui-tool docker image" echo "Building sui-tools docker image" echo "Dockerfile: \t$DOCKERFILE" echo "docker context: $REPO_ROOT"