[antithesis] Enable workload instrumentation (ava-labs#3059)

Signed-off-by: marun <[email protected]>
Co-authored-by: Stephen Buttolph <[email protected]>

go.mod

@@ -9,6 +9,7 @@ go 1.21.10
 require (
 	github.com/DataDog/zstd v1.5.2
 	github.com/NYTimes/gziphandler v1.1.1
+	github.com/antithesishq/antithesis-sdk-go v0.3.8
 	github.com/ava-labs/coreth v0.13.5-remove-optional-gatherer.2
 	github.com/ava-labs/ledger-avalanche/go v0.0.0-20231102202641-ae2ebdaeac34
 	github.com/btcsuite/btcd/btcutil v1.1.3
@@ -158,7 +159,7 @@ require (
 	go.uber.org/multierr v1.10.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/tools v0.16.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -59,6 +59,8 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
+github.com/antithesishq/antithesis-sdk-go v0.3.8 h1:OvGoHxIcOXFJLyn9IJQ5DzByZ3YVAWNBc394ObzDRb8=
+github.com/antithesishq/antithesis-sdk-go v0.3.8/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/ava-labs/coreth v0.13.5-remove-optional-gatherer.2 h1:RX9DcvgWxq42B2aiGzk77Y8w2bcB7ApO/Cdj9hA6QoE=
 github.com/ava-labs/coreth v0.13.5-remove-optional-gatherer.2/go.mod h1:cm5c12xo5NiTgtbmeduv8i2nYdzgkczz9Wm3yiwwTRU=
@@ -700,8 +702,6 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -916,8 +916,8 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
-golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
+golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

scripts/build_antithesis_images.sh

@@ -41,19 +41,32 @@ function build_images {
   local node_image_name="${base_image_name}-node:${TAG}"
   local workload_image_name="${base_image_name}-workload:${TAG}"
   local config_image_name="${base_image_name}-config:${TAG}"
+  # The same builder image is used to build node and workload images for all test
+  # setups. It is not intended to be pushed.
+  local builder_image_name="antithesis-avalanchego-builder:${TAG}"
 
   # Define dockerfiles
   local base_dockerfile="${AVALANCHE_PATH}/tests/antithesis/${test_setup}/Dockerfile"
+  local builder_dockerfile="${base_dockerfile}.builder-instrumented"
   local node_dockerfile="${base_dockerfile}.node"
+  # Working directory for instrumented builds
+  local builder_workdir="/avalanchego_instrumented/customer"
   if [[ "$(go env GOARCH)" == "arm64" ]]; then
-    # Antithesis instrumentation is only supported on amd64. On apple silicon (arm64), the
-    # uninstrumented Dockerfile will be used to build the node image to enable local test
-    # development.
+    # Antithesis instrumentation is only supported on amd64. On apple silicon (arm64),
+    # uninstrumented Dockerfiles will be used to enable local test development.
+    builder_dockerfile="${base_dockerfile}.builder-uninstrumented"
     node_dockerfile="${uninstrumented_node_dockerfile}"
+    # Working directory for uninstrumented builds
+    builder_workdir="/build"
   fi
 
   # Define default build command
-  local docker_cmd="docker buildx build --build-arg GO_VERSION=${GO_VERSION} --build-arg NODE_IMAGE=${node_image_name}"
+  local docker_cmd="docker buildx build\
+ --build-arg GO_VERSION=${GO_VERSION}\
+ --build-arg NODE_IMAGE=${node_image_name}\
+ --build-arg BUILDER_IMAGE=${builder_image_name}\
+ --build-arg BUILDER_WORKDIR=${builder_workdir}\
+ --build-arg TAG=${TAG}"
 
   if [[ "${test_setup}" == "xsvm" ]]; then
     # The xsvm node image is built on the avalanchego node image, which is assumed to have already been
@@ -69,6 +82,13 @@ function build_images {
     docker_cmd="${docker_cmd} --push"
   fi
 
+  if [[ "${test_setup}" == "avalanchego" ]]; then
+    # Build the image that enables compiling golang binaries for the node and workload
+    # image builds. The builder image is intended to enable building instrumented binaries
+    # if built on amd64 and non-instrumented binaries if built on arm64.
+    ${docker_cmd} -t "${builder_image_name}" -f "${builder_dockerfile}" "${AVALANCHE_PATH}"
+  fi
+
   # Build node image first to allow the workload image to use it.
   ${docker_cmd} -t "${node_image_name}" -f "${node_dockerfile}" "${AVALANCHE_PATH}"
 

scripts/build_test.sh

@@ -9,6 +9,12 @@ source "$AVALANCHE_PATH"/scripts/constants.sh
 
 EXCLUDED_TARGETS="| grep -v /mocks | grep -v proto | grep -v tests/e2e | grep -v tests/upgrade"
 
+if [[ "$(go env GOOS)" == "windows" ]]; then
+  # Test discovery for the antithesis test setups is broken due to
+  # their dependence on the linux-only Antithesis SDK.
+  EXCLUDED_TARGETS="${EXCLUDED_TARGETS} | grep -v tests/antithesis"
+fi
+
 TEST_TARGETS="$(eval "go list ./... ${EXCLUDED_TARGETS}")"
 
 # shellcheck disable=SC2086

tests/antithesis/README.md

@@ -49,6 +49,21 @@ In addition, github workflows are suggested to ensure
 `scripts/tests.build_antithesis_images.sh` runs against PRs and
 `scripts/build_antithesis_images.sh` runs against pushes.
 
+### Use of a builder image
+
+To simplify building instrumented (for running in CI) and
+non-instrumented (for running locally) versions of the workload and
+node images, a common builder image is used. If on an amd64 host,
+`tests/antithesis/avalanchego/Dockerfile.builder-instrumented` is used
+to create an instrumented builder. On an arm64 host,
+`tests/antithesis/avalanchego/Dockerfile.builder-uninstrumented` is
+used to create an uninstrumented builder. In both cases, the builder
+image is based on the default golang image and will include the source
+code necessary to build the node and workload binaries. The
+alternative would require duplicating builder setup for instrumented
+and non-instrumented builds for the workload and node images of each
+test setup.
+
 ## Troubleshooting a test setup
 
 ### Running a workload directly

tests/antithesis/avalanchego/Dockerfile.builder-instrumented

@@ -0,0 +1,46 @@
+# The version is supplied as a build argument rather than hard-coded
+# to minimize the cost of version changes.
+ARG GO_VERSION
+
+# Antithesis: Getting the Antithesis golang instrumentation library
+FROM docker.io/antithesishq/go-instrumentor AS instrumentor
+
+# ============= Instrumentation Stage ================
+FROM golang:$GO_VERSION-bullseye
+
+WORKDIR /build
+# Copy and download avalanche dependencies using go mod
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+# Copy the code into the container
+COPY . .
+
+# Ensure pre-existing builds are not available for inclusion in the final image
+RUN [ -d ./build ] && rm -rf ./build/* || true
+
+# Keep the commit hash to easily verify the exact version that is running
+RUN git rev-parse HEAD > ./commit_hash.txt
+
+# Copy the instrumentor and supporting files to their correct locations
+COPY --from=instrumentor /opt/antithesis /opt/antithesis
+COPY --from=instrumentor /opt/antithesis/lib /lib
+
+# Create the destination output directory for the instrumented code
+RUN mkdir -p /avalanchego_instrumented
+
+# Park the .git file in a safe location
+RUN mkdir -p /opt/tmp/
+RUN cp -r .git /opt/tmp/
+
+# Instrument avalanchego
+RUN /opt/antithesis/bin/goinstrumentor \
+    -stderrthreshold=INFO \
+    -antithesis /opt/antithesis/instrumentation \
+    . \
+    /avalanchego_instrumented
+
+WORKDIR /avalanchego_instrumented/customer
+RUN go mod download
+RUN ln -s /opt/tmp/.git .git

tests/antithesis/avalanchego/Dockerfile.builder-uninstrumented

@@ -0,0 +1,17 @@
+# The version is supplied as a build argument rather than hard-coded
+# to minimize the cost of version changes.
+ARG GO_VERSION
+
+FROM golang:$GO_VERSION-bullseye
+
+WORKDIR /build
+# Copy and download avalanche dependencies using go mod
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+# Copy the code into the container
+COPY . .
+
+# Ensure pre-existing builds are not available for inclusion in the final image
+RUN [ -d ./build ] && rm -rf ./build/* || true

tests/antithesis/avalanchego/Dockerfile.node

@@ -1,46 +1,11 @@
-# The version is supplied as a build argument rather than hard-coded
-# to minimize the cost of version changes.
-ARG GO_VERSION
-
-# Antithesis: Getting the Antithesis golang instrumentation library
-FROM docker.io/antithesishq/go-instrumentor AS instrumentor
+# TAG should identify the builder image
+ARG TAG
 
 # ============= Compilation Stage ================
-FROM golang:$GO_VERSION-bullseye AS builder
-
-WORKDIR /build
-# Copy and download avalanche dependencies using go mod
-COPY go.mod .
-COPY go.sum .
-RUN go mod download
-
-# Copy the code into the container
-COPY . .
-
-# Keep the commit hash to easily verify the exact version that is running
-RUN git rev-parse HEAD > ./commit_hash.txt
-
-# Copy the instrumentor and supporting files to their correct locations
-COPY --from=instrumentor /opt/antithesis /opt/antithesis
-COPY --from=instrumentor /opt/antithesis/lib /lib
-
-# Create the destination output directory for the instrumented code
-RUN mkdir -p /avalanchego_instrumented
-
-# Park the .git file in a safe location
-RUN mkdir -p /opt/tmp/
-RUN cp -r .git /opt/tmp/
-
-# Instrument avalanchego
-RUN /opt/antithesis/bin/goinstrumentor \
-    -stderrthreshold=INFO \
-    -antithesis /opt/antithesis/instrumentation \
-    . \
-    /avalanchego_instrumented
+FROM antithesis-avalanchego-builder:$TAG AS builder
 
+# The workdir is hard-coded since this Dockerfile is only intended for instrumented builds.
 WORKDIR /avalanchego_instrumented/customer
-RUN go mod download
-RUN ln -s /opt/tmp/.git .git
 
 # Build avalanchego with race detection (-r) enabled.
 RUN ./scripts/build.sh -r

tests/antithesis/avalanchego/Dockerfile.workload

@@ -1,21 +1,16 @@
-# The version is supplied as a build argument rather than hard-coded
-# to minimize the cost of version changes.
-ARG GO_VERSION
+# TAG should identify the builder image
+ARG TAG
 
 # NODE_IMAGE needs to identify an existing node image and should include the tag
 ARG NODE_IMAGE
 
 # ============= Compilation Stage ================
-FROM golang:$GO_VERSION-bullseye AS builder
+FROM antithesis-avalanchego-builder:$TAG AS builder
 
-WORKDIR /build
-# Copy and download avalanche dependencies using go mod
-COPY go.mod .
-COPY go.sum .
-RUN go mod download
+# The builder workdir will vary between instrumented and non-instrumented builders
+ARG BUILDER_WORKDIR
 
-# Copy the code into the container
-COPY . .
+WORKDIR $BUILDER_WORKDIR
 
 # Build the workload
 RUN ./scripts/build_antithesis_avalanchego_workload.sh
@@ -24,7 +19,10 @@ RUN ./scripts/build_antithesis_avalanchego_workload.sh
 # Base the workflow on the node image to support bootstrap testing
 FROM $NODE_IMAGE AS execution
 
+# The builder workdir will vary between instrumented and non-instrumented builders
+ARG BUILDER_WORKDIR
+
 # Copy the executable into the container
-COPY --from=builder /build/build/antithesis-avalanchego-workload ./workload
+COPY --from=builder $BUILDER_WORKDIR/build/antithesis-avalanchego-workload ./workload
 
 CMD [ "./workload" ]