[docker] pass through profile and features flags (aptos-labs#3824)

* [docker] pass through profile and features flags

* [helm][genesis] more options for genesis

* docker: set FEATURES and PROFILE on the correct stage

Co-authored-by: Christian Theilemann <[email protected]>

.github/workflows/run-forge.yaml

@@ -48,12 +48,10 @@ on:
       FORGE_ENABLE_FAILPOINTS:
         required: false
         type: string
-        default: false
         description: Whether to use failpoints images. If this option is true, and you plan to manually pass in IMAGE_TAGs, note that the tagging schema is different
       FORGE_ENABLE_PERFORMANCE:
         required: false
         type: string
-        default: false
         description: Whether to use performance images. If this option is true, and you plan to manually pass in IMAGE_TAGs, note that the tagging schema is different
 
 env:

docker/build-rust-all.sh

@@ -6,22 +6,26 @@ set -e
 PROFILE=${PROFILE:-release}
 FEATURES=${FEATURES:-""}
 
+echo "Building all rust-based docker images"
+echo "PROFILE: $PROFILE"
+echo "FEATURES: $FEATURES"
+
 # Build all the rust binaries
 cargo build --profile=$PROFILE \
-        -p aptos \
-        -p aptos-faucet \
-        -p aptos-indexer \
-        -p aptos-sf-indexer \
-        -p aptos-node \
-        -p aptos-node-checker \
-        -p aptos-openapi-spec-generator \
-        -p aptos-telemetry-service \
-        -p aptos-vfn-check-client \
-        -p backup-cli \
-        -p db-bootstrapper \
-        -p forge-cli \
-        -p transaction-emitter \
-        "$@"
+    -p aptos \
+    -p aptos-faucet \
+    -p aptos-indexer \
+    -p aptos-sf-indexer \
+    -p aptos-node \
+    -p aptos-node-checker \
+    -p aptos-openapi-spec-generator \
+    -p aptos-telemetry-service \
+    -p aptos-vfn-check-client \
+    -p backup-cli \
+    -p db-bootstrapper \
+    -p forge-cli \
+    -p transaction-emitter \
+    "$@"
 
 # Build and overwrite the aptos-node binary with features if specified
 if [ -n "$FEATURES" ]; then
@@ -50,10 +54,9 @@ BINS=(
 
 mkdir dist
 
-for BIN in "${BINS[@]}"
-do
+for BIN in "${BINS[@]}"; do
     cp target/$PROFILE/$BIN dist/$BIN
 done
 
 # Build the Aptos Move framework and place it in dist. It can be found afterwards in the current directory.
-( cd dist && cargo run --package framework -- release )
+(cd dist && cargo run --package framework -- release)

docker/docker-bake-rust-all.hcl

@@ -47,25 +47,6 @@ variable "PROFILE" {
 }
 variable "FEATURES" {
   // Cargo features to enable, as a comma separated string
-  default = ""
-}
-
-
-target "builder" {
-  target     = "builder"
-  dockerfile = "docker/rust-all.Dockerfile"
-  context    = "."
-  cache-from = generate_cache_from("builder")
-  cache-to   = generate_cache_to("builder")
-  tags       = generate_tags("builder")
-  args = {
-    PROFILE            = "${PROFILE}"
-    FEATURES           = "${FEATURES}"
-    GIT_SHA            = "${GIT_SHA}"
-    GIT_BRANCH         = "${GIT_BRANCH}"
-    GIT_TAG            = "${GIT_TAG}"
-    BUILT_VIA_BUILDKIT = "true"
-  }
 }
 
 group "all" {
@@ -88,7 +69,6 @@ target "_common" {
   context    = "."
   cache-from = flatten([
     // need to repeat all images here until https://github.com/docker/buildx/issues/934 is resolved
-    generate_cache_from("builder"),
     generate_cache_from("validator"),
     generate_cache_from("indexer"),
     generate_cache_from("node-checker"),
@@ -106,6 +86,8 @@ target "_common" {
     "org.label-schema.git-sha"        = "${GIT_SHA}"
   }
   args = {
+    PROFILE            = "${PROFILE}"
+    FEATURES           = "${FEATURES}"
     GIT_SHA            = "${GIT_SHA}"
     GIT_BRANCH         = "${GIT_BRANCH}"
     GIT_TAG            = "${GIT_TAG}"

docker/rust-all.Dockerfile

@@ -37,7 +37,7 @@ ENV PROFILE ${PROFILE}
 ARG FEATURES
 ENV FEATURES ${FEATURES}
 
-RUN docker/build-rust-all.sh && rm -rf $CARGO_HOME && rm -rf target 
+RUN PROFILE=$PROFILE FEATURES=$FEATURES docker/build-rust-all.sh && rm -rf $CARGO_HOME && rm -rf target 
 
 ### Validator Image ###
 FROM debian-base AS validator

terraform/aptos-node-testnet/main.tf

@@ -92,9 +92,13 @@ provider "kubernetes" {
   token                  = module.validator.aws_eks_cluster_auth_token
 }
 
+locals {
+  genesis_helm_chart_path = "${path.module}/../helm/genesis"
+}
+
 resource "helm_release" "genesis" {
   name        = "genesis"
-  chart       = "${path.module}/../helm/genesis"
+  chart       = local.genesis_helm_chart_path
   max_history = 5
   wait        = false
 
@@ -122,4 +126,9 @@ resource "helm_release" "genesis" {
     }),
     jsonencode(var.genesis_helm_values)
   ]
+  # inspired by https://stackoverflow.com/a/66501021 to trigger redeployment whenever any of the charts file contents change.
+  set {
+    name  = "chart_sha1"
+    value = sha1(join("", [for f in fileset(local.genesis_helm_chart_path, "**") : filesha1("${local.genesis_helm_chart_path}/${f}")]))
+  }
 }

terraform/helm/genesis/templates/genesis.yaml

@@ -92,7 +92,7 @@ spec:
         imagePullPolicy: {{ .Values.genesis.image.pullPolicy }}
         workingDir: /tmp
         command:
-        - sh
+        - bash
         - -c
         - |-
           set -e

testsuite/forge-test-runner-template.yaml

@@ -18,7 +18,14 @@ spec:
     - -c
     - |
       ulimit -n 1048576
-      forge --suite {FORGE_TEST_SUITE} --duration-secs {FORGE_RUNNER_DURATION_SECS} test k8s-swarm --image-tag {IMAGE_TAG} --upgrade-image-tag {UPGRADE_IMAGE_TAG} --namespace {FORGE_NAMESPACE} {REUSE_ARGS} {KEEP_ARGS} {ENABLE_HAPROXY_ARGS}
+      forge \
+        --suite {FORGE_TEST_SUITE} \
+        --duration-secs {FORGE_RUNNER_DURATION_SECS} {NUM_VALIDATORS_ARGS} {NUM_VALIDATOR_FULLNODES_ARGS} \
+        test k8s-swarm \
+          --image-tag {IMAGE_TAG} \
+          --upgrade-image-tag {UPGRADE_IMAGE_TAG} \
+          --namespace {FORGE_NAMESPACE} {REUSE_ARGS} {KEEP_ARGS} {ENABLE_HAPROXY_ARGS}
+
     env:
     - name: FORGE_TRIGGERED_BY
       value: {FORGE_TRIGGERED_BY}

testsuite/forge.py

@@ -492,6 +492,8 @@ class ForgeContext:
     reuse_args: Sequence[str]
     keep_args: Sequence[str]
     haproxy_args: Sequence[str]
+    num_validators_args: Sequence[str]
+    num_validator_fullnodes_args: Sequence[str]
 
     # aws related options
     aws_account_num: str
@@ -823,6 +825,8 @@ def run(self, context: ForgeContext) -> ForgeResult:
                     "--",
                     "--suite",
                     context.forge_test_suite,
+                    *context.num_validators_args,
+                    *context.num_validator_fullnodes_args,
                     "--duration-secs",
                     context.forge_runner_duration_secs,
                     "test",
@@ -901,9 +905,17 @@ def run(self, context: ForgeContext) -> ForgeResult:
             AWS_ACCOUNT_NUM=context.aws_account_num,
             AWS_REGION=context.aws_region,
             FORGE_NAMESPACE=context.forge_namespace,
-            REUSE_ARGS=context.reuse_args if context.reuse_args else "",
-            KEEP_ARGS=context.keep_args if context.keep_args else "",
-            ENABLE_HAPROXY_ARGS=context.haproxy_args if context.haproxy_args else "",
+            REUSE_ARGS=" ".join(context.reuse_args) if context.reuse_args else "",
+            KEEP_ARGS=" ".join(context.keep_args) if context.keep_args else "",
+            ENABLE_HAPROXY_ARGS=" ".join(context.haproxy_args)
+            if context.haproxy_args
+            else "",
+            NUM_VALIDATORS_ARGS=" ".join(context.num_validators_args)
+            if context.num_validators_args
+            else "",
+            NUM_VALIDATOR_FULLNODES_ARGS=" ".join(context.num_validator_fullnodes_args)
+            if context.num_validator_fullnodes_args
+            else "",
             FORGE_TRIGGERED_BY=forge_triggered_by,
         )
 
@@ -1177,6 +1189,9 @@ def sanitize_forge_resource_name(forge_resource: str) -> str:
 # forge test runner customization
 @envoption("FORGE_RUNNER_MODE", "k8s")
 @envoption("FORGE_CLUSTER_NAME")
+# these override the args in forge-cli
+@envoption("FORGE_NUM_VALIDATORS")
+@envoption("FORGE_NUM_VALIDATOR_FULLNODES")
 @envoption("FORGE_NAMESPACE_KEEP")
 @envoption("FORGE_NAMESPACE_REUSE")
 @envoption("FORGE_ENABLE_HAPROXY")
@@ -1211,6 +1226,8 @@ def test(
     aws_auth_script: Optional[str],
     forge_runner_mode: str,
     forge_cluster_name: Optional[str],
+    forge_num_validators: int,
+    forge_num_validator_fullnodes: int,
     forge_namespace_keep: Optional[str],
     forge_namespace_reuse: Optional[str],
     forge_enable_failpoints: Optional[str],
@@ -1292,6 +1309,10 @@ def test(
 
     assert forge_namespace is not None, "Forge namespace is required"
 
+    # These features and profile flags are set as strings
+    forge_enable_failpoints = forge_enable_failpoints == "true"
+    forge_enable_performance = forge_enable_performance == "true"
+
     if forge_test_suite == "compat":
         # Compat uses 2 image tags
         default_latest_image, second_latest_image = list(
@@ -1342,6 +1363,15 @@ def test(
         reuse_args=["--reuse"] if forge_namespace_reuse else [],
         keep_args=["--keep"] if forge_namespace_keep else [],
         haproxy_args=["--enable-haproxy"] if forge_enable_haproxy else [],
+        num_validators_args=["--num-validators", forge_num_validators]
+        if forge_num_validators
+        else [],
+        num_validator_fullnodes_args=[
+            "--num-validator-fullnodes",
+            forge_num_validator_fullnodes,
+        ]
+        if forge_num_validator_fullnodes
+        else [],
         aws_account_num=aws_account_num,
         aws_region=aws_region,
         forge_image_tag=forge_image_tag,