[gha] run multiple forge tests on nightly

EnriqueTejeda · Jul 30, 2022 · cdfa305 · cdfa305
1 parent 2259562
commit cdfa305
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 23 deletions.
diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml
@@ -141,7 +141,6 @@ jobs:
     secrets: inherit
     with:
       GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
-      merge_or_canary: ${{ github.event.pull_request.auto_merge != null && 'merge' || 'canary' }}
       # Use the cache ID as the Forge namespace so we can limit Forge test concurrency on k8s, since Forge
       # test lifecycle is separate from that of GHA. This protects us from the case where many Forge tests are triggered
       # by this GHA. If there is a Forge namespace collision, Forge will pre-empt the existing test running in the namespace.

diff --git a/.github/workflows/pre-release-continuous-test.yaml b/.github/workflows/pre-release-continuous-test.yaml
@@ -10,14 +10,21 @@ on:
     branches:
       - pre-release-continuous-test
   schedule:
-    # Run hourly
-    - cron: "0 * * * *"
+    # Run every 3 hours
+    - cron: "0 */3 * * *"
 
 jobs:
-  run-forge:
+  # run two concurrent forge test jobs on the same cluster
+  # they must use different namespaces, or they will preempt each other
+  run-forge-0:
     uses: ./.github/workflows/run-forge.yaml
     secrets: inherit
     with:
-      GIT_SHA: ${{ github.sha }}
-      merge_or_canary: canary
-      FORGE_NAMESPACE: continuous
+      FORGE_NAMESPACE: forge-continuous-0
+      FORGE_CLUSTER_NAME: aptos-forge-1
+  run-forge-1:
+    uses: ./.github/workflows/run-forge.yaml
+    secrets: inherit
+    with:
+      FORGE_NAMESPACE: forge-continuous-1
+      FORGE_CLUSTER_NAME: aptos-forge-1
diff --git a/.github/workflows/run-forge.yaml b/.github/workflows/run-forge.yaml
@@ -5,17 +5,18 @@ on:
   workflow_call:
     inputs:
       GIT_SHA:
-        required: true
+        required: false
         type: string
         description:
-      merge_or_canary:
-        required: true
-        type: string
-        description: "indicate whether this is a forge run for an auto-merge or a canary, must be `merge` or `canary`"
       FORGE_NAMESPACE:
         required: true
         type: string
         description: The Forge k8s namespace to be used for test. This value should manage Forge test concurrency. It may be truncated.
+      FORGE_CLUSTER_NAME:
+        required: true
+        type: string
+        default: aptos-forge-0
+        description: The Forge k8s cluster to be used for test
 
 env:
   AWS_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
@@ -25,7 +26,7 @@ env:
   IMAGE_TAG: ${{ inputs.GIT_SHA }}
   FORGE_ENABLED: ${{ secrets.FORGE_ENABLED }}
   FORGE_BLOCKING: ${{ secrets.FORGE_BLOCKING }}
-  FORGE_CLUSTER_NAME: ${{ secrets.FORGE_CLUSTER_NAME }}
+  FORGE_CLUSTER_NAME: ${{ inputs.FORGE_CLUSTER_NAME }}
   FORGE_OUTPUT: forge_output.txt
   FORGE_REPORT: forge_report.json
   FORGE_COMMENT: forge_comment.txt
@@ -44,6 +45,8 @@ jobs:
         if: env.FORGE_ENABLED == 'true'
         with:
           ref: ${{ inputs.GIT_SHA }}
+          # get the last 10 commits if IMAGE_TAG is not specified
+          fetch-depth: env.IMAGE_TAG != null && 0 || 10
       - name: Set kubectl context
         if: env.FORGE_ENABLED == 'true'
         run: aws eks update-kubeconfig --name $FORGE_CLUSTER_NAME

diff --git a/testsuite/run_forge.sh b/testsuite/run_forge.sh
@@ -76,17 +76,34 @@ set_forge_namespace() {
 
 # Set an image tag to use
 set_image_tag() {
-    if [ -z "$IMAGE_TAG" ]; then
-        IMAGE_TAG_DEFAULT=$(git rev-parse HEAD)
-        echo "IMAGE_TAG not set, defaulting to current HEAD commit as tag: ${IMAGE_TAG_DEFAULT}"
-        IMAGE_TAG=${IMAGE_TAG_DEFAULT}
-    fi
     echo "Ensure image exists"
-    img=$(aws ecr describe-images --repository-name="aptos/validator" --image-ids=imageTag=$IMAGE_TAG)
-    if [ $? != 0 ]; then
-        echo "IMAGE_TAG does not exist in ECR: ${IMAGE_TAG}. Make sure your commit has been pushed to GitHub previously."
-        echo "If you're trying to run the code from your PR, apply the label 'CICD:build-images' and wait for the builds to finish."
-        exit 1
+    # if IMAGE_TAG not set, check the last few commits on HEAD
+    if [ -z "$IMAGE_TAG" ]; then
+        echo "IMAGE_TAG not set, trying the latest commits before HEAD"
+        commit_threshold=5
+        for i in $(seq 0 $commit_threshold); do
+            IMAGE_TAG_DEFAULT=$(git rev-parse HEAD~$i)
+            echo "Trying tag: ${IMAGE_TAG_DEFAULT}"
+            git log --format=%B -n 1 $IMAGE_TAG_DEFAULT
+            img=$(aws ecr describe-images --repository-name="aptos/validator" --image-ids=imageTag=$IMAGE_TAG_DEFAULT)
+            if [ "$?" -eq 0 ]; then
+                echo "Image tag exists. Using tag: ${IMAGE_TAG_DEFAULT}"
+                IMAGE_TAG=$IMAGE_TAG_DEFAULT
+                return 0
+            fi
+        done
+        # if IMAGE_TAG still not set after checking HEAD,
+        if [ -z "$IMAGE_TAG"]; then
+            echo "None of the last ${commit_threshold} commits have been built and pushed"
+            exit 1
+        fi
+    else
+        img=$(aws ecr describe-images --repository-name="aptos/validator" --image-ids=imageTag=$IMAGE_TAG)
+        if [ "$?" -ne 0 ]; then
+            echo "IMAGE_TAG does not exist in ECR: ${IMAGE_TAG}. Make sure your commit has been pushed to GitHub previously."
+            echo "If you're trying to run the code from your PR, apply the label 'CICD:build-images' and wait for the builds to finish."
+            exit 1
+        fi
     fi
 }