HARMONY-1564: Add retry and debug for bootstrap-harmony and wait for …

…giovanni-adapter to be ready.
cbrownstevenson · Jan 25, 2024 · d19caaa · d19caaa
1 parent 1b742ac
commit d19caaa
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 1 deletion.
diff --git a/.github/workflows/harmony-in-a-box.yml b/.github/workflows/harmony-in-a-box.yml
@@ -55,7 +55,11 @@ jobs:
       run: docker images
 
     - name: Start harmony in a box
-      run: KUBE_CONTEXT=colima EXEC_CONTEXT=workflow bin/bootstrap-harmony
+      run: |
+        source bin/helper
+        export KUBE_CONTEXT=colima
+        export EXEC_CONTEXT=workflow
+        retry_command 5 bin/bootstrap-harmony
       env:
         OAUTH_CLIENT_ID: ${{ secrets.OAUTH_CLIENT_ID }}
         OAUTH_UID: ${{ secrets.OAUTH_UID }}
@@ -87,10 +91,31 @@ jobs:
           exit 1
         fi
 
+    - name: Wait for Giovanni pod to be ready
+      run: |
+        for i in {1..10}; do
+          kubectl -n harmony get pods
+          running_count=$(kubectl -n harmony get pods -l name="giovanni-adapter" --no-headers | grep "2/2" | wc -l)
+          if [ "$running_count" -eq 1 ]; then
+            echo "Giovanni pod is ready! Continue with the workflow."
+            break
+          else
+            echo "Giovanni pod is not ready, wait..."
+            sleep 60
+          fi
+        done
+
+        # If after 10 attempts, running_count is still not 1, fail the workflow
+        if [ "$running_count" -ne 1 ]; then
+          echo "Failed: Running count is not 1 after 10 attempts."
+          exit 1
+        fi
+
     - name: Test Giovanni request
       id: giovanni
       run: |
         GIOVANNI_RESP=$(curl -Ln -bj "http://localhost:3000/C1225808238-GES_DISC/ogc-api-coverages/1.0.0/collections/Grid%2FprecipitationCal/coverage/rangeset?format=text%2Fcsv&point=0.76,-3.8&subset=time(%222020-01-06T12%3A00%3A00Z%22%3A%222020-01-06T16%3A00%3A00Z%22)")
+        echo "Giovanni request response: $GIOVANNI_RESP"
         GIOVANNI_URL=$(echo "$GIOVANNI_RESP" | jq -r '.links[] | select(.rel == "self") | .href')
         echo "status_url: $GIOVANNI_URL"
         echo "status_url=$GIOVANNI_URL" >> "$GITHUB_OUTPUT"

diff --git a/bin/helper b/bin/helper
@@ -12,6 +12,14 @@ retry_command() {
 
     if [ $count -lt $retries ]; then
       echo "Retrying command ($count/$retries)..."
+      # add additional debug info when the command is bin/bootstrap-harmony
+      if [[ "$@" == "bin/bootstrap-harmony" ]]; then
+        echo "Get extra debug info for command bin/bootstrap-harmony..."
+        kubectl cluster-info
+        kubectl -n harmony get pods
+        top -l 1 -s 0 | grep PhysMem
+        df -h
+      fi
       sleep 5
     else
       echo "Retry limit exceeded. Exiting..."