[circleci] retry docker-compose testnet startup

cmepthuklolka1 · May 10, 2022 · 337ca6b · 337ca6b
1 parent 8224594
commit 337ca6b
Showing 1 changed file with 20 additions and 15 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -418,30 +418,35 @@ commands:
             echo 'export APTOS_NODE_URL="http://localhost:8080"' >> $BASH_ENV
             echo 'export APTOS_FAUCET_URL="http://localhost:8000"' >> $BASH_ENV
       - run:
-          name: Start local testnet docker-compose
+          name: Start local testnet docker-compose and verify up
           shell: /bin/bash
           working_directory: docker/compose/validator-testnet
           command: |
             export VALIDATOR_IMAGE_REPO=${AWS_ECR_ACCOUNT_URL}/aptos/validator
             export FAUCET_IMAGE_REPO=${AWS_ECR_ACCOUNT_URL}/aptos/faucet
-            docker-compose up -d
-      - run:
-          name: Verify testnet docker-compose up
-          shell: /bin/bash
-          working_directory: docker/compose/validator-testnet
-          command: |
             DOCKER_COMPOSE_EXIT_CODE=1
-            for i in $(seq 60); do
-              curl -s -w "\n%{http_code}\n" localhost:8080 | tee >(tail -1 > validator_ret.txt)
-              curl -s -w "\n%{http_code}\n" -X POST 'localhost:8000/mint?pub_key=459c77a38803bd53f3adee52703810e3a74fd7c46952c497e75afb0a7932586d&amount=20000000' \
-                | tee >(tail -1 > faucet_ret.txt)
-              if [ "$(cat validator_ret.txt)" = "200" ] && [ "$(cat faucet_ret.txt)" = "200" ]; then
-                echo "Both validator and faucet healthy"
-                DOCKER_COMPOSE_EXIT_CODE=0
+
+            # try a total of 5 times (5 min timeout) to bring up a docker-compose testnet
+            for i in $(seq 1 5); do
+              docker-compose down || true # try bringing down ay existing testnets
+              docker-compose up -d
+              for j in $(seq 60); do
+                curl -s -w "\n%{http_code}\n" localhost:8080 | tee >(tail -1 > validator_ret.txt)
+                curl -s -w "\n%{http_code}\n" -X POST 'localhost:8000/mint?pub_key=459c77a38803bd53f3adee52703810e3a74fd7c46952c497e75afb0a7932586d&amount=20000000' \
+                  | tee >(tail -1 > faucet_ret.txt)
+                if [ "$(cat validator_ret.txt)" = "200" ] && [ "$(cat faucet_ret.txt)" = "200" ]; then
+                  echo "Both validator and faucet healthy"
+                  DOCKER_COMPOSE_EXIT_CODE=0
+                  break
+                fi
+                sleep 1
+              done
+              if [ "$DOCKER_COMPOSE_EXIT_CODE" = "0" ]; then
                 break
               fi
-              sleep 1
+              echo "DOCKER_COMPOSE_EXIT_CODE = $DOCKER_COMPOSE_EXIT_CODE, trying compose again..."
             done
+
             echo "export DOCKER_COMPOSE_EXIT_CODE=$DOCKER_COMPOSE_EXIT_CODE" >> $BASH_ENV
       - run:
           name: Check docker-compose resources