Add default CI multi-seed sim

.circleci/config.yml

@@ -136,6 +136,24 @@ jobs:
             export PATH="$GOBIN:$PATH"
             make test_sim_gaia_fast
 
+  test_sim_gaia_multi_seed:
+    <<: *defaults
+    parallelism: 1
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - checkout
+      - run:
+          name: dependencies
+          command: |
+            export PATH="$GOBIN:$PATH"
+            make get_vendor_deps
+      - run:
+          name: Test multi-seed Gaia simulation
+          command: |
+            export PATH="$GOBIN:$PATH"
+            make test_sim_gaia_multi_seed
+
   test_cover:
     <<: *defaults
     parallelism: 4

Makefile

@@ -162,9 +162,9 @@ test_sim_gaia_fast:
 	@echo "Running quick Gaia simulation. This may take several minutes..."
 	@go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=400 -SimulationBlockSize=200 -SimulationCommit=true -v -timeout 24h
 
-test_sim_gaia_full:
-	@echo "Running full multi-seed Gaia simulation. This may take awhile!"
-	@sh scripts/multisim.sh
+test_sim_gaia_multi_seed:
+	@echo "Running multi-seed Gaia simulation. This may take awhile!"
+	@sh scripts/multisim.sh 10
 
 SIM_NUM_BLOCKS ?= 210
 SIM_BLOCK_SIZE ?= 200
@@ -241,4 +241,4 @@ localnet-stop:
 check_tools check_dev_tools get_tools get_dev_tools get_vendor_deps draw_deps test test_cli test_unit \
 test_cover test_lint benchmark devdoc_init devdoc devdoc_save devdoc_update \
 build-linux build-docker-gaiadnode localnet-start localnet-stop \
-format check-ledger test_sim_gaia_nondeterminism test_sim_modules test_sim_gaia_fast test_sim_gaia_slow update_tools update_dev_tools
+format check-ledger test_sim_gaia_nondeterminism test_sim_modules test_sim_gaia_fast test_sim_gaia_multi_seed update_tools update_dev_tools

scripts/multisim.sh

@@ -1,8 +1,10 @@
 #!/bin/bash
 
 seeds=(1 2 4 7 9 20 32 123 4728 37827 981928 87821 891823782 989182 89182391)
+blocks=$1
 
-echo "Running multi-seed simulation with seeds: ${seeds[@]}"
+echo "Running multi-seed simulation with seeds ${seeds[@]}"
+echo "Running $blocks blocks per seed"
 echo "Edit scripts/multisim.sh to add new seeds. Keeping parameters in the file makes failures easy to reproduce."
 echo "This script will kill all sub-simulations on SIGINT/SIGTERM/EXIT (i.e. Ctrl-C)."
 
@@ -16,7 +18,7 @@ sim() {
 	echo "Running full Gaia simulation with seed $seed. This may take awhile!"
   file="$tmpdir/gaia-simulation-seed-$seed-date-$(date -Iseconds -u).stdout"
   echo "Writing stdout to $file..."
-	go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=1000 \
+	go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=$blocks \
     -SimulationVerbose=true -SimulationCommit=true -SimulationSeed=$seed -v -timeout 24h > $file
 }
 
@@ -37,10 +39,12 @@ i=0
 for pid in ${pids[*]}; do
   wait $pid
   last=$?
+  seed=${seeds[${i}]}
   if [ $last -ne 0 ]; then
-    seed=${seeds[${i}]}
     echo "Simulation with seed $seed failed!"
     code=1
+  else
+    echo "Simulation with seed $seed OK!"
   fi
   i=$(($i+1))
 done