Merge PR cosmos#2930: Simulation spring cleaning

* Update PENDING.md
* Add simple period for expensive invariants
* Remove individual module simulations
* Simulate a few more blocks
* Add README explaining reason for shell scripts
* Deduplicate scripts, log exact replication command on failure
* Refactor invariants to take sdk.Context instead of baseapp.BaseApp
* Reference all issues in PENDING.md entry
* Remove no longer used simulation.RandSetup
* Bug fixes
* Address @rigelrozanski comments
* Fix typo

.circleci/config.yml

@@ -90,24 +90,6 @@ jobs:
             make test_cli
             make test_examples
 
-  test_sim_modules:
-    <<: *defaults
-    parallelism: 1
-    steps:
-      - attach_workspace:
-          at: /tmp/workspace
-      - checkout
-      - run:
-          name: dependencies
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make get_vendor_deps
-      - run:
-          name: Test individual module simulations
-          command: |
-            export PATH="$GOBIN:$PATH"
-            make test_sim_modules
-
   test_sim_gaia_nondeterminism:
     <<: *defaults
     parallelism: 1
@@ -307,9 +289,6 @@ workflows:
       - integration_tests:
           requires:
             - setup_dependencies
-      - test_sim_modules:
-          requires:
-            - setup_dependencies
       - test_sim_gaia_nondeterminism:
           requires:
             - setup_dependencies

Makefile

@@ -164,29 +164,25 @@ test_unit:
 test_race:
 	@VERSION=$(VERSION) go test -race $(PACKAGES_NOSIMULATION)
 
-test_sim_modules:
-	@echo "Running individual module simulations..."
-	@go test $(PACKAGES_SIMTEST)
-
 test_sim_gaia_nondeterminism:
 	@echo "Running nondeterminism test..."
 	@go test ./cmd/gaia/app -run TestAppStateDeterminism -SimulationEnabled=true -v -timeout 10m
 
 test_sim_gaia_fast:
 	@echo "Running quick Gaia simulation. This may take several minutes..."
-	@go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=500 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -v -timeout 24h
+	@go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -v -timeout 24h
 
 test_sim_gaia_import_export:
 	@echo "Running Gaia import/export simulation. This may take several minutes..."
-	@bash scripts/import-export-sim.sh 50
+	@bash scripts/multisim.sh 50 TestGaiaImportExport
 
 test_sim_gaia_simulation_after_import:
 	@echo "Running Gaia simulation-after-import. This may take several minutes..."
-	@bash scripts/simulation-after-import.sh 50
+	@bash scripts/multisim.sh 50 TestGaiaSimulationAfterImport
 
 test_sim_gaia_multi_seed:
 	@echo "Running multi-seed Gaia simulation. This may take awhile!"
-	@bash scripts/multisim.sh 25
+	@bash scripts/multisim.sh 50 TestFullGaiaSimulation
 
 SIM_NUM_BLOCKS ?= 500
 SIM_BLOCK_SIZE ?= 200

PENDING.md

@@ -74,6 +74,7 @@ IMPROVEMENTS
   - [#110](https://github.com/tendermint/devops/issues/110) Updated CircleCI job to trigger website build when cosmos docs are updated.
 
 * SDK
+ - [x/mock/simulation] \#2832, \#2885, \#2873, \#2902 Simulation cleanup
  - [x/mock/simulation] [\#2720] major cleanup, introduction of helper objects, reorganization
  - \#2821 Codespaces are now strings
  - [types] #2776 Improve safety of `Coin` and `Coins` types. Various functions

cmd/gaia/app/invariants.go

@@ -8,6 +8,7 @@ import (
 	distrsim "github.com/cosmos/cosmos-sdk/x/distribution/simulation"
 	"github.com/cosmos/cosmos-sdk/x/mock/simulation"
 	stakesim "github.com/cosmos/cosmos-sdk/x/stake/simulation"
+	abci "github.com/tendermint/tendermint/abci/types"
 )
 
 func (app *GaiaApp) runtimeInvariants() []simulation.Invariant {
@@ -23,8 +24,9 @@ func (app *GaiaApp) runtimeInvariants() []simulation.Invariant {
 func (app *GaiaApp) assertRuntimeInvariants() {
 	invariants := app.runtimeInvariants()
 	start := time.Now()
+	ctx := app.NewContext(false, abci.Header{Height: app.LastBlockHeight() + 1})
 	for _, inv := range invariants {
-		if err := inv(app.BaseApp); err != nil {
+		if err := inv(ctx); err != nil {
 			panic(fmt.Errorf("invariant broken: %s", err))
 		}
 	}

cmd/gaia/app/sim_test.go

@@ -40,6 +40,7 @@ var (
 	enabled   bool
 	verbose   bool
 	commit    bool
+	period    int
 )
 
 func init() {
@@ -49,6 +50,7 @@ func init() {
 	flag.BoolVar(&enabled, "SimulationEnabled", false, "Enable the simulation")
 	flag.BoolVar(&verbose, "SimulationVerbose", false, "Verbose log output")
 	flag.BoolVar(&commit, "SimulationCommit", false, "Have the simulation commit")
+	flag.IntVar(&period, "SimulationPeriod", 100, "Run slow invariants only once every period assertions")
 }
 
 func appStateFn(r *rand.Rand, accs []simulation.Account) json.RawMessage {
@@ -185,12 +187,12 @@ func testAndRunTxs(app *GaiaApp) []simulation.WeightedOperation {
 
 func invariants(app *GaiaApp) []simulation.Invariant {
 	return []simulation.Invariant{
-		banksim.NonnegativeBalanceInvariant(app.accountKeeper),
-		govsim.AllInvariants(),
-		distrsim.AllInvariants(app.distrKeeper, app.stakeKeeper),
-		stakesim.AllInvariants(app.bankKeeper, app.stakeKeeper,
-			app.feeCollectionKeeper, app.distrKeeper, app.accountKeeper),
-		slashingsim.AllInvariants(),
+		simulation.PeriodicInvariant(banksim.NonnegativeBalanceInvariant(app.accountKeeper), period, 0),
+		simulation.PeriodicInvariant(govsim.AllInvariants(), period, 0),
+		simulation.PeriodicInvariant(distrsim.AllInvariants(app.distrKeeper, app.stakeKeeper), period, 0),
+		simulation.PeriodicInvariant(stakesim.AllInvariants(app.bankKeeper, app.stakeKeeper,
+			app.feeCollectionKeeper, app.distrKeeper, app.accountKeeper), period, 0),
+		simulation.PeriodicInvariant(slashingsim.AllInvariants(), period, 0),
 	}
 }
 
@@ -219,7 +221,6 @@ func BenchmarkFullGaiaSimulation(b *testing.B) {
 	_, err := simulation.SimulateFromSeed(
 		b, app.BaseApp, appStateFn, seed,
 		testAndRunTxs(app),
-		[]simulation.RandSetup{},
 		invariants(app), // these shouldn't get ran
 		numBlocks,
 		blockSize,
@@ -262,7 +263,6 @@ func TestFullGaiaSimulation(t *testing.T) {
 	_, err := simulation.SimulateFromSeed(
 		t, app.BaseApp, appStateFn, seed,
 		testAndRunTxs(app),
-		[]simulation.RandSetup{},
 		invariants(app),
 		numBlocks,
 		blockSize,
@@ -304,7 +304,6 @@ func TestGaiaImportExport(t *testing.T) {
 	_, err := simulation.SimulateFromSeed(
 		t, app.BaseApp, appStateFn, seed,
 		testAndRunTxs(app),
-		[]simulation.RandSetup{},
 		invariants(app),
 		numBlocks,
 		blockSize,
@@ -401,7 +400,6 @@ func TestGaiaSimulationAfterImport(t *testing.T) {
 	stopEarly, err := simulation.SimulateFromSeed(
 		t, app.BaseApp, appStateFn, seed,
 		testAndRunTxs(app),
-		[]simulation.RandSetup{},
 		invariants(app),
 		numBlocks,
 		blockSize,
@@ -447,7 +445,6 @@ func TestGaiaSimulationAfterImport(t *testing.T) {
 	_, err = simulation.SimulateFromSeed(
 		t, newApp.BaseApp, appStateFn, seed,
 		testAndRunTxs(newApp),
-		[]simulation.RandSetup{},
 		invariants(newApp),
 		numBlocks,
 		blockSize,
@@ -479,13 +476,11 @@ func TestAppStateDeterminism(t *testing.T) {
 			simulation.SimulateFromSeed(
 				t, app.BaseApp, appStateFn, seed,
 				testAndRunTxs(app),
-				[]simulation.RandSetup{},
 				[]simulation.Invariant{},
 				50,
 				100,
 				true,
 			)
-			//app.Commit()
 			appHash := app.LastCommitID().Hash
 			appHashList[j] = appHash
 		}

scripts/README.md

@@ -0,0 +1,3 @@
+Generally we should avoid shell scripting and write tests purely in Golang.
+However, some libraries are not Goroutine-safe (e.g. app simulations cannot be run safely in parallel),
+and OS-native threading may be more efficient for many parallel simulations, so we use shell scripts here.

scripts/multisim.sh

@@ -3,9 +3,11 @@
 seeds=(1 2 4 7 9 20 32 123 124 582 1893 2989 3012 4728 37827 981928 87821 891823782 989182 89182391 \
 11 22 44 77 99 2020 3232 123123 124124 582582 18931893 29892989 30123012 47284728 37827)
 blocks=$1
+testname=$2
 
 echo "Running multi-seed simulation with seeds ${seeds[@]}"
 echo "Running $blocks blocks per seed"
+echo "Running test $testname"
 echo "Edit scripts/multisim.sh to add new seeds. Keeping parameters in the file makes failures easy to reproduce."
 echo "This script will kill all sub-simulations on SIGINT/SIGTERM (i.e. Ctrl-C)."
 
@@ -16,10 +18,10 @@ echo "Using temporary log directory: $tmpdir"
 
 sim() {
   seed=$1
-	echo "Running full Gaia simulation with seed $seed. This may take awhile!"
+	echo "Running Gaia simulation with seed $seed. This may take awhile!"
   file="$tmpdir/gaia-simulation-seed-$seed-date-$(date -u +"%Y-%m-%dT%H:%M:%S+00:00").stdout"
   echo "Writing stdout to $file..."
-	go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=$blocks \
+	go test ./cmd/gaia/app -run $testname -SimulationEnabled=true -SimulationNumBlocks=$blocks \
     -SimulationVerbose=true -SimulationCommit=true -SimulationSeed=$seed -v -timeout 24h > $file
 }
 
@@ -44,6 +46,7 @@ for pid in ${pids[*]}; do
   if [ $last -ne 0 ]
   then
     echo "Simulation with seed $seed failed!"
+    echo "To replicate, run 'go test ./cmd/gaia/app -run $testname -SimulationEnabled=true -SimulationNumBlocks=$blocks -SimulationVerbose=true -SimulationCommit=true -SimulationSeed=$seed -v -timeout 24h'"
     code=1
   else
     echo "Simulation with seed $seed OK"

x/bank/simulation/invariants.go

@@ -4,18 +4,15 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/cosmos/cosmos-sdk/baseapp"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/cosmos/cosmos-sdk/x/auth"
 	"github.com/cosmos/cosmos-sdk/x/mock"
 	"github.com/cosmos/cosmos-sdk/x/mock/simulation"
-	abci "github.com/tendermint/tendermint/abci/types"
 )
 
 // NonnegativeBalanceInvariant checks that all accounts in the application have non-negative balances
 func NonnegativeBalanceInvariant(mapper auth.AccountKeeper) simulation.Invariant {
-	return func(app *baseapp.BaseApp) error {
-		ctx := app.NewContext(false, abci.Header{})
+	return func(ctx sdk.Context) error {
 		accts := mock.GetAllAccounts(mapper, ctx)
 		for _, acc := range accts {
 			coins := acc.GetCoins()
@@ -32,8 +29,7 @@ func NonnegativeBalanceInvariant(mapper auth.AccountKeeper) simulation.Invariant
 // TotalCoinsInvariant checks that the sum of the coins across all accounts
 // is what is expected
 func TotalCoinsInvariant(mapper auth.AccountKeeper, totalSupplyFn func() sdk.Coins) simulation.Invariant {
-	return func(app *baseapp.BaseApp) error {
-		ctx := app.NewContext(false, abci.Header{})
+	return func(ctx sdk.Context) error {
 		totalCoins := sdk.Coins{}
 
 		chkAccount := func(acc auth.Account) bool {