all: skip noisy/faulty benchmarks + add b.ReportAllocs for every benc…

…hmark (cosmos#8856)

* Skips very noisy benchmarks that end up running only for b.N=1 because
their entire time is spent in setup, and varying parameters doesn't change
much given that the number of stores is what dominates the expense. To
ensure we can provide reliable benchmarks, progressively for the project,
skip these until there is a proper re-work of what the benchmarks need to do

* Previously sub-benchmarks: b.Run(...) did not b.ReportAllocs() due to a faulty
assumption that invoking b.ReportAllocs() at the top would be inherited by
all sub-benchmarks. This change fixes that

Fixes cosmos#8779
Fixes cosmos#8855

Co-authored-by: Alessio Treglia <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

crypto/armor_test.go

@@ -158,12 +158,11 @@ func TestUnarmorInfoBytesErrors(t *testing.T) {
 }
 
 func BenchmarkBcryptGenerateFromPassword(b *testing.B) {
-	b.ReportAllocs()
-
 	passphrase := []byte("passphrase")
 	for securityParam := 9; securityParam < 16; securityParam++ {
 		param := securityParam
 		b.Run(fmt.Sprintf("benchmark-security-param-%d", param), func(b *testing.B) {
+			b.ReportAllocs()
 			saltBytes := tmcrypto.CRandBytes(16)
 			b.ResetTimer()
 			for i := 0; i < b.N; i++ {

crypto/types/compact_bit_array_test.go

@@ -233,6 +233,7 @@ func BenchmarkNumTrueBitsBefore(b *testing.B) {
 	ba, _ := randCompactBitArray(100)
 
 	b.Run("new", func(b *testing.B) {
+		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
 			ba.NumTrueBitsBefore(90)
 		}

store/rootmulti/store_test.go

@@ -689,6 +689,8 @@ func BenchmarkMultistoreSnapshotRestore1M(b *testing.B) {
 }
 
 func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) {
+	b.Skip("Noisy with slow setup time, please see https://github.com/cosmos/cosmos-sdk/issues/8855.")
+
 	b.ReportAllocs()
 	b.StopTimer()
 	source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys)
@@ -717,6 +719,8 @@ func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) {
 }
 
 func benchmarkMultistoreSnapshotRestore(b *testing.B, stores uint8, storeKeys uint64) {
+	b.Skip("Noisy with slow setup time, please see https://github.com/cosmos/cosmos-sdk/issues/8855.")
+
 	b.ReportAllocs()
 	b.StopTimer()
 	source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys)

types/coin_benchmark_test.go

@@ -13,6 +13,7 @@ func BenchmarkCoinsAdditionIntersect(b *testing.B) {
 	b.ReportAllocs()
 	benchmarkingFunc := func(numCoinsA int, numCoinsB int) func(b *testing.B) {
 		return func(b *testing.B) {
+			b.ReportAllocs()
 			coinsA := Coins(make([]Coin, numCoinsA))
 			coinsB := Coins(make([]Coin, numCoinsB))
 
@@ -43,6 +44,7 @@ func BenchmarkCoinsAdditionNoIntersect(b *testing.B) {
 	b.ReportAllocs()
 	benchmarkingFunc := func(numCoinsA int, numCoinsB int) func(b *testing.B) {
 		return func(b *testing.B) {
+			b.ReportAllocs()
 			coinsA := Coins(make([]Coin, numCoinsA))
 			coinsB := Coins(make([]Coin, numCoinsB))
 

x/auth/ante/sigverify_benchmark_test.go

@@ -27,13 +27,15 @@ func BenchmarkSig(b *testing.B) {
 	b.ResetTimer()
 
 	b.Run("secp256k1", func(b *testing.B) {
+		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
 			ok := pkK.VerifySignature(msg, sigK)
 			require.True(ok)
 		}
 	})
 
 	b.Run("secp256r1", func(b *testing.B) {
+		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
 			ok := pkR.VerifySignature(msg, sigR)
 			require.True(ok)

x/auth/keeper/keeper_bench_test.go

@@ -26,6 +26,7 @@ func BenchmarkAccountMapperGetAccountFound(b *testing.B) {
 }
 
 func BenchmarkAccountMapperSetAccount(b *testing.B) {
+	b.ReportAllocs()
 	app, ctx := createTestApp(false)
 
 	b.ResetTimer()

x/bank/bench_test.go

@@ -60,6 +60,7 @@ func BenchmarkOneBankSendTxPerBlock(b *testing.B) {
 }
 
 func BenchmarkOneBankMultiSendTxPerBlock(b *testing.B) {
+	b.ReportAllocs()
 	// Add an account at genesis
 	acc := authtypes.BaseAccount{
 		Address: addr1.String(),