Sequencer bench tests (0xPolygonHermez#426)

* working on bench tests for the sequencer

* working on bench tests for the sequencer

* working on bench tests for the sequencer

* fixed getPendingTxs request

* fixed lint errors

* change defer order

* fixed pr comments

* deleted redundant batchProcessorLock

* fixed go import

aggregator/aggregator.go

@@ -132,13 +132,13 @@ func (a *Aggregator) Start() {
 			}
 
 			// 4. send zki + txs to the prover
-			stateRootConsolidated, err := a.State.GetStateRootByBatchNumber(lastConsolidatedBatch.Number().Uint64())
+			stateRootConsolidated, err := a.State.GetStateRootByBatchNumber(a.ctx, lastConsolidatedBatch.Number().Uint64())
 			if err != nil {
 				log.Warnf("failed to get current state root, err: %v", err)
 				continue
 			}
 
-			stateRootToConsolidate, err := a.State.GetStateRootByBatchNumber(batchToConsolidate.Number().Uint64())
+			stateRootToConsolidate, err := a.State.GetStateRootByBatchNumber(a.ctx, batchToConsolidate.Number().Uint64())
 			if err != nil {
 				log.Warnf("failed to get state root to consolidate, err: %v", err)
 				continue

etherman/etherman.go

@@ -201,7 +201,6 @@ func (etherMan *Client) sendBatch(ctx context.Context, opts *bind.TransactOpts,
 		log.Error("error coverting hex string to []byte. Error: ", err)
 		return nil, errors.New("error coverting hex string to []byte. Error: " + err.Error())
 	}
-
 	tx, err := etherMan.PoE.SendBatch(etherMan.auth, callData, maticAmount)
 	if err != nil {
 		return nil, err

go.mod

@@ -3,10 +3,11 @@ module github.com/hermeznetwork/hermez-core
 go 1.17
 
 require (
-	github.com/ethereum/go-ethereum v1.10.12
+	github.com/ethereum/go-ethereum v1.10.16
 	github.com/gobuffalo/packr/v2 v2.8.3
 	github.com/hermeznetwork/tracerr v0.3.2
 	github.com/iden3/go-iden3-crypto v0.0.13
+	github.com/jackc/pgconn v1.10.0
 	github.com/jackc/pgtype v1.8.1
 	github.com/jackc/pgx/v4 v4.13.0
 	github.com/mitchellh/mapstructure v1.4.3
@@ -29,7 +30,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea // indirect
+	github.com/deckarep/golang-set v1.8.0 // indirect
 	github.com/edsrzf/mmap-go v1.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-kit/kit v0.9.0 // indirect
@@ -48,13 +49,11 @@ require (
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/holiman/uint256 v1.2.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
-	github.com/jackc/pgconn v1.10.0 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgproto3/v2 v2.1.1 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
 	github.com/jackc/puddle v1.1.3 // indirect
-	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/karrick/godirwalk v1.16.1 // indirect
 	github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e // indirect
 	github.com/magiconair/properties v1.8.5 // indirect

go.sum

@@ -160,8 +160,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dchest/blake512 v1.0.0/go.mod h1:FV1x7xPPLWukZlpDpWQ88rF/SFwZ5qbskrzhLMB92JI=
-github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0=
-github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
+github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4=
+github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
 github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
 github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M=
 github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw=
@@ -185,8 +185,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/ethereum/go-ethereum v1.10.12 h1:el/KddB3gLEsnNgGQ3SQuZuiZjwnFTYHe5TwUet5Om4=
-github.com/ethereum/go-ethereum v1.10.12/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw=
+github.com/ethereum/go-ethereum v1.10.16 h1:3oPrumn0bCW/idjcxMn5YYVCdK7VzJYIvwGZUGLEaoc=
+github.com/ethereum/go-ethereum v1.10.16/go.mod h1:Anj6cxczl+AHy63o4X9O8yWNHuN5wMpfb8MAnHkWn7Y=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c=
@@ -333,7 +333,7 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
+github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
@@ -447,7 +447,6 @@ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0f
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
-github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
 github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU=
@@ -466,7 +465,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
-github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
+github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
 github.com/karrick/godirwalk v1.15.8/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=

pool/postgres.go

@@ -64,6 +64,7 @@ func (p *PostgresPool) GetPendingTxs(ctx context.Context) ([]Transaction, error)
 	if err != nil {
 		return nil, err
 	}
+	defer rows.Close()
 
 	txs := make([]Transaction, 0, len(rows.RawValues()))
 	for rows.Next() {
@@ -139,6 +140,8 @@ func (p *PostgresPool) GetGasPrice(ctx context.Context) (uint64, error) {
 		return 0, err
 	}
 
+	defer rows.Close()
+
 	gasPrice := uint64(0)
 
 	for rows.Next() {

sequencer/sequencer.go

@@ -87,11 +87,14 @@ func NewSequencer(cfg Config, pool txPool, state state.State, ethMan etherman) (
 
 // Start starts the sequencer
 func (s *Sequencer) Start() {
+	ticker := time.NewTicker(s.cfg.IntervalToProposeBatch.Duration)
+	defer ticker.Stop()
 	// Infinite for loop:
 	for {
+		s.tryProposeBatch()
 		select {
-		case <-time.After(s.cfg.IntervalToProposeBatch.Duration):
-			s.tryProposeBatch()
+		case <-ticker.C:
+			// nothing
 		case <-s.ctx.Done():
 			return
 		}
@@ -172,7 +175,6 @@ func (s *Sequencer) tryProposeBatch() {
 			return
 		}
 		log.Infof("batch proposal sent successfully: %s", sendBatchTx.Hash().Hex())
-
 		// update txs in the pool as selected
 		err = s.Pool.UpdateTxsState(s.ctx, selectedTxsHashes, pool.TxStateSelected)
 		if err != nil {

sequencer/strategy/txprofitabilitychecker/txprofitabilitychecker.go

@@ -115,7 +115,7 @@ func NewTxProfitabilityCheckerAcceptAll(ethman etherman, state state.State, inte
 	}
 }
 
-// IsProfitable always returns true
+// IsProfitable always returns true, until it's failed to get sequencer collateral
 func (pc *AcceptAll) IsProfitable(ctx context.Context, txs []*types.Transaction) (bool, *big.Int, error) {
 	collateral, err := pc.EthMan.GetCurrentSequencerCollateral()
 	if err != nil {

sequencer/strategy/txselector/txselector.go

@@ -57,8 +57,8 @@ func NewTxSelectorBase(cfg Config) TxSelector {
 }
 
 // SelectTxs process txs and split valid txs into batches of txs. This process should be completed in less than selectionTime
-func (t *Base) SelectTxs(batchProcessor batchProcessor, pendingTxs []pool.Transaction, sequencerAddress common.Address) ([]*types.Transaction, []string, []string, error) {
-	sortedTxs := t.TxSorter.SortTxs(pendingTxs)
+func (b *Base) SelectTxs(batchProcessor batchProcessor, pendingTxs []pool.Transaction, sequencerAddress common.Address) ([]*types.Transaction, []string, []string, error) {
+	sortedTxs := b.TxSorter.SortTxs(pendingTxs)
 	var (
 		selectedTxs                         []*types.Transaction
 		selectedTxsHashes, invalidTxsHashes []string

state/pgstatestorage/pgstatestorage.go

@@ -465,6 +465,8 @@ func (s *PostgresStorage) GetTxsByBatchNum(ctx context.Context, batchNum uint64)
 		return nil, err
 	}
 
+	defer rows.Close()
+
 	txs := make([]*types.Transaction, 0, len(rows.RawValues()))
 	var (
 		encoded string

state/state.go

@@ -31,7 +31,7 @@ type State interface {
 	EstimateGas(transaction *types.Transaction) uint64
 	GetNonce(address common.Address, batchNumber uint64) (uint64, error)
 	SetGenesis(ctx context.Context, genesis Genesis) error
-	GetStateRootByBatchNumber(batchNumber uint64) ([]byte, error)
+	GetStateRootByBatchNumber(ctx context.Context, batchNumber uint64) ([]byte, error)
 	Storage
 }
 
@@ -99,17 +99,17 @@ func NewState(cfg Config, storage Storage, tree merkletree) State {
 
 // NewBatchProcessor creates a new batch processor
 func (s *BasicState) NewBatchProcessor(sequencerAddress common.Address, lastBatchNumber uint64) (*BasicBatchProcessor, error) {
+	ctx := context.Background()
+
 	// init correct state root from previous batch
-	stateRoot, err := s.GetStateRootByBatchNumber(lastBatchNumber)
+	stateRoot, err := s.GetStateRootByBatchNumber(ctx, lastBatchNumber)
 	if err != nil {
 		return nil, err
 	}
 
-	ctx := context.Background()
-
 	// Get Sequencer's Chain ID
 	chainID := s.cfg.DefaultChainID
-	sq, err := s.GetSequencer(context.Background(), sequencerAddress)
+	sq, err := s.GetSequencer(ctx, sequencerAddress)
 	if err == nil {
 		chainID = sq.ChainID.Uint64()
 	}
@@ -149,9 +149,7 @@ func (s *BasicState) GetStateRoot(ctx context.Context, virtual bool) ([]byte, er
 }
 
 // GetStateRootByBatchNumber returns state root by batch number from the MT
-func (s *BasicState) GetStateRootByBatchNumber(batchNumber uint64) ([]byte, error) {
-	ctx := context.Background()
-
+func (s *BasicState) GetStateRootByBatchNumber(ctx context.Context, batchNumber uint64) ([]byte, error) {
 	batch, err := s.GetBatchByNumber(ctx, batchNumber)
 	if err != nil {
 		return nil, err
@@ -166,7 +164,8 @@ func (s *BasicState) GetStateRootByBatchNumber(batchNumber uint64) ([]byte, erro
 
 // GetBalance from a given address
 func (s *BasicState) GetBalance(address common.Address, batchNumber uint64) (*big.Int, error) {
-	root, err := s.GetStateRootByBatchNumber(batchNumber)
+	ctx := context.Background()
+	root, err := s.GetStateRootByBatchNumber(ctx, batchNumber)
 	if err != nil {
 		return nil, err
 	}
@@ -176,7 +175,8 @@ func (s *BasicState) GetBalance(address common.Address, batchNumber uint64) (*bi
 
 // GetCode from a given address
 func (s *BasicState) GetCode(address common.Address, batchNumber uint64) ([]byte, error) {
-	root, err := s.GetStateRootByBatchNumber(batchNumber)
+	ctx := context.Background()
+	root, err := s.GetStateRootByBatchNumber(ctx, batchNumber)
 	if err != nil {
 		return nil, err
 	}
@@ -259,7 +259,8 @@ func (s *BasicState) SetGenesis(ctx context.Context, genesis Genesis) error {
 
 // GetNonce returns the nonce of the given account at the given batch number
 func (s *BasicState) GetNonce(address common.Address, batchNumber uint64) (uint64, error) {
-	root, err := s.GetStateRootByBatchNumber(batchNumber)
+	ctx := context.Background()
+	root, err := s.GetStateRootByBatchNumber(ctx, batchNumber)
 	if err != nil {
 		return 0, err
 	}
@@ -276,7 +277,8 @@ func (s *BasicState) GetNonce(address common.Address, batchNumber uint64) (uint6
 
 // GetStorageAt from a given address
 func (s *BasicState) GetStorageAt(address common.Address, position common.Hash, batchNumber uint64) (*big.Int, error) {
-	root, err := s.GetStateRootByBatchNumber(batchNumber)
+	ctx := context.Background()
+	root, err := s.GetStateRootByBatchNumber(ctx, batchNumber)
 	if err != nil {
 		return nil, err
 	}

state/state_test.go

@@ -479,7 +479,7 @@ func TestStateTransition(t *testing.T) {
 			err = st.SetGenesis(ctx, genesis)
 			require.NoError(t, err)
 
-			root, err := st.GetStateRootByBatchNumber(0)
+			root, err := st.GetStateRootByBatchNumber(ctx, 0)
 			require.NoError(t, err)
 
 			for gaddr, gbalance := range genesis.Balances {
@@ -559,7 +559,7 @@ func TestStateTransition(t *testing.T) {
 				assert.Equal(t, transaction.Hash(), transactions[0].Hash())
 			}
 
-			root, err = st.GetStateRootByBatchNumber(batch.Number().Uint64())
+			root, err = st.GetStateRootByBatchNumber(ctx, batch.Number().Uint64())
 			require.NoError(t, err)
 
 			// Check new roots
@@ -700,7 +700,7 @@ func TestReceipts(t *testing.T) {
 			err = st.SetGenesis(ctx, genesis)
 			require.NoError(t, err)
 
-			root, err := st.GetStateRootByBatchNumber(0)
+			root, err := st.GetStateRootByBatchNumber(ctx, 0)
 			require.NoError(t, err)
 
 			for gaddr, gbalance := range genesis.Balances {
@@ -780,7 +780,7 @@ func TestReceipts(t *testing.T) {
 				assert.Equal(t, transaction.Hash(), transactions[0].Hash())
 			}
 
-			root, err = st.GetStateRootByBatchNumber(batch.Number().Uint64())
+			root, err = st.GetStateRootByBatchNumber(ctx, batch.Number().Uint64())
 			require.NoError(t, err)
 
 			// Check new roots
@@ -885,7 +885,7 @@ func TestStateErrors(t *testing.T) {
 	_, err = st.GetNonce(addr, 0)
 	require.Equal(t, state.ErrNotFound, err)
 
-	_, err = st.GetStateRootByBatchNumber(0)
+	_, err = st.GetStateRootByBatchNumber(ctx, 0)
 	require.Equal(t, state.ErrNotFound, err)
 
 	_, err = st.GetLastBlock(ctx)