e2e test (okx#117)

.github/workflows/test.yml

@@ -23,4 +23,4 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Test
-      run: make test
+      run: make test-full

Makefile

@@ -17,10 +17,17 @@ build: ## Build the binary
 	$(GOENVVARS) go build $(LDFLAGS) -o $(GOBIN)/$(GOBINARY) $(GOCMD)
 
 .PHONY: test
-test: ## runs tests
+test: ## runs only short tests without checking race conditions
 	$(STOPDB) || true
 	$(STARTDB)
-	go test -race -p 1 ./...
+	go test -short -p 1 ./...
+	$(STOPDB)
+
+.PHONY: test-full
+test-full: ## runs all tests checking race conditions
+	$(STOPDB) || true
+	$(STARTDB)
+	go test -race -p 1 -timeout 180s ./...
 	$(STOPDB)
 
 .PHONY: install-linter
@@ -32,7 +39,7 @@ lint: ## runs linter
 	$$(go env GOPATH)/bin/golangci-lint run --timeout=5m -E whitespace -E gosec -E gci -E misspell -E gomnd -E gofmt -E goimports -E golint --exclude-use-default=false --max-same-issues 0
 
 .PHONY: deploy
-deploy: lint test build ## Validate and create the binary to be deployed
+deploy: lint test-full build ## Validate and create the binary to be deployed
 
 .PHONY: run
 run: ## Runs the application

aggregator/config.go

@@ -2,8 +2,6 @@ package aggregator
 
 import (
 	"time"
-
-	"github.com/hermeznetwork/hermez-core/etherman"
 )
 
 // Duration is a wrapper type that parses time duration from text.
@@ -26,7 +24,4 @@ type Config struct {
 	// IntervalToConsolidateState is the time the aggregator waits until
 	// trying to consolidate a new state
 	IntervalToConsolidateState Duration `mapstructure:"IntervalToConsolidateState"`
-
-	// Etherman is the configuration required by etherman to interact with L1
-	Etherman etherman.Config
 }

cmd/config.toml

@@ -9,38 +9,28 @@ Password = "polygon"
 Host = "localhost"
 Port = "5432"
 
+[Etherman]
+URL = "http://localhost"
+PoEAddress = "0xb1D0Dc8E2Ce3a93EB2b32f4C7c3fD9dDAf1211FA"
+PrivateKeyPath = "../test/test.keystore"
+PrivateKeyPassword = "testonly"
+
 [RPC]
 Host = "0.0.0.0"
 Port = 8123
-ChainID = 2576980377
 
 [Synchronizer]
 GenesisBlock = 1
 
-[Synchronizer.Etherman]
-URL = "http://localhost"
-PoEAddress = "0xb1D0Dc8E2Ce3a93EB2b32f4C7c3fD9dDAf1211FA"
-PrivateKeyPath = "../test/test.keystore"
-PrivateKeyPassword = "testonly"
-
 [Sequencer]
 IntervalToProposeBatch = "15s"
 SyncedBlockDif = 1
 
-[Sequencer.Etherman]
-URL = "http://localhost"
-PoEAddress = "0xb1D0Dc8E2Ce3a93EB2b32f4C7c3fD9dDAf1211FA"
-PrivateKeyPath = "../test/test.keystore"
-PrivateKeyPassword = "testonly"
-
 [Aggregator]
 IntervalToConsolidateState = "3s"
 
-[Aggregator.Etherman]
-URL = "http://localhost"
-PoEAddress = "0xb1D0Dc8E2Ce3a93EB2b32f4C7c3fD9dDAf1211FA"
-PrivateKeyPath = "../test/test.keystore"
-PrivateKeyPassword = "testonly"
-
 [Prover]
-ProverURI = "0.0.0.0:50051"
+ProverURI = "0.0.0.0:50051"
+
+[State]
+Arity = 16

cmd/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"math/big"
@@ -16,11 +17,12 @@ import (
 	"github.com/hermeznetwork/hermez-core/etherman"
 	"github.com/hermeznetwork/hermez-core/jsonrpc"
 	"github.com/hermeznetwork/hermez-core/log"
-	"github.com/hermeznetwork/hermez-core/mocks"
 	"github.com/hermeznetwork/hermez-core/pool"
 	"github.com/hermeznetwork/hermez-core/sequencer"
 	"github.com/hermeznetwork/hermez-core/state"
+	"github.com/hermeznetwork/hermez-core/state/tree"
 	"github.com/hermeznetwork/hermez-core/synchronizer"
+	"github.com/iden3/go-iden3-crypto/poseidon"
 	"github.com/urfave/cli/v2"
 )
 
@@ -75,28 +77,36 @@ func start(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	// c := config.Load()
+
 	setupLog(c.Log)
+
 	runMigrations(c.Database)
-	etherman, err := newSimulatedEtherman(c.Synchronizer.Etherman)
+
+	etherman, err := newSimulatedEtherman(c.Etherman)
 	if err != nil {
 		log.Fatal(err)
 		return err
 	}
-	state := mocks.NewState()
-	pool, err := pool.NewPostgresPool(c.Database)
+
+	sqlDB, err := db.NewSQLDB(c.Database)
 	if err != nil {
 		log.Fatal(err)
 		return err
 	}
+	mt := tree.NewMerkleTree(sqlDB, c.State.Arity, poseidon.Hash)
+	tr := tree.NewStateTree(mt, []byte{})
+	st := state.NewState(sqlDB, tr)
+
+	pool, err := pool.NewPostgresPool(c.Database)
 	if err != nil {
 		log.Fatal(err)
 		return err
 	}
+
 	//proverClient, conn := newProverClient(c.Prover)
-	go runSynchronizer(c.Synchronizer, etherman, state)
-	go runJSONRpcServer(c.RPC, pool, state)
-	go runSequencer(c.Sequencer, etherman, pool, state)
+	go runSynchronizer(c.Synchronizer, etherman, st)
+	go runJSONRpcServer(c.RPC, c.Etherman, pool, st)
+	go runSequencer(c.Sequencer, etherman, pool, st)
 	//go runAggregator(c.Aggregator, etherman, proverClient, state)
 	//waitSignal(conn)
 	waitSignal()
@@ -156,8 +166,30 @@ func runSynchronizer(c synchronizer.Config, etherman *etherman.ClientEtherMan, s
 	}
 }
 
-func runJSONRpcServer(jc jsonrpc.Config, pool pool.Pool, state state.State) {
-	if err := jsonrpc.NewServer(jc, pool, state).Start(); err != nil {
+func runJSONRpcServer(jc jsonrpc.Config, ec etherman.Config, pool pool.Pool, st state.State) {
+	var err error
+	var seq *state.Sequencer
+
+	key, err := newKeyFromKeystore(ec.PrivateKeyPath, ec.PrivateKeyPassword)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	seqAddress := key.Address
+
+	const intervalToCheckSequencerRegistrationInSeconds = 10
+
+	for {
+		seq, err = st.GetSequencer(context.Background(), seqAddress)
+		if err != nil {
+			log.Warnf("Make sure the address %s has been registered in the smart contract as a sequencer, err: %v", seqAddress.Hex(), err)
+			time.Sleep(intervalToCheckSequencerRegistrationInSeconds * time.Second)
+			continue
+		}
+		break
+	}
+
+	if err := jsonrpc.NewServer(jc, seq.ChainID.Uint64(), pool, st).Start(); err != nil {
 		log.Fatal(err)
 	}
 }
@@ -193,9 +225,8 @@ func waitSignal() {
 	}
 }
 
-func newAuthFromKeystore(path, password string) (*bind.TransactOpts, error) {
+func newKeyFromKeystore(path, password string) (*keystore.Key, error) {
 	if path == "" && password == "" {
-		log.Info("lol")
 		return nil, nil
 	}
 	keystoreEncrypted, err := ioutil.ReadFile(filepath.Clean(path))
@@ -206,6 +237,14 @@ func newAuthFromKeystore(path, password string) (*bind.TransactOpts, error) {
 	if err != nil {
 		return nil, err
 	}
+	return key, nil
+}
+
+func newAuthFromKeystore(path, password string) (*bind.TransactOpts, error) {
+	key, err := newKeyFromKeystore(path, password)
+	if err != nil {
+		log.Fatal(err)
+	}
 	log.Info("addr: ", key.Address.Hex())
 	auth, err := bind.NewKeyedTransactorWithChainID(key.PrivateKey, big.NewInt(1337)) //nolint:gomnd
 	if err != nil {

config/config.go

@@ -8,10 +8,12 @@ import (
 
 	"github.com/hermeznetwork/hermez-core/aggregator"
 	"github.com/hermeznetwork/hermez-core/db"
+	"github.com/hermeznetwork/hermez-core/etherman"
 	"github.com/hermeznetwork/hermez-core/jsonrpc"
 	logger "github.com/hermeznetwork/hermez-core/log"
 	"github.com/hermeznetwork/hermez-core/proverclient"
 	"github.com/hermeznetwork/hermez-core/sequencer"
+	"github.com/hermeznetwork/hermez-core/state"
 	"github.com/hermeznetwork/hermez-core/synchronizer"
 	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/viper"
@@ -21,11 +23,13 @@ import (
 type Config struct {
 	Log          logger.Config
 	Database     db.Config
+	Etherman     etherman.Config
 	Synchronizer synchronizer.Config
 	RPC          jsonrpc.Config
 	Sequencer    sequencer.Config
 	Aggregator   aggregator.Config
 	Prover       proverclient.Config
+	State        state.Config
 }
 
 // Load loads the configuration

config/default.go

@@ -13,39 +13,29 @@ Password = "polygon"
 Host = "localhost"
 Port = "5432"
 
-[RPC]
-Host = "0.0.0.0"
-Port = 8123
-ChainID = 2576980377
+[Etherman]
+URL = "http://localhost"
+PoEAddress = "0xb1D0Dc8E2Ce3a93EB2b32f4C7c3fD9dDAf1211FA"
+PrivateKeyPath = "../test/test.keystore"
+PrivateKeyPassword = "testonly"
 
 [Synchronizer]
 GenesisBlock = 1
 
-[Synchronizer.Etherman]
-URL = "http://localhost"
-PoEAddress = "0xb1D0Dc8E2Ce3a93EB2b32f4C7c3fD9dDAf1211FA"
-PrivateKeyPath = "./test/test.keystore"
-PrivateKeyPassword = "testonly"
+[RPC]
+Host = "0.0.0.0"
+Port = 8123
 
 [Sequencer]
 IntervalToProposeBatch = "15s"
 SyncedBlockDif = 1
 
-[Sequencer.Etherman]
-URL = "http://localhost"
-PoEAddress = "0xb1D0Dc8E2Ce3a93EB2b32f4C7c3fD9dDAf1211FA"
-PrivateKeyPath = "../test/test.keystore"
-PrivateKeyPassword = "testonly"
-
 [Aggregator]
 IntervalToConsolidateState = "3s"
 
-[Aggregator.Etherman]
-URL = "http://localhost"
-PoEAddress = "0xb1D0Dc8E2Ce3a93EB2b32f4C7c3fD9dDAf1211FA"
-PrivateKeyPath = "../test/test.keystore"
-PrivateKeyPassword = "testonly"
-
 [Prover]
 ProverURI = "0.0.0.0:50051"
+
+[State]
+Arity = 16
 `

encoding/encoding.go

@@ -10,6 +10,8 @@ import (
 )
 
 const (
+	// Base10 decimal base
+	Base10 = 10
 	// BitSize64 64 bits
 	BitSize64 = 64
 	// TenToThePowerOf9 is 10 ^ 9 which represents a gWei

etherman/etherman.go

@@ -24,9 +24,15 @@ import (
 )
 
 var (
-	newBatchEventSignatureHash    = crypto.Keccak256Hash([]byte("SendBatch(uint256,address)"))
-	consolidateBatchSignatureHash = crypto.Keccak256Hash([]byte("VerifyBatch(uint256,address)"))
-	newSequencerSignatureHash     = crypto.Keccak256Hash([]byte("SetSequencer(address,string)"))
+	newBatchEventSignatureHash        = crypto.Keccak256Hash([]byte("SendBatch(uint256,address)"))
+	consolidateBatchSignatureHash     = crypto.Keccak256Hash([]byte("VerifyBatch(uint256,address)"))
+	newSequencerSignatureHash         = crypto.Keccak256Hash([]byte("SetSequencer(address,string)"))
+	ownershipTransferredSignatureHash = crypto.Keccak256Hash([]byte("OwnershipTransferred(address,address)"))
+)
+
+var (
+	// ErrNotFound is used when the object is not found
+	ErrNotFound = errors.New("Not found")
 )
 
 // EtherMan represents an Ethereum Manager
@@ -76,7 +82,10 @@ func NewEtherman(cfg Config, auth *bind.TransactOpts) (*ClientEtherMan, error) {
 func (etherMan *ClientEtherMan) EthBlockByNumber(ctx context.Context, blockNumber uint64) (*types.Block, error) {
 	block, err := etherMan.EtherClient.BlockByNumber(ctx, new(big.Int).SetUint64(blockNumber))
 	if err != nil {
-		return &types.Block{}, err
+		if errors.Is(err, ethereum.NotFound) || err.Error() == "block does not exist in blockchain" {
+			return nil, ErrNotFound
+		}
+		return nil, err
 	}
 	return block, nil
 }
@@ -200,6 +209,9 @@ func (etherMan *ClientEtherMan) readEvents(ctx context.Context, query ethereum.F
 			log.Warn("error processing event: ", err, vLog)
 			continue
 		}
+		if block == nil {
+			continue
+		}
 		if b, exists := blocks[block.BlockHash]; exists {
 			b.Batches = append(blocks[block.BlockHash].Batches, block.Batches...)
 			b.NewSequencers = append(blocks[block.BlockHash].NewSequencers, block.NewSequencers...)
@@ -225,6 +237,8 @@ func (etherMan *ClientEtherMan) processEvent(ctx context.Context, vLog types.Log
 		batch.Sequencer = common.BytesToAddress(vLog.Topics[2].Bytes())
 		var head types.Header
 		head.TxHash = vLog.TxHash
+		head.Difficulty = big.NewInt(0)
+		head.Number = big.NewInt(0).SetUint64(batch.BatchNumber)
 		batch.Header = &head
 		block.BlockNumber = vLog.BlockNumber
 		batch.BlockNumber = vLog.BlockNumber
@@ -295,6 +309,9 @@ func (etherMan *ClientEtherMan) processEvent(ctx context.Context, vLog types.Log
 		sequencer.ChainID = se.ChainID
 		block.NewSequencers = append(block.NewSequencers, sequencer)
 		return &block, nil
+	case ownershipTransferredSignatureHash:
+		log.Debug("Unhandled event: OwnershipTransferred: ", vLog)
+		return nil, nil
 	}
 	return nil, fmt.Errorf("Event not registered")
 }

go.mod

@@ -5,14 +5,12 @@ go 1.16
 require (
 	github.com/btcsuite/btcd v0.21.0-beta // indirect
 	github.com/ethereum/go-ethereum v1.10.12
-	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/gobuffalo/packr/v2 v2.8.3
 	github.com/hermeznetwork/tracerr v0.3.2
 	github.com/holiman/uint256 v1.2.0
 	github.com/iden3/go-iden3-crypto v0.0.11
 	github.com/jackc/pgx/v4 v4.13.0
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
-	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/mitchellh/mapstructure v1.4.3
 	github.com/rubenv/sql-migrate v0.0.0-20211023115951-9f02b1e13857
 	github.com/spf13/viper v1.9.0
@@ -26,5 +24,4 @@ require (
 	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/grpc v1.40.0
 	google.golang.org/protobuf v1.27.1
-	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 )