Feature/pool (0xPolygonHermez#60)

Add the pool persistence for transactions and gas price

.gitignore

@@ -0,0 +1 @@
+/dist/

Makefile

@@ -0,0 +1,31 @@
+.PHONY: build
+build: lint test ## Build the binary
+	go build -o ./dist/hezcore ./cmd/main.go
+
+.PHONY: test
+test: ## runs tests
+	go test ./... -p 1
+
+.PHONY: install-linter
+install-linter: ## install linter
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.30.0
+
+.PHONY: lint
+lint: ## runs linter
+	$$(go env GOPATH)/bin/golangci-lint run --timeout=5m -E whitespace -E gosec -E gci -E misspell -E gomnd -E gofmt -E goimports -E golint --exclude-use-default=false --max-same-issues 0
+
+.PHONY: run-db
+run-db: ## runs the db instance
+	docker run --rm -p 5432:5432 -e POSTGRES_DB="polygon-hermez" -e POSTGRES_USER="hermez" -e POSTGRES_PASSWORD="polygon" -d postgres
+
+## Help display.
+## Pulls comments from beside commands and prints a nicely formatted
+## display with the commands and their usage information.
+.DEFAULT_GOAL := help
+
+.PHONY: help
+help: ## Prints this help
+		@grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \
+		| sort \
+		| awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+

cmd/main.go

@@ -4,77 +4,86 @@ import (
 	"os"
 	"os/signal"
 
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/hermeznetwork/hermez-core/aggregator"
 	"github.com/hermeznetwork/hermez-core/config"
-	"github.com/hermeznetwork/hermez-core/etherman"
+	"github.com/hermeznetwork/hermez-core/db"
 	"github.com/hermeznetwork/hermez-core/jsonrpc"
 	"github.com/hermeznetwork/hermez-core/log"
 	"github.com/hermeznetwork/hermez-core/mocks"
-	"github.com/hermeznetwork/hermez-core/sequencer"
-	"github.com/hermeznetwork/hermez-core/synchronizer"
+	"github.com/hermeznetwork/hermez-core/pool"
 )
 
 func main() {
 	c := config.Load()
 	setupLog(c.Log)
-	go runJSONRpcServer(c.RPC)
-	go runSequencer(c.Sequencer)
-	go runAggregator(c.Aggregator)
+	runMigrations(c.Database)
+	go runJSONRpcServer(c.RPC, c.Database)
+	// go runSequencer(c.Sequencer)
+	// go runAggregator(c.Aggregator)
 	waitSignal()
 }
 
 func setupLog(c log.Config) {
 	log.Init(c)
 }
 
-func runJSONRpcServer(c jsonrpc.Config) {
-	p := mocks.NewPool()
-	s := mocks.NewState()
-
-	if err := jsonrpc.NewServer(c, p, s).Start(); err != nil {
+func runMigrations(c db.Config) {
+	err := db.RunMigrations(c)
+	if err != nil {
 		log.Fatal(err)
 	}
 }
 
-func runSequencer(c sequencer.Config) {
-	p := mocks.NewPool()
-	s := mocks.NewState()
-	e, err := etherman.NewEtherman(c.Etherman)
+func runJSONRpcServer(jc jsonrpc.Config, dc db.Config) {
+	p, err := pool.NewPostgresPool(dc)
 	if err != nil {
 		log.Fatal(err)
 	}
-	sy, err := synchronizer.NewSynchronizer(e, s)
-	if err != nil {
-		log.Fatal(err)
-	}
-	seq, err := sequencer.NewSequencer(c, p, s, e, sy)
-	if err != nil {
-		log.Fatal(err)
-	}
-	seq.Start()
-}
 
-func runAggregator(c aggregator.Config) {
-	// TODO: have more readable variables
 	s := mocks.NewState()
-	bp := s.NewBatchProcessor(common.Hash{}, false)
-	e, err := etherman.NewEtherman(c.Etherman)
-	if err != nil {
-		log.Fatal(err)
-	}
-	sy, err := synchronizer.NewSynchronizer(e, s)
-	if err != nil {
-		log.Fatal(err)
-	}
-	pc := aggregator.NewProverClient()
-	agg, err := aggregator.NewAggregator(c, s, bp, e, sy, pc)
-	if err != nil {
+
+	if err := jsonrpc.NewServer(jc, p, s).Start(); err != nil {
 		log.Fatal(err)
 	}
-	agg.Start()
 }
 
+// func runSequencer(c sequencer.Config) {
+// 	p := mocks.NewPool()
+// 	s := mocks.NewState()
+// 	e, err := etherman.NewEtherman(c.Etherman)
+// 	if err != nil {
+// 		log.Fatal(err)
+// 	}
+// 	sy, err := synchronizer.NewSynchronizer(e, s)
+// 	if err != nil {
+// 		log.Fatal(err)
+// 	}
+// 	seq, err := sequencer.NewSequencer(c, p, s, e, sy)
+// 	if err != nil {
+// 		log.Fatal(err)
+// 	}
+// 	seq.Start()
+// }
+
+// func runAggregator(c aggregator.Config) {
+// 	// TODO: have more readable variables
+// 	s := mocks.NewState()
+// 	bp := s.NewBatchProcessor(common.Hash{}, false)
+// 	e, err := etherman.NewEtherman(c.Etherman)
+// 	if err != nil {
+// 		log.Fatal(err)
+// 	}
+// 	sy, err := synchronizer.NewSynchronizer(e, s)
+// 	if err != nil {
+// 		log.Fatal(err)
+// 	}
+// 	pc := aggregator.NewProverClient()
+// 	agg, err := aggregator.NewAggregator(c, s, bp, e, sy, pc)
+// 	if err != nil {
+// 		log.Fatal(err)
+// 	}
+// 	agg.Start()
+// }
+
 func waitSignal() {
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt)

config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"github.com/hermeznetwork/hermez-core/aggregator"
+	"github.com/hermeznetwork/hermez-core/db"
 	"github.com/hermeznetwork/hermez-core/etherman"
 	"github.com/hermeznetwork/hermez-core/jsonrpc"
 	"github.com/hermeznetwork/hermez-core/log"
@@ -11,6 +12,7 @@ import (
 // Config represents the configuration of the entire Hermez Node
 type Config struct {
 	Log        log.Config
+	Database   db.Config
 	RPC        jsonrpc.Config
 	Sequencer  sequencer.Config
 	Aggregator aggregator.Config
@@ -25,11 +27,18 @@ func Load() Config {
 			Level:   "debug",
 			Outputs: []string{"stdout"},
 		},
+		Database: db.Config{
+			Database: "polygon-hermez",
+			User:     "hermez",
+			Password: "polygon",
+			Host:     "localhost",
+			Port:     "5432",
+		},
 		RPC: jsonrpc.Config{
 			Host: "",
 			Port: 8123,
 
-			ChainID: 2576980377, // 0x99999999
+			ChainID: 2576980377, // 0x99999999,
 		},
 		Sequencer: sequencer.Config{
 			Etherman: etherman.Config{},

db/config.go

@@ -0,0 +1,19 @@
+package db
+
+// Config provide fields to configure the pool
+type Config struct {
+	// Database name
+	Database string
+
+	// User name
+	User string
+
+	// Password of the user
+	Password string
+
+	// Host address
+	Host string
+
+	// Port Number
+	Port string
+}

db/db.go

@@ -1,12 +1,36 @@
-package database
+package db
 
 import (
 	"context"
 
+	"github.com/gobuffalo/packr/v2"
+	"github.com/hermeznetwork/hermez-core/log"
+	"github.com/jackc/pgx/v4"
 	"github.com/jackc/pgx/v4/pgxpool"
+	"github.com/jackc/pgx/v4/stdlib"
+	migrate "github.com/rubenv/sql-migrate"
 )
 
 // NewSQLDB creates a new SQL DB
-func NewSQLDB(dbName, dbUser, dbPassword, dbHost, dbPort string) (*pgxpool.Pool, error) {
-	return pgxpool.Connect(context.Background(), "postgres://"+dbUser+":"+dbPassword+"@"+dbHost+":"+dbPort+"/"+dbName)
+func NewSQLDB(cfg Config) (*pgxpool.Pool, error) {
+	return pgxpool.Connect(context.Background(), "postgres://"+cfg.User+":"+cfg.Password+"@"+cfg.Host+":"+cfg.Port+"/"+cfg.Database)
+}
+
+// RunMigrations will execute pending migrations if needed to keep
+// the database updated with the latest changes
+func RunMigrations(cfg Config) error {
+	c, err := pgx.ParseConfig("postgres://" + cfg.User + ":" + cfg.Password + "@" + cfg.Host + ":" + cfg.Port + "/" + cfg.Database)
+	if err != nil {
+		return err
+	}
+	db := stdlib.OpenDB(*c)
+
+	var migrations = &migrate.PackrMigrationSource{Box: packr.New("hermez-db-migrations", "./migrations")}
+	nMigrations, err := migrate.Exec(db, "postgres", migrations, migrate.Up)
+	if err != nil {
+		return err
+	}
+
+	log.Info("successfully ran ", nMigrations, " migrations Up")
+	return nil
 }

db/db_test.go

@@ -1,34 +1,44 @@
-package database
+package db
 
 import (
 	"context"
 	"testing"
 
-	dbutils "github.com/hermeznetwork/hermez-core/test/db"
+	"github.com/hermeznetwork/hermez-core/log"
+	"github.com/hermeznetwork/hermez-core/test/dbutils"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-const (
-	dbName     = "testing"
-	dbUser     = "hermez"
-	dbPassword = "password"
-)
+var cfg = Config{
+	Database: "testing",
+	User:     "hermez",
+	Password: "password",
+	Host:     "localhost",
+	Port:     "5432",
+}
 
 func TestDBService(t *testing.T) {
-	// Start DB Server
-	err := dbutils.StartPostgreSQL(dbName, dbUser, dbPassword, "./migrations/0001.sql")
+	log.Init(log.Config{
+		Level:   "debug",
+		Outputs: []string{"stdout"},
+	})
+
+	err := dbutils.StartPostgreSQL(cfg.Database, cfg.User, cfg.Password, "")
+	require.NoError(t, err)
+
+	err = RunMigrations(cfg)
 	require.NoError(t, err)
 
-	db, err := NewSQLDB(dbName, dbUser, dbPassword, dbutils.DBHost, dbutils.DBPort)
+	db, err := NewSQLDB(cfg)
 	require.NoError(t, err)
 
 	var result uint
 	err = db.QueryRow(context.Background(), "select count(*) from block").Scan(&result)
 	require.NoError(t, err)
 	assert.Equal(t, result, uint(0))
 
-	db.Close()
+	db.Close() //nolint:gosec,errcheck
 
 	// Stop DB Server
 	err = dbutils.StopPostgreSQL()

db/migrations/0002.sql

@@ -0,0 +1,18 @@
+-- +migrate Up
+
+CREATE SCHEMA pool
+
+CREATE TABLE pool.txs (
+    hash      VARCHAR PRIMARY KEY,
+    encoded   VARCHAR,
+    decoded   jsonb,
+    state     varchar(15)
+);
+
+-- create json indexes to query ordered by nonce and by tx state
+
+CREATE TABLE pool.gas_price (
+    item_id SERIAL PRIMARY KEY,
+    price DECIMAL(78,0),
+    timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL
+);

go.mod

@@ -5,14 +5,16 @@ go 1.16
 require (
 	github.com/btcsuite/btcd v0.21.0-beta // indirect
 	github.com/ethereum/go-ethereum v1.10.12
+	github.com/gobuffalo/packr/v2 v2.8.3
 	github.com/hermeznetwork/tracerr v0.3.2
 	github.com/holiman/uint256 v1.2.0
 	github.com/iden3/go-iden3-crypto v0.0.11
 	github.com/jackc/pgx/v4 v4.13.0
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/rubenv/sql-migrate v0.0.0-20211023115951-9f02b1e13857
 	github.com/stretchr/testify v1.7.0
 	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
-	go.uber.org/zap v1.16.0
+	github.com/ziutek/mymysql v1.5.4 // indirect
+	go.uber.org/zap v1.17.0
 	golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3 // indirect
-	google.golang.org/protobuf v1.25.0 // indirect
 )