Release/v0.2.0 to develop (0xPolygonHermez#2322)

Merge release/0.2.0

.github/workflows/jsonschema.yml

@@ -0,0 +1,61 @@
+---
+name: JSON schema
+on:
+  push:
+    branches:
+      - main
+      - master
+      - develop
+      - update-external-dependencies
+      - 'release/**'
+  pull_request:
+
+jobs:
+  json-schema:
+    strategy:
+      matrix:
+        go-version: [ 1.19.x ]
+        goarch: [ "amd64" ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      # https://github.com/actions/checkout#Checkout-pull-request-HEAD-commit-instead-of-merge-commit
+      # Checkout pull request HEAD commit instead of merge commit
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Install Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: ${{ matrix.go-version }}
+      env:
+        GOARCH: ${{ matrix.goarch }}
+
+    - uses: actions/setup-python@v1
+    - uses: BSFishy/pip-action@v1
+      with:
+        packages: |
+          json-schema-for-humans
+
+    - name: Check if JSON schema and generated doc is up to date
+      run: |
+          EXPECTED_DIFF=""
+          NOT_UPDATED_MSG="JSON Schema is not up to date, run 'make config-doc-gen' before creating the PR"
+
+          echo "Checking if JSON schema is up to date..."
+          make GENERATE_DOC_PATH=/tmp/ config-doc-gen
+          for CHECK_FILE in "node-config-schema.json" "node-config-doc.md" "node-config-doc.html" "custom_network-config-schema.json" "custom_network-config-doc.md" "custom_network-config-doc.html"; do
+            EXPECTED_FILE=tmp/$CHECK_FILE
+            REAL_FILE=docs/config-file/$CHECK_FILE
+            echo "checking $CHECK_FILE ...."
+            diff /tmp/$CHECK_FILE docs/config-file/$CHECK_FILE
+            if [ $? -ne 0  ]; then
+              echo "  FAILED file $CHECK_FILE!"
+              exit 1
+            fi
+            echo "checked $CHECK_FILE OK"
+          done
+
+          echo "Everything up to date"
+

.gitignore

@@ -16,4 +16,6 @@
 .env
 out.dat
 
-cmd/__debug_bin
+cmd/__debug_bin
+
+.venv

Dockerfile

@@ -12,8 +12,9 @@ RUN cd /src/db && packr2
 RUN cd /src && make build
 
 # CONTAINER FOR RUNNING BINARY
-FROM alpine:3.16.0
+FROM alpine:3.18.0
 COPY --from=build /src/dist/zkevm-node /app/zkevm-node
 COPY --from=build /src/config/environments/testnet/testnet.node.config.toml /app/example.config.toml
+RUN apk update && apk add postgresql15-client
 EXPOSE 8123
 CMD ["/bin/sh", "-c", "/app/zkevm-node run"]

Makefile

@@ -20,6 +20,15 @@ LDFLAGS += -X 'github.com/0xPolygonHermez/zkevm-node.GitRev=$(GITREV)'
 LDFLAGS += -X 'github.com/0xPolygonHermez/zkevm-node.GitBranch=$(GITBRANCH)'
 LDFLAGS += -X 'github.com/0xPolygonHermez/zkevm-node.BuildDate=$(DATE)'
 
+# Variables
+VENV           = .venv
+VENV_PYTHON    = $(VENV)/bin/python
+SYSTEM_PYTHON  = $(or $(shell which python3), $(shell which python))
+PYTHON         = $(or $(wildcard $(VENV_PYTHON)), "install_first_venv")
+GENERATE_SCHEMA_DOC = $(VENV)/bin/generate-schema-doc
+GENERATE_DOC_PATH=  "docs/config-file/"
+GENERATE_DOC_TEMPLATES_PATH=  "docs/config-file/templates/"
+
 .PHONY: build
 build: ## Builds the binary locally into ./dist
 	$(GOENVVARS) go build -ldflags "all=$(LDFLAGS)" -o $(GOBIN)/$(GOBINARY) $(GOCMD)
@@ -54,6 +63,52 @@ install-linter: ## Installs the linter
 lint: ## Runs the linter
 	export "GOROOT=$$(go env GOROOT)" && $$(go env GOPATH)/bin/golangci-lint run
 
+
+$(VENV_PYTHON):
+	rm -rf $(VENV)
+	$(SYSTEM_PYTHON) -m venv $(VENV)
+
+venv: $(VENV_PYTHON)
+
+# https://stackoverflow.com/questions/24736146/how-to-use-virtualenv-in-makefile
+.PHONY: install-config-doc-gen
+$(GENERATE_SCHEMA_DOC): $(VENV_PYTHON)
+	$(PYTHON) -m pip install --upgrade pip
+	$(PYTHON) -m pip install json-schema-for-humans
+
+PHONY: config-doc-gen
+config-doc-gen: config-doc-node config-doc-custom_network ## Generate config file's json-schema for node and custom_network  and documentation
+	#
+
+.PHONY: config-doc-node
+config-doc-node: $(GENERATE_SCHEMA_DOC) ## Generate config file's json-schema for node and documentation
+	go run ./cmd generate-json-schema --config-file=node --output=$(GENERATE_DOC_PATH)node-config-schema.json
+	$(GENERATE_SCHEMA_DOC) --config show_breadcrumbs=true \
+							--config footer_show_time=false \
+							--config expand_buttons=true \
+							--config custom_template_path=$(GENERATE_DOC_TEMPLATES_PATH)/js/base.html \
+							$(GENERATE_DOC_PATH)node-config-schema.json \
+							$(GENERATE_DOC_PATH)node-config-doc.html
+	$(GENERATE_SCHEMA_DOC)  --config custom_template_path=$(GENERATE_DOC_TEMPLATES_PATH)/md/base.md \
+							--config footer_show_time=false \
+							$(GENERATE_DOC_PATH)node-config-schema.json \
+							$(GENERATE_DOC_PATH)node-config-doc.md
+
+.PHONY: config-doc-custom_network
+config-doc-custom_network: $(GENERATE_SCHEMA_DOC) ## Generate config file's json-schema for custom_network and documentation
+	go run ./cmd generate-json-schema --config-file=custom_network --output=$(GENERATE_DOC_PATH)custom_network-config-schema.json
+	$(GENERATE_SCHEMA_DOC) --config show_breadcrumbs=true --config footer_show_time=false \
+							--config expand_buttons=true \
+							--config custom_template_path=$(GENERATE_DOC_TEMPLATES_PATH)/js/base.html \
+							$(GENERATE_DOC_PATH)custom_network-config-schema.json \
+							$(GENERATE_DOC_PATH)custom_network-config-doc.html
+	$(GENERATE_SCHEMA_DOC)  --config custom_template_path=$(GENERATE_DOC_TEMPLATES_PATH)/md/base.md \
+							--config footer_show_time=false \
+							--config example_format=JSON \
+							$(GENERATE_DOC_PATH)custom_network-config-schema.json \
+							$(GENERATE_DOC_PATH)custom_network-config-doc.md
+
+
 .PHONY: update-external-dependencies
 update-external-dependencies: ## Updates external dependencies like images, test vectors or proto files
 	go run ./scripts/cmd/... updatedeps
@@ -65,7 +120,7 @@ install-git-hooks: ## Moves hook files to the .git/hooks directory
 .PHONY: generate-code-from-proto
 generate-code-from-proto: ## Generates code from proto files
 	cd proto/src/proto/hashdb/v1 && protoc --proto_path=. --proto_path=../../../../include --go_out=../../../../../merkletree/pb --go-grpc_out=../../../../../merkletree/pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative hashdb.proto
-	cd proto/src/proto/executor/v1 && protoc --proto_path=. --go_out=../../../../../state/runtime/executor/pb --go-grpc_out=../../../../../state/runtime/executor/pb --go-grpc_opt=paths=source_relative --go_opt=paths=source_relative executor.proto
+	cd proto/src/proto/executor/v1 && protoc --proto_path=. --go_out=../../../../../state/runtime/executor --go-grpc_out=../../../../../state/runtime/executor --go-grpc_opt=paths=source_relative --go_opt=paths=source_relative executor.proto
 	cd proto/src/proto/aggregator/v1 && protoc --proto_path=. --proto_path=../../../../include --go_out=../../../../../aggregator/pb --go-grpc_out=../../../../../aggregator/pb --go-grpc_opt=paths=source_relative --go_opt=paths=source_relative aggregator.proto
 
 ## Help display.

cmd/jsonschema.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"github.com/0xPolygonHermez/zkevm-node/config"
+	"github.com/urfave/cli/v2"
+)
+
+func genJSONSchema(cli *cli.Context) error {
+	file_config := cli.String(config.FlagDocumentationFileType)
+	output := cli.String(config.FlagOutputFile)
+	switch file_config {
+	case NODE_CONFIGFILE:
+		{
+			generator := config.NewNodeConfigJsonSchemaGenerater()
+			return generator.GenerateJsonSchemaAndWriteToFile(cli, output)
+		}
+	case NETWORK_CONFIGFILE:
+		{
+			generator := config.NewNetworkConfigJsonSchemaGenerater()
+			return generator.GenerateJsonSchemaAndWriteToFile(cli, output)
+		}
+	default:
+		panic("Not supported this config file: " + file_config)
+	}
+}

cmd/main.go

@@ -30,6 +30,13 @@ const (
 	SEQUENCE_SENDER = "sequence-sender"
 )
 
+const (
+	// NODE_CONFIGFILE name to identify the node config-file
+	NODE_CONFIGFILE = "node"
+	// NETWORK_CONFIGFILE name to identify the netowk_custom (genesis) config-file
+	NETWORK_CONFIGFILE = "custom_network"
+)
+
 var (
 	configFileFlag = cli.StringFlag{
 		Name:     config.FlagCfg,
@@ -75,6 +82,16 @@ var (
 		Usage:    "Blocks the migrations in stateDB to not run them",
 		Required: false,
 	}
+	outputFileFlag = cli.StringFlag{
+		Name:     config.FlagOutputFile,
+		Usage:    "Indicate the output file",
+		Required: true,
+	}
+	documentationFileTypeFlag = cli.StringFlag{
+		Name:     config.FlagDocumentationFileType,
+		Usage:    fmt.Sprintf("Indicate the type of file to generate json-schema: %v,%v ", NODE_CONFIGFILE, NETWORK_CONFIGFILE),
+		Required: true,
+	}
 )
 
 func main() {
@@ -149,6 +166,12 @@ func main() {
 			Action:  dumpState,
 			Flags:   dumpStateFlags,
 		},
+		{
+			Name:   "generate-json-schema",
+			Usage:  "Generate the json-schema for the configuration file, and store it on docs/schema.json",
+			Action: genJSONSchema,
+			Flags:  []cli.Flag{&outputFileFlag, &documentationFileTypeFlag},
+		},
 		{
 			Name:    "snapshot",
 			Aliases: []string{"snap"},

cmd/run.go

@@ -32,7 +32,6 @@ import (
 	"github.com/0xPolygonHermez/zkevm-node/sequencesender"
 	"github.com/0xPolygonHermez/zkevm-node/state"
 	"github.com/0xPolygonHermez/zkevm-node/state/runtime/executor"
-	executorpb "github.com/0xPolygonHermez/zkevm-node/state/runtime/executor/pb"
 	"github.com/0xPolygonHermez/zkevm-node/synchronizer"
 	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -126,11 +125,9 @@ func start(cliCtx *cli.Context) error {
 	}
 
 	currentForkID := forkIDIntervals[len(forkIDIntervals)-1].ForkId
-	log.Infof("Fork ID read from POE SC = %v", currentForkID)
+	log.Infof("Fork ID read from POE SC = %v", forkIDIntervals[len(forkIDIntervals)-1].ForkId)
 	c.Aggregator.ChainID = l2ChainID
 	c.Aggregator.ForkId = currentForkID
-	c.Sequencer.DBManager.ForkID = currentForkID
-	c.Sequencer.Finalizer.ForkID = currentForkID
 	log.Infof("Chain ID read from POE SC = %v", l2ChainID)
 
 	ctx := context.Background()
@@ -218,7 +215,7 @@ func start(cliCtx *cli.Context) error {
 			if poolInstance == nil {
 				poolInstance = createPool(c.Pool, l2ChainID, st, eventLog)
 			}
-			go runSynchronizer(*c, etherman, etm, st, poolInstance)
+			go runSynchronizer(*c, etherman, etm, st, poolInstance, eventLog)
 		case ETHTXMANAGER:
 			ev.Component = event.Component_EthTxManager
 			ev.Description = "Running eth tx manager service"
@@ -286,7 +283,7 @@ func newEtherman(c config.Config) (*etherman.Client, error) {
 	return etherman, nil
 }
 
-func runSynchronizer(cfg config.Config, etherman *etherman.Client, ethTxManager *ethtxmanager.Client, st *state.State, pool *pool.Pool) {
+func runSynchronizer(cfg config.Config, etherman *etherman.Client, ethTxManager *ethtxmanager.Client, st *state.State, pool *pool.Pool, eventLog *event.EventLog) {
 	var trustedSequencerURL string
 	var err error
 	if !cfg.IsTrustedSequencer {
@@ -305,7 +302,7 @@ func runSynchronizer(cfg config.Config, etherman *etherman.Client, ethTxManager
 
 	sy, err := synchronizer.NewSynchronizer(
 		cfg.IsTrustedSequencer, etherman, st, pool, ethTxManager,
-		zkEVMClient, cfg.NetworkConfig.Genesis, cfg.Synchronizer,
+		zkEVMClient, eventLog, cfg.NetworkConfig.Genesis, cfg.Synchronizer,
 	)
 	if err != nil {
 		log.Fatal(err)
@@ -457,7 +454,7 @@ func newState(ctx context.Context, c *config.Config, l2ChainID uint64, forkIDInt
 	stateDb := state.NewPostgresStorage(sqlDB)
 
 	// Executor
-	var executorClient executorpb.ExecutorServiceClient
+	var executorClient executor.ExecutorServiceClient
 	if needsExecutor {
 		executorClient, _, _ = executor.NewExecutorClient(ctx, c.Executor)
 	}

config/config.go

@@ -48,32 +48,74 @@ const (
 	FlagPassword = "password"
 	// FlagMigrations is the flag for migrations.
 	FlagMigrations = "migrations"
+	// FlagOutputFile is the flag for the output file
+	FlagOutputFile = "output"
 	// FlagMaxAmount is the flag to avoid to use the flag FlagAmount
 	FlagMaxAmount = "max-amount"
+	// FlagDocumentationFileType is the flag for the choose which file generate json-schema
+	FlagDocumentationFileType = "config-file"
 )
 
-// Config represents the configuration of the entire Hermez Node
+/*
+Config represents the configuration of the entire Hermez Node
+The file is [TOML format]
+You could find some examples:
+  - `config/environments/local/local.node.config.toml`: running a permisionless node
+  - `config/environments/mainnet/public.node.config.toml`
+  - `config/environments/public/public.node.config.toml`
+  - `test/config/test.node.config.toml`: configuration for a trusted node used in CI
+
+[TOML format]: https://en.wikipedia.org/wiki/TOML
+*/
 type Config struct {
-	IsTrustedSequencer     bool   `mapstructure:"IsTrustedSequencer"`
+	// This define is a trusted node (`true`) or a permission less (`false`). If you don't known
+	// set to `false`
+	IsTrustedSequencer bool `mapstructure:"IsTrustedSequencer"`
+	// Last batch number before  a forkid change (fork upgrade). That implies that
+	// greater batch numbers are going to be trusted but no virtualized neither verified.
+	// So after the batch number `ForkUpgradeBatchNumber` is virtualized and verified you could update
+	// the system (SC,...) to new forkId and remove this value to allow the system to keep
+	// Virtualizing and verifying the new batchs.
+	// Check issue [#2236](https://github.com/0xPolygonHermez/zkevm-node/issues/2236) to known more
+	// This value overwrite `SequenceSender.ForkUpgradeBatchNumber`
 	ForkUpgradeBatchNumber uint64 `mapstructure:"ForkUpgradeBatchNumber"`
-	ForkUpgradeNewForkId   uint64 `mapstructure:"ForkUpgradeNewForkId"`
-	Log                    log.Config
-	Etherman               etherman.Config
-	EthTxManager           ethtxmanager.Config
-	Pool                   pool.Config
-	RPC                    jsonrpc.Config
-	Synchronizer           synchronizer.Config
-	Sequencer              sequencer.Config
-	SequenceSender         sequencesender.Config
-	Aggregator             aggregator.Config
-	NetworkConfig          NetworkConfig
-	L2GasPriceSuggester    gasprice.Config
-	Executor               executor.Config
-	MTClient               merkletree.Config
-	StateDB                db.Config
-	Metrics                metrics.Config
-	EventLog               event.Config
-	HashDB                 db.Config
+	// Which is the new forkId
+	ForkUpgradeNewForkId uint64 `mapstructure:"ForkUpgradeNewForkId"`
+	// Configure Log level for all the services, allow also to store the logs in a file
+	Log log.Config
+	// Configuration of the etherman (client for access L1)
+	Etherman etherman.Config
+	// Configuration for ethereum transaction manager
+	EthTxManager ethtxmanager.Config
+	// Pool service configuration
+	Pool pool.Config
+	// Configuration for RPC service. THis one offers a extended Ethereum JSON-RPC API interface to interact with the node
+	RPC jsonrpc.Config
+	// Configuration of service `Syncrhonizer`. For this service is also really important the value of `IsTrustedSequencer`
+	// because depending of this values is going to ask to a trusted node for trusted transactions or not
+	Synchronizer synchronizer.Config
+	// Configuration of the sequencer service
+	Sequencer sequencer.Config
+	// Configuration of the sequence sender service
+	SequenceSender sequencesender.Config
+	// Configuration of the aggregator service
+	Aggregator aggregator.Config
+	// Configuration of the genesis of the network. This is used to known the initial state of the network
+	NetworkConfig NetworkConfig
+	// Configuration of the gas price suggester service
+	L2GasPriceSuggester gasprice.Config
+	// Configuration of the executor service
+	Executor executor.Config
+	// Configuration of the merkle tree client service. Not use in the node, only for testing
+	MTClient merkletree.Config
+	// Configuration of the state database connection
+	StateDB db.Config
+	// Configuration of the metrics service, basically is where is going to publish the metrics
+	Metrics metrics.Config
+	// Configuration of the event database connection
+	EventLog event.Config
+	// Configuration of the hash database connection
+	HashDB db.Config
 }
 
 // Default parses the default configuration values.