Merge v0.3.0 into develop (0xPolygonHermez#2474)

* fix null effective_percentage * fix forkID calculation * fix script * generate json-schema + docs for node config file and network_custom * fix unittest * Hotfixv0.1.4 to v0.2.0 (0xPolygonHermez#2255) * Hotfix v0.1.4 to main (0xPolygonHermez#2250) * fix concurrent web socket writes * fix eth_syncing * fix custom trace internal tx call error handling and update prover * add test to custom tracer depth issue; fix internal call error and gas used * fix custom tracer for internal tx with error and no more steps after it * remove debug code * Make max grpc message size configurable (0xPolygonHermez#2179) * make max grpc message size configurable * fix state tests * fix tests * fix tests * get SequencerNodeURI from SC if empty and not IsTrustedSequencer * Optimize trace (0xPolygonHermez#2183) * optimize trace * fix memory reading * update docker image * update prover image * fix converter * fix memory * fix step memory * fix structlogs * fix structlogs * fix structlogs * fix structlogs * fix structlogs * fix structlogs * fix structlogs * fix structlogs * update prover image * fix struclogs * fix memory size * fix memory size * fix memory size * refactor memory resize * refactor memory resize * move log for the best fitting tx (0xPolygonHermez#2192) * fix load zkCounters from pool * remove unnecessary log.info * add custom tracer support to CREATES opcode without depth increase (0xPolygonHermez#2213) * logs * fix getting stateroot from previous batch (GetWIPBatch) * logs * Fix GetWipBatch when previous last batch is a forced batch * fix forcedBatch trusted state * Revert "fix getting stateroot from previous batch (GetWIPBatch)" This reverts commit 860f0e7. * force GHA * add pool limits (0xPolygonHermez#2189) * Hotfix/batch l2 data (0xPolygonHermez#2223) * Fix BatchL2Data * Force GHA * remove failed txs from the pool limit check (0xPolygonHermez#2233) * debug trace by batch number via external rpc requests (0xPolygonHermez#2235) * fix trace batch remote requests in parallel limitation (0xPolygonHermez#2244) * Added RPC.TraceBatchUseHTTPS config parameter * fix executor version --------- Co-authored-by: tclemos <[email protected]> Co-authored-by: tclemos <[email protected]> Co-authored-by: Toni Ramírez <[email protected]> Co-authored-by: agnusmor <[email protected]> Co-authored-by: agnusmor <[email protected]> Co-authored-by: Thiago Coimbra Lemos <[email protected]> * fix test * fix test --------- Co-authored-by: tclemos <[email protected]> Co-authored-by: tclemos <[email protected]> Co-authored-by: Toni Ramírez <[email protected]> Co-authored-by: agnusmor <[email protected]> Co-authored-by: agnusmor <[email protected]> Co-authored-by: Thiago Coimbra Lemos <[email protected]> * Effective GasPrice refactor+fixes (0xPolygonHermez#2247) * effective GasPrice refactor * bugs fixes and finalizer tests fixes * fix typo * fix calculate effective gasprice percentage * fix test gas price * Fix/0xPolygonHermez#2257 effective gas price receipt (0xPolygonHermez#2258) * effective gas price returned by the rpc in the receipt * linter * bugfix: fixing l2blocks timestamp for the fist batch (0xPolygonHermez#2260) * bugfix: fixing l2blocks timestamp for the fist batch Signed-off-by: Nikolay Nedkov <[email protected]> * fix finalizer unit test --------- Signed-off-by: Nikolay Nedkov <[email protected]> * add more comments, and removed fields PrivateKeyPath and PrivateKeyPassword from etherman.Config that are not in use * add info to git action * add info to git action * fix github action * updated comments * updated comments * Fix/0xPolygonHermez#2263 gas used (0xPolygonHermez#2264) * fix fea2scalar and gas used * suggestion * fix fea2scalar * suggestion * Fix pending tx when duplicate nonce (0xPolygonHermez#2270) * fix pending tx when duplicate nonce * set pool.transaction.failed_reason to NULL when updating an existing tx * add more log details when adding tx to AddrQueue * fix query to add tx to the pool. Fix lint errors * change failed_reason for tx discarded due duplicate nonce * Only return a tx from the pool if tx is in pending status (0xPolygonHermez#2273) * Return a tx from the pool only if it is * fix TestGetTransactionByHash --------- Co-authored-by: agnusmor <[email protected]> * fix documentation with config file * improve: adding check to skip appending effectivePercentage if current forkId is under 5. Signed-off-by: Nikolay Nedkov <[email protected]> * Fiex effectiveGasprice unsigned txs with forkId lower than 5 (0xPolygonHermez#2278) * feat: adding functionality to stop sequencer on specific batch num from config param. Signed-off-by: Nikolay Nedkov <[email protected]> * patch: adding print for X-Real-IP in JSON-RPC Signed-off-by: Nikolay Nedkov <[email protected]> * Fix checkIfSynced (0xPolygonHermez#2289) * [Rehashing] Check logs order and fix blockhash and blockNumber in the log conversion (0xPolygonHermez#2280) * fix and check order * linter * flushID synchronizer (0xPolygonHermez#2287) * FlushID in synchronizer * linter * fix logs * commnets * executor error refactor (0xPolygonHermez#2299) * handle invalid rlp ROM error (0xPolygonHermez#2297) * add maxL2GasPrice (0xPolygonHermez#2294) * add maxL2GasPrice * fix * fix * add test * document parameter * update description * Error refactor (0xPolygonHermez#2302) * error refactor * refactor * Fix replaced tx as failed when duplicated nonce (0xPolygonHermez#2308) * Fix UpdateTxStatus for replacedTx * Fix adding tx with same nonce on AddrQueue * log reprocess need (0xPolygonHermez#2309) * log reprocess need * Update finalizer.go * Feature/2300 synchronizer detect if executor restart (0xPolygonHermez#2306) * detect if executor restarts and stop synchonizer * Update prover images (0xPolygonHermez#2311) * update prover image * update prover images * change executor param * Update testnet.prover.config.json * Update test.permissionless.prover.config.json * Update test.prover.config.json * Update public.prover.config.json * prover params * prover params * prover params * update prover images * add doc, and fix dockers to be able to use snap/restore feature (0xPolygonHermez#2315) * add doc, and fix dockers to be able to use snap/restore feature * add doc for snap/restore feature --------- Co-authored-by: Toni Ramírez <[email protected]> * Update docker-compose.yml * Update docker-compose.yml * do not add tx to the pool in case err != nil * do not add tx into the pool if a fatal error in the executor happens during pre execution * fix dbMultiWriteSinglePosition config value * workarround for the error error closing batch * workarround for the error error closing batch * workarround for the error error closing batch * workaround for the error of closing batch, another case * `Worker`'s `AddTxTracker` Bug Fix (0xPolygonHermez#2343) * bugfix: Resolve Function Bug in Worker Module Signed-off-by: Nikolay Nedkov <[email protected]> * improve: improving the wait for pending txs to be for only the txs for the current address. Signed-off-by: Nikolay Nedkov <[email protected]> --------- Signed-off-by: Nikolay Nedkov <[email protected]> * rename config files (0xPolygonHermez#2349) * fix closing batch + logs (0xPolygonHermez#2348) * fix closing batch + logs * fix * log description * typo errors * fix error: failed to store transactions for batch due to duplicate key * test * typo * Update README.md * Update release.yml * bugfix: fixing place where we need to increment the wg per address for pending txs Signed-off-by: Nikolay Nedkov <[email protected]> * Store batchL2Data when the batch is opened (0xPolygonHermez#2358) * add GasPriceMarginFactor and MaxGasPrice to eth-tx-manager (0xPolygonHermez#2360) * add GasPriceMarginFactor and MaxGasPrice to eth-tx-manager * add logs, fix config * update config file documentation --------- Co-authored-by: joanestebanr <[email protected]> * bugfix: attaching missing TxTracker.From to pending txs to store for forced batches. (0xPolygonHermez#2365) Signed-off-by: Nikolay Nedkov <[email protected]> * Update README.md * improve: adding logs (0xPolygonHermez#2373) * improve: adding logs Signed-off-by: Nikolay Nedkov <[email protected]> * adding more logs Signed-off-by: Nikolay Nedkov <[email protected]> * adding more logs okx#2 Signed-off-by: Nikolay Nedkov <[email protected]> --------- Signed-off-by: Nikolay Nedkov <[email protected]> * bugfix: fixing finalizer's handling. (0xPolygonHermez#2375) Signed-off-by: Nikolay Nedkov <[email protected]> * Update README.md * change hashdb go package * new hashdb interface * aggregator pb refactor * new prover image * change prover config * update prover image * update to latest proto and prover image * Refactor nonce calculation for addQueue (0xPolygonHermez#2382) * refactor nonce * fix * fix * fix script * check to avoid data inconsistencies (0xPolygonHermez#2387) * check to avoid data inconsistencies * check batchL2Data * names in the logs * Refactor: avoid delete addrQueue if it has pending txs to store (0xPolygonHermez#2391) * refactor delete addrQueue only if not pending txs to store * fix finalizer test * fix olsStateRoot in handleForcedTxsProcessResp * Update sequencer/addrqueue.go Co-authored-by: Alonso Rodriguez <[email protected]> --------- Co-authored-by: Toni Ramírez <[email protected]> Co-authored-by: Alonso Rodriguez <[email protected]> * Sort txs in worker by gasPrice (remove efficiency sort) (0xPolygonHermez#2392) * Sort txs in worker by GasPrice (remove efficiency sort) * update config docs --------- Co-authored-by: Toni Ramírez <[email protected]> * use useMainExecGenerated (0xPolygonHermez#2393) * Fix store forced batch tx (0xPolygonHermez#2394) * l2coinbase (0xPolygonHermez#2400) * l2coinbase * add default config * add support config fields that are common.Address * docs * prover image --------- Co-authored-by: joanestebanr <[email protected]> * Check flushID != 0 (0xPolygonHermez#2406) * Show tx.GasPrice in the worker logs (instead of tx.Cost) (0xPolygonHermez#2416) * Check flushID != 0 in Sequencer (0xPolygonHermez#2415) * check flushid != 0 in sequencer * Use f.halt instead of log.Fatal to report that flushid is 0 Co-authored-by: Toni Ramírez <[email protected]> * fix lint --------- Co-authored-by: Toni Ramírez <[email protected]> * update config params for Prover v2.1.0 (0xPolygonHermez#2418) * cherry-pick 0xPolygonHermez#2385 and 0xPolygonHermez#2396 from develop into v0.2.6 (0xPolygonHermez#2412) * fix http request instance null for websocket requests (0xPolygonHermez#2385) * fix ws subscribe to get filtered log notifications (0xPolygonHermez#2396) * new block endpoints and improvements to batch endpoint (0xPolygonHermez#2411) * Add forced batches tx to addrQueue (0xPolygonHermez#2398) * add forced batches tx to addrQueue * fix test * fix test * fix test * fix test * fix test * fix test * fix test * fix test * fix test * refactor * fix test * fix test * fix test * fix test * fix test * fixes * fixes * fixes * fixes * fixes * fixes * fixes * fixes * fixes * fixes * fixes * fixe hash and from * fixe hash and from * fixe hash and from * fixe hash and from * fix test * fix test * fix test * fix test * fix test * fix test * fix test * improve tests * improve tests * improve tests * improve tests * improve tests * refactor * refactor * improve logs * bugifx: adding missing tx.BreakEvenGasPrice nil check Signed-off-by: Nikolay Nedkov <[email protected]> * Sync halt (0xPolygonHermez#2428) * cherry-pick 0xPolygonHermez#2245 and 0xPolygonHermez#2424 from develop into v0.2.6 (0xPolygonHermez#2447) * fix safe and finalized l2 block to consider l1 safe and finalized blocks respectively (0xPolygonHermez#2245) * fix and add tests for safe and finalized l2 blocks (0xPolygonHermez#2424) * New executor errors refactor (0xPolygonHermez#2438) * wip * new errors * retry on executor db error * new prover images * fix comment * update hasdh proto and prover images * handle excutor db error * update test * update test * update test * update test * refactor error check in unsigned tx * Reprocess full batch in parallel (sanity check) (0xPolygonHermez#2425) * reprocess full batch in parallel (sanity check) * update doc * update reprocessFullBatch logs * Speed up deleting batches from stateDB creating an index for state.receipt.block_num (0xPolygonHermez#2457) * receipt deletion index * receipt deletion index * Feature/0xPolygonHermez#2429 fork id improvement (0xPolygonHermez#2450) * db table + tests * GetForks func modified to get them by range * Sync forkID * forkIDIntervals and forkID genesis * linter * docs * Avoid resetForkID in trustedNode * fix test group 9 * suggestions * doc and mocks * fix check storedFlushID (0xPolygonHermez#2458) * remove stored flush id 0 (0xPolygonHermez#2459) * Feature/0xPolygonHermez#2403 snap (0xPolygonHermez#2404) * Path snapshot command * restore * readme * options used by dbeaver * #2429_forkID_improvement: #2429_forkID_improvement: * fix * fix postgres version to v15 * fix permissionless init script * bugfix: removing measuring of metrics from async batch reprocessing f… (0xPolygonHermez#2461) * bugfix: removing measuring of metrics from async batch reprocessing for executor. Signed-off-by: Nikolay Nedkov <[email protected]> * fixing unit tests Signed-off-by: Nikolay Nedkov <[email protected]> --------- Signed-off-by: Nikolay Nedkov <[email protected]> * update prover images (0xPolygonHermez#2473) * Update production-setup.md * update doc * fix jsonrpc tests * fixes state.db * update doc again * remove obsolete config * docs one more time... --------- Signed-off-by: Nikolay Nedkov <[email protected]> Co-authored-by: joanestebanr <[email protected]> Co-authored-by: Alonso Rodriguez <[email protected]> Co-authored-by: tclemos <[email protected]> Co-authored-by: tclemos <[email protected]> Co-authored-by: agnusmor <[email protected]> Co-authored-by: agnusmor <[email protected]> Co-authored-by: Thiago Coimbra Lemos <[email protected]> Co-authored-by: joanestebanr <[email protected]> Co-authored-by: Nikolay Nedkov <[email protected]>
imnotsure-x · Aug 29, 2023 · 5bc323d · 5bc323d
1 parent 648af0a
commit 5bc323d
Show file tree

Hide file tree

Showing 65 changed files with 3,821 additions and 851 deletions.
diff --git a/ci/e2e-group9/forced_batches_vector_group1_test.go b/ci/e2e-group9/forced_batches_vector_group1_test.go
@@ -24,7 +24,7 @@ import (
 )
 
 const (
-	forkID5 = 5
+	forkID5 uint64 = 5
 )
 
 func TestForcedBatchesVectorFiles(t *testing.T) {
@@ -64,6 +64,7 @@ func TestForcedBatchesVectorFiles(t *testing.T) {
 				log.Info("###################")
 				genesisActions := vectors.GenerateGenesisActions(testCase.Genesis)
 				require.NoError(t, opsman.SetGenesis(genesisActions))
+				require.NoError(t, opsman.SetForkID(forkID5))
 				require.NoError(t, opsman.Setup())
 
 				// Check initial root

diff --git a/cmd/readme.md b/cmd/readme.md
@@ -2,10 +2,10 @@
 
 ### Create snapshots
 ```
-go run ./cmd snapshot --cfg config/environments/local/local.node.config.toml
+go run ./cmd snapshot --cfg config/environments/local/local.node.config.toml --output ./folder/
 ```
 
 ### Restore snapshots
 ```
-go run ./cmd restore --cfg config/environments/local/local.node.config.toml -is zkevmpubliccorestatedb_1685614455_v0.1.0_undefined.sql.tar.gz -ih zkevmpublicstatedb_1685615051_v0.1.0_undefined.sql.tar.gz
+go run ./cmd restore --cfg config/environments/local/local.node.config.toml -is ./folder/zkevmpubliccorestatedb_1685614455_v0.1.0_undefined.sql.tar.gz -ih ./folder/zkevmpublicstatedb_1685615051_v0.1.0_undefined.sql.tar.gz
 ```
diff --git a/cmd/restore.go b/cmd/restore.go
@@ -1,7 +1,12 @@
 package main
 
 import (
+	"bufio"
 	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
 	"strconv"
 	"strings"
 
@@ -45,9 +50,16 @@ func restore(ctx *cli.Context) error {
 		return errors.New("stateDB input file must end in .sql.tar.gz")
 	}
 
-	// Run migrations to create schemas and tables
-	runStateMigrations(c.State.DB)
-
+	d, err := db.NewSQLDB(c.State.DB)
+	if err != nil {
+		log.Error("error conecting to stateDB. Error: ", err)
+		return err
+	}
+	_, err = d.Exec(ctx.Context, "DROP SCHEMA IF EXISTS state CASCADE; DROP TABLE IF EXISTS gorp_migrations;")
+	if err != nil {
+		log.Error("error dropping state schema or migration table. Error: ", err)
+		return err
+	}
 	port, err := strconv.Atoi(c.State.DB.Port)
 	if err != nil {
 		log.Error("error converting port to int. Error: ", err)
@@ -64,12 +76,11 @@ func restore(ctx *cli.Context) error {
 		log.Error("error: ", err)
 		return err
 	}
-	restore.Role = c.State.DB.User
-	restore.Schemas = append(restore.Schemas, "state")
+	params := []string{"--no-owner", "--no-acl", "--format=c"}
 	log.Info("Restore stateDB snapshot started, please wait...")
-	restoreExec := restore.Exec(inputFileStateDB, pg.ExecOptions{StreamPrint: false})
+	restoreExec := execCommand(restore, inputFileStateDB, pg.ExecOptions{StreamPrint: false}, params)
 	if restoreExec.Error != nil {
-		log.Error("error restoring snapshot. Error: ", restoreExec.Error.Err)
+		log.Error("error restoring stateDB snapshot. Error: ", restoreExec.Error.Err)
 		log.Debug("restoreExec.Output: ", restoreExec.Output)
 		return err
 	}
@@ -84,12 +95,12 @@ func restore(ctx *cli.Context) error {
 		log.Error("error converting port to int. Error: ", err)
 		return err
 	}
-	d, err := db.NewSQLDB(c.HashDB)
+	d, err = db.NewSQLDB(c.HashDB)
 	if err != nil {
 		log.Error("error conecting to hashdb. Error: ", err)
 		return err
 	}
-	_, err = d.Exec(ctx.Context, "DROP SCHEMA IF EXISTS state CASCADE; CREATE SCHEMA IF NOT EXISTS state;")
+	_, err = d.Exec(ctx.Context, "DROP SCHEMA IF EXISTS state CASCADE;")
 	if err != nil {
 		log.Error("error dropping and creating state schema. Error: ", err)
 		return err
@@ -105,17 +116,62 @@ func restore(ctx *cli.Context) error {
 		log.Error("error: ", err)
 		return err
 	}
-	restore.Role = c.HashDB.User
-	restore.Schemas = append(restore.Schemas, "state")
-	restore.Options = []string{"--no-owner", "--no-acl"}
+
 	log.Info("Restore HashDB snapshot started, please wait...")
-	restoreExec = restore.Exec(inputFileHashDB, pg.ExecOptions{StreamPrint: false})
+	restoreExec = execCommand(restore, inputFileHashDB, pg.ExecOptions{StreamPrint: false}, params)
 	if restoreExec.Error != nil {
-		log.Error("error restoring snapshot. Error: ", restoreExec.Error.Err)
+		log.Error("error restoring hashDB snapshot. Error: ", restoreExec.Error.Err)
 		log.Debug("restoreExec.Output: ", restoreExec.Output)
 		return err
 	}
 
 	log.Info("Restore HashDB snapshot success")
 	return nil
 }
+
+func execCommand(x *pg.Restore, filename string, opts pg.ExecOptions, params []string) pg.Result {
+	result := pg.Result{}
+	options := append(params, x.Postgres.Parse()...)
+	options = append(options, fmt.Sprintf("%s%s", x.Path, filename))
+	log.Debug("Options: ", options)
+
+	result.FullCommand = strings.Join(options, " ")
+	cmd := exec.Command(pg.PGRestoreCmd, options...) //nolint:gosec
+	cmd.Env = append(os.Environ(), x.EnvPassword)
+	stderrIn, _ := cmd.StderrPipe()
+	go func(stderrIn io.ReadCloser, opts pg.ExecOptions, result *pg.Result) {
+		output := ""
+		reader := bufio.NewReader(stderrIn)
+		for {
+			line, err := reader.ReadString('\n')
+			if err != nil {
+				if errors.Is(err, io.EOF) {
+					result.Output = output
+					break
+				}
+				result.Error = &pg.ResultError{Err: fmt.Errorf("error reading output: %w", err), CmdOutput: output}
+				break
+			}
+
+			if opts.StreamPrint {
+				_, err = fmt.Fprint(opts.StreamDestination, line)
+				if err != nil {
+					result.Error = &pg.ResultError{Err: fmt.Errorf("error writing output: %w", err), CmdOutput: output}
+					break
+				}
+			}
+
+			output += line
+		}
+	}(stderrIn, opts, &result)
+	err := cmd.Start()
+	if err != nil {
+		result.Error = &pg.ResultError{Err: err, CmdOutput: result.Output}
+	}
+	err = cmd.Wait()
+	if exitError, ok := err.(*exec.ExitError); ok {
+		result.Error = &pg.ResultError{Err: err, ExitCode: exitError.ExitCode(), CmdOutput: result.Output}
+	}
+
+	return result
+}