Merge branch 'upstream/v2.61.13-RC1' into zjg/merge-v2.61.13-RC1

Author: zjg555543

.github/workflows/ci_zkevm.yml

@@ -5,12 +5,13 @@ on:
       - dev
       - zkevm
       - zkevm-2.60
+      - stable/**
   pull_request:
     branches:
       - dev
       - zkevm
       - zkevm-2.60
-      - stable*
+      - stable/**
     types:
       - opened
       - reopened

.github/workflows/doc-rpc.yml

@@ -4,10 +4,12 @@ on:
     branches:
       - dev
       - zkevm
+      - stable/**
   pull_request:
     branches:
       - dev
       - zkevm
+      - stable/**
     types:
       - opened
       - reopened

.github/workflows/test-resequence.yml

@@ -4,10 +4,12 @@ on:
     branches:
       - dev
       - zkevm
+      - stable/**
   pull_request:
     branches:
       - dev
       - zkevm
+      - stable/**
     types:
       - opened
       - reopened

.github/workflows/test-unwinds.yml

@@ -5,6 +5,7 @@ on:
       - dev
       - zkevm
       - zkevm-2.60
+      - stable/**
   workflow_dispatch:
 
 jobs:

cmd/rpcdaemon/cli/config.go

@@ -144,6 +144,7 @@ func RootCommand() (*cobra.Command, *httpcfg.HttpCfg) {
 	rootCmd.PersistentFlags().DurationVar(&cfg.OverlayReplayBlockTimeout, "rpc.overlay.replayblocktimeout", rpccfg.DefaultOverlayReplayBlockTimeout, "Maximum amount of time to wait for the answer to replay a single block when called from an overlay_getLogs call.")
 	rootCmd.PersistentFlags().IntVar(&cfg.BatchLimit, utils.RpcBatchLimit.Name, utils.RpcBatchLimit.Value, utils.RpcBatchLimit.Usage)
 	rootCmd.PersistentFlags().IntVar(&cfg.ReturnDataLimit, utils.RpcReturnDataLimit.Name, utils.RpcReturnDataLimit.Value, utils.RpcReturnDataLimit.Usage)
+	rootCmd.PersistentFlags().Uint64Var(&cfg.LogsMaxRange, utils.RpcLogsMaxRange.Name, utils.RpcLogsMaxRange.Value, utils.RpcLogsMaxRange.Usage)
 	rootCmd.PersistentFlags().BoolVar(&cfg.AllowUnprotectedTxs, utils.AllowUnprotectedTxs.Name, utils.AllowUnprotectedTxs.Value, utils.AllowUnprotectedTxs.Usage)
 	rootCmd.PersistentFlags().IntVar(&cfg.MaxGetProofRewindBlockCount, utils.RpcMaxGetProofRewindBlockCount.Name, utils.RpcMaxGetProofRewindBlockCount.Value, utils.RpcMaxGetProofRewindBlockCount.Usage)
 	rootCmd.PersistentFlags().Uint64Var(&cfg.OtsMaxPageSize, utils.OtsSearchMaxCapFlag.Name, utils.OtsSearchMaxCapFlag.Value, utils.OtsSearchMaxCapFlag.Usage)

cmd/rpcdaemon/cli/httpcfg/http_cfg.go

@@ -83,10 +83,11 @@ type HttpCfg struct {
 	LogDirVerbosity string
 	LogDirPath      string
 
-	BatchLimit                  int  // Maximum number of requests in a batch
-	ReturnDataLimit             int  // Maximum number of bytes returned from calls (like eth_call)
-	AllowUnprotectedTxs         bool // Whether to allow non EIP-155 protected transactions  txs over RPC
-	MaxGetProofRewindBlockCount int  //Max GetProof rewind block count
+	BatchLimit                  int    // Maximum number of requests in a batch
+	ReturnDataLimit             int    // Maximum number of bytes returned from calls (like eth_call)
+	LogsMaxRange                uint64 // Maximum number of logs that can be requested in a single call
+	AllowUnprotectedTxs         bool   // Whether to allow non EIP-155 protected transactions  txs over RPC
+	MaxGetProofRewindBlockCount int    //Max GetProof rewind block count
 	// Ots API
 	OtsMaxPageSize uint64
 

cmd/utils/flags.go

@@ -415,6 +415,11 @@ var (
 		Usage: "L2 datastreamer endpoint",
 		Value: "",
 	}
+	L2DataStreamerUseTLSFlag = cli.BoolFlag{
+		Name:  "zkevm.l2-datastreamer-use-tls",
+		Usage: "Use TLS connection to L2 datastreamer endpoint",
+		Value: false,
+	}
 	L2DataStreamerTimeout = cli.StringFlag{
 		Name:  "zkevm.l2-datastreamer-timeout",
 		Usage: "The time to wait for data to arrive from the stream before reporting an error (0s doesn't check)",
@@ -591,6 +596,11 @@ var (
 		Usage: "A comma separated list of grpc addresses that host executors",
 		Value: "",
 	}
+	ExecutorEnabled = cli.BoolFlag{
+		Name:  "zkevm.executor-enabled",
+		Usage: "Enables the executor. Used for testing limbo, when executor-urls are set, but we don't want to use them, only in limbo to verify limbo transactions. For this case, set it to false. Defaulted to true",
+		Value: true,
+	}
 	ExecutorStrictMode = cli.BoolFlag{
 		Name:  "zkevm.executor-strict",
 		Usage: "Defaulted to true to ensure you must set some executor URLs, bypass this restriction by setting to false",
@@ -612,6 +622,12 @@ var (
 		Usage: "A size of the memdb used on witness generation in format \"2GB\". Might fail generation for older batches if not enough for the unwind.",
 		Value: datasizeFlagValue(2 * datasize.GB),
 	}
+	WitnessUnwindLimit = cli.Uint64Flag{
+		Name:  "zkevm.witness-unwind-limit",
+		Usage: "The maximum number of blocks the witness generation can unwind",
+		Value: 500_000,
+	}
+
 	ExecutorMaxConcurrentRequests = cli.IntFlag{
 		Name:  "zkevm.executor-max-concurrent-requests",
 		Usage: "The maximum number of concurrent requests to the executor",
@@ -667,6 +683,11 @@ var (
 		Usage: "Allow the sequencer to proceed transactions with 0 gas price",
 		Value: false,
 	}
+	RejectLowGasPriceTransactions = cli.BoolFlag{
+		Name:  "zkevm.reject-low-gas-price-transactions",
+		Usage: "Reject the sequencer to proceed transactions with low gas price",
+		Value: false,
+	}
 	AllowPreEIP155Transactions = cli.BoolFlag{
 		Name:  "zkevm.allow-pre-eip155-transactions",
 		Usage: "Allow the sequencer to proceed pre-EIP155 transactions",
@@ -788,6 +809,11 @@ var (
 		Usage: "Contracts that will have all of their storage added to the witness every time",
 		Value: "",
 	}
+	BadTxAllowance = cli.Uint64Flag{
+		Name:  "zkevm.bad-tx-allowance",
+		Usage: "The maximum number of times a transaction that consumes too many counters to fit into a batch will be attempted before it is rejected outright by eth_sendRawTransaction",
+		Value: 2,
+	}
 	ACLPrintHistory = cli.IntFlag{
 		Name:  "acl.print-history",
 		Usage: "Number of entries to print from the ACL history on node start up",
@@ -835,6 +861,11 @@ var (
 		Usage: "Maximum number of bytes returned from eth_call or similar invocations",
 		Value: 100_000,
 	}
+	RpcLogsMaxRange = cli.Uint64Flag{
+		Name:  "rpc.logs.maxrange",
+		Usage: "Maximum range of logs that can be requested in a single call",
+		Value: 1000,
+	}
 	HTTPTraceFlag = cli.BoolFlag{
 		Name:  "http.trace",
 		Usage: "Print all HTTP requests to logs with INFO level",
@@ -2376,6 +2407,8 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *nodecfg.Config, cfg *ethconfig.C
 
 	if ctx.IsSet(TxPoolGossipDisableFlag.Name) {
 		cfg.DisableTxPoolGossip = ctx.Bool(TxPoolGossipDisableFlag.Name)
+	} else {
+		cfg.DisableTxPoolGossip = txpoolcfg.DefaultConfig.NoGossip
 	}
 }
 

core/vm/zk_batch_counters.go

@@ -88,6 +88,24 @@ func (bcc *BatchCounterCollector) AddNewTransactionCounters(txCounters *Transact
 	return bcc.CheckForOverflow(false) //no need to calculate the merkle proof here
 }
 
+// SingleTransactionOverflowCheck is used to check if a single transaction would overflow the batch counters
+// by creating a new counter collector and priming it for a single block with just this transaction
+// in it.  There is an assumption here that the tx counters being passed in have already been through execution
+func (bcc *BatchCounterCollector) SingleTransactionOverflowCheck(txCounters *TransactionCounter) (bool, error) {
+	err := txCounters.CalculateRlp()
+	if err != nil {
+		return true, err
+	}
+
+	bcc.transactions = append(bcc.transactions, txCounters)
+	bcc.UpdateRlpCountersCache(txCounters)
+	bcc.blockCount = 1 // simulate a single block
+
+	bcc.UpdateExecutionAndProcessingCountersCache(txCounters)
+
+	return bcc.CheckForOverflow(false) //no need to calculate the merkle proof here
+}
+
 func (bcc *BatchCounterCollector) RemovePreviousTransactionCounters() {
 	lastTx := bcc.transactions[len(bcc.transactions)-1]
 	bcc.UndoTransactionCountersCache(lastTx)
@@ -275,12 +293,15 @@ func (bcc *BatchCounterCollector) CombineCollectorsNoChanges() Counters {
 		}
 	}
 
+	txCounters := NewCounters()
 	for _, tx := range bcc.transactions {
-		txCounters := tx.CombineCounters()
+		_ = tx.CombineCountersInto(&txCounters)
 		for k, v := range txCounters {
 			combined[k].used += v.used
 			combined[k].remaining -= v.used
 		}
+
+		txCounters.NullateUsed()
 	}
 
 	return combined

core/vm/zk_counters.go

@@ -83,6 +83,12 @@ func NewCountersFromUsedArray(used []int) *Counters {
 	return &res
 }
 
+func (c Counters) NullateUsed() {
+	for _, v := range c {
+		v.used = 0
+	}
+}
+
 func (c Counters) UsedAsString() string {
 	res := fmt.Sprintf("[%s: %v]", CounterKeyNames[SHA], c[SHA].used)
 	res += fmt.Sprintf("[%s: %v]", CounterKeyNames[A], c[A].used)

core/vm/zk_transaction_counters.go

@@ -52,11 +52,19 @@ func NewTransactionCounter(transaction types.Transaction, smtMaxLevel int, forkI
 
 func (tc *TransactionCounter) CombineCounters() Counters {
 	combined := NewCounters()
+
+	_ = tc.CombineCountersInto(&combined)
+
+	return combined
+}
+
+func (tc *TransactionCounter) CombineCountersInto(combined *Counters) *Counters {
 	for k := range tc.rlpCounters.counters {
 		val := tc.rlpCounters.counters[k].used + tc.executionCounters.counters[k].used + tc.processingCounters.counters[k].used
-		combined[k] = &Counter{
-			used: val,
+		if (*combined)[k] == nil {
+			(*combined)[k] = &Counter{used: 0}
 		}
+		(*combined)[k].used = val
 	}
 
 	return combined

erigon-lib/.github/workflows/ci.yml

@@ -3,12 +3,12 @@ on:
   push:
     branches:
       - main
-      - stable
+      - stable/**
       - alpha
   pull_request:
     branches:
       - main
-      - stable
+      - stable/**
       - alpha
 env:
   CGO_ENABLED: "1"

erigon-lib/kv/mdbx/kv_mdbx.go

@@ -409,7 +409,7 @@ func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) {
 		writeTxLimiter: opts.writeTxLimiter,
 
 		txsCountMutex:         txsCountMutex,
-		txsAllDoneOnCloseCond: sync.NewCond(txsCountMutex),
+		txsAllDoneOnCloseCond: sync.NewCond(&sync.Mutex{}),
 
 		leakDetector: dbg.NewLeakDetector("db."+opts.label.String(), dbg.SlowTx()),
 
@@ -696,16 +696,19 @@ func (db *MdbxKV) trackTxBegin() bool {
 }
 
 func (db *MdbxKV) hasTxsAllDoneAndClosed() bool {
+	db.txsCountMutex.Lock()
+	defer db.txsCountMutex.Unlock()
 	return (db.txsCount == 0) && db.closed.Load()
 }
 
 func (db *MdbxKV) trackTxEnd() {
 	db.txsCountMutex.Lock()
-	defer db.txsCountMutex.Unlock()
 
 	if db.txsCount > 0 {
 		db.txsCount--
+		db.txsCountMutex.Unlock()
 	} else {
+		db.txsCountMutex.Unlock()
 		panic("MdbxKV: unmatched trackTxEnd")
 	}
 
@@ -715,9 +718,6 @@ func (db *MdbxKV) trackTxEnd() {
 }
 
 func (db *MdbxKV) waitTxsAllDoneOnClose() {
-	db.txsCountMutex.Lock()
-	defer db.txsCountMutex.Unlock()
-
 	for !db.hasTxsAllDoneAndClosed() {
 		db.txsAllDoneOnCloseCond.Wait()
 	}
@@ -751,16 +751,15 @@ func (db *MdbxKV) BeginRo(ctx context.Context) (txn kv.Tx, err error) {
 		// otherwise carry on
 	}
 
-	if !db.trackTxBegin() {
-		return nil, fmt.Errorf("db closed")
-	}
-
 	// will return nil err if context is cancelled (may appear to acquire the semaphore)
 	if semErr := db.readTxLimiter.Acquire(ctx, 1); semErr != nil {
-		db.trackTxEnd()
 		return nil, fmt.Errorf("mdbx.MdbxKV.BeginRo: roTxsLimiter error %w", semErr)
 	}
 
+	if !db.trackTxBegin() {
+		return nil, fmt.Errorf("db closed")
+	}
+
 	defer func() {
 		if txn == nil {
 			// on error, or if there is whatever reason that we don't return a tx,

erigon-lib/kv/mdbx/kv_mdbx_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -29,9 +30,10 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"sync"
+
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon-lib/kv/order"
-	"sync"
 )
 
 func BaseCaseDB(t *testing.T) kv.RwDB {
@@ -1101,28 +1103,48 @@ func TestDeadlock(t *testing.T) {
 	}).MapSize(128 * datasize.MB).MustOpen()
 	t.Cleanup(db.Close)
 
+	maxGoroutines := 10_000 // Limit the number of concurrent goroutines
+	sem := make(chan struct{}, maxGoroutines)
+
+	var outerErr error
+	stop := false
 	wg := sync.WaitGroup{}
 	for i := 0; i < 300_000; i++ {
+		if stop {
+			break
+		}
+		sem <- struct{}{}
 		wg.Add(1)
 		go func(idx int) {
+			defer func() {
+				<-sem
+				wg.Done()
+			}()
 			ctx := context.Background()
 			// create a write transaction every X requests
-			if idx%5 == 0 {
+			if idx%3 == 0 {
 				tx, err := db.BeginRw(ctx)
 				if err != nil {
-					t.Fatal(err)
+					fmt.Println(err)
+					stop = true
+					outerErr = err
+					return
 				}
 				defer tx.Rollback()
 			} else {
 				tx, err := db.BeginRo(ctx)
 				if err != nil {
-					t.Fatal(err)
+					fmt.Println(err)
+					stop = true
+					outerErr = err
 				}
 				defer tx.Rollback()
 			}
-			wg.Done()
 		}(i)
 	}
 
 	wg.Wait()
+	if outerErr != nil {
+		t.Error(outerErr)
+	}
 }

erigon-lib/kv/tables.go

@@ -548,6 +548,7 @@ const (
 	TablePoolLimbo                    = "PoolLimbo"
 	BATCH_ENDS                        = "batch_ends"
 	WITNESS_CACHE                     = "witness_cache"
+	BAD_TX_HASHES                     = "bad_tx_hashes"
 	//Diagnostics tables
 	DiagSystemInfo = "DiagSystemInfo"
 	DiagSyncStages = "DiagSyncStages"
@@ -793,6 +794,7 @@ var ChaindataTables = []string{
 	TablePoolLimbo,
 	BATCH_ENDS,
 	WITNESS_CACHE,
+	BAD_TX_HASHES,
 }
 
 const (