quiet logging for witness generation unwinds (0xPolygonHermez#1407)

hexoscott · revitteth · web-flow · commit 1dcac1bf185d · 2024-11-05T10:28:50.000Z
* quiet logging for witness generation unwinds

* fix - integration

---------

Co-authored-by: Max Revitt &lt;max@revitt.consulting&gt;
diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go
@@ -1213,7 +1213,7 @@ func stageHashState(db kv.RwDB, ctx context.Context, logger log.Logger) error {
 	cfg := stagedsync.StageHashStateCfg(db, dirs, historyV3, agg)
 	if unwind > 0 {
 		u := sync.NewUnwindState(stages.HashState, s.BlockNumber-unwind, s.BlockNumber)
-		err = stagedsync.UnwindHashStateStage(u, s, tx, cfg, ctx, logger)
+		err = stagedsync.UnwindHashStateStage(u, s, tx, cfg, ctx, logger, false)
 		if err != nil {
 			return err
 		}
diff --git a/cmd/integration/commands/state_stages.go b/cmd/integration/commands/state_stages.go
@@ -473,7 +473,7 @@ func loopIh(db kv.RwDB, ctx context.Context, unwind uint64, logger log.Logger) e
 	to := execStage.BlockNumber - unwind
 	_ = sync.SetCurrentStage(stages.HashState)
 	u := &stagedsync.UnwindState{ID: stages.HashState, UnwindPoint: to}
-	if err = stagedsync.UnwindHashStateStage(u, stage(sync, tx, nil, stages.HashState), tx, stagedsync.StageHashStateCfg(db, dirs, historyV3, agg), ctx, logger); err != nil {
+	if err = stagedsync.UnwindHashStateStage(u, stage(sync, tx, nil, stages.HashState), tx, stagedsync.StageHashStateCfg(db, dirs, historyV3, agg), ctx, logger, false); err != nil {
 		return err
 	}
 	_ = sync.SetCurrentStage(stages.IntermediateHashes)
diff --git a/eth/stagedsync/default_stages.go b/eth/stagedsync/default_stages.go
@@ -138,7 +138,7 @@ func DefaultStages(ctx context.Context,
 				return SpawnHashStateStage(s, txc.Tx, hashState, ctx, logger)
 			},
 			Unwind: func(firstCycle bool, u *UnwindState, s *StageState, txc wrap.TxContainer, logger log.Logger) error {
-				return UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger)
+				return UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger, false)
 			},
 			Prune: func(firstCycle bool, p *PruneState, tx kv.RwTx, logger log.Logger) error {
 				return PruneHashStateStage(p, tx, hashState, ctx)
@@ -318,7 +318,7 @@ func PipelineStages(ctx context.Context, snapshots SnapshotsCfg, blockHashCfg Bl
 				return SpawnHashStateStage(s, txc.Tx, hashState, ctx, logger)
 			},
 			Unwind: func(firstCycle bool, u *UnwindState, s *StageState, txc wrap.TxContainer, logger log.Logger) error {
-				return UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger)
+				return UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger, false)
 			},
 			Prune: func(firstCycle bool, p *PruneState, tx kv.RwTx, logger log.Logger) error {
 				return PruneHashStateStage(p, tx, hashState, ctx)
@@ -527,7 +527,7 @@ func UploaderPipelineStages(ctx context.Context, snapshots SnapshotsCfg, headers
 				return SpawnHashStateStage(s, txc.Tx, hashState, ctx, logger)
 			},
 			Unwind: func(firstCycle bool, u *UnwindState, s *StageState, txc wrap.TxContainer, logger log.Logger) error {
-				return UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger)
+				return UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger, false)
 			},
 			Prune: func(firstCycle bool, p *PruneState, tx kv.RwTx, logger log.Logger) error {
 				return PruneHashStateStage(p, tx, hashState, ctx)
@@ -701,7 +701,7 @@ func StateStages(ctx context.Context, headers HeadersCfg, bodies BodiesCfg, bloc
 				return SpawnHashStateStage(s, txc.Tx, hashState, ctx, logger)
 			},
 			Unwind: func(firstCycle bool, u *UnwindState, s *StageState, txc wrap.TxContainer, logger log.Logger) error {
-				return UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger)
+				return UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger, false)
 			},
 		},
 		{
diff --git a/eth/stagedsync/stage_hashstate.go b/eth/stagedsync/stage_hashstate.go
@@ -102,7 +102,7 @@ func SpawnHashStateStage(s *StageState, tx kv.RwTx, cfg HashStateCfg, ctx contex
 	return nil
 }
 
-func UnwindHashStateStage(u *UnwindState, s *StageState, tx kv.RwTx, cfg HashStateCfg, ctx context.Context, logger log.Logger) (err error) {
+func UnwindHashStateStage(u *UnwindState, s *StageState, tx kv.RwTx, cfg HashStateCfg, ctx context.Context, logger log.Logger, quiet bool) (err error) {
 	useExternalTx := tx != nil
 	if !useExternalTx {
 		tx, err = cfg.db.BeginRw(ctx)
@@ -113,7 +113,7 @@ func UnwindHashStateStage(u *UnwindState, s *StageState, tx kv.RwTx, cfg HashSta
 	}
 
 	logPrefix := u.LogPrefix()
-	if err = unwindHashStateStageImpl(logPrefix, u, s, tx, cfg, ctx, logger); err != nil {
+	if err = unwindHashStateStageImpl(logPrefix, u, s, tx, cfg, ctx, logger, quiet); err != nil {
 		return err
 	}
 	if err = u.Done(tx); err != nil {
@@ -127,7 +127,7 @@ func UnwindHashStateStage(u *UnwindState, s *StageState, tx kv.RwTx, cfg HashSta
 	return nil
 }
 
-func unwindHashStateStageImpl(logPrefix string, u *UnwindState, s *StageState, tx kv.RwTx, cfg HashStateCfg, ctx context.Context, logger log.Logger) error {
+func unwindHashStateStageImpl(logPrefix string, u *UnwindState, s *StageState, tx kv.RwTx, cfg HashStateCfg, ctx context.Context, logger log.Logger, quiet bool) error {
 	// Currently it does not require unwinding because it does not create any Intermediate Hash records
 	// and recomputes the state root from scratch
 	prom := NewPromoter(tx, cfg.dirs, ctx, logger)
@@ -143,13 +143,13 @@ func unwindHashStateStageImpl(logPrefix string, u *UnwindState, s *StageState, t
 		}
 		return nil
 	}
-	if err := prom.Unwind(logPrefix, s, u, false /* storage */, true /* codes */); err != nil {
+	if err := prom.Unwind(logPrefix, s, u, false /* storage */, true /* codes */, quiet); err != nil {
 		return err
 	}
-	if err := prom.Unwind(logPrefix, s, u, false /* storage */, false /* codes */); err != nil {
+	if err := prom.Unwind(logPrefix, s, u, false /* storage */, false /* codes */, quiet); err != nil {
 		return err
 	}
-	if err := prom.Unwind(logPrefix, s, u, true /* storage */, false /* codes */); err != nil {
+	if err := prom.Unwind(logPrefix, s, u, true /* storage */, false /* codes */, quiet); err != nil {
 		return err
 	}
 	return nil
@@ -844,7 +844,7 @@ func (p *Promoter) UnwindOnHistoryV3(logPrefix string, agg *state.Aggregator, un
 	return collector.Load(p.tx, kv.HashedAccounts, etl.IdentityLoadFunc, etl.TransformArgs{Quit: p.ctx.Done()})
 }
 
-func (p *Promoter) Unwind(logPrefix string, s *StageState, u *UnwindState, storage bool, codes bool) error {
+func (p *Promoter) Unwind(logPrefix string, s *StageState, u *UnwindState, storage bool, codes bool, quiet bool) error {
 	var changeSetBucket string
 	if storage {
 		changeSetBucket = kv.StorageChangeSet
@@ -854,7 +854,9 @@ func (p *Promoter) Unwind(logPrefix string, s *StageState, u *UnwindState, stora
 	from := s.BlockNumber
 	to := u.UnwindPoint
 
-	p.logger.Info(fmt.Sprintf("[%s] Unwinding started", logPrefix), "from", from, "to", to, "storage", storage, "codes", codes)
+	if !quiet {
+		p.logger.Info(fmt.Sprintf("[%s] Unwinding started", logPrefix), "from", from, "to", to, "storage", storage, "codes", codes)
+	}
 
 	startkey := hexutility.EncodeTs(to + 1)
 
diff --git a/eth/stagedsync/stage_hashstate_test.go b/eth/stagedsync/stage_hashstate_test.go
@@ -106,7 +106,7 @@ func TestUnwindHashed(t *testing.T) {
 	}
 	u := &UnwindState{UnwindPoint: 50}
 	s := &StageState{BlockNumber: 100}
-	err = unwindHashStateStageImpl("logPrefix", u, s, tx2, StageHashStateCfg(db2, dirs, historyV3, nil), context.Background(), logger)
+	err = unwindHashStateStageImpl("logPrefix", u, s, tx2, StageHashStateCfg(db2, dirs, historyV3, nil), context.Background(), logger, false)
 	if err != nil {
 		t.Errorf("error while unwind state: %v", err)
 	}
@@ -227,7 +227,7 @@ func TestUnwindHashStateShutdown(t *testing.T) {
 
 			u := &UnwindState{UnwindPoint: 5}
 			s := &StageState{BlockNumber: 10}
-			if err = unwindHashStateStageImpl("logPrefix", u, s, tx, cfg, ctx, logger); !errors.Is(err, tc.errExp) {
+			if err = unwindHashStateStageImpl("logPrefix", u, s, tx, cfg, ctx, logger, false); !errors.Is(err, tc.errExp) {
 				t.Errorf("error does not match expected error while shutdown unwindHashStateStageImpl, got: %v, expected: %v", err, tc.errExp)
 			}
 
diff --git a/turbo/jsonrpc/eth_call.go b/turbo/jsonrpc/eth_call.go
@@ -362,7 +362,7 @@ func (api *APIImpl) GetProof(ctx context.Context, address libcommon.Address, sto
 		stageState := &stagedsync.StageState{BlockNumber: latestBlock}
 
 		hashStageCfg := stagedsync.StageHashStateCfg(nil, api.dirs, api.historyV3(batch), api._agg)
-		if err := stagedsync.UnwindHashStateStage(unwindState, stageState, batch, hashStageCfg, ctx, api.logger); err != nil {
+		if err := stagedsync.UnwindHashStateStage(unwindState, stageState, batch, hashStageCfg, ctx, api.logger, true); err != nil {
 			return nil, err
 		}
 
diff --git a/zk/stages/stages.go b/zk/stages/stages.go
@@ -125,7 +125,7 @@ func SequencerZkStages(
 				return stages.SpawnHashStateStage(s, txc.Tx, hashState, ctx, logger)
 			},
 			Unwind: func(firstCycle bool, u *stages.UnwindState, s *stages.StageState, txc wrap.TxContainer, logger log.Logger) error {
-				return stages.UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger)
+				return stages.UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger, false)
 			},
 			Prune: func(firstCycle bool, p *stages.PruneState, tx kv.RwTx, logger log.Logger) error {
 				return stages.PruneHashStateStage(p, tx, hashState, ctx)
@@ -328,7 +328,7 @@ func DefaultZkStages(
 				return stages.SpawnHashStateStage(s, txc.Tx, hashState, ctx, logger)
 			},
 			Unwind: func(firstCycle bool, u *stages.UnwindState, s *stages.StageState, txc wrap.TxContainer, logger log.Logger) error {
-				return stages.UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger)
+				return stages.UnwindHashStateStage(u, s, txc.Tx, hashState, ctx, logger, false)
 			},
 			Prune: func(firstCycle bool, p *stages.PruneState, tx kv.RwTx, logger log.Logger) error {
 				return stages.PruneHashStateStage(p, tx, hashState, ctx)
diff --git a/zk/witness/witness.go b/zk/witness/witness.go
@@ -223,7 +223,7 @@ func (g *Generator) generateWitness(tx kv.Tx, ctx context.Context, batchNum uint
 		stageState := &stagedsync.StageState{BlockNumber: latestBlock}
 
 		hashStageCfg := stagedsync.StageHashStateCfg(nil, g.dirs, g.historyV3, g.agg)
-		if err := stagedsync.UnwindHashStateStage(unwindState, stageState, batch, hashStageCfg, ctx, log.New()); err != nil {
+		if err := stagedsync.UnwindHashStateStage(unwindState, stageState, batch, hashStageCfg, ctx, log.New(), true); err != nil {
 			return nil, fmt.Errorf("unwind hash state: %w", err)
 		}
 

Original file line number	Diff line number	Diff line change
`@@ -1213,7 +1213,7 @@ func stageHashState(db kv.RwDB, ctx context.Context, logger log.Logger) error {`
`1213`	`1213`	`cfg := stagedsync.StageHashStateCfg(db, dirs, historyV3, agg)`
`1214`	`1214`	`if unwind > 0 {`
`1215`	`1215`	`u := sync.NewUnwindState(stages.HashState, s.BlockNumber-unwind, s.BlockNumber)`
`1216`		`- err = stagedsync.UnwindHashStateStage(u, s, tx, cfg, ctx, logger)`
	`1216`	`+ err = stagedsync.UnwindHashStateStage(u, s, tx, cfg, ctx, logger, false)`
`1217`	`1217`	`if err != nil {`
`1218`	`1218`	`return err`
`1219`	`1219`	`}`
Original file line number	Diff line number	Diff line change
`@@ -473,7 +473,7 @@ func loopIh(db kv.RwDB, ctx context.Context, unwind uint64, logger log.Logger) e`
`473`	`473`	`to := execStage.BlockNumber - unwind`
`474`	`474`	`_ = sync.SetCurrentStage(stages.HashState)`
`475`	`475`	`u := &stagedsync.UnwindState{ID: stages.HashState, UnwindPoint: to}`
`476`		`- if err = stagedsync.UnwindHashStateStage(u, stage(sync, tx, nil, stages.HashState), tx, stagedsync.StageHashStateCfg(db, dirs, historyV3, agg), ctx, logger); err != nil {`
	`476`	`+ if err = stagedsync.UnwindHashStateStage(u, stage(sync, tx, nil, stages.HashState), tx, stagedsync.StageHashStateCfg(db, dirs, historyV3, agg), ctx, logger, false); err != nil {`
`477`	`477`	`return err`
`478`	`478`	`}`
`479`	`479`	`_ = sync.SetCurrentStage(stages.IntermediateHashes)`
Original file line number	Diff line number	Diff line change
`@@ -102,7 +102,7 @@ func SpawnHashStateStage(s *StageState, tx kv.RwTx, cfg HashStateCfg, ctx contex`
`102`	`102`	`return nil`
`103`	`103`	`}`
`104`	`104`
`105`		`-func UnwindHashStateStage(u UnwindState, s StageState, tx kv.RwTx, cfg HashStateCfg, ctx context.Context, logger log.Logger) (err error) {`
	`105`	`+func UnwindHashStateStage(u UnwindState, s StageState, tx kv.RwTx, cfg HashStateCfg, ctx context.Context, logger log.Logger, quiet bool) (err error) {`
`106`	`106`	`useExternalTx := tx != nil`
`107`	`107`	`if !useExternalTx {`
`108`	`108`	`tx, err = cfg.db.BeginRw(ctx)`
`@@ -113,7 +113,7 @@ func UnwindHashStateStage(u UnwindState, s StageState, tx kv.RwTx, cfg HashSta`
`113`	`113`	`}`
`114`	`114`
`115`	`115`	`logPrefix := u.LogPrefix()`
`116`		`- if err = unwindHashStateStageImpl(logPrefix, u, s, tx, cfg, ctx, logger); err != nil {`
	`116`	`+ if err = unwindHashStateStageImpl(logPrefix, u, s, tx, cfg, ctx, logger, quiet); err != nil {`
`117`	`117`	`return err`
`118`	`118`	`}`
`119`	`119`	`if err = u.Done(tx); err != nil {`
`@@ -127,7 +127,7 @@ func UnwindHashStateStage(u UnwindState, s StageState, tx kv.RwTx, cfg HashSta`
`127`	`127`	`return nil`
`128`	`128`	`}`
`129`	`129`
`130`		`-func unwindHashStateStageImpl(logPrefix string, u UnwindState, s StageState, tx kv.RwTx, cfg HashStateCfg, ctx context.Context, logger log.Logger) error {`
	`130`	`+func unwindHashStateStageImpl(logPrefix string, u UnwindState, s StageState, tx kv.RwTx, cfg HashStateCfg, ctx context.Context, logger log.Logger, quiet bool) error {`
`131`	`131`	`// Currently it does not require unwinding because it does not create any Intermediate Hash records`
`132`	`132`	`// and recomputes the state root from scratch`
`133`	`133`	`prom := NewPromoter(tx, cfg.dirs, ctx, logger)`
`@@ -143,13 +143,13 @@ func unwindHashStateStageImpl(logPrefix string, u UnwindState, s StageState, t`
`143`	`143`	`}`
`144`	`144`	`return nil`
`145`	`145`	`}`
`146`		`- if err := prom.Unwind(logPrefix, s, u, false /* storage /, true / codes */); err != nil {`
	`146`	`+ if err := prom.Unwind(logPrefix, s, u, false /* storage /, true / codes */, quiet); err != nil {`
`147`	`147`	`return err`
`148`	`148`	`}`
`149`		`- if err := prom.Unwind(logPrefix, s, u, false /* storage /, false / codes */); err != nil {`
	`149`	`+ if err := prom.Unwind(logPrefix, s, u, false /* storage /, false / codes */, quiet); err != nil {`
`150`	`150`	`return err`
`151`	`151`	`}`
`152`		`- if err := prom.Unwind(logPrefix, s, u, true /* storage /, false / codes */); err != nil {`
	`152`	`+ if err := prom.Unwind(logPrefix, s, u, true /* storage /, false / codes */, quiet); err != nil {`
`153`	`153`	`return err`
`154`	`154`	`}`
`155`	`155`	`return nil`
`@@ -844,7 +844,7 @@ func (p Promoter) UnwindOnHistoryV3(logPrefix string, agg state.Aggregator, un`
`844`	`844`	`return collector.Load(p.tx, kv.HashedAccounts, etl.IdentityLoadFunc, etl.TransformArgs{Quit: p.ctx.Done()})`
`845`	`845`	`}`
`846`	`846`
`847`		`-func (p Promoter) Unwind(logPrefix string, s StageState, u *UnwindState, storage bool, codes bool) error {`
	`847`	`+func (p Promoter) Unwind(logPrefix string, s StageState, u *UnwindState, storage bool, codes bool, quiet bool) error {`
`848`	`848`	`var changeSetBucket string`
`849`	`849`	`if storage {`
`850`	`850`	`changeSetBucket = kv.StorageChangeSet`
`@@ -854,7 +854,9 @@ func (p Promoter) Unwind(logPrefix string, s StageState, u *UnwindState, stora`
`854`	`854`	`from := s.BlockNumber`
`855`	`855`	`to := u.UnwindPoint`
`856`	`856`
`857`		`- p.logger.Info(fmt.Sprintf("[%s] Unwinding started", logPrefix), "from", from, "to", to, "storage", storage, "codes", codes)`
	`857`	`+ if !quiet {`
	`858`	`+ p.logger.Info(fmt.Sprintf("[%s] Unwinding started", logPrefix), "from", from, "to", to, "storage", storage, "codes", codes)`
	`859`	`+ }`
`858`	`860`
`859`	`861`	`startkey := hexutility.EncodeTs(to + 1)`
`860`	`862`
Original file line number	Diff line number	Diff line change
`@@ -106,7 +106,7 @@ func TestUnwindHashed(t *testing.T) {`
`106`	`106`	`}`
`107`	`107`	`u := &UnwindState{UnwindPoint: 50}`
`108`	`108`	`s := &StageState{BlockNumber: 100}`
`109`		`- err = unwindHashStateStageImpl("logPrefix", u, s, tx2, StageHashStateCfg(db2, dirs, historyV3, nil), context.Background(), logger)`
	`109`	`+ err = unwindHashStateStageImpl("logPrefix", u, s, tx2, StageHashStateCfg(db2, dirs, historyV3, nil), context.Background(), logger, false)`
`110`	`110`	`if err != nil {`
`111`	`111`	`t.Errorf("error while unwind state: %v", err)`
`112`	`112`	`}`
`@@ -227,7 +227,7 @@ func TestUnwindHashStateShutdown(t *testing.T) {`
`227`	`227`
`228`	`228`	`u := &UnwindState{UnwindPoint: 5}`
`229`	`229`	`s := &StageState{BlockNumber: 10}`
`230`		`- if err = unwindHashStateStageImpl("logPrefix", u, s, tx, cfg, ctx, logger); !errors.Is(err, tc.errExp) {`
	`230`	`+ if err = unwindHashStateStageImpl("logPrefix", u, s, tx, cfg, ctx, logger, false); !errors.Is(err, tc.errExp) {`
`231`	`231`	`t.Errorf("error does not match expected error while shutdown unwindHashStateStageImpl, got: %v, expected: %v", err, tc.errExp)`
`232`	`232`	`}`
`233`	`233`
Original file line number	Diff line number	Diff line change
`@@ -362,7 +362,7 @@ func (api *APIImpl) GetProof(ctx context.Context, address libcommon.Address, sto`
`362`	`362`	`stageState := &stagedsync.StageState{BlockNumber: latestBlock}`
`363`	`363`
`364`	`364`	`hashStageCfg := stagedsync.StageHashStateCfg(nil, api.dirs, api.historyV3(batch), api._agg)`
`365`		`- if err := stagedsync.UnwindHashStateStage(unwindState, stageState, batch, hashStageCfg, ctx, api.logger); err != nil {`
	`365`	`+ if err := stagedsync.UnwindHashStateStage(unwindState, stageState, batch, hashStageCfg, ctx, api.logger, true); err != nil {`
`366`	`366`	`return nil, err`
`367`	`367`	`}`
`368`	`368`
Original file line number	Diff line number	Diff line change
`@@ -223,7 +223,7 @@ func (g *Generator) generateWitness(tx kv.Tx, ctx context.Context, batchNum uint`
`223`	`223`	`stageState := &stagedsync.StageState{BlockNumber: latestBlock}`
`224`	`224`
`225`	`225`	`hashStageCfg := stagedsync.StageHashStateCfg(nil, g.dirs, g.historyV3, g.agg)`
`226`		`- if err := stagedsync.UnwindHashStateStage(unwindState, stageState, batch, hashStageCfg, ctx, log.New()); err != nil {`
	`226`	`+ if err := stagedsync.UnwindHashStateStage(unwindState, stageState, batch, hashStageCfg, ctx, log.New(), true); err != nil {`
`227`	`227`	`return nil, fmt.Errorf("unwind hash state: %w", err)`
`228`	`228`	`}`
`229`	`229`