forked from dymensionxyz/dymint
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add pruning mechanism that deletes old blocks and commits (dyme…
- Loading branch information
Showing
15 changed files
with
424 additions
and
139 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| package block | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "sync/atomic" | ||
| ) | ||
|
|
||
| func (m *Manager) pruneBlocks(retainHeight int64) (uint64, error) { | ||
| syncTarget := atomic.LoadUint64(&m.syncTarget) | ||
|
|
||
| if retainHeight > int64(syncTarget) { | ||
| return 0, fmt.Errorf("cannot prune uncommitted blocks") | ||
| } | ||
|
|
||
| pruned, err := m.store.PruneBlocks(retainHeight) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("failed to prune block store: %w", err) | ||
| } | ||
|
|
||
| //TODO: prune state/indexer and state/txindexer?? | ||
|
|
||
| return pruned, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| package store | ||
|
|
||
| import "fmt" | ||
|
|
||
| // PruneBlocks removes block up to (but not including) a height. It returns number of blocks pruned. | ||
| func (s *DefaultStore) PruneBlocks(heightInt int64) (uint64, error) { | ||
| if heightInt <= 0 { | ||
| return 0, fmt.Errorf("height must be greater than 0") | ||
| } | ||
|
|
||
| height := uint64(heightInt) | ||
| if height > s.Height() { | ||
| return 0, fmt.Errorf("cannot prune beyond the latest height %v", s.height) | ||
| } | ||
| base := s.Base() | ||
| if height < base { | ||
| return 0, fmt.Errorf("cannot prune to height %v, it is lower than base height %v", | ||
| height, base) | ||
| } | ||
|
|
||
| pruned := uint64(0) | ||
| batch := s.db.NewBatch() | ||
| defer batch.Discard() | ||
|
|
||
| flush := func(batch Batch, base uint64) error { | ||
| err := batch.Commit() | ||
| if err != nil { | ||
| return fmt.Errorf("failed to prune up to height %v: %w", base, err) | ||
| } | ||
| s.SetBase(base) | ||
| return nil | ||
| } | ||
|
|
||
| for h := base; h < height; h++ { | ||
| hash, err := s.loadHashFromIndex(h) | ||
| if err != nil { | ||
| continue | ||
| } | ||
| if err := batch.Delete(getBlockKey(hash)); err != nil { | ||
| return 0, err | ||
| } | ||
| if err := batch.Delete(getCommitKey(hash)); err != nil { | ||
| return 0, err | ||
| } | ||
| if err := batch.Delete(getIndexKey(h)); err != nil { | ||
| return 0, err | ||
| } | ||
| if err := batch.Delete(getResponsesKey(h)); err != nil { | ||
| return 0, err | ||
| } | ||
| if err := batch.Delete(getValidatorsKey(h)); err != nil { | ||
| return 0, err | ||
| } | ||
|
|
||
| pruned++ | ||
|
|
||
| // flush every 1000 blocks to avoid batches becoming too large | ||
| if pruned%1000 == 0 && pruned > 0 { | ||
| err := flush(batch, h) | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| batch = s.db.NewBatch() | ||
| defer batch.Discard() | ||
| } | ||
| } | ||
|
|
||
| err := flush(batch, height) | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| return pruned, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| package store_test | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/dymensionxyz/dymint/store" | ||
| "github.com/dymensionxyz/dymint/testutil" | ||
| "github.com/dymensionxyz/dymint/types" | ||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestStorePruning(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| pruningHeight := uint64(3) | ||
|
|
||
| cases := []struct { | ||
| name string | ||
| blocks []*types.Block | ||
| pruningHeight uint64 | ||
| expectedBase uint64 | ||
| expectedHeight uint64 | ||
| shouldError bool | ||
| }{ | ||
| {"blocks with pruning", []*types.Block{ | ||
| testutil.GetRandomBlock(1, 0), | ||
| testutil.GetRandomBlock(2, 0), | ||
| testutil.GetRandomBlock(3, 0), | ||
| testutil.GetRandomBlock(4, 0), | ||
| testutil.GetRandomBlock(5, 0), | ||
| }, pruningHeight, pruningHeight, 5, false}, | ||
| {"blocks out of order", []*types.Block{ | ||
| testutil.GetRandomBlock(2, 0), | ||
| testutil.GetRandomBlock(3, 0), | ||
| testutil.GetRandomBlock(1, 0), | ||
| }, pruningHeight, pruningHeight, 3, false}, | ||
| {"with a gap", []*types.Block{ | ||
| testutil.GetRandomBlock(1, 0), | ||
| testutil.GetRandomBlock(9, 0), | ||
| testutil.GetRandomBlock(10, 0), | ||
| }, pruningHeight, pruningHeight, 10, false}, | ||
| {"pruning beyond latest height", []*types.Block{ | ||
| testutil.GetRandomBlock(1, 0), | ||
| testutil.GetRandomBlock(2, 0), | ||
| }, pruningHeight, 1, 2, true}, // should error because pruning height > latest height | ||
| {"pruning height 0", []*types.Block{ | ||
| testutil.GetRandomBlock(1, 0), | ||
| testutil.GetRandomBlock(2, 0), | ||
| testutil.GetRandomBlock(3, 0), | ||
| }, 0, 1, 3, true}, | ||
| } | ||
|
|
||
| for _, c := range cases { | ||
| t.Run(c.name, func(t *testing.T) { | ||
| assert := assert.New(t) | ||
| bstore := store.New(store.NewDefaultInMemoryKVStore()) | ||
| assert.Equal(uint64(0), bstore.Height()) | ||
|
|
||
| for _, block := range c.blocks { | ||
| _, err := bstore.SaveBlock(block, &types.Commit{}, nil) | ||
| bstore.SetHeight(block.Header.Height) | ||
| assert.NoError(err) | ||
| } | ||
|
|
||
| _, err := bstore.PruneBlocks(int64(c.pruningHeight)) | ||
| if c.shouldError { | ||
| assert.Error(err) | ||
| } else { | ||
| assert.NoError(err) | ||
| assert.Equal(pruningHeight, bstore.Base()) | ||
| assert.Equal(c.expectedHeight, bstore.Height()) | ||
| assert.Equal(c.expectedBase, bstore.Base()) | ||
|
|
||
| // Check if pruned blocks are really removed from the store | ||
| for h := uint64(1); h < pruningHeight; h++ { | ||
| _, err := bstore.LoadBlock(h) | ||
| assert.Error(err, "Block at height %d should be pruned", h) | ||
|
|
||
| _, err = bstore.LoadBlockResponses(h) | ||
| assert.Error(err, "BlockResponse at height %d should be pruned", h) | ||
|
|
||
| _, err = bstore.LoadCommit(h) | ||
| assert.Error(err, "Commit at height %d should be pruned", h) | ||
| } | ||
| } | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| //TODO: prune twice |
Oops, something went wrong.