forked from cosmos/cosmos-sdk
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
perf(orm): add benchmarks (cosmos#11525)
## Description This PR adds benchmarks for inserting, updating, deleting and getting a simple entity compared with handwritten versions of this code. Two simple optimizations were found with these benchmarks and included in this PR: * correctly compute buffer size in `KeyCodec.EncodeKey` to avoid growing the buffer * use `value.String()` instead of `value.Interface().(string)` as the latter causes an additional allocation Other potential places for optimization can be seen by running CPU and memory profiles. In particular, update is significantly slower because the ORM assumes the existing value always needs to be decoded to keep indexes up-to-date which is not the case in this simple example. In the future, the ORM code generator could potentially generate more code up to the point of essentially mimicking the manual versions if performance tradeoffs become the bottleneck. As a follow-up, we should do benchmarks using an IAVL tree as the backing store to see if the ORM overhead is significant in a more real-world scenario. --- ### Author Checklist *All items are required. Please add a note to the item if the item is not applicable and please add links to any relevant follow up issues.* I have... - [ ] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title - [ ] added `!` to the type prefix if API or client breaking change - [ ] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#pr-targeting)) - [ ] provided a link to the relevant issue or specification - [ ] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/master/docs/building-modules) - [ ] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#testing) - [ ] added a changelog entry to `CHANGELOG.md` - [ ] included comments for [documenting Go code](https://blog.golang.org/godoc) - [ ] updated the relevant documentation or specification - [ ] reviewed "Files changed" and left comments if necessary - [ ] confirmed all CI checks have passed ### Reviewers Checklist *All items are required. Please add a note if the item is not applicable and please add your handle next to the items reviewed if you only reviewed selected items.* I have... - [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title - [ ] confirmed `!` in the type prefix if API or client breaking change - [ ] confirmed all author checklist items have been addressed - [ ] reviewed state machine logic - [ ] reviewed API design and naming - [ ] reviewed documentation is accurate - [ ] reviewed tests and test coverage - [ ] manually tested (if applicable)
- Loading branch information
Showing
4 changed files
with
340 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
package testkv | ||
|
||
import ( | ||
"testing" | ||
|
||
dbm "github.com/tendermint/tm-db" | ||
"gotest.tools/v3/assert" | ||
|
||
"github.com/cosmos/cosmos-sdk/orm/model/ormtable" | ||
) | ||
|
||
func NewGoLevelDBBackend(t testing.TB) ormtable.Backend { | ||
db, err := dbm.NewGoLevelDB("test", t.TempDir()) | ||
assert.NilError(t, err) | ||
return ormtable.NewBackend(ormtable.BackendOptions{ | ||
CommitmentStore: db, | ||
}) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,315 @@ | ||
package ormtable_test | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"testing" | ||
|
||
"google.golang.org/protobuf/proto" | ||
|
||
"github.com/cosmos/cosmos-sdk/orm/internal/testkv" | ||
"github.com/cosmos/cosmos-sdk/orm/testing/ormtest" | ||
|
||
dbm "github.com/tendermint/tm-db" | ||
"gotest.tools/v3/assert" | ||
|
||
"github.com/cosmos/cosmos-sdk/orm/internal/testpb" | ||
"github.com/cosmos/cosmos-sdk/orm/model/ormtable" | ||
"github.com/cosmos/cosmos-sdk/orm/types/kv" | ||
) | ||
|
||
func initBalanceTable(t testing.TB) testpb.BalanceTable { | ||
table, err := ormtable.Build(ormtable.Options{ | ||
MessageType: (&testpb.Balance{}).ProtoReflect().Type(), | ||
}) | ||
assert.NilError(t, err) | ||
|
||
balanceTable, err := testpb.NewBalanceTable(table) | ||
assert.NilError(t, err) | ||
|
||
return balanceTable | ||
} | ||
|
||
func BenchmarkMemory(b *testing.B) { | ||
bench(b, func(tb testing.TB) ormtable.Backend { | ||
return ormtest.NewMemoryBackend() | ||
}) | ||
} | ||
|
||
func BenchmarkLevelDB(b *testing.B) { | ||
bench(b, testkv.NewGoLevelDBBackend) | ||
} | ||
|
||
func bench(b *testing.B, newBackend func(testing.TB) ormtable.Backend) { | ||
b.Run("insert", func(b *testing.B) { | ||
b.StopTimer() | ||
ctx := ormtable.WrapContextDefault(newBackend(b)) | ||
b.StartTimer() | ||
benchInsert(b, ctx) | ||
}) | ||
b.Run("update", func(b *testing.B) { | ||
b.StopTimer() | ||
ctx := ormtable.WrapContextDefault(newBackend(b)) | ||
benchInsert(b, ctx) | ||
b.StartTimer() | ||
benchUpdate(b, ctx) | ||
}) | ||
b.Run("get", func(b *testing.B) { | ||
b.StopTimer() | ||
ctx := ormtable.WrapContextDefault(newBackend(b)) | ||
benchInsert(b, ctx) | ||
b.StartTimer() | ||
benchGet(b, ctx) | ||
}) | ||
b.Run("delete", func(b *testing.B) { | ||
b.StopTimer() | ||
ctx := ormtable.WrapContextDefault(newBackend(b)) | ||
benchInsert(b, ctx) | ||
b.StartTimer() | ||
benchDelete(b, ctx) | ||
}) | ||
} | ||
|
||
func benchInsert(b *testing.B, ctx context.Context) { | ||
balanceTable := initBalanceTable(b) | ||
for i := 0; i < b.N; i++ { | ||
assert.NilError(b, balanceTable.Insert(ctx, &testpb.Balance{ | ||
Address: fmt.Sprintf("acct%d", i), | ||
Denom: "bar", | ||
Amount: 10, | ||
})) | ||
} | ||
} | ||
|
||
func benchUpdate(b *testing.B, ctx context.Context) { | ||
balanceTable := initBalanceTable(b) | ||
for i := 0; i < b.N; i++ { | ||
assert.NilError(b, balanceTable.Update(ctx, &testpb.Balance{ | ||
Address: fmt.Sprintf("acct%d", i), | ||
Denom: "bar", | ||
Amount: 11, | ||
})) | ||
} | ||
} | ||
|
||
func benchGet(b *testing.B, ctx context.Context) { | ||
balanceTable := initBalanceTable(b) | ||
for i := 0; i < b.N; i++ { | ||
balance, err := balanceTable.Get(ctx, fmt.Sprintf("acct%d", i), "bar") | ||
assert.NilError(b, err) | ||
assert.Equal(b, uint64(10), balance.Amount) | ||
} | ||
} | ||
|
||
func benchDelete(b *testing.B, ctx context.Context) { | ||
balanceTable := initBalanceTable(b) | ||
for i := 0; i < b.N; i++ { | ||
assert.NilError(b, balanceTable.Delete(ctx, &testpb.Balance{ | ||
Address: fmt.Sprintf("acct%d", i), | ||
Denom: "bar", | ||
})) | ||
} | ||
} | ||
|
||
// | ||
// Manually written versions of insert, update, delete and get for testpb.Balance | ||
// | ||
|
||
const ( | ||
addressDenomPrefix byte = iota | ||
denomAddressPrefix | ||
) | ||
|
||
func insertBalance(store kv.Store, balance *testpb.Balance) error { | ||
denom := balance.Denom | ||
balance.Denom = "" | ||
addr := balance.Address | ||
balance.Address = "" | ||
|
||
addressDenomKey := []byte{addressDenomPrefix} | ||
addressDenomKey = append(addressDenomKey, []byte(addr)...) | ||
addressDenomKey = append(addressDenomKey, 0x0) | ||
addressDenomKey = append(addressDenomKey, []byte(denom)...) | ||
has, err := store.Has(addressDenomKey) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
if has { | ||
return fmt.Errorf("already exists") | ||
} | ||
|
||
bz, err := proto.Marshal(balance) | ||
if err != nil { | ||
return err | ||
} | ||
balance.Denom = denom | ||
balance.Address = addr | ||
|
||
err = store.Set(addressDenomKey, bz) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
// set denom address index | ||
denomAddressKey := []byte{denomAddressPrefix} | ||
denomAddressKey = append(denomAddressKey, []byte(balance.Denom)...) | ||
denomAddressKey = append(denomAddressKey, 0x0) | ||
denomAddressKey = append(denomAddressKey, []byte(balance.Address)...) | ||
err = store.Set(denomAddressKey, []byte{}) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func updateBalance(store kv.Store, balance *testpb.Balance) error { | ||
denom := balance.Denom | ||
balance.Denom = "" | ||
addr := balance.Address | ||
balance.Address = "" | ||
bz, err := proto.Marshal(balance) | ||
if err != nil { | ||
return err | ||
} | ||
balance.Denom = denom | ||
balance.Address = addr | ||
|
||
addressDenomKey := []byte{addressDenomPrefix} | ||
addressDenomKey = append(addressDenomKey, []byte(addr)...) | ||
addressDenomKey = append(addressDenomKey, 0x0) | ||
addressDenomKey = append(addressDenomKey, []byte(denom)...) | ||
|
||
return store.Set(addressDenomKey, bz) | ||
} | ||
|
||
func deleteBalance(store kv.Store, balance *testpb.Balance) error { | ||
denom := balance.Denom | ||
addr := balance.Address | ||
|
||
addressDenomKey := []byte{addressDenomPrefix} | ||
addressDenomKey = append(addressDenomKey, []byte(addr)...) | ||
addressDenomKey = append(addressDenomKey, 0x0) | ||
addressDenomKey = append(addressDenomKey, []byte(denom)...) | ||
err := store.Delete(addressDenomKey) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
denomAddressKey := []byte{denomAddressPrefix} | ||
denomAddressKey = append(denomAddressKey, []byte(balance.Denom)...) | ||
denomAddressKey = append(denomAddressKey, 0x0) | ||
denomAddressKey = append(denomAddressKey, []byte(balance.Address)...) | ||
return store.Delete(denomAddressKey) | ||
} | ||
|
||
func getBalance(store kv.Store, address, denom string) (*testpb.Balance, error) { | ||
addressDenomKey := []byte{addressDenomPrefix} | ||
addressDenomKey = append(addressDenomKey, []byte(address)...) | ||
addressDenomKey = append(addressDenomKey, 0x0) | ||
addressDenomKey = append(addressDenomKey, []byte(denom)...) | ||
|
||
bz, err := store.Get(addressDenomKey) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
if bz == nil { | ||
return nil, fmt.Errorf("not found") | ||
} | ||
|
||
var balance = testpb.Balance{} | ||
err = proto.Unmarshal(bz, &balance) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
balance.Address = address | ||
balance.Denom = denom | ||
|
||
return &balance, nil | ||
} | ||
|
||
func BenchmarkManualInsertMemory(b *testing.B) { | ||
benchManual(b, func() (dbm.DB, error) { | ||
return dbm.NewMemDB(), nil | ||
}) | ||
} | ||
|
||
func BenchmarkManualInsertLevelDB(b *testing.B) { | ||
benchManual(b, func() (dbm.DB, error) { | ||
return dbm.NewGoLevelDB("test", b.TempDir()) | ||
}) | ||
} | ||
|
||
func benchManual(b *testing.B, newStore func() (dbm.DB, error)) { | ||
b.Run("insert", func(b *testing.B) { | ||
b.StopTimer() | ||
store, err := newStore() | ||
assert.NilError(b, err) | ||
b.StartTimer() | ||
benchManualInsert(b, store) | ||
}) | ||
b.Run("update", func(b *testing.B) { | ||
b.StopTimer() | ||
store, err := newStore() | ||
assert.NilError(b, err) | ||
benchManualInsert(b, store) | ||
b.StartTimer() | ||
benchManualUpdate(b, store) | ||
}) | ||
b.Run("get", func(b *testing.B) { | ||
b.StopTimer() | ||
store, err := newStore() | ||
assert.NilError(b, err) | ||
benchManualInsert(b, store) | ||
b.StartTimer() | ||
benchManualGet(b, store) | ||
}) | ||
b.Run("delete", func(b *testing.B) { | ||
b.StopTimer() | ||
store, err := newStore() | ||
assert.NilError(b, err) | ||
benchManualInsert(b, store) | ||
b.StartTimer() | ||
benchManualDelete(b, store) | ||
}) | ||
} | ||
|
||
func benchManualInsert(b *testing.B, store kv.Store) { | ||
for i := 0; i < b.N; i++ { | ||
assert.NilError(b, insertBalance(store, &testpb.Balance{ | ||
Address: fmt.Sprintf("acct%d", i), | ||
Denom: "bar", | ||
Amount: 10, | ||
})) | ||
} | ||
} | ||
|
||
func benchManualUpdate(b *testing.B, store kv.Store) { | ||
for i := 0; i < b.N; i++ { | ||
assert.NilError(b, updateBalance(store, &testpb.Balance{ | ||
Address: fmt.Sprintf("acct%d", i), | ||
Denom: "bar", | ||
Amount: 11, | ||
})) | ||
} | ||
} | ||
|
||
func benchManualDelete(b *testing.B, store kv.Store) { | ||
for i := 0; i < b.N; i++ { | ||
assert.NilError(b, deleteBalance(store, &testpb.Balance{ | ||
Address: fmt.Sprintf("acct%d", i), | ||
Denom: "bar", | ||
})) | ||
} | ||
} | ||
|
||
func benchManualGet(b *testing.B, store kv.Store) { | ||
for i := 0; i < b.N; i++ { | ||
balance, err := getBalance(store, fmt.Sprintf("acct%d", i), "bar") | ||
assert.NilError(b, err) | ||
assert.Equal(b, uint64(10), balance.Amount) | ||
} | ||
} |