feat: Add Table-Store (aka ORM) package - Genesis (cosmos#10481)

<!--
The default pull request template is for types feat, fix, or refactor.
For other templates, add one of the following parameters to the url:
- template=docs.md
- template=other.md
-->

## Description

closes: cosmos#9237
ref: cosmos#9156

This PR is a follow-up of cosmos#10451, cosmos#10415 and cosmos#9751 and the last PR for adding the ORM package to `x/group`.
It adds import/export genesis features.

---

### Author Checklist

*All items are required. Please add a note to the item if the item is not applicable and
please add links to any relevant follow up issues.*

I have...

- [x] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] added `!` to the type prefix if API or client breaking change
- [x] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#pr-targeting))
- [x] provided a link to the relevant issue or specification
- [ ] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/master/docs/building-modules)
- [x] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#testing)
- [ ] added a changelog entry to `CHANGELOG.md`
- [x] included comments for [documenting Go code](https://blog.golang.org/godoc)
- [x] updated the relevant documentation or specification
- [x] reviewed "Files changed" and left comments if necessary
- [ ] confirmed all CI checks have passed

### Reviewers Checklist

*All items are required. Please add a note if the item is not applicable and please add
your handle next to the items reviewed if you only reviewed selected items.*

I have...

- [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] confirmed `!` in the type prefix if API or client breaking change
- [ ] confirmed all author checklist items have been addressed 
- [ ] reviewed state machine logic
- [ ] reviewed API design and naming
- [ ] reviewed documentation is accurate
- [ ] reviewed tests and test coverage
- [ ] manually tested (if applicable)

x/group/go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/cosmos/cosmos-sdk v0.44.0
 	github.com/gogo/protobuf v1.3.3
 	github.com/stretchr/testify v1.7.0
-	google.golang.org/grpc v1.41.0
+	google.golang.org/grpc v1.42.0
 	google.golang.org/protobuf v1.27.1
 	pgregory.net/rapid v0.4.7
 )
@@ -20,10 +20,10 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd v0.22.0-beta // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/confio/ics23/go v0.6.6 // indirect
 	github.com/cosmos/btcutil v1.0.4 // indirect
-	github.com/cosmos/iavl v0.17.1 // indirect
+	github.com/cosmos/iavl v0.17.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
 	github.com/dgraph-io/ristretto v0.1.0 // indirect
@@ -32,10 +32,10 @@ require (
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-kit/kit v0.10.0 // indirect
 	github.com/go-logfmt/logfmt v0.5.0 // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/btree v1.0.0 // indirect
+	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/gtank/merlin v0.1.1 // indirect
@@ -44,7 +44,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jmhodges/levigo v1.0.0 // indirect
-	github.com/klauspost/compress v1.12.3 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/lazyledger/smt v0.2.1-0.20210709230900-03ea40719554 // indirect
 	github.com/libp2p/go-buffer-pool v0.0.2 // indirect
@@ -75,8 +75,8 @@ require (
 	github.com/tendermint/tendermint v0.34.14 // indirect
 	go.etcd.io/bbolt v1.3.5 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
-	golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f // indirect
-	golang.org/x/sys v0.0.0-20210903071746-97244b99971b // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
+	golang.org/x/sys v0.0.0-20211113001501-0c823b97ae02 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect

x/group/internal/orm/auto_uint64.go

@@ -3,9 +3,13 @@ package orm
 import (
 	"github.com/cosmos/cosmos-sdk/codec"
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/cosmos/cosmos-sdk/types/errors"
 )
 
-var _ Indexable = &AutoUInt64Table{}
+var (
+	_ Indexable       = &AutoUInt64Table{}
+	_ TableExportable = &AutoUInt64Table{}
+)
 
 // AutoUInt64Table is the table type with an auto incrementing ID.
 type AutoUInt64Table struct {
@@ -106,3 +110,27 @@ func (a AutoUInt64Table) PrefixScan(store sdk.KVStore, start, end uint64) (Itera
 func (a AutoUInt64Table) ReversePrefixScan(store sdk.KVStore, start uint64, end uint64) (Iterator, error) {
 	return a.table.ReversePrefixScan(store, EncodeSequence(start), EncodeSequence(end))
 }
+
+// Sequence returns the sequence used by this table
+func (a AutoUInt64Table) Sequence() Sequence {
+	return a.seq
+}
+
+// Export stores all the values in the table in the passed ModelSlicePtr and
+// returns the current value of the associated sequence.
+func (a AutoUInt64Table) Export(store sdk.KVStore, dest ModelSlicePtr) (uint64, error) {
+	_, err := a.table.Export(store, dest)
+	if err != nil {
+		return 0, err
+	}
+	return a.seq.CurVal(store), nil
+}
+
+// Import clears the table and initializes it from the given data interface{}.
+// data should be a slice of structs that implement PrimaryKeyed.
+func (a AutoUInt64Table) Import(store sdk.KVStore, data interface{}, seqValue uint64) error {
+	if err := a.seq.InitVal(store, seqValue); err != nil {
+		return errors.Wrap(err, "sequence")
+	}
+	return a.table.Import(store, data, seqValue)
+}

x/group/internal/orm/genesis.go

@@ -0,0 +1,19 @@
+package orm
+
+import (
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+// TableExportable
+type TableExportable interface {
+	// Export stores all the values in the table in the passed
+	// ModelSlicePtr. If the table has an associated sequence, then its
+	// current value is returned, otherwise 0 is returned by default.
+	Export(store sdk.KVStore, dest ModelSlicePtr) (uint64, error)
+
+	// Import clears the table and initializes it from the given data
+	// interface{}. data should be a slice of structs that implement
+	// PrimaryKeyed. The seqValue is optional and only
+	// used with tables that have an associated sequence.
+	Import(store sdk.KVStore, data interface{}, seqValue uint64) error
+}

x/group/internal/orm/genesis_test.go

@@ -0,0 +1,57 @@
+package orm
+
+import (
+	"testing"
+
+	"github.com/cosmos/cosmos-sdk/codec"
+	"github.com/cosmos/cosmos-sdk/codec/types"
+	"github.com/cosmos/cosmos-sdk/testutil/testdata"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestImportExportTableData(t *testing.T) {
+	interfaceRegistry := types.NewInterfaceRegistry()
+	cdc := codec.NewProtoCodec(interfaceRegistry)
+
+	table, err := NewAutoUInt64Table(AutoUInt64TablePrefix, AutoUInt64TableSeqPrefix, &testdata.TableModel{}, cdc)
+	require.NoError(t, err)
+
+	ctx := NewMockContext()
+	store := ctx.KVStore(sdk.NewKVStoreKey("test"))
+
+	tms := []*testdata.TableModel{
+		{
+			Id:       1,
+			Name:     "my test 1",
+			Number:   123,
+			Metadata: []byte("metadata 1"),
+		},
+		{
+			Id:       2,
+			Name:     "my test 2",
+			Number:   456,
+			Metadata: []byte("metadata 2"),
+		},
+	}
+
+	err = table.Import(store, tms, 2)
+	require.NoError(t, err)
+
+	for _, g := range tms {
+		var loaded testdata.TableModel
+		_, err := table.GetOne(store, g.Id, &loaded)
+		require.NoError(t, err)
+
+		require.Equal(t, g, &loaded)
+	}
+
+	var exported []*testdata.TableModel
+	seq, err := table.Export(store, &exported)
+	require.NoError(t, err)
+	require.Equal(t, seq, uint64(2))
+
+	for i, g := range exported {
+		require.Equal(t, g, tms[i])
+	}
+}

x/group/internal/orm/orm_scenario_test.go

@@ -1,6 +1,7 @@
 package orm
 
 import (
+	"bytes"
 	"encoding/binary"
 	"fmt"
 	"testing"
@@ -15,6 +16,9 @@ import (
 	"github.com/cosmos/cosmos-sdk/testutil/testdata"
 )
 
+// Testing ORM with arbitrary metadata length
+const metadataLen = 10
+
 func TestKeeperEndToEndWithAutoUInt64Table(t *testing.T) {
 	interfaceRegistry := types.NewInterfaceRegistry()
 	cdc := codec.NewProtoCodec(interfaceRegistry)
@@ -275,6 +279,130 @@ func TestGasCostsPrimaryKeyTable(t *testing.T) {
 	}
 }
 
+func TestExportImportStateAutoUInt64Table(t *testing.T) {
+	interfaceRegistry := types.NewInterfaceRegistry()
+	cdc := codec.NewProtoCodec(interfaceRegistry)
+
+	ctx := NewMockContext()
+	store := ctx.KVStore(sdk.NewKVStoreKey("test"))
+
+	k := NewTestKeeper(cdc)
+
+	testRecordsNum := 10
+	for i := 1; i <= testRecordsNum; i++ {
+		tm := testdata.TableModel{
+			Id:       uint64(i),
+			Name:     fmt.Sprintf("my test %d", i),
+			Metadata: bytes.Repeat([]byte{byte(i)}, metadataLen),
+		}
+
+		rowID, err := k.autoUInt64Table.Create(store, &tm)
+		require.NoError(t, err)
+		require.Equal(t, uint64(i), rowID)
+	}
+	var tms []*testdata.TableModel
+	seqVal, err := k.autoUInt64Table.Export(store, &tms)
+	require.NoError(t, err)
+	require.Equal(t, seqVal, uint64(testRecordsNum))
+
+	// when a new db seeded
+	ctx = NewMockContext()
+	store = ctx.KVStore(sdk.NewKVStoreKey("test"))
+
+	err = k.autoUInt64Table.Import(store, tms, seqVal)
+	require.NoError(t, err)
+
+	// then all data is set again
+	for i := 1; i <= testRecordsNum; i++ {
+		require.True(t, k.autoUInt64Table.Has(store, uint64(i)))
+		var loaded testdata.TableModel
+		rowID, err := k.autoUInt64Table.GetOne(store, uint64(i), &loaded)
+		require.NoError(t, err)
+
+		require.Equal(t, RowID(EncodeSequence(uint64(i))), rowID)
+		assert.Equal(t, fmt.Sprintf("my test %d", i), loaded.Name)
+		exp := bytes.Repeat([]byte{byte(i)}, metadataLen)
+		assert.Equal(t, exp, loaded.Metadata)
+
+		// and also the indexes
+		exists, err := k.autoUInt64TableModelByMetadataIndex.Has(store, exp)
+		require.NoError(t, err)
+		require.True(t, exists)
+
+		it, err := k.autoUInt64TableModelByMetadataIndex.Get(store, exp)
+		require.NoError(t, err)
+		var all []testdata.TableModel
+		ReadAll(it, &all)
+		require.Len(t, all, 1)
+		assert.Equal(t, loaded, all[0])
+	}
+	require.Equal(t, uint64(testRecordsNum), k.autoUInt64Table.Sequence().CurVal(store))
+}
+
+func TestExportImportStatePrimaryKeyTable(t *testing.T) {
+	interfaceRegistry := types.NewInterfaceRegistry()
+	cdc := codec.NewProtoCodec(interfaceRegistry)
+
+	ctx := NewMockContext()
+	store := ctx.KVStore(sdk.NewKVStoreKey("test"))
+
+	k := NewTestKeeper(cdc)
+
+	testRecordsNum := 10
+	testRecords := make([]testdata.TableModel, testRecordsNum)
+	for i := 1; i <= testRecordsNum; i++ {
+		tm := testdata.TableModel{
+			Id:       uint64(i),
+			Name:     fmt.Sprintf("my test %d", i),
+			Number:   uint64(i - 1),
+			Metadata: bytes.Repeat([]byte{byte(i)}, metadataLen),
+		}
+
+		err := k.primaryKeyTable.Create(store, &tm)
+		require.NoError(t, err)
+		testRecords[i-1] = tm
+	}
+	var tms []*testdata.TableModel
+	_, err := k.primaryKeyTable.Export(store, &tms)
+	require.NoError(t, err)
+
+	// when a new db seeded
+	ctx = NewMockContext()
+	store = ctx.KVStore(sdk.NewKVStoreKey("test"))
+
+	err = k.primaryKeyTable.Import(store, tms, 0)
+	require.NoError(t, err)
+
+	// then all data is set again
+	it, err := k.primaryKeyTable.PrefixScan(store, nil, nil)
+	require.NoError(t, err)
+	var loaded []testdata.TableModel
+	keys, err := ReadAll(it, &loaded)
+	require.NoError(t, err)
+	for i := range keys {
+		assert.Equal(t, PrimaryKey(&testRecords[i]), keys[i].Bytes())
+	}
+	assert.Equal(t, testRecords, loaded)
+
+	// all indexes setup
+	for _, v := range testRecords {
+		assertIndex(t, store, k.primaryKeyTableModelByNameIndex, v, v.Name)
+		assertIndex(t, store, k.primaryKeyTableModelByNumberIndex, v, v.Number)
+		assertIndex(t, store, k.primaryKeyTableModelByMetadataIndex, v, v.Metadata)
+	}
+}
+
+func assertIndex(t *testing.T, store sdk.KVStore, index Index, v testdata.TableModel, searchKey interface{}) {
+	it, err := index.Get(store, searchKey)
+	require.NoError(t, err)
+
+	var loaded []testdata.TableModel
+	keys, err := ReadAll(it, &loaded)
+	require.NoError(t, err)
+	assert.Equal(t, []RowID{PrimaryKey(&v)}, keys)
+	assert.Equal(t, []testdata.TableModel{v}, loaded)
+}
+
 func first(t *testing.T, it Iterator) ([]byte, testdata.TableModel) {
 	var loaded testdata.TableModel
 	key, err := First(it, &loaded)

x/group/internal/orm/primary_key.go

@@ -5,7 +5,10 @@ import (
 	sdk "github.com/cosmos/cosmos-sdk/types"
 )
 
-var _ Indexable = &PrimaryKeyTable{}
+var (
+	_ Indexable       = &PrimaryKeyTable{}
+	_ TableExportable = &PrimaryKeyTable{}
+)
 
 // PrimaryKeyTable provides simpler object style orm methods without passing database RowIDs.
 // Entries are persisted and loaded with a reference to their unique primary key.
@@ -144,3 +147,14 @@ func (a PrimaryKeyTable) PrefixScan(store sdk.KVStore, start, end []byte) (Itera
 func (a PrimaryKeyTable) ReversePrefixScan(store sdk.KVStore, start, end []byte) (Iterator, error) {
 	return a.table.ReversePrefixScan(store, start, end)
 }
+
+// Export stores all the values in the table in the passed ModelSlicePtr.
+func (a PrimaryKeyTable) Export(store sdk.KVStore, dest ModelSlicePtr) (uint64, error) {
+	return a.table.Export(store, dest)
+}
+
+// Import clears the table and initializes it from the given data interface{}.
+// data should be a slice of structs that implement PrimaryKeyed.
+func (a PrimaryKeyTable) Import(store sdk.KVStore, data interface{}, seqValue uint64) error {
+	return a.table.Import(store, data, seqValue)
+}

x/group/internal/orm/spec/01_table.md

@@ -15,6 +15,8 @@ The `table` struct does not:
 - optimize Gas usage conditions
 The `table` struct is private, so that we only have custom tables built on top of it, that do satisfy these requirements.
 
+`table` provides methods for exporting (using a [`PrefixScan` `Iterator`](03_iterator_pagination.md#iterator)) and importing genesis data. For the import to be successful, objects have to be aware of their primary key by implementing the [`PrimaryKeyed`](#primarykeyed) interface.
+
 ## AutoUInt64Table
 
 `AutoUInt64Table` is a table type with an auto incrementing `uint64` ID.
@@ -27,6 +29,8 @@ It's based on the `Sequence` struct which is a persistent unique key generator b
 
 `PrimaryKeyTable` provides simpler object style orm methods where are persisted and loaded with a reference to their unique primary key.
 
+### PrimaryKeyed
+
 The model provided for creating a `PrimaryKeyTable` should implement the `PrimaryKeyed` interface:
 
 +++ https://github.com/cosmos/cosmos-sdk/blob/9f78f16ae75cc42fc5fe636bde18a453ba74831f/x/group/internal/orm/primary_key.go#L28-L41