Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Index selectors #96

Merged
merged 45 commits into from
Apr 11, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
3771e81
Query.Intersect API & test
marino39 Mar 20, 2023
fa6c2ef
Naive implementation of Query.Intersects
marino39 Mar 20, 2023
a220743
Improve Query.Intersect benchmark
marino39 Mar 20, 2023
8968be7
Query.Intersect add sort / filter / offset / limit
marino39 Mar 20, 2023
c14e8ea
Index.Iter implementation
marino39 Mar 21, 2023
4dac79b
Use Index.Iter in Table Scan functions
marino39 Mar 21, 2023
490f432
Extract index update functions from Table to Index
marino39 Mar 21, 2023
e5db7e4
Extract index update functions from Table to Index: p2
marino39 Mar 21, 2023
13e42f5
KeySuccessor args change
marino39 Mar 21, 2023
a4bd17a
Fix: allocations is Table.PrimaryKey
marino39 Mar 21, 2023
ac937b9
Optimize Index.OnUpdate, so it requires less writes to database
marino39 Mar 22, 2023
3a4ee78
Use new Index callbacks in Table.reindex func
marino39 Mar 22, 2023
7a50e94
Add Intersect function to Index and use it in Query
marino39 Mar 22, 2023
80b2d1d
Benchmark Query.Intersect
marino39 Mar 23, 2023
e16bdc8
Move Index.Intersects to be map based because of poor performance
marino39 Mar 23, 2023
d8aa922
Reduce number of allocations in Index.Intersect
marino39 Mar 23, 2023
11e01d1
Add Query.Intersect().Order() test
marino39 Mar 23, 2023
fd5ce51
Query.Intersect().After() implementation
marino39 Mar 23, 2023
365ad1c
Query.After now works with Query.Order and Query.Intersect
marino39 Mar 24, 2023
323c907
Make sure that intersect queries do not have any modifiers like limit…
marino39 Mar 27, 2023
d39c691
Index selectors API proposition
marino39 Mar 27, 2023
d98773b
Merge branch 'master' into new_index_selectors
marino39 Mar 29, 2023
6f43140
Selector struct impl
marino39 Mar 29, 2023
8aa787f
Selector interface based approach
marino39 Mar 30, 2023
2790aa8
Table / Index.Intersect ported to use Selector
marino39 Mar 30, 2023
6afb4a8
Add table utils unit test
marino39 Mar 30, 2023
58fe15e
Use Selector in Query
marino39 Mar 30, 2023
863df97
Add files to .gitignore
marino39 Mar 30, 2023
cda655e
Fix: Example app
marino39 Mar 30, 2023
80b0f91
Cleanup
marino39 Mar 30, 2023
f63efaf
Table.Get SelectorRange implementation
marino39 Mar 31, 2023
0ca9793
Table.Get SelectorPoints implementation
marino39 Mar 31, 2023
a002c93
Table.Get SelectorPoints implementation -- add one more benchmark
marino39 Mar 31, 2023
9abce7e
Table.Get SelectorRanges implementation
marino39 Apr 4, 2023
0781cc7
Index.Iter unit tests
marino39 Apr 5, 2023
cdcb795
Index.Iter unit tests - PrimaryIndex
marino39 Apr 5, 2023
f38b78a
Index.Iter unit tests - new concept of Point Iter for primary index &…
marino39 Apr 5, 2023
a319514
Index.Iter restore SelectorPoints impl & unit tests
marino39 Apr 6, 2023
3f29335
Index OnInsert/OnUpdate/OnDelete unit tests
marino39 Apr 6, 2023
e8b19d2
Fix: Index.OnUpdate not properly updating indexes
marino39 Apr 6, 2023
40d552c
Update README.md
marino39 Apr 7, 2023
4e25ee3
Update comment in selector.go for SelectorRange and SelectorRanges
marino39 Apr 7, 2023
699851e
Fix: Issue with Query, SelectorRanges and Query.After
marino39 Apr 7, 2023
3bde6d3
Remove no longer valid test case
marino39 Apr 7, 2023
f8a2ab1
Query tests with different selectors and Query.After
marino39 Apr 11, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Table.Get SelectorRange implementation
  • Loading branch information
marino39 committed Mar 31, 2023
commit f63efafba64e8c687450a7f0e5c8de9312af161c
6 changes: 3 additions & 3 deletions filter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ func TestFilter_Upsert(t *testing.T) {

mFilter.On("MayContain", mock.Anything, mock.Anything).Return(true).Once()

dbTr, err := tokenBalanceTable.Get(NewSelectorPoint(tokenBalanceAccountUpdated))
dbTr, err := tokenBalanceTable.Get(context.Background(), NewSelectorPoint(tokenBalanceAccountUpdated))
require.NoError(t, err)
assert.Equal(t, tokenBalanceAccountUpdated, dbTr[0])

Expand Down Expand Up @@ -270,7 +270,7 @@ func TestFilter_Get(t *testing.T) {

mFilter.On("MayContain", mock.Anything, mock.Anything).Return(false).Once()

_, err := tokenBalanceTable.Get(NewSelectorPoint(tokenBalanceAccount))
_, err := tokenBalanceTable.Get(context.Background(), NewSelectorPoint(tokenBalanceAccount))
require.Error(t, err)

mFilter.On("MayContain", mock.Anything, mock.Anything).Return(false).Once()
Expand All @@ -281,7 +281,7 @@ func TestFilter_Get(t *testing.T) {

mFilter.On("MayContain", mock.Anything, mock.Anything).Return(true).Once()

dbTr, err := tokenBalanceTable.Get(NewSelectorPoint(tokenBalanceAccount))
dbTr, err := tokenBalanceTable.Get(context.Background(), NewSelectorPoint(tokenBalanceAccount))
require.NoError(t, err)
assert.Equal(t, tokenBalanceAccount, dbTr[0])

Expand Down
2 changes: 2 additions & 0 deletions index.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,8 @@ func (idx *Index[T]) Iter(table Table[T], selector Selector[T], optBatch ...Batc
upperBound = encodeIndexKey(table, up, idx, keyBufferPool.Get()[:0])
if bytes.Equal(lowerBound, upperBound) {
upperBound = keySuccessor(lowerBound[0:_KeyPrefixSplitIndex(lowerBound)], upperBound[:0])
} else {
upperBound = keySuccessor(upperBound, upperBound[:0])
}
default:
panic("invalid selector type")
Expand Down
13 changes: 11 additions & 2 deletions table.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ type TableInfo interface {
}

type TableGetter[T any] interface {
Get(sel Selector[T], optBatch ...Batch) ([]T, error)
Get(ctx context.Context, sel Selector[T], optBatch ...Batch) ([]T, error)
}

type TableExistChecker[T any] interface {
Expand Down Expand Up @@ -816,7 +816,7 @@ func (t *_table[T]) exist(key []byte, batch Batch, iter Iterator) bool {
return iter.SeekGE(key) && bytes.Equal(iter.Key(), key)
}

func (t *_table[T]) Get(sel Selector[T], optBatch ...Batch) ([]T, error) {
func (t *_table[T]) Get(ctx context.Context, sel Selector[T], optBatch ...Batch) ([]T, error) {
var batch Batch
if len(optBatch) > 0 && optBatch[0] != nil {
batch = optBatch[0]
Expand Down Expand Up @@ -851,6 +851,15 @@ func (t *_table[T]) Get(sel Selector[T], optBatch ...Batch) ([]T, error) {
}

return []T{rtr}, nil
case SelectorTypeRange:
var trs []T

err := t.ScanIndex(ctx, t.primaryIndex, sel, &trs, optBatch...)
if err != nil {
return nil, err
}

return trs, nil
default:
return nil, fmt.Errorf("invalid selector type")
}
Expand Down
96 changes: 95 additions & 1 deletion table_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,100 @@ func TestBondTable_SerializerOption(t *testing.T) {
assert.NotNil(t, tokenBalanceTable.Serializer())
}

func TestBondTable_Get(t *testing.T) {
db := setupDatabase()
defer tearDownDatabase(db)

const (
TokenBalanceTableID TableID = 0xC0
)

tokenBalanceTable := NewTable[*TokenBalance](TableOptions[*TokenBalance]{
DB: db,
TableID: TokenBalanceTableID,
TableName: "token_balance",
TablePrimaryKeyFunc: func(builder KeyBuilder, tb *TokenBalance) []byte {
return builder.AddUint64Field(tb.ID).Bytes()
},
Serializer: &serializers.JsonSerializer{},
})
require.NotNil(t, tokenBalanceTable)

// token balances to insert
tokenBalanceAccount1 := &TokenBalance{
ID: 1,
AccountID: 1,
ContractAddress: "0xtestContract",
AccountAddress: "0xtestAccount",
Balance: 5,
}

err := tokenBalanceTable.Insert(context.Background(), []*TokenBalance{tokenBalanceAccount1})
require.NoError(t, err)

// get token balance
tokenBalance, err := tokenBalanceTable.Get(context.Background(), NewSelectorPoint(&TokenBalance{ID: tokenBalanceAccount1.ID}))
require.NoError(t, err)
assert.Equal(t, tokenBalanceAccount1, tokenBalance[0])

// get token balance with non-existing id
tokenBalance, err = tokenBalanceTable.Get(context.Background(), NewSelectorPoint(&TokenBalance{ID: 2}))
require.Error(t, err)
assert.Nil(t, tokenBalance)
}

func TestBondTable_Get_Range(t *testing.T) {
db := setupDatabase()
defer tearDownDatabase(db)

const (
TokenBalanceTableID TableID = 0xC0
)

tokenBalanceTable := NewTable[*TokenBalance](TableOptions[*TokenBalance]{
DB: db,
TableID: TokenBalanceTableID,
TableName: "token_balance",
TablePrimaryKeyFunc: func(builder KeyBuilder, tb *TokenBalance) []byte {
return builder.AddUint64Field(tb.ID).Bytes()
},
Serializer: &serializers.JsonSerializer{},
})
require.NotNil(t, tokenBalanceTable)

// token balances to insert
expectedTokenBalances := []*TokenBalance{
{
ID: 1,
AccountID: 1,
ContractAddress: "0xtestContract",
AccountAddress: "0xtestAccount",
Balance: 5,
},
{
ID: 2,
AccountID: 1,
ContractAddress: "0xtestContract",
AccountAddress: "0xtestAccount",
Balance: 5,
},
}

err := tokenBalanceTable.Insert(context.Background(), expectedTokenBalances)
require.NoError(t, err)

// get token balances with range
tokenBalances, err := tokenBalanceTable.Get(context.Background(), NewSelectorRange(&TokenBalance{ID: 1}, &TokenBalance{ID: 2}))
require.NoError(t, err)
require.Equal(t, len(expectedTokenBalances), len(tokenBalances))
assert.Equal(t, expectedTokenBalances, tokenBalances)

// get token balance with non-existing id range
tokenBalances, err = tokenBalanceTable.Get(context.Background(), NewSelectorRange(&TokenBalance{ID: 3}, &TokenBalance{ID: 4}))
require.NoError(t, err)
assert.Equal(t, 0, len(tokenBalances))
}

func TestBondTable_Insert(t *testing.T) {
db := setupDatabase()
defer tearDownDatabase(db)
Expand Down Expand Up @@ -1162,7 +1256,7 @@ func TestBond_Batch(t *testing.T) {
exist := tokenBalanceTable.Exist(&TokenBalance{ID: 1}, batch)
require.True(t, exist)

tokenBalance, err := tokenBalanceTable.Get(NewSelectorPoint(&TokenBalance{ID: 1}), batch)
tokenBalance, err := tokenBalanceTable.Get(context.Background(), NewSelectorPoint(&TokenBalance{ID: 1}), batch)
require.NoError(t, err)
require.NotNil(t, tokenBalance)

Expand Down