Skip to content

Commit

Permalink
les, light: fix CHT trie retrievals (#16039)
Browse files Browse the repository at this point in the history 
* les, light: fix CHT trie retrievals

* les, light: minor polishes, test remote CHT retrievals

* les, light: deterministic nodeset rlp, bloombits test skeleton

* les: add an event emission to the les bloombits test

* les: drop dead tester code
  • Loading branch information
@karalabe
karalabe authored Feb 11, 2018
1 parent 5cf75a3 commit 7a0019c
Show file tree
Hide file tree
Showing 10 changed files with 210 additions and 119 deletions.
44 changes: 16 additions & 28 deletions les/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -790,10 +790,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
break
}
}
proofs := nodes.NodeList()
bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost)
pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost)
return p.SendProofsV2(req.ReqID, bv, proofs)
return p.SendProofsV2(req.ReqID, bv, nodes.NodeList())

case ProofsV1Msg:
if pm.odr == nil {
Expand Down Expand Up @@ -856,15 +855,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
if reject(uint64(reqCnt), MaxHelperTrieProofsFetch) {
return errResp(ErrRequestRejected, "")
}
trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix))
for _, req := range req.Reqs {
if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil {
sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.ChtV1Frequency-1)
sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1)
if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) {
statedb, err := pm.blockchain.State()
if err != nil {
continue
}
trie, err := statedb.Database().OpenTrie(root)
trie, err := trie.New(root, trieDb)
if err != nil {
continue
}
Expand All @@ -878,7 +874,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
if bytes += proof.DataSize() + estHeaderRlpSize; bytes >= softResponseLimit {
break
}

}
}
}
Expand Down Expand Up @@ -910,20 +905,16 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
lastIdx uint64
lastType uint
root common.Hash
statedb *state.StateDB
trie state.Trie
auxTrie *trie.Trie
)

nodes := light.NewNodeSet()

for _, req := range req.Reqs {
if trie == nil || req.HelperTrieType != lastType || req.TrieIdx != lastIdx {
statedb, trie, lastType, lastIdx = nil, nil, req.HelperTrieType, req.TrieIdx
if auxTrie == nil || req.Type != lastType || req.TrieIdx != lastIdx {
auxTrie, lastType, lastIdx = nil, req.Type, req.TrieIdx

if root, _ = pm.getHelperTrie(req.HelperTrieType, req.TrieIdx); root != (common.Hash{}) {
if statedb, _ = pm.blockchain.State(); statedb != nil {
trie, _ = statedb.Database().OpenTrie(root)
}
var prefix string
if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) {
auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix)))
}
}
if req.AuxReq == auxRoot {
Expand All @@ -934,8 +925,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
auxData = append(auxData, data)
auxBytes += len(data)
} else {
if trie != nil {
trie.Prove(req.Key, req.FromLevel, nodes)
if auxTrie != nil {
auxTrie.Prove(req.Key, req.FromLevel, nodes)
}
if req.AuxReq != 0 {
data := pm.getHelperTrieAuxData(req)
Expand All @@ -947,10 +938,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
break
}
}
proofs := nodes.NodeList()
bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost)
pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost)
return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: proofs, AuxData: auxData})
return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData})

case HeaderProofsMsg:
if pm.odr == nil {
Expand Down Expand Up @@ -1123,7 +1113,7 @@ func (pm *ProtocolManager) getAccount(statedb *state.StateDB, root, hash common.
func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, string) {
switch id {
case htCanonical:
sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.ChtFrequency-1)
sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.CHTFrequencyClient-1)
return light.GetChtV2Root(pm.chainDb, idx, sectionHead), light.ChtTablePrefix
case htBloomBits:
sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.BloomTrieFrequency-1)
Expand All @@ -1134,10 +1124,8 @@ func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, stri

// getHelperTrieAuxData returns requested auxiliary data for the given HelperTrie request
func (pm *ProtocolManager) getHelperTrieAuxData(req HelperTrieReq) []byte {
if req.HelperTrieType == htCanonical && req.AuxReq == auxHeader {
if len(req.Key) != 8 {
return nil
}
switch {
case req.Type == htCanonical && req.AuxReq == auxHeader && len(req.Key) == 8:
blockNum := binary.BigEndian.Uint64(req.Key)
hash := core.GetCanonicalHash(pm.chainDb, blockNum)
return core.GetHeaderRLP(pm.chainDb, hash, blockNum)
Expand Down
158 changes: 117 additions & 41 deletions les/handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package les

import (
"bytes"
"encoding/binary"
"math/big"
"math/rand"
"testing"
Expand Down Expand Up @@ -45,27 +45,8 @@ func expectResponse(r p2p.MsgReader, msgcode, reqID, bv uint64, data interface{}
return p2p.ExpectMsg(r, msgcode, resp{reqID, bv, data})
}

func testCheckProof(t *testing.T, exp *light.NodeSet, got light.NodeList) {
if exp.KeyCount() > len(got) {
t.Errorf("proof has fewer nodes than expected")
return
}
if exp.KeyCount() < len(got) {
t.Errorf("proof has more nodes than expected")
return
}
for _, node := range got {
n, _ := exp.Get(crypto.Keccak256(node))
if !bytes.Equal(n, node) {
t.Errorf("proof contents mismatch")
return
}
}
}

// Tests that block headers can be retrieved from a remote chain based on user queries.
func TestGetBlockHeadersLes1(t *testing.T) { testGetBlockHeaders(t, 1) }

func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) }

func testGetBlockHeaders(t *testing.T, protocol int) {
Expand Down Expand Up @@ -196,7 +177,6 @@ func testGetBlockHeaders(t *testing.T, protocol int) {

// Tests that block contents can be retrieved from a remote chain based on their hashes.
func TestGetBlockBodiesLes1(t *testing.T) { testGetBlockBodies(t, 1) }

func TestGetBlockBodiesLes2(t *testing.T) { testGetBlockBodies(t, 2) }

func testGetBlockBodies(t *testing.T, protocol int) {
Expand Down Expand Up @@ -274,7 +254,6 @@ func testGetBlockBodies(t *testing.T, protocol int) {

// Tests that the contract codes can be retrieved based on account addresses.
func TestGetCodeLes1(t *testing.T) { testGetCode(t, 1) }

func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) }

func testGetCode(t *testing.T, protocol int) {
Expand Down Expand Up @@ -309,7 +288,6 @@ func testGetCode(t *testing.T, protocol int) {

// Tests that the transaction receipts can be retrieved based on hashes.
func TestGetReceiptLes1(t *testing.T) { testGetReceipt(t, 1) }

func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) }

func testGetReceipt(t *testing.T, protocol int) {
Expand Down Expand Up @@ -338,7 +316,6 @@ func testGetReceipt(t *testing.T, protocol int) {

// Tests that trie merkle proofs can be retrieved
func TestGetProofsLes1(t *testing.T) { testGetProofs(t, 1) }

func TestGetProofsLes2(t *testing.T) { testGetProofs(t, 2) }

func testGetProofs(t *testing.T, protocol int) {
Expand Down Expand Up @@ -389,27 +366,126 @@ func testGetProofs(t *testing.T, protocol int) {
case 2:
cost := peer.GetRequestCost(GetProofsV2Msg, len(proofreqs))
sendRequest(peer.app, GetProofsV2Msg, 42, cost, proofreqs)
msg, err := peer.app.ReadMsg()
if err != nil {
t.Errorf("Message read error: %v", err)
}
var resp struct {
ReqID, BV uint64
Data light.NodeList
}
if err := msg.Decode(&resp); err != nil {
t.Errorf("reply decode error: %v", err)
if err := expectResponse(peer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil {
t.Errorf("proofs mismatch: %v", err)
}
if msg.Code != ProofsV2Msg {
t.Errorf("Message code mismatch")
}
}

// Tests that CHT proofs can be correctly retrieved.
func TestGetCHTProofsLes1(t *testing.T) { testGetCHTProofs(t, 1) }
func TestGetCHTProofsLes2(t *testing.T) { testGetCHTProofs(t, 2) }

func testGetCHTProofs(t *testing.T, protocol int) {
// Figure out the client's CHT frequency
frequency := uint64(light.CHTFrequencyClient)
if protocol == 1 {
frequency = uint64(light.CHTFrequencyServer)
}
// Assemble the test environment
db, _ := ethdb.NewMemDatabase()
pm := newTestProtocolManagerMust(t, false, int(frequency)+light.HelperTrieProcessConfirmations, testChainGen, nil, nil, db)
bc := pm.blockchain.(*core.BlockChain)
peer, _ := newTestPeer(t, "peer", protocol, pm, true)
defer peer.close()

// Wait a while for the CHT indexer to process the new headers
time.Sleep(100 * time.Millisecond * time.Duration(frequency/light.CHTFrequencyServer)) // Chain indexer throttling
time.Sleep(250 * time.Millisecond) // CI tester slack

// Assemble the proofs from the different protocols
header := bc.GetHeaderByNumber(frequency)
rlp, _ := rlp.EncodeToBytes(header)

key := make([]byte, 8)
binary.BigEndian.PutUint64(key, frequency)

proofsV1 := []ChtResp{{
Header: header,
}}
proofsV2 := HelperTrieResps{
AuxData: [][]byte{rlp},
}
switch protocol {
case 1:
root := light.GetChtRoot(db, 0, bc.GetHeaderByNumber(frequency-1).Hash())
trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix)))

var proof light.NodeList
trie.Prove(key, 0, &proof)
proofsV1[0].Proof = proof

case 2:
root := light.GetChtV2Root(db, 0, bc.GetHeaderByNumber(frequency-1).Hash())
trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix)))
trie.Prove(key, 0, &proofsV2.Proofs)
}
// Assemble the requests for the different protocols
requestsV1 := []ChtReq{{
ChtNum: 1,
BlockNum: frequency,
}}
requestsV2 := []HelperTrieReq{{
Type: htCanonical,
TrieIdx: 0,
Key: key,
AuxReq: auxHeader,
}}
// Send the proof request and verify the response
switch protocol {
case 1:
cost := peer.GetRequestCost(GetHeaderProofsMsg, len(requestsV1))
sendRequest(peer.app, GetHeaderProofsMsg, 42, cost, requestsV1)
if err := expectResponse(peer.app, HeaderProofsMsg, 42, testBufLimit, proofsV1); err != nil {
t.Errorf("proofs mismatch: %v", err)
}
if resp.ReqID != 42 {
t.Errorf("ReqID mismatch")
case 2:
cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requestsV2))
sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requestsV2)
if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofsV2); err != nil {
t.Errorf("proofs mismatch: %v", err)
}
if resp.BV != testBufLimit {
t.Errorf("BV mismatch")
}
}

// Tests that bloombits proofs can be correctly retrieved.
func TestGetBloombitsProofs(t *testing.T) {
// Assemble the test environment
db, _ := ethdb.NewMemDatabase()
pm := newTestProtocolManagerMust(t, false, light.BloomTrieFrequency+256, testChainGen, nil, nil, db)
bc := pm.blockchain.(*core.BlockChain)
peer, _ := newTestPeer(t, "peer", 2, pm, true)
defer peer.close()

// Wait a while for the bloombits indexer to process the new headers
time.Sleep(100 * time.Millisecond * time.Duration(light.BloomTrieFrequency/4096)) // Chain indexer throttling
time.Sleep(250 * time.Millisecond) // CI tester slack

// Request and verify each bit of the bloom bits proofs
for bit := 0; bit < 2048; bit++ {
// Assemble therequest and proofs for the bloombits
key := make([]byte, 10)

binary.BigEndian.PutUint16(key[:2], uint16(bit))
binary.BigEndian.PutUint64(key[2:], uint64(light.BloomTrieFrequency))

requests := []HelperTrieReq{{
Type: htBloomBits,
TrieIdx: 0,
Key: key,
}}
var proofs HelperTrieResps

root := light.GetBloomTrieRoot(db, 0, bc.GetHeaderByNumber(light.BloomTrieFrequency-1).Hash())
trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.BloomTrieTablePrefix)))
trie.Prove(key, 0, &proofs.Proofs)

// Send the proof request and verify the response
cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requests))
sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requests)
if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil {
t.Errorf("bit %d: proofs mismatch: %v", bit, err)
}
testCheckProof(t, proofsV2, resp.Data)
}
}

Expand Down
26 changes: 22 additions & 4 deletions les/helper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/eth"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/les/flowcontrol"
Expand All @@ -55,6 +56,9 @@ var (
testContractCodeDeployed = testContractCode[16:]
testContractDeployed = uint64(2)

testEventEmitterCode = common.Hex2Bytes("60606040523415600e57600080fd5b7f57050ab73f6b9ebdd9f76b8d4997793f48cf956e965ee070551b9ca0bb71584e60405160405180910390a160358060476000396000f3006060604052600080fd00a165627a7a723058203f727efcad8b5811f8cb1fc2620ce5e8c63570d697aef968172de296ea3994140029")
testEventEmitterAddr common.Address

testBufLimit = uint64(100)
)

Expand Down Expand Up @@ -85,15 +89,19 @@ func testChainGen(i int, block *core.BlockGen) {
// In block 2, the test bank sends some more ether to account #1.
// acc1Addr passes it on to account #2.
// acc1Addr creates a test contract.
tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey)
// acc1Addr creates a test event.
nonce := block.TxNonce(acc1Addr)

tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey)
tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key)
nonce++
tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, acc1Key)
testContractAddr = crypto.CreateAddress(acc1Addr, nonce)
tx3, _ := types.SignTx(types.NewContractCreation(nonce+1, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, acc1Key)
testContractAddr = crypto.CreateAddress(acc1Addr, nonce+1)
tx4, _ := types.SignTx(types.NewContractCreation(nonce+2, big.NewInt(0), 200000, big.NewInt(0), testEventEmitterCode), signer, acc1Key)
testEventEmitterAddr = crypto.CreateAddress(acc1Addr, nonce+2)
block.AddTx(tx1)
block.AddTx(tx2)
block.AddTx(tx3)
block.AddTx(tx4)
case 2:
// Block 3 is empty but was mined by account #2.
block.SetCoinbase(acc2Addr)
Expand Down Expand Up @@ -147,6 +155,16 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor
chain, _ = light.NewLightChain(odr, gspec.Config, engine)
} else {
blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{})

chtIndexer := light.NewChtIndexer(db, false)
chtIndexer.Start(blockchain)

bbtIndexer := light.NewBloomTrieIndexer(db, false)

bloomIndexer := eth.NewBloomIndexer(db, params.BloomBitsBlocks)
bloomIndexer.AddChildIndexer(bbtIndexer)
bloomIndexer.Start(blockchain)

gchain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, blocks, generator)
if _, err := blockchain.InsertChain(gchain); err != nil {
panic(err)
Expand Down
Loading

0 comments on commit 7a0019c

Please sign in to comment.