Skip to content

Commit

Permalink
updated leaf format for blind tree and fixed a few bugs (keybase#20636)
Browse files Browse the repository at this point in the history
* updated leaf format for blind tree and fixed a few bugs

* cleanup
  • Loading branch information
AMarcedone authored Oct 29, 2019
1 parent 3cea570 commit 0dece15
Showing 7 changed files with 115 additions and 43 deletions.
18 changes: 9 additions & 9 deletions go/blindtree/values.go
Original file line number Diff line number Diff line change
@@ -29,7 +29,7 @@ var _ codec.Selfer = &BlindMerkleValue{}

func (t *BlindMerkleValue) CodecEncodeSelf(e *codec.Encoder) {
switch t.ValueType {
case ValueTypeTeamV1, ValueTypeStringForTesting:
case ValueTypeTeamV1, ValueTypeStringForTesting, ValueTypeEmpty:
// pass
default:
panic("Unknown merkle value type")
@@ -44,7 +44,7 @@ func (t *BlindMerkleValue) CodecDecodeSelf(d *codec.Decoder) {
case ValueTypeEmpty:
// Nothing to do here
case ValueTypeTeamV1:
var v Teamv1Value
var v TeamV1Value
d.MustDecode(&v)
t.InnerValue = v
case ValueTypeStringForTesting:
@@ -56,19 +56,19 @@ func (t *BlindMerkleValue) CodecDecodeSelf(d *codec.Decoder) {
}
}

type SigID []byte

type Teamv1HiddenTail sig3.LinkID

type Teamv1Value struct {
type TeamV1Value struct {
_struct struct{} `codec:",toarray"` //nolint
Tails map[keybase1.SeqType]Teamv1HiddenTail
Tails map[keybase1.SeqType]sig3.Tail
}

func BlindMerkleValueStringForTesting(s string) BlindMerkleValue {
return BlindMerkleValue{ValueType: ValueTypeStringForTesting, InnerValue: s}
}

func BlindMerkleValueTeamV1(v Teamv1Value) BlindMerkleValue {
func BlindMerkleValueTeamV1(v TeamV1Value) BlindMerkleValue {
return BlindMerkleValue{ValueType: ValueTypeTeamV1, InnerValue: v}
}

func BlindMerkleValueEmpty() BlindMerkleValue {
return BlindMerkleValue{ValueType: ValueTypeEmpty, InnerValue: nil}
}
13 changes: 8 additions & 5 deletions go/blindtree/values_test.go
Original file line number Diff line number Diff line change
@@ -5,16 +5,16 @@ import (
"testing"

"github.com/keybase/client/go/msgpack"

"github.com/keybase/client/go/protocol/keybase1"
"github.com/keybase/client/go/sig3"

"github.com/stretchr/testify/require"
)

func TestEncodeMerkleValues(t *testing.T) {
fakeTail := Teamv1HiddenTail([32]byte{0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04})
fakeTeamValue := Teamv1Value{
Tails: map[keybase1.SeqType]Teamv1HiddenTail{keybase1.SeqType_TEAM_PRIVATE_HIDDEN: fakeTail},
fakeTail := sig3.Tail{Seqno: keybase1.Seqno(2), Hash: [32]byte{0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04, 0x01, 0x02, 0x03, 0x04}}
fakeTeamValue := TeamV1Value{
Tails: map[keybase1.SeqType]sig3.Tail{keybase1.SeqType_TEAM_PRIVATE_HIDDEN: fakeTail},
}

encodingTests := []struct {
@@ -26,6 +26,7 @@ func TestEncodeMerkleValues(t *testing.T) {
{"Bong", BlindMerkleValueStringForTesting("Bong"), ValueTypeStringForTesting},
{"", BlindMerkleValueStringForTesting(""), ValueTypeStringForTesting},
{fakeTeamValue, BlindMerkleValueTeamV1(fakeTeamValue), ValueTypeTeamV1},
{nil, BlindMerkleValueEmpty(), ValueTypeEmpty},
}

for _, et := range encodingTests {
@@ -48,9 +49,11 @@ func TestEncodeMerkleValues(t *testing.T) {
require.True(t, ok, "Got type %T", dec.InnerValue)
require.EqualValues(t, et.Value, s)
case ValueTypeTeamV1:
s, ok := dec.InnerValue.(Teamv1Value)
s, ok := dec.InnerValue.(TeamV1Value)
require.True(t, ok, "Got type %T", dec.InnerValue)
require.EqualValues(t, et.Value, s)
case ValueTypeEmpty:
require.Nil(t, dec.InnerValue)
default:
t.Errorf("The test does not suppor type %v", et.Type)
}
7 changes: 4 additions & 3 deletions go/merkletree2/interfaces.go
Original file line number Diff line number Diff line change
@@ -78,7 +78,8 @@ type StorageEngineWithBlinding interface {
// Transaction references a DB transaction.
type Transaction interface{}

type GetLatestValueWithProofResponse struct {
Value EncodedValue
Proof MerkleInclusionProof `codec:"r,omitempty"`
type GetValueWithProofResponse struct {
_struct struct{} `codec:",toarray"` //nolint
Value EncodedValue `codec:"v"`
Proof MerkleInclusionProof `codec:"r,omitempty"`
}
4 changes: 2 additions & 2 deletions go/merkletree2/proof_verifier.go
Original file line number Diff line number Diff line change
@@ -59,7 +59,7 @@ func (m *MerkleProofVerifier) verifyInclusionOrExclusionProof(ctx logger.Context

// Reconstruct the leaf node if necessary
var nodeHash Hash
if kvp.Value != nil || len(proof.OtherPairsInLeaf) > 0 {
if kvp.Value != nil || proof.OtherPairsInLeaf != nil {
valueToInsert := false
leafHashesLength := len(proof.OtherPairsInLeaf)
if kvp.Value != nil {
@@ -130,7 +130,7 @@ func (m *MerkleProofVerifier) verifyInclusionOrExclusionProof(ctx logger.Context

// Check the rootHash computed matches the expected value.
if !rootHash.Equal(expRootHash) {
return NewProofVerificationFailedError(fmt.Errorf("expected rootHash does not match the computed one: expected %X but got %X", expRootHash, rootHash))
return NewProofVerificationFailedError(fmt.Errorf("expected rootHash does not match the computed one (for key: %X, value: %v): expected %X but got %X", kvp.Key, kvp.Value, expRootHash, rootHash))
}

// Success!
3 changes: 3 additions & 0 deletions go/merkletree2/storage_engine_for_test.go
Original file line number Diff line number Diff line change
@@ -282,6 +282,9 @@ func (i *InMemoryStorageEngine) LookupKEVPairsUnderPosition(ctx logger.ContextIn
}
}
}
if len(kvps) == 0 {
return nil, nil, NewKeyNotFoundError()
}
return kvps, seqnos, nil
}

57 changes: 36 additions & 21 deletions go/merkletree2/tree.go
Original file line number Diff line number Diff line change
@@ -140,15 +140,19 @@ var _ codec.Selfer = &Node{}
func (n *Node) CodecEncodeSelf(e *codec.Encoder) {
if n.INodes != nil && n.LeafHashes != nil && len(n.INodes) > 0 && len(n.LeafHashes) > 0 {
panic("Cannot Encode a node with both Inodes and LeafHashes")
} else if n.INodes != nil && len(n.INodes) > 0 {
}

if n.INodes != nil && len(n.INodes) > 0 {
e.MustEncode(NodeTypeINode)
e.MustEncode(n.INodes)
} else if n.LeafHashes != nil && len(n.LeafHashes) > 0 {
e.MustEncode(NodeTypeLeaf)
e.MustEncode(n.LeafHashes)
} else {
e.MustEncode(nil)
return
}

// Note: we encode empty nodes (with empty or nil LeafHashes) as leaf nodes.
// This is so we can represent a tree with no values as a single (empty)
// leaf node.
e.MustEncode(NodeTypeLeaf)
e.MustEncode(n.LeafHashes)
}

func (n *Node) CodecDecodeSelf(d *codec.Decoder) {
@@ -446,11 +450,18 @@ func (t *Tree) GetKeyValuePair(ctx logger.ContextInterface, tr Transaction, s Se
}

// A MerkleInclusionProof proves that a specific key value pair is stored in a
// merkle tree, given the RootMetadata hash of such tree.
// merkle tree, given the RootMetadata hash of such tree. It can also be used to
// prove that a specific key is not part of the tree (we call this an exclusion
// or absence proof)
type MerkleInclusionProof struct {
_struct struct{} `codec:",toarray"` //nolint
KeySpecificSecret KeySpecificSecret `codec:"k"`
OtherPairsInLeaf []KeyHashPair `codec:"l"`
// When this struct is used as an exclusion proof, OtherPairsInLeaf is set
// to nil if the proof ends at an internal node, and set to a slice of
// length 0 or more if the proof ends at a (possibly empty) leaf node. In
// particular, a tree with no keys is encoded with the root being the only
// (empty leaf) node.
OtherPairsInLeaf []KeyHashPair `codec:"l"`
// SiblingHashesOnPath are ordered by level from the farthest to the closest
// to the root, and lexicographically within each level.
SiblingHashesOnPath []Hash `codec:"s"`
@@ -600,7 +611,6 @@ func (t *Tree) getEncodedValueWithInclusionProofOrExclusionProof(ctx logger.Cont
}

var kevps []KeyEncodedValuePair
var seqnos []Seqno

// We have two cases: either the node at leafPos is actually a leaf
// (which might or not contain the key which we are trying to look up),
@@ -617,7 +627,7 @@ func (t *Tree) getEncodedValueWithInclusionProofOrExclusionProof(ctx logger.Cont
if t.cfg.MaxValuesPerLeaf == 1 {
// Try to avoid LookupKEVPairsUnderPosition if possible as it is a range
// query and thus more expensive.
kevp, s, err := t.eng.LookupKEVPair(ctx, tr, s, k)
kevp, _, err := t.eng.LookupKEVPair(ctx, tr, s, k)
if err != nil {
// KeyNotFoundError is ignored as the inclusion proof we
// produce will prove that the key is not in the tree.
@@ -626,7 +636,6 @@ func (t *Tree) getEncodedValueWithInclusionProofOrExclusionProof(ctx logger.Cont
}
} else {
kevps = append(kevps, KeyEncodedValuePair{Key: k, Value: kevp})
seqnos = append(seqnos, s)
}
}

@@ -636,40 +645,46 @@ func (t *Tree) getEncodedValueWithInclusionProofOrExclusionProof(ctx logger.Cont
if len(kevps) == 0 {
// Lookup hashes of key value pairs stored at the same leaf.
// These pairs are ordered by key.
kevps, seqnos, err = t.eng.LookupKEVPairsUnderPosition(ctx, tr, s, leafPos)
kevps, _, err = t.eng.LookupKEVPairsUnderPosition(ctx, tr, s, leafPos)
if err != nil {
return nil, MerkleInclusionProof{}, err
// KeyNotFoundError is ignored. This would happen when we are
// trying to produce an absence proof on an empty tree: there
// would be a leaf node containing no keys.
if _, keyNotFound := err.(KeyNotFoundError); !keyNotFound {
return nil, MerkleInclusionProof{}, err
}
kevps = make([]KeyEncodedValuePair, 0)
}
}
}

var msMap map[Seqno]MasterSecret
if t.cfg.UseBlindedValueHashes && len(seqnos) > 0 {
msMap, err = t.eng.(StorageEngineWithBlinding).LookupMasterSecrets(ctx, tr, seqnos)
var ms MasterSecret
if t.cfg.UseBlindedValueHashes && len(kevps) > 0 {
msMap, err := t.eng.(StorageEngineWithBlinding).LookupMasterSecrets(ctx, tr, []Seqno{s})
if err != nil {
return nil, MerkleInclusionProof{}, err
}
ms = msMap[s]
}

// OtherPairsInLeaf will have length equal to kevps - 1 in an inclusion
// proof, and kevps in an absence proof.
if len(kevps) > 0 {
if kevps != nil {
proof.OtherPairsInLeaf = make([]KeyHashPair, 0, len(kevps))
}

for i, kevpi := range kevps {
for _, kevpi := range kevps {
if kevpi.Key.Equal(k) {
val = kevpi.Value
if t.cfg.UseBlindedValueHashes {
proof.KeySpecificSecret = t.cfg.Encoder.ComputeKeySpecificSecret(msMap[seqnos[i]], k)
proof.KeySpecificSecret = t.cfg.Encoder.ComputeKeySpecificSecret(ms, k)
} else {
proof.KeySpecificSecret = nil
}
continue
}

var hash Hash
hash, err = t.cfg.Encoder.HashKeyEncodedValuePairWithKeySpecificSecret(kevpi, t.cfg.Encoder.ComputeKeySpecificSecret(msMap[seqnos[i]], kevpi.Key))
hash, err = t.cfg.Encoder.HashKeyEncodedValuePairWithKeySpecificSecret(kevpi, t.cfg.Encoder.ComputeKeySpecificSecret(ms, kevpi.Key))
if err != nil {
return nil, MerkleInclusionProof{}, err
}
56 changes: 53 additions & 3 deletions go/merkletree2/tree_test.go
Original file line number Diff line number Diff line change
@@ -349,7 +349,6 @@ func TestHonestMerkleProofsVerifySuccesfully(t *testing.T) {
eVal, proof, err = tree.GetEncodedValueWithInclusionOrExclusionProofFromRootHash(NewLoggerContextTodoForTesting(t), nil, rootHash2, kvp.Key)
require.NoError(t, err)
require.Nil(t, eVal)
t.Logf("proof: %+v", proof)
require.NoError(t, verifier.VerifyExclusionProof(NewLoggerContextTodoForTesting(t), kvp.Key, &proof, rootHash2))

for _, kvp := range test.kvps1[:len(test.kvps1)-1] {
@@ -643,6 +642,51 @@ func TestSomeMaliciousExclusionProofsFail(t *testing.T) {
}
}

func TestExclProofOnEmptyTree(t *testing.T) {
config1bitU, config2bitsU, config3bitsU := getTreeCfgsWith1_2_3BitsPerIndexUnblinded(t)
config1bitB, config2bitsB, config3bitsB := getTreeCfgsWith1_2_3BitsPerIndexBlinded(t)
defaultStep := 2
kvps1_1bit, _, _ := getSampleKVPS1bit()
kvps1_3bits, _, _ := getSampleKVPS3bits()

config3bits2valsPerLeafU, err := NewConfig(IdentityHasher{}, false, 3, 2, 3, ConstructStringValueContainer)
require.NoError(t, err)
config3bits2valsPerLeafB, err := NewConfig(IdentityHasherBlinded{}, true, 3, 2, 3, ConstructStringValueContainer)
require.NoError(t, err)

tests := []struct {
cfg Config
kvps1 []KeyValuePair
extraKey Key
}{
{config1bitU, kvps1_1bit, []byte{0xaa}},
{config2bitsU, kvps1_1bit, []byte{0xaa}},
{config3bitsU, kvps1_3bits, []byte{0xaa, 0xaa, 0xaa}},
{config3bits2valsPerLeafU, kvps1_3bits, []byte{0xaa, 0xaa, 0xaa}},
{config1bitB, kvps1_1bit, []byte{0xaa}},
{config2bitsB, kvps1_1bit, []byte{0xaa}},
{config3bitsB, kvps1_3bits, []byte{0xaa, 0xaa, 0xaa}},
{config3bits2valsPerLeafB, kvps1_3bits, []byte{0xaa, 0xaa, 0xaa}},
}

for _, test := range tests {
t.Run(fmt.Sprintf("%v bits %v values per leaf tree (blinded %v)", test.cfg.BitsPerIndex, test.cfg.MaxValuesPerLeaf, test.cfg.UseBlindedValueHashes), func(t *testing.T) {
tree, err := NewTree(test.cfg, defaultStep, NewInMemoryStorageEngine(test.cfg), RootVersionV1)
require.NoError(t, err)
verifier := MerkleProofVerifier{cfg: test.cfg}

s1, rootHash1, err := tree.Build(NewLoggerContextTodoForTesting(t), nil, nil, nil)
require.NoError(t, err)
require.EqualValues(t, 1, s1)

eVal, proof, err := tree.GetEncodedValueWithInclusionOrExclusionProofFromRootHash(NewLoggerContextTodoForTesting(t), nil, rootHash1, test.extraKey)
require.NoError(t, err)
require.Nil(t, eVal)
require.NoError(t, verifier.VerifyExclusionProof(NewLoggerContextTodoForTesting(t), test.extraKey, &proof, rootHash1))
})
}
}

func TestVerifyInclusionProofFailureBranches(t *testing.T) {

cfg, err := NewConfig(IdentityHasherBlinded{}, true, 2, 4, 2, ConstructStringValueContainer)
@@ -832,9 +876,15 @@ func TestGetLatestRoot(t *testing.T) {

func TestNodeEncodingBasic(t *testing.T) {
n := Node{}
_, err := msgpack.EncodeCanonical(&n)
enc, err := msgpack.EncodeCanonical(&n)
require.NoError(t, err)

var den Node
err = msgpack.Decode(&den, enc)
require.NoError(t, err)
require.Equal(t, n, den)
require.NotEqual(t, Node{LeafHashes: []KeyHashPair{}}, den)

n.INodes = make([]Hash, 2)
_, err = msgpack.EncodeCanonical(&n)
require.NoError(t, err)
@@ -845,7 +895,7 @@ func TestNodeEncodingBasic(t *testing.T) {
require.Contains(t, err.Error(), "msgpack encode error")

n = Node{INodes: []Hash{Hash([]byte{0x01, 0x02}), Hash([]byte{0x03, 0x04})}}
enc, err := msgpack.EncodeCanonical(&n)
enc, err = msgpack.EncodeCanonical(&n)
require.NoError(t, err)

t.Logf("enc : %X", enc)

0 comments on commit 0dece15

Please sign in to comment.