add exclusion proofs to merkletree2 (keybase#20388)

* use a single handle for each merkletree2 encoder

* implement exclusion proofs

* added rootHash input to StoreRootMetadata method

* remove dead code

* fix error message

* bug fix

* exported two position functions to make testing merklestorage easier

* Added some comments

go/merkletree2/encoders.go

@@ -46,11 +46,7 @@ func NewBlindedSHA512_256v1Encoder() *BlindedSHA512_256v1Encoder {
 	mh.WriteExt = true
 	mh.Canonical = true
 
-	var mh2 codec.MsgpackHandle
-	mh2.WriteExt = true
-	mh2.Canonical = true
-
-	return &BlindedSHA512_256v1Encoder{enc: codec.NewEncoderBytes(nil, &mh), dec: codec.NewDecoderBytes(nil, &mh2)}
+	return &BlindedSHA512_256v1Encoder{enc: codec.NewEncoderBytes(nil, &mh), dec: codec.NewDecoderBytes(nil, &mh)}
 }
 
 func (e *BlindedSHA512_256v1Encoder) Encode(o interface{}) (out []byte, err error) {

go/merkletree2/errors.go

@@ -42,6 +42,19 @@ func NewProofVerificationFailedError(reason error) ProofVerificationFailedError
 	return ProofVerificationFailedError{reason: reason}
 }
 
+// NodeNotFoundError is returned by a StorageEngine when trying to fetch an internal node
+// which is not part of the tree at a specific Seqno.
+type NodeNotFoundError struct{}
+
+func (e NodeNotFoundError) Error() string {
+	return fmt.Sprintf("Node not found.")
+}
+
+// NewNodeNotFoundError returns a new error
+func NewNodeNotFoundError() NodeNotFoundError {
+	return NodeNotFoundError{}
+}
+
 // KeyNotFoundError is returned when trying to fetch a key which is not part of
 // the tree at a specific Seqno.
 type KeyNotFoundError struct{}

go/merkletree2/interfaces.go

@@ -10,11 +10,18 @@ type StorageEngine interface {
 	// StoreKVPairs stores the []KeyEncodedValuePair in the tree.
 	StoreKEVPairs(logger.ContextInterface, Transaction, Seqno, []KeyEncodedValuePair) error
 
+	// StoreNode stores the Hash (of a tree node) at the provided Position,
+	// Seqno in the tree.
 	StoreNode(logger.ContextInterface, Transaction, Seqno, *Position, Hash) error
 
+	// StoreNode takes multiple pairs of a position and a hash, and stores each
+	// hash (of a tree node) at the corresponding position and at the supplied
+	// Seqno in the tree.
 	StoreNodes(logger.ContextInterface, Transaction, Seqno, []PositionHashPair) error
 
-	StoreRootMetadata(logger.ContextInterface, Transaction, RootMetadata) error
+	// StoreRootMetadata stores the supplied RootMetadata, along with the
+	// corresponding Hash.
+	StoreRootMetadata(logger.ContextInterface, Transaction, RootMetadata, Hash) error
 
 	// LookupLatestRoot returns the latest root metadata and sequence number in
 	// the tree. If no root is found, then a NoLatestRootFound error is returned.
@@ -23,6 +30,9 @@ type StorageEngine interface {
 	// If there is no root for the specified Seqno, an InvalidSeqnoError is returned.
 	LookupRoot(logger.ContextInterface, Transaction, Seqno) (RootMetadata, error)
 
+	// Returns a RootMetadata given its Hash.
+	LookupRootFromHash(logger.ContextInterface, Transaction, Hash) (RootMetadata, error)
+
 	// LookupRoots returns the RootMetadata objects in the tree at the
 	// supplied Seqnos, ordered by seqno.
 	LookupRoots(logger.ContextInterface, Transaction, []Seqno) ([]RootMetadata, error)

go/merkletree2/position.go

@@ -14,11 +14,11 @@ import (
 // (0b00000101).
 type Position big.Int
 
-func (t *Config) getRootPosition() *Position {
+func (t *Config) GetRootPosition() *Position {
 	return (*Position)(big.NewInt(1))
 }
 
-func (t *Config) getChild(p *Position, c ChildIndex) *Position {
+func (t *Config) GetChild(p *Position, c ChildIndex) *Position {
 	var q big.Int
 	q.Lsh((*big.Int)(p), uint(t.BitsPerIndex))
 	q.Bits()[0] = q.Bits()[0] | big.Word(c)

go/merkletree2/position_test.go

@@ -18,15 +18,15 @@ func TestEncoding(t *testing.T) {
 		bin string
 		p   Position
 	}{
-		{config1bit, "1", *config1bit.getRootPosition()},
-		{config2bits, "1", *config2bits.getRootPosition()},
-		{config3bits, "1", *config3bits.getRootPosition()},
-		{config1bit, "10", *config1bit.getChild(config1bit.getRootPosition(), 0)},
-		{config2bits, "100", *config2bits.getChild(config2bits.getRootPosition(), 0)},
-		{config3bits, "1000", *config3bits.getChild(config3bits.getRootPosition(), 0)},
-		{config1bit, "101", *config1bit.getChild(config1bit.getChild(config1bit.getRootPosition(), 0), 1)},
-		{config2bits, "10011", *config2bits.getChild(config2bits.getChild(config2bits.getRootPosition(), 0), 3)},
-		{config3bits, "1000101", *config3bits.getChild(config3bits.getChild(config3bits.getRootPosition(), 0), 5)},
+		{config1bit, "1", *config1bit.GetRootPosition()},
+		{config2bits, "1", *config2bits.GetRootPosition()},
+		{config3bits, "1", *config3bits.GetRootPosition()},
+		{config1bit, "10", *config1bit.GetChild(config1bit.GetRootPosition(), 0)},
+		{config2bits, "100", *config2bits.GetChild(config2bits.GetRootPosition(), 0)},
+		{config3bits, "1000", *config3bits.GetChild(config3bits.GetRootPosition(), 0)},
+		{config1bit, "101", *config1bit.GetChild(config1bit.GetChild(config1bit.GetRootPosition(), 0), 1)},
+		{config2bits, "10011", *config2bits.GetChild(config2bits.GetChild(config2bits.GetRootPosition(), 0), 3)},
+		{config3bits, "1000101", *config3bits.GetChild(config3bits.GetChild(config3bits.GetRootPosition(), 0), 5)},
 	}
 
 	for _, et := range encodingTests {
@@ -65,7 +65,7 @@ func TestGetAndUpdateParentAndGetChild(t *testing.T) {
 			parent, err := makePositionFromStringForTesting(test.parent)
 			require.NoError(t, err)
 			require.True(t, test.c.getParent(&child).Equals(&parent))
-			require.True(t, test.c.getChild(&parent, test.i).Equals(&child))
+			require.True(t, test.c.GetChild(&parent, test.i).Equals(&child))
 			parentInPlace := child.Clone()
 			test.c.updateToParent(parentInPlace)
 			require.True(t, parentInPlace.Equals(&parent))

go/merkletree2/proof_verifier.go

@@ -15,52 +15,80 @@ func NewMerkleProofVerifier(c Config) MerkleProofVerifier {
 	return MerkleProofVerifier{cfg: c}
 }
 
-func (m *MerkleProofVerifier) VerifyInclusionProof(ctx logger.ContextInterface, kvp KeyValuePair, proof *MerkleInclusionProof, expRootHash Hash) error {
+func (m *MerkleProofVerifier) VerifyInclusionProof(ctx logger.ContextInterface, kvp KeyValuePair, proof *MerkleInclusionProof, expRootHash Hash) (err error) {
+	if kvp.Value == nil {
+		return NewProofVerificationFailedError(fmt.Errorf("Keys cannot have nil values in the tree"))
+	}
+	return m.verifyInclusionOrExclusionProof(ctx, kvp, proof, expRootHash)
+}
+
+// VerifyExclusionProof uses a MerkleInclusionProof to assert that a specific key is not part of the tree
+func (m *MerkleProofVerifier) VerifyExclusionProof(ctx logger.ContextInterface, k Key, proof *MerkleInclusionProof, expRootHash Hash) (err error) {
+	return m.verifyInclusionOrExclusionProof(ctx, KeyValuePair{Key: k}, proof, expRootHash)
+}
+
+// if kvp.Value == nil, this functions checks that kvp.Key is not included in the tree. Otherwise, it checks that kvp is included in the tree.
+func (m *MerkleProofVerifier) verifyInclusionOrExclusionProof(ctx logger.ContextInterface, kvp KeyValuePair, proof *MerkleInclusionProof, expRootHash Hash) (err error) {
 	if proof == nil {
 		return NewProofVerificationFailedError(fmt.Errorf("nil proof"))
 	}
 
-	if proof.RootMetadataNoHash.RootVersion != RootVersionV1 {
-		return NewProofVerificationFailedError(libkb.NewAppOutdatedError(fmt.Errorf("RootVersion %v is not supported (this client can only handle V1)", proof.RootMetadataNoHash.RootVersion)))
+	var kvpHash Hash
+	// Hash the key value pair if necessary
+	if kvp.Value != nil {
+		kvpHash, err = m.cfg.Encoder.HashKeyValuePairWithKeySpecificSecret(kvp, proof.KeySpecificSecret)
+		if err != nil {
+			return NewProofVerificationFailedError(err)
+		}
 	}
 
-	// Hash the key value pair
-	kvpHash, err := m.cfg.Encoder.HashKeyValuePairWithKeySpecificSecret(kvp, proof.KeySpecificSecret)
-	if err != nil {
-		return NewProofVerificationFailedError(err)
+	if proof.RootMetadataNoHash.RootVersion != RootVersionV1 {
+		return NewProofVerificationFailedError(libkb.NewAppOutdatedError(fmt.Errorf("RootVersion %v is not supported (this client can only handle V1)", proof.RootMetadataNoHash.RootVersion)))
 	}
 
 	if len(kvp.Key) != m.cfg.KeysByteLength {
 		return NewProofVerificationFailedError(fmt.Errorf("Key has wrong length for this tree: %v (expected %v)", len(kvp.Key), m.cfg.KeysByteLength))
 	}
 
-	if len(proof.OtherPairsInLeaf)+1 > m.cfg.MaxValuesPerLeaf {
+	// inclusion proofs for existing values can have at most MaxValuesPerLeaf - 1
+	// other pairs in the leaf, while exclusion proofs can have at most
+	// MaxValuesPerLeaf.
+	if (kvp.Value != nil && len(proof.OtherPairsInLeaf)+1 > m.cfg.MaxValuesPerLeaf) || (kvp.Value == nil && len(proof.OtherPairsInLeaf) > m.cfg.MaxValuesPerLeaf) {
 		return NewProofVerificationFailedError(fmt.Errorf("Too many keys in leaf: %v > %v", len(proof.OtherPairsInLeaf)+1, m.cfg.MaxValuesPerLeaf))
 	}
 
-	// Reconstruct the leaf node
-	leaf := Node{LeafHashes: make([]KeyHashPair, len(proof.OtherPairsInLeaf)+1)}
-
-	// LeafHashes is obtained by adding kvp into OtherPairsInLeaf while maintaining sorted order
-	for i, j, isInserted := 0, 0, false; i < len(proof.OtherPairsInLeaf)+1; i++ {
-		if (j < len(proof.OtherPairsInLeaf) && !isInserted && proof.OtherPairsInLeaf[j].Key.Cmp(kvp.Key) > 0) || j >= len(proof.OtherPairsInLeaf) {
-			leaf.LeafHashes[i] = KeyHashPair{Key: kvp.Key, Hash: kvpHash}
-			isInserted = true
-		} else {
-			leaf.LeafHashes[i] = proof.OtherPairsInLeaf[j]
-			j++
+	// Reconstruct the leaf node if necessary
+	var nodeHash Hash
+	if kvp.Value != nil || len(proof.OtherPairsInLeaf) > 0 {
+		valueToInsert := false
+		leafHashesLength := len(proof.OtherPairsInLeaf)
+		if kvp.Value != nil {
+			leafHashesLength++
+			valueToInsert = true
 		}
+		leaf := Node{LeafHashes: make([]KeyHashPair, leafHashesLength)}
+
+		// LeafHashes is obtained by adding kvp into OtherPairsInLeaf while maintaining sorted order
+		for i, j := 0, 0; i < leafHashesLength; i++ {
+			if (j < len(proof.OtherPairsInLeaf) && valueToInsert && proof.OtherPairsInLeaf[j].Key.Cmp(kvp.Key) > 0) || j >= len(proof.OtherPairsInLeaf) {
+				leaf.LeafHashes[i] = KeyHashPair{Key: kvp.Key, Hash: kvpHash}
+				valueToInsert = false
+			} else {
+				leaf.LeafHashes[i] = proof.OtherPairsInLeaf[j]
+				j++
+			}
 
-		// Ensure all the KeyHashPairs in the leaf node are different
-		if i > 0 && leaf.LeafHashes[i-1].Key.Cmp(leaf.LeafHashes[i].Key) >= 0 {
-			return NewProofVerificationFailedError(fmt.Errorf("Error in Leaf Key ordering or duplicated key: %v >= %v", leaf.LeafHashes[i-1].Key, leaf.LeafHashes[i].Key))
+			// Ensure all the KeyHashPairs in the leaf node are different
+			if i > 0 && leaf.LeafHashes[i-1].Key.Cmp(leaf.LeafHashes[i].Key) >= 0 {
+				return NewProofVerificationFailedError(fmt.Errorf("Error in Leaf Key ordering or duplicated key: %v >= %v", leaf.LeafHashes[i-1].Key, leaf.LeafHashes[i].Key))
+			}
 		}
-	}
 
-	// Recompute the hashes on the nodes on the path from the leaf to the root.
-	_, nodeHash, err := m.cfg.Encoder.EncodeAndHashGeneric(leaf)
-	if err != nil {
-		return NewProofVerificationFailedError(err)
+		// Recompute the hashes on the nodes on the path from the leaf to the root.
+		_, nodeHash, err = m.cfg.Encoder.EncodeAndHashGeneric(leaf)
+		if err != nil {
+			return NewProofVerificationFailedError(err)
+		}
 	}
 
 	sibH := proof.SiblingHashesOnPath

go/merkletree2/storage_engine_for_test.go

@@ -17,6 +17,7 @@ type InMemoryStorageEngine struct {
 	SortedKVPRs      []*KVPRecord
 	Nodes            map[string]*NodeRecord
 	MasterSecretsMap map[Seqno]MasterSecret
+	ReverseRootMap   map[string]Seqno
 
 	// used to make prefix queries efficient. Not otherwise necessary
 	//PositionToKeys map[string](map[string]bool)
@@ -60,6 +61,7 @@ func NewInMemoryStorageEngine(cfg Config) *InMemoryStorageEngine {
 	i.MasterSecretsMap = make(map[Seqno]MasterSecret)
 	i.Roots = make(map[Seqno]RootMetadata)
 	i.Nodes = make(map[string]*NodeRecord)
+	i.ReverseRootMap = make(map[string]Seqno)
 	i.cfg = cfg
 	return &i
 }
@@ -114,8 +116,9 @@ func (i *InMemoryStorageEngine) StoreNode(c logger.ContextInterface, t Transacti
 	return nil
 }
 
-func (i *InMemoryStorageEngine) StoreRootMetadata(c logger.ContextInterface, t Transaction, r RootMetadata) error {
+func (i *InMemoryStorageEngine) StoreRootMetadata(c logger.ContextInterface, t Transaction, r RootMetadata, h Hash) error {
 	i.Roots[r.Seqno] = r
+	i.ReverseRootMap[string(h)] = r.Seqno
 	return nil
 }
 
@@ -140,6 +143,14 @@ func (i *InMemoryStorageEngine) LookupRoot(c logger.ContextInterface, t Transact
 	return RootMetadata{}, NewInvalidSeqnoError(s, fmt.Errorf("No root at seqno %v", s))
 }
 
+func (i *InMemoryStorageEngine) LookupRootFromHash(c logger.ContextInterface, t Transaction, h Hash) (RootMetadata, error) {
+	s, found := i.ReverseRootMap[string(h)]
+	if found {
+		return i.Roots[s], nil
+	}
+	return RootMetadata{}, NewInvalidSeqnoError(s, fmt.Errorf("No root at seqno %v", s))
+}
+
 func (i *InMemoryStorageEngine) LookupRoots(c logger.ContextInterface, t Transaction, seqnos []Seqno) (roots []RootMetadata, err error) {
 	seqnosSorted := make([]Seqno, len(seqnos))
 	copy(seqnosSorted, seqnos)
@@ -159,6 +170,9 @@ func (i *InMemoryStorageEngine) LookupRoots(c logger.ContextInterface, t Transac
 }
 
 func (i *InMemoryStorageEngine) LookupRootHashes(c logger.ContextInterface, t Transaction, seqnos []Seqno) (hashes []Hash, err error) {
+	if len(seqnos) == 0 {
+		return nil, fmt.Errorf("No seqnos requested")
+	}
 	seqnosSorted := make([]Seqno, len(seqnos))
 	copy(seqnosSorted, seqnos)
 	sort.Sort(SeqnoSortedAsInt(seqnosSorted))
@@ -182,14 +196,14 @@ func (i *InMemoryStorageEngine) LookupRootHashes(c logger.ContextInterface, t Tr
 func (i *InMemoryStorageEngine) LookupNode(c logger.ContextInterface, t Transaction, s Seqno, p *Position) (Hash, error) {
 	node, found := i.Nodes[string(p.GetBytes())]
 	if !found {
-		return nil, errors.New("No node found")
+		return nil, NewNodeNotFoundError()
 	}
 	for ; node != nil; node = node.next {
 		if node.s <= s {
 			return node.h, nil
 		}
 	}
-	return nil, errors.New("No node version found")
+	return nil, NewNodeNotFoundError()
 }
 
 func (i *InMemoryStorageEngine) LookupNodes(c logger.ContextInterface, t Transaction, s Seqno, positions []Position) (res []PositionHashPair, err error) {