From 9b0edf7876d2de357d08f0a746732cf1f65ffea1 Mon Sep 17 00:00:00 2001 From: Aaron Gao Date: Thu, 17 Jun 2021 14:37:32 -1000 Subject: [PATCH] [jf+exec] add batch_update method to Jellyfish merkle to leverage execution result Closes: #8574 --- Cargo.lock | 11 +- Cargo.toml | 1 - common/nibble/Cargo.toml | 19 - crypto/crypto/src/hash.rs | 8 + crypto/crypto/src/unit_tests/hash_test.rs | 15 + execution/executor/src/lib.rs | 12 +- execution/executor/src/types.rs | 11 + storage/diemdb-benchmark/src/lib.rs | 1 + storage/diemdb/src/lib.rs | 14 +- storage/diemdb/src/pruner/test.rs | 1 + storage/diemdb/src/state_store/mod.rs | 10 +- .../src/state_store/state_store_test.rs | 3 +- storage/diemdb/src/test_helper.rs | 1 + storage/jellyfish-merkle/Cargo.toml | 5 +- storage/jellyfish-merkle/src/iterator/mod.rs | 7 +- .../src/jellyfish_merkle_test.rs | 83 +++-- storage/jellyfish-merkle/src/lib.rs | 342 +++++++++++++++++- storage/jellyfish-merkle/src/node_type/mod.rs | 8 +- .../src/node_type/node_type_test.rs | 4 +- storage/jellyfish-merkle/src/restore/mod.rs | 6 +- .../src/tree_cache/tree_cache_test.rs | 6 +- storage/scratchpad/src/sparse_merkle/mod.rs | 115 +++++- types/src/lib.rs | 1 + .../src/lib.rs => types/src/nibble/mod.rs | 6 + .../src/nibble}/nibble_path/mod.rs | 5 +- .../nibble}/nibble_path/nibble_path_test.rs | 2 +- types/src/proptest_types.rs | 1 + types/src/transaction/mod.rs | 8 + 28 files changed, 596 insertions(+), 110 deletions(-) delete mode 100644 common/nibble/Cargo.toml rename common/nibble/src/lib.rs => types/src/nibble/mod.rs (84%) rename {storage/jellyfish-merkle/src => types/src/nibble}/nibble_path/mod.rs (99%) rename {storage/jellyfish-merkle/src => types/src/nibble}/nibble_path/nibble_path_test.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index eb3b6dde366a8..37c271b659ac8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1885,9 +1885,9 @@ dependencies = [ "diem-crypto-derive", "diem-infallible", "diem-metrics", - "diem-nibble", "diem-types", "diem-workspace-hack", + "itertools 0.10.0", "mirai-annotations", "num-derive", "num-traits", @@ -2152,15 +2152,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "diem-nibble" -version = "0.1.0" -dependencies = [ - "diem-workspace-hack", - "proptest", - "serde", -] - [[package]] name = "diem-node" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index b7e24b2996546..567d7842f2b96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ members = [ "common/logger/derive", "common/metrics", "common/metrics-core", - "common/nibble", "common/num-variants", "common/proptest-helpers", "common/proxy", diff --git a/common/nibble/Cargo.toml b/common/nibble/Cargo.toml deleted file mode 100644 index ef01cf54c8a01..0000000000000 --- a/common/nibble/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "diem-nibble" -version = "0.1.0" -authors = ["Diem Association "] -description = "Diem diem-nibble" -repository = "https://github.com/diem/diem" -homepage = "https://diem.com" -license = "Apache-2.0" -publish = false -edition = "2018" - -[dependencies] -diem-workspace-hack = { path = "../workspace-hack" } -proptest = { version = "1.0.0", optional = true } -serde = { version = "1.0.124", features = ["derive"] } - -[features] -default = [] -fuzzing = ["proptest"] diff --git a/crypto/crypto/src/hash.rs b/crypto/crypto/src/hash.rs index 973a33c5b440c..2f844cb8de703 100644 --- a/crypto/crypto/src/hash.rs +++ b/crypto/crypto/src/hash.rs @@ -212,6 +212,14 @@ impl HashValue { (self.hash[pos] >> bit) & 1 != 0 } + /// Returns the `index`-th nibble in the bytes. + pub fn nibble(&self, index: usize) -> u8 { + assume!(index < Self::LENGTH * 2); // assumed precondition + let pos = index / 2; + let shift = if index % 2 == 0 { 4 } else { 0 }; + (self.hash[pos] >> shift) & 0x0f + } + /// Returns a `HashValueBitIterator` over all the bits that represent this `HashValue`. pub fn iter_bits(&self) -> HashValueBitIterator<'_> { HashValueBitIterator::new(self) diff --git a/crypto/crypto/src/unit_tests/hash_test.rs b/crypto/crypto/src/unit_tests/hash_test.rs index b878fbf49d10c..cc57861ed4890 100644 --- a/crypto/crypto/src/unit_tests/hash_test.rs +++ b/crypto/crypto/src/unit_tests/hash_test.rs @@ -138,6 +138,21 @@ fn test_fmt_binary() { } } +#[test] +fn test_get_nibble() { + let mut bytes = [0u8; HashValue::LENGTH]; + let mut nibbles = vec![]; + for byte in bytes.iter_mut().take(HashValue::LENGTH) { + *byte = rand::thread_rng().gen(); + nibbles.push(*byte >> 4); + nibbles.push(*byte & 0x0f); + } + let hash = HashValue::new(bytes); + for (i, nibble) in nibbles.iter().enumerate().take(HashValue::LENGTH * 2) { + assert_eq!(hash.nibble(i), *nibble); + } +} + #[test] fn test_common_prefix_bits_len() { { diff --git a/execution/executor/src/lib.rs b/execution/executor/src/lib.rs index 196b1c64e138e..d117fc3dd4473 100644 --- a/execution/executor/src/lib.rs +++ b/execution/executor/src/lib.rs @@ -292,7 +292,7 @@ where }) .collect::>>()?; - let (txn_state_roots, current_state_tree) = parent_trees + let (roots_with_node_hashes, current_state_tree) = parent_trees .state_tree() .serial_update( txn_blobs @@ -307,9 +307,9 @@ where ) .expect("Failed to update state tree."); - for ((vm_output, txn), (state_tree_hash, blobs)) in itertools::zip_eq( + for ((vm_output, txn), ((state_tree_hash, new_node_hashes), blobs)) in itertools::zip_eq( itertools::zip_eq(vm_outputs.into_iter(), transactions.iter()).take(transaction_count), - itertools::zip_eq(txn_state_roots, txn_blobs), + itertools::zip_eq(roots_with_node_hashes, txn_blobs), ) { let event_tree = { let event_hashes: Vec<_> = @@ -353,6 +353,7 @@ where txn_data.push(TransactionData::new( blobs, + new_node_hashes, vm_output.events().to_vec(), vm_output.status().clone(), state_tree_hash, @@ -368,6 +369,7 @@ where txn_data.resize( transactions.len(), TransactionData::new( + HashMap::new(), HashMap::new(), vec![], TransactionStatus::Retry, @@ -558,6 +560,7 @@ where txns_to_commit.push(TransactionToCommit::new( txn, txn_data.account_blobs().clone(), + Some(txn_data.jf_node_hashes().clone()), txn_data.events().to_vec(), txn_data.gas_used(), recorded_status, @@ -891,6 +894,7 @@ impl BlockExecutor for Executor { txns_to_keep.push(TransactionToCommit::new( txn.clone(), txn_data.account_blobs().clone(), + Some(txn_data.jf_node_hashes().clone()), txn_data.events().to_vec(), txn_data.gas_used(), recorded_status.clone(), @@ -945,7 +949,6 @@ impl BlockExecutor for Executor { // Skip duplicate txns that are already persistent. let txns_to_commit = &txns_to_keep[num_txns_to_skip as usize..]; - let num_txns_to_commit = txns_to_commit.len() as u64; { let _timer = DIEM_EXECUTOR_SAVE_TRANSACTIONS_SECONDS.start_timer(); @@ -957,6 +960,7 @@ impl BlockExecutor for Executor { "Injected error in commit_blocks" ))) }); + self.db.writer.save_transactions( txns_to_commit, first_version_to_commit, diff --git a/execution/executor/src/types.rs b/execution/executor/src/types.rs index f73f070382e9b..364907e27a865 100644 --- a/execution/executor/src/types.rs +++ b/execution/executor/src/types.rs @@ -9,6 +9,7 @@ use diem_types::{ account_state_blob::AccountStateBlob, contract_event::ContractEvent, epoch_state::EpochState, + nibble::nibble_path::NibblePath, on_chain_config, proof::accumulator::InMemoryAccumulator, transaction::{TransactionStatus, Version}, @@ -26,6 +27,10 @@ pub struct TransactionData { /// new blob. account_blobs: HashMap, + /// Each entry in this map represents the the hash of a newly generated jellyfish node + /// and its corresponding nibble path. + jf_node_hashes: HashMap, + /// The list of events emitted during this transaction. events: Vec, @@ -48,6 +53,7 @@ pub struct TransactionData { impl TransactionData { pub fn new( account_blobs: HashMap, + jf_node_hashes: HashMap, events: Vec, status: TransactionStatus, state_root_hash: HashValue, @@ -57,6 +63,7 @@ impl TransactionData { ) -> Self { TransactionData { account_blobs, + jf_node_hashes, events, status, state_root_hash, @@ -70,6 +77,10 @@ impl TransactionData { &self.account_blobs } + pub fn jf_node_hashes(&self) -> &HashMap { + &self.jf_node_hashes + } + pub fn events(&self) -> &[ContractEvent] { &self.events } diff --git a/storage/diemdb-benchmark/src/lib.rs b/storage/diemdb-benchmark/src/lib.rs index 5ee1295e62a56..c2d8188af3bcc 100644 --- a/storage/diemdb-benchmark/src/lib.rs +++ b/storage/diemdb-benchmark/src/lib.rs @@ -66,6 +66,7 @@ fn gen_txn_to_commit( TransactionToCommit::new( txn, states, + None, vec![], /* events */ 0, /* gas_used */ KeptVMStatus::Executed, diff --git a/storage/diemdb/src/lib.rs b/storage/diemdb/src/lib.rs index 0fe4271aa6d99..eb786c6a8b32b 100644 --- a/storage/diemdb/src/lib.rs +++ b/storage/diemdb/src/lib.rs @@ -539,9 +539,17 @@ impl DiemDB { .iter() .map(|txn_to_commit| txn_to_commit.account_states().clone()) .collect::>(); - let state_root_hashes = - self.state_store - .put_account_state_sets(account_state_sets, first_version, &mut cs)?; + + let node_hashes = txns_to_commit + .iter() + .map(|txn_to_commit| txn_to_commit.jf_node_hashes()) + .collect::>>(); + let state_root_hashes = self.state_store.put_account_state_sets( + account_state_sets, + node_hashes, + first_version, + &mut cs, + )?; // Event updates. Gather event accumulator root hashes. let event_root_hashes = zip_eq(first_version..=last_version, txns_to_commit) diff --git a/storage/diemdb/src/pruner/test.rs b/storage/diemdb/src/pruner/test.rs index d237c87fb2a94..65af1e259c1ef 100644 --- a/storage/diemdb/src/pruner/test.rs +++ b/storage/diemdb/src/pruner/test.rs @@ -18,6 +18,7 @@ fn put_account_state_set( let root = state_store .put_account_state_sets( vec![account_state_set.into_iter().collect::>()], + None, version, &mut cs, ) diff --git a/storage/diemdb/src/state_store/mod.rs b/storage/diemdb/src/state_store/mod.rs index e98d252d926c9..dabd8ed0823d8 100644 --- a/storage/diemdb/src/state_store/mod.rs +++ b/storage/diemdb/src/state_store/mod.rs @@ -15,12 +15,11 @@ use crate::{ }; use anyhow::Result; use diem_crypto::HashValue; -use diem_jellyfish_merkle::{ - node_type::NodeKey, JellyfishMerkleTree, TreeReader, TreeWriter, ROOT_NIBBLE_HEIGHT, -}; +use diem_jellyfish_merkle::{node_type::NodeKey, JellyfishMerkleTree, TreeReader, TreeWriter}; use diem_types::{ account_address::{AccountAddress, HashAccountAddress}, account_state_blob::AccountStateBlob, + nibble::{nibble_path::NibblePath, ROOT_NIBBLE_HEIGHT}, proof::{SparseMerkleProof, SparseMerkleRangeProof}, transaction::Version, }; @@ -67,6 +66,7 @@ impl StateStore { pub fn put_account_state_sets( &self, account_state_sets: Vec>, + node_hashes: Option>>, first_version: Version, cs: &mut ChangeSet, ) -> Result> { @@ -80,8 +80,8 @@ impl StateStore { }) .collect::>(); - let (new_root_hash_vec, tree_update_batch) = - JellyfishMerkleTree::new(self).put_value_sets(blob_sets, first_version)?; + let (new_root_hash_vec, tree_update_batch) = JellyfishMerkleTree::new(self) + .batch_put_value_sets(blob_sets, node_hashes, first_version)?; let num_versions = new_root_hash_vec.len(); assert_eq!(num_versions, tree_update_batch.node_stats.len()); diff --git a/storage/diemdb/src/state_store/state_store_test.rs b/storage/diemdb/src/state_store/state_store_test.rs index 30d469b54ac15..29b261a55baa9 100644 --- a/storage/diemdb/src/state_store/state_store_test.rs +++ b/storage/diemdb/src/state_store/state_store_test.rs @@ -24,6 +24,7 @@ fn put_account_state_set( let root = store .put_account_state_sets( vec![account_state_set.into_iter().collect::>()], + None, version, &mut cs, ) @@ -368,7 +369,7 @@ fn init_store(store: &StateStore, input: impl Iterator = std::iter::once((key, value)).collect(); store - .put_account_state_sets(vec![account_state_set], i as Version, &mut cs) + .put_account_state_sets(vec![account_state_set], None, i as Version, &mut cs) .unwrap(); store.db.write_schemas(cs.batch).unwrap(); } diff --git a/storage/diemdb/src/test_helper.rs b/storage/diemdb/src/test_helper.rs index e816a0c1df191..eb93b4f4d5894 100644 --- a/storage/diemdb/src/test_helper.rs +++ b/storage/diemdb/src/test_helper.rs @@ -33,6 +33,7 @@ fn to_blocks_to_commit( let txn_hash = txn_to_commit.transaction().hash(); let state_root_hash = db.state_store.put_account_state_sets( vec![txn_to_commit.account_states().clone()], + None, cur_ver, &mut cs, )?[0]; diff --git a/storage/jellyfish-merkle/Cargo.toml b/storage/jellyfish-merkle/Cargo.toml index 419036b4e8ebe..6198319c3a5ec 100644 --- a/storage/jellyfish-merkle/Cargo.toml +++ b/storage/jellyfish-merkle/Cargo.toml @@ -12,6 +12,7 @@ edition = "2018" [dependencies] anyhow = "1.0.38" byteorder = "1.4.3" +itertools = { version = "0.10.0", default-features = false } mirai-annotations = "1.10.1" num-derive = "0.3.3" num-traits = "0.2.14" @@ -27,7 +28,6 @@ diem-crypto = { path = "../../crypto/crypto" } diem-crypto-derive = { path = "../../crypto/crypto-derive" } diem-infallible = { path = "../../common/infallible" } diem-metrics = { path = "../../common/metrics" } -diem-nibble = { path = "../../common/nibble" } diem-types = { path = "../../types" } diem-workspace-hack = { path = "../../common/workspace-hack" } @@ -37,9 +37,8 @@ proptest = "1.0.0" proptest-derive = "0.3.0" diem-crypto = { path = "../../crypto/crypto", features = ["fuzzing"] } -diem-nibble = { path = "../../common/nibble", features = ["fuzzing"] } diem-types = { path = "../../types", features = ["fuzzing"] } [features] default = [] -fuzzing = ["proptest", "rand", "proptest-derive", "diem-crypto/fuzzing", "diem-types/fuzzing", "diem-nibble/fuzzing"] +fuzzing = ["proptest", "rand", "proptest-derive", "diem-crypto/fuzzing", "diem-types/fuzzing"] diff --git a/storage/jellyfish-merkle/src/iterator/mod.rs b/storage/jellyfish-merkle/src/iterator/mod.rs index 570b424314985..9a52b26519915 100644 --- a/storage/jellyfish-merkle/src/iterator/mod.rs +++ b/storage/jellyfish-merkle/src/iterator/mod.rs @@ -10,14 +10,15 @@ mod iterator_test; use crate::{ - nibble_path::NibblePath, node_type::{InternalNode, Node, NodeKey}, TreeReader, }; use anyhow::{format_err, Result}; use diem_crypto::HashValue; -use diem_nibble::Nibble; -use diem_types::transaction::Version; +use diem_types::{ + nibble::{nibble_path::NibblePath, Nibble}, + transaction::Version, +}; use std::{marker::PhantomData, sync::Arc}; /// `NodeVisitInfo` keeps track of the status of an internal node during the iteration process. It diff --git a/storage/jellyfish-merkle/src/jellyfish_merkle_test.rs b/storage/jellyfish-merkle/src/jellyfish_merkle_test.rs index 13932fa0f4c5c..d36d84aa3a734 100644 --- a/storage/jellyfish-merkle/src/jellyfish_merkle_test.rs +++ b/storage/jellyfish-merkle/src/jellyfish_merkle_test.rs @@ -8,8 +8,7 @@ use crate::test_helper::{ test_get_with_proof_with_distinct_last_nibble, ValueBlob, }; use diem_crypto::HashValue; -use diem_nibble::Nibble; -use diem_types::transaction::PRE_GENESIS_VERSION; +use diem_types::{nibble::Nibble, transaction::PRE_GENESIS_VERSION}; use mock_tree_store::MockTreeStore; use proptest::prelude::*; use rand::{rngs::StdRng, Rng, SeedableRng}; @@ -36,10 +35,12 @@ fn test_insert_to_empty_tree() { let key = HashValue::random(); let value = ValueBlob::from(vec![1u8, 2u8, 3u8, 4u8]); + // batch version let (_new_root_hash, batch) = tree - .put_value_set(vec![(key, value.clone())], 0 /* version */) + .batch_put_value_sets(vec![vec![(key, value.clone())]], None, 0 /* version */) .unwrap(); assert!(batch.stale_node_index_batch.is_empty()); + db.write_tree_update_batch(batch).unwrap(); assert_eq!(tree.get(key, 0).unwrap().unwrap(), value); @@ -59,8 +60,13 @@ fn test_insert_to_pre_genesis() { let tree = JellyfishMerkleTree::new(&db); let key2 = update_nibble(&key1, 0, 15); let value2 = ValueBlob::from(vec![3u8, 4u8]); + // batch version let (_root_hash, batch) = tree - .put_value_set(vec![(key2, value2.clone())], 0 /* version */) + .batch_put_value_sets( + vec![vec![(key2, value2.clone())]], + None, + 0, /* version */ + ) .unwrap(); // Check pre-genesis node prunes okay. @@ -84,7 +90,11 @@ fn test_insert_at_leaf_with_internal_created() { let value1 = ValueBlob::from(vec![1u8, 2u8]); let (_root0_hash, batch) = tree - .put_value_set(vec![(key1, value1.clone())], 0 /* version */) + .batch_put_value_sets( + vec![vec![(key1, value1.clone())]], + None, + 0, /* version */ + ) .unwrap(); assert!(batch.stale_node_index_batch.is_empty()); @@ -97,7 +107,11 @@ fn test_insert_at_leaf_with_internal_created() { let value2 = ValueBlob::from(vec![3u8, 4u8]); let (_root1_hash, batch) = tree - .put_value_set(vec![(key2, value2.clone())], 1 /* version */) + .batch_put_value_sets( + vec![vec![(key2, value2.clone())]], + None, + 1, /* version */ + ) .unwrap(); assert_eq!(batch.stale_node_index_batch.len(), 1); db.write_tree_update_batch(batch).unwrap(); @@ -147,7 +161,11 @@ fn test_insert_at_leaf_with_multiple_internals_created() { let value1 = ValueBlob::from(vec![1u8, 2u8]); let (_root0_hash, batch) = tree - .put_value_set(vec![(key1, value1.clone())], 0 /* version */) + .batch_put_value_sets( + vec![vec![(key1, value1.clone())]], + None, + 0, /* version */ + ) .unwrap(); db.write_tree_update_batch(batch).unwrap(); assert_eq!(tree.get(key1, 0).unwrap().unwrap(), value1); @@ -158,7 +176,11 @@ fn test_insert_at_leaf_with_multiple_internals_created() { let value2 = ValueBlob::from(vec![3u8, 4u8]); let (_root1_hash, batch) = tree - .put_value_set(vec![(key2, value2.clone())], 1 /* version */) + .batch_put_value_sets( + vec![vec![(key2, value2.clone())]], + None, + 1, /* version */ + ) .unwrap(); db.write_tree_update_batch(batch).unwrap(); assert_eq!(tree.get(key1, 0).unwrap().unwrap(), value1); @@ -217,7 +239,11 @@ fn test_insert_at_leaf_with_multiple_internals_created() { // 3. Update leaf2 with new value let value2_update = ValueBlob::from(vec![5u8, 6u8]); let (_root2_hash, batch) = tree - .put_value_set(vec![(key2, value2_update.clone())], 2 /* version */) + .batch_put_value_sets( + vec![vec![(key2, value2_update.clone())]], + None, + 2, /* version */ + ) .unwrap(); db.write_tree_update_batch(batch).unwrap(); assert!(tree.get(key2, 0).unwrap().is_none()); @@ -295,7 +321,7 @@ fn test_batch_insertion() { .for_each(|(k, v)| assert_eq!(tree.get(*k, version).unwrap().unwrap(), *v)) }; - // Insert as one batch. + // Insert as one batch and update one by one. { let db = MockTreeStore::default(); let tree = JellyfishMerkleTree::new(&db); @@ -313,7 +339,9 @@ fn test_batch_insertion() { let db = MockTreeStore::default(); let tree = JellyfishMerkleTree::new(&db); - let (_roots, batch) = tree.put_value_sets(batches, 0 /* first_version */).unwrap(); + let (_roots, batch) = tree + .batch_put_value_sets(batches, None, 0 /* first_version */) + .unwrap(); db.write_tree_update_batch(batch).unwrap(); verify_fn(&tree, 6); @@ -434,13 +462,14 @@ fn test_non_existence() { let key3 = update_nibble(&key1, 2, 3); let value3 = ValueBlob::from(vec![3u8]); - let (root, batch) = tree - .put_value_set( - vec![ + let (roots, batch) = tree + .batch_put_value_sets( + vec![vec![ (key1, value1.clone()), (key2, value2.clone()), (key3, value3.clone()), - ], + ]], + None, 0, /* version */ ) .unwrap(); @@ -457,21 +486,21 @@ fn test_non_existence() { let non_existing_key = update_nibble(&key1, 0, 1); let (value, proof) = tree.get_with_proof(non_existing_key, 0).unwrap(); assert_eq!(value, None); - assert!(proof.verify(root, non_existing_key, None).is_ok()); + assert!(proof.verify(roots[0], non_existing_key, None).is_ok()); } // 2. Non-existing node at non-root internal node { let non_existing_key = update_nibble(&key1, 1, 15); let (value, proof) = tree.get_with_proof(non_existing_key, 0).unwrap(); assert_eq!(value, None); - assert!(proof.verify(root, non_existing_key, None).is_ok()); + assert!(proof.verify(roots[0], non_existing_key, None).is_ok()); } // 3. Non-existing node at leaf node { let non_existing_key = update_nibble(&key1, 2, 4); let (value, proof) = tree.get_with_proof(non_existing_key, 0).unwrap(); assert_eq!(value, None); - assert!(proof.verify(root, non_existing_key, None).is_ok()); + assert!(proof.verify(roots[0], non_existing_key, None).is_ok()); } } @@ -533,7 +562,9 @@ fn test_put_value_sets() { } value_sets.push(keyed_value_set); } - let (root_hashes, batch) = tree.put_value_sets(value_sets, 0 /* version */).unwrap(); + let (root_hashes, batch) = tree + .batch_put_value_sets(value_sets, None, 0 /* version */) + .unwrap(); assert_eq!(root_hashes, root_hashes_one_by_one); assert_eq!(batch, batch_one_by_one); } @@ -555,13 +586,15 @@ fn many_keys_get_proof_and_verify_tree_root(seed: &[u8], num_keys: usize) { kvs.push((key, value)); } - let (root, batch) = tree.put_value_set(kvs.clone(), 0 /* version */).unwrap(); + let (roots, batch) = tree + .batch_put_value_sets(vec![kvs.clone()], None, 0 /* version */) + .unwrap(); db.write_tree_update_batch(batch).unwrap(); for (k, v) in &kvs { let (value, proof) = tree.get_with_proof(*k, 0).unwrap(); assert_eq!(value.unwrap(), *v); - assert!(proof.verify(root, *k, Some(v)).is_ok()); + assert!(proof.verify(roots[0], *k, Some(v)).is_ok()); } } @@ -592,9 +625,9 @@ fn many_versions_get_proof_and_verify_tree_root(seed: &[u8], num_versions: usize for (idx, kvs) in kvs.iter().enumerate() { let (root, batch) = tree - .put_value_set(vec![(kvs.0, kvs.1.clone())], idx as Version) + .batch_put_value_sets(vec![vec![(kvs.0, kvs.1.clone())]], None, idx as Version) .unwrap(); - roots.push(root); + roots.push(root[0]); db.write_tree_update_batch(batch).unwrap(); } @@ -602,9 +635,9 @@ fn many_versions_get_proof_and_verify_tree_root(seed: &[u8], num_versions: usize for (idx, kvs) in kvs.iter().enumerate() { let version = (num_versions + idx) as Version; let (root, batch) = tree - .put_value_set(vec![(kvs.0, kvs.2.clone())], version) + .batch_put_value_sets(vec![vec![(kvs.0, kvs.2.clone())]], None, version) .unwrap(); - roots.push(root); + roots.push(root[0]); db.write_tree_update_batch(batch).unwrap(); } diff --git a/storage/jellyfish-merkle/src/lib.rs b/storage/jellyfish-merkle/src/lib.rs index a49185f03daf9..8b42fcfe9f690 100644 --- a/storage/jellyfish-merkle/src/lib.rs +++ b/storage/jellyfish-merkle/src/lib.rs @@ -74,7 +74,6 @@ mod jellyfish_merkle_test; pub mod metrics; #[cfg(any(test, feature = "fuzzing"))] mod mock_tree_store; -mod nibble_path; pub mod node_type; pub mod restore; #[cfg(any(test, feature = "fuzzing"))] @@ -83,12 +82,14 @@ mod tree_cache; use anyhow::{bail, ensure, format_err, Result}; use diem_crypto::{hash::CryptoHash, HashValue}; -use diem_nibble::Nibble; use diem_types::{ + nibble::{ + nibble_path::{skip_common_prefix, NibbleIterator, NibblePath}, + Nibble, ROOT_NIBBLE_HEIGHT, + }, proof::{SparseMerkleProof, SparseMerkleRangeProof}, transaction::Version, }; -use nibble_path::{skip_common_prefix, NibbleIterator, NibblePath}; use node_type::{Child, Children, InternalNode, LeafNode, Node, NodeKey}; #[cfg(any(test, feature = "fuzzing"))] use proptest::arbitrary::Arbitrary; @@ -96,7 +97,7 @@ use proptest::arbitrary::Arbitrary; use proptest_derive::Arbitrary; use serde::{de::DeserializeOwned, Serialize}; use std::{ - collections::{BTreeMap, BTreeSet}, + collections::{BTreeMap, BTreeSet, HashMap}, marker::PhantomData, }; use thiserror::Error; @@ -108,9 +109,6 @@ pub struct MissingRootError { pub version: Version, } -/// The hardcoded maximum height of a [`JellyfishMerkleTree`] in nibbles. -pub const ROOT_NIBBLE_HEIGHT: usize = HashValue::LENGTH * 2; - /// `TreeReader` defines the interface between /// [`JellyfishMerkleTree`](struct.JellyfishMerkleTree.html) /// and underlying storage holding nodes. @@ -185,6 +183,50 @@ pub struct TreeUpdateBatch { pub node_stats: Vec, } +/// An iterator that iterates the index range (inclusive) of each different nibble at given +/// `nibble_idx` of all the keys in a sorted key-value pairs. +struct NibbleRangeIterator<'a, V> { + sorted_kvs: &'a [(HashValue, V)], + nibble_idx: usize, + pos: usize, +} + +impl<'a, V> NibbleRangeIterator<'a, V> { + fn new(sorted_kvs: &'a [(HashValue, V)], nibble_idx: usize) -> Self { + assert!(nibble_idx < ROOT_NIBBLE_HEIGHT); + NibbleRangeIterator { + sorted_kvs, + nibble_idx, + pos: 0, + } + } +} + +impl<'a, V> std::iter::Iterator for NibbleRangeIterator<'a, V> { + type Item = (usize, usize); + + fn next(&mut self) -> Option { + let left = self.pos; + if self.pos < self.sorted_kvs.len() { + let cur_nibble: u8 = self.sorted_kvs[left].0.nibble(self.nibble_idx); + let (mut i, mut j) = (left, self.sorted_kvs.len() - 1); + // Find the last index of the cur_nibble. + while i < j { + let mid = j - (j - i) / 2; + if self.sorted_kvs[mid].0.nibble(self.nibble_idx) > cur_nibble { + j = mid - 1; + } else { + i = mid; + } + } + self.pos = i + 1; + Some((left, i)) + } else { + None + } + } +} + /// The Jellyfish Merkle tree data structure. See [`crate`] for description. pub struct JellyfishMerkleTree<'a, R, V> { reader: &'a R, @@ -204,6 +246,281 @@ where } } + /// Get the node hash from the cache if exists, otherwise compute it. + fn get_hash( + node_key: &NodeKey, + node: &Node, + hash_cache: &Option<&HashMap>, + ) -> HashValue { + if let Some(cache) = hash_cache { + match cache.get(node_key.nibble_path()) { + Some(hash) => *hash, + None => unreachable!("{:?} can not be found in hash cache", node_key), + } + } else { + node.hash() + } + } + + /// The batch version of `put_value_sets`. + pub fn batch_put_value_sets( + &self, + value_sets: Vec>, + node_hashes: Option>>, + first_version: Version, + ) -> Result<(Vec, TreeUpdateBatch)> { + let mut tree_cache = TreeCache::new(self.reader, first_version)?; + let hash_sets: Vec<_> = match node_hashes { + Some(hashes) => hashes.into_iter().map(Some).collect(), + None => (0..value_sets.len()).map(|_| None).collect(), + }; + + for (idx, (value_set, hash_set)) in + itertools::zip_eq(value_sets.into_iter(), hash_sets.into_iter()).enumerate() + { + assert!( + !value_set.is_empty(), + "Transactions that output empty write set should not be included.", + ); + let version = first_version + idx as u64; + let deduped_and_sorted_kvs = value_set + .into_iter() + .collect::>() + .into_iter() + .collect::>(); + let root_node_key = tree_cache.get_root_node_key().clone(); + let (new_root_node_key, _) = Self::batch_insert_at( + root_node_key, + version, + deduped_and_sorted_kvs.as_slice(), + 0, + &hash_set, + &mut tree_cache, + )?; + tree_cache.set_root_node_key(new_root_node_key); + + // Freezes the current cache to make all contents in the current cache immutable. + tree_cache.freeze(); + } + + Ok(tree_cache.into()) + } + + fn batch_insert_at( + mut node_key: NodeKey, + version: Version, + kvs: &[(HashValue, V)], + depth: usize, + hash_cache: &Option<&HashMap>, + tree_cache: &mut TreeCache, + ) -> Result<(NodeKey, Node)> { + assert!(!kvs.is_empty()); + + let node = tree_cache.get_node(&node_key)?; + Ok(match node { + Node::Internal(internal_node) => { + // We always delete the existing internal node here because it will not be referenced anyway + // since this version. + tree_cache.delete_node(&node_key, false /* is_leaf */); + + // Reuse the current `InternalNode` in memory to create a new internal node. + let mut children: Children = internal_node.clone().into(); + + // Traverse all the path touched by `kvs` from this internal node. + for (left, right) in NibbleRangeIterator::new(kvs, depth) { + // Traverse downwards from this internal node recursively by splitting the updates into + // each child index + let child_index = kvs[left].0.get_nibble(depth); + + let (new_child_node_key, new_child_node) = + match internal_node.child(child_index) { + Some(child) => { + let child_node_key = + node_key.gen_child_node_key(child.version, child_index); + Self::batch_insert_at( + child_node_key, + version, + &kvs[left..=right], + depth + 1, + hash_cache, + tree_cache, + )? + } + None => { + let new_child_node_key = + node_key.gen_child_node_key(version, child_index); + Self::batch_create_subtree( + new_child_node_key, + version, + &kvs[left..=right], + depth + 1, + hash_cache, + tree_cache, + )? + } + }; + + children.insert( + child_index, + Child::new( + Self::get_hash(&new_child_node_key, &new_child_node, hash_cache), + version, + new_child_node.is_leaf(), + ), + ); + } + let new_internal_node = InternalNode::new(children); + + node_key.set_version(version); + + // Cache this new internal node. + tree_cache.put_node(node_key.clone(), new_internal_node.clone().into())?; + (node_key, new_internal_node.into()) + } + Node::Leaf(leaf_node) => { + // We are on a leaf node but trying to insert another node, so we may diverge. + // We always delete the existing leaf node here because it will not be referenced anyway + // since this version. + tree_cache.delete_node(&node_key, true /* is_leaf */); + node_key.set_version(version); + Self::batch_create_subtree_with_existing_leaf( + node_key, version, leaf_node, kvs, depth, hash_cache, tree_cache, + )? + } + Node::Null => { + if !node_key.nibble_path().is_empty() { + bail!( + "Null node exists for non-root node with node_key {:?}", + node_key + ); + } + + if node_key.version() == version { + tree_cache.delete_node(&node_key, false /* is_leaf */); + } + Self::batch_create_subtree( + NodeKey::new_empty_path(version), + version, + kvs, + depth, + hash_cache, + tree_cache, + )? + } + }) + } + + fn batch_create_subtree_with_existing_leaf( + node_key: NodeKey, + version: Version, + existing_leaf_node: LeafNode, + kvs: &[(HashValue, V)], + depth: usize, + hash_cache: &Option<&HashMap>, + tree_cache: &mut TreeCache, + ) -> Result<(NodeKey, Node)> { + let existing_leaf_key = existing_leaf_node.account_key(); + + if kvs.len() == 1 && kvs[0].0 == existing_leaf_key { + let new_leaf_node = Node::new_leaf(existing_leaf_key, kvs[0].1.clone()); + tree_cache.put_node(node_key.clone(), new_leaf_node.clone())?; + Ok((node_key, new_leaf_node)) + } else { + let existing_leaf_bucket = existing_leaf_key.get_nibble(depth); + let mut isolated_existing_leaf = true; + let mut children = Children::new(); + for (left, right) in NibbleRangeIterator::new(kvs, depth) { + let child_index = kvs[left].0.get_nibble(depth); + let child_node_key = node_key.gen_child_node_key(version, child_index); + let (new_child_node_key, new_child_node) = if existing_leaf_bucket == child_index { + isolated_existing_leaf = false; + Self::batch_create_subtree_with_existing_leaf( + child_node_key, + version, + existing_leaf_node.clone(), + &kvs[left..=right], + depth + 1, + hash_cache, + tree_cache, + )? + } else { + Self::batch_create_subtree( + child_node_key, + version, + &kvs[left..=right], + depth + 1, + hash_cache, + tree_cache, + )? + }; + children.insert( + child_index, + Child::new( + Self::get_hash(&new_child_node_key, &new_child_node, hash_cache), + version, + new_child_node.is_leaf(), + ), + ); + } + if isolated_existing_leaf { + let existing_leaf_node_key = + node_key.gen_child_node_key(version, existing_leaf_bucket); + children.insert( + existing_leaf_bucket, + Child::new(existing_leaf_node.hash(), version, true /* is_leaf */), + ); + + tree_cache.put_node(existing_leaf_node_key, existing_leaf_node.into())?; + } + + let new_internal_node = InternalNode::new(children); + + tree_cache.put_node(node_key.clone(), new_internal_node.clone().into())?; + Ok((node_key, new_internal_node.into())) + } + } + + fn batch_create_subtree( + node_key: NodeKey, + version: Version, + kvs: &[(HashValue, V)], + depth: usize, + hash_cache: &Option<&HashMap>, + tree_cache: &mut TreeCache, + ) -> Result<(NodeKey, Node)> { + if kvs.len() == 1 { + let new_leaf_node = Node::new_leaf(kvs[0].0, kvs[0].1.clone()); + tree_cache.put_node(node_key.clone(), new_leaf_node.clone())?; + Ok((node_key, new_leaf_node)) + } else { + let mut children = Children::new(); + for (left, right) in NibbleRangeIterator::new(kvs, depth) { + let child_index = kvs[left].0.get_nibble(depth); + let child_node_key = node_key.gen_child_node_key(version, child_index); + let (new_child_node_key, new_child_node) = Self::batch_create_subtree( + child_node_key, + version, + &kvs[left..=right], + depth + 1, + hash_cache, + tree_cache, + )?; + children.insert( + child_index, + Child::new( + Self::get_hash(&new_child_node_key, &new_child_node, hash_cache), + version, + new_child_node.is_leaf(), + ), + ); + } + let new_internal_node = InternalNode::new(children); + + tree_cache.put_node(node_key.clone(), new_internal_node.clone().into())?; + Ok((node_key, new_internal_node.into())) + } + } + /// This is a convenient function that calls /// [`put_value_sets`](struct.JellyfishMerkleTree.html#method.put_value_sets) with a single /// `keyed_value_set`. @@ -213,7 +530,8 @@ where value_set: Vec<(HashValue, V)>, version: Version, ) -> Result<(HashValue, TreeUpdateBatch)> { - let (root_hashes, tree_update_batch) = self.put_value_sets(vec![value_set], version)?; + let (root_hashes, tree_update_batch) = + self.batch_put_value_sets(vec![value_set], None, version)?; assert_eq!( root_hashes.len(), 1, @@ -492,7 +810,7 @@ where ); let internal_node = InternalNode::new(children); - let mut next_internal_node = internal_node.clone(); + let mut next_internal_node: Node = internal_node.clone().into(); tree_cache.put_node(node_key.clone(), internal_node.into())?; for _i in 0..num_common_nibbles_below_internal { @@ -506,11 +824,11 @@ where Child::new(next_internal_node.hash(), version, false /* is_leaf */), ); let internal_node = InternalNode::new(children); - next_internal_node = internal_node.clone(); + next_internal_node = internal_node.clone().into(); tree_cache.put_node(node_key.clone(), internal_node.into())?; } - Ok((node_key, next_internal_node.into())) + Ok((node_key, next_internal_node)) } /// Helper function for creating leaf nodes. Returns the newly created leaf node. @@ -675,7 +993,7 @@ impl NibbleExt for HashValue { mod test { use super::NibbleExt; use diem_crypto::hash::{HashValue, TestOnlyHash}; - use diem_nibble::Nibble; + use diem_types::nibble::Nibble; #[test] fn test_common_prefix_nibbles_len() { diff --git a/storage/jellyfish-merkle/src/node_type/mod.rs b/storage/jellyfish-merkle/src/node_type/mod.rs index 29989249a4841..fe1e5e2d9e2a1 100644 --- a/storage/jellyfish-merkle/src/node_type/mod.rs +++ b/storage/jellyfish-merkle/src/node_type/mod.rs @@ -12,19 +12,15 @@ #[cfg(test)] mod node_type_test; -use crate::{ - metrics::{DIEM_JELLYFISH_INTERNAL_ENCODED_BYTES, DIEM_JELLYFISH_LEAF_ENCODED_BYTES}, - nibble_path::NibblePath, - ROOT_NIBBLE_HEIGHT, -}; +use crate::metrics::{DIEM_JELLYFISH_INTERNAL_ENCODED_BYTES, DIEM_JELLYFISH_LEAF_ENCODED_BYTES}; use anyhow::{ensure, Context, Result}; use byteorder::{BigEndian, LittleEndian, ReadBytesExt, WriteBytesExt}; use diem_crypto::{ hash::{CryptoHash, SPARSE_MERKLE_PLACEHOLDER_HASH}, HashValue, }; -use diem_nibble::Nibble; use diem_types::{ + nibble::{nibble_path::NibblePath, Nibble, ROOT_NIBBLE_HEIGHT}, proof::{SparseMerkleInternalNode, SparseMerkleLeafNode}, transaction::Version, }; diff --git a/storage/jellyfish-merkle/src/node_type/node_type_test.rs b/storage/jellyfish-merkle/src/node_type/node_type_test.rs index 6c99406f5da11..9294eea380da0 100644 --- a/storage/jellyfish-merkle/src/node_type/node_type_test.rs +++ b/storage/jellyfish-merkle/src/node_type/node_type_test.rs @@ -5,13 +5,13 @@ use super::{ deserialize_u64_varint, serialize_u64_varint, Child, Children, InternalNode, NodeDecodeError, NodeKey, }; -use crate::{nibble_path::NibblePath, test_helper::ValueBlob}; +use crate::test_helper::ValueBlob; use diem_crypto::{ hash::{CryptoHash, SPARSE_MERKLE_PLACEHOLDER_HASH}, HashValue, }; -use diem_nibble::Nibble; use diem_types::{ + nibble::{nibble_path::NibblePath, Nibble}, proof::{SparseMerkleInternalNode, SparseMerkleLeafNode}, transaction::Version, }; diff --git a/storage/jellyfish-merkle/src/restore/mod.rs b/storage/jellyfish-merkle/src/restore/mod.rs index f8317011cb991..bd210305ae8e9 100644 --- a/storage/jellyfish-merkle/src/restore/mod.rs +++ b/storage/jellyfish-merkle/src/restore/mod.rs @@ -8,7 +8,6 @@ mod restore_test; use crate::{ - nibble_path::{NibbleIterator, NibblePath}, node_type::{ get_child_and_sibling_half_start, Child, Children, InternalNode, LeafNode, Node, NodeKey, }, @@ -19,8 +18,11 @@ use diem_crypto::{ hash::{CryptoHash, SPARSE_MERKLE_PLACEHOLDER_HASH}, HashValue, }; -use diem_nibble::Nibble; use diem_types::{ + nibble::{ + nibble_path::{NibbleIterator, NibblePath}, + Nibble, + }, proof::{SparseMerkleInternalNode, SparseMerkleRangeProof}, transaction::Version, }; diff --git a/storage/jellyfish-merkle/src/tree_cache/tree_cache_test.rs b/storage/jellyfish-merkle/src/tree_cache/tree_cache_test.rs index 7e6668433985c..f2816feac2848 100644 --- a/storage/jellyfish-merkle/src/tree_cache/tree_cache_test.rs +++ b/storage/jellyfish-merkle/src/tree_cache/tree_cache_test.rs @@ -2,11 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use super::*; -use crate::{ - mock_tree_store::MockTreeStore, nibble_path::NibblePath, node_type::Node, - test_helper::ValueBlob, NodeKey, -}; +use crate::{mock_tree_store::MockTreeStore, node_type::Node, test_helper::ValueBlob, NodeKey}; use diem_crypto::HashValue; +use diem_types::nibble::nibble_path::NibblePath; fn random_leaf_with_key(next_version: Version) -> (Node, NodeKey) { let address = HashValue::random(); diff --git a/storage/scratchpad/src/sparse_merkle/mod.rs b/storage/scratchpad/src/sparse_merkle/mod.rs index e19e284ccc4b9..df24c2da73874 100644 --- a/storage/scratchpad/src/sparse_merkle/mod.rs +++ b/storage/scratchpad/src/sparse_merkle/mod.rs @@ -88,8 +88,16 @@ use diem_crypto::{ hash::{CryptoHash, SPARSE_MERKLE_PLACEHOLDER_HASH}, HashValue, }; -use diem_types::proof::{SparseMerkleInternalNode, SparseMerkleLeafNode, SparseMerkleProof}; -use std::{borrow::Borrow, cmp, collections::BTreeMap, sync::Arc}; +use diem_types::{ + nibble::{nibble_path::NibblePath, ROOT_NIBBLE_HEIGHT}, + proof::{SparseMerkleInternalNode, SparseMerkleLeafNode, SparseMerkleProof}, +}; +use std::{ + borrow::Borrow, + cmp, + collections::{BTreeMap, BTreeSet, HashMap}, + sync::Arc, +}; /// `AccountStatus` describes the result of querying an account from this SparseMerkleTree. #[derive(Debug, Eq, PartialEq)] @@ -218,16 +226,111 @@ where &self, update_batch: Vec>, proof_reader: &impl ProofRead, - ) -> Result<(Vec, Self), UpdateError> { + ) -> Result<(Vec<(HashValue, HashMap)>, Self), UpdateError> { let mut current_state_tree = self.clone(); - let mut result_hashes = Vec::with_capacity(update_batch.len()); + let mut result = Vec::with_capacity(update_batch.len()); for updates in update_batch { + // sort and dedup the accounts + let accounts = updates + .iter() + .map(|(account, _)| *account) + .collect::>() + .into_iter() + .collect::>(); current_state_tree = current_state_tree.batch_update(updates, proof_reader)?; - result_hashes.push(current_state_tree.root_hash()); + result.push(( + current_state_tree.root_hash(), + current_state_tree.generate_node_hashes(accounts), + )); } - Ok((result_hashes, current_state_tree)) + Ok((result, current_state_tree)) } + /// This is a helper function that compares an updated in-memory sparse merkle with the + /// current on-disk jellyfish sparse merkle to get the hashes of newly generated nodes. + pub fn generate_node_hashes( + &self, + // must be sorted + touched_accounts: Vec, + ) -> HashMap { + let mut node_hashes = HashMap::new(); + let mut nibble_path = NibblePath::new(vec![]); + Self::collect_new_hashes( + touched_accounts.as_slice(), + self.root_weak(), + 0, /* depth in nibble */ + 0, /* level within a nibble*/ + &mut nibble_path, + &mut node_hashes, + ); + node_hashes + } + + /// Recursively generate the partial node update batch of jellyfish merkle + fn collect_new_hashes( + keys: &[HashValue], + subtree: SubTree, + depth_in_nibble: usize, + level_within_nibble: usize, + cur_nibble_path: &mut NibblePath, + node_hashes: &mut HashMap, + ) { + assert!(depth_in_nibble <= ROOT_NIBBLE_HEIGHT); + if keys.is_empty() { + return; + } + + if level_within_nibble == 0 { + if depth_in_nibble != 0 { + cur_nibble_path + .push(NibblePath::new(keys[0].to_vec()).get_nibble(depth_in_nibble - 1)); + } + node_hashes.insert(cur_nibble_path.clone(), subtree.hash()); + } + match subtree.get_node_if_in_mem().expect("must exist").borrow() { + Node::Internal(internal_node) => { + let (next_nibble_depth, next_level_within_nibble) = if level_within_nibble == 3 { + (depth_in_nibble + 1, 0) + } else { + (depth_in_nibble, level_within_nibble + 1) + }; + let pivot = partition( + &keys.iter().map(|k| (*k, ())).collect::>()[..], + depth_in_nibble * 4 + level_within_nibble, + ); + Self::collect_new_hashes( + &keys[..pivot], + internal_node.left.weak(), + next_nibble_depth, + next_level_within_nibble, + cur_nibble_path, + node_hashes, + ); + Self::collect_new_hashes( + &keys[pivot..], + internal_node.right.weak(), + next_nibble_depth, + next_level_within_nibble, + cur_nibble_path, + node_hashes, + ); + } + Node::Leaf(leaf_node) => { + assert_eq!(keys.len(), 1); + assert_eq!(keys[0], leaf_node.key); + matches!(leaf_node.value, LeafValue::Value(_)); + if level_within_nibble != 0 { + let mut leaf_nibble_path = cur_nibble_path.clone(); + leaf_nibble_path + .push(NibblePath::new(keys[0].to_vec()).get_nibble(depth_in_nibble)); + node_hashes.insert(leaf_nibble_path, subtree.hash()); + } + } + } + if level_within_nibble == 0 && depth_in_nibble != 0 { + cur_nibble_path.pop(); + } + } /// Constructs a new Sparse Merkle Tree, returns the SMT root hash after each update and the /// final SMT root. Since the tree is immutable, existing tree remains the same and may /// share parts with the new, returned tree. Unlike `serial_update', intermediate trees aren't diff --git a/types/src/lib.rs b/types/src/lib.rs index 0f27bdec1ca42..34372fdfdc808 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -21,6 +21,7 @@ pub mod ledger_info; pub mod mempool_status; pub mod move_resource; pub mod network_address; +pub mod nibble; pub mod on_chain_config; pub mod proof; #[cfg(any(test, feature = "fuzzing"))] diff --git a/common/nibble/src/lib.rs b/types/src/nibble/mod.rs similarity index 84% rename from common/nibble/src/lib.rs rename to types/src/nibble/mod.rs index 9e98708564001..17f70d3a16684 100644 --- a/common/nibble/src/lib.rs +++ b/types/src/nibble/mod.rs @@ -5,11 +5,17 @@ //! `Nibble` represents a four-bit unsigned integer. +pub mod nibble_path; + +use diem_crypto::HashValue; #[cfg(feature = "fuzzing")] use proptest::prelude::*; use serde::{Deserialize, Serialize}; use std::fmt; +/// The hardcoded maximum height of a state merkle tree in nibbles. +pub const ROOT_NIBBLE_HEIGHT: usize = HashValue::LENGTH * 2; + #[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] pub struct Nibble(u8); diff --git a/storage/jellyfish-merkle/src/nibble_path/mod.rs b/types/src/nibble/nibble_path/mod.rs similarity index 99% rename from storage/jellyfish-merkle/src/nibble_path/mod.rs rename to types/src/nibble/nibble_path/mod.rs index accfe50b6f53c..17a94c7900604 100644 --- a/storage/jellyfish-merkle/src/nibble_path/mod.rs +++ b/types/src/nibble/nibble_path/mod.rs @@ -7,8 +7,7 @@ #[cfg(test)] mod nibble_path_test; -use crate::ROOT_NIBBLE_HEIGHT; -use diem_nibble::Nibble; +use crate::nibble::{Nibble, ROOT_NIBBLE_HEIGHT}; use mirai_annotations::*; #[cfg(any(test, feature = "fuzzing"))] use proptest::{collection::vec, prelude::*}; @@ -153,7 +152,7 @@ impl NibblePath { } /// Get the i-th nibble. - fn get_nibble(&self, i: usize) -> Nibble { + pub fn get_nibble(&self, i: usize) -> Nibble { assert!(i < self.num_nibbles); Nibble::from((self.bytes[i / 2] >> (if i % 2 == 1 { 0 } else { 4 })) & 0xf) } diff --git a/storage/jellyfish-merkle/src/nibble_path/nibble_path_test.rs b/types/src/nibble/nibble_path/nibble_path_test.rs similarity index 99% rename from storage/jellyfish-merkle/src/nibble_path/nibble_path_test.rs rename to types/src/nibble/nibble_path/nibble_path_test.rs index 69054009c5b80..be8e7ed7f584c 100644 --- a/storage/jellyfish-merkle/src/nibble_path/nibble_path_test.rs +++ b/types/src/nibble/nibble_path/nibble_path_test.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use super::{arb_internal_nibble_path, skip_common_prefix, NibblePath}; -use diem_nibble::Nibble; +use crate::nibble::Nibble; use proptest::prelude::*; #[test] diff --git a/types/src/proptest_types.rs b/types/src/proptest_types.rs index d7f73d4ba3da7..895e9e813d24e 100644 --- a/types/src/proptest_types.rs +++ b/types/src/proptest_types.rs @@ -825,6 +825,7 @@ impl TransactionToCommitGen { TransactionToCommit::new( Transaction::UserTransaction(transaction), account_states, + None, events, self.gas_used, self.status, diff --git a/types/src/transaction/mod.rs b/types/src/transaction/mod.rs index 19451756cb79d..32cd20677b49c 100644 --- a/types/src/transaction/mod.rs +++ b/types/src/transaction/mod.rs @@ -9,6 +9,7 @@ use crate::{ chain_id::ChainId, contract_event::ContractEvent, ledger_info::LedgerInfo, + nibble::nibble_path::NibblePath, proof::{accumulator::InMemoryAccumulator, TransactionInfoWithProof, TransactionListProof}, transaction::authenticator::{AccountAuthenticator, TransactionAuthenticator}, vm_status::{DiscardedVMStatus, KeptVMStatus, StatusCode, StatusType, VMStatus}, @@ -991,6 +992,7 @@ impl Display for TransactionInfo { pub struct TransactionToCommit { transaction: Transaction, account_states: HashMap, + jf_node_hashes: Option>, events: Vec, gas_used: u64, status: KeptVMStatus, @@ -1000,6 +1002,7 @@ impl TransactionToCommit { pub fn new( transaction: Transaction, account_states: HashMap, + jf_node_hashes: Option>, events: Vec, gas_used: u64, status: KeptVMStatus, @@ -1007,6 +1010,7 @@ impl TransactionToCommit { TransactionToCommit { transaction, account_states, + jf_node_hashes, events, gas_used, status, @@ -1021,6 +1025,10 @@ impl TransactionToCommit { &self.account_states } + pub fn jf_node_hashes(&self) -> Option<&HashMap> { + self.jf_node_hashes.as_ref() + } + pub fn events(&self) -> &[ContractEvent] { &self.events }