From 1e28753f386daffc75414b214e3cfd3a8bed2b9f Mon Sep 17 00:00:00 2001 From: howardwu Date: Mon, 29 Nov 2021 23:03:29 -0800 Subject: [PATCH] Add prover storage functionality --- .dockerignore | 1 + .gitignore | 1 + Cargo.lock | 34 +++++++++++++++++----------------- Cargo.toml | 4 ++-- src/environment/mod.rs | 2 +- src/main.rs | 4 ++-- src/network/prover.rs | 18 +++++++++++++++--- src/network/server.rs | 9 ++++++--- src/node.rs | 17 +++++++++++++++-- src/rpc/rpc.rs | 8 ++++---- storage/Cargo.toml | 2 +- storage/src/state/ledger.rs | 18 ++++-------------- storage/src/state/prover.rs | 10 ++++++++++ storage/src/state/tests.rs | 14 +++++++------- testing/Cargo.toml | 2 +- 15 files changed, 87 insertions(+), 57 deletions(-) diff --git a/.dockerignore b/.dockerignore index 966653d68a..03226244d9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ **/target **/.DS_Store **/.ledger* +**/.prover* **inner.proving* **outer.proving* **posw.proving* diff --git a/.gitignore b/.gitignore index 966653d68a..03226244d9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ **/target **/.DS_Store **/.ledger* +**/.prover* **inner.proving* **outer.proving* **posw.proving* diff --git a/Cargo.lock b/Cargo.lock index 6294dd83f2..78d73223b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,9 +24,9 @@ dependencies = [ [[package]] name = "aleo-std" -version = "0.1.1" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09209c7e5c7c91e93e1676481d3c14cd3c89ba0073ae0e20ff447cae3dae3829" +checksum = "64301eafbaf71ae281e6eb535eba3998b9292faadcb02552ed14487429867810" dependencies = [ "aleo-std-storage", "aleo-std-time", @@ -36,9 +36,9 @@ dependencies = [ [[package]] name = "aleo-std-storage" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d57acc81ab9c7f9adba6416b80fe1b45596027a3b2a78ebd850793bbe95ca2" +checksum = "312e7383f8f6b64bf0abbfcc2c9526bbbb5409dc7a0db9bb460748cd4ad12f14" dependencies = [ "dirs", ] @@ -1923,7 +1923,7 @@ dependencies = [ [[package]] name = "snarkvm" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "snarkvm-dpc", "snarkvm-utilities", @@ -1932,7 +1932,7 @@ dependencies = [ [[package]] name = "snarkvm-algorithms" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "anyhow", "blake2", @@ -1961,7 +1961,7 @@ dependencies = [ [[package]] name = "snarkvm-curves" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "derivative", "rand", @@ -1975,7 +1975,7 @@ dependencies = [ [[package]] name = "snarkvm-derives" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "proc-macro-crate", "proc-macro-error", @@ -1987,7 +1987,7 @@ dependencies = [ [[package]] name = "snarkvm-dpc" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "anyhow", "base58", @@ -2018,7 +2018,7 @@ dependencies = [ [[package]] name = "snarkvm-fields" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "anyhow", "derivative", @@ -2031,7 +2031,7 @@ dependencies = [ [[package]] name = "snarkvm-gadgets" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "anyhow", "derivative", @@ -2051,7 +2051,7 @@ dependencies = [ [[package]] name = "snarkvm-marlin" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "bincode", "blake2", @@ -2077,7 +2077,7 @@ dependencies = [ [[package]] name = "snarkvm-parameters" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "aleo-std", "anyhow", @@ -2094,7 +2094,7 @@ dependencies = [ [[package]] name = "snarkvm-polycommit" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "derivative", "digest", @@ -2112,12 +2112,12 @@ dependencies = [ [[package]] name = "snarkvm-profiler" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" [[package]] name = "snarkvm-r1cs" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "anyhow", "cfg-if", @@ -2133,7 +2133,7 @@ dependencies = [ [[package]] name = "snarkvm-utilities" version = "0.7.5" -source = "git+https://github.com/AleoHQ/snarkVM.git?rev=29bfb3a#29bfb3ac537c75bb51ef55e7318a6f48def35182" +source = "git+https://github.com/AleoHQ/snarkVM.git?rev=8937da0#8937da089069df28f01c1aa68e5d3b9e0cfb1705" dependencies = [ "anyhow", "bincode", diff --git a/Cargo.toml b/Cargo.toml index 8eb66473c8..fb2ea3e39e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ default = [] test = [] [dependencies] -snarkvm = { git = "https://github.com/AleoHQ/snarkVM.git", rev = "29bfb3a" } +snarkvm = { git = "https://github.com/AleoHQ/snarkVM.git", rev = "8937da0" } #snarkvm = { path = "../snarkVM" } bytes = "1.0.0" @@ -38,7 +38,7 @@ path = "./storage" version = "2.0.0" [dependencies.aleo-std] -version = "0.1.1" +version = "0.1.4" [dependencies.anyhow] version = "1" diff --git a/src/environment/mod.rs b/src/environment/mod.rs index e8b40162e7..29edc5f8d3 100644 --- a/src/environment/mod.rs +++ b/src/environment/mod.rs @@ -62,7 +62,7 @@ pub trait Environment: 'static + Clone + Debug + Default + Send + Sync { const SYNC_NODES: [&'static str; 13] = ["127.0.0.1:4131", "127.0.0.1:4133", "127.0.0.1:4134", "127.0.0.1:4135", "127.0.0.1:4136", "127.0.0.1:4137", "127.0.0.1:4138", "127.0.0.1:4139", "127.0.0.1:4140", "127.0.0.1:4141", "127.0.0.1:4142", "127.0.0.1:4143", "127.0.0.1:4144"]; /// The duration in seconds to sleep in between heartbeat executions. - const HEARTBEAT_IN_SECS: u64 = 9; + const HEARTBEAT_IN_SECS: u64 = 12; /// The maximum duration in seconds permitted for establishing a connection with a node, /// before dropping the connection; it should be no greater than the `HEARTBEAT_IN_SECS`. const CONNECTION_TIMEOUT_IN_SECS: u64 = 1; diff --git a/src/main.rs b/src/main.rs index 035c1867bd..58172e6ee2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,13 +38,13 @@ fn main() -> Result<()> { .enable_all() .thread_stack_size(8 * 1024 * 1024) .worker_threads((num_cpus::get() / 8 * 2).max(1)) - .max_blocking_threads((num_cpus::get() / 8).max(1)) + .max_blocking_threads((num_cpus::get() / 8 * 2).max(1)) .build()?; // Initialize the parallelization parameters. rayon::ThreadPoolBuilder::new() .stack_size(8 * 1024 * 1024) - .num_threads((num_cpus::get() / 8 * 3).max(1)) + .num_threads((num_cpus::get() / 8 * 5).max(1)) .build_global() .unwrap(); diff --git a/src/network/prover.rs b/src/network/prover.rs index 1497485853..83cda45050 100644 --- a/src/network/prover.rs +++ b/src/network/prover.rs @@ -25,6 +25,7 @@ use crate::{ PeersRequest, PeersRouter, }; +use snarkos_storage::{storage::Storage, ProverState}; use snarkvm::dpc::prelude::*; use anyhow::Result; @@ -32,6 +33,7 @@ use rand::thread_rng; use rayon::{ThreadPool, ThreadPoolBuilder}; use std::{ net::SocketAddr, + path::Path, sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -65,6 +67,8 @@ pub enum ProverRequest { /// #[derive(Debug)] pub struct Prover { + /// The state storage of the prover. + state: Arc>, /// The thread pool for the miner. miner: Arc, /// The prover router of the node. @@ -85,8 +89,9 @@ pub struct Prover { impl Prover { /// Initializes a new instance of the prover. - pub async fn new( + pub async fn open + Copy>( tasks: &mut Tasks>, + path: P, miner: Option>, local_ip: SocketAddr, status: &Status, @@ -105,6 +110,7 @@ impl Prover { // Initialize the prover. let prover = Arc::new(Self { + state: Arc::new(ProverState::open_writer::(path)?), miner: Arc::new(pool), prover_router, memory_pool: RwLock::new(MemoryPool::new()), @@ -149,6 +155,7 @@ impl Prover { prover.status.update(State::Mining); // Prepare the unconfirmed transactions, terminator, and status. + let state = prover.state.clone(); let miner = prover.miner.clone(); let canon = prover.ledger_reader.clone(); // This is *safe* as the ledger only reads. let unconfirmed_transactions = prover.memory_pool.read().await.transactions(); @@ -177,12 +184,17 @@ impl Prover { status.update(State::Ready); match result { - Ok(Ok(block)) => { + Ok(Ok((block, coinbase_record))) => { debug!("Miner has found an unconfirmed candidate for block {}", block.height()); + // Store the coinbase record. + if let Err(error) = state.add_coinbase_record(&coinbase_record) { + warn!("[Miner] Failed to store coinbase record - {}", error); + } + // Broadcast the next block. let request = LedgerRequest::UnconfirmedBlock(local_ip, block, prover_router.clone()); if let Err(error) = ledger_router.send(request).await { - warn!("Failed to broadcast mined block: {}", error); + warn!("Failed to broadcast mined block - {}", error); } } Ok(Err(error)) | Err(error) => trace!("{}", error), diff --git a/src/network/server.rs b/src/network/server.rs index b7ace63e01..fb02073abd 100644 --- a/src/network/server.rs +++ b/src/network/server.rs @@ -68,7 +68,9 @@ impl Server { }; // Initialize the ledger storage path. - let storage_path = node.storage_path(local_ip); + let ledger_storage_path = node.ledger_storage_path(local_ip); + // Initialize the prover storage path. + let prover_storage_path = node.prover_storage_path(local_ip); // Initialize the status indicator. let status = Status::new(); // Initialize the terminator bit. @@ -77,10 +79,11 @@ impl Server { // Initialize a new instance for managing peers. let peers = Peers::new(&mut tasks, local_ip, None, &status).await; // Initialize a new instance for managing the ledger. - let ledger = Ledger::::open::(&mut tasks, &storage_path, &status, &terminator, peers.router()).await?; + let ledger = Ledger::::open::(&mut tasks, &ledger_storage_path, &status, &terminator, peers.router()).await?; // Initialize a new instance for managing the prover. - let prover = Prover::new( + let prover = Prover::open::( &mut tasks, + &prover_storage_path, miner, local_ip, &status, diff --git a/src/node.rs b/src/node.rs index 7316b52963..fcecbeb873 100644 --- a/src/node.rs +++ b/src/node.rs @@ -100,8 +100,8 @@ impl Node { } } - /// Returns the storage path of the node. - pub(crate) fn storage_path(&self, _local_ip: SocketAddr) -> PathBuf { + /// Returns the storage path of the ledger. + pub(crate) fn ledger_storage_path(&self, _local_ip: SocketAddr) -> PathBuf { cfg_if::cfg_if! { if #[cfg(feature = "test")] { // Tests may use any available ports, and removes the storage artifacts afterwards, @@ -113,6 +113,19 @@ impl Node { } } + /// Returns the storage path of the prover. + pub(crate) fn prover_storage_path(&self, _local_ip: SocketAddr) -> PathBuf { + cfg_if::cfg_if! { + if #[cfg(feature = "test")] { + // Tests may use any available ports, and removes the storage artifacts afterwards, + // so that there is no need to adhere to a specific number assignment logic. + PathBuf::from(format!("/tmp/snarkos-test-prover-{}", _local_ip.port())) + } else { + aleo_std::aleo_prover_dir(self.network, self.dev) + } + } + } + async fn start_server(&self) -> Result<()> { let miner = match (E::NODE_TYPE, &self.miner) { (NodeType::Miner, Some(address)) => { diff --git a/src/rpc/rpc.rs b/src/rpc/rpc.rs index 2e6ef71205..9d1747b7a9 100644 --- a/src/rpc/rpc.rs +++ b/src/rpc/rpc.rs @@ -709,7 +709,7 @@ mod tests { let address = account.address(); // Mine the next block. - let block_1 = ledger_state + let (block_1, _) = ledger_state .mine_next_block(address, true, &[], &terminator, rng) .expect("Failed to mine"); ledger_state.add_next_block(&block_1).expect("Failed to add next block to ledger"); @@ -829,7 +829,7 @@ mod tests { let address = account.address(); // Mine the next block. - let block_1 = ledger_state + let (block_1, _) = ledger_state .mine_next_block(address, true, &[], &terminator, rng) .expect("Failed to mine"); ledger_state.add_next_block(&block_1).expect("Failed to add next block to ledger"); @@ -989,7 +989,7 @@ mod tests { let address = account.address(); // Mine the next block. - let block_1 = ledger_state + let (block_1, _) = ledger_state .mine_next_block(address, true, &[], &terminator, &mut rng) .expect("Failed to mine"); ledger_state.add_next_block(&block_1).expect("Failed to add next block to ledger"); @@ -1161,7 +1161,7 @@ mod tests { let address = account.address(); // Initialize a new transaction. - let transaction = Transaction::::new_coinbase(address, AleoAmount(1234), true, &mut rng) + let (transaction, _) = Transaction::::new_coinbase(address, AleoAmount(1234), true, &mut rng) .expect("Failed to create a coinbase transaction"); // Initialize a new rpc. diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 91d770560b..bde0d25a44 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -17,7 +17,7 @@ license = "GPL-3.0" edition = "2018" [dependencies] -snarkvm = { git = "https://github.com/AleoHQ/snarkVM.git", rev = "29bfb3a" } +snarkvm = { git = "https://github.com/AleoHQ/snarkVM.git", rev = "8937da0" } #snarkvm = { path = "../../snarkVM" } [dependencies.anyhow] diff --git a/storage/src/state/ledger.rs b/storage/src/state/ledger.rs index 35c19edd34..36014935bb 100644 --- a/storage/src/state/ledger.rs +++ b/storage/src/state/ledger.rs @@ -579,7 +579,7 @@ impl LedgerState { transactions: &[Transaction], terminator: &AtomicBool, rng: &mut R, - ) -> Result> { + ) -> Result<(Block, Record)> { // Prepare the new block. let previous_block_hash = self.latest_block_hash(); let block_height = self.latest_block_height() + 1; @@ -598,7 +598,7 @@ impl LedgerState { // Craft a coinbase transaction. let amount = Block::::block_reward(block_height); - let coinbase_transaction = Transaction::::new_coinbase(recipient, amount, is_public, rng)?; + let (coinbase_transaction, coinbase_record) = Transaction::::new_coinbase(recipient, amount, is_public, rng)?; // Filter the transactions to ensure they are new, and append the coinbase transaction. // TODO (howardwu): Improve the performance and design of this. @@ -638,7 +638,7 @@ impl LedgerState { terminator, rng, ) { - Ok(block) => Ok(block), + Ok(block) => Ok((block, coinbase_record)), Err(error) => Err(anyhow!("Unable to mine the next block: {}", error)), } } @@ -1335,7 +1335,6 @@ struct TransactionState { transitions: DataMap)>, serial_numbers: DataMap, commitments: DataMap, - events: DataMap>>, } impl TransactionState { @@ -1346,7 +1345,6 @@ impl TransactionState { transitions: storage.open_map("transitions")?, serial_numbers: storage.open_map("serial_numbers")?, commitments: storage.open_map("commitments")?, - events: storage.open_map("events")?, }) } @@ -1414,7 +1412,7 @@ impl TransactionState { }; } - Transaction::from(*N::inner_circuit_id(), ledger_root, transitions, vec![]) + Transaction::from(*N::inner_circuit_id(), ledger_root, transitions) } /// Returns the transaction metadata for a given transaction ID. @@ -1457,10 +1455,6 @@ impl TransactionState { self.commitments.insert(commitment, &transition_id)?; } } - - // Insert the transaction events. - self.events.insert(&transaction_id, transaction.events())?; - Ok(()) } } @@ -1495,10 +1489,6 @@ impl TransactionState { self.commitments.remove(commitment)?; } } - - // Remove the transaction events. - self.events.remove(transaction_id)?; - Ok(()) } } diff --git a/storage/src/state/prover.rs b/storage/src/state/prover.rs index 087952b911..0c47c03e3a 100644 --- a/storage/src/state/prover.rs +++ b/storage/src/state/prover.rs @@ -55,6 +55,11 @@ impl ProverState { self.coinbase.contains_record(commitment) } + /// Returns all coinbase records in storage. + pub fn to_coinbase_records(&self) -> Vec> { + self.coinbase.to_records() + } + /// Returns the coinbase record for a given commitment. pub fn get_coinbase_record(&self, commitment: &N::Commitment) -> Result> { self.coinbase.get_record(commitment) @@ -90,6 +95,11 @@ impl CoinbaseState { self.records.contains_key(commitment) } + /// Returns all records in storage. + fn to_records(&self) -> Vec> { + self.records.values().collect() + } + /// Returns the record for a given commitment. fn get_record(&self, commitment: &N::Commitment) -> Result> { match self.records.get(commitment)? { diff --git a/storage/src/state/tests.rs b/storage/src/state/tests.rs index 189090ff61..feaa4e42c9 100644 --- a/storage/src/state/tests.rs +++ b/storage/src/state/tests.rs @@ -75,7 +75,7 @@ fn test_add_next_block() { let address = account.address(); // Mine the next block. - let block = ledger + let (block, _) = ledger .mine_next_block(address, true, &[], &terminator, rng) .expect("Failed to mine"); ledger.add_next_block(&block).expect("Failed to add next block to ledger"); @@ -123,7 +123,7 @@ fn test_remove_last_block() { let address = account.address(); // Mine the next block. - let block = ledger + let (block, _) = ledger .mine_next_block(address, true, &[], &terminator, rng) .expect("Failed to mine"); ledger.add_next_block(&block).expect("Failed to add next block to ledger"); @@ -169,14 +169,14 @@ fn test_remove_last_2_blocks() { let address = account.address(); // Mine the next block. - let block_1 = ledger + let (block_1, _) = ledger .mine_next_block(address, true, &[], &terminator, rng) .expect("Failed to mine"); ledger.add_next_block(&block_1).expect("Failed to add next block to ledger"); assert_eq!(1, ledger.latest_block_height()); // Mine the next block. - let block_2 = ledger + let (block_2, _) = ledger .mine_next_block(address, true, &[], &terminator, rng) .expect("Failed to mine"); ledger.add_next_block(&block_2).expect("Failed to add next block to ledger"); @@ -222,7 +222,7 @@ fn test_get_block_locators() { let address = account.address(); // Mine the next block. - let block_1 = ledger + let (block_1, _) = ledger .mine_next_block(address, true, &[], &terminator, rng) .expect("Failed to mine"); ledger.add_next_block(&block_1).expect("Failed to add next block to ledger"); @@ -239,7 +239,7 @@ fn test_get_block_locators() { ); // Mine the next block. - let block_2 = ledger + let (block_2, _) = ledger .mine_next_block(address, true, &[], &terminator, rng) .expect("Failed to mine"); ledger.add_next_block(&block_2).expect("Failed to add next block to ledger"); @@ -256,7 +256,7 @@ fn test_get_block_locators() { ); // Mine the next block. - let block_3 = ledger + let (block_3, _) = ledger .mine_next_block(address, true, &[], &terminator, rng) .expect("Failed to mine"); ledger.add_next_block(&block_3).expect("Failed to add next block to ledger"); diff --git a/testing/Cargo.toml b/testing/Cargo.toml index d5fb951748..f6a1eae632 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -25,7 +25,7 @@ path = "../storage" [dependencies.snarkvm] git = "https://github.com/AleoHQ/snarkVM.git" -rev = "29bfb3a" +rev = "8937da0" #path = "../../snarkVM" [dependencies.anyhow]