Skip to content

Commit

Permalink
Multi get transaction blocks (MystenLabs#10379)
Browse files Browse the repository at this point in the history
## Description 

I believe we can reuse `multi_get_transactions` as it calls
`multi_get_transaction_blocks` under the hood

## Test Plan 

How did you test the new or updated feature?

---
If your changes are not user-facing and not a breaking change, you can
skip the following section. Otherwise, please indicate what changed, and
then add to the Release Notes section as highlighted during the release
process.

### Type of Change (Check all that apply)

- [ ] user-visible impact
- [ ] breaking change for a client SDKs
- [ ] breaking change for FNs (FN binary must upgrade)
- [ ] breaking change for validators or node operators (must upgrade
binaries)
- [ ] breaking change for on-chain data layout
- [ ] necessitate either a data wipe or data migration

### Release notes
  • Loading branch information
wlmyng authored Apr 5, 2023
1 parent c97f7b6 commit 2bcdb6d
Show file tree
Hide file tree
Showing 5 changed files with 110 additions and 3 deletions.
5 changes: 5 additions & 0 deletions crates/sui-rpc-loadgen/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ cargo run --bin sui-rpc-loadgen -- --urls "http://127.0.0.1:9000" --num-threads
cargo run --bin sui-rpc-loadgen -- --urls "http://127.0.0.1:9000" "http://127.0.0.1:9000" --num-threads 4 query-transaction-blocks --address-type from
```

### Multi Get Transaction Blocks
```bash
cargo run --bin sui-rpc-loadgen -- --urls "http://127.0.0.1:9000" "http://127.0.0.1:9000" --num-threads 4 multi-get-transaction-blocks
```

### Multi Get Objects

```bash
Expand Down
16 changes: 15 additions & 1 deletion crates/sui-rpc-loadgen/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ use tracing::info;

use crate::load_test::{LoadTest, LoadTestConfig};
use crate::payload::{
load_addresses_from_file, load_objects_from_file, Command, RpcCommandProcessor, SignerInfo,
load_addresses_from_file, load_digests_from_file, load_objects_from_file, Command,
RpcCommandProcessor, SignerInfo,
};

#[derive(Parser)]
Expand Down Expand Up @@ -101,6 +102,11 @@ pub enum ClapCommand {
#[clap(flatten)]
common: CommonOptions,
},
#[clap(name = "multi-get-transaction-blocks")]
MultiGetTransactionBlocks {
#[clap(flatten)]
common: CommonOptions,
},
#[clap(name = "multi-get-objects")]
MultiGetObjects {
#[clap(flatten)]
Expand Down Expand Up @@ -214,6 +220,14 @@ async fn main() -> Result<(), Box<dyn Error>> {
false,
)
}
ClapCommand::MultiGetTransactionBlocks { common } => {
let digests = load_digests_from_file(expand_path(&opts.data_directory));
(
Command::new_multi_get_transaction_blocks(digests),
common,
false,
)
}
ClapCommand::GetAllBalances { common, chunk_size } => {
let addresses = load_addresses_from_file(expand_path(&opts.data_directory));
(
Expand Down
22 changes: 20 additions & 2 deletions crates/sui-rpc-loadgen/src/payload/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ mod get_checkpoints;
mod get_object;
mod get_reference_gas_price;
mod multi_get_objects;
mod multi_get_transaction_blocks;
mod pay_sui;
mod query_transactions;
mod rpc_command_processor;
Expand All @@ -17,11 +18,14 @@ use anyhow::Result;
use async_trait::async_trait;
use core::default::Default;
use std::time::Duration;
use sui_types::{base_types::SuiAddress, messages_checkpoint::CheckpointSequenceNumber};
use sui_types::{
base_types::SuiAddress, digests::TransactionDigest,
messages_checkpoint::CheckpointSequenceNumber,
};

use crate::load_test::LoadTestConfig;
pub use rpc_command_processor::{
load_addresses_from_file, load_objects_from_file, RpcCommandProcessor,
load_addresses_from_file, load_digests_from_file, load_objects_from_file, RpcCommandProcessor,
};
use sui_types::base_types::ObjectID;

Expand Down Expand Up @@ -109,6 +113,14 @@ impl Command {
}
}

pub fn new_multi_get_transaction_blocks(digests: Vec<TransactionDigest>) -> Self {
let multi_get_transaction_blocks = MultiGetTransactionBlocks { digests };
Self {
data: CommandData::MultiGetTransactionBlocks(multi_get_transaction_blocks),
..Default::default()
}
}

pub fn new_multi_get_objects(object_ids: Vec<ObjectID>) -> Self {
let multi_get_objects = MultiGetObjects { object_ids };
Self {
Expand Down Expand Up @@ -165,6 +177,7 @@ pub enum CommandData {
GetCheckpoints(GetCheckpoints),
PaySui(PaySui),
QueryTransactionBlocks(QueryTransactionBlocks),
MultiGetTransactionBlocks(MultiGetTransactionBlocks),
MultiGetObjects(MultiGetObjects),
GetObject(GetObject),
GetAllBalances(GetAllBalances),
Expand Down Expand Up @@ -200,6 +213,11 @@ pub struct QueryTransactionBlocks {
pub addresses: Vec<SuiAddress>,
}

#[derive(Clone)]
pub struct MultiGetTransactionBlocks {
pub digests: Vec<TransactionDigest>,
}

#[derive(Clone, EnumString)]
#[strum(serialize_all = "lowercase")]
pub enum AddressQueryType {
Expand Down
37 changes: 37 additions & 0 deletions crates/sui-rpc-loadgen/src/payload/multi_get_transaction_blocks.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Copyright (c) Mysten Labs, Inc.
// SPDX-License-Identifier: Apache-2.0

use anyhow::Result;

use crate::payload::{MultiGetTransactionBlocks, ProcessPayload, RpcCommandProcessor, SignerInfo};
use async_trait::async_trait;
use futures::future::join_all;

use super::validation::{check_transactions, chunk_entities};

#[async_trait]
impl<'a> ProcessPayload<'a, &'a MultiGetTransactionBlocks> for RpcCommandProcessor {
async fn process(
&'a self,
op: &'a MultiGetTransactionBlocks,
_signer_info: &Option<SignerInfo>,
) -> Result<()> {
let clients = self.get_clients().await?;
let digests = &op.digests;

if op.digests.is_empty() {
panic!("No digests provided, skipping query");
}

let chunks = chunk_entities(digests, None);
let chunk_futures = chunks.into_iter().map(|chunk| {
let clients = clients.clone();
async move {
check_transactions(&clients, &chunk, false, false).await;
}
});
join_all(chunk_futures).await;

Ok(())
}
}
33 changes: 33 additions & 0 deletions crates/sui-rpc-loadgen/src/payload/rpc_command_processor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ use crate::payload::{
Payload, ProcessPayload, Processor, QueryTransactionBlocks, SignerInfo,
};

use super::MultiGetTransactionBlocks;

pub(crate) const DEFAULT_GAS_BUDGET: u64 = 500_000_000;
pub(crate) const DEFAULT_LARGE_GAS_BUDGET: u64 = 50_000_000_000;
pub(crate) const MAX_NUM_NEW_OBJECTS_IN_SINGLE_TRANSACTION: usize = 120;
Expand Down Expand Up @@ -80,6 +82,7 @@ impl RpcCommandProcessor {
CommandData::GetCheckpoints(ref v) => self.process(v, signer_info).await,
CommandData::PaySui(ref v) => self.process(v, signer_info).await,
CommandData::QueryTransactionBlocks(ref v) => self.process(v, signer_info).await,
CommandData::MultiGetTransactionBlocks(ref v) => self.process(v, signer_info).await,
CommandData::MultiGetObjects(ref v) => self.process(v, signer_info).await,
CommandData::GetObject(ref v) => self.process(v, signer_info).await,
CommandData::GetAllBalances(ref v) => self.process(v, signer_info).await,
Expand Down Expand Up @@ -265,6 +268,13 @@ impl Processor for RpcCommandProcessor {
divide_query_transaction_blocks_tasks(data, config.num_threads).await
}
}
CommandData::MultiGetTransactionBlocks(data) => {
if !config.divide_tasks {
vec![config.command.clone(); config.num_threads]
} else {
divide_multi_get_transaction_blocks_tasks(data, config.num_threads).await
}
}
CommandData::GetAllBalances(data) => {
if !config.divide_tasks {
vec![config.command.clone(); config.num_threads]
Expand Down Expand Up @@ -384,6 +394,13 @@ pub fn load_objects_from_file(filepath: String) -> Vec<ObjectID> {
objects
}

pub fn load_digests_from_file(filepath: String) -> Vec<TransactionDigest> {
let path = format!("{}/{}", filepath, CacheType::TransactionDigest);
let digests: Vec<TransactionDigest> =
read_data_from_file(&path).expect("Failed to read transaction digests");
digests
}

fn read_data_from_file<T: DeserializeOwned>(file_path: &str) -> Result<T, anyhow::Error> {
let mut path_buf = PathBuf::from(file_path);

Expand Down Expand Up @@ -461,6 +478,22 @@ async fn divide_query_transaction_blocks_tasks(
.collect()
}

async fn divide_multi_get_transaction_blocks_tasks(
data: &MultiGetTransactionBlocks,
num_chunks: usize,
) -> Vec<Command> {
let chunk_size = if data.digests.len() < num_chunks {
1
} else {
data.digests.len() as u64 / num_chunks as u64
};
let chunked = chunk_entities(data.digests.as_slice(), Some(chunk_size as usize));
chunked
.into_iter()
.map(Command::new_multi_get_transaction_blocks)
.collect()
}

async fn divide_get_all_balances_tasks(data: &GetAllBalances, num_threads: usize) -> Vec<Command> {
let per_thread_size = if data.addresses.len() < num_threads {
1
Expand Down

0 comments on commit 2bcdb6d

Please sign in to comment.