From d89edb561d019c0c99baf9aa61d4e634b90a731c Mon Sep 17 00:00:00 2001 From: Greg Nazario Date: Mon, 2 May 2022 22:21:43 -0700 Subject: [PATCH] [aptos-cli] Build initial framework for Genesis Tooling Built up the ability to read from a local git repo or github to pull users for Genesis. Additionally, added ability to push to both and generate all keys needed to run a validator. --- Cargo.lock | 2 + crates/aptos/Cargo.toml | 2 + crates/aptos/src/common/init.rs | 12 +- crates/aptos/src/common/types.rs | 30 ++++ crates/aptos/src/common/utils.rs | 10 ++ crates/aptos/src/genesis/config.rs | 81 +++++++++++ crates/aptos/src/genesis/git.rs | 216 +++++++++++++++++++++++++++++ crates/aptos/src/genesis/keys.rs | 94 +++++++++++++ crates/aptos/src/genesis/mod.rs | 83 +++++++++++ crates/aptos/src/lib.rs | 10 +- x.toml | 1 + 11 files changed, 527 insertions(+), 14 deletions(-) create mode 100644 crates/aptos/src/genesis/config.rs create mode 100644 crates/aptos/src/genesis/git.rs create mode 100644 crates/aptos/src/genesis/keys.rs create mode 100644 crates/aptos/src/genesis/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 5253715bc7f88..2636ec15f8fda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -155,7 +155,9 @@ dependencies = [ "anyhow", "aptos-config", "aptos-crypto", + "aptos-github-client", "aptos-logger", + "aptos-management", "aptos-rest-client", "aptos-sdk", "aptos-secure-storage", diff --git a/crates/aptos/Cargo.toml b/crates/aptos/Cargo.toml index c03a99f2f8ea4..353bea176fb4a 100644 --- a/crates/aptos/Cargo.toml +++ b/crates/aptos/Cargo.toml @@ -33,7 +33,9 @@ uuid = { version = "0.8.2", features = ["v4", "serde"] } aptos-config = { path = "../../config" } aptos-crypto = { path = "../aptos-crypto" } aptos-logger = { path = "../aptos-logger" } +aptos-management = { path = "../../config/management" } aptos-secure-storage = { path = "../../secure/storage" } +aptos-github-client = { path = "../../secure/storage/github" } aptos-telemetry = { path = "../aptos-telemetry" } aptos-temppath = { path = "../aptos-temppath" } aptos-transaction-builder = { path = "../../sdk/transaction-builder" } diff --git a/crates/aptos/src/common/init.rs b/crates/aptos/src/common/init.rs index 570467a52b748..254f5f1936d35 100644 --- a/crates/aptos/src/common/init.rs +++ b/crates/aptos/src/common/init.rs @@ -8,7 +8,7 @@ use crate::{ account_address_from_public_key, CliCommand, CliConfig, CliError, CliTypedResult, EncodingOptions, PrivateKeyInputOptions, ProfileConfig, ProfileOptions, PromptOptions, }, - utils::prompt_yes_with_override, + utils::{prompt_yes_with_override, read_line}, }, op::key::GenerateKey, }; @@ -175,13 +175,3 @@ impl CliCommand<()> for InitTool { Ok(()) } } - -/// Reads a line from input -fn read_line(input_name: &'static str) -> CliTypedResult { - let mut input_buf = String::new(); - let _ = std::io::stdin() - .read_line(&mut input_buf) - .map_err(|err| CliError::IO(input_name.to_string(), err))?; - - Ok(input_buf) -} diff --git a/crates/aptos/src/common/types.rs b/crates/aptos/src/common/types.rs index 523851cc631da..a78732166325b 100644 --- a/crates/aptos/src/common/types.rs +++ b/crates/aptos/src/common/types.rs @@ -78,6 +78,36 @@ impl CliError { } } +impl From for CliError { + fn from(e: aptos_config::config::Error) -> Self { + CliError::UnexpectedError(e.to_string()) + } +} + +impl From for CliError { + fn from(e: aptos_github_client::Error) -> Self { + CliError::UnexpectedError(e.to_string()) + } +} + +impl From for CliError { + fn from(e: serde_yaml::Error) -> Self { + CliError::UnexpectedError(e.to_string()) + } +} + +impl From for CliError { + fn from(e: base64::DecodeError) -> Self { + CliError::UnexpectedError(e.to_string()) + } +} + +impl From for CliError { + fn from(e: std::string::FromUtf8Error) -> Self { + CliError::UnexpectedError(e.to_string()) + } +} + /// Config saved to `.aptos/config.yaml` #[derive(Debug, Serialize, Deserialize)] pub struct CliConfig { diff --git a/crates/aptos/src/common/utils.rs b/crates/aptos/src/common/utils.rs index ac257c30d40e8..1d1a883c3f9d6 100644 --- a/crates/aptos/src/common/utils.rs +++ b/crates/aptos/src/common/utils.rs @@ -276,3 +276,13 @@ pub async fn submit_transaction( pub fn current_dir() -> PathBuf { env::current_dir().unwrap() } + +/// Reads a line from input +pub fn read_line(input_name: &'static str) -> CliTypedResult { + let mut input_buf = String::new(); + let _ = std::io::stdin() + .read_line(&mut input_buf) + .map_err(|err| CliError::IO(input_name.to_string(), err))?; + + Ok(input_buf) +} diff --git a/crates/aptos/src/genesis/config.rs b/crates/aptos/src/genesis/config.rs new file mode 100644 index 0000000000000..4d0da8dc0f605 --- /dev/null +++ b/crates/aptos/src/genesis/config.rs @@ -0,0 +1,81 @@ +// Copyright (c) Aptos +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + common::types::{CliError, CliTypedResult}, + genesis::git::from_yaml, +}; +use aptos_crypto::{ed25519::Ed25519PublicKey, x25519}; +use aptos_types::{chain_id::ChainId, network_address::DnsName}; +use serde::{Deserialize, Serialize}; +use std::{fs::File, io::Read, path::PathBuf, str::FromStr}; + +/// Template for setting up Github for Genesis +/// +#[derive(Debug, Deserialize, Serialize)] +pub struct Layout { + /// Root key for the blockchain + /// TODO: In the future, we won't need a root key + pub root_key: Ed25519PublicKey, + /// List of usernames or identifiers + pub users: Vec, + /// ChainId for the target network + pub chain_id: ChainId, + /// Modules folder + pub modules_folder: String, +} + +impl Layout { + /// Read the layout from a YAML file on disk + pub fn from_disk(path: &PathBuf) -> CliTypedResult { + let mut file = + File::open(&path).map_err(|e| CliError::IO(path.display().to_string(), e))?; + let mut contents = String::new(); + file.read_to_string(&mut contents) + .map_err(|e| CliError::IO(path.display().to_string(), e))?; + from_yaml(&contents) + } +} + +/// A set of configuration needed to add a Validator to genesis +/// +#[derive(Debug, Serialize, Deserialize)] +pub struct ValidatorConfiguration { + /// Key used for signing in consensus + pub consensus_key: Ed25519PublicKey, + /// Key used for signing transactions with the account + pub account_key: Ed25519PublicKey, + /// Public key used for network identity (same as account address) + pub network_key: x25519::PublicKey, + /// Host for validator which can be an IP or a DNS name + pub validator_host: HostAndPort, + /// Host for full node which can be an IP or a DNS name and is optional + pub full_node_host: Option, +} + +/// Combined Host (DnsName or IP) and port +#[derive(Debug, Serialize, Deserialize)] +pub struct HostAndPort { + pub host: DnsName, + pub port: u16, +} + +impl FromStr for HostAndPort { + type Err = CliError; + + fn from_str(s: &str) -> Result { + let parts: Vec<_> = s.split(':').collect(); + if parts.len() != 2 { + Err(CliError::CommandArgumentError( + "Invalid host and port, must be of the form 'host:port` e.g. '127.0.0.1:6180'" + .to_string(), + )) + } else { + let host = DnsName::from_str(*parts.get(0).unwrap()) + .map_err(|e| CliError::CommandArgumentError(e.to_string()))?; + let port = u16::from_str(parts.get(1).unwrap()) + .map_err(|e| CliError::CommandArgumentError(e.to_string()))?; + Ok(HostAndPort { host, port }) + } + } +} diff --git a/crates/aptos/src/genesis/git.rs b/crates/aptos/src/genesis/git.rs new file mode 100644 index 0000000000000..3efab3c247bd5 --- /dev/null +++ b/crates/aptos/src/genesis/git.rs @@ -0,0 +1,216 @@ +// Copyright (c) Aptos +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + common::types::{CliError, CliTypedResult}, + genesis::config::Layout, + CliCommand, +}; +use aptos_config::config::Token; +use aptos_github_client::Client as GithubClient; +use async_trait::async_trait; +use clap::Parser; +use serde::{de::DeserializeOwned, Serialize}; +use std::{ + io::{Read, Write}, + path::PathBuf, + str::FromStr, +}; + +pub const LAYOUT_NAME: &str = "layout"; + +/// Setup a shared Github repository for Genesis +/// +#[derive(Parser)] +pub struct SetupGit { + #[clap(flatten)] + git_options: GitOptions, + /// Path to `Layout` which defines where all the files are + #[clap(long, parse(from_os_str))] + layout_path: PathBuf, +} + +#[async_trait] +impl CliCommand<()> for SetupGit { + fn command_name(&self) -> &'static str { + "SetupGit" + } + + async fn execute(self) -> CliTypedResult<()> { + let layout = Layout::from_disk(&self.layout_path)?; + + // Upload layout file to ensure we can read later + let client = self.git_options.get_client()?; + client.put(LAYOUT_NAME, &layout)?; + + // Make a place for the modules to be uploaded + client.create_dir(&layout.modules_folder)?; + + Ok(()) + } +} + +#[derive(Clone, Debug)] +pub struct GithubRepo { + owner: String, + repository: String, +} + +impl FromStr for GithubRepo { + type Err = CliError; + + fn from_str(s: &str) -> Result { + let parts: Vec<_> = s.split('/').collect(); + if parts.len() != 2 { + Err(CliError::CommandArgumentError("Invalid repository must be of the form 'owner/repository` e.g. 'aptos-labs/aptos-core'".to_string())) + } else { + Ok(GithubRepo { + owner: parts.get(0).unwrap().to_string(), + repository: parts.get(0).unwrap().to_string(), + }) + } + } +} + +#[derive(Clone, Parser)] +pub struct GitOptions { + /// Github repository e.g. 'aptos-labs/aptos-core' + #[clap(long)] + github_repository: Option, + /// Github repository branch e.g. main + #[clap(long, default_value = "main")] + github_branch: String, + /// Path to Github API token. Token must have repo:* permissions + #[clap(long, parse(from_os_str))] + github_token_path: Option, + /// Path to local git repo. + #[clap(long, parse(from_os_str))] + local_repository_path: Option, +} + +impl GitOptions { + pub fn get_client(self) -> CliTypedResult { + if self.github_repository.is_none() + && self.github_token_path.is_none() + && self.local_repository_path.is_some() + { + Ok(GitClient::local(self.local_repository_path.unwrap())) + } else if self.github_repository.is_some() + && self.github_token_path.is_some() + && self.local_repository_path.is_none() + { + GitClient::github( + self.github_repository.unwrap(), + self.github_branch, + self.github_token_path.unwrap(), + ) + } else { + Err(CliError::CommandArgumentError("Must provide either only --local-repository-path or both --github-repository and --github-token-path".to_string())) + } + } +} + +/// A Git client for abstracting away local vs Github +/// +/// Note: Writes do not commit locally +pub enum GitClient { + Local(PathBuf), + Github(GithubClient), +} + +impl GitClient { + pub fn local(path: PathBuf) -> GitClient { + GitClient::Local(path) + } + + pub fn github( + repository: GithubRepo, + branch: String, + token_path: PathBuf, + ) -> CliTypedResult { + let token = Token::FromDisk(token_path).read_token()?; + Ok(GitClient::Github(GithubClient::new( + repository.owner, + repository.repository, + branch, + token, + ))) + } + + /// Retrieves an object as a YAML encoded file from the appropriate storage + pub fn get(&self, name: &str) -> CliTypedResult { + match self { + GitClient::Local(local_repository_path) => { + let path = local_repository_path.join(format!("{}.yml", name)); + let mut file = std::fs::File::open(path.clone()) + .map_err(|e| CliError::IO(path.display().to_string(), e))?; + let mut contents = String::new(); + file.read_to_string(&mut contents) + .map_err(|e| CliError::IO(path.display().to_string(), e))?; + from_yaml(&contents) + } + GitClient::Github(client) => { + from_base64_encoded_yaml(&client.get_file(&format!("{}.yml", name))?) + } + } + } + + /// Puts an object as a YAML encoded file to the appropriate storage + pub fn put(&self, name: &str, input: &T) -> CliTypedResult<()> { + match self { + GitClient::Local(local_repository_path) => { + let path = local_repository_path.join(format!("{}.yml", name)); + let mut file = if path.exists() { + std::fs::File::open(path.clone()) + .map_err(|e| CliError::IO(path.display().to_string(), e))? + } else { + std::fs::File::create(path.clone()) + .map_err(|e| CliError::IO(path.display().to_string(), e))? + }; + + file.write_all(to_yaml(input)?.as_bytes()) + .map_err(|e| CliError::IO(path.display().to_string(), e))?; + } + GitClient::Github(client) => { + client.put(&format!("{}.yml", name), &to_base64_encoded_yaml(input)?)?; + } + } + + Ok(()) + } + + pub fn create_dir(&self, name: &str) -> CliTypedResult<()> { + match self { + GitClient::Local(local_repository_path) => { + let path = local_repository_path.join(name); + if path.exists() && path.is_dir() { + // Do nothing + } else { + std::fs::create_dir(path.clone()) + .map_err(|e| CliError::IO(path.display().to_string(), e))? + }; + } + GitClient::Github(_) => { + // There's no such thing as an empty directory in Git, so do nothing + } + } + + Ok(()) + } +} + +pub fn to_yaml(input: &T) -> CliTypedResult { + Ok(serde_yaml::to_string(input)?) +} + +pub fn from_yaml(input: &str) -> CliTypedResult { + Ok(serde_yaml::from_str(input)?) +} + +pub fn to_base64_encoded_yaml(input: &T) -> CliTypedResult { + Ok(base64::encode(to_yaml(input)?)) +} + +pub fn from_base64_encoded_yaml(input: &str) -> CliTypedResult { + from_yaml(&String::from_utf8(base64::decode(input)?)?) +} diff --git a/crates/aptos/src/genesis/keys.rs b/crates/aptos/src/genesis/keys.rs new file mode 100644 index 0000000000000..b0d73d7978da5 --- /dev/null +++ b/crates/aptos/src/genesis/keys.rs @@ -0,0 +1,94 @@ +// Copyright (c) Aptos +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + common::types::{CliTypedResult, EncodingType}, + genesis::{ + config::{HostAndPort, ValidatorConfiguration}, + git::GitOptions, + }, + op::key, + CliCommand, +}; +use aptos_crypto::{ed25519::Ed25519PrivateKey, x25519, PrivateKey}; +use async_trait::async_trait; +use clap::Parser; +use std::path::PathBuf; + +const ACCOUNT_KEY_FILE: &str = "account.key"; +const CONSENSUS_KEY_FILE: &str = "consensus.key"; +const NETWORK_KEY_FILE: &str = "network.key"; + +/// Generate account key, consensus key, and network key for a validator +#[derive(Parser)] +pub struct GenerateKeys { + #[clap(long, parse(from_os_str))] + output_path: PathBuf, +} + +#[async_trait] +impl CliCommand> for GenerateKeys { + fn command_name(&self) -> &'static str { + "GenerateKeys" + } + + async fn execute(self) -> CliTypedResult> { + let account_key_path = self.output_path.join(ACCOUNT_KEY_FILE); + let consensus_key_path = self.output_path.join(CONSENSUS_KEY_FILE); + let network_key_path = self.output_path.join(NETWORK_KEY_FILE); + let _ = key::GenerateKey::generate_ed25519(EncodingType::Hex, &account_key_path).await?; + let _ = key::GenerateKey::generate_ed25519(EncodingType::Hex, &consensus_key_path).await?; + let _ = key::GenerateKey::generate_x25519(EncodingType::Hex, &network_key_path).await?; + Ok(vec![account_key_path, consensus_key_path, network_key_path]) + } +} + +/// Upload ValidatorCredentials +#[derive(Parser)] +pub struct SetValidatorConfiguration { + /// Username + #[clap(long)] + username: String, + #[clap(flatten)] + git_options: GitOptions, + /// Path to credentials + #[clap(long, parse(from_os_str), default_value = ".aptos/")] + credentials_path: PathBuf, + /// Host and port pair for the validator e.g. 127.0.0.1:6180 + #[clap(long)] + validator_host: HostAndPort, + /// Host and port pair for the fullnode e.g. 127.0.0.1:6180 + #[clap(long)] + full_node_host: Option, +} + +#[async_trait] +impl CliCommand<()> for SetValidatorConfiguration { + fn command_name(&self) -> &'static str { + "SetValidatorConfiguration" + } + + async fn execute(self) -> CliTypedResult<()> { + let account_key_path = self.credentials_path.join(ACCOUNT_KEY_FILE); + let consensus_key_path = self.credentials_path.join(CONSENSUS_KEY_FILE); + let network_key_path = self.credentials_path.join(NETWORK_KEY_FILE); + let account_key: Ed25519PrivateKey = + EncodingType::Hex.load_key(ACCOUNT_KEY_FILE, &account_key_path)?; + let consensus_key: Ed25519PrivateKey = + EncodingType::Hex.load_key(CONSENSUS_KEY_FILE, &consensus_key_path)?; + let network_key: x25519::PrivateKey = + EncodingType::Hex.load_key(NETWORK_KEY_FILE, &network_key_path)?; + + let credentials = ValidatorConfiguration { + consensus_key: consensus_key.public_key(), + account_key: account_key.public_key(), + network_key: network_key.public_key(), + validator_host: self.validator_host, + full_node_host: self.full_node_host, + }; + + self.git_options + .get_client()? + .put(&self.username, &credentials) + } +} diff --git a/crates/aptos/src/genesis/mod.rs b/crates/aptos/src/genesis/mod.rs new file mode 100644 index 0000000000000..4064fe221d4fb --- /dev/null +++ b/crates/aptos/src/genesis/mod.rs @@ -0,0 +1,83 @@ +// Copyright (c) Aptos +// SPDX-License-Identifier: Apache-2.0 + +pub mod config; +pub mod git; +pub mod keys; + +use crate::{ + common::types::CliTypedResult, + genesis::{ + config::{Layout, ValidatorConfiguration}, + git::{GitOptions, LAYOUT_NAME}, + }, + CliCommand, CliResult, +}; +use aptos_crypto::ed25519::Ed25519PublicKey; +use aptos_types::chain_id::ChainId; +use async_trait::async_trait; +use clap::Parser; +use serde::Serialize; + +/// Tool for setting up and building the Genesis transaction +/// +#[derive(Parser)] +pub enum GenesisTool { + GenerateGenesis(GenerateGenesis), + GenerateKeys(keys::GenerateKeys), + SetupGit(git::SetupGit), + SetValidatorConfiguration(keys::SetValidatorConfiguration), +} + +impl GenesisTool { + pub async fn execute(self) -> CliResult { + match self { + GenesisTool::GenerateGenesis(tool) => tool.execute_serialized().await, + GenesisTool::GenerateKeys(tool) => tool.execute_serialized().await, + GenesisTool::SetupGit(tool) => tool.execute_serialized_success().await, + GenesisTool::SetValidatorConfiguration(tool) => tool.execute_serialized_success().await, + } + } +} + +/// Generate genesis from a git repo +#[derive(Parser)] +pub struct GenerateGenesis { + #[clap(flatten)] + github_options: GitOptions, +} + +#[async_trait] +impl CliCommand for GenerateGenesis { + fn command_name(&self) -> &'static str { + "GenerateGenesis" + } + + async fn execute(self) -> CliTypedResult { + // TODO: Generate genesis, this right now just reads all users + fetch_genesis_info(self.github_options) + } +} + +pub fn fetch_genesis_info(git_options: GitOptions) -> CliTypedResult { + let client = git_options.get_client()?; + let layout: Layout = client.get(LAYOUT_NAME)?; + + let mut configs = Vec::new(); + for user in &layout.users { + configs.push(client.get(user)?); + } + + Ok(GenesisInfo { + chain_id: layout.chain_id, + root_key: layout.root_key, + participants: configs, + }) +} + +#[derive(Debug, Serialize)] +pub struct GenesisInfo { + chain_id: ChainId, + root_key: Ed25519PublicKey, + participants: Vec, +} diff --git a/crates/aptos/src/lib.rs b/crates/aptos/src/lib.rs index 1842f292d8655..773164f6e95bd 100644 --- a/crates/aptos/src/lib.rs +++ b/crates/aptos/src/lib.rs @@ -6,6 +6,7 @@ pub mod account; pub mod common; pub mod config; +pub mod genesis; pub mod move_tool; pub mod op; @@ -21,11 +22,13 @@ pub enum Tool { Account(account::AccountTool), #[clap(subcommand)] Config(config::ConfigTool), - Init(common::init::InitTool), #[clap(subcommand)] - Move(move_tool::MoveTool), + Genesis(genesis::GenesisTool), + Init(common::init::InitTool), #[clap(subcommand)] Key(op::key::KeyTool), + #[clap(subcommand)] + Move(move_tool::MoveTool), } impl Tool { @@ -34,9 +37,10 @@ impl Tool { Tool::Account(tool) => tool.execute().await, Tool::Config(tool) => tool.execute().await, // TODO: Replace entirely with config init + Tool::Genesis(tool) => tool.execute().await, Tool::Init(tool) => tool.execute_serialized_success().await, - Tool::Move(tool) => tool.execute().await, Tool::Key(tool) => tool.execute().await, + Tool::Move(tool) => tool.execute().await, } } } diff --git a/x.toml b/x.toml index 8de3976968a3d..b8940a1786b76 100644 --- a/x.toml +++ b/x.toml @@ -228,6 +228,7 @@ root-members = [ root-members = [ "backup-cli", "db-bootstrapper", + "aptos", "aptos-genesis-tool", "aptos-node", "aptos-operational-tool",