[aptos-cli] Build initial framework for Genesis Tooling

Built up the ability to read from a local git repo or github
to pull users for Genesis.  Additionally, added ability to
push to both and generate all keys needed to run a validator.

crates/aptos/Cargo.toml

@@ -33,7 +33,9 @@ uuid = { version = "0.8.2", features = ["v4", "serde"] }
 aptos-config = { path = "../../config" }
 aptos-crypto = { path = "../aptos-crypto" }
 aptos-logger = { path = "../aptos-logger" }
+aptos-management = { path = "../../config/management" }
 aptos-secure-storage = { path = "../../secure/storage" }
+aptos-github-client = { path = "../../secure/storage/github" }
 aptos-telemetry = { path = "../aptos-telemetry" }
 aptos-temppath = { path = "../aptos-temppath" }
 aptos-transaction-builder = { path = "../../sdk/transaction-builder" }

crates/aptos/src/common/init.rs

@@ -8,7 +8,7 @@ use crate::{
             account_address_from_public_key, CliCommand, CliConfig, CliError, CliTypedResult,
             EncodingOptions, PrivateKeyInputOptions, ProfileConfig, ProfileOptions, PromptOptions,
         },
-        utils::prompt_yes_with_override,
+        utils::{prompt_yes_with_override, read_line},
     },
     op::key::GenerateKey,
 };
@@ -175,13 +175,3 @@ impl CliCommand<()> for InitTool {
         Ok(())
     }
 }
-
-/// Reads a line from input
-fn read_line(input_name: &'static str) -> CliTypedResult<String> {
-    let mut input_buf = String::new();
-    let _ = std::io::stdin()
-        .read_line(&mut input_buf)
-        .map_err(|err| CliError::IO(input_name.to_string(), err))?;
-
-    Ok(input_buf)
-}

crates/aptos/src/common/types.rs

@@ -78,6 +78,36 @@ impl CliError {
     }
 }
 
+impl From<aptos_config::config::Error> for CliError {
+    fn from(e: aptos_config::config::Error) -> Self {
+        CliError::UnexpectedError(e.to_string())
+    }
+}
+
+impl From<aptos_github_client::Error> for CliError {
+    fn from(e: aptos_github_client::Error) -> Self {
+        CliError::UnexpectedError(e.to_string())
+    }
+}
+
+impl From<serde_yaml::Error> for CliError {
+    fn from(e: serde_yaml::Error) -> Self {
+        CliError::UnexpectedError(e.to_string())
+    }
+}
+
+impl From<base64::DecodeError> for CliError {
+    fn from(e: base64::DecodeError) -> Self {
+        CliError::UnexpectedError(e.to_string())
+    }
+}
+
+impl From<std::string::FromUtf8Error> for CliError {
+    fn from(e: std::string::FromUtf8Error) -> Self {
+        CliError::UnexpectedError(e.to_string())
+    }
+}
+
 /// Config saved to `.aptos/config.yaml`
 #[derive(Debug, Serialize, Deserialize)]
 pub struct CliConfig {

crates/aptos/src/common/utils.rs

@@ -276,3 +276,13 @@ pub async fn submit_transaction(
 pub fn current_dir() -> PathBuf {
     env::current_dir().unwrap()
 }
+
+/// Reads a line from input
+pub fn read_line(input_name: &'static str) -> CliTypedResult<String> {
+    let mut input_buf = String::new();
+    let _ = std::io::stdin()
+        .read_line(&mut input_buf)
+        .map_err(|err| CliError::IO(input_name.to_string(), err))?;
+
+    Ok(input_buf)
+}

crates/aptos/src/genesis/config.rs

@@ -0,0 +1,81 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{
+    common::types::{CliError, CliTypedResult},
+    genesis::git::from_yaml,
+};
+use aptos_crypto::{ed25519::Ed25519PublicKey, x25519};
+use aptos_types::{chain_id::ChainId, network_address::DnsName};
+use serde::{Deserialize, Serialize};
+use std::{fs::File, io::Read, path::PathBuf, str::FromStr};
+
+/// Template for setting up Github for Genesis
+///
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Layout {
+    /// Root key for the blockchain
+    /// TODO: In the future, we won't need a root key
+    pub root_key: Ed25519PublicKey,
+    /// List of usernames or identifiers
+    pub users: Vec<String>,
+    /// ChainId for the target network
+    pub chain_id: ChainId,
+    /// Modules folder
+    pub modules_folder: String,
+}
+
+impl Layout {
+    /// Read the layout from a YAML file on disk
+    pub fn from_disk(path: &PathBuf) -> CliTypedResult<Self> {
+        let mut file =
+            File::open(&path).map_err(|e| CliError::IO(path.display().to_string(), e))?;
+        let mut contents = String::new();
+        file.read_to_string(&mut contents)
+            .map_err(|e| CliError::IO(path.display().to_string(), e))?;
+        from_yaml(&contents)
+    }
+}
+
+/// A set of configuration needed to add a Validator to genesis
+///
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ValidatorConfiguration {
+    /// Key used for signing in consensus
+    pub consensus_key: Ed25519PublicKey,
+    /// Key used for signing transactions with the account
+    pub account_key: Ed25519PublicKey,
+    /// Public key used for network identity (same as account address)
+    pub network_key: x25519::PublicKey,
+    /// Host for validator which can be an IP or a DNS name
+    pub validator_host: HostAndPort,
+    /// Host for full node which can be an IP or a DNS name and is optional
+    pub full_node_host: Option<HostAndPort>,
+}
+
+/// Combined Host (DnsName or IP) and port
+#[derive(Debug, Serialize, Deserialize)]
+pub struct HostAndPort {
+    pub host: DnsName,
+    pub port: u16,
+}
+
+impl FromStr for HostAndPort {
+    type Err = CliError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let parts: Vec<_> = s.split(':').collect();
+        if parts.len() != 2 {
+            Err(CliError::CommandArgumentError(
+                "Invalid host and port, must be of the form 'host:port` e.g. '127.0.0.1:6180'"
+                    .to_string(),
+            ))
+        } else {
+            let host = DnsName::from_str(*parts.get(0).unwrap())
+                .map_err(|e| CliError::CommandArgumentError(e.to_string()))?;
+            let port = u16::from_str(parts.get(1).unwrap())
+                .map_err(|e| CliError::CommandArgumentError(e.to_string()))?;
+            Ok(HostAndPort { host, port })
+        }
+    }
+}

crates/aptos/src/genesis/git.rs

@@ -0,0 +1,216 @@
+// Copyright (c) Aptos
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{
+    common::types::{CliError, CliTypedResult},
+    genesis::config::Layout,
+    CliCommand,
+};
+use aptos_config::config::Token;
+use aptos_github_client::Client as GithubClient;
+use async_trait::async_trait;
+use clap::Parser;
+use serde::{de::DeserializeOwned, Serialize};
+use std::{
+    io::{Read, Write},
+    path::PathBuf,
+    str::FromStr,
+};
+
+pub const LAYOUT_NAME: &str = "layout";
+
+/// Setup a shared Github repository for Genesis
+///
+#[derive(Parser)]
+pub struct SetupGit {
+    #[clap(flatten)]
+    git_options: GitOptions,
+    /// Path to `Layout` which defines where all the files are
+    #[clap(long, parse(from_os_str))]
+    layout_path: PathBuf,
+}
+
+#[async_trait]
+impl CliCommand<()> for SetupGit {
+    fn command_name(&self) -> &'static str {
+        "SetupGit"
+    }
+
+    async fn execute(self) -> CliTypedResult<()> {
+        let layout = Layout::from_disk(&self.layout_path)?;
+
+        // Upload layout file to ensure we can read later
+        let client = self.git_options.get_client()?;
+        client.put(LAYOUT_NAME, &layout)?;
+
+        // Make a place for the modules to be uploaded
+        client.create_dir(&layout.modules_folder)?;
+
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct GithubRepo {
+    owner: String,
+    repository: String,
+}
+
+impl FromStr for GithubRepo {
+    type Err = CliError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let parts: Vec<_> = s.split('/').collect();
+        if parts.len() != 2 {
+            Err(CliError::CommandArgumentError("Invalid repository must be of the form 'owner/repository` e.g. 'aptos-labs/aptos-core'".to_string()))
+        } else {
+            Ok(GithubRepo {
+                owner: parts.get(0).unwrap().to_string(),
+                repository: parts.get(0).unwrap().to_string(),
+            })
+        }
+    }
+}
+
+#[derive(Clone, Parser)]
+pub struct GitOptions {
+    /// Github repository e.g. 'aptos-labs/aptos-core'
+    #[clap(long)]
+    github_repository: Option<GithubRepo>,
+    /// Github repository branch e.g. main
+    #[clap(long, default_value = "main")]
+    github_branch: String,
+    /// Path to Github API token.  Token must have repo:* permissions
+    #[clap(long, parse(from_os_str))]
+    github_token_path: Option<PathBuf>,
+    /// Path to local git repo.
+    #[clap(long, parse(from_os_str))]
+    local_repository_path: Option<PathBuf>,
+}
+
+impl GitOptions {
+    pub fn get_client(self) -> CliTypedResult<GitClient> {
+        if self.github_repository.is_none()
+            && self.github_token_path.is_none()
+            && self.local_repository_path.is_some()
+        {
+            Ok(GitClient::local(self.local_repository_path.unwrap()))
+        } else if self.github_repository.is_some()
+            && self.github_token_path.is_some()
+            && self.local_repository_path.is_none()
+        {
+            GitClient::github(
+                self.github_repository.unwrap(),
+                self.github_branch,
+                self.github_token_path.unwrap(),
+            )
+        } else {
+            Err(CliError::CommandArgumentError("Must provide either only --local-repository-path or both --github-repository and --github-token-path".to_string()))
+        }
+    }
+}
+
+/// A Git client for abstracting away local vs Github
+///
+/// Note: Writes do not commit locally
+pub enum GitClient {
+    Local(PathBuf),
+    Github(GithubClient),
+}
+
+impl GitClient {
+    pub fn local(path: PathBuf) -> GitClient {
+        GitClient::Local(path)
+    }
+
+    pub fn github(
+        repository: GithubRepo,
+        branch: String,
+        token_path: PathBuf,
+    ) -> CliTypedResult<GitClient> {
+        let token = Token::FromDisk(token_path).read_token()?;
+        Ok(GitClient::Github(GithubClient::new(
+            repository.owner,
+            repository.repository,
+            branch,
+            token,
+        )))
+    }
+
+    /// Retrieves an object as a YAML encoded file from the appropriate storage
+    pub fn get<T: DeserializeOwned>(&self, name: &str) -> CliTypedResult<T> {
+        match self {
+            GitClient::Local(local_repository_path) => {
+                let path = local_repository_path.join(format!("{}.yml", name));
+                let mut file = std::fs::File::open(path.clone())
+                    .map_err(|e| CliError::IO(path.display().to_string(), e))?;
+                let mut contents = String::new();
+                file.read_to_string(&mut contents)
+                    .map_err(|e| CliError::IO(path.display().to_string(), e))?;
+                from_yaml(&contents)
+            }
+            GitClient::Github(client) => {
+                from_base64_encoded_yaml(&client.get_file(&format!("{}.yml", name))?)
+            }
+        }
+    }
+
+    /// Puts an object as a YAML encoded file to the appropriate storage
+    pub fn put<T: Serialize + ?Sized>(&self, name: &str, input: &T) -> CliTypedResult<()> {
+        match self {
+            GitClient::Local(local_repository_path) => {
+                let path = local_repository_path.join(format!("{}.yml", name));
+                let mut file = if path.exists() {
+                    std::fs::File::open(path.clone())
+                        .map_err(|e| CliError::IO(path.display().to_string(), e))?
+                } else {
+                    std::fs::File::create(path.clone())
+                        .map_err(|e| CliError::IO(path.display().to_string(), e))?
+                };
+
+                file.write_all(to_yaml(input)?.as_bytes())
+                    .map_err(|e| CliError::IO(path.display().to_string(), e))?;
+            }
+            GitClient::Github(client) => {
+                client.put(&format!("{}.yml", name), &to_base64_encoded_yaml(input)?)?;
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn create_dir(&self, name: &str) -> CliTypedResult<()> {
+        match self {
+            GitClient::Local(local_repository_path) => {
+                let path = local_repository_path.join(name);
+                if path.exists() && path.is_dir() {
+                    // Do nothing
+                } else {
+                    std::fs::create_dir(path.clone())
+                        .map_err(|e| CliError::IO(path.display().to_string(), e))?
+                };
+            }
+            GitClient::Github(_) => {
+                // There's no such thing as an empty directory in Git, so do nothing
+            }
+        }
+
+        Ok(())
+    }
+}
+
+pub fn to_yaml<T: Serialize + ?Sized>(input: &T) -> CliTypedResult<String> {
+    Ok(serde_yaml::to_string(input)?)
+}
+
+pub fn from_yaml<T: DeserializeOwned>(input: &str) -> CliTypedResult<T> {
+    Ok(serde_yaml::from_str(input)?)
+}
+
+pub fn to_base64_encoded_yaml<T: Serialize + ?Sized>(input: &T) -> CliTypedResult<String> {
+    Ok(base64::encode(to_yaml(input)?))
+}
+
+pub fn from_base64_encoded_yaml<T: DeserializeOwned>(input: &str) -> CliTypedResult<T> {
+    from_yaml(&String::from_utf8(base64::decode(input)?)?)
+}