[node config] allow initial set of Move modules to be chosen at genesis

The set of Move modules published at genesis is currently hardcoded. This makes it difficult to reuse the Diem software in a setting that requires a different set of initial Move modules or a different genesis state.

This PR adds the bytecode for initial Move modules (represented as list of hex strings) to the `layout` part of the genesis builder, then creates the genesis transaction using this list.

Closes: aptos-labs#8726

config/global-constants/src/lib.rs

@@ -25,3 +25,4 @@ pub const VALIDATOR_NETWORK_KEY: &str = "validator_network";
 pub const SAFETY_DATA: &str = "safety_data";
 pub const WAYPOINT: &str = "waypoint";
 pub const GENESIS_WAYPOINT: &str = "genesis-waypoint";
+pub const MOVE_MODULES: &str = "move-modules";

config/management/genesis/Cargo.toml

@@ -23,6 +23,7 @@ generate-key = { path = "../../generate-key" }
 bcs = "0.1.2"
 diem-config = { path = "../.."}
 diem-crypto = { path = "../../../crypto/crypto" }
+diem-framework-releases = { path = "../../../language/diem-framework/releases"}
 diem-global-constants = { path = "../../global-constants" }
 diem-management = { path = ".." }
 diem-network-address-encryption = {path = "../network-address-encryption"}

config/management/genesis/README.md

@@ -4,16 +4,17 @@ The `diem-genesis-tool` provides a tool for the genesis ceremony of the Diem blo
 
 * A diem root account that maintains the set of validator owners, validator operators, and the active validator set.
 * A treasury compliance account that maintains VASPs, DDs, and other related topics.
+* The initial set of Move modules published at genesis.
 * Validator owners (OW) that have accounts on the blockchain. These accounts contain a validator configuration and specify a validator operator.
 * Validator operators (OP) that have accounts on the blockchain. These accounts have the ability to manipulate validator configuration.
-
 ## Generating Genesis
 
 The process for starting organization of the planned and current functionality includes:
 
 * Initialization ceremony
   * The association sets up a secure-backend for data uploads, `shared storage`, e.g., GitHub.  The association then distributes credentials for each OW and OP.
   * The association generates its `diem root key` and shares the public key to the `shared storage`.
+  * The association uploads the initial set of Move modules to `shared storage`.
   * Each OW will generate a private `owner key` and share the public key to the `shared storage`.
   * Each OP will generate a private `operator key` and share the public key to the `shared storage`.
 * Validator initialization
@@ -98,13 +99,21 @@ cargo run -p diem-genesis-tool -- \
     --config config_file.yaml \
     --path $PATH_TO_LAYOUT
 ```
+* The association will publish the initial set of Move modules to `shared storage`:
+```
+cargo run -p diem-genesis-tool -- \
+    set-move-modules \
+    --config config_file.yaml \
+    --dir $MOVE_MODULES_DIR
+```
+This should be a directory containing only Move bytecode files (`.mv` extension).
 * The association will publish the the `diem root`  public key to the `shared storage`:
 ```
 cargo run -p diem-genesis-tool -- \
     diem-root-key \
     --config config_file.yaml
 ```
-* The association will publish the the `diem root`  public key to the `shared storage`:
+* The association will publish the the `treasury compliance` public key to the `shared storage`:
 ```
 cargo run -p diem-genesis-tool -- \
     diem-treasury-compliance-key \

config/management/genesis/src/builder.rs

@@ -4,7 +4,9 @@
 use crate::layout::Layout;
 use anyhow::Result;
 use diem_crypto::ed25519::Ed25519PublicKey;
-use diem_global_constants::{DIEM_ROOT_KEY, OPERATOR_KEY, OWNER_KEY, TREASURY_COMPLIANCE_KEY};
+use diem_global_constants::{
+    DIEM_ROOT_KEY, MOVE_MODULES, OPERATOR_KEY, OWNER_KEY, TREASURY_COMPLIANCE_KEY,
+};
 use diem_management::constants::{self, VALIDATOR_CONFIG, VALIDATOR_OPERATOR};
 use diem_secure_storage::{KVStorage, Namespaced};
 use diem_types::{
@@ -48,6 +50,19 @@ impl<S: KVStorage> GenesisBuilder<S> {
         Layout::parse(&raw_layout).map_err(Into::into)
     }
 
+    pub fn set_move_modules(&mut self, modules: Vec<Vec<u8>>) -> Result<()> {
+        self.with_namespace_mut(constants::COMMON_NS)
+            .set(MOVE_MODULES, modules)
+            .map_err(Into::into)
+    }
+
+    pub fn move_modules(&self) -> Result<Vec<Vec<u8>>> {
+        self.with_namespace(constants::COMMON_NS)
+            .get(MOVE_MODULES)
+            .map(|r| r.value)
+            .map_err(Into::into)
+    }
+
     pub fn set_root_key(&mut self, root_key: Ed25519PublicKey) -> Result<()> {
         let layout = self.layout()?;
         self.with_namespace_mut(&layout.diem_root)
@@ -199,6 +214,7 @@ impl<S: KVStorage> GenesisBuilder<S> {
         let diem_root_key = self.root_key()?;
         let treasury_compliance_key = self.treasury_compliance_key()?;
         let validators = self.validators()?;
+        let move_modules = self.move_modules()?;
 
         // Only have an allowlist of stdlib scripts
         let script_policy = None;
@@ -207,6 +223,7 @@ impl<S: KVStorage> GenesisBuilder<S> {
             diem_root_key,
             treasury_compliance_key,
             &validators,
+            &move_modules,
             script_policy,
             chain_id,
         );

config/management/genesis/src/command.rs

@@ -22,6 +22,8 @@ pub enum Command {
     OwnerKey(crate::key::OwnerKey),
     #[structopt(about = "Submits a Layout doc to a shared storage")]
     SetLayout(crate::layout::SetLayout),
+    #[structopt(about = "Submits Move module bytecodes to a shared storage")]
+    SetMoveModules(crate::move_modules::SetMoveModules),
     #[structopt(about = "Sets the validator operator chosen by the owner")]
     SetOperator(crate::validator_operator::ValidatorOperator),
     #[structopt(about = "Submits an Ed25519PublicKey for the treasury root")]
@@ -41,6 +43,7 @@ pub enum CommandName {
     OperatorKey,
     OwnerKey,
     SetLayout,
+    SetMoveModules,
     SetOperator,
     TreasuryComplianceKey,
     ValidatorConfig,
@@ -57,6 +60,7 @@ impl From<&Command> for CommandName {
             Command::OperatorKey(_) => CommandName::OperatorKey,
             Command::OwnerKey(_) => CommandName::OwnerKey,
             Command::SetLayout(_) => CommandName::SetLayout,
+            Command::SetMoveModules(_) => CommandName::SetMoveModules,
             Command::SetOperator(_) => CommandName::SetOperator,
             Command::TreasuryComplianceKey(_) => CommandName::TreasuryComplianceKey,
             Command::ValidatorConfig(_) => CommandName::ValidatorConfig,
@@ -75,6 +79,7 @@ impl std::fmt::Display for CommandName {
             CommandName::OperatorKey => "operator-key",
             CommandName::OwnerKey => "owner-key",
             CommandName::SetLayout => "set-layout",
+            CommandName::SetMoveModules => "set-move-modules",
             CommandName::SetOperator => "set-operator",
             CommandName::TreasuryComplianceKey => "treasury-compliance-key",
             CommandName::ValidatorConfig => "validator-config",
@@ -96,6 +101,7 @@ impl Command {
             Command::OperatorKey(_) => self.operator_key().map(|_| "Success!".to_string()),
             Command::OwnerKey(_) => self.owner_key().map(|_| "Success!".to_string()),
             Command::SetLayout(_) => self.set_layout().map(|_| "Success!".to_string()),
+            Command::SetMoveModules(_) => self.set_move_modules().map(|_| "Success!".to_string()),
             Command::SetOperator(_) => self.set_operator().map(|_| "Success!".to_string()),
             Command::TreasuryComplianceKey(_) => self
                 .treasury_compliance_key()
@@ -133,6 +139,10 @@ impl Command {
         execute_command!(self, Command::SetLayout, CommandName::SetLayout)
     }
 
+    pub fn set_move_modules(self) -> Result<Vec<Vec<u8>>, Error> {
+        execute_command!(self, Command::SetMoveModules, CommandName::SetMoveModules)
+    }
+
     pub fn set_operator(self) -> Result<String, Error> {
         execute_command!(self, Command::SetOperator, CommandName::SetOperator)
     }
@@ -213,7 +223,23 @@ pub mod tests {
             .set_layout(temppath.path().to_str().unwrap())
             .unwrap();
 
-        // Step 2) Upload the root keys:
+        // Step 2) Upload the Move modules
+        let tempdir = diem_temppath::TempPath::new();
+        tempdir.create_as_dir().unwrap();
+        for b in diem_framework_releases::current_module_blobs() {
+            let mut temppath =
+                diem_temppath::TempPath::new_with_temp_dir(tempdir.path().to_path_buf());
+            temppath.create_as_file().unwrap();
+            temppath.persist(); // otherwise, file will disappear before we call set_move_modules
+            let mut file = File::create(temppath.path()).unwrap();
+            file.write_all(b).unwrap();
+            file.sync_all().unwrap();
+        }
+        helper
+            .set_move_modules(tempdir.path().to_str().unwrap())
+            .unwrap();
+
+        // Step 3) Upload the root keys:
         helper.initialize_by_idx(dave_ns.into(), storage_idx);
         helper
             .diem_root_key(dave_ns, &(dave_ns.to_string() + shared))
@@ -222,7 +248,7 @@ pub mod tests {
             .treasury_compliance_key(dave_ns, &(dave_ns.to_string() + shared))
             .unwrap();
 
-        // Step 3) Upload each owner key (except carol, she'll have auth_key [0; 32]):
+        // Step 4) Upload each owner key (except carol, she'll have auth_key [0; 32]):
         for ns in [alice_ns, bob_ns, carol_ns].iter() {
             let ns = (*ns).to_string();
             let ns_shared = (*ns).to_string() + shared;
@@ -234,7 +260,7 @@ pub mod tests {
             }
         }
 
-        // Step 4) Upload each operator key:
+        // Step 5) Upload each operator key:
         for ns in [operator_alice_ns, operator_bob_ns, operator_carol_ns].iter() {
             let ns = (*ns).to_string();
             let ns_shared = (*ns).to_string() + shared;
@@ -244,15 +270,15 @@ pub mod tests {
             helper.operator_key(&ns, &ns_shared).unwrap();
         }
 
-        // Step 5) Set the operator for each owner:
+        // Step 6) Set the operator for each owner:
         for ns in [alice_ns, bob_ns, carol_ns].iter() {
             let ns_shared = (*ns).to_string() + shared;
 
             let operator_name = format!("operator_{}", ns_shared);
             helper.set_operator(&operator_name, &ns_shared).unwrap();
         }
 
-        // Step 6) Upload a signed validator config transaction for each operator:
+        // Step 7) Upload a signed validator config transaction for each operator:
         for ns in [operator_alice_ns, operator_bob_ns, operator_carol_ns].iter() {
             let ns = (*ns).to_string();
             let ns_shared = (*ns).to_string() + shared;
@@ -271,7 +297,7 @@ pub mod tests {
                 .unwrap();
         }
 
-        // Step 7) Produce genesis
+        // Step 8) Produce genesis
         let genesis_path = diem_temppath::TempPath::new();
         genesis_path.create_as_file().unwrap();
         helper

config/management/genesis/src/config_builder.rs

@@ -121,9 +121,12 @@ impl BuildSwarm for FullnodeBuilder {
 pub fn test_config() -> (NodeConfig, Ed25519PrivateKey) {
     let path = TempPath::new();
     path.create_as_dir().unwrap();
-    let builder = ValidatorBuilder::new(path.path())
-        .num_validators(1)
-        .template(NodeConfig::default_for_validator());
+    let builder = ValidatorBuilder::new(
+        path.path(),
+        diem_framework_releases::current_module_blobs().to_vec(),
+    )
+    .num_validators(1)
+    .template(NodeConfig::default_for_validator());
     let (mut configs, key) = builder.build_swarm().unwrap();
 
     let mut config = configs.swap_remove(0);

config/management/genesis/src/lib.rs

@@ -8,6 +8,7 @@ pub mod command;
 mod genesis;
 mod key;
 pub mod layout;
+mod move_modules;
 pub mod validator_builder;
 mod validator_config;
 mod validator_operator;

config/management/genesis/src/move_modules.rs

@@ -0,0 +1,54 @@
+// Copyright (c) The Diem Core Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::builder::GenesisBuilder;
+use diem_management::{config::ConfigPath, error::Error, secure_backend::SharedBackend};
+use diem_secure_storage::Storage;
+use std::{fs, path::PathBuf};
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub struct SetMoveModules {
+    #[structopt(flatten)]
+    config: ConfigPath,
+    // Directory containing Move bytecode (.mv) files to use in genesis
+    #[structopt(long)]
+    dir: PathBuf,
+    #[structopt(flatten)]
+    backend: SharedBackend,
+}
+
+impl SetMoveModules {
+    pub fn execute(self) -> Result<Vec<Vec<u8>>, Error> {
+        let mut move_modules = vec![];
+        // collect all Move bytecode files located immediately under self.dir
+        for dir_entry in
+            fs::read_dir(self.dir.clone()).map_err(|e| Error::UnexpectedError(e.to_string()))?
+        {
+            let path = dir_entry
+                .map_err(|e| Error::UnexpectedError(e.to_string()))?
+                .path();
+            if path.is_dir() {
+                return Err(Error::UnexpectedError(format!(
+                    "Subdirectory {:?} found under Move bytecode modules directory. All bytecode files must be located directly under the modules directory {:?}", path, self.dir)));
+            }
+            move_modules.push(fs::read(path).map_err(|e| Error::UnexpectedError(e.to_string()))?)
+        }
+        let config = self
+            .config
+            .load()?
+            .override_shared_backend(&self.backend.shared_backend)?;
+
+        // In order to not break cli compatibility we need to clear the namespace set via cli since
+        // it was previously ignored.
+        let mut shared_backend = config.shared_backend;
+        shared_backend.clear_namespace();
+
+        let storage = Storage::from(&shared_backend);
+        GenesisBuilder::new(storage)
+            .set_move_modules(move_modules.clone())
+            .map_err(|e| Error::StorageWriteError("shared", "move_modules", e.to_string()))?;
+
+        Ok(move_modules)
+    }
+}

config/management/genesis/src/storage_helper.rs

@@ -256,6 +256,26 @@ impl StorageHelper {
         command.set_layout()
     }
 
+    #[cfg(test)]
+    pub fn set_move_modules(&self, dir: &str) -> Result<Vec<Vec<u8>>, Error> {
+        println!("setting move modules with dir {}", dir);
+        let args = format!(
+            "
+                diem-genesis-tool
+                set-move-modules
+                --dir {dir}
+                --shared-backend backend={backend};\
+                    path={storage_path}
+            ",
+            dir = dir,
+            backend = DISK,
+            storage_path = self.path_string(),
+        );
+
+        let command = Command::from_iter(args.split_whitespace());
+        command.set_move_modules()
+    }
+
     pub fn set_operator(&self, operator_name: &str, shared_ns: &str) -> Result<String, Error> {
         let args = format!(
             "