[data model] rendering Move object contents as JSON

- Upgrade Move version to import diem/move@1e5e36c. This also required some unrelated (but minor) updates in adapter.rs
- Use the new Move logic to add object.rs utility functions for (1) getting the type layout of a Move object, and (2) converting Move/FastX objects to JSON (which can be done once you have a type layout)

Note: this PR does not yet yet these utility functions via authority or client service API's, since that would require a fairly large refactoring. There are a few different approaches we could use here:
- Expose a new authority API for getting a `MoveStructLayout` given a `StructTag`. The client service can then use this to turn an `Object` into JSON locally, and expose its own API that does this. This is probably the path of least resistance, but it may require the client service to either store the `MoveTypeLayout` for all of its objects or frequently query the authority API to get layouts.
- Expose a new authority API for getting an `Object` in JSON instead of as a Rust `Object`. This is maybe a bit more convenient for the client service and other consumers of the authority API, but more work for the authority + perhaps muddies the waters around which API's it wants to use.
- Keep the authority API's as-is, but ask the client service to store (or repeatedly query for) all the Move packages relevant to its object types + their transitive dependencies. This would allow the client serive to comput its own `MoveStructLayout`. This is also a reasonable approach, and we'll eventually need it if we want to do local execution (since the VM will also need these packages).

fastpay/Cargo.toml

@@ -35,15 +35,15 @@ fastx-adapter = { path = "../fastx_programmability/adapter" }
 fastx-network = { path = "../network_utils" }
 fastx-types = { path = "../fastx_types" }
 
-move-package = { git = "https://github.com/diem/move", rev = "62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-core-types = { git = "https://github.com/diem/move", rev = "62b5a5075378ae6a7102bbfc1fb33b57ba6125d2", features = ["address20"] }
-move-bytecode-verifier = { git = "https://github.com/diem/move", rev = "62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-
 rustyline = "9.1.2"
 rustyline-derive = "0.6.0"
 colored = "2.0.0"
 unescape = "0.1.0"
 
+move-package = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-core-types = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5", features=["address20"] }
+move-bytecode-verifier = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+
 [[bin]]
 name = "client"
 path = "src/client.rs"
@@ -62,4 +62,4 @@ path = "src/wallet.rs"
 
 [[bin]]
 name = "fastx"
-path = "src/fastx.rs"
+path = "src/fastx.rs"

fastpay_core/Cargo.toml

@@ -27,10 +27,10 @@ fastx-framework = { path = "../fastx_programmability/framework" }
 fastx-network = { path = "../network_utils" }
 fastx-types = { path = "../fastx_types" }
 
-move-binary-format = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-core-types = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2", features=["address20"] }
-move-package = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-vm-runtime = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
+move-binary-format = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-core-types = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5", features=["address20"] }
+move-package = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-vm-runtime = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
 
 typed-store = { git = "https://github.com/MystenLabs/mysten-infra", rev = "d04c29e686aba380e6f9fc0c60a2dfdb974c5f8a"}
 

fastpay_core/src/authority_aggregator.rs

@@ -924,7 +924,7 @@ where
             .await?;
 
         let (_obj_ref, tx_digest) = object_map.keys().last().unwrap();
-        return Ok(transaction_map[tx_digest].clone());
+        Ok(transaction_map[tx_digest].clone())
     }
 
     /// Find the highest sequence number that is known to a quorum of authorities.

fastx_programmability/adapter/Cargo.toml

@@ -13,16 +13,17 @@ bcs = "0.1.3"
 once_cell = "1.9.0"
 structopt = "0.3.26"
 
-move-binary-format = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-bytecode-utils = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-bytecode-verifier = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-core-types = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2", features=["address20"] }
-move-cli = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-vm-runtime = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
+move-binary-format = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-bytecode-utils = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-bytecode-verifier = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-core-types = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5", features=["address20"] }
+move-cli = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-vm-runtime = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-vm-types = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
 
 fastx-framework = { path = "../framework" }
 fastx-verifier = { path = "../verifier" }
 fastx-types = { path = "../../fastx_types" }
 
 [dev-dependencies]
-move-package = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
+move-package = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }

fastx_programmability/adapter/src/adapter.rs

@@ -111,15 +111,17 @@ pub fn execute<E: Debug, S: ResourceResolver<Error = E> + ModuleResolver<Error =
     };
 
     // TODO: Update Move gas constants to reflect the gas fee on fastx.
-    let mut gas_status =
-        match get_gas_status(Some(gas_budget)).map_err(|e| FastPayError::GasBudgetTooHigh {
+    let cost_table = &move_vm_types::gas_schedule::INITIAL_COST_SCHEDULE;
+    let mut gas_status = match get_gas_status(cost_table, Some(gas_budget)).map_err(|e| {
+        FastPayError::GasBudgetTooHigh {
             error: e.to_string(),
-        }) {
-            Ok(ok) => ok,
-            Err(err) => {
-                exec_failure!(gas::MIN_MOVE_CALL_GAS, err);
-            }
-        };
+        }
+    }) {
+        Ok(ok) => ok,
+        Err(err) => {
+            exec_failure!(gas::MIN_MOVE_CALL_GAS, err);
+        }
+    };
     let session = vm.new_session(state_view);
     match session.execute_function_for_effects(
         &module_id,
@@ -253,7 +255,8 @@ pub fn verify_and_link<
     let vm = MoveVM::new(natives)
         .expect("VM creation only fails if natives are invalid, and we created the natives");
     // Note: VM does not do any gas metering on publish code path, so setting budget to None is fine
-    let mut gas_status = get_gas_status(None)
+    let cost_table = &move_vm_types::gas_schedule::INITIAL_COST_SCHEDULE;
+    let mut gas_status = get_gas_status(cost_table, None)
         .expect("Can only fail if gas budget is too high, and we didn't supply one");
     let mut session = vm.new_session(state_view);
     // TODO(https://github.com/MystenLabs/fastnft/issues/69): avoid this redundant serialization by exposing VM API that allows us to run the linker directly on `Vec<CompiledModule>`

fastx_programmability/framework/Cargo.toml

@@ -15,15 +15,15 @@ num_enum = "0.5.6"
 fastx-types = { path = "../../fastx_types" }
 fastx-verifier = { path = "../verifier" }
 
-move-binary-format = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-bytecode-verifier = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-cli = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-core-types = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2", features=["address20"] }
-move-package = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-stdlib = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-unit-test = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-vm-runtime = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-vm-types = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
+move-binary-format = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-bytecode-verifier = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-cli = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-core-types = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5", features=["address20"] }
+move-package = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-stdlib = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-unit-test = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-vm-runtime = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-vm-types = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
 
 
 [package.metadata.cargo-udeps.ignore]

fastx_programmability/verifier/Cargo.toml

@@ -8,8 +8,8 @@ license = "Apache-2.0"
 publish = false
 
 [dependencies]
-move-binary-format = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-bytecode-verifier = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-core-types = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2", features=["address20"] }
+move-binary-format = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-bytecode-verifier = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-core-types = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5", features=["address20"] }
 
 fastx-types = { path = "../../fastx_types" }

fastx_types/Cargo.toml

@@ -20,11 +20,16 @@ sha3 = "0.9"
 thiserror = "1.0.30"
 hex = "0.4.3"
 serde_bytes = "0.11.5"
+serde_json = "1.0.78"
 serde_with = "1.11.0"
+signature = "1.5.0"
 static_assertions = "1.1.0"
 
 typed-store = { git = "https://github.com/MystenLabs/mysten-infra", rev ="d04c29e686aba380e6f9fc0c60a2dfdb974c5f8a"}
 
-move-binary-format = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2" }
-move-core-types = { git = "https://github.com/diem/move", rev="62b5a5075378ae6a7102bbfc1fb33b57ba6125d2", features=["address20"] }
-signature = "1.5.0"
+move-binary-format = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-bytecode-utils = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-core-types = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5", features=["address20"] }
+move-disassembler = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+move-ir-types = { git = "https://github.com/diem/move", rev="1e5e36cb1fede8073c8688886a0943bb5bfe40d5" }
+

fastx_types/src/error.rs

@@ -200,6 +200,10 @@ pub enum FastPayError {
     )]
     QuorumNotReached { errors: Vec<FastPayError> },
 
+    // Errors returned by authority and client read API's
+    #[error("Failure serializing object in the requested format")]
+    ObjectSerializationError,
+
     // Client side error
     #[error("Client state has a different pending transaction.")]
     ConcurrentTransactionError,

fastx_types/src/object.rs

@@ -1,17 +1,25 @@
 // Copyright (c) Mysten Labs
 // SPDX-License-Identifier: Apache-2.0
 
+use move_binary_format::binary_views::BinaryIndexedView;
+use move_bytecode_utils::layout::TypeLayoutBuilder;
+use move_bytecode_utils::module_cache::GetModule;
+use move_core_types::language_storage::TypeTag;
+use move_core_types::value::{MoveStruct, MoveStructLayout, MoveTypeLayout};
+use move_disassembler::disassembler::Disassembler;
+use move_ir_types::location::Spanned;
 use serde::{Deserialize, Serialize};
 use serde_bytes::ByteBuf;
+use serde_json::{json, Value};
 use serde_with::{serde_as, Bytes};
 use std::collections::BTreeMap;
 use std::convert::{TryFrom, TryInto};
 use std::fmt::{Debug, Display, Formatter};
 
-use crate::error::FastPayError;
 use move_binary_format::CompiledModule;
 use move_core_types::{account_address::AccountAddress, language_storage::StructTag};
 
+use crate::error::FastPayError;
 use crate::{
     base_types::{
         sha3_hash, Authenticator, BcsSignable, FastPayAddress, ObjectDigest, ObjectID, ObjectRef,
@@ -39,6 +47,15 @@ const ID_END_INDEX: usize = AccountAddress::LENGTH;
 /// Index marking the end of the object's version + the beginning of type-specific data
 const VERSION_END_INDEX: usize = ID_END_INDEX + 8;
 
+/// Different schemes for converting a Move object to JSON
+pub struct ObjectFormatOptions {
+    /// If true, include the type of each object as well as its fields; e.g.:
+    /// `{ "fields": { "f": 20, "g": { "fields" { "h": true }, "type": "0x0::MyModule::MyNestedType" }, "type": "0x0::MyModule::MyType" }`
+    ///  If false, include field names only; e.g.:
+    /// `{ "f": 20, "g": { "h": true } }`
+    include_types: bool,
+}
+
 impl MoveObject {
     pub fn new(type_: StructTag, contents: Vec<u8>) -> Self {
         Self {
@@ -63,7 +80,6 @@ impl MoveObject {
         &self.contents[VERSION_END_INDEX..]
     }
 
-    ///
     pub fn id_version_contents(&self) -> &[u8] {
         &self.contents[..VERSION_END_INDEX]
     }
@@ -120,6 +136,36 @@ impl MoveObject {
     pub fn freeze(&mut self) {
         self.read_only = true;
     }
+
+    /// Get a `MoveStructLayout` for `self`.
+    /// The `resolver` value must contain the module that declares `self.type_` and the (transitive)
+    /// dependencies of `self.type_` in order for this to succeed. Failure will result in an `ObjectSerializationError`
+    pub fn get_layout(
+        &self,
+        format: ObjectFormatOptions,
+        resolver: &impl GetModule,
+    ) -> Result<MoveStructLayout, FastPayError> {
+        let type_ = TypeTag::Struct(self.type_.clone());
+        let layout = if format.include_types {
+            TypeLayoutBuilder::build_with_types(&type_, resolver)
+        } else {
+            TypeLayoutBuilder::build_with_fields(&type_, resolver)
+        }
+        .map_err(|_e| FastPayError::ObjectSerializationError)?;
+        match layout {
+            MoveTypeLayout::Struct(l) => Ok(l),
+            _ => unreachable!(
+                "We called build_with_types on Struct type, should get a struct layout"
+            ),
+        }
+    }
+
+    /// Convert `self` to the JSON representation dictated by `layout`.
+    pub fn to_json(&self, layout: &MoveStructLayout) -> Result<Value, FastPayError> {
+        let move_value = MoveStruct::simple_deserialize(&self.contents, layout)
+            .map_err(|_e| FastPayError::ObjectSerializationError)?;
+        serde_json::to_value(&move_value).map_err(|_e| FastPayError::ObjectSerializationError)
+    }
 }
 
 // TODO: Make MovePackage a NewType so that we can implement functions on it.
@@ -168,6 +214,38 @@ impl Data {
             Package(_) => None,
         }
     }
+
+    /// Convert `self` to the JSON representation dictated by `format`.
+    /// If `self` is a Move value, the `resolver` value must contain the module that declares `self.type_` and the (transitive)
+    /// dependencies of `self.type_` in order for this to succeed. Failure will result in an `ObjectSerializationError`
+    pub fn to_json(
+        &self,
+        format: ObjectFormatOptions,
+        resolver: &impl GetModule,
+    ) -> Result<Value, FastPayError> {
+        use Data::*;
+        match self {
+            Move(m) => {
+                let layout = m.get_layout(format, resolver)?;
+                m.to_json(&layout)
+            }
+            Package(p) => {
+                let mut disassembled = serde_json::Map::new();
+                for (name, bytecode) in p {
+                    let module = CompiledModule::deserialize(bytecode)
+                        .expect("Adapter publish flow ensures that this bytecode deserializes");
+                    let view = BinaryIndexedView::Module(&module);
+                    let d = Disassembler::from_view(view, Spanned::unsafe_no_loc(()).loc)
+                        .map_err(|_e| FastPayError::ObjectSerializationError)?;
+                    let bytecode_str = d
+                        .disassemble()
+                        .map_err(|_e| FastPayError::ObjectSerializationError)?;
+                    disassembled.insert(name.to_string(), Value::String(bytecode_str));
+                }
+                Ok(Value::Object(disassembled))
+            }
+        }
+    }
 }
 
 #[derive(Eq, PartialEq, Debug, Clone, Deserialize, Serialize, Hash)]
@@ -321,6 +399,22 @@ impl Object {
             previous_transaction: TransactionDigest::genesis(),
         }
     }
+
+    /// Convert `self` to the JSON representation dictated by `format`.
+    /// If `self` is a Move value, the `resolver` value must contain the module that declares `self.type_` and the (transitive)
+    /// dependencies of `self.type_` in order for this to succeed. Failure will result in an `ObjectSerializationError`
+    pub fn to_json(
+        &self,
+        format: ObjectFormatOptions,
+        resolver: &impl GetModule,
+    ) -> Result<Value, FastPayError> {
+        let contents = self.data.to_json(format, resolver)?;
+        let owner = serde_json::to_value(&self.owner)
+            .map_err(|_e| FastPayError::ObjectSerializationError)?;
+        let previous_transaction = serde_json::to_value(&self.previous_transaction)
+            .map_err(|_e| FastPayError::ObjectSerializationError)?;
+        Ok(json!({ "contents": contents, "owner": owner, "tx_digest": previous_transaction }))
+    }
 }
 
 pub enum ObjectRead {