[objects] client stores object layouts and uses them to pretty-print …

…objects

- Add layouts to the client store
- By default, get_object requests ask for the object layout
- Object layouts from the response get propagated to the store
- When the CLI prints an object with --deep, it uses the layout (if available) to render the full object contents in JSON. We will need to do something similar in the REST API for get_object, but that is not yet in `main`

sui/src/wallet_commands.rs

@@ -166,9 +166,11 @@ impl WalletCommands {
                 let client_state = context.get_or_create_client_state(owner)?;
                 let object_read = client_state.get_object_info(*id).await?;
                 let object = object_read.object()?;
-                println!("{}", object);
                 if *deep {
-                    println!("Full Info: {:#?}", object);
+                    let layout = object_read.layout()?;
+                    println!("{}", object.to_json(layout)?);
+                } else {
+                    println!("{}", object);
                 }
             }
             WalletCommands::Call {

sui_core/src/authority_aggregator.rs

@@ -5,7 +5,8 @@ use crate::authority_client::AuthorityAPI;
 use crate::safe_client::SafeClient;
 
 use futures::{future, StreamExt};
-use sui_types::object::{Object, ObjectRead};
+use move_core_types::value::MoveStructLayout;
+use sui_types::object::{Object, ObjectFormatOptions, ObjectRead};
 use sui_types::{
     base_types::*,
     committee::Committee,
@@ -339,7 +340,11 @@ where
         (
             BTreeMap<
                 (ObjectRef, TransactionDigest),
-                (Option<Object>, Vec<(AuthorityName, Option<SignedOrder>)>),
+                (
+                    Option<Object>,
+                    Option<MoveStructLayout>,
+                    Vec<(AuthorityName, Option<SignedOrder>)>,
+                ),
             >,
             HashMap<TransactionDigest, CertifiedOrder>,
         ),
@@ -398,7 +403,11 @@ where
         let mut error_list = Vec::new();
         let mut object_map = BTreeMap::<
             (ObjectRef, TransactionDigest),
-            (Option<Object>, Vec<(AuthorityName, Option<SignedOrder>)>),
+            (
+                Option<Object>,
+                Option<MoveStructLayout>,
+                Vec<(AuthorityName, Option<SignedOrder>)>,
+            ),
         >::new();
         let mut certificates = HashMap::new();
 
@@ -427,8 +436,7 @@ where
                 // None if the object was deleted at this authority
                 //
                 // NOTE: here we could also be gathering the locked orders to see if we could make a cert.
-                // TODO: pass along layout_option so the client can store it
-                let (object_option, signed_order_option, _layout_option) =
+                let (object_option, signed_order_option, layout_option) =
                     if let Some(ObjectResponse {
                         object,
                         lock,
@@ -443,8 +451,8 @@ where
                 // Update the map with the information from this authority
                 let entry = object_map
                     .entry((object_ref, transaction_digest))
-                    .or_insert((object_option, Vec::new()));
-                entry.1.push((name, signed_order_option));
+                    .or_insert((object_option, layout_option, Vec::new()));
+                entry.2.push((name, signed_order_option));
 
                 if let Some(cert) = cert_option {
                     certificates.insert(cert.order.digest(), cert);
@@ -545,7 +553,7 @@ where
         timeout_after_quorum: Duration,
     ) -> Result<
         (
-            Vec<(Object, Option<CertifiedOrder>)>,
+            Vec<(Object, Option<MoveStructLayout>, Option<CertifiedOrder>)>,
             Vec<(ObjectRef, Option<CertifiedOrder>)>,
         ),
         SuiError,
@@ -572,8 +580,10 @@ where
             // If more that one version of an object is available, we update all authorities with it.
             while !aggregate_object_info.is_empty() {
                 // This will be the very latest object version, because object_ref is ordered this way.
-                let ((object_ref, transaction_digest), (object_option, object_authorities)) =
-                    aggregate_object_info.pop().unwrap(); // safe due to check above
+                let (
+                    (object_ref, transaction_digest),
+                    (object_option, layout_option, object_authorities),
+                ) = aggregate_object_info.pop().unwrap(); // safe due to check above
 
                 // NOTE: Here we must check that the object is indeed an input to this transaction
                 //       but for the moment lets do the happy case.
@@ -584,7 +594,7 @@ where
                     //       Otherwise report the authority as potentially faulty.
 
                     if let Some(obj) = object_option {
-                        active_objects.push((obj, None));
+                        active_objects.push((obj, layout_option, None));
                     }
                     // Cannot be that the genesis contributes to deleted objects
 
@@ -611,7 +621,7 @@ where
 
                 // Return the latest version of an object, or a deleted object
                 match object_option {
-                    Some(obj) => active_objects.push((obj, Some(cert))),
+                    Some(obj) => active_objects.push((obj, layout_option, Some(cert))),
                     None => deleted_objects.push((object_ref, Some(cert))),
                 }
 
@@ -654,7 +664,7 @@ where
         timeout_after_quorum: Duration,
     ) -> Result<
         (
-            Vec<(Object, Option<CertifiedOrder>)>,
+            Vec<(Object, Option<MoveStructLayout>, Option<CertifiedOrder>)>,
             Vec<(ObjectRef, Option<CertifiedOrder>)>,
         ),
         SuiError,
@@ -973,7 +983,9 @@ where
             .await?;
         let mut object_ref_stack: Vec<_> = object_map.into_iter().collect();
 
-        while let Some(((obj_ref, tx_digest), (obj_option, authorities))) = object_ref_stack.pop() {
+        while let Some(((obj_ref, tx_digest), (obj_option, layout_option, authorities))) =
+            object_ref_stack.pop()
+        {
             let stake: usize = authorities
                 .iter()
                 .map(|(name, _)| self.committee.weight(name))
@@ -985,7 +997,11 @@ where
                     return Ok(ObjectRead::Deleted(obj_ref));
                 } else {
                     // safe due to check
-                    return Ok(ObjectRead::Exists(obj_ref, obj_option.unwrap()));
+                    return Ok(ObjectRead::Exists(
+                        obj_ref,
+                        obj_option.unwrap(),
+                        layout_option,
+                    ));
                 }
             } else if cert_map.contains_key(&tx_digest) {
                 // If we have less stake telling us about the latest state of an object
@@ -1035,7 +1051,11 @@ where
                         return Ok(ObjectRead::Deleted(obj_ref));
                     } else {
                         // safe due to check
-                        return Ok(ObjectRead::Exists(obj_ref, obj_option.unwrap()));
+                        return Ok(ObjectRead::Exists(
+                            obj_ref,
+                            obj_option.unwrap(),
+                            layout_option,
+                        ));
                     }
                 }
             }
@@ -1078,8 +1098,8 @@ where
         let request = ObjectInfoRequest {
             object_id,
             request_sequence_number: None,
-            // TODO: allow caller to specify layout
-            request_layout: None,
+            // TODO: allow caller to decide whether they want the layout, and which options. For now, we always ask, and get the default format
+            request_layout: Some(ObjectFormatOptions::default()),
         };
 
         // For now assume all authorities. Assume they're all honest

sui_core/src/client.rs

@@ -578,7 +578,7 @@ where
             .sync_all_owned_objects(self.address, Duration::from_secs(60))
             .await?;
 
-        for (object, option_cert) in active_object_certs {
+        for (object, option_layout, option_cert) in active_object_certs {
             let object_ref = object.to_object_reference();
             let (object_id, sequence_number, _) = object_ref;
             self.store
@@ -590,6 +590,13 @@ where
                     .certificates
                     .insert(&cert.order.digest(), &cert)?;
             }
+            // Save the object layout, if any
+            if let Some(layout) = option_layout {
+                if let Some(type_) = object.type_() {
+                    // TODO: sanity check to add: if we're overwriting an old layout, it should be the same as the new one
+                    self.store.object_layouts.insert(type_, &layout)?;
+                }
+            }
         }
 
         Ok(())

sui_core/src/client/client_store.rs

@@ -1,4 +1,6 @@
 use super::*;
+use move_core_types::language_storage::StructTag;
+use move_core_types::value::MoveStructLayout;
 use rocksdb::{DBWithThreadMode, MultiThreaded};
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -11,6 +13,7 @@ const OBJ_REF_CF_NAME: &str = "object_refs";
 const TX_DIGEST_TO_CERT_CF_NAME: &str = "object_certs";
 const PENDING_ORDERS_CF_NAME: &str = "pending_orders";
 const OBJECT_CF_NAME: &str = "objects";
+const OBJECT_LAYOUTS_CF_NAME: &str = "object_layouts";
 
 const MANAGED_ADDRESS_PATHS_CF_NAME: &str = "managed_address_paths";
 const MANAGED_ADDRESS_SUBDIR: &str = "managed_addresses";
@@ -103,6 +106,10 @@ pub struct ClientSingleAddressStore {
     /// Map from object ref to actual object to track object history
     /// There can be duplicates and we never delete objects
     pub objects: DBMap<ObjectRef, Object>,
+    /// Map from Move object type to the layout of the object's Move contents.
+    /// Should contain the layouts for all object types in `objects`, and
+    /// possibly more.
+    pub object_layouts: DBMap<StructTag, MoveStructLayout>,
 }
 
 impl ClientSingleAddressStore {
@@ -117,6 +124,7 @@ impl ClientSingleAddressStore {
                 OBJ_REF_CF_NAME,
                 TX_DIGEST_TO_CERT_CF_NAME,
                 OBJECT_CF_NAME,
+                OBJECT_LAYOUTS_CF_NAME,
             ],
         );
 
@@ -127,6 +135,7 @@ impl ClientSingleAddressStore {
             object_refs: client_store::reopen_db(&db, OBJ_REF_CF_NAME),
             object_certs: client_store::reopen_db(&db, TX_DIGEST_TO_CERT_CF_NAME),
             objects: client_store::reopen_db(&db, OBJECT_CF_NAME),
+            object_layouts: client_store::reopen_db(&db, OBJECT_LAYOUTS_CF_NAME),
         }
     }
 }

sui_core/src/unit_tests/client_tests.rs

@@ -286,7 +286,7 @@ async fn init_local_authorities(
 }
 
 #[cfg(test)]
-fn init_local_authorities_bad_1(
+async fn init_local_authorities_bad_1(
     count: usize,
 ) -> (BTreeMap<AuthorityName, LocalAuthorityClient>, Committee) {
     let mut key_pairs = Vec::new();
@@ -313,12 +313,13 @@ fn init_local_authorities_bad_1(
         let mut opts = rocksdb::Options::default();
         opts.set_max_open_files(max_files_client_tests());
         let store = Arc::new(AuthorityStore::open(path, Some(opts)));
-        let state = AuthorityState::new_without_genesis_for_testing(
+        let state = AuthorityState::new_with_genesis_modules(
             committee.clone(),
             address,
             Box::pin(secret),
             store,
-        );
+        )
+        .await;
         clients.insert(address, LocalAuthorityClient::new(state));
     }
     (clients, committee)
@@ -424,7 +425,7 @@ async fn init_local_client_state(
 async fn init_local_client_state_with_bad_authority(
     object_ids: Vec<Vec<ObjectID>>,
 ) -> ClientState<LocalAuthorityClient> {
-    let (authority_clients, committee) = init_local_authorities_bad_1(object_ids.len());
+    let (authority_clients, committee) = init_local_authorities_bad_1(object_ids.len()).await;
     let mut client = make_client(authority_clients.clone(), committee);
     fund_account(
         authority_clients.values().collect(),
@@ -2046,7 +2047,7 @@ async fn test_sync_all_owned_objects() {
         2,
         owned_object
             .iter()
-            .filter(|(o, _)| o.owner.is_address(&client1.address()))
+            .filter(|(o, _, _)| o.owner.is_address(&client1.address()))
             .count()
     );
 }
@@ -2309,7 +2310,7 @@ async fn test_address_manager() {
 
     // Try adding new addresses to manage
     let (address, secret) = get_key_pair();
-    let secret2 = secret.copy();
+    let _secret2 = secret.copy();
     let secret = Box::pin(secret);
     let (authority_clients, committee) = init_local_authorities(4).await;
     let gas_object1 = ObjectID::random();

sui_types/src/coin.rs

@@ -5,6 +5,7 @@ use move_core_types::{
     ident_str,
     identifier::IdentStr,
     language_storage::{StructTag, TypeTag},
+    value::{MoveFieldLayout, MoveStructLayout, MoveTypeLayout},
 };
 use serde::{Deserialize, Serialize};
 
@@ -53,4 +54,17 @@ impl Coin {
     pub fn to_bcs_bytes(&self) -> Vec<u8> {
         bcs::to_bytes(&self).unwrap()
     }
+
+    pub fn layout(type_param: StructTag) -> MoveStructLayout {
+        MoveStructLayout::WithTypes {
+            type_: Self::type_(type_param),
+            fields: vec![
+                MoveFieldLayout::new(
+                    ident_str!("id").to_owned(),
+                    MoveTypeLayout::Struct(ID::layout()),
+                ),
+                MoveFieldLayout::new(ident_str!("value").to_owned(), MoveTypeLayout::U64),
+            ],
+        }
+    }
 }

sui_types/src/gas_coin.rs

@@ -5,6 +5,7 @@ use move_core_types::{
     ident_str,
     identifier::IdentStr,
     language_storage::{StructTag, TypeTag},
+    value::MoveStructLayout,
 };
 use serde::{Deserialize, Serialize};
 use std::convert::{TryFrom, TryInto};
@@ -69,6 +70,10 @@ impl GasCoin {
     pub fn to_object(&self) -> MoveObject {
         MoveObject::new(Self::type_(), self.to_bcs_bytes())
     }
+
+    pub fn layout() -> MoveStructLayout {
+        Coin::layout(Self::type_())
+    }
 }
 impl TryFrom<&MoveObject> for GasCoin {
     type Error = SuiError;

sui_types/src/id.rs

@@ -1,7 +1,12 @@
 // Copyright (c) Mysten Labs
 // SPDX-License-Identifier: Apache-2.0
 
-use move_core_types::{ident_str, identifier::IdentStr, language_storage::StructTag};
+use move_core_types::{
+    ident_str,
+    identifier::IdentStr,
+    language_storage::StructTag,
+    value::{MoveFieldLayout, MoveStructLayout, MoveTypeLayout},
+};
 use serde::{Deserialize, Serialize};
 
 use crate::{
@@ -11,6 +16,7 @@ use crate::{
 
 pub const ID_MODULE_NAME: &IdentStr = ident_str!("ID");
 pub const ID_STRUCT_NAME: &IdentStr = ID_MODULE_NAME;
+pub const ID_BYTES_STRUCT_NAME: &IdentStr = ident_str!("IDBytes");
 
 /// Rust version of the Move FastX::ID::ID type
 #[derive(Debug, Serialize, Deserialize)]
@@ -53,10 +59,42 @@ impl ID {
     pub fn to_bcs_bytes(&self) -> Vec<u8> {
         bcs::to_bytes(&self).unwrap()
     }
+
+    pub fn layout() -> MoveStructLayout {
+        MoveStructLayout::WithTypes {
+            type_: Self::type_(),
+            fields: vec![
+                MoveFieldLayout::new(
+                    ident_str!("id").to_owned(),
+                    MoveTypeLayout::Struct(IDBytes::layout()),
+                ),
+                MoveFieldLayout::new(ident_str!("version").to_owned(), MoveTypeLayout::U64),
+            ],
+        }
+    }
 }
 
 impl IDBytes {
     pub fn new(bytes: ObjectID) -> Self {
         Self { bytes }
     }
+
+    pub fn type_() -> StructTag {
+        StructTag {
+            address: SUI_FRAMEWORK_ADDRESS,
+            name: ID_BYTES_STRUCT_NAME.to_owned(),
+            module: ID_MODULE_NAME.to_owned(),
+            type_params: Vec::new(),
+        }
+    }
+
+    pub fn layout() -> MoveStructLayout {
+        MoveStructLayout::WithTypes {
+            type_: Self::type_(),
+            fields: vec![MoveFieldLayout::new(
+                ident_str!("bytes").to_owned(),
+                MoveTypeLayout::Vector(Box::new(MoveTypeLayout::U8)),
+            )],
+        }
+    }
 }