[Storage] RIP AccountStateBlob

Closes: aptos-labs#853

api/src/accounts.rs

@@ -20,12 +20,12 @@ use aptos_types::{
 };
 
 use anyhow::Result;
-use aptos_types::{account_state_blob::AccountStateBlob, account_view::AccountView};
+use aptos_types::account_view::AccountView;
 use move_core_types::{
     identifier::Identifier, language_storage::StructTag, move_resource::MoveStructType,
     value::MoveValue,
 };
-use std::convert::{TryFrom, TryInto};
+use std::convert::TryInto;
 use warp::{filters::BoxedFilter, Filter, Rejection, Reply};
 
 // GET /accounts/<address>
@@ -38,16 +38,6 @@ pub fn get_account(context: Context) -> BoxedFilter<(impl Reply,)> {
         .boxed()
 }
 
-// GET /accounts/<address>/blob
-pub fn get_account_state_blob(context: Context) -> BoxedFilter<(impl Reply,)> {
-    warp::path!("accounts" / AddressParam / "blob")
-        .and(warp::get())
-        .and(context.filter())
-        .and_then(handle_get_account_state_blob)
-        .with(metrics("get_account_state_blob"))
-        .boxed()
-}
-
 // GET /accounts/<address>/resources
 pub fn get_account_resources(context: Context) -> BoxedFilter<(impl Reply,)> {
     warp::path!("accounts" / AddressParam / "resources")
@@ -82,14 +72,6 @@ async fn handle_get_account(
     Ok(Account::new(None, address, context)?.account()?)
 }
 
-async fn handle_get_account_state_blob(
-    address: AddressParam,
-    context: Context,
-) -> Result<impl Reply, Rejection> {
-    fail_point("endpoint_get_account_state_blob")?;
-    Ok(Account::new(None, address, context)?.account_state_blob()?)
-}
-
 async fn handle_get_account_resources(
     ledger_version: Option<LedgerVersionParam>,
     address: AddressParam,
@@ -152,15 +134,6 @@ impl Account {
         Response::new(self.latest_ledger_info, &account)
     }
 
-    pub fn account_state_blob(self) -> Result<impl Reply, Error> {
-        let state = self
-            .context
-            .get_account_state(self.address.into(), self.ledger_version)?
-            .ok_or_else(|| self.account_not_found())?;
-        let blob: Vec<u8> = AccountStateBlob::try_from(&state)?.into();
-        Response::new(self.latest_ledger_info, &blob)
-    }
-
     pub fn resources(self) -> Result<impl Reply, Error> {
         let resources = self
             .context

api/src/index.rs

@@ -31,7 +31,6 @@ pub fn routes(context: Context) -> impl Filter<Extract = impl Reply, Error = Inf
         .or(accounts::get_account(context.clone()))
         .or(accounts::get_account_resources(context.clone()))
         .or(accounts::get_account_modules(context.clone()))
-        .or(accounts::get_account_state_blob(context.clone()))
         .or(transactions::get_transaction(context.clone()))
         .or(transactions::get_transactions(context.clone()))
         .or(transactions::get_account_transactions(context.clone()))

api/src/tests/accounts_test.rs

@@ -61,17 +61,6 @@ async fn test_account_resources_response() {
     context.check_golden_output(resp);
 }
 
-// Unstable due to framework changes
-#[ignore]
-#[tokio::test]
-async fn test_account_state_blob_response() {
-    let mut context = new_test_context(current_function_name!());
-    let address = "0xA550C18";
-
-    let resp = context.get(&account_blob(address)).await;
-    context.check_golden_output(resp);
-}
-
 // Unstable due to framework changes
 #[ignore]
 #[tokio::test]
@@ -219,10 +208,6 @@ fn account_resources(address: &str) -> String {
     format!("/accounts/{}/resources", address)
 }
 
-fn account_blob(address: &str) -> String {
-    format!("/accounts/{}/blob", address)
-}
-
 fn account_resources_with_ledger_version(address: &str, ledger_version: i128) -> String {
     format!("{}?version={}", account_resources(address), ledger_version)
 }

config/management/operational/Cargo.toml

@@ -39,6 +39,7 @@ aptos-transaction-builder = { path = "../../../sdk/transaction-builder" }
 aptos-types = { path = "../../../types" }
 aptos-workspace-hack = { path = "../../../crates/aptos-workspace-hack" }
 fallible = { path = "../../../crates/fallible" }
+move-deps = { path = "../../../aptos-move/move-deps", features = ["address32"] }
 netcore = { path = "../../../network/netcore" }
 network = { path = "../../../network" }
 

config/management/operational/src/rest_client.rs

@@ -5,12 +5,16 @@ use crate::{TransactionContext, TransactionStatus};
 use aptos_management::error::Error;
 use aptos_rest_client::Client;
 use aptos_types::{
-    account_address::AccountAddress, account_config, account_config::AccountResource,
-    account_state::AccountState, account_state_blob::AccountStateBlob, account_view::AccountView,
-    transaction::SignedTransaction, validator_config::ValidatorConfig,
+    account_address::AccountAddress,
+    account_config,
+    account_config::AccountResource,
+    on_chain_config::{access_path_for_config, OnChainConfig},
+    transaction::SignedTransaction,
+    validator_config::ValidatorConfig,
     validator_info::ValidatorInfo,
 };
-use std::convert::TryFrom;
+use move_deps::move_core_types::move_resource::MoveStructType;
+use serde::de::DeserializeOwned;
 
 /// A wrapper around JSON RPC for error handling
 pub struct RestClient {
@@ -36,28 +40,29 @@ impl RestClient {
         ))
     }
 
-    pub async fn account_state(&self, account: AccountAddress) -> Result<AccountState, Error> {
-        let result = self.client.get_account_state_blob(account).await;
-        let account_state_blob: AccountStateBlob = result
-            .map_err(|e| Error::RestReadError("account-state", e.to_string()))?
-            .inner()
-            .clone()
-            .into();
-
-        AccountState::try_from(&account_state_blob)
-            .map_err(|e| Error::RestReadError("account-state", e.to_string()))
+    pub async fn get_resource<T: DeserializeOwned>(
+        &self,
+        address: AccountAddress,
+        resource_type: &str,
+    ) -> Result<T, Error> {
+        Ok(self
+            .client
+            .get_resource(address, resource_type)
+            .await
+            .map_err(|e| Error::RestReadError("get_resource", e.to_string()))?
+            .into_inner())
     }
 
     pub async fn validator_config(
         &self,
         account: AccountAddress,
     ) -> Result<ValidatorConfig, Error> {
-        resource(
-            "validator-config-resource",
-            self.account_state(account)
-                .await?
-                .get_validator_config_resource(),
-        )
+        let access_path = ValidatorConfig::struct_tag().access_vector();
+        let resource_type = std::str::from_utf8(&access_path)
+            .map_err(|e| Error::UnableToParse("Unable to form resource type", e.to_string()))?;
+
+        let validator_config: ValidatorConfig = self.get_resource(account, resource_type).await?;
+        resource("validator-config-resource", Ok(Some(validator_config)))
     }
 
     /// This method returns all validator infos currently registered in the validator set of the
@@ -68,45 +73,44 @@ impl RestClient {
         account: Option<AccountAddress>,
     ) -> Result<Vec<ValidatorInfo>, Error> {
         let validator_set_account = account_config::validator_set_address();
-        let validator_set = self
-            .account_state(validator_set_account)
-            .await?
-            .get_validator_set();
-
-        match validator_set {
-            Ok(Some(validator_set)) => {
-                let mut validator_infos = vec![];
-                for validator_info in validator_set.payload() {
-                    if let Some(account) = account {
-                        if validator_info.account_address() == &account {
-                            validator_infos.push(validator_info.clone());
-                        }
-                    } else {
-                        validator_infos.push(validator_info.clone());
-                    }
-                }
+        let access_path =
+            access_path_for_config(aptos_types::on_chain_config::ValidatorSet::CONFIG_ID).path;
+        let resource_type = std::str::from_utf8(&access_path)
+            .map_err(|e| Error::RestReadError("Unable to form resource type", e.to_string()))?;
+
+        let validator_set: aptos_types::on_chain_config::ValidatorSet = self
+            .get_resource(validator_set_account, resource_type)
+            .await?;
 
-                if validator_infos.is_empty() {
-                    return Err(Error::UnexpectedError(
-                        "No validator sets were found!".to_string(),
-                    ));
+        let mut validator_infos = vec![];
+        for validator_info in validator_set.payload() {
+            if let Some(account) = account {
+                if validator_info.account_address() == &account {
+                    validator_infos.push(validator_info.clone());
                 }
-                Ok(validator_infos)
+            } else {
+                validator_infos.push(validator_info.clone());
             }
-            Ok(None) => Err(Error::RestReadError(
-                "validator-set",
-                "not present".to_string(),
-            )),
-            Err(e) => Err(Error::RestReadError("validator-set", e.to_string())),
         }
+
+        if validator_infos.is_empty() {
+            return Err(Error::UnexpectedError(
+                "No validator sets were found!".to_string(),
+            ));
+        }
+        Ok(validator_infos)
     }
 
     pub async fn account_resource(
         &self,
         account: AccountAddress,
     ) -> Result<AccountResource, Error> {
-        let account_state = self.account_state(account).await?;
-        resource("account-resource", account_state.get_account_resource())
+        let access_path = AccountResource::struct_tag().access_vector();
+        let resource_type = std::str::from_utf8(&access_path)
+            .map_err(|e| Error::UnableToParse("Unable to form resource type", e.to_string()))?;
+
+        let account_resource: AccountResource = self.get_resource(account, resource_type).await?;
+        resource("account-resource", Ok(Some(account_resource)))
     }
 
     pub async fn sequence_number(&self, account: AccountAddress) -> Result<u64, Error> {

config/seed-peer-generator/src/utils.rs

@@ -3,16 +3,15 @@
 
 #![forbid(unsafe_code)]
 
-use std::convert::TryFrom;
-
 use anyhow::Error;
 use aptos_config::config::{Peer, PeerRole, PeerSet};
 use aptos_logger::prelude::*;
 use aptos_rest_client::Client;
 use aptos_types::{
-    account_config::aptos_root_address, account_state::AccountState,
-    account_state_blob::AccountStateBlob, account_view::AccountView,
-    network_address::NetworkAddress, on_chain_config::ValidatorSet, validator_info::ValidatorInfo,
+    account_config::aptos_root_address,
+    network_address::NetworkAddress,
+    on_chain_config::{access_path_for_config, OnChainConfig, ValidatorSet},
+    validator_info::ValidatorInfo,
     PeerId,
 };
 
@@ -39,14 +38,11 @@ pub(crate) fn to_fullnode_addresses(
 fn get_validator_set(client_endpoint: String) -> anyhow::Result<ValidatorSet> {
     let client = Client::new(url::Url::parse(&client_endpoint)?);
     let rt = tokio::runtime::Runtime::new().unwrap();
-    let blob = rt.block_on(client.get_account_state_blob(aptos_root_address()))?;
-    let account_state_blob: AccountStateBlob = blob.inner().clone().into();
-    let account_state = AccountState::try_from(&account_state_blob)?;
-    if let Some(val) = account_state.get_validator_set()? {
-        Ok(val)
-    } else {
-        Err(Error::msg("No validator set"))
-    }
+    let validator_set_response = rt.block_on(client.get_resource::<ValidatorSet>(
+        aptos_root_address(),
+        std::str::from_utf8(&access_path_for_config(ValidatorSet::CONFIG_ID).path)?,
+    ))?;
+    Ok(validator_set_response.inner().clone())
 }
 
 // TODO: Merge with OnchainDiscovery

crates/aptos-rest-client/src/lib.rs

@@ -15,6 +15,7 @@ use serde_json::{json, Value};
 use state::State;
 use std::time::Duration;
 use url::Url;
+
 pub mod error;
 pub mod faucet;
 pub use faucet::FaucetClient;
@@ -242,18 +243,6 @@ impl Client {
         self.json(response).await
     }
 
-    pub async fn get_account_state_blob(
-        &self,
-        address: AccountAddress,
-    ) -> Result<Response<Vec<u8>>> {
-        let url = self.base_url.join(&format!("accounts/{}/blob", address))?;
-
-        let response = self.inner.get(url).send().await?;
-        let (response, state) = self.check_response(response).await?;
-        let blob = response.json().await?;
-        Ok(Response::new(blob, state))
-    }
-
     pub async fn get_account_resources(
         &self,
         address: AccountAddress,

documentation/specifications/common/authenticated_data_structures.md

@@ -286,9 +286,9 @@ The previous section described Merkle accumulator and sparse Merkle tree that fo
 
 ### Ledger State
 
-The ledger state represents the ground truth about the Aptos ecosystem, including the quantity of coins held by each user at a given version. Each validator must know the ledger state at the latest version in order to execute new transactions. Specifically, the ledger state is a map from `AccountAddress` to a binary blob `AccountStateBlob` that represents the state of an account.
+The ledger state represents the ground truth about the Aptos ecosystem, including the quantity of coins held by each user at a given version. Each validator must know the ledger state at the latest version in order to execute new transactions. Specifically, the ledger state is a map from `StateKey` to a `StateValue` that represents a resource associated with an account.
 
-Each `AccountAddress` is hashed to a 256-bit `HashValue`. Then each `(hash(address), account_state_blob)` tuple is inserted into a sparse Merkle tree as a key-value pair. Based on the properties of cryptographic hash functions, we can assume that the resulting sparse Merkle tree is balanced no matter how the addresses are generated. This sparse Merkle tree represents the entire ledger state and the root hash of this tree is the state root hash, which means the state root hash is the authenticator for any account state at the same version.
+Each `StateKey` is hashed to a 256-bit `HashValue`. Then each `(hash(state_key), state_value)` tuple is inserted into a sparse Merkle tree as a key-value pair. Based on the properties of cryptographic hash functions, we can assume that the resulting sparse Merkle tree is balanced no matter how the addresses are generated. This sparse Merkle tree represents the entire ledger state and the root hash of this tree is the state root hash, which means the state root hash is the authenticator for any resource at the same version.
 
 ```rust
 type LedgerState = SparseMerkleTree<AccountStateBlob>;
@@ -300,7 +300,7 @@ When the state tree is updated after the execution of a transaction and a new tr
 
 #### Accounts
 
-At the logical level, an account is a collection of Move resources and modules. At the physical level, an account is an ordered map of access paths to byte array values. TODO: link to definition of access path. When an account is stored in the ledger state, the ordered map is serialized using Binary Canonical Serialization (BCS) to create a binary blob `AccountStateBlob` that gets inserted into a sparse Merkle tree.
+At the logical level, an account is a collection of Move resources and modules. At the physical level, an account is an ordered map of state key to state values. TODO: link to definition of access path. When an account is stored in the ledger state, individual resource in the account is serialized using Binary Canonical Serialization (BCS) to create a `StateValue` that gets inserted into a sparse Merkle tree.
 
 ```rust
 type Path = Vec<u8>;
@@ -312,17 +312,22 @@ struct AccessPath {
 
 type AccountState = BTreeMap<Path, Vec<u8>>;
 
-struct AccountStateBlob {
-    blob: Vec<u8>,
+pub enum StateKey {
+  AccessPath(AccessPath),
+  TableItem {
+    handle: u128,
+    key: Vec<u8>,
+  },
+  // Only used for testing
+  Raw(Vec<u8>),
 }
 
-impl From<AccountState> for AccountStateBlob {
-    fn from(account_state: AccountState) -> Self {
-        Self {
-            blob: bcs::to_bytes(&account_state),
-        }
-    }
+pub struct StateValue {
+  pub maybe_bytes: Option<Vec<u8>>,
+  hash: HashValue,
 }
+
+
 ```
 
 ### Transaction and Transaction Output

documentation/specifications/db_backup/spec.md

@@ -111,7 +111,7 @@ pub struct StateSnapshotChunk {
     /// key of the last account in this chunk.
     pub last_key: HashValue,
     /// Repeated `len(record) + record` where `record` is BCS serialized tuple
-    /// `(key, account_state_blob)`
+    /// `(key, state_value)`
     pub blobs: FileHandle,
     /// BCS serialized `SparseMerkleRangeProof` that proves this chunk adds up to the root hash
     /// indicated in the backup (`StateSnapshotBackup::root_hash`).

storage/backup/backup-cli/src/backup_types/state_snapshot/manifest.rs

@@ -19,7 +19,7 @@ pub struct StateSnapshotChunk {
     /// key of the last account in this chunk.
     pub last_key: HashValue,
     /// Repeated `len(record) + record` where `record` is BCS serialized tuple
-    /// `(key, account_state_blob)`
+    /// `(key, state_value)`
     pub blobs: FileHandle,
     /// BCS serialized `SparseMerkleRangeProof` that proves this chunk adds up to the root hash
     /// indicated in the backup (`StateSnapshotBackup::root_hash`).