Skip to content

Commit

Permalink
Encode data using base64, encode ids using Hex for human readable ser…
Browse files Browse the repository at this point in the history
…ialisation (MystenLabs#1304)

* serialize data using base64, ids using Hex for human readable serialization

* BytesOrHex, BytesOrBase64 refactoring
Serialise to bytes instead of tuplearray

* Use default serde for AuthoritySignature instead of Bytes

* add more info to error

* Update readable_serde.rs

Hyphenate compound modifiers, remove caps where not needed

* address PR comments

* * use const generic instead of macro
* replace base64 with base64ct

Co-authored-by: Clay-Mysten <[email protected]>
  • Loading branch information
patrickkuo and Clay-Mysten authored Apr 12, 2022
1 parent f3dea88 commit f7aa79c
Show file tree
Hide file tree
Showing 7 changed files with 259 additions and 54 deletions.
2 changes: 1 addition & 1 deletion sui_types/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ edition = "2021"

[dependencies]
anyhow = { version = "1.0.56", features = ["backtrace"] }
base64 = "0.13.0"
bcs = "0.1.3"
bincode = "1.3.3"
itertools = "0.10.3"
Expand All @@ -27,6 +26,7 @@ signature = "1.5.0"
static_assertions = "1.1.0"
opentelemetry = { version = "0.17", features = ["rt-tokio"] }
parking_lot = "0.12.0"
base64ct = { version = "1.5.0", features = ["alloc"] }

name_variant = { git = "https://github.com/MystenLabs/mysten-infra", rev = "97a056f85555fa2afe497d6abb7cf6bf8faa63cf" }
typed-store = { git = "https://github.com/MystenLabs/mysten-infra", rev = "e44bca4513a6ff6c97399cd79e82e4bc00571ac3" }
Expand Down
43 changes: 7 additions & 36 deletions sui_types/src/base_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,17 @@ use std::collections::{HashMap, HashSet};
use std::convert::{TryFrom, TryInto};
use std::fmt;

use crate::readable_serde::BytesOrBase64;
use crate::readable_serde::BytesOrHex;
use ed25519_dalek::Digest;
use hex::FromHex;
use move_core_types::account_address::AccountAddress;
use move_core_types::ident_str;
use move_core_types::identifier::IdentStr;
use opentelemetry::{global, Context};
use rand::Rng;
use serde::{de::Error as _, Deserialize, Serialize};
use serde_with::serde_as;
use serde_with::Bytes;
use serde::de::Error;
use serde::{Deserialize, Serialize};
use sha3::Sha3_256;

use crate::crypto::PublicKeyBytes;
Expand Down Expand Up @@ -42,9 +43,8 @@ pub struct ObjectID(AccountAddress);
pub type ObjectRef = (ObjectID, SequenceNumber, ObjectDigest);

pub const SUI_ADDRESS_LENGTH: usize = ObjectID::LENGTH;
#[serde_as]
#[derive(Eq, Default, PartialEq, Ord, PartialOrd, Copy, Clone, Hash, Serialize, Deserialize)]
pub struct SuiAddress(#[serde_as(as = "Bytes")] [u8; SUI_ADDRESS_LENGTH]);
pub struct SuiAddress(#[serde(with = "BytesOrHex")] [u8; SUI_ADDRESS_LENGTH]);

impl SuiAddress {
pub fn to_vec(&self) -> Vec<u8> {
Expand Down Expand Up @@ -133,13 +133,11 @@ pub const TRANSACTION_DIGEST_LENGTH: usize = 32;
pub const OBJECT_DIGEST_LENGTH: usize = 32;

/// A transaction will have a (unique) digest.
#[serde_as]
#[derive(Eq, PartialEq, Ord, PartialOrd, Copy, Clone, Hash, Serialize, Deserialize)]
pub struct TransactionDigest(#[serde_as(as = "Bytes")] [u8; TRANSACTION_DIGEST_LENGTH]);
pub struct TransactionDigest(#[serde(with = "BytesOrBase64")] [u8; TRANSACTION_DIGEST_LENGTH]);
// Each object has a unique digest
#[serde_as]
#[derive(Eq, PartialEq, Ord, PartialOrd, Copy, Clone, Hash, Serialize, Deserialize)]
pub struct ObjectDigest(#[serde_as(as = "Bytes")] pub [u8; 32]); // We use SHA3-256 hence 32 bytes here
pub struct ObjectDigest(#[serde(with = "BytesOrBase64")] pub [u8; 32]); // We use SHA3-256 hence 32 bytes here

pub const TX_CONTEXT_MODULE_NAME: &IdentStr = ident_str!("TxContext");
pub const TX_CONTEXT_STRUCT_NAME: &IdentStr = TX_CONTEXT_MODULE_NAME;
Expand Down Expand Up @@ -345,33 +343,6 @@ impl fmt::Display for SuiAddress {
}
}

pub fn address_as_base64<S>(address: &SuiAddress, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::ser::Serializer,
{
serializer.serialize_str(&encode_address(address))
}

pub fn address_from_base64<'de, D>(deserializer: D) -> Result<SuiAddress, D::Error>
where
D: serde::de::Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
let value = decode_address(&s).map_err(|err| serde::de::Error::custom(err.to_string()))?;
Ok(value)
}

pub fn encode_address(address: &SuiAddress) -> String {
base64::encode(&address.0[..])
}

pub fn decode_address(s: &str) -> Result<SuiAddress, anyhow::Error> {
let value = base64::decode(s)?;
let mut address = [0u8; SUI_ADDRESS_LENGTH];
address.copy_from_slice(&value[..SUI_ADDRESS_LENGTH]);
Ok(SuiAddress(address))
}

pub fn dbg_addr(name: u8) -> SuiAddress {
let addr = [name; SUI_ADDRESS_LENGTH];
SuiAddress(addr)
Expand Down
26 changes: 12 additions & 14 deletions sui_types/src/crypto.rs
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
// Copyright (c) 2022, Mysten Labs, Inc.
// SPDX-License-Identifier: Apache-2.0
use anyhow::Error;
use base64ct::{Base64, Encoding};
use std::borrow::Borrow;
use std::collections::HashMap;

use crate::base_types::{AuthorityName, SuiAddress};
use crate::error::{SuiError, SuiResult};
use crate::readable_serde::BytesOrBase64;
use ed25519_dalek as dalek;
use ed25519_dalek::{Digest, PublicKey, Verifier};
use once_cell::sync::OnceCell;
use rand::rngs::OsRng;
use serde::{Deserialize, Serialize};
use serde_with::{serde_as, Bytes};
use sha3::Sha3_256;

use crate::base_types::{AuthorityName, SuiAddress};
use crate::error::{SuiError, SuiResult};

// TODO: Make sure secrets are not copyable and movable to control where they are in memory
#[derive(Debug)]
pub struct KeyPair {
Expand Down Expand Up @@ -49,7 +49,7 @@ impl Serialize for KeyPair {
where
S: serde::ser::Serializer,
{
serializer.serialize_str(&base64::encode(&self.key_pair.to_bytes()))
serializer.serialize_str(&Base64::encode_string(&self.key_pair.to_bytes()))
}
}

Expand All @@ -59,7 +59,8 @@ impl<'de> Deserialize<'de> for KeyPair {
D: serde::de::Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
let value = base64::decode(&s).map_err(|err| serde::de::Error::custom(err.to_string()))?;
let value =
Base64::decode_vec(&s).map_err(|err| serde::de::Error::custom(err.to_string()))?;
let key = dalek::Keypair::from_bytes(&value)
.map_err(|err| serde::de::Error::custom(err.to_string()))?;
Ok(KeyPair {
Expand Down Expand Up @@ -87,9 +88,8 @@ impl signature::Signer<AuthoritySignature> for KeyPair {
}
}

#[serde_as]
#[derive(Eq, Default, PartialEq, Ord, PartialOrd, Copy, Clone, Hash, Serialize, Deserialize)]
pub struct PublicKeyBytes(#[serde_as(as = "Bytes")] [u8; dalek::PUBLIC_KEY_LENGTH]);
pub struct PublicKeyBytes(#[serde(with = "BytesOrBase64")] [u8; dalek::PUBLIC_KEY_LENGTH]);

impl PublicKeyBytes {
pub fn to_vec(&self) -> Vec<u8> {
Expand Down Expand Up @@ -165,9 +165,8 @@ pub fn get_key_pair_from_bytes(bytes: &[u8]) -> (SuiAddress, KeyPair) {
pub const SUI_SIGNATURE_LENGTH: usize =
ed25519_dalek::PUBLIC_KEY_LENGTH + ed25519_dalek::SIGNATURE_LENGTH;

#[serde_as]
#[derive(Eq, PartialEq, Copy, Clone, Serialize, Deserialize)]
pub struct Signature(#[serde_as(as = "Bytes")] [u8; SUI_SIGNATURE_LENGTH]);
pub struct Signature(#[serde(with = "BytesOrBase64")] [u8; SUI_SIGNATURE_LENGTH]);

impl AsRef<[u8]> for Signature {
fn as_ref(&self) -> &[u8] {
Expand All @@ -185,8 +184,8 @@ impl signature::Signature for Signature {

impl std::fmt::Debug for Signature {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
let s = base64::encode(self.signature_bytes());
let p = base64::encode(self.public_key_bytes());
let s = Base64::encode_string(self.signature_bytes());
let p = Base64::encode_string(self.public_key_bytes());
write!(f, "{s}@{p}")?;
Ok(())
}
Expand Down Expand Up @@ -299,8 +298,7 @@ impl Signature {
/// A signature emitted by an authority. It's useful to decouple this from user signatures,
/// as their set of supported schemes will probably diverge
#[derive(Debug, Eq, PartialEq, Copy, Clone, Serialize, Deserialize)]

pub struct AuthoritySignature(pub dalek::Signature);
pub struct AuthoritySignature(#[serde(with = "BytesOrBase64")] pub dalek::Signature);
impl AsRef<[u8]> for AuthoritySignature {
fn as_ref(&self) -> &[u8] {
self.0.as_ref()
Expand Down
1 change: 1 addition & 0 deletions sui_types/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ pub mod id;
pub mod messages;
pub mod move_package;
pub mod object;
pub mod readable_serde;
pub mod serialize;
pub mod storage;

Expand Down
3 changes: 2 additions & 1 deletion sui_types/src/messages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Copyright (c) 2022, Mysten Labs, Inc.
// SPDX-License-Identifier: Apache-2.0

use base64ct::{Base64, Encoding};
use std::fmt::Write;
use std::fmt::{Display, Formatter};
use std::{
Expand Down Expand Up @@ -316,7 +317,7 @@ where
}

pub fn to_base64(&self) -> String {
base64::encode(self.to_bytes())
Base64::encode_string(&self.to_bytes())
}
}

Expand Down
153 changes: 153 additions & 0 deletions sui_types/src/readable_serde.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
// Copyright (c) 2022, Mysten Labs, Inc.
// SPDX-License-Identifier: Apache-2.0

use anyhow::anyhow;
use std::fmt::Display;

use base64ct::{Base64, Encoding as _};
use serde;
use serde::de::{Deserialize, Deserializer, Error};
use serde::ser::Serializer;
use serde::Serialize;
use serde_with::{Bytes, DeserializeAs, SerializeAs};

/// Encode bytes to hex for human-readable serializer and deserializer,
/// serde to bytes for non-human-readable serializer and deserializer.
pub trait BytesOrHex<'de>: Sized {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
Self: AsRef<[u8]>,
{
serialize_to_bytes_or_encode(serializer, self.as_ref(), Encoding::Hex)
}
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>;
}

/// Encode bytes to Base64 for human-readable serializer and deserializer,
/// serde to array tuple for non-human-readable serializer and deserializer.
pub trait BytesOrBase64<'de>: Sized {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
Self: AsRef<[u8]>,
{
serialize_to_bytes_or_encode(serializer, self.as_ref(), Encoding::Base64)
}
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>;
}

fn deserialize_from_bytes_or_decode<'de, D>(
deserializer: D,
encoding: Encoding,
) -> Result<Vec<u8>, D::Error>
where
D: Deserializer<'de>,
{
if deserializer.is_human_readable() {
let s = String::deserialize(deserializer)?;
encoding.decode(s).map_err(to_custom_error::<'de, D, _>)
} else {
Bytes::deserialize_as(deserializer)
}
}

fn serialize_to_bytes_or_encode<S>(
serializer: S,
data: &[u8],
encoding: Encoding,
) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
if serializer.is_human_readable() {
encoding.encode(data).serialize(serializer)
} else {
Bytes::serialize_as(&data, serializer)
}
}

impl<'de> BytesOrBase64<'de> for ed25519_dalek::Signature {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
Self: AsRef<[u8]>,
{
if serializer.is_human_readable() {
Encoding::Base64.encode(self.as_ref()).serialize(serializer)
} else {
<Self as Serialize>::serialize(self, serializer)
}
}
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
if deserializer.is_human_readable() {
let s = String::deserialize(deserializer)?;
let value = Encoding::Base64
.decode(s)
.map_err(to_custom_error::<'de, D, _>)?;
Self::try_from(value.as_slice()).map_err(to_custom_error::<'de, D, _>)
} else {
<Self as Deserialize>::deserialize(deserializer)
}
}
}

fn to_custom_error<'de, D, E>(e: E) -> D::Error
where
E: Display,
D: Deserializer<'de>,
{
D::Error::custom(format!("byte deserialization failed, cause by: {}", e))
}

impl<'de, const N: usize> BytesOrHex<'de> for [u8; N] {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = deserialize_from_bytes_or_decode(deserializer, Encoding::Hex)?;
let mut array = [0u8; N];
array.copy_from_slice(&value[..N]);
Ok(array)
}
}

impl<'de, const N: usize> BytesOrBase64<'de> for [u8; N] {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = deserialize_from_bytes_or_decode(deserializer, Encoding::Base64)?;
let mut array = [0u8; N];
array.copy_from_slice(&value[..N]);
Ok(array)
}
}

enum Encoding {
Base64,
Hex,
}

impl Encoding {
fn decode(&self, s: String) -> Result<Vec<u8>, anyhow::Error> {
Ok(match self {
Encoding::Base64 => Base64::decode_vec(&s).map_err(|e| anyhow!(e))?,
Encoding::Hex => hex::decode(s)?,
})
}

fn encode<T: AsRef<[u8]>>(&self, data: T) -> String {
match self {
Encoding::Base64 => Base64::encode_string(data.as_ref()),
Encoding::Hex => hex::encode(data),
}
}
}
Loading

0 comments on commit f7aa79c

Please sign in to comment.