Skip to content

Commit

Permalink
src/serialize.h: base serialization level endianness neutrality
Browse files Browse the repository at this point in the history 
Serialization type-safety and endianness compatibility.
  • Loading branch information
@laanwj
laanwj committed Mar 6, 2015
1 parent 4e853aa commit 01f9c34
Show file tree
Hide file tree
Showing 2 changed files with 149 additions and 76 deletions.
201 changes: 125 additions & 76 deletions src/serialize.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
#include <utility>
#include <vector>

#include "compat/endian.h"

class CScript;

static const unsigned int MAX_SIZE = 0x02000000;
Expand Down Expand Up @@ -71,6 +73,79 @@ inline const T* end_ptr(const std::vector<T,TAl>& v)
return v.empty() ? NULL : (&v[0] + v.size());
}

/*
* Lowest-level serialization and conversion.
* @note Sizes of these types are verified in the tests
*/
template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
{
s.write((char*)&obj, 1);
}
template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
{
obj = htole16(obj);
s.write((char*)&obj, 2);
}
template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
{
obj = htole32(obj);
s.write((char*)&obj, 4);
}
template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
{
obj = htole64(obj);
s.write((char*)&obj, 8);
}
template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
{
uint8_t obj;
s.read((char*)&obj, 1);
return obj;
}
template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
{
uint16_t obj;
s.read((char*)&obj, 2);
return le16toh(obj);
}
template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
{
uint32_t obj;
s.read((char*)&obj, 4);
return le32toh(obj);
}
template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
{
uint64_t obj;
s.read((char*)&obj, 8);
return le64toh(obj);
}
inline uint64_t ser_double_to_uint64(double x)
{
union { double x; uint64_t y; } tmp;
tmp.x = x;
return tmp.y;
}
inline uint32_t ser_float_to_uint32(float x)
{
union { float x; uint32_t y; } tmp;
tmp.x = x;
return tmp.y;
}
inline double ser_uint64_to_double(uint64_t y)
{
union { double x; uint64_t y; } tmp;
tmp.y = y;
return tmp.x;
}
inline float ser_uint32_to_float(uint32_t y)
{
union { float x; uint32_t y; } tmp;
tmp.y = y;
return tmp.x;
}


/////////////////////////////////////////////////////////////////
//
// Templates for serializing to anything that looks like a stream,
Expand Down Expand Up @@ -108,59 +183,48 @@ enum
SerializationOp(s, CSerActionUnserialize(), nType, nVersion); \
}



/*
* Basic Types
*/
#define WRITEDATA(s, obj) s.write((char*)&(obj), sizeof(obj))
#define READDATA(s, obj) s.read((char*)&(obj), sizeof(obj))

inline unsigned int GetSerializeSize(char a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(signed char a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(unsigned char a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(signed short a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(unsigned short a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(signed int a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(unsigned int a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(signed long a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(unsigned long a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(signed long long a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(unsigned long long a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(float a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(double a, int, int=0) { return sizeof(a); }

template<typename Stream> inline void Serialize(Stream& s, char a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, signed char a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, unsigned char a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, signed short a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, unsigned short a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, signed int a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, unsigned int a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, signed long a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, unsigned long a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, signed long long a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, unsigned long long a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, float a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, double a, int, int=0) { WRITEDATA(s, a); }

template<typename Stream> inline void Unserialize(Stream& s, char& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, signed char& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, unsigned char& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, signed short& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, unsigned short& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, signed int& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, unsigned int& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, signed long& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, unsigned long& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, signed long long& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, unsigned long long& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, float& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, double& a, int, int=0) { READDATA(s, a); }
inline unsigned int GetSerializeSize(char a, int, int=0) { return 1; }
inline unsigned int GetSerializeSize(int8_t a, int, int=0) { return 1; }
inline unsigned int GetSerializeSize(uint8_t a, int, int=0) { return 1; }
inline unsigned int GetSerializeSize(int16_t a, int, int=0) { return 2; }
inline unsigned int GetSerializeSize(uint16_t a, int, int=0) { return 2; }
inline unsigned int GetSerializeSize(int32_t a, int, int=0) { return 4; }
inline unsigned int GetSerializeSize(uint32_t a, int, int=0) { return 4; }
inline unsigned int GetSerializeSize(int64_t a, int, int=0) { return 8; }
inline unsigned int GetSerializeSize(uint64_t a, int, int=0) { return 8; }
inline unsigned int GetSerializeSize(float a, int, int=0) { return 4; }
inline unsigned int GetSerializeSize(double a, int, int=0) { return 8; }

template<typename Stream> inline void Serialize(Stream& s, char a, int, int=0) { ser_writedata8(s, a); } // TODO Get rid of bare char
template<typename Stream> inline void Serialize(Stream& s, int8_t a, int, int=0) { ser_writedata8(s, a); }
template<typename Stream> inline void Serialize(Stream& s, uint8_t a, int, int=0) { ser_writedata8(s, a); }
template<typename Stream> inline void Serialize(Stream& s, int16_t a, int, int=0) { ser_writedata16(s, a); }
template<typename Stream> inline void Serialize(Stream& s, uint16_t a, int, int=0) { ser_writedata16(s, a); }
template<typename Stream> inline void Serialize(Stream& s, int32_t a, int, int=0) { ser_writedata32(s, a); }
template<typename Stream> inline void Serialize(Stream& s, uint32_t a, int, int=0) { ser_writedata32(s, a); }
template<typename Stream> inline void Serialize(Stream& s, int64_t a, int, int=0) { ser_writedata64(s, a); }
template<typename Stream> inline void Serialize(Stream& s, uint64_t a, int, int=0) { ser_writedata64(s, a); }
template<typename Stream> inline void Serialize(Stream& s, float a, int, int=0) { ser_writedata32(s, ser_float_to_uint32(a)); }
template<typename Stream> inline void Serialize(Stream& s, double a, int, int=0) { ser_writedata64(s, ser_double_to_uint64(a)); }

template<typename Stream> inline void Unserialize(Stream& s, char& a, int, int=0) { a = ser_readdata8(s); } // TODO Get rid of bare char
template<typename Stream> inline void Unserialize(Stream& s, int8_t& a, int, int=0) { a = ser_readdata8(s); }
template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a, int, int=0) { a = ser_readdata8(s); }
template<typename Stream> inline void Unserialize(Stream& s, int16_t& a, int, int=0) { a = ser_readdata16(s); }
template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a, int, int=0) { a = ser_readdata16(s); }
template<typename Stream> inline void Unserialize(Stream& s, int32_t& a, int, int=0) { a = ser_readdata32(s); }
template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a, int, int=0) { a = ser_readdata32(s); }
template<typename Stream> inline void Unserialize(Stream& s, int64_t& a, int, int=0) { a = ser_readdata64(s); }
template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a, int, int=0) { a = ser_readdata64(s); }
template<typename Stream> inline void Unserialize(Stream& s, float& a, int, int=0) { a = ser_uint32_to_float(ser_readdata32(s)); }
template<typename Stream> inline void Unserialize(Stream& s, double& a, int, int=0) { a = ser_uint64_to_double(ser_readdata64(s)); }

inline unsigned int GetSerializeSize(bool a, int, int=0) { return sizeof(char); }
template<typename Stream> inline void Serialize(Stream& s, bool a, int, int=0) { char f=a; WRITEDATA(s, f); }
template<typename Stream> inline void Unserialize(Stream& s, bool& a, int, int=0) { char f; READDATA(s, f); a=f; }
template<typename Stream> inline void Serialize(Stream& s, bool a, int, int=0) { char f=a; ser_writedata8(s, f); }
template<typename Stream> inline void Unserialize(Stream& s, bool& a, int, int=0) { char f=ser_readdata8(s); a=f; }



Expand All @@ -187,64 +251,50 @@ void WriteCompactSize(Stream& os, uint64_t nSize)
{
if (nSize < 253)
{
unsigned char chSize = nSize;
WRITEDATA(os, chSize);
ser_writedata8(os, nSize);
}
else if (nSize <= std::numeric_limits<unsigned short>::max())
{
unsigned char chSize = 253;
unsigned short xSize = nSize;
WRITEDATA(os, chSize);
WRITEDATA(os, xSize);
ser_writedata8(os, 253);
ser_writedata16(os, nSize);
}
else if (nSize <= std::numeric_limits<unsigned int>::max())
{
unsigned char chSize = 254;
unsigned int xSize = nSize;
WRITEDATA(os, chSize);
WRITEDATA(os, xSize);
ser_writedata8(os, 254);
ser_writedata32(os, nSize);
}
else
{
unsigned char chSize = 255;
uint64_t xSize = nSize;
WRITEDATA(os, chSize);
WRITEDATA(os, xSize);
ser_writedata8(os, 255);
ser_writedata64(os, nSize);
}
return;
}

template<typename Stream>
uint64_t ReadCompactSize(Stream& is)
{
unsigned char chSize;
READDATA(is, chSize);
uint8_t chSize = ser_readdata8(is);
uint64_t nSizeRet = 0;
if (chSize < 253)
{
nSizeRet = chSize;
}
else if (chSize == 253)
{
unsigned short xSize;
READDATA(is, xSize);
nSizeRet = xSize;
nSizeRet = ser_readdata16(is);
if (nSizeRet < 253)
throw std::ios_base::failure("non-canonical ReadCompactSize()");
}
else if (chSize == 254)
{
unsigned int xSize;
READDATA(is, xSize);
nSizeRet = xSize;
nSizeRet = ser_readdata32(is);
if (nSizeRet < 0x10000u)
throw std::ios_base::failure("non-canonical ReadCompactSize()");
}
else
{
uint64_t xSize;
READDATA(is, xSize);
nSizeRet = xSize;
nSizeRet = ser_readdata64(is);
if (nSizeRet < 0x100000000ULL)
throw std::ios_base::failure("non-canonical ReadCompactSize()");
}
Expand Down Expand Up @@ -303,7 +353,7 @@ void WriteVarInt(Stream& os, I n)
len++;
}
do {
WRITEDATA(os, tmp[len]);
ser_writedata8(os, tmp[len]);
} while(len--);
}

Expand All @@ -312,8 +362,7 @@ I ReadVarInt(Stream& is)
{
I n = 0;
while(true) {
unsigned char chData;
READDATA(is, chData);
unsigned char chData = ser_readdata8(is);
n = (n << 7) | (chData & 0x7F);
if (chData & 0x80)
n++;
Expand Down
24 changes: 24 additions & 0 deletions src/test/serialize_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,30 @@ using namespace std;

BOOST_AUTO_TEST_SUITE(serialize_tests)

BOOST_AUTO_TEST_CASE(sizes)
{
BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(char(0), 0));
BOOST_CHECK_EQUAL(sizeof(int8_t), GetSerializeSize(int8_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(uint8_t), GetSerializeSize(uint8_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(int16_t), GetSerializeSize(int16_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(uint16_t), GetSerializeSize(uint16_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(int32_t), GetSerializeSize(int32_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(uint32_t), GetSerializeSize(uint32_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(int64_t), GetSerializeSize(int64_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(uint64_t), GetSerializeSize(uint64_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(float), GetSerializeSize(float(0), 0));
BOOST_CHECK_EQUAL(sizeof(double), GetSerializeSize(double(0), 0));

// Bool is serialized as char
BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(bool(0), 0));
}

BOOST_AUTO_TEST_CASE(floats)
{
// TODO ser_uint32_to_float, ser_uint64_to_double
// TODO ser_float_to_uint32, ser_double_to_uint64
}

BOOST_AUTO_TEST_CASE(varints)
{
// encode
Expand Down

0 comments on commit 01f9c34

Please sign in to comment.