1196 serialization

CMakeLists.txt

@@ -51,6 +51,7 @@ LIST(APPEND BOOST_COMPONENTS thread
                              chrono
                              unit_test_framework
                              context)
+# boost::endian is also required, but FindBoost can't handle header-only libs
 SET( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 
 IF( WIN32 )

README.md

@@ -55,7 +55,7 @@ We recommend building on Ubuntu 16.04 LTS (64-bit)
     git submodule sync --recursive
     git submodule update --init --recursive
 
-**NOTE:** Versions of [Boost](http://www.boost.org/) 1.57 through 1.69 are supported. Newer versions may work, but
+**NOTE:** Versions of [Boost](http://www.boost.org/) 1.58 through 1.69 are supported. Newer versions may work, but
 have not been tested. If your system came pre-installed with a version of Boost that you do not wish to use, you may
 manually build your preferred version and use it with BitShares by specifying it on the CMake command line.
 

libraries/chain/block_database.cpp

@@ -24,14 +24,19 @@
 #include <graphene/chain/block_database.hpp>
 #include <graphene/protocol/fee_schedule.hpp>
 #include <fc/io/raw.hpp>
+#include <boost/endian/buffers.hpp>
 
 namespace graphene { namespace chain {
 
 struct index_entry
 {
-   uint64_t      block_pos = 0;
-   uint32_t      block_size = 0;
-   block_id_type block_id;
+   index_entry() {
+      block_pos = 0;
+      block_size = 0;
+   };
+   boost::endian::little_uint64_buf_t block_pos;
+   boost::endian::little_uint32_buf_t block_size;
+   block_id_type                      block_id;
 };
  }}
 FC_REFLECT( graphene::chain::index_entry, (block_pos)(block_size)(block_id) );
@@ -125,7 +130,7 @@ bool block_database::contains( const block_id_type& id )const
    _block_num_to_pos.seekg( index_pos );
    _block_num_to_pos.read( (char*)&e, sizeof(e) );
 
-   return e.block_id == id && e.block_size > 0;
+   return e.block_id == id && e.block_size.value() > 0;
 }
 
 block_id_type block_database::fetch_block_id( uint32_t block_num )const
@@ -159,10 +164,10 @@ optional<signed_block> block_database::fetch_optional( const block_id_type& id )
 
       if( e.block_id != id ) return optional<signed_block>();
 
-      vector<char> data( e.block_size );
-      _blocks.seekg( e.block_pos );
-      if (e.block_size)
-         _blocks.read( data.data(), e.block_size );
+      vector<char> data( e.block_size.value() );
+      _blocks.seekg( e.block_pos.value() );
+      if (e.block_size.value())
+         _blocks.read( data.data(), e.block_size.value() );
       auto result = fc::raw::unpack<signed_block>(data);
       FC_ASSERT( result.id() == e.block_id );
       return result;
@@ -189,9 +194,9 @@ optional<signed_block> block_database::fetch_by_number( uint32_t block_num )cons
       _block_num_to_pos.seekg( index_pos, _block_num_to_pos.beg );
       _block_num_to_pos.read( (char*)&e, sizeof(e) );
 
-      vector<char> data( e.block_size );
-      _blocks.seekg( e.block_pos );
-      _blocks.read( data.data(), e.block_size );
+      vector<char> data( e.block_size.value() );
+      _blocks.seekg( e.block_pos.value() );
+      _blocks.read( data.data(), e.block_size.value() );
       auto result = fc::raw::unpack<signed_block>(data);
       FC_ASSERT( result.id() == e.block_id );
       return result;
@@ -224,14 +229,14 @@ optional<index_entry> block_database::last_index_entry()const {
          pos -= sizeof(index_entry);
          _block_num_to_pos.seekg( pos );
          _block_num_to_pos.read( (char*)&e, sizeof(e) );
-         if( _block_num_to_pos.gcount() == sizeof(e) && e.block_size > 0
-                && int64_t(e.block_pos + e.block_size) <= blocks_size )
+         if( _block_num_to_pos.gcount() == sizeof(e) && e.block_size.value() > 0
+                && int64_t(e.block_pos.value() + e.block_size.value()) <= blocks_size )
             try
             {
-               vector<char> data( e.block_size );
-               _blocks.seekg( e.block_pos );
-               _blocks.read( data.data(), e.block_size );
-               if( _blocks.gcount() == long(e.block_size) )
+               vector<char> data( e.block_size.value() );
+               _blocks.seekg( e.block_pos.value() );
+               _blocks.read( data.data(), e.block_size.value() );
+               if( _blocks.gcount() == long(e.block_size.value()) )
                {
                   const signed_block block = fc::raw::unpack<signed_block>(data);
                   if( block.id() == e.block_id )

libraries/chain/db_block.cpp

@@ -38,6 +38,7 @@
 
 #include <graphene/protocol/fee_schedule.hpp>
 
+#include <fc/io/raw.hpp>
 #include <fc/thread/parallel.hpp>
 
 namespace graphene { namespace chain {
@@ -661,7 +662,7 @@ processed_transaction database::_apply_transaction(const signed_transaction& trx
          const auto& tapos_block_summary = block_summary_id_type( trx.ref_block_num )(*this);
 
          //Verify TaPoS block summary has correct ID prefix, and that this block's time is not past the expiration
-         FC_ASSERT( trx.ref_block_prefix == tapos_block_summary.block_id._hash[1] );
+         FC_ASSERT( trx.ref_block_prefix == tapos_block_summary.block_id._hash[1].value() );
       }
 
       fc::time_point_sec now = head_block_time();

libraries/chain/include/graphene/chain/global_property_object.hpp

@@ -22,13 +22,14 @@
  * THE SOFTWARE.
  */
 #pragma once
-#include <fc/uint128.hpp>
 
 #include <graphene/protocol/chain_parameters.hpp>
 #include <graphene/chain/types.hpp>
 #include <graphene/chain/database.hpp>
 #include <graphene/db/object.hpp>
 
+#include <fc/uint128.hpp>
+
 namespace graphene { namespace chain {
 
    /**

libraries/net/include/graphene/net/message.hpp

@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 #pragma once
+#include <boost/endian/buffers.hpp>
 #include <fc/array.hpp>
 #include <fc/io/varint.hpp>
 #include <fc/network/ip.hpp>
@@ -39,8 +40,8 @@ namespace graphene { namespace net {
    */
   struct message_header
   {
-     uint32_t  size;   // number of bytes in message, capped at MAX_MESSAGE_SIZE
-     uint32_t  msg_type;  // every channel gets a 16 bit message type specifier
+     boost::endian::little_uint32_buf_t size;   // number of bytes in message, capped at MAX_MESSAGE_SIZE
+     boost::endian::little_uint32_buf_t msg_type;  // every channel gets a 16 bit message type specifier
   };
 
   typedef fc::uint160_t message_hash_type;
@@ -85,7 +86,7 @@ namespace graphene { namespace net {
      T as()const 
      {
          try {
-          FC_ASSERT( msg_type == T::type );
+          FC_ASSERT( msg_type.value() == T::type );
           T tmp;
           if( data.size() )
           {
@@ -103,7 +104,7 @@ namespace graphene { namespace net {
               "error unpacking network message as a '${type}'  ${x} !=? ${msg_type}", 
               ("type", fc::get_typename<T>::name() )
               ("x", T::type)
-              ("msg_type", msg_type)
+              ("msg_type", msg_type.value())
               );
      }
   };

libraries/net/message_oriented_connection.cpp

@@ -153,24 +153,23 @@ namespace graphene { namespace net {
       try
       {
         message m;
+        char buffer[BUFFER_SIZE];
         while( true )
         {
-          char buffer[BUFFER_SIZE];
           _sock.read(buffer, BUFFER_SIZE);
           _bytes_received += BUFFER_SIZE;
           memcpy((char*)&m, buffer, sizeof(message_header));
+          FC_ASSERT( m.size.value() <= MAX_MESSAGE_SIZE, "", ("m.size",m.size.value())("MAX_MESSAGE_SIZE",MAX_MESSAGE_SIZE) );
 
-          FC_ASSERT( m.size <= MAX_MESSAGE_SIZE, "", ("m.size",m.size)("MAX_MESSAGE_SIZE",MAX_MESSAGE_SIZE) );
-
-          size_t remaining_bytes_with_padding = 16 * ((m.size - LEFTOVER + 15) / 16);
+          size_t remaining_bytes_with_padding = 16 * ((m.size.value() - LEFTOVER + 15) / 16);
           m.data.resize(LEFTOVER + remaining_bytes_with_padding); //give extra 16 bytes to allow for padding added in send call
           std::copy(buffer + sizeof(message_header), buffer + sizeof(buffer), m.data.begin());
           if (remaining_bytes_with_padding)
           {
             _sock.read(&m.data[LEFTOVER], remaining_bytes_with_padding);
             _bytes_received += remaining_bytes_with_padding;
           }
-          m.data.resize(m.size); // truncate off the padding bytes
+          m.data.resize(m.size.value()); // truncate off the padding bytes
 
           _last_message_received_time = fc::time_point::now();
 
@@ -255,14 +254,14 @@ namespace graphene { namespace net {
 
       try
       {
-        size_t size_of_message_and_header = sizeof(message_header) + message_to_send.size;
-        if( message_to_send.size > MAX_MESSAGE_SIZE )
+        size_t size_of_message_and_header = sizeof(message_header) + message_to_send.size.value();
+        if( message_to_send.size.value() > MAX_MESSAGE_SIZE )
            elog("Trying to send a message larger than MAX_MESSAGE_SIZE. This probably won't work...");
         //pad the message we send to a multiple of 16 bytes
         size_t size_with_padding = 16 * ((size_of_message_and_header + 15) / 16);
         std::unique_ptr<char[]> padded_message(new char[size_with_padding]);
         memcpy(padded_message.get(), (char*)&message_to_send, sizeof(message_header));
-        memcpy(padded_message.get() + sizeof(message_header), message_to_send.data.data(), message_to_send.size );
+        memcpy(padded_message.get() + sizeof(message_header), message_to_send.data.data(), message_to_send.size.value() );
         _sock.write(padded_message.get(), size_with_padding);
         _sock.flush();
         _bytes_sent += size_with_padding;

libraries/net/node.cpp

@@ -1257,10 +1257,10 @@ namespace graphene { namespace net { namespace detail {
       VERIFY_CORRECT_THREAD();
       message_hash_type message_hash = received_message.id();
       dlog("handling message ${type} ${hash} size ${size} from peer ${endpoint}",
-           ("type", graphene::net::core_message_type_enum(received_message.msg_type))("hash", message_hash)
+           ("type", graphene::net::core_message_type_enum(received_message.msg_type.value()))("hash", message_hash)
            ("size", received_message.size)
            ("endpoint", originating_peer->get_remote_endpoint()));
-      switch ( received_message.msg_type )
+      switch ( received_message.msg_type.value() )
       {
       case core_message_type_enum::hello_message_type:
         on_hello_message(originating_peer, received_message.as<hello_message>());
@@ -1320,8 +1320,8 @@ namespace graphene { namespace net { namespace detail {
       default:
         // ignore any message in between core_message_type_first and _last that we don't handle above
         // to allow us to add messages in the future
-        if (received_message.msg_type < core_message_type_enum::core_message_type_first ||
-            received_message.msg_type > core_message_type_enum::core_message_type_last)
+        if (received_message.msg_type.value() < core_message_type_enum::core_message_type_first ||
+            received_message.msg_type.value() > core_message_type_enum::core_message_type_last)
           process_ordinary_message(originating_peer, received_message, message_hash);
         break;
       }
@@ -2306,7 +2306,7 @@ namespace graphene { namespace net { namespace detail {
 
       for (const message& reply : reply_messages)
       {
-        if (reply.msg_type == block_message_type)
+        if (reply.msg_type.value() == block_message_type)
           originating_peer->send_item(item_id(block_message_type, reply.as<graphene::net::block_message>().block_id));
         else
           originating_peer->send_message(reply);
@@ -3363,7 +3363,7 @@ namespace graphene { namespace net { namespace detail {
       fc::time_point message_receive_time = fc::time_point::now();
 
       // only process it if we asked for it
-      auto iter = originating_peer->items_requested_from_peer.find( item_id(message_to_process.msg_type, message_hash) );
+      auto iter = originating_peer->items_requested_from_peer.find( item_id(message_to_process.msg_type.value(), message_hash) );
       if( iter == originating_peer->items_requested_from_peer.end() )
       {
         wlog( "received a message I didn't ask for from peer ${endpoint}, disconnecting from peer",
@@ -3383,7 +3383,7 @@ namespace graphene { namespace net { namespace detail {
         fc::time_point message_validated_time;
         try
         {
-          if (message_to_process.msg_type == trx_message_type)
+          if (message_to_process.msg_type.value() == trx_message_type)
           {
             trx_message transaction_message_to_process = message_to_process.as<trx_message>();
             dlog("passing message containing transaction ${trx} to client", ("trx", transaction_message_to_process.trx.id()));
@@ -3401,7 +3401,7 @@ namespace graphene { namespace net { namespace detail {
         {
           wlog( "client rejected message sent by peer ${peer}, ${e}", ("peer", originating_peer->get_remote_endpoint() )("e", e) );
           // record it so we don't try to fetch this item again
-          _recently_failed_items.insert(peer_connection::timestamped_item_id(item_id(message_to_process.msg_type, message_hash ), fc::time_point::now()));
+          _recently_failed_items.insert(peer_connection::timestamped_item_id(item_id(message_to_process.msg_type.value(), message_hash ), fc::time_point::now()));
           return;
         }
 
@@ -4434,13 +4434,13 @@ namespace graphene { namespace net { namespace detail {
     {
       VERIFY_CORRECT_THREAD();
       fc::uint160_t hash_of_message_contents;
-      if( item_to_broadcast.msg_type == graphene::net::block_message_type )
+      if( item_to_broadcast.msg_type.value() == graphene::net::block_message_type )
       {
         graphene::net::block_message block_message_to_broadcast = item_to_broadcast.as<graphene::net::block_message>();
         hash_of_message_contents = block_message_to_broadcast.block_id; // for debugging
         _most_recent_blocks_accepted.push_back( block_message_to_broadcast.block_id );
       }
-      else if( item_to_broadcast.msg_type == graphene::net::trx_message_type )
+      else if( item_to_broadcast.msg_type.value() == graphene::net::trx_message_type )
       {
         graphene::net::trx_message transaction_message_to_broadcast = item_to_broadcast.as<graphene::net::trx_message>();
         hash_of_message_contents = transaction_message_to_broadcast.trx.id(); // for debugging
@@ -4449,7 +4449,7 @@ namespace graphene { namespace net { namespace detail {
       message_hash_type hash_of_item_to_broadcast = item_to_broadcast.id();
 
       _message_cache.cache_message( item_to_broadcast, hash_of_item_to_broadcast, propagation_data, hash_of_message_contents );
-      _new_inventory.insert( item_id(item_to_broadcast.msg_type, hash_of_item_to_broadcast ) );
+      _new_inventory.insert( item_id(item_to_broadcast.msg_type.value(), hash_of_item_to_broadcast ) );
       trigger_advertise_inventory_loop();
     }
 
@@ -4827,9 +4827,9 @@ namespace graphene { namespace net { namespace detail {
       try
       {
         const message& message_to_deliver = destination_node->messages_to_deliver.front();
-        if (message_to_deliver.msg_type == trx_message_type)
+        if (message_to_deliver.msg_type.value() == trx_message_type)
           destination_node->delegate->handle_transaction(message_to_deliver.as<trx_message>());
-        else if (message_to_deliver.msg_type == block_message_type)
+        else if (message_to_deliver.msg_type.value() == block_message_type)
         {
           std::vector<fc::uint160_t> contained_transaction_message_ids;
           destination_node->delegate->handle_block(message_to_deliver.as<block_message>(), false, contained_transaction_message_ids);

libraries/protocol/block.cpp

@@ -21,9 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include <boost/endian/conversion.hpp>
 #include <graphene/protocol/block.hpp>
 #include <fc/io/raw.hpp>
-#include <fc/bitutil.hpp>
 #include <algorithm>
 
 namespace graphene { namespace protocol {
@@ -34,15 +34,15 @@ namespace graphene { namespace protocol {
 
    uint32_t block_header::num_from_id(const block_id_type& id)
    {
-      return fc::endian_reverse_u32(id._hash[0]);
+      return boost::endian::endian_reverse(id._hash[0].value());
    }
 
    const block_id_type& signed_block_header::id()const
    {
-      if( !_block_id._hash[0] )
+      if( !_block_id._hash[0].value() )
       {
          auto tmp = fc::sha224::hash( *this );
-         tmp._hash[0] = fc::endian_reverse_u32(block_num()); // store the block num in the ID, 160 bits is plenty for the hash
+         tmp._hash[0] = boost::endian::endian_reverse(block_num()); // store the block num in the ID, 160 bits is plenty for the hash
          static_assert( sizeof(tmp._hash[0]) == 4, "should be 4 bytes" );
          memcpy(_block_id._hash, tmp._hash, std::min(sizeof(_block_id), sizeof(tmp)));
       }
@@ -72,7 +72,7 @@ namespace graphene { namespace protocol {
       if( transactions.size() == 0 ) 
          return empty_checksum;
 
-      if( !_calculated_merkle_root._hash[0] )
+      if( !_calculated_merkle_root._hash[0].value() )
       {
          vector<digest_type> ids;
          ids.resize( transactions.size() );

libraries/protocol/include/graphene/protocol/address.hpp

@@ -62,12 +62,6 @@ namespace graphene { namespace protocol {
 
        explicit operator std::string()const; ///< converts to base58 + checksum
 
-       friend size_t hash_value( const address& v ) { 
-          const void* tmp = static_cast<const void*>(v.addr._hash+2);
-
-          const size_t* tmp2 = reinterpret_cast<const size_t*>(tmp);
-          return *tmp2;
-       }
        fc::ripemd160 addr;
    };
    inline bool operator == ( const address& a, const address& b ) { return a.addr == b.addr; }
@@ -82,18 +76,5 @@ namespace fc
    void from_variant( const fc::variant& var,  graphene::protocol::address& vo, uint32_t max_depth = 1 );
 }
 
-namespace std
-{
-   template<>
-   struct hash<graphene::protocol::address>
-   {
-       public:
-         size_t operator()(const graphene::protocol::address &a) const
-         {
-            return (uint64_t(a.addr._hash[0])<<32) | uint64_t( a.addr._hash[0] );
-         }
-   };
-}
-
 #include <fc/reflect/reflect.hpp>
 FC_REFLECT( graphene::protocol::address, (addr) )