use pyrlp>=1.0.0 (#153)

* bump rlp>=1.0.0; move utils to tools/rlp_utils.py

* fix all unittests because of pyrlp>=1.0.0

* move rlp_utils to quarkchain/rlp/

.circleci/config.yml

@@ -60,15 +60,15 @@ jobs:
             pypy3 -m unittest
 
       - run:
-          name: Run EVM Tests
+          name: Run P2P Tests
           command: |
-            git apply fixtures/0001-Necessary-changes-to-let-evm-tests-pass.patch
-            pypy3 quarkchain/evm/tests/test_state.py fixtures/GeneralStateTests/
+            pypy3 -m pytest quarkchain/p2p
 
       - run:
-          name: Run P2P Tests
+          name: Run EVM Tests
           command: |
-            pypy3 -m pytest quarkchain/p2p
+            git apply fixtures/0001-Necessary-changes-to-let-evm-tests-pass.patch
+            pypy3 quarkchain/evm/tests/test_state.py fixtures/GeneralStateTests/
 
       - store_artifacts:
           path: test-reports

devp2p/app.py

@@ -6,7 +6,7 @@
 from .slogging import get_logger
 from devp2p import utils
 from devp2p import crypto
-from rlp.utils import decode_hex
+from quarkchain.rlp.utils import decode_hex
 from devp2p import __version__
 log = get_logger('app')
 

devp2p/app_helper.py

@@ -5,7 +5,7 @@
 from devp2p.discovery import NodeDiscovery
 from devp2p.crypto import privtopub as privtopub_raw, sha3
 from devp2p.utils import host_port_pubkey_to_uri, update_config_with_defaults
-from rlp.utils import encode_hex
+from quarkchain.rlp.utils import encode_hex
 import gevent
 import copy
 from .slogging import get_logger

devp2p/crypto.py

@@ -5,7 +5,7 @@
 import pyelliptic
 
 import bitcoin
-from rlp.utils import str_to_bytes, safe_ord, ascii_chr
+from quarkchain.rlp.utils import str_to_bytes, safe_ord, ascii_chr
 from hashlib import sha256
 import struct
 from coincurve import PrivateKey, PublicKey

devp2p/discovery.py

@@ -8,7 +8,7 @@
 import gevent.socket
 import ipaddress
 import rlp
-from rlp.utils import decode_hex, is_integer, str_to_bytes, bytes_to_str, safe_ord
+from quarkchain.rlp.utils import decode_hex, is_integer, str_to_bytes, bytes_to_str, safe_ord
 from gevent.server import DatagramServer
 
 from devp2p import slogging

devp2p/examples/full_app.py

@@ -7,7 +7,7 @@
 from devp2p.utils import colors, COLOR_END
 from devp2p import app_helper
 import rlp
-from rlp.utils import encode_hex, decode_hex, is_integer
+from quarkchain.rlp.utils import encode_hex, decode_hex, is_integer
 import gevent
 try:
     import ethereum.slogging as slogging

devp2p/kademlia.py

@@ -23,7 +23,7 @@
 import logging
 from .crypto import sha3
 from .utils import big_endian_to_int
-from rlp.utils import encode_hex, is_integer, str_to_bytes
+from quarkchain.rlp.utils import encode_hex, is_integer, str_to_bytes
 
 log = slogging.get_logger('p2p.discovery.kademlia')
 

devp2p/multiplexer.py

@@ -1,7 +1,7 @@
 from gevent.queue import Queue
 from collections import OrderedDict
 import rlp
-from rlp.utils import str_to_bytes, is_integer
+from quarkchain.rlp.utils import str_to_bytes, is_integer
 import struct
 import sys
 

devp2p/peer.py

@@ -12,7 +12,7 @@
 from devp2p import slogging
 import gevent.socket
 from devp2p import rlpxcipher
-from rlp.utils import decode_hex
+from quarkchain.rlp.utils import decode_hex
 
 log = slogging.get_logger('p2p.peer')
 

devp2p/peermanager.py

@@ -14,7 +14,7 @@
 from .peer import Peer
 from devp2p import crypto
 from devp2p import utils
-from rlp.utils import decode_hex
+from quarkchain.rlp.utils import decode_hex
 
 from devp2p import slogging
 log = slogging.get_logger('p2p.peermgr')

devp2p/rlpxcipher.py

@@ -4,7 +4,7 @@
 import os
 import rlp
 from rlp import sedes
-from rlp.utils import safe_ord, str_to_bytes, ascii_chr
+from quarkchain.rlp.utils import safe_ord, str_to_bytes, ascii_chr
 from devp2p.crypto import sha3
 from Crypto.Hash import keccak
 from devp2p.crypto import ECCx

devp2p/tests/test_crypto.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 from devp2p import crypto
-from rlp.utils import decode_hex
+from quarkchain.rlp.utils import decode_hex
 import random
 import pytest
 

devp2p/tests/test_discovery.py

@@ -2,7 +2,7 @@
 from devp2p import kademlia
 from devp2p import crypto
 from devp2p.app import BaseApp
-from rlp.utils import decode_hex, encode_hex
+from quarkchain.rlp.utils import decode_hex, encode_hex
 from devp2p.utils import remove_chars
 import pytest
 import gevent

devp2p/tests/test_ecies.py

@@ -1,5 +1,5 @@
 import devp2p.crypto as crypto
-from rlp.utils import decode_hex
+from quarkchain.rlp.utils import decode_hex
 
 
 def test_ecies_enc():

devp2p/tests/test_go_handshake.py

@@ -4,7 +4,7 @@
 
 from devp2p.rlpxcipher import RLPxSession
 from devp2p.crypto import ECCx, privtopub
-from rlp.utils import decode_hex
+from quarkchain.rlp.utils import decode_hex
 test_values = \
     {
         "initiator_private_key": "5e173f6ac3c669587538e7727cf19b782a4f2fda07c1eaa662c593e5e85e3051",

devp2p/tests/test_go_signature.py

@@ -1,5 +1,5 @@
 from devp2p.crypto import ecdsa_sign, mk_privkey, privtopub, ecdsa_recover, ECCx
-from rlp.utils import decode_hex
+from quarkchain.rlp.utils import decode_hex
 import pyelliptic
 
 

devp2p/tests/test_p2pprotocol.py

@@ -4,7 +4,7 @@
 from devp2p.multiplexer import Packet
 from devp2p.utils import remove_chars
 import pytest
-from rlp.utils import decode_hex
+from quarkchain.rlp.utils import decode_hex
 # notify peer of successfulll handshake!
 # so other protocols get registered
 # so other protocols can do their handshake

devp2p/tests/test_peer.py

@@ -11,7 +11,7 @@
 import time
 import gevent
 import copy
-from rlp.utils import encode_hex, decode_hex, str_to_bytes
+from quarkchain.rlp.utils import encode_hex, decode_hex, str_to_bytes
 
 
 def get_connected_apps():

devp2p/tests/test_peermanager.py

@@ -1,7 +1,7 @@
 from devp2p import peermanager
 from devp2p import crypto
 from devp2p.app import BaseApp
-from rlp.utils import encode_hex
+from quarkchain.rlp.utils import encode_hex
 import devp2p.muxsession
 import rlp
 import devp2p.p2p_protocol

devp2p/tests/test_rlpxsession.py

@@ -2,7 +2,7 @@
 from devp2p.crypto import mk_privkey, ECCx, sha3
 from devp2p.multiplexer import Multiplexer, Packet
 from devp2p.utils import remove_chars
-from rlp.utils import decode_hex, str_to_bytes
+from quarkchain.rlp.utils import decode_hex, str_to_bytes
 import struct
 
 

devp2p/utils.py

@@ -1,8 +1,8 @@
 from __future__ import print_function
 import struct
 import rlp
-from rlp.utils import encode_hex, decode_hex, str_to_bytes, bytes_to_str
-from rlp.utils import safe_ord
+from quarkchain.rlp.utils import encode_hex, decode_hex, str_to_bytes, bytes_to_str
+from quarkchain.rlp.utils import safe_ord
 import collections
 import sys
 

fixtures/0001-Necessary-changes-to-let-evm-tests-pass.patch

@@ -14,10 +14,10 @@ python3 quarkchain/evm/tests/test_state.py fixtures/GeneralStateTests/
  3 files changed, 8 insertions(+), 8 deletions(-)
 
 diff --git a/quarkchain/evm/messages.py b/quarkchain/evm/messages.py
-index e59be92..7af6160 100644
+index 0a32dd3..f043e1f 100644
 --- a/quarkchain/evm/messages.py
 +++ b/quarkchain/evm/messages.py
-@@ -154,8 +154,8 @@ def validate_transaction(state, tx):
+@@ -136,8 +136,8 @@ def validate_transaction(state, tx):
      if not tx.sender:  # sender is set and validated on Transaction initialization
          raise UnsignedTransaction(tx)
 
@@ -28,7 +28,7 @@ index e59be92..7af6160 100644
 
      # (2) the transaction nonce is valid (equivalent to the
      #     sender account's current nonce);
-@@ -201,7 +201,7 @@ def apply_message(state, msg=None, **kwargs):
+@@ -183,7 +183,7 @@ def apply_message(state, msg=None, **kwargs):
      return bytearray_to_bytestr(data) if result else None
 
 
@@ -37,7 +37,7 @@ index e59be92..7af6160 100644
      """tx_wrapper_hash is the hash for quarkchain.core.Transaction
      TODO: remove quarkchain.core.Transaction wrapper and use evm.Transaction directly
      """
-@@ -442,9 +442,7 @@ def _apply_msg(ext, msg, code):
+@@ -424,9 +424,7 @@ def _apply_msg(ext, msg, code):
 
 
  def mk_contract_address(sender, full_shard_id, nonce):
@@ -49,36 +49,62 @@ index e59be92..7af6160 100644
 
  def create_contract(ext, msg):
 diff --git a/quarkchain/evm/state.py b/quarkchain/evm/state.py
-index 600f091..4a47b2c 100644
+index ca4094b..0541f25 100644
 --- a/quarkchain/evm/state.py
 +++ b/quarkchain/evm/state.py
-@@ -56,15 +56,15 @@ class Account(rlp.Serializable):
+@@ -55,24 +55,23 @@ class _Account(rlp.Serializable):
          ('balance', big_endian_int),
          ('storage', trie_root),
          ('code_hash', hash32),
 -        ('full_shard_id', BigEndianInt(4)),
      ]
 
+
+ class Account(rlp.Serializable):
+
 -    def __init__(self, nonce, balance, storage, code_hash, full_shard_id, env, address, db=None):
 +    def __init__(self, nonce, balance, storage, code_hash, full_shard_id=0, env=None, address=None, db=None):
          self.db = env.db if db is None else db
          assert isinstance(db, Db)
          self.env = env
          self.address = address
--        super(Account, self).__init__(nonce, balance, storage, code_hash, full_shard_id)
+
+-        acc = _Account(nonce, balance, storage, code_hash, full_shard_id)
++        acc = _Account(nonce, balance, storage, code_hash)
+         self.nonce = acc.nonce
+         self.balance = acc.balance
+         self.storage = acc.storage
+         self.code_hash = acc.code_hash
+-        self.full_shard_id = acc.full_shard_id
 +        self.full_shard_id = full_shard_id
-+        super(Account, self).__init__(nonce, balance, storage, code_hash)
+
          self.storage_cache = {}
          self.storage_trie = SecureTrie(Trie(self.db))
-         self.storage_trie.root_hash = self.storage
+@@ -194,7 +193,6 @@ class State:
+                 balance=o.balance,
+                 storage=o.storage,
+                 code_hash=o.code_hash,
+-                full_shard_id=o.full_shard_id,
+                 env=self.env,
+                 address=address,
+                 db=self.db,
+@@ -399,7 +397,7 @@ class State:
+                 self.deletes.extend(acct.storage_trie.deletes)
+                 self.changed[addr] = True
+                 if self.account_exists(addr) or allow_empties:
+-                    _acct = _Account(acct.nonce, acct.balance, acct.storage, acct.code_hash, acct.full_shard_id)
++                    _acct = _Account(acct.nonce, acct.balance, acct.storage, acct.code_hash)
+                     self.trie.update(addr, rlp.encode(_acct))
+                     if self.executing_on_head:
+                         self.db.put(b'address:' + addr, rlp.encode(_acct))
 diff --git a/quarkchain/evm/transactions.py b/quarkchain/evm/transactions.py
-index 7ee7221..6836d93 100644
+index c4758ec..02be464 100644
 --- a/quarkchain/evm/transactions.py
 +++ b/quarkchain/evm/transactions.py
-@@ -72,7 +72,7 @@ class Transaction(rlp.Serializable):
+@@ -71,7 +71,7 @@ class Transaction(rlp.Serializable):
+
      def __init__(self, nonce, gasprice, startgas, to, value, data,
                   v=0, r=0, s=0, from_full_shard_id=0, to_full_shard_id=0, network_id=1, version=0):
-         self.data = None
 -        self.shard_size = 0
 +        self.shard_size = 1
 

quarkchain/evm/config.py

@@ -1,4 +1,4 @@
-from rlp.utils import decode_hex
+from quarkchain.rlp.utils import decode_hex
 
 from quarkchain.evm import utils
 from quarkchain.db import InMemoryDb, Db

quarkchain/evm/messages.py

@@ -7,7 +7,7 @@
 from quarkchain.evm.utils import int256, safe_ord, bytearray_to_bytestr
 from rlp.sedes import big_endian_int, binary, CountableList, BigEndianInt
 from rlp.sedes.binary import Binary
-from rlp.utils import decode_hex, encode_hex
+from quarkchain.rlp.utils import decode_hex, encode_hex
 from quarkchain.evm import utils  # FIXME: use eth_utils
 from quarkchain.evm import bloom  # FIXME: use eth_bloom
 from quarkchain.evm import transactions
@@ -92,40 +92,22 @@ class Receipt(rlp.Serializable):
         ("contract_full_shard_id", BigEndianInt(4)),
     ]
 
-    def __init__(
-        self,
-        state_root,
-        gas_used,
-        logs,
-        contract_address,
-        contract_full_shard_id,
-        bloom=None,
-    ):
-        # does not call super.__init__ as bloom should not be an attribute but
-        # a property
-        self.state_root = state_root
-        self.gas_used = gas_used
-        self.logs = logs
-        if bloom is not None and bloom != self.bloom:
-            raise ValueError("Invalid bloom filter")
-        self.contract_address = contract_address
-        self.contract_full_shard_id = contract_full_shard_id
-        self._cached_rlp = None
-        self._mutable = True
-
     @property
     def bloom(self):
         bloomables = [x.bloomables() for x in self.logs]
         return bloom.bloom_from_list(utils.flatten(bloomables))
 
 
 def mk_receipt(state, success, logs, contract_address, contract_full_shard_id):
+    bloomables = [x.bloomables() for x in logs]
+    ret_bloom = bloom.bloom_from_list(utils.flatten(bloomables))
     o = Receipt(
-        b"\x01" if success else b"",
-        state.gas_used,
-        logs,
-        contract_address,
-        contract_full_shard_id,
+        state_root=b"\x01" if success else b"",
+        gas_used=state.gas_used,
+        bloom=ret_bloom,
+        logs=logs,
+        contract_address=contract_address,
+        contract_full_shard_id=contract_full_shard_id,
     )
     return o
 

quarkchain/evm/specials.py

@@ -1,7 +1,7 @@
 # -*- coding: utf8 -*-
 from py_ecc.secp256k1 import privtopub, ecdsa_raw_recover, N as secp256k1n
 import hashlib
-from rlp.utils import ascii_chr
+from quarkchain.rlp.utils import ascii_chr
 
 from quarkchain.evm import utils, opcodes
 from quarkchain.evm.utils import safe_ord, decode_hex, encode_int32