Add Decred support (kyuupichan#550)

* Refactor reorg_hashes function

* Add Decred support

electrumx/lib/coins.py

@@ -43,7 +43,7 @@
 from electrumx.lib.hash import HASHX_LEN
 from electrumx.lib.script import ScriptPubKey, OpCodes
 import electrumx.lib.tx as lib_tx
-from electrumx.server.block_processor import BlockProcessor
+import electrumx.server.block_processor as block_proc
 import electrumx.server.daemon as daemon
 from electrumx.server.session import ElectrumX, DashElectrumX
 
@@ -69,7 +69,7 @@ class Coin(object):
     SESSIONCLS = ElectrumX
     DESERIALIZER = lib_tx.Deserializer
     DAEMON = daemon.Daemon
-    BLOCK_PROCESSOR = BlockProcessor
+    BLOCK_PROCESSOR = block_proc.BlockProcessor
     MEMPOOL_HISTOGRAM_REFRESH_SECS = 500
     XPUB_VERBYTES = bytes('????', 'utf-8')
     XPRV_VERBYTES = bytes('????', 'utf-8')
@@ -1676,6 +1676,65 @@ def block_header(cls, block, height):
         return deserializer.read_header(height, cls.BASIC_HEADER_SIZE)
 
 
+class Decred(Coin):
+    NAME = "Decred"
+    SHORTNAME = "DCR"
+    NET = "mainnet"
+    XPUB_VERBYTES = bytes.fromhex("02fda926")
+    XPRV_VERBYTES = bytes.fromhex("02fda4e8")
+    P2PKH_VERBYTE = bytes.fromhex("073f")
+    P2SH_VERBYTES = [bytes.fromhex("071a")]
+    WIF_BYTE = bytes.fromhex("230e")
+    GENESIS_HASH = ('298e5cc3d985bfe7f81dc135f360abe0'
+                    '89edd4396b86d2de66b0cef42b21d980')
+    BASIC_HEADER_SIZE = 180
+    HEADER_HASH = lib_tx.DeserializerDecred.blake256
+    DESERIALIZER = lib_tx.DeserializerDecred
+    DAEMON = daemon.DecredDaemon
+    BLOCK_PROCESSOR = block_proc.DecredBlockProcessor
+    ENCODE_CHECK = partial(Base58.encode_check,
+                           hash_fn=lib_tx.DeserializerDecred.blake256d)
+    DECODE_CHECK = partial(Base58.decode_check,
+                           hash_fn=lib_tx.DeserializerDecred.blake256d)
+    HEADER_UNPACK = struct.Struct('<i32s32s32sH6sHBBIIQIIII32sI').unpack_from
+    TX_COUNT = 4629388
+    TX_COUNT_HEIGHT = 260628
+    TX_PER_BLOCK = 17
+    REORG_LIMIT = 1000
+    RPC_PORT = 9109
+
+    @classmethod
+    def header_hash(cls, header):
+        '''Given a header return the hash.'''
+        return cls.HEADER_HASH(header)
+
+    @classmethod
+    def block(cls, raw_block, height):
+        '''Return a Block namedtuple given a raw block and its height.'''
+        if height > 0:
+            return super().block(raw_block, height)
+        else:
+            return Block(raw_block, cls.block_header(raw_block, height), [])
+
+    @classmethod
+    def electrum_header(cls, header, height):
+        labels = ('version', 'prev_block_hash', 'merkle_root', 'stake_root',
+                  'vote_bits', 'final_state', 'voters', 'fresh_stake',
+                  'revocations', 'pool_size', 'bits', 'sbits', 'block_height',
+                  'size', 'timestamp', 'nonce', 'extra_data', 'stake_version')
+        values = cls.HEADER_UNPACK(header)
+        h = dict(zip(labels, values))
+
+        # Convert some values
+        assert h['block_height'] == height
+        h['prev_block_hash'] = hash_to_hex_str(h['prev_block_hash'])
+        h['merkle_root'] = hash_to_hex_str(h['merkle_root'])
+        h['stake_root'] = hash_to_hex_str(h['stake_root'])
+        h['final_state'] = h['final_state'].hex()
+        h['extra_data'] = h['extra_data'].hex()
+        return h
+
+
 class Axe(Dash):
     NAME = "Axe"
     SHORTNAME = "AXE"

electrumx/lib/tx.py

@@ -29,6 +29,7 @@
 
 
 from collections import namedtuple
+from struct import pack
 
 from electrumx.lib.hash import sha256, double_sha256, hash_to_hex_str
 from electrumx.lib.util import (
@@ -428,8 +429,6 @@ class TxInputDcr(namedtuple("TxInput", "prev_hash prev_idx tree sequence")):
 
     @cachedproperty
     def is_coinbase(self):
-        # The previous output of a coin base must have a max value index and a
-        # zero hash.
         return (self.prev_hash == TxInputDcr.ZERO and
                 self.prev_idx == TxInputDcr.MINUS_1)
 
@@ -440,36 +439,52 @@ def __str__(self):
 
 
 class TxOutputDcr(namedtuple("TxOutput", "value version pk_script")):
-    '''Class representing a transaction output.'''
+    '''Class representing a Decred transaction output.'''
     pass
 
 
 class TxDcr(namedtuple("Tx", "version inputs outputs locktime expiry "
                              "witness")):
-    '''Class representing transaction that has a time field.'''
+    '''Class representing a Decred  transaction.'''
 
     @cachedproperty
     def is_coinbase(self):
         return self.inputs[0].is_coinbase
 
 
 class DeserializerDecred(Deserializer):
-
     @staticmethod
     def blake256(data):
         from blake256.blake256 import blake_hash
         return blake_hash(data)
 
+    @staticmethod
+    def blake256d(data):
+        from blake256.blake256 import blake_hash
+        return blake_hash(blake_hash(data))
+
+    def read_tx(self):
+        return self._read_tx_parts(produce_hash=False)[0]
+
+    def read_tx_and_hash(self):
+        tx, tx_hash, vsize = self._read_tx_parts()
+        return tx, tx_hash
+
+    def read_tx_and_vsize(self):
+        tx, tx_hash, vsize = self._read_tx_parts(produce_hash=False)
+        return tx, vsize
+
     def read_tx_block(self):
         '''Returns a list of (deserialized_tx, tx_hash) pairs.'''
-        read_tx = self.read_tx
-        txs = [read_tx() for _ in range(self._read_varint())]
-        stxs = [read_tx() for _ in range(self._read_varint())]
+        read = self.read_tx_and_hash
+        txs = [read() for _ in range(self._read_varint())]
+        stxs = [read() for _ in range(self._read_varint())]
         return txs + stxs
 
-    def _read_inputs(self):
-        read_input = self._read_input
-        return [read_input() for i in range(self._read_varint())]
+    def read_tx_tree(self):
+        '''Returns a list of deserialized_tx without tx hashes.'''
+        read_tx = self.read_tx
+        return [read_tx() for _ in range(self._read_varint())]
 
     def _read_input(self):
         return TxInputDcr(
@@ -479,10 +494,6 @@ def _read_input(self):
             self._read_le_uint32(),  # sequence
         )
 
-    def _read_outputs(self):
-        read_output = self._read_output
-        return [read_output() for _ in range(self._read_varint())]
-
     def _read_output(self):
         return TxOutputDcr(
             self._read_le_int64(),  # value
@@ -502,20 +513,34 @@ def _read_witness_field(self):
         script = self._read_varbytes()
         return value_in, block_height, block_index, script
 
-    def read_tx(self):
+    def _read_tx_parts(self, produce_hash=True):
         start = self.cursor
         version = self._read_le_int32()
         inputs = self._read_inputs()
         outputs = self._read_outputs()
         locktime = self._read_le_uint32()
         expiry = self._read_le_uint32()
-        no_witness_tx = b'\x01\x00\x01\x00' + self.binary[start+4:self.cursor]
+        end_prefix = self.cursor
         witness = self._read_witness(len(inputs))
+
+        # Drop the coinbase-like input from a vote tx as it creates problems
+        # with UTXOs lookups and mempool management
+        if inputs[0].is_coinbase and len(inputs) > 1:
+            inputs = inputs[1:]
+
+        if produce_hash:
+            # TxSerializeNoWitness << 16 == 0x10000
+            no_witness_header = pack('<I', 0x10000 | (version & 0xffff))
+            prefix_tx = no_witness_header + self.binary[start+4:end_prefix]
+            tx_hash = self.blake256(prefix_tx)
+        else:
+            tx_hash = None
+
         return TxDcr(
             version,
             inputs,
             outputs,
             locktime,
             expiry,
             witness
-        ), DeserializerDecred.blake256(no_witness_tx)
+        ), tx_hash, self.cursor - start

electrumx/server/block_processor.py

@@ -263,6 +263,16 @@ async def reorg_hashes(self, count):
         The hashes are returned in order of increasing height.  Start
         is the height of the first hash, last of the last.
         '''
+        start, count = self.calc_reorg_range(count)
+        last = start + count - 1
+        s = '' if count == 1 else 's'
+        self.logger.info(f'chain was reorganised replacing {count:,d} '
+                         f'block{s} at heights {start:,d}-{last:,d}')
+
+        return start, last, self.fs_block_hashes(start, count)
+
+    async def calc_reorg_range(self, count):
+        '''Calculate the reorg range'''
 
         def diff_pos(hashes1, hashes2):
             '''Returns the index of the first difference in the hash lists.
@@ -291,12 +301,7 @@ def diff_pos(hashes1, hashes2):
         else:
             start = (self.height - count) + 1
 
-        last = start + count - 1
-        s = '' if count == 1 else 's'
-        self.logger.info(f'chain was reorganised replacing {count:,d} '
-                         f'block{s} at heights {start:,d}-{last:,d}')
-
-        return start, last, self.fs_block_hashes(start, count)
+        return start, count
 
     def flush_state(self, batch):
         '''Flush chain state to the batch.'''
@@ -826,3 +831,13 @@ def force_chain_reorg(self, count):
             self.blocks_event.set()
             return True
         return False
+
+
+class DecredBlockProcessor(BlockProcessor):
+    async def calc_reorg_range(self, count):
+        start, count = super().calc_reorg_range(count)
+        if start > 0:
+            # A reorg in Decred can invalidate the previous block
+            start -= 1
+            count += 1
+        return start, count

electrumx/server/daemon.py

@@ -17,8 +17,10 @@
 
 import aiohttp
 
-from electrumx.lib.util import int_to_varint, hex_to_bytes, class_logger
-from electrumx.lib.hash import hex_str_to_hash
+from electrumx.lib.util import int_to_varint, hex_to_bytes, class_logger, \
+    unpack_uint16_from
+from electrumx.lib.hash import hex_str_to_hash, hash_to_hex_str
+from electrumx.lib.tx import DeserializerDecred
 from aiorpcx import JSONRPC
 
 
@@ -365,3 +367,87 @@ def timestamp_safe(self, t):
         if isinstance(t, int):
             return t
         return timegm(strptime(t, "%Y-%m-%d %H:%M:%S %Z"))
+
+
+class DecredDaemon(Daemon):
+    async def raw_blocks(self, hex_hashes):
+        '''Return the raw binary blocks with the given hex hashes.'''
+
+        params_iterable = ((h, False) for h in hex_hashes)
+        blocks = await self._send_vector('getblock', params_iterable)
+
+        raw_blocks = []
+        valid_tx_tree = {}
+        for block in blocks:
+            # Convert to bytes from hex
+            raw_block = hex_to_bytes(block)
+            raw_blocks.append(raw_block)
+            # Check if previous block is valid
+            prev = self.prev_hex_hash(raw_block)
+            votebits = unpack_uint16_from(raw_block[100:102])[0]
+            valid_tx_tree[prev] = self.is_valid_tx_tree(votebits)
+
+        processed_raw_blocks = []
+        for hash, raw_block in zip(hex_hashes, raw_blocks):
+            if hash in valid_tx_tree:
+                is_valid = valid_tx_tree[hash]
+            else:
+                # Do something complicated to figure out if this block is valid
+                header = await self._send_single('getblockheader', (hash, ))
+                if 'nextblockhash' not in header:
+                    raise DaemonError(f'Could not find next block for {hash}')
+                next_hash = header['nextblockhash']
+                next_header = await self._send_single('getblockheader',
+                                                      (next_hash, ))
+                is_valid = self.is_valid_tx_tree(next_header['votebits'])
+
+            if is_valid:
+                processed_raw_blocks.append(raw_block)
+            else:
+                # If this block is invalid remove the normal transactions
+                self.logger.info(f'block {hash} is invalidated')
+                processed_raw_blocks.append(self.strip_tx_tree(raw_block))
+
+        return processed_raw_blocks
+
+    @staticmethod
+    def prev_hex_hash(raw_block):
+        return hash_to_hex_str(raw_block[4:36])
+
+    @staticmethod
+    def is_valid_tx_tree(votebits):
+        # Check if previous block was invalidated.
+        return bool(votebits & (1 << 0) != 0)
+
+    def strip_tx_tree(self, raw_block):
+        c = self.coin
+        assert issubclass(c.DESERIALIZER, DeserializerDecred)
+        d = c.DESERIALIZER(raw_block, start=c.BASIC_HEADER_SIZE)
+        d.read_tx_tree()  # Skip normal transactions
+        # Create a fake block without any normal transactions
+        return raw_block[:c.BASIC_HEADER_SIZE] + b'\x00' + raw_block[d.cursor:]
+
+    async def height(self):
+        height = await super().height()
+        if height > 0:
+            # Lie about the daemon height as the current tip can be invalidated
+            height -= 1
+            self._height = height
+        return height
+
+    async def mempool_hashes(self):
+        mempool = await super().mempool_hashes()
+        # Add current tip transactions to the 'fake' mempool.
+        real_height = await self._send_single('getblockcount')
+        tip_hash = await self._send_single('getblockhash', (real_height,))
+        tip = await self.deserialised_block(tip_hash)
+        # Add normal transactions except coinbase
+        mempool += tip['tx'][1:]
+        # Add stake transactions if applicable
+        mempool += tip.get('stx', [])
+        return mempool
+
+    def client_session(self):
+        # FIXME allow self signed certificates
+        connector = aiohttp.TCPConnector(verify_ssl=False)
+        return aiohttp.ClientSession(connector=connector)

tests/blocks/decred_mainnet_100.json

@@ -0,0 +1,15 @@
+{
+  "hash": "0000000000017dd91008ec7c0ea63749b81d9a5188d9efc8d2d8cc0bdcff4d2a",
+  "size": 382,
+  "height": 100,
+  "merkleroot": "5c49629cefa3d5eb640a3236f6e970386e0b0826a5d33d566de36aec534fa93d",
+  "stakeroot": "0000000000000000000000000000000000000000000000000000000000000000",
+  "tx": [
+    "c813acfcad624ccf19e6240358b95cbbb1b728ee94557556dc373cceae1a7e4b"
+  ],
+  "time": 1454961067,
+  "nonce": 3396292691,
+  "bits": "1b01ffff",
+  "previousblockhash": "000000000000dcecdf2c1ae9bb3e2e3135e7765b1902938ff67e2be489ab8131",
+  "block": "010000003181ab89e42b7ef68f9302195b76e735312e3ebbe91a2cdfecdc0000000000003da94f53ec6ae36d563dd3a526080b6e3870e9f636320a64ebd5a3ef9c62495c000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000ffff011b00c2eb0b00000000640000007e010000abf1b85653506fca9885f1c26941ecf1010000000000000000000000000000000000000000000000000000000101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff00ffffffff03fa1a981200000000000017a914f5916158e3e2c4551c1796708db8367207ed13bb8700000000000000000000266a2464000000000000000000000000000000000000000000000000000000733ea5b290c04d1fdea1906f0000000000001976a9145b98376242c78de2003e7940d7e44270c39b83eb88ac000000000000000001d8bc28820000000000000000ffffffff0800002f646372642f00"
+}