Merge branch 'segwit'

bitcoin/script.py

@@ -17,6 +17,8 @@
 from base58 import b58decode
 from util import dblsha
 
+WitnessMagic = b'\xaa\x21\xa9\xed'
+
 def _Address2PKH(addr):
 	try:
 		addr = b58decode(addr, 25)
@@ -43,6 +45,13 @@ def toAddress(cls, addr):
 		elif ver == 5 or ver == 196:
 			return b'\xa9\x14' + pubkeyhash + b'\x87'
 		raise ValueError('invalid address version')
+
+	@classmethod
+	def commitment(cls, commitment):
+		clen = len(commitment)
+		if clen > 0x4b:
+			raise NotImplementedError
+		return b'\x6a' + bytes((clen,)) + commitment
 
 def countSigOps(s):
 	# FIXME: don't count data as ops

bitcoin/txn.py

@@ -22,10 +22,16 @@
 _nullprev = b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0'
 
 class Txn:
-	def __init__(self, data=None):
+	def __init__(self, data=None, txid=None):
 		if data:
 			self.data = data
-			self.idhash()
+			if txid:
+				self.txid = txid
+			else:
+				try:
+					self.idhash()
+				except NotImplementedError:
+					pass
 
 	@classmethod
 	def new(cls):
@@ -49,6 +55,9 @@ def addOutput(self, amount, pkScript):
 		self.outputs.append( (amount, pkScript) )
 
 	def disassemble(self, retExtra = False):
+		if self.data[4:6] == b'\0\1':
+			raise NotImplementedError
+
 		self.version = unpack('<L', self.data[:4])[0]
 		rc = [4]
 
@@ -115,7 +124,21 @@ def assemble(self):
 		self.idhash()
 
 	def idhash(self):
+		if self.data[4:6] == b'\0\1':
+			if hasattr(self, 'txid'):
+				del self.txid
+			raise NotImplementedError
 		self.txid = dblsha(self.data)
+		if hasattr(self, 'witness_hash'):
+			del self.witness_hash
+
+	def withash(self):
+		self.witness_hash = dblsha(self.data)
+
+	def get_witness_hash(self):
+		if not hasattr(self, 'witness_hash'):
+			self.withash()
+		return self.witness_hash
 
 # Txn tests
 def _test():

eloipool.py

@@ -95,15 +95,18 @@ def RaiseRedFlags(reason):
 	pass
 
 
-from bitcoin.script import BitcoinScript
+from bitcoin.script import BitcoinScript, WitnessMagic
 from bitcoin.txn import Txn
 from base58 import b58decode
 from binascii import b2a_hex
 from struct import pack
 import subprocess
 from time import time
 
-def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True, prevBlockHex = None):
+def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True, prevBlockHex = None, witness_commitment = NotImplemented):
+	if witness_commitment is NotImplemented:
+		raise NotImplementedError
+
 	txn = Txn.new()
 
 	if useCoinbaser and hasattr(config, 'CoinbaserCmd') and config.CoinbaserCmd:
@@ -131,6 +134,10 @@ def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True, prevBlockHex = None):
 	pkScript = BitcoinScript.toAddress(config.TrackerAddr)
 	txn.addOutput(coinbaseValue, pkScript)
 
+	# SegWit commitment
+	if not witness_commitment is None:
+		txn.addOutput(0, BitcoinScript.commitment(WitnessMagic + witness_commitment))
+
 	# TODO
 	# TODO: red flag on dupe coinbase
 	return txn

jsonrpc_getblocktemplate.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from binascii import b2a_hex
+from bitcoin.script import BitcoinScript, WitnessMagic
 from copy import deepcopy
 from jsonrpcserver import JSONRPCHandler
 from time import time
@@ -37,10 +38,6 @@ def final_init(server):
 		'target': '00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff',
 		'version': 3,
 		'submitold': True,
-
-		# Bitcoin-specific:
-		'sigoplimit': 20000,
-		'sizelimit': 1000000,
 	}
 	def doJSON_getblocktemplate(self, params):
 		if 'mode' in params and params['mode'] != 'template':
@@ -66,9 +63,16 @@ def doJSON_getblocktemplate(self, params):
 		else:
 			rv['longpollid'] = str(self.server.LPId)
 		tl = []
+		SegwitTemplate = False
+		for rule in merkleTree.MP['rules']:
+			if rule == 'segwit' or rule == '!segwit':
+				SegwitTemplate = True
+				break
 		for txn in merkleTree.data[1:]:
 			txno = {}
 			txno['data'] = b2a_hex(txn.data).decode('ascii')
+			if SegwitTemplate:
+				txno['txid'] = b2a_hex(txn.txid[::-1]).decode('ascii')
 			tl.append(txno)
 		rv['transactions'] = tl
 		now = int(time())
@@ -82,6 +86,9 @@ def doJSON_getblocktemplate(self, params):
 		rv['target'] = '%064x' % (target,)
 		t = deepcopy(merkleTree.data[0])
 		t.setCoinbase(cb)
+		if not merkleTree.witness_commitment is None:
+			assert t.outputs[-1] == (0, BitcoinScript.commitment(WitnessMagic + merkleTree.witness_commitment))
+			t.outputs.pop()
 		t.assemble()
 		txno = {}
 		txno['data'] = b2a_hex(t.data).decode('ascii')

merklemaker.py

@@ -16,7 +16,7 @@
 
 from binascii import b2a_hex
 import bitcoin.script
-from bitcoin.script import countSigOps
+from bitcoin.script import BitcoinScript, countSigOps
 from bitcoin.txn import Txn
 from bitcoin.varlen import varlenEncode, varlenDecode
 from collections import deque
@@ -32,11 +32,12 @@
 import threading
 from time import sleep, time
 import traceback
+import util
 
 _makeCoinbase = [0, 0]
 _filecounter = 0
 
-SupportedRules = ('csv',)
+SupportedRules = ('csv', 'segwit')
 
 def SplitRuleFlag(ruleflag):
 	MandatoryRule = (ruleflag[0] == '!')
@@ -45,17 +46,36 @@ def SplitRuleFlag(ruleflag):
 	else:
 		return (False, ruleflag)
 
+def CalculateWitnessCommitment(txnobjs, nonce, force=False):
+	gentx_withash = nonce
+	withashes = (gentx_withash,) + tuple(a.get_witness_hash() for a in txnobjs[1:])
+
+	if not force:
+		txids = (gentx_withash,) + tuple(a.txid for a in txnobjs[1:])
+		if withashes == txids:
+			# Unnecessary
+			return None
+
+	wmr = MerkleTree(data=withashes).merkleRoot()
+	commitment = util.dblsha(wmr + nonce)
+	return commitment
+
 def MakeBlockHeader(MRD):
 	(merkleRoot, merkleTree, coinbase, prevBlock, bits) = MRD[:5]
 	BlockVersionBytes = merkleTree.MP['_BlockVersionBytes']
 	timestamp = pack('<L', int(time()))
 	hdr = BlockVersionBytes + prevBlock + merkleRoot + timestamp + bits + b'iolE'
 	return hdr
 
-def assembleBlock(blkhdr, txlist):
+def assembleBlock(blkhdr, txlist, wantGenTxNonce=None):
 	payload = blkhdr
 	payload += varlenEncode(len(txlist))
-	for tx in txlist:
+	gentxdata = txlist[0].data
+	assert gentxdata[4:6] != b'\0\1'
+	if wantGenTxNonce:
+		gentxdata = gentxdata[:4] + b'\0\1' + gentxdata[4:-4] + b'\x01\x20' + wantGenTxNonce + gentxdata[-4:]
+	payload += gentxdata
+	for tx in txlist[1:]:
 		payload += tx.data
 	return payload
 
@@ -94,6 +114,8 @@ def __init__(self, *a, **k):
 		self.currentBlock = (None, None, None)
 		self.lastBlock = (None, None, None)
 		self.SubsidyAlgo = lambda height: 5000000000 >> (height // 210000)
+		self.WitnessNonce = b'\0' * 0x20
+		self.ForceWitnessCommitment = False
 
 	def _prepare(self):
 		self.UseTemplateChecks = True
@@ -181,11 +203,13 @@ def UpdateClearMerkleTree(self, MT, MP):
 
 	def createClearMerkleTree(self, height):
 		subsidy = self.SubsidyAlgo(height)
-		cbtxn = self.makeCoinbaseTxn(subsidy, False)
+		cbtxn = self.makeCoinbaseTxn(subsidy, False, witness_commitment=None)
+		cbtxn.setCoinbase(b'\0\0')  # necessary to avoid triggering segwit marker+flags
 		cbtxn.assemble()
 		MT = MerkleTree([cbtxn])
 		if self.currentMerkleTree:
 			self.UpdateClearMerkleTree(MT, self.currentMerkleTree.MP)
+		MT.witness_commitment = None
 		return MT
 
 	def updateBlock(self, newBlock, height = None, bits = None, _HBH = None):
@@ -295,15 +319,17 @@ def _APOT(self, txninfopot, MP, POTInfo):
 		return True
 
 	def _makeBlockSafe(self, MP, txnlist, txninfo):
+		sizelimit = MP.get('sizelimit', 1000000) - 0x10000  # 64 KB breathing room
 		blocksize = sum(map(len, txnlist)) + 80
-		while blocksize > 934464:  # 1 "MB" limit - 64 KB breathing room
+		while blocksize > sizelimit:
 			txnsize = len(txnlist[-1])
 			self._trimBlock(MP, txnlist, txninfo, 'SizeLimit', lambda x: 'Making blocks over 1 MB size limit (%d bytes; %s)' % (blocksize, x))
 			blocksize -= txnsize
 
 		# NOTE: This check doesn't work at all without BIP22 transaction obj format
+		sigoplimit = MP.get('sigoplimit', 20000) - 0x200  # 512 sigop breathing room
 		blocksigops = sum(a.get('sigops', 0) for a in txninfo)
-		while blocksigops > 19488:  # 20k limit - 0x200 breathing room
+		while blocksigops > sigoplimit:
 			txnsigops = txninfo[-1]['sigops']
 			self._trimBlock(MP, txnlist, txninfo, 'SigOpLimit', lambda x: 'Making blocks over 20k SigOp limit (%d; %s)' % (blocksigops, x))
 			blocksigops -= txnsigops
@@ -429,20 +455,27 @@ def _ProcessGBT(self, MP, TS = None):
 			self.logger.error('Template from \'%s\' should be trimmed, but requires unsupported rule(s)', TS['name'])
 			return None
 
-		cbtxn = self.makeCoinbaseTxn(MP['coinbasevalue'], prevBlockHex = MP['previousblockhash'])
+		txnobjs = [None]
+		for i in range(len(txnlist)):
+			iinfo = txninfo[i]
+			ka = {}
+			if 'txid' in iinfo:
+				ka['txid'] = bytes.fromhex(iinfo['txid'])[::-1]
+			txnobjs.append(Txn(data=txnlist[i], **ka))
+
+		witness_commitment = CalculateWitnessCommitment(txnobjs, self.WitnessNonce, force=self.ForceWitnessCommitment)
+
+		cbtxn = self.makeCoinbaseTxn(MP['coinbasevalue'], prevBlockHex = MP['previousblockhash'], witness_commitment=witness_commitment)
 		cbtxn.setCoinbase(b'\0\0')
 		cbtxn.assemble()
-		txnlist.insert(0, cbtxn.data)
-		txninfo.insert(0, {
-		})
-
-		txnlist = [a for a in map(Txn, txnlist[1:])]
-		txnlist.insert(0, cbtxn)
-		txnlist = list(txnlist)
-		newMerkleTree = MerkleTree(txnlist)
+		txnobjs[0] = cbtxn
+
+		txnobjs = list(txnobjs)
+		newMerkleTree = MerkleTree(txnobjs)
 		newMerkleTree.POTInfo = MP.get('POTInfo')
 		newMerkleTree.MP = MP
 		newMerkleTree.oMP = oMP
+		newMerkleTree.witness_commitment = witness_commitment
 
 		return newMerkleTree
 
@@ -470,10 +503,13 @@ def _CheckTemplate(self, newMerkleTree, TS):
 		coinbase = self.makeCoinbase(height=height)
 		cbtxn.setCoinbase(coinbase)
 		cbtxn.assemble()
+		newMerkleTree.recalculate(True)
 		merkleRoot = newMerkleTree.merkleRoot()
 		MRD = (merkleRoot, newMerkleTree, coinbase, prevBlock, bits)
 		blkhdr = MakeBlockHeader(MRD)
-		data = assembleBlock(blkhdr, txnlist)
+		need_gentx_nonce = (not newMerkleTree.witness_commitment is None)
+		# 0.13 doesn't allow segwit-enabled block proposals without the generation transaction in witness form with its nonce
+		data = assembleBlock(blkhdr, txnlist, wantGenTxNonce=self.WitnessNonce if need_gentx_nonce else None)
 		ProposeReq = {
 			"mode": "proposal",
 			"data": b2a_hex(data).decode('utf8'),
@@ -856,9 +892,9 @@ def MBS(LO = 0):
 	txninfo[2]['fee'] = 0
 	assert MBS(1) == (MP, txnlist, txninfo)
 	# _ProcessGBT tests
-	def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True, prevBlockHex = None):
+	def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True, prevBlockHex = None, witness_commitment=None):
 		txn = Txn.new()
-		txn.addOutput(coinbaseValue, b'')
+		txn.addOutput(coinbaseValue, BitcoinScript.commitment(witness_commitment) if witness_commitment else b'')
 		return txn
 	MM.makeCoinbaseTxn = makeCoinbaseTxn
 	MM.updateBlock = lambda *a, **ka: None

merkletree.py

@@ -38,9 +38,11 @@ def recalculate(self, detailed=False):
 			PreL = [None]
 			StartL = 2
 		Ll = len(L)
+		if isinstance(L[1] if Ll > 1 else L[0], Txn):
+			L = list(map(lambda a: a.txid if a else a, L))
+		else:
+			L = list(L)
 		if detailed or Ll > 1:
-			if isinstance(L[1] if Ll > 1 else L[0], Txn):
-				L = list(map(lambda a: a.txid if a else a, L))
 			while True:
 				if detailed:
 					detail += L