Bitcoin Core v0.17.0 compatibility patch

- support new label API
- support new signrawtransactionwithkey RPC method

doc/release-notes/release-notes-v0.9.9.md

@@ -4,6 +4,10 @@
 
  - Full Ethereum (`adef0b3`), Ethereum Classic (`d4eb8f6`) and ERC20 token (`881d559`) support
 
+   Testing level for this feature has moved from EXPERIMENTAL to BETA
+
    For usage details, see https://github.com/mmgen/mmgen/wiki/Altcoin-and-Forkcoin-Support
 
+   NOTE: This release is compatible only with Bitcoin Core v0.16.3 and older. A compatibility patch for v0.17.0 and newer will be included in forthcoming sub-release 0.9.9a
+
    This is a Linux-only release

doc/release-notes/release-notes-v0.9.9a.md

@@ -0,0 +1,5 @@
+### MMGen Version 0.9.9a Release Notes
+
+   Compatibility release for Bitcoin Core v0.17.0    
+   - support new label API
+   - support new signrawtransactionwithkey RPC method

mmgen/addr.py

@@ -904,8 +904,12 @@ re-import your addresses.
 	@classmethod
 	def get_tw_data(cls):
 		vmsg('Getting address data from tracking wallet')
-		accts = g.rpch.listaccounts(0,True)
-		alists = g.rpch.getaddressesbyaccount([[k] for k in accts],batch=True)
+		if 'label_api' in g.rpch.caps:
+			accts = g.rpch.listlabels()
+			alists = [a.keys() for a in g.rpch.getaddressesbylabel([[k] for k in accts],batch=True)]
+		else:
+			accts = g.rpch.listaccounts(0,True)
+			alists = g.rpch.getaddressesbyaccount([[k] for k in accts],batch=True)
 		return zip(accts,alists)
 
 	def add_tw_data(self):

mmgen/altcoins/eth/tw.py

@@ -139,7 +139,7 @@ class EthereumTrackingWallet(TrackingWallet):
 		return OrderedDict(map(lambda x: (x['mmid'],{'addr':x['addr'],'comment':x['comment']}), self.sorted_list()))
 
 	@write_mode
-	def import_label(self,coinaddr,lbl):
+	def set_label(self,coinaddr,lbl):
 		for addr,d in self.data_root().items():
 			if addr == coinaddr:
 				d['comment'] = lbl.comment
@@ -192,8 +192,8 @@ Actions:         [q]uit view, [p]rint to file, pager [v]iew, [w]ide view,
 	key_mappings = {
 		'a':'s_amt','d':'s_addr','r':'d_reverse','M':'s_twmmid',
 		'm':'d_mmid','e':'d_redraw',
-		'q':'a_quit','p':'a_print','v':'a_view','w':'a_view_wide','l':'a_lbl_add',
-		'R':'a_addr_remove' }
+		'q':'a_quit','p':'a_print','v':'a_view','w':'a_view_wide',
+		'l':'a_lbl_add','R':'a_addr_remove' }
 
 	def do_sort(self,key=None,reverse=False):
 		if key == 'txid': return

mmgen/exception.py

@@ -24,3 +24,4 @@ mmgen.exception: Exception classes for the MMGen suite
 class UnrecognizedTokenSymbol(Exception): pass
 class TokenNotInBlockchain(Exception): pass
 class UserNonConfirmation(Exception): pass
+class RPCFailure(Exception): pass

mmgen/globalvars.py

@@ -80,7 +80,6 @@ class g(object):
 	accept_defaults      = False
 	chain                = None # set by first call to rpc_init()
 	chains               = 'mainnet','testnet','regtest'
-	daemon_version       = None # set by first call to rpc_init()
 	rpc_host             = ''
 	rpc_port             = 0
 	rpc_user             = ''

mmgen/main.py

@@ -62,4 +62,5 @@ def launch(what):
 
 				from mmgen.util import die,ydie
 				if type(e).__name__ == 'UserNonConfirmation': die(1,m)
-				else: ydie(2,u'\nERROR: ' + m)
+				if type(e).__name__ == 'RPCFailure': ydie(2,m)
+				ydie(2,u'\nERROR: ' + m)

mmgen/protocol.py

@@ -103,10 +103,6 @@ class BitcoinProtocol(MMGenObject):
 	def get_protocol_by_chain(chain):
 		return CoinProtocol(g.coin,{'mainnet':False,'testnet':True,'regtest':True}[chain])
 
-	@staticmethod
-	def get_rpc_coin_amt_type():
-		return (float,str)[g.daemon_version>=120000]
-
 	@classmethod
 	def cap(cls,s): return s in cls.caps
 

mmgen/regtest.py

@@ -80,7 +80,8 @@ def test_daemon():
 	p = start_cmd('cli','getblockcount',quiet=True)
 	err = process_output(p,silent=True)[1]
 	ret,state = p.wait(),None
-	if "error: couldn't connect" in err: state = 'stopped'
+	if "error: couldn't connect" in err or "error: Could not connect" in err:
+		state = 'stopped'
 	if not state: state = ('busy','ready')[ret==0]
 	return state
 

mmgen/rpc.py

@@ -28,8 +28,6 @@ from decimal import Decimal
 def dmsg_rpc(s):
 	if g.debug_rpc: msg(s)
 
-class RPCFailure(Exception): pass
-
 class CoinDaemonRPCConnection(object):
 
 	auth = True
@@ -77,15 +75,15 @@ class CoinDaemonRPCConnection(object):
 	# Batch mode:  call with list of arg lists as first argument
 	# kwargs are for local use and are not passed to server
 
-	# By default, dies with an error msg on all errors and exceptions
-	# on_fail is one of 'die' (default), 'return', 'silent', 'raise'
+	# By default, raises RPCFailure exception with an error msg on all errors and exceptions
+	# on_fail is one of 'raise' (default), 'return', 'silent' or 'die'
 	# With on_fail='return', returns 'rpcfail',(resp_object,(die_args))
 	def request(self,cmd,*args,**kwargs):
 
 		if os.getenv('MMGEN_RPC_FAIL_ON_COMMAND') == cmd:
 			cmd = 'badcommand_' + cmd
 
-		cf = { 'timeout':g.http_timeout, 'batch':False, 'on_fail':'die' }
+		cf = { 'timeout':g.http_timeout, 'batch':False, 'on_fail':'raise' }
 
 		for k in cf:
 			if k in kwargs and kwargs[k]: cf[k] = kwargs[k]
@@ -112,10 +110,11 @@ class CoinDaemonRPCConnection(object):
 		dmsg_rpc('=== request() debug ===')
 		dmsg_rpc('    RPC POST data ==> {}\n'.format(p))
 
+		parent = self
 		class MyJSONEncoder(json.JSONEncoder):
 			def default(self,obj):
 				if isinstance(obj,g.proto.coin_amt):
-					return g.proto.get_rpc_coin_amt_type()(obj)
+					return parent.coin_amt_type(obj)
 				return json.JSONEncoder.default(self,obj)
 
 		http_hdr = { 'Content-Type': 'application/json' }
@@ -180,6 +179,7 @@ class CoinDaemonRPCConnection(object):
 		'estimatefee',
 		'estimatesmartfee',
 		'getaddressesbyaccount',
+		'getaddressesbylabel',
 		'getbalance',
 		'getblock',
 		'getblockchaininfo',
@@ -196,9 +196,12 @@ class CoinDaemonRPCConnection(object):
 		'gettransaction',
 		'importaddress',
 		'listaccounts',
+		'listlabels',
 		'listunspent',
+		'setlabel',
 		'sendrawtransaction',
 		'signrawtransaction',
+		'signrawtransactionwithkey', # method new to Core v0.17.0
 		'validateaddress',
 		'walletpassphrase',
 	)

mmgen/tw.py

@@ -110,9 +110,10 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 		if not us_rpc: die(0,self.wmsg['no_spendable_outputs'])
 		confs_per_day = 60*60*24 / g.proto.secs_per_block
 		tr_rpc = []
+		lbl_id = ('account','label')['label_api' in g.rpch.caps]
 		for o in us_rpc:
-			if not 'account' in o: continue          # coinbase outputs have no account field
-			l = TwLabel(o['account'],on_fail='silent')
+			if not lbl_id in o: continue          # coinbase outputs have no account field
+			l = TwLabel(o[lbl_id],on_fail='silent')
 			if l:
 				o.update({
 					'twmmid': l.mmid,
@@ -403,10 +404,11 @@ class TwAddrList(MMGenDict):
 		self.total = g.proto.coin_amt('0')
 		rpc_init()
 
+		lbl_id = ('account','label')['label_api' in g.rpch.caps]
 		for d in g.rpch.listunspent(0):
-			if not 'account' in d: continue  # skip coinbase outputs with missing account
+			if not lbl_id in d: continue  # skip coinbase outputs with missing account
 			if d['confirmations'] < minconf: continue
-			label = TwLabel(d['account'],on_fail='silent')
+			label = TwLabel(d[lbl_id],on_fail='silent')
 			if label:
 				if usr_addr_list and (label.mmid not in usr_addr_list): continue
 				if label.mmid in self:
@@ -425,10 +427,14 @@ class TwAddrList(MMGenDict):
 		if showempty or all_labels:
 			# for compatibility with old mmids, must use raw RPC rather than native data for matching
 			# args: minconf,watchonly, MUST use keys() so we get list, not dict
-			acct_list = g.rpch.listaccounts(0,True).keys() # raw list, no 'L'
+			if 'label_api' in g.rpch.caps:
+				acct_list = g.rpch.listlabels()
+				acct_addrs = [a.keys() for a in g.rpch.getaddressesbylabel([[k] for k in acct_list],batch=True)]
+			else:
+				acct_list = g.rpch.listaccounts(0,True).keys() # raw list, no 'L'
+				acct_addrs = g.rpch.getaddressesbyaccount([[a] for a in acct_list],batch=True) # use raw list here
 			acct_labels = MMGenList([TwLabel(a,on_fail='silent') for a in acct_list])
 			check_dup_mmid(acct_labels)
-			acct_addrs = g.rpch.getaddressesbyaccount([[a] for a in acct_list],batch=True) # use raw list here
 			assert len(acct_list) == len(acct_addrs),(
 				'listaccounts() and getaddressesbyaccount() not equal in length')
 			addr_pairs = zip(acct_labels,acct_addrs)
@@ -442,6 +448,11 @@ class TwAddrList(MMGenDict):
 					if showbtcaddrs:
 						self[label.mmid]['addr'] = CoinAddr(addr_arr[0])
 
+	def raw_list(self):
+		return [((k if k.type == 'mmgen' else 'Non-MMGen'),self[k]['addr'],self[k]['amt']) for k in self]
+
+	def coinaddr_list(self): return [self[k]['addr'] for k in self]
+
 	def format(self,showbtcaddrs,sort,show_age,show_days):
 		out = ['Chain: '+green(g.chain.upper())] if g.chain != 'mainnet' else []
 		fs = u'{{mid}}{} {{cmt}} {{amt}}{}'.format(('',' {addr}')[showbtcaddrs],('',' {age}')[show_age])
@@ -519,16 +530,17 @@ class TrackingWallet(MMGenObject):
 	def write(self): pass
 
 	def is_in_wallet(self,addr):
-		d = g.rpch.validateaddress(addr)
-		return d['iswatchonly'] and 'account' in d
+		return addr in TwAddrList([],0,True,True,True).coinaddr_list()
 
 	@write_mode
-	def import_label(self,coinaddr,lbl):
-		# NOTE: this works because importaddress() removes the old account before
-		# associating the new account with the address.
-		# Will be replaced by setlabel() with new RPC label API
-		# RPC args: addr,label,rescan[=true],p2sh[=none]
-		return g.rpch.importaddress(coinaddr,lbl,False,on_fail='return')
+	def set_label(self,coinaddr,lbl):
+		if 'label_api' in g.rpch.caps:
+			return g.rpch.setlabel(coinaddr,lbl,on_fail='return')
+		else:
+			# NOTE: this works because importaddress() removes the old account before
+			# associating the new account with the address.
+			# RPC args: addr,label,rescan[=true],p2sh[=none]
+			return g.rpch.importaddress(coinaddr,lbl,False,on_fail='return')
 
 	# returns on failure
 	@write_mode
@@ -568,7 +580,7 @@ class TrackingWallet(MMGenObject):
 
 		lbl = TwLabel(mmaddr + ('',' '+cmt)[bool(cmt)],on_fail=on_fail)
 
-		ret = self.import_label(coinaddr,lbl)
+		ret = self.set_label(coinaddr,lbl)
 
 		from mmgen.rpc import rpc_error,rpc_errmsg
 		if rpc_error(ret):
@@ -609,8 +621,9 @@ class TwGetBalance(MMGenObject):
 
 	def create_data(self):
 		# 0: unconfirmed, 1: below minconf, 2: confirmed, 3: spendable
+		lbl_id = ('account','label')['label_api' in g.rpch.caps]
 		for d in g.rpch.listunspent(0):
-			try:    lbl = TwLabel(d['account'],on_fail='silent')
+			try: lbl = TwLabel(d[lbl_id],on_fail='silent')
 			except: lbl = None
 			if lbl:
 				if lbl.mmid.type == 'mmgen':

mmgen/tx.py

@@ -496,7 +496,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 
 	def get_rel_fee_from_network(self): # rel_fee is in BTC/kB
 		try:
-			ret = g.rpch.estimatesmartfee(opt.tx_confs,on_fail='raise')
+			ret = g.rpch.estimatesmartfee(opt.tx_confs)
 			rel_fee = ret['feerate'] if 'feerate' in ret else -2
 			fe_type = 'estimatesmartfee'
 		except:
@@ -714,36 +714,35 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 
 		msg_r('Signing transaction{}...'.format(tx_num_str))
 		wifs = [d.sec.wif for d in keys]
-		ret = g.rpch.signrawtransaction(self.hex,sig_data,wifs,g.proto.sighash_type,on_fail='return')
 
-		from mmgen.rpc import rpc_error,rpc_errmsg
-		if rpc_error(ret):
-			errmsg = rpc_errmsg(ret)
-			if 'Invalid sighash param' in errmsg:
+		try:
+			ret = g.rpch.signrawtransactionwithkey(self.hex,wifs,sig_data,g.proto.sighash_type) \
+				if 'sign_with_key' in g.rpch.caps else \
+					g.rpch.signrawtransaction(self.hex,sig_data,wifs,g.proto.sighash_type)
+		except Exception as e:
+			if 'Invalid sighash param' in e.message:
 				m  = 'This is not the BCH chain.'
 				m += "\nRe-run the script without the --coin=bch option."
 			else:
-				m = errmsg
+				m = e.message
 			msg(yellow(m))
 			return False
+
+		if ret['complete']:
+			self.hex = ret['hex']
+			self.compare_size_and_estimated_size()
+			dt = DeserializedTX(self.hex)
+			self.check_hex_tx_matches_mmgen_tx(dt)
+			self.coin_txid = CoinTxID(dt['txid'],on_fail='return')
+			self.check_sigs(dt)
+			assert self.coin_txid == g.rpch.decoderawtransaction(self.hex)['txid'],(
+										'txid mismatch (after signing)')
+			msg('OK')
+			return True
 		else:
-			if ret['complete']:
-#				Msg(pretty_hexdump(unhexlify(self.hex),cols=16)) # DEBUG
-#				pmsg(make_chksum_6(unhexlify(self.hex)).upper())
-				self.hex = ret['hex']
-				self.compare_size_and_estimated_size()
-				dt = DeserializedTX(self.hex)
-				self.check_hex_tx_matches_mmgen_tx(dt)
-				self.coin_txid = CoinTxID(dt['txid'],on_fail='return')
-				self.check_sigs(dt)
-				assert self.coin_txid == g.rpch.decoderawtransaction(self.hex)['txid'],(
-											'txid mismatch (after signing)')
-				msg('OK')
-				return True
-			else:
-				msg('failed\n{} returned the following errors:'.format(g.proto.daemon_name.capitalize()))
-				msg(repr(ret['errors']))
-				return False
+			msg('failed\n{} returned the following errors:'.format(g.proto.daemon_name.capitalize()))
+			msg(repr(ret['errors']))
+			return False
 
 	def mark_raw(self):
 		self.desc = 'transaction'

mmgen/util.py

@@ -847,12 +847,13 @@ def rpc_init_parity():
 				g.rpc_host or 'localhost',
 				g.rpc_port or g.proto.rpc_port)
 
-	if not g.daemon_version: # First call
-		g.daemon_version = g.rpch.parity_versionInfo()['version'] # fail immediately if daemon is geth
-		g.chain = g.rpch.parity_chain().replace(' ','_')
-		if g.token:
-			(g.token,g.dcoin) = resolve_token_arg(g.token)
+	g.rpch.daemon_version = g.rpch.parity_versionInfo()['version'] # fail immediately if daemon is geth
+	g.rpch.coin_amt_type = str
+	g.chain = g.rpch.parity_chain().replace(' ','_')
+	if g.token:
+		(g.token,g.dcoin) = resolve_token_arg(g.token)
 
+	g.rpch.caps = ()
 	return g.rpch
 
 def rpc_init_bitcoind():
@@ -887,19 +888,25 @@ def rpc_init_bitcoind():
 				g.rpc_password or cfg['rpcpassword'],
 				auth_cookie=get_coin_daemon_auth_cookie())
 
-	if not g.daemon_version: # First call
-		if g.bob or g.alice:
-			import regtest as rt
-			rt.user(('alice','bob')[g.bob],quiet=True)
-		g.daemon_version = int(conn.getnetworkinfo()['version'])
-		g.chain = conn.getblockchaininfo()['chain']
-		if g.chain != 'regtest': g.chain += 'net'
-		assert g.chain in g.chains
-		check_chaintype_mismatch()
+	if g.bob or g.alice:
+		import regtest as rt
+		rt.user(('alice','bob')[g.bob],quiet=True)
+	conn.daemon_version = int(conn.getnetworkinfo()['version'])
+	conn.coin_amt_type = (float,str)[conn.daemon_version>=120000]
+	g.chain = conn.getblockchaininfo()['chain']
+	if g.chain != 'regtest': g.chain += 'net'
+	assert g.chain in g.chains
+	check_chaintype_mismatch()
 
 	if g.chain == 'mainnet': # skip this for testnet, as Genesis block may change
 		check_chainfork_mismatch(conn)
 
+	conn.caps = ()
+	for func,cap in (
+		('setlabel','label_api'),
+		('signrawtransactionwithkey','sign_with_key') ):
+		if len(conn.request('help',func).split('\n')) > 3:
+			conn.caps += (cap,)
 	return conn
 
 def rpc_init(reinit=False):

test/test.py

@@ -164,6 +164,7 @@ opt.popen_spawn = True # popen has issues, so use popen_spawn always
 
 if not opt.system: os.environ['PYTHONPATH'] = repo_root
 
+lbl_id = ('account','label')[g.coin=='BTC'] # update as other coins adopt Core's label API
 ref_subdir = '' if g.proto.base_coin == 'BTC' else 'ethereum_classic' if g.coin == 'ETC' else g.proto.name
 altcoin_pfx = '' if g.proto.base_coin == 'BTC' else '-'+g.proto.base_coin
 tn_ext = ('','.testnet')[g.testnet]
@@ -847,7 +848,7 @@ cmd_group['regtest'] = (
 	('regtest_alice_add_label_badaddr2','adding a label with invalid address for this chain'),
 	('regtest_alice_add_label_badaddr3','adding a label with wrong MMGen address'),
 	('regtest_alice_add_label_badaddr4','adding a label with wrong coin address'),
-	('regtest_alice_add_label_rpcfail','RPC failure code'),
+	('regtest_alice_bal_rpcfail','RPC failure code'),
 	('regtest_alice_send_estimatefee','tx creation with no fee on command line'),
 	('regtest_generate',           'mining a block'),
 	('regtest_bob_bal6',           "Bob's balance"),
@@ -1313,7 +1314,7 @@ def create_fake_unspent_entry(coinaddr,al_id=None,idx=None,lbl=None,non_mmgen=Fa
 					'bech32': (g.proto.witness_vernum_hex+'14','') }[k]
 	amt1,amt2 = {'btc':(10,40),'bch':(10,40),'ltc':(1000,4000)}[coin_sel]
 	ret = {
-		'account': '{}:{}'.format(g.proto.base_coin.lower(),coinaddr) if non_mmgen \
+		lbl_id: '{}:{}'.format(g.proto.base_coin.lower(),coinaddr) if non_mmgen \
 			else (u'{}:{}{}'.format(al_id,idx,lbl)),
 		'vout': int(getrandnum(4) % 8),
 		'txid': unicode(hexlify(os.urandom(32))),
@@ -1913,9 +1914,9 @@ class MMGenTestSuite(object):
 		if cmdline_inputs:
 			from mmgen.tx import TwLabel
 			cmd_args = ['--inputs={},{},{},{},{},{}'.format(
-				TwLabel(dfake[0]['account']).mmid,dfake[1]['address'],
-				TwLabel(dfake[2]['account']).mmid,dfake[3]['address'],
-				TwLabel(dfake[4]['account']).mmid,dfake[5]['address']
+				TwLabel(dfake[0][lbl_id]).mmid,dfake[1]['address'],
+				TwLabel(dfake[2][lbl_id]).mmid,dfake[3]['address'],
+				TwLabel(dfake[4][lbl_id]).mmid,dfake[5]['address']
 				),'--outdir='+trash_dir] + cmd_args[1:]
 		end_silence()
 
@@ -3101,11 +3102,14 @@ class MMGenTestSuite(object):
 		return self.regtest_alice_add_label_badaddr(name,addr,
 			"Address '{}' not found in tracking wallet".format(addr))
 
-	def regtest_alice_add_label_rpcfail(self,name):
+	def regtest_alice_bal_rpcfail(self,name):
 		addr = self.regtest_user_sid('alice') + ':C:2'
-		os.environ['MMGEN_RPC_FAIL_ON_COMMAND'] = 'importaddress'
-		self.regtest_alice_add_label_badaddr(name,addr,'Label could not be added')
+		os.environ['MMGEN_RPC_FAIL_ON_COMMAND'] = 'listunspent'
+		t = MMGenExpect(name,'mmgen-tool',['--alice','getbalance'])
 		os.environ['MMGEN_RPC_FAIL_ON_COMMAND'] = ''
+		t.expect('Method not found')
+		t.read()
+		ok()
 
 	def regtest_alice_remove_label1(self,name):
 		sid = self.regtest_user_sid('alice')