From 0408c4e3047677e7ae64594037c7af04e0435e29 Mon Sep 17 00:00:00 2001 From: MMGen Date: Fri, 19 Oct 2018 00:30:04 +0000 Subject: [PATCH] Bitcoin Core v0.17.0 compatibility patch - support new label API - support new signrawtransactionwithkey RPC method --- doc/release-notes/release-notes-v0.9.9.md | 4 ++ doc/release-notes/release-notes-v0.9.9a.md | 5 +++ mmgen/addr.py | 8 +++- mmgen/altcoins/eth/tw.py | 6 +-- mmgen/exception.py | 1 + mmgen/globalvars.py | 1 - mmgen/main.py | 3 +- mmgen/protocol.py | 4 -- mmgen/regtest.py | 3 +- mmgen/rpc.py | 15 ++++--- mmgen/tw.py | 45 +++++++++++++-------- mmgen/tx.py | 47 +++++++++++----------- mmgen/util.py | 35 +++++++++------- test/test.py | 20 +++++---- 14 files changed, 117 insertions(+), 80 deletions(-) create mode 100644 doc/release-notes/release-notes-v0.9.9a.md diff --git a/doc/release-notes/release-notes-v0.9.9.md b/doc/release-notes/release-notes-v0.9.9.md index 38b25d17..b912b14d 100644 --- a/doc/release-notes/release-notes-v0.9.9.md +++ b/doc/release-notes/release-notes-v0.9.9.md @@ -4,6 +4,10 @@ - Full Ethereum (`adef0b3`), Ethereum Classic (`d4eb8f6`) and ERC20 token (`881d559`) support + Testing level for this feature has moved from EXPERIMENTAL to BETA + For usage details, see https://github.com/mmgen/mmgen/wiki/Altcoin-and-Forkcoin-Support + NOTE: This release is compatible only with Bitcoin Core v0.16.3 and older. A compatibility patch for v0.17.0 and newer will be included in forthcoming sub-release 0.9.9a + This is a Linux-only release diff --git a/doc/release-notes/release-notes-v0.9.9a.md b/doc/release-notes/release-notes-v0.9.9a.md new file mode 100644 index 00000000..9ab96ae3 --- /dev/null +++ b/doc/release-notes/release-notes-v0.9.9a.md @@ -0,0 +1,5 @@ +### MMGen Version 0.9.9a Release Notes + + Compatibility release for Bitcoin Core v0.17.0 + - support new label API + - support new signrawtransactionwithkey RPC method diff --git a/mmgen/addr.py b/mmgen/addr.py index bca63aa3..3fbd7b12 100755 --- a/mmgen/addr.py +++ b/mmgen/addr.py @@ -904,8 +904,12 @@ re-import your addresses. @classmethod def get_tw_data(cls): vmsg('Getting address data from tracking wallet') - accts = g.rpch.listaccounts(0,True) - alists = g.rpch.getaddressesbyaccount([[k] for k in accts],batch=True) + if 'label_api' in g.rpch.caps: + accts = g.rpch.listlabels() + alists = [a.keys() for a in g.rpch.getaddressesbylabel([[k] for k in accts],batch=True)] + else: + accts = g.rpch.listaccounts(0,True) + alists = g.rpch.getaddressesbyaccount([[k] for k in accts],batch=True) return zip(accts,alists) def add_tw_data(self): diff --git a/mmgen/altcoins/eth/tw.py b/mmgen/altcoins/eth/tw.py index 84e21ad1..00143004 100755 --- a/mmgen/altcoins/eth/tw.py +++ b/mmgen/altcoins/eth/tw.py @@ -139,7 +139,7 @@ class EthereumTrackingWallet(TrackingWallet): return OrderedDict(map(lambda x: (x['mmid'],{'addr':x['addr'],'comment':x['comment']}), self.sorted_list())) @write_mode - def import_label(self,coinaddr,lbl): + def set_label(self,coinaddr,lbl): for addr,d in self.data_root().items(): if addr == coinaddr: d['comment'] = lbl.comment @@ -192,8 +192,8 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, key_mappings = { 'a':'s_amt','d':'s_addr','r':'d_reverse','M':'s_twmmid', 'm':'d_mmid','e':'d_redraw', - 'q':'a_quit','p':'a_print','v':'a_view','w':'a_view_wide','l':'a_lbl_add', - 'R':'a_addr_remove' } + 'q':'a_quit','p':'a_print','v':'a_view','w':'a_view_wide', + 'l':'a_lbl_add','R':'a_addr_remove' } def do_sort(self,key=None,reverse=False): if key == 'txid': return diff --git a/mmgen/exception.py b/mmgen/exception.py index 7fb09219..e98d0bea 100755 --- a/mmgen/exception.py +++ b/mmgen/exception.py @@ -24,3 +24,4 @@ mmgen.exception: Exception classes for the MMGen suite class UnrecognizedTokenSymbol(Exception): pass class TokenNotInBlockchain(Exception): pass class UserNonConfirmation(Exception): pass +class RPCFailure(Exception): pass diff --git a/mmgen/globalvars.py b/mmgen/globalvars.py index bc678fea..004ac760 100755 --- a/mmgen/globalvars.py +++ b/mmgen/globalvars.py @@ -80,7 +80,6 @@ class g(object): accept_defaults = False chain = None # set by first call to rpc_init() chains = 'mainnet','testnet','regtest' - daemon_version = None # set by first call to rpc_init() rpc_host = '' rpc_port = 0 rpc_user = '' diff --git a/mmgen/main.py b/mmgen/main.py index 446cb6d9..5685f420 100755 --- a/mmgen/main.py +++ b/mmgen/main.py @@ -62,4 +62,5 @@ def launch(what): from mmgen.util import die,ydie if type(e).__name__ == 'UserNonConfirmation': die(1,m) - else: ydie(2,u'\nERROR: ' + m) + if type(e).__name__ == 'RPCFailure': ydie(2,m) + ydie(2,u'\nERROR: ' + m) diff --git a/mmgen/protocol.py b/mmgen/protocol.py index 2aa66cd1..c7d09e42 100755 --- a/mmgen/protocol.py +++ b/mmgen/protocol.py @@ -103,10 +103,6 @@ class BitcoinProtocol(MMGenObject): def get_protocol_by_chain(chain): return CoinProtocol(g.coin,{'mainnet':False,'testnet':True,'regtest':True}[chain]) - @staticmethod - def get_rpc_coin_amt_type(): - return (float,str)[g.daemon_version>=120000] - @classmethod def cap(cls,s): return s in cls.caps diff --git a/mmgen/regtest.py b/mmgen/regtest.py index 237bde00..af7ce606 100755 --- a/mmgen/regtest.py +++ b/mmgen/regtest.py @@ -80,7 +80,8 @@ def test_daemon(): p = start_cmd('cli','getblockcount',quiet=True) err = process_output(p,silent=True)[1] ret,state = p.wait(),None - if "error: couldn't connect" in err: state = 'stopped' + if "error: couldn't connect" in err or "error: Could not connect" in err: + state = 'stopped' if not state: state = ('busy','ready')[ret==0] return state diff --git a/mmgen/rpc.py b/mmgen/rpc.py index 7508872b..b5b13362 100755 --- a/mmgen/rpc.py +++ b/mmgen/rpc.py @@ -28,8 +28,6 @@ from decimal import Decimal def dmsg_rpc(s): if g.debug_rpc: msg(s) -class RPCFailure(Exception): pass - class CoinDaemonRPCConnection(object): auth = True @@ -77,15 +75,15 @@ class CoinDaemonRPCConnection(object): # Batch mode: call with list of arg lists as first argument # kwargs are for local use and are not passed to server - # By default, dies with an error msg on all errors and exceptions - # on_fail is one of 'die' (default), 'return', 'silent', 'raise' + # By default, raises RPCFailure exception with an error msg on all errors and exceptions + # on_fail is one of 'raise' (default), 'return', 'silent' or 'die' # With on_fail='return', returns 'rpcfail',(resp_object,(die_args)) def request(self,cmd,*args,**kwargs): if os.getenv('MMGEN_RPC_FAIL_ON_COMMAND') == cmd: cmd = 'badcommand_' + cmd - cf = { 'timeout':g.http_timeout, 'batch':False, 'on_fail':'die' } + cf = { 'timeout':g.http_timeout, 'batch':False, 'on_fail':'raise' } for k in cf: if k in kwargs and kwargs[k]: cf[k] = kwargs[k] @@ -112,10 +110,11 @@ class CoinDaemonRPCConnection(object): dmsg_rpc('=== request() debug ===') dmsg_rpc(' RPC POST data ==> {}\n'.format(p)) + parent = self class MyJSONEncoder(json.JSONEncoder): def default(self,obj): if isinstance(obj,g.proto.coin_amt): - return g.proto.get_rpc_coin_amt_type()(obj) + return parent.coin_amt_type(obj) return json.JSONEncoder.default(self,obj) http_hdr = { 'Content-Type': 'application/json' } @@ -180,6 +179,7 @@ class CoinDaemonRPCConnection(object): 'estimatefee', 'estimatesmartfee', 'getaddressesbyaccount', + 'getaddressesbylabel', 'getbalance', 'getblock', 'getblockchaininfo', @@ -196,9 +196,12 @@ class CoinDaemonRPCConnection(object): 'gettransaction', 'importaddress', 'listaccounts', + 'listlabels', 'listunspent', + 'setlabel', 'sendrawtransaction', 'signrawtransaction', + 'signrawtransactionwithkey', # method new to Core v0.17.0 'validateaddress', 'walletpassphrase', ) diff --git a/mmgen/tw.py b/mmgen/tw.py index d441de62..ac202e62 100755 --- a/mmgen/tw.py +++ b/mmgen/tw.py @@ -110,9 +110,10 @@ watch-only wallet using '{}-addrimport' and then re-run this program. if not us_rpc: die(0,self.wmsg['no_spendable_outputs']) confs_per_day = 60*60*24 / g.proto.secs_per_block tr_rpc = [] + lbl_id = ('account','label')['label_api' in g.rpch.caps] for o in us_rpc: - if not 'account' in o: continue # coinbase outputs have no account field - l = TwLabel(o['account'],on_fail='silent') + if not lbl_id in o: continue # coinbase outputs have no account field + l = TwLabel(o[lbl_id],on_fail='silent') if l: o.update({ 'twmmid': l.mmid, @@ -403,10 +404,11 @@ class TwAddrList(MMGenDict): self.total = g.proto.coin_amt('0') rpc_init() + lbl_id = ('account','label')['label_api' in g.rpch.caps] for d in g.rpch.listunspent(0): - if not 'account' in d: continue # skip coinbase outputs with missing account + if not lbl_id in d: continue # skip coinbase outputs with missing account if d['confirmations'] < minconf: continue - label = TwLabel(d['account'],on_fail='silent') + label = TwLabel(d[lbl_id],on_fail='silent') if label: if usr_addr_list and (label.mmid not in usr_addr_list): continue if label.mmid in self: @@ -425,10 +427,14 @@ class TwAddrList(MMGenDict): if showempty or all_labels: # for compatibility with old mmids, must use raw RPC rather than native data for matching # args: minconf,watchonly, MUST use keys() so we get list, not dict - acct_list = g.rpch.listaccounts(0,True).keys() # raw list, no 'L' + if 'label_api' in g.rpch.caps: + acct_list = g.rpch.listlabels() + acct_addrs = [a.keys() for a in g.rpch.getaddressesbylabel([[k] for k in acct_list],batch=True)] + else: + acct_list = g.rpch.listaccounts(0,True).keys() # raw list, no 'L' + acct_addrs = g.rpch.getaddressesbyaccount([[a] for a in acct_list],batch=True) # use raw list here acct_labels = MMGenList([TwLabel(a,on_fail='silent') for a in acct_list]) check_dup_mmid(acct_labels) - acct_addrs = g.rpch.getaddressesbyaccount([[a] for a in acct_list],batch=True) # use raw list here assert len(acct_list) == len(acct_addrs),( 'listaccounts() and getaddressesbyaccount() not equal in length') addr_pairs = zip(acct_labels,acct_addrs) @@ -442,6 +448,11 @@ class TwAddrList(MMGenDict): if showbtcaddrs: self[label.mmid]['addr'] = CoinAddr(addr_arr[0]) + def raw_list(self): + return [((k if k.type == 'mmgen' else 'Non-MMGen'),self[k]['addr'],self[k]['amt']) for k in self] + + def coinaddr_list(self): return [self[k]['addr'] for k in self] + def format(self,showbtcaddrs,sort,show_age,show_days): out = ['Chain: '+green(g.chain.upper())] if g.chain != 'mainnet' else [] fs = u'{{mid}}{} {{cmt}} {{amt}}{}'.format(('',' {addr}')[showbtcaddrs],('',' {age}')[show_age]) @@ -519,16 +530,17 @@ class TrackingWallet(MMGenObject): def write(self): pass def is_in_wallet(self,addr): - d = g.rpch.validateaddress(addr) - return d['iswatchonly'] and 'account' in d + return addr in TwAddrList([],0,True,True,True).coinaddr_list() @write_mode - def import_label(self,coinaddr,lbl): - # NOTE: this works because importaddress() removes the old account before - # associating the new account with the address. - # Will be replaced by setlabel() with new RPC label API - # RPC args: addr,label,rescan[=true],p2sh[=none] - return g.rpch.importaddress(coinaddr,lbl,False,on_fail='return') + def set_label(self,coinaddr,lbl): + if 'label_api' in g.rpch.caps: + return g.rpch.setlabel(coinaddr,lbl,on_fail='return') + else: + # NOTE: this works because importaddress() removes the old account before + # associating the new account with the address. + # RPC args: addr,label,rescan[=true],p2sh[=none] + return g.rpch.importaddress(coinaddr,lbl,False,on_fail='return') # returns on failure @write_mode @@ -568,7 +580,7 @@ class TrackingWallet(MMGenObject): lbl = TwLabel(mmaddr + ('',' '+cmt)[bool(cmt)],on_fail=on_fail) - ret = self.import_label(coinaddr,lbl) + ret = self.set_label(coinaddr,lbl) from mmgen.rpc import rpc_error,rpc_errmsg if rpc_error(ret): @@ -609,8 +621,9 @@ class TwGetBalance(MMGenObject): def create_data(self): # 0: unconfirmed, 1: below minconf, 2: confirmed, 3: spendable + lbl_id = ('account','label')['label_api' in g.rpch.caps] for d in g.rpch.listunspent(0): - try: lbl = TwLabel(d['account'],on_fail='silent') + try: lbl = TwLabel(d[lbl_id],on_fail='silent') except: lbl = None if lbl: if lbl.mmid.type == 'mmgen': diff --git a/mmgen/tx.py b/mmgen/tx.py index 4044c6a3..a91f256b 100755 --- a/mmgen/tx.py +++ b/mmgen/tx.py @@ -496,7 +496,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam def get_rel_fee_from_network(self): # rel_fee is in BTC/kB try: - ret = g.rpch.estimatesmartfee(opt.tx_confs,on_fail='raise') + ret = g.rpch.estimatesmartfee(opt.tx_confs) rel_fee = ret['feerate'] if 'feerate' in ret else -2 fe_type = 'estimatesmartfee' except: @@ -714,36 +714,35 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam msg_r('Signing transaction{}...'.format(tx_num_str)) wifs = [d.sec.wif for d in keys] - ret = g.rpch.signrawtransaction(self.hex,sig_data,wifs,g.proto.sighash_type,on_fail='return') - from mmgen.rpc import rpc_error,rpc_errmsg - if rpc_error(ret): - errmsg = rpc_errmsg(ret) - if 'Invalid sighash param' in errmsg: + try: + ret = g.rpch.signrawtransactionwithkey(self.hex,wifs,sig_data,g.proto.sighash_type) \ + if 'sign_with_key' in g.rpch.caps else \ + g.rpch.signrawtransaction(self.hex,sig_data,wifs,g.proto.sighash_type) + except Exception as e: + if 'Invalid sighash param' in e.message: m = 'This is not the BCH chain.' m += "\nRe-run the script without the --coin=bch option." else: - m = errmsg + m = e.message msg(yellow(m)) return False + + if ret['complete']: + self.hex = ret['hex'] + self.compare_size_and_estimated_size() + dt = DeserializedTX(self.hex) + self.check_hex_tx_matches_mmgen_tx(dt) + self.coin_txid = CoinTxID(dt['txid'],on_fail='return') + self.check_sigs(dt) + assert self.coin_txid == g.rpch.decoderawtransaction(self.hex)['txid'],( + 'txid mismatch (after signing)') + msg('OK') + return True else: - if ret['complete']: -# Msg(pretty_hexdump(unhexlify(self.hex),cols=16)) # DEBUG -# pmsg(make_chksum_6(unhexlify(self.hex)).upper()) - self.hex = ret['hex'] - self.compare_size_and_estimated_size() - dt = DeserializedTX(self.hex) - self.check_hex_tx_matches_mmgen_tx(dt) - self.coin_txid = CoinTxID(dt['txid'],on_fail='return') - self.check_sigs(dt) - assert self.coin_txid == g.rpch.decoderawtransaction(self.hex)['txid'],( - 'txid mismatch (after signing)') - msg('OK') - return True - else: - msg('failed\n{} returned the following errors:'.format(g.proto.daemon_name.capitalize())) - msg(repr(ret['errors'])) - return False + msg('failed\n{} returned the following errors:'.format(g.proto.daemon_name.capitalize())) + msg(repr(ret['errors'])) + return False def mark_raw(self): self.desc = 'transaction' diff --git a/mmgen/util.py b/mmgen/util.py index c650dcfc..754f4ee4 100755 --- a/mmgen/util.py +++ b/mmgen/util.py @@ -847,12 +847,13 @@ def rpc_init_parity(): g.rpc_host or 'localhost', g.rpc_port or g.proto.rpc_port) - if not g.daemon_version: # First call - g.daemon_version = g.rpch.parity_versionInfo()['version'] # fail immediately if daemon is geth - g.chain = g.rpch.parity_chain().replace(' ','_') - if g.token: - (g.token,g.dcoin) = resolve_token_arg(g.token) + g.rpch.daemon_version = g.rpch.parity_versionInfo()['version'] # fail immediately if daemon is geth + g.rpch.coin_amt_type = str + g.chain = g.rpch.parity_chain().replace(' ','_') + if g.token: + (g.token,g.dcoin) = resolve_token_arg(g.token) + g.rpch.caps = () return g.rpch def rpc_init_bitcoind(): @@ -887,19 +888,25 @@ def rpc_init_bitcoind(): g.rpc_password or cfg['rpcpassword'], auth_cookie=get_coin_daemon_auth_cookie()) - if not g.daemon_version: # First call - if g.bob or g.alice: - import regtest as rt - rt.user(('alice','bob')[g.bob],quiet=True) - g.daemon_version = int(conn.getnetworkinfo()['version']) - g.chain = conn.getblockchaininfo()['chain'] - if g.chain != 'regtest': g.chain += 'net' - assert g.chain in g.chains - check_chaintype_mismatch() + if g.bob or g.alice: + import regtest as rt + rt.user(('alice','bob')[g.bob],quiet=True) + conn.daemon_version = int(conn.getnetworkinfo()['version']) + conn.coin_amt_type = (float,str)[conn.daemon_version>=120000] + g.chain = conn.getblockchaininfo()['chain'] + if g.chain != 'regtest': g.chain += 'net' + assert g.chain in g.chains + check_chaintype_mismatch() if g.chain == 'mainnet': # skip this for testnet, as Genesis block may change check_chainfork_mismatch(conn) + conn.caps = () + for func,cap in ( + ('setlabel','label_api'), + ('signrawtransactionwithkey','sign_with_key') ): + if len(conn.request('help',func).split('\n')) > 3: + conn.caps += (cap,) return conn def rpc_init(reinit=False): diff --git a/test/test.py b/test/test.py index 6cb93b14..c3d0126d 100755 --- a/test/test.py +++ b/test/test.py @@ -164,6 +164,7 @@ opt.popen_spawn = True # popen has issues, so use popen_spawn always if not opt.system: os.environ['PYTHONPATH'] = repo_root +lbl_id = ('account','label')[g.coin=='BTC'] # update as other coins adopt Core's label API ref_subdir = '' if g.proto.base_coin == 'BTC' else 'ethereum_classic' if g.coin == 'ETC' else g.proto.name altcoin_pfx = '' if g.proto.base_coin == 'BTC' else '-'+g.proto.base_coin tn_ext = ('','.testnet')[g.testnet] @@ -847,7 +848,7 @@ cmd_group['regtest'] = ( ('regtest_alice_add_label_badaddr2','adding a label with invalid address for this chain'), ('regtest_alice_add_label_badaddr3','adding a label with wrong MMGen address'), ('regtest_alice_add_label_badaddr4','adding a label with wrong coin address'), - ('regtest_alice_add_label_rpcfail','RPC failure code'), + ('regtest_alice_bal_rpcfail','RPC failure code'), ('regtest_alice_send_estimatefee','tx creation with no fee on command line'), ('regtest_generate', 'mining a block'), ('regtest_bob_bal6', "Bob's balance"), @@ -1313,7 +1314,7 @@ def create_fake_unspent_entry(coinaddr,al_id=None,idx=None,lbl=None,non_mmgen=Fa 'bech32': (g.proto.witness_vernum_hex+'14','') }[k] amt1,amt2 = {'btc':(10,40),'bch':(10,40),'ltc':(1000,4000)}[coin_sel] ret = { - 'account': '{}:{}'.format(g.proto.base_coin.lower(),coinaddr) if non_mmgen \ + lbl_id: '{}:{}'.format(g.proto.base_coin.lower(),coinaddr) if non_mmgen \ else (u'{}:{}{}'.format(al_id,idx,lbl)), 'vout': int(getrandnum(4) % 8), 'txid': unicode(hexlify(os.urandom(32))), @@ -1913,9 +1914,9 @@ class MMGenTestSuite(object): if cmdline_inputs: from mmgen.tx import TwLabel cmd_args = ['--inputs={},{},{},{},{},{}'.format( - TwLabel(dfake[0]['account']).mmid,dfake[1]['address'], - TwLabel(dfake[2]['account']).mmid,dfake[3]['address'], - TwLabel(dfake[4]['account']).mmid,dfake[5]['address'] + TwLabel(dfake[0][lbl_id]).mmid,dfake[1]['address'], + TwLabel(dfake[2][lbl_id]).mmid,dfake[3]['address'], + TwLabel(dfake[4][lbl_id]).mmid,dfake[5]['address'] ),'--outdir='+trash_dir] + cmd_args[1:] end_silence() @@ -3101,11 +3102,14 @@ class MMGenTestSuite(object): return self.regtest_alice_add_label_badaddr(name,addr, "Address '{}' not found in tracking wallet".format(addr)) - def regtest_alice_add_label_rpcfail(self,name): + def regtest_alice_bal_rpcfail(self,name): addr = self.regtest_user_sid('alice') + ':C:2' - os.environ['MMGEN_RPC_FAIL_ON_COMMAND'] = 'importaddress' - self.regtest_alice_add_label_badaddr(name,addr,'Label could not be added') + os.environ['MMGEN_RPC_FAIL_ON_COMMAND'] = 'listunspent' + t = MMGenExpect(name,'mmgen-tool',['--alice','getbalance']) os.environ['MMGEN_RPC_FAIL_ON_COMMAND'] = '' + t.expect('Method not found') + t.read() + ok() def regtest_alice_remove_label1(self,name): sid = self.regtest_user_sid('alice')