From f9a483f34f7a0ac28421b60359328b218c302b3f Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 10 May 2020 14:07:54 +0000 Subject: [PATCH] asyncio/aiohttp support Asynchronous HTTP significantly speeds up operations involving multiple JSON-RPC calls to the server, such as tracking wallet views for wallets with a large number of outputs. This patch adds base-level asyncio infrastructure plus aiohttp support to all applicable MMGen commands. The aiohttp package is not currently supported by MSYS2, so Windows users will have to choose one of the other backends ('curl' is the default). Tested on: Linux, Armbian, Windows; Python 3.6, 3.7, 3.8 New user features: - configurable RPC backends via the 'rpc_backend' option. Supported options are 'aiohttp' (Linux-only), 'httplib', 'requests' and 'curl' - configurable RPC queue size via the 'aiohttp_rpc_queue_len' option The patch also includes a rewrite/redesign of large parts of the MMGen code base, most importantly: - rpc.py - full rewrite of RPC library, new RPCBackends class - main_addrimport.py - full rewrite - main_autosign.py - LED code now handled by new LEDControl class - eth/tw.py, eth/tx.py - reworked logic for resolving token symbols and addresses - eth/tx.py - separate classes for signed and unsigned transactions Testing: # Set a backend (choose one): $ export MMGEN_RPC_BACKEND='aiohttp' # Linux-only $ export MMGEN_RPC_BACKEND='curl' # Windows $ export MMGEN_RPC_BACKEND='httplib' # compare performance with 'aiohttp' # Bitcoin: $ test/unit_tests.py rpc btc $ test/test.py main regtest autosign # Ethereum: $ test/unit_tests.py rpc eth $ test/tooltest2.py --coin=eth --testnet=1 txview $ test/test.py --coin=eth ethdev # Monero wallet: $ test/unit_tests.py rpc xmr_wallet $ test/test-release.sh -F xmr --- data_files/mmgen.cfg | 11 + mmgen/addr.py | 40 +- mmgen/altcoins/eth/contract.py | 107 +-- mmgen/altcoins/eth/tw.py | 224 +++--- mmgen/altcoins/eth/tx.py | 420 +++++----- mmgen/daemon.py | 53 +- mmgen/exception.py | 1 + mmgen/globalvars.py | 17 +- mmgen/main_addrimport.py | 201 ++--- mmgen/main_autosign.py | 163 ++-- mmgen/main_split.py | 32 +- mmgen/main_tool.py | 3 + mmgen/main_txbump.py | 75 +- mmgen/main_txcreate.py | 12 +- mmgen/main_txdo.py | 29 +- mmgen/main_txsend.py | 43 +- mmgen/main_txsign.py | 65 +- mmgen/obj.py | 6 + mmgen/opts.py | 21 +- mmgen/rpc.py | 739 ++++++++++-------- mmgen/tool.py | 106 +-- mmgen/tw.py | 167 ++-- mmgen/tx.py | 368 +++++---- mmgen/txsign.py | 4 +- mmgen/util.py | 66 +- setup.py | 1 + .../B472BD-ETH[23.45495,40000].testnet.sigtx | 7 + test/ref/ethereum/tracking-wallet-v2.json | 2 +- test/test.py | 6 +- test/test_py_d/ts_autosign.py | 104 ++- test/test_py_d/ts_ethdev.py | 74 +- test/test_py_d/ts_main.py | 6 +- test/test_py_d/ts_regtest.py | 9 +- test/tooltest2.py | 5 +- test/unit_tests_d/ut_rpc.py | 108 +++ test/unit_tests_d/ut_tx_deserialize.py | 34 +- 36 files changed, 1818 insertions(+), 1511 deletions(-) create mode 100644 test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.sigtx create mode 100755 test/unit_tests_d/ut_rpc.py diff --git a/data_files/mmgen.cfg b/data_files/mmgen.cfg index 3dd5a9f4..89365b43 100644 --- a/data_files/mmgen.cfg +++ b/data_files/mmgen.cfg @@ -35,6 +35,17 @@ # Uncomment to override 'rpcpassword' from coin daemon config file: # rpc_password mypassword +# Choose the backend to use for JSON-RPC connections. Valid choices are +# 'httplib', 'requests', 'curl', 'aiohttp' (Linux only) or 'auto' (defaults +# to curl for Windows/MSYS2 and httplib for Linux): +# rpc_backend auto + +# Increase to allow aiohttp to make more simultaneous RPC connections to the +# daemon. Must be no greater than the 'rpcworkqueue' value in effect on the +# currently running bitcoind (DEFAULT_HTTP_WORKQUEUE = 16). Values over 32 +# may produce little benefit or even reduce performance: +# aiohttp_rpc_queue_len 16 + # Uncomment to set the coin daemon datadir: # daemon_data_dir /path/to/datadir diff --git a/mmgen/addr.py b/mmgen/addr.py index 03fa2734..3e22ddfb 100755 --- a/mmgen/addr.py +++ b/mmgen/addr.py @@ -1023,10 +1023,8 @@ re-import your addresses. def __new__(cls,*args,**kwargs): return MMGenObject.__new__(altcoin_subclass(cls,'tw','AddrData')) - def __init__(self,source=None,wallet=None): + def __init__(self,*args,**kwargs): self.al_ids = {} - if source == 'tw': - self.add_tw_data(wallet) def seed_ids(self): return list(self.al_ids.keys()) @@ -1048,30 +1046,34 @@ re-import your addresses. return (list(d.values())[0][0]) if d else None @classmethod - def get_tw_data(cls,wallet=None): + async def get_tw_data(cls,wallet=None): vmsg('Getting address data from tracking wallet') if 'label_api' in g.rpc.caps: - accts = g.rpc.listlabels() - alists = [list(a.keys()) for a in g.rpc.getaddressesbylabel([[k] for k in accts],batch=True)] + accts = await g.rpc.call('listlabels') + ll = await g.rpc.batch_call('getaddressesbylabel',[(k,) for k in accts]) + alists = [list(a.keys()) for a in ll] else: - accts = g.rpc.listaccounts(0,True) - alists = g.rpc.getaddressesbyaccount([[k] for k in accts],batch=True) + accts = await g.rpc.call('listaccounts',0,True) + alists = await g.rpc.batch_call('getaddressesbyaccount',[(k,) for k in accts]) return list(zip(accts,alists)) - def add_tw_data(self,wallet): - d,out,i = self.get_tw_data(wallet),{},0 - for acct,addr_array in d: + async def add_tw_data(self,wallet): + + twd = await type(self).get_tw_data(wallet) + out,i = {},0 + for acct,addr_array in twd: l = TwLabel(acct,on_fail='silent') if l and l.mmid.type == 'mmgen': obj = l.mmid.obj - i += 1 if len(addr_array) != 1: die(2,self.msgs['too_many_acct_addresses'].format(acct)) al_id = AddrListID(SeedID(sid=obj.sid),MMGenAddrType(obj.mmtype)) if al_id not in out: out[al_id] = [] out[al_id].append(AddrListEntry(idx=obj.idx,addr=addr_array[0],label=l.comment)) - vmsg('{n} {pnm} addresses found, {m} accounts total'.format(n=i,pnm=pnm,m=len(d))) + i += 1 + + vmsg('{n} {pnm} addresses found, {m} accounts total'.format(n=i,pnm=pnm,m=len(twd))) for al_id in out: self.add(AddrList(al_id=al_id,adata=AddrListList(sorted(out[al_id],key=lambda a: a.idx)))) @@ -1087,3 +1089,15 @@ re-import your addresses. for al_id in self.al_ids: d.update(self.al_ids[al_id].make_reverse_dict(coinaddrs)) return d + +class TwAddrData(AddrData,metaclass=aInitMeta): + + def __new__(cls,*args,**kwargs): + return MMGenObject.__new__(altcoin_subclass(cls,'tw','TwAddrData')) + + def __init__(self,*args,**kwargs): + pass + + async def __ainit__(self,wallet=None): + self.al_ids = {} + await self.add_tw_data(wallet) diff --git a/mmgen/altcoins/eth/contract.py b/mmgen/altcoins/eth/contract.py index 1f89e235..ae7f3f03 100755 --- a/mmgen/altcoins/eth/contract.py +++ b/mmgen/altcoins/eth/contract.py @@ -25,7 +25,7 @@ from . import rlp from mmgen.globalvars import g from mmgen.common import * -from mmgen.obj import MMGenObject,CoinAddr,TokenAddr,CoinTxID,ETHAmt +from mmgen.obj import MMGenObject,CoinAddr,TokenAddr,CoinTxID,ETHAmt,aInitMeta from mmgen.util import msg try: @@ -39,21 +39,7 @@ def parse_abi(s): def create_method_id(sig): return keccak_256(sig.encode()).hexdigest()[:8] -class Token(MMGenObject): # ERC20 - - _decimals = None - - # Test that token is in the blockchain by calling constructor w/o decimals arg - def __init__(self,addr,decimals=None): - self.addr = TokenAddr(addr) - if decimals: - self._decimals = decimals - else: - rpc_init() - self.decimals() - if not self._decimals: - raise TokenNotInBlockchain("Token '{}' not in blockchain".format(addr)) - self.base_unit = Decimal('10') ** -self._decimals +class TokenBase(MMGenObject): # ERC20 @staticmethod def transferdata2sendaddr(data): # online @@ -62,53 +48,50 @@ class Token(MMGenObject): # ERC20 def transferdata2amt(self,data): # online return ETHAmt(int(parse_abi(data)[-1],16) * self.base_unit) - def do_call(self,method_sig,method_args='',toUnit=False): + async def do_call(self,method_sig,method_args='',toUnit=False): data = create_method_id(method_sig) + method_args if g.debug: msg('ETH_CALL {}: {}'.format(method_sig,'\n '.join(parse_abi(data)))) - ret = g.rpc.eth_call({ 'to': '0x'+self.addr, 'data': '0x'+data }) + ret = await g.rpc.call('eth_call',{ 'to': '0x'+self.addr, 'data': '0x'+data }) if toUnit: return int(ret,16) * self.base_unit else: return ret - def balance(self,acct_addr): - return ETHAmt(self.do_call('balanceOf(address)',acct_addr.rjust(64,'0'),toUnit=True)) + async def get_balance(self,acct_addr): + return ETHAmt(await self.do_call('balanceOf(address)',acct_addr.rjust(64,'0'),toUnit=True)) def strip(self,s): return ''.join([chr(b) for b in s if 32 <= b <= 127]).strip() - # TODO: make these properties - def decimals(self): - if self._decimals == None: - res = self.do_call('decimals()') - try: - assert res[:2] == '0x' - self._decimals = int(res[2:],16) - except: - msg("RPC call to decimals() failed (returned '{}')".format(res)) - return None - return self._decimals + async def get_name(self): + return self.strip(bytes.fromhex((await self.do_call('name()'))[2:])) - def name(self): - return self.strip(bytes.fromhex(self.do_call('name()')[2:])) + async def get_symbol(self): + return self.strip(bytes.fromhex((await self.do_call('symbol()'))[2:])) - def symbol(self): - return self.strip(bytes.fromhex(self.do_call('symbol()')[2:])) + async def get_decimals(self): + ret = await self.do_call('decimals()') + try: + assert ret[:2] == '0x' + return int(ret,16) + except: + msg("RPC call to decimals() failed (returned '{}')".format(ret)) + return None - def total_supply(self): - return self.do_call('totalSupply()',toUnit=True) + async def get_total_supply(self): + return await self.do_call('totalSupply()',toUnit=True) - def info(self): + async def info(self): fs = '{:15}{}\n' * 5 return fs.format('token address:', self.addr, - 'token symbol:', self.symbol(), - 'token name:', self.name(), - 'decimals:', self.decimals(), - 'total supply:', self.total_supply()) + 'token symbol:', await self.get_symbol(), + 'token name:', await self.get_name(), + 'decimals:', self.decimals, + 'total supply:', await self.get_total_supply()) - def code(self): - return g.rpc.eth_getCode('0x'+self.addr)[2:] + async def code(self): + return (await g.rpc.call('eth_getCode','0x'+self.addr))[2:] def create_data(self,to_addr,amt,method_sig='transfer(address,uint256)',from_addr=None): from_arg = from_addr.rjust(64,'0') if from_addr else '' @@ -126,13 +109,13 @@ class Token(MMGenObject): # ERC20 'nonce': nonce, 'data': bytes.fromhex(data) } - def txsign(self,tx_in,key,from_addr,chain_id=None): + async def txsign(self,tx_in,key,from_addr,chain_id=None): from .pyethereum.transactions import Transaction if chain_id is None: chain_id_method = ('parity_chainId','eth_chainId')['eth_chainId' in g.rpc.caps] - chain_id = int(g.rpc.request(chain_id_method),16) + chain_id = int(await g.rpc.call(chain_id_method),16) tx = Transaction(**tx_in).sign(key,chain_id) hex_tx = rlp.encode(tx).hex() coin_txid = CoinTxID(tx.hash.hex()) @@ -147,18 +130,38 @@ class Token(MMGenObject): # ERC20 # The following are used for token deployment only: - def txsend(self,hex_tx): - return g.rpc.eth_sendRawTransaction('0x'+hex_tx).replace('0x','',1) + async def txsend(self,hex_tx): + return (await g.rpc.call('eth_sendRawTransaction','0x'+hex_tx)).replace('0x','',1) - def transfer( self,from_addr,to_addr,amt,key,start_gas,gasPrice, + async def transfer( self,from_addr,to_addr,amt,key,start_gas,gasPrice, method_sig='transfer(address,uint256)', from_addr2=None, return_data=False): tx_in = self.make_tx_in( from_addr,to_addr,amt, start_gas,gasPrice, - nonce = int(g.rpc.parity_nextNonce('0x'+from_addr),16), + nonce = int(await g.rpc.call('parity_nextNonce','0x'+from_addr),16), method_sig = method_sig, from_addr2 = from_addr2 ) - (hex_tx,coin_txid) = self.txsign(tx_in,key,from_addr) - return self.txsend(hex_tx) + (hex_tx,coin_txid) = await self.txsign(tx_in,key,from_addr) + return await self.txsend(hex_tx) + +class Token(TokenBase): + + def __init__(self,addr,decimals): + self.addr = TokenAddr(addr) + assert isinstance(decimals,int),f'decimals param must be int instance, not {type(decimals)}' + self.decimals = decimals + self.base_unit = Decimal('10') ** -self.decimals + +class TokenResolve(TokenBase,metaclass=aInitMeta): + + def __init__(self,addr): + return super().__init__() + + async def __ainit__(self,addr): + self.addr = TokenAddr(addr) + decimals = await self.get_decimals() # requires self.addr! + if not decimals: + raise TokenNotInBlockchain(f'Token {addr!r} not in blockchain') + Token.__init__(self,addr,decimals) diff --git a/mmgen/altcoins/eth/tw.py b/mmgen/altcoins/eth/tw.py index 3bc2c1aa..28daa69b 100755 --- a/mmgen/altcoins/eth/tw.py +++ b/mmgen/altcoins/eth/tw.py @@ -22,27 +22,17 @@ altcoins.eth.tw: Ethereum tracking wallet and related classes for the MMGen suit from mmgen.common import * from mmgen.obj import ETHAmt,TwLabel,is_coin_addr,is_mmgen_id,MMGenListItem,ListItemAttr,ImmutableAttr -from mmgen.tw import TrackingWallet,TwAddrList,TwUnspentOutputs -from mmgen.addr import AddrData -from .contract import Token +from mmgen.tw import TrackingWallet,TwAddrList,TwUnspentOutputs,TwGetBalance +from mmgen.addr import AddrData,TwAddrData +from .contract import Token,TokenResolve class EthereumTrackingWallet(TrackingWallet): - caps = () + caps = ('batch',) data_key = 'accounts' use_tw_file = True - def __init__(self,mode='r',no_rpc=False): - TrackingWallet.__init__(self,mode=mode) - - for v in self.data['tokens'].values(): - self.conv_types(v) - - if g.token and not is_coin_addr(g.token): - ret = self.sym2addr(g.token,no_rpc=no_rpc) - if ret: g.token = ret - - def is_in_wallet(self,addr): + async def is_in_wallet(self,addr): return addr in self.data_root def init_empty(self): @@ -84,14 +74,17 @@ class EthereumTrackingWallet(TrackingWallet): self.force_write() msg('{} upgraded successfully!'.format(self.desc)) - # Don't call rpc_init() for Ethereum, because it may create a wallet instance - def rpc_init(self): pass - - def rpc_get_balance(self,addr): - return ETHAmt(int(g.rpc.eth_getBalance('0x'+addr),16),'wei') + async def rpc_get_balance(self,addr): + return ETHAmt(int(await g.rpc.call('eth_getBalance','0x'+addr),16),'wei') @write_mode - def import_address(self,addr,label,foo): + async def batch_import_address(self,args_list): + for arg_list in args_list: + await self.import_address(*arg_list) + return args_list + + @write_mode + async def import_address(self,addr,label,foo): r = self.data_root if addr in r: if not r[addr]['mmid'] and label.mmid: @@ -101,7 +94,7 @@ class EthereumTrackingWallet(TrackingWallet): r[addr] = { 'mmid': label.mmid, 'comment': label.comment } @write_mode - def remove_address(self,addr): + async def remove_address(self,addr): r = self.data_root if is_coin_addr(addr): @@ -109,7 +102,7 @@ class EthereumTrackingWallet(TrackingWallet): elif is_mmgen_id(addr): have_match = lambda k: r[k]['mmid'] == addr else: - die(1,"'{}' is not an Ethereum address or MMGen ID".format(addr)) + die(1,f'{addr!r} is not an Ethereum address or MMGen ID') for k in r: if have_match(k): @@ -119,46 +112,30 @@ class EthereumTrackingWallet(TrackingWallet): self.write() return ret else: - m = "Address '{}' not found in '{}' section of tracking wallet" - msg(m.format(addr,self.data_root_desc)) + msg(f'Address {addr!r} not found in {self.data_root_desc!r} section of tracking wallet') return None @write_mode - def set_label(self,coinaddr,lbl): + async def set_label(self,coinaddr,lbl): for addr,d in list(self.data_root.items()): if addr == coinaddr: d['comment'] = lbl.comment self.write() return None - else: # emulate on_fail='return' of RPC library - m = "Address '{}' not found in '{}' section of tracking wallet" - return ('rpcfail',(None,2,m.format(coinaddr,self.data_root_desc))) - - def addr2sym(self,req_addr): + else: + msg(f'Address {coinaddr!r} not found in {self.data_root_desc!r} section of tracking wallet') + return False + async def addr2sym(self,req_addr): for addr in self.data['tokens']: if addr == req_addr: - ret = self.data['tokens'][addr]['params'].get('symbol') - if ret: return ret - else: break - - self.token_obj = Token(req_addr) - ret = self.token_obj.symbol().upper() - self.force_set_token_param(req_addr,'symbol',ret) - return ret - - def sym2addr(self,sym,no_rpc=False): - - for addr in self.data['tokens']: - if self.data['tokens'][addr]['params'].get('symbol') == sym.upper(): - return addr - - if no_rpc: + return self.data['tokens'][addr]['params']['symbol'] + else: return None + async def sym2addr(self,sym): for addr in self.data['tokens']: - if Token(addr).symbol().upper() == sym.upper(): - self.force_set_token_param(addr,'symbol',sym.upper()) + if self.data['tokens'][addr]['params']['symbol'] == sym.upper(): return addr else: return None @@ -168,17 +145,6 @@ class EthereumTrackingWallet(TrackingWallet): return self.data['tokens'][token]['params'].get(param) return None - def force_set_token_param(self,*args,**kwargs): - mode_save = self.mode - self.mode = 'w' - self.set_token_param(*args,**kwargs) - self.mode = mode_save - - @write_mode - def set_token_param(self,token,param,val): - if token in self.data['tokens']: - self.data['tokens'][token]['params'][param] = val - class EthereumTokenTrackingWallet(EthereumTrackingWallet): desc = 'Ethereum token tracking wallet' @@ -186,29 +152,32 @@ class EthereumTokenTrackingWallet(EthereumTrackingWallet): symbol = None cur_eth_balances = {} - def __init__(self,mode='r',no_rpc=False): - EthereumTrackingWallet.__init__(self,mode=mode,no_rpc=no_rpc) + async def __ainit__(self,mode='r'): + await super().__ainit__(mode=mode) - self.desc = 'Ethereum token tracking wallet' + for v in self.data['tokens'].values(): + self.conv_types(v) if not is_coin_addr(g.token): - raise UnrecognizedTokenSymbol('Specified token {!r} could not be resolved!'.format(g.token)) + g.token = await self.sym2addr(g.token) # returns None on failure - if mode == 'r' and not g.token in self.data['tokens']: + if not is_coin_addr(g.token): + if self.importing: + m = 'When importing addresses for a new token, the token must be specified by address, not symbol.' + raise InvalidTokenAddress(f'{g.token!r}: invalid token address\n{m}') + else: + raise UnrecognizedTokenSymbol(f'Specified token {g.token!r} could not be resolved!') + + if g.token in self.data['tokens']: + self.decimals = self.data['tokens'][g.token]['params']['decimals'] + self.symbol = self.data['tokens'][g.token]['params']['symbol'] + elif not self.importing: raise TokenNotInWallet('Specified token {!r} not in wallet!'.format(g.token)) self.token = g.token + g.dcoin = self.symbol - if self.token in self.data['tokens']: - for k in ('decimals','symbol'): - setattr(self,k,self.get_param(k)) - if getattr(self,k) == None: - setattr(self,k,getattr(Token(self.token,self.decimals),k)()) - if getattr(self,k) != None: - self.set_param(k,getattr(self,k)) - self.write() - - def is_in_wallet(self,addr): + async def is_in_wallet(self,addr): return addr in self.data['tokens'][self.token] @property @@ -217,43 +186,39 @@ class EthereumTokenTrackingWallet(EthereumTrackingWallet): @property def data_root_desc(self): - return 'token ' + Token(self.token,self.decimals).symbol() + return 'token ' + self.get_param('symbol') - @write_mode - def add_token(self,token): - msg("Adding token '{}' to tracking wallet.".format(token)) - self.data['tokens'][token] = { 'params': {} } + async def rpc_get_balance(self,addr): + return await Token(self.token,self.decimals).get_balance(addr) - @write_mode - def import_address(self,*args,**kwargs): - if self.token not in self.data['tokens']: - self.add_token(self.token) - EthereumTrackingWallet.import_address(self,*args,**kwargs) - - def rpc_get_balance(self,addr): - return Token(self.token,self.decimals).balance(addr) - - def get_eth_balance(self,addr,force_rpc=False): + async def get_eth_balance(self,addr,force_rpc=False): cache = self.cur_eth_balances - data_root = self.data['accounts'] - ret = None if force_rpc else self.get_cached_balance(addr,cache,data_root) + r = self.data['accounts'] + ret = None if force_rpc else self.get_cached_balance(addr,cache,r) if ret == None: - ret = EthereumTrackingWallet.rpc_get_balance(self,addr) - self.cache_balance(addr,ret,cache,data_root) + ret = await super().rpc_get_balance(addr) + self.cache_balance(addr,ret,cache,r) return ret - def force_set_param(self,*args,**kwargs): - mode_save = self.mode - self.mode = 'w' - self.set_param(*args,**kwargs) - self.mode = mode_save + def get_param(self,param): + return self.data['tokens'][self.token]['params'][param] @write_mode - def set_param(self,param,val): - self.data['tokens'][self.token]['params'][param] = val - - def get_param(self,param): - return self.data['tokens'][self.token]['params'].get(param) + async def import_token(tw): + """ + Token 'symbol' and 'decimals' values are resolved from the network by the system just + once, upon token import. Thereafter, token address, symbol and decimals are resolved + either from the tracking wallet (online operations) or transaction file (when signing). + """ + if not g.token in tw.data['tokens']: + t = await TokenResolve(g.token) + tw.token = g.token + tw.data['tokens'][tw.token] = { + 'params': { + 'symbol': await t.get_symbol(), + 'decimals': t.decimals + } + } # No unspent outputs with Ethereum, but naming must be consistent class EthereumTwUnspentOutputs(TwUnspentOutputs): @@ -277,23 +242,23 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, 'q':'a_quit','p':'a_print','v':'a_view','w':'a_view_wide', 'l':'a_lbl_add','D':'a_addr_delete','R':'a_balance_refresh' } - def __init__(self,*args,**kwargs): + async def __ainit__(self,*args,**kwargs): if g.use_cached_balances: self.hdr_fmt += '\n' + yellow('WARNING: Using cached balances. These may be out of date!') - TwUnspentOutputs.__init__(self,*args,**kwargs) + await TwUnspentOutputs.__ainit__(self,*args,**kwargs) def do_sort(self,key=None,reverse=False): if key == 'txid': return super().do_sort(key=key,reverse=reverse) - def get_unspent_rpc(self): + async def get_unspent_rpc(self): wl = self.wallet.sorted_list if self.addrs: wl = [d for d in wl if d['addr'] in self.addrs] return [{ 'account': TwLabel(d['mmid']+' '+d['comment'],on_fail='raise'), 'address': d['addr'], - 'amount': self.wallet.get_balance(d['addr']), + 'amount': await self.wallet.get_balance(d['addr']), 'confirmations': 0, # TODO } for d in wl] @@ -311,6 +276,9 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, def age_disp(self,o,age_fmt): # TODO return None + def age_disp(self,o,age_fmt): # TODO + return None + class EthereumTokenTwUnspentOutputs(EthereumTwUnspentOutputs): disp_type = 'token' @@ -320,18 +288,18 @@ class EthereumTokenTwUnspentOutputs(EthereumTwUnspentOutputs): def get_display_precision(self): return 10 # truncate precision for narrow display - def get_unspent_data(self): - super().get_unspent_data() + async def get_unspent_data(self,*args,**kwargs): + await super().get_unspent_data(*args,**kwargs) for e in self.unspent: - e.amt2 = self.wallet.get_eth_balance(e.addr) + e.amt2 = await self.wallet.get_eth_balance(e.addr) class EthereumTwAddrList(TwAddrList): has_age = False - def __init__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None): + async def __ainit__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None): - self.wallet = wallet or TrackingWallet(mode='w') + self.wallet = wallet or await TrackingWallet(mode='w') tw_dict = self.wallet.mmid_ordered_dict self.total = g.proto.coin_amt('0') @@ -341,7 +309,7 @@ class EthereumTwAddrList(TwAddrList): label = TwLabel(mmid+' '+d['comment'],on_fail='raise') if usr_addr_list and (label.mmid not in usr_addr_list): continue - bal = self.wallet.get_balance(d['addr']) + bal = await self.wallet.get_balance(d['addr']) if bal == 0 and not showempty: if not label.comment or not all_labels: continue @@ -352,18 +320,20 @@ class EthereumTwAddrList(TwAddrList): self[label.mmid]['amt'] += bal self.total += bal -class EthereumTokenTwAddrList(EthereumTwAddrList): pass + del self.wallet + +class EthereumTokenTwAddrList(EthereumTwAddrList): + pass -from mmgen.tw import TwGetBalance class EthereumTwGetBalance(TwGetBalance): fs = '{w:13} {c}\n' # TODO - for now, just suppress display of meaningless data - def __init__(self,*args,**kwargs): - self.wallet = TrackingWallet(mode='w') - TwGetBalance.__init__(self,*args,**kwargs) + async def __ainit__(self,*args,**kwargs): + self.wallet = await TrackingWallet(mode='w') + await TwGetBalance.__ainit__(self,*args,**kwargs) - def create_data(self): + async def create_data(self): data = self.wallet.mmid_ordered_dict for d in data: if d.type == 'mmgen': @@ -374,20 +344,24 @@ class EthereumTwGetBalance(TwGetBalance): key = 'Non-MMGen' conf_level = 2 # TODO - amt = self.wallet.get_balance(data[d]['addr']) + amt = await self.wallet.get_balance(data[d]['addr']) self.data['TOTAL'][conf_level] += amt self.data[key][conf_level] += amt -class EthereumTokenTwGetBalance(EthereumTwGetBalance): pass + del self.wallet -class EthereumAddrData(AddrData): +class EthereumTwAddrData(TwAddrData): @classmethod - def get_tw_data(cls,wallet=None): + async def get_tw_data(cls,wallet=None): vmsg('Getting address data from tracking wallet') - tw = (wallet or TrackingWallet()).mmid_ordered_dict + tw = (wallet or await TrackingWallet()).mmid_ordered_dict # emulate the output of RPC 'listaccounts' and 'getaddressesbyaccount' return [(mmid+' '+d['comment'],[d['addr']]) for mmid,d in list(tw.items())] +class EthereumTokenTwGetBalance(EthereumTwGetBalance): pass +class EthereumTokenTwAddrData(EthereumTwAddrData): pass + +class EthereumAddrData(AddrData): pass class EthereumTokenAddrData(EthereumAddrData): pass diff --git a/mmgen/altcoins/eth/tx.py b/mmgen/altcoins/eth/tx.py index 480e0b90..04d88f17 100755 --- a/mmgen/altcoins/eth/tx.py +++ b/mmgen/altcoins/eth/tx.py @@ -24,7 +24,9 @@ import json from mmgen.common import * from mmgen.obj import * -from mmgen.tx import MMGenTX,MMGenBumpTX,MMGenSplitTX +from mmgen.tx import MMGenTX,MMGenBumpTX,MMGenSplitTX,MMGenTxForSigning +from mmgen.tw import TrackingWallet +from .contract import Token class EthereumMMGenTX(MMGenTX): desc = 'Ethereum transaction' @@ -49,7 +51,7 @@ class EthereumMMGenTX(MMGenTX): usr_contract_data = HexStr('') def __init__(self,*args,**kwargs): - super().__init__(*args,**kwargs) + MMGenTX.__init__(self,*args,**kwargs) if hasattr(opt,'tx_gas') and opt.tx_gas: self.tx_gas = self.start_gas = ETHAmt(int(opt.tx_gas),'wei') if hasattr(opt,'contract_data') and opt.contract_data: @@ -58,9 +60,12 @@ class EthereumMMGenTX(MMGenTX): self.usr_contract_data = HexStr(open(opt.contract_data).read().strip()) self.disable_fee_check = True + def check_txfile_hex_data(self): + pass + @classmethod - def get_exec_status(cls,txid,silent=False): - d = g.rpc.eth_getTransactionReceipt('0x'+txid) + async def get_exec_status(cls,txid,silent=False): + d = await g.rpc.call('eth_getTransactionReceipt','0x'+txid) if not silent: if 'contractAddress' in d and d['contractAddress']: msg('Contract address: {}'.format(d['contractAddress'].replace('0x',''))) @@ -84,46 +89,35 @@ class EthereumMMGenTX(MMGenTX): return True return False - # hex data if signed, json if unsigned - def check_txfile_hex_data(self): - if self.check_sigs(): - from .pyethereum.transactions import Transaction - from . import rlp - etx = rlp.decode(bytes.fromhex(self.hex),Transaction) - d = etx.to_dict() # ==> hex values have '0x' prefix, 0 is '0x' - for k in ('sender','to','data'): - if k in d: d[k] = d[k].replace('0x','',1) - o = { 'from': CoinAddr(d['sender']), - 'to': CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is token address - 'amt': ETHAmt(d['value'],'wei'), - 'gasPrice': ETHAmt(d['gasprice'],'wei'), - 'startGas': ETHAmt(d['startgas'],'wei'), - 'nonce': ETHNonce(d['nonce']), - 'data': HexStr(d['data']) } - if o['data'] and not o['to']: # token- or contract-creating transaction - o['token_addr'] = TokenAddr(etx.creates.hex()) # NB: could be a non-token contract address - self.disable_fee_check = True - txid = CoinTxID(etx.hash.hex()) - assert txid == self.coin_txid,"txid in tx.hex doesn't match value in MMGen transaction file" - else: - d = json.loads(self.hex) - o = { 'from': CoinAddr(d['from']), - 'to': CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is sendto address - 'amt': ETHAmt(d['amt']), - 'gasPrice': ETHAmt(d['gasPrice']), - 'startGas': ETHAmt(d['startGas']), - 'nonce': ETHNonce(d['nonce']), - 'chainId': Int(d['chainId']), - 'data': HexStr(d['data']) } + def parse_txfile_hex_data(self): + from .pyethereum.transactions import Transaction + from . import rlp + etx = rlp.decode(bytes.fromhex(self.hex),Transaction) + d = etx.to_dict() # ==> hex values have '0x' prefix, 0 is '0x' + for k in ('sender','to','data'): + if k in d: d[k] = d[k].replace('0x','',1) + o = { + 'from': CoinAddr(d['sender']), + 'to': CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is token address + 'amt': ETHAmt(d['value'],'wei'), + 'gasPrice': ETHAmt(d['gasprice'],'wei'), + 'startGas': ETHAmt(d['startgas'],'wei'), + 'nonce': ETHNonce(d['nonce']), + 'data': HexStr(d['data']) } + if o['data'] and not o['to']: # token- or contract-creating transaction + o['token_addr'] = TokenAddr(etx.creates.hex()) # NB: could be a non-token contract address + self.disable_fee_check = True + txid = CoinTxID(etx.hash.hex()) + assert txid == self.coin_txid,"txid in tx.hex doesn't match value in MMGen transaction file" self.tx_gas = o['startGas'] # approximate, but better than nothing self.fee = self.fee_rel2abs(o['gasPrice'].toWei()) self.txobj = o return d # 'token_addr','decimals' required by Token subclass - def get_nonce(self): - return ETHNonce(int(g.rpc.parity_nextNonce('0x'+self.inputs[0].addr),16)) + async def get_nonce(self): + return ETHNonce(int(await g.rpc.call('parity_nextNonce','0x'+self.inputs[0].addr),16)) - def make_txobj(self): # called by create_raw() + async def make_txobj(self): # called by create_raw() chain_id_method = ('parity_chainId','eth_chainId')['eth_chainId' in g.rpc.caps] self.txobj = { 'from': self.inputs[0].addr, @@ -131,20 +125,20 @@ class EthereumMMGenTX(MMGenTX): 'amt': self.outputs[0].amt if self.outputs else ETHAmt('0'), 'gasPrice': self.usr_rel_fee or self.fee_abs2rel(self.fee,to_unit='eth'), 'startGas': self.start_gas, - 'nonce': self.get_nonce(), - 'chainId': Int(g.rpc.request(chain_id_method),16), + 'nonce': await self.get_nonce(), + 'chainId': Int(await g.rpc.call(chain_id_method),16), 'data': self.usr_contract_data, } # Instead of serializing tx data as with BTC, just create a JSON dump. # This complicates things but means we avoid using the rlp library to deserialize the data, # thus removing an attack vector - def create_raw(self): + async def create_raw(self): assert len(self.inputs) == 1,'Transaction has more than one input!' o_num = len(self.outputs) o_ok = 0 if self.usr_contract_data else 1 assert o_num == o_ok,'Transaction has {} output{} (should have {})'.format(o_num,suf(o_num),o_ok) - self.make_txobj() + await self.make_txobj() odict = { k: str(v) for k,v in self.txobj.items() if k != 'token_to' } self.hex = json.dumps(odict) self.update_txid() @@ -156,9 +150,6 @@ class EthereumMMGenTX(MMGenTX): assert not is_hex_str(self.hex),'update_txid() must be called only when self.hex is not hex data' self.txid = MMGenTxID(make_chksum_6(self.hex).upper()) - def get_blockcount(self): - return Int(g.rpc.eth_blockNumber(),16) - def process_cmd_args(self,cmd_args,ad_f,ad_w): lc = len(cmd_args) if lc == 0 and self.usr_contract_data and not 'Token' in type(self).__name__: @@ -185,7 +176,9 @@ class EthereumMMGenTX(MMGenTX): return [int(reply)] # coin-specific fee routines: - def get_relay_fee(self): return ETHAmt('0') # TODO + @property + def relay_fee(self): + return ETHAmt('0') # TODO # given absolute fee in ETH, return gas price in Gwei using tx_gas def fee_abs2rel(self,abs_fee,to_unit='Gwei'): @@ -194,8 +187,8 @@ class EthereumMMGenTX(MMGenTX): return ret if to_unit == 'eth' else ret.to_unit(to_unit,show_decimal=True) # get rel_fee (gas price) from network, return in native wei - def get_rel_fee_from_network(self): - return Int(g.rpc.eth_gasPrice(),16),'eth_gasPrice' # ==> rel_fee,fe_type + async def get_rel_fee_from_network(self): + return Int(await g.rpc.call('eth_gasPrice'),16),'eth_gasPrice' # ==> rel_fee,fe_type # given rel fee and units, return absolute fee using tx_gas def convert_fee_spec(self,foo,units,amt,unit): @@ -264,7 +257,7 @@ class EthereumMMGenTX(MMGenTX): def format_view_rel_fee(self,terse): return '' def format_view_verbose_footer(self): return '' # TODO - def resolve_g_token_from_tx_file(self): + def resolve_g_token_from_txfile(self): die(2,"The '--token' option must be specified for token transaction files") def final_inputs_ok_msg(self,change_amt): @@ -272,7 +265,126 @@ class EthereumMMGenTX(MMGenTX): chg = '0' if (self.outputs and self.outputs[0].is_chg) else change_amt return m.format(ETHAmt(chg).hl(),g.coin) - def do_sign(self,wif,tx_num_str): + async def get_status(self,status=False): + + class r(object): pass + + async def is_in_mempool(): + if not 'full_node' in g.rpc.caps: + return False + return '0x'+self.coin_txid in [x['hash'] for x in await g.rpc.call('parity_pendingTransactions')] + + async def is_in_wallet(): + d = await g.rpc.call('eth_getTransactionReceipt','0x'+self.coin_txid) + if d and 'blockNumber' in d and d['blockNumber'] is not None: + r.confs = 1 + int(await g.rpc.call('eth_blockNumber'),16) - int(d['blockNumber'],16) + r.exec_status = int(d['status'],16) + return True + return False + + if await is_in_mempool(): + msg('Transaction is in mempool' if status else 'Warning: transaction is in mempool!') + return + + if status: + if await is_in_wallet(): + if self.txobj['data']: + cd = capfirst(self.contract_desc) + if r.exec_status == 0: + msg('{} failed to execute!'.format(cd)) + else: + msg('{} successfully executed with status {}'.format(cd,r.exec_status)) + die(0,'Transaction has {} confirmation{}'.format(r.confs,suf(r.confs))) + die(1,'Transaction is neither in mempool nor blockchain!') + + async def send(self,prompt_user=True,exit_on_fail=False): + + if not self.marked_signed(): + die(1,'Transaction is not signed!') + + self.check_correct_chain(on_fail='die') + + fee = self.fee_rel2abs(self.txobj['gasPrice'].toWei()) + + if not self.disable_fee_check and (fee > g.proto.max_tx_fee): + die(2,'Transaction fee ({}) greater than {} max_tx_fee ({} {})!'.format( + fee,g.proto.name.capitalize(),g.proto.max_tx_fee,g.coin)) + + await self.get_status() + + if prompt_user: + self.confirm_send() + + if g.bogus_send: + ret = None + else: + try: + ret = await g.rpc.call('eth_sendRawTransaction','0x'+self.hex) + except: + raise + ret = False + + if ret == False: + msg(red('Send of MMGen transaction {} failed'.format(self.txid))) + if exit_on_fail: + sys.exit(1) + return False + else: + if g.bogus_send: + m = 'BOGUS transaction NOT sent: {}' + else: + m = 'Transaction sent: {}' + assert ret == '0x'+self.coin_txid,'txid mismatch (after sending)' + self.desc = 'sent transaction' + msg(m.format(self.coin_txid.hl())) + self.add_timestamp() + self.add_blockcount() + return True + + async def get_cmdline_input_addrs(self): + ret = [] + if opt.inputs: + r = (await TrackingWallet()).data_root # must create new instance here + m = 'Address {!r} not in tracking wallet' + for i in opt.inputs.split(','): + if is_mmgen_id(i): + for addr in r: + if r[addr]['mmid'] == i: + ret.append(addr) + break + else: + raise UserAddressNotInWallet(m.format(i)) + elif is_coin_addr(i): + if not i in r: + raise UserAddressNotInWallet(m.format(i)) + ret.append(i) + else: + die(1,"'{}': not an MMGen ID or coin address".format(i)) + return ret + + def print_contract_addr(self): + if 'token_addr' in self.txobj: + msg('Contract address: {}'.format(self.txobj['token_addr'].hl())) + +class EthereumMMGenTxForSigning(EthereumMMGenTX,MMGenTxForSigning): + + def parse_txfile_hex_data(self): + d = json.loads(self.hex) + o = { + 'from': CoinAddr(d['from']), + 'to': CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is sendto address + 'amt': ETHAmt(d['amt']), + 'gasPrice': ETHAmt(d['gasPrice']), + 'startGas': ETHAmt(d['startGas']), + 'nonce': ETHNonce(d['nonce']), + 'chainId': Int(d['chainId']), + 'data': HexStr(d['data']) } + self.tx_gas = o['startGas'] # approximate, but better than nothing + self.fee = self.fee_rel2abs(o['gasPrice'].toWei()) + self.txobj = o + return d # 'token_addr','decimals' required by Token subclass + + async def do_sign(self,wif,tx_num_str): o = self.txobj o_conv = { 'to': bytes.fromhex(o['to']), @@ -299,7 +411,7 @@ class EthereumMMGenTX(MMGenTX): assert self.check_sigs(),'Signature check failed' - def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception + async def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception if self.marked_signed(): msg('Transaction is already signed!') @@ -311,7 +423,7 @@ class EthereumMMGenTX(MMGenTX): msg_r('Signing transaction{}...'.format(tx_num_str)) try: - self.do_sign(keys[0].sec.wif,tx_num_str) + await self.do_sign(keys[0].sec.wif,tx_num_str) msg('OK') return True except Exception as e: @@ -322,103 +434,6 @@ class EthereumMMGenTX(MMGenTX): ymsg('\n'+''.join(traceback.format_exception(*sys.exc_info()))) return False - def get_status(self,status=False): - - class r(object): pass - - def is_in_mempool(): - if not 'full_node' in g.rpc.caps: - return False - return '0x'+self.coin_txid in [x['hash'] for x in g.rpc.parity_pendingTransactions()] - - def is_in_wallet(): - d = g.rpc.eth_getTransactionReceipt('0x'+self.coin_txid) - if d and 'blockNumber' in d and d['blockNumber'] is not None: - r.confs = 1 + int(g.rpc.eth_blockNumber(),16) - int(d['blockNumber'],16) - r.exec_status = int(d['status'],16) - return True - return False - - if is_in_mempool(): - msg('Transaction is in mempool' if status else 'Warning: transaction is in mempool!') - return - - if status: - if is_in_wallet(): - if self.txobj['data']: - cd = capfirst(self.contract_desc) - if r.exec_status == 0: - msg('{} failed to execute!'.format(cd)) - else: - msg('{} successfully executed with status {}'.format(cd,r.exec_status)) - die(0,'Transaction has {} confirmation{}'.format(r.confs,suf(r.confs))) - die(1,'Transaction is neither in mempool nor blockchain!') - - def send(self,prompt_user=True,exit_on_fail=False): - - if not self.marked_signed(): - die(1,'Transaction is not signed!') - - self.check_correct_chain(on_fail='die') - - fee = self.fee_rel2abs(self.txobj['gasPrice'].toWei()) - - if not self.disable_fee_check and (fee > g.proto.max_tx_fee): - die(2,'Transaction fee ({}) greater than {} max_tx_fee ({} {})!'.format( - fee,g.proto.name.capitalize(),g.proto.max_tx_fee,g.coin)) - - self.get_status() - - if prompt_user: - self.confirm_send() - - ret = None if g.bogus_send else g.rpc.eth_sendRawTransaction('0x'+self.hex,on_fail='return') - - from mmgen.rpc import rpc_error,rpc_errmsg - if rpc_error(ret): - msg(yellow(rpc_errmsg(ret))) - msg(red('Send of MMGen transaction {} failed'.format(self.txid))) - if exit_on_fail: - sys.exit(1) - return False - else: - if g.bogus_send: - m = 'BOGUS transaction NOT sent: {}' - else: - m = 'Transaction sent: {}' - assert ret == '0x'+self.coin_txid,'txid mismatch (after sending)' - self.desc = 'sent transaction' - msg(m.format(self.coin_txid.hl())) - self.add_timestamp() - self.add_blockcount() - return True - - def get_cmdline_input_addrs(self): - ret = [] - if opt.inputs: - from mmgen.tw import TrackingWallet - r = TrackingWallet().data_root # must create new instance here - m = 'Address {!r} not in tracking wallet' - for i in opt.inputs.split(','): - if is_mmgen_id(i): - for addr in r: - if r[addr]['mmid'] == i: - ret.append(addr) - break - else: - raise UserAddressNotInWallet(m.format(i)) - elif is_coin_addr(i): - if not i in r: - raise UserAddressNotInWallet(m.format(i)) - ret.append(i) - else: - die(1,"'{}': not an MMGen ID or coin address".format(i)) - return ret - - def print_contract_addr(self): - if 'token_addr' in self.txobj: - msg('Contract address: {}'.format(self.txobj['token_addr'].hl())) - class EthereumTokenMMGenTX(EthereumMMGenTX): desc = 'Ethereum token transaction' contract_desc = 'token contract' @@ -427,26 +442,18 @@ class EthereumTokenMMGenTX(EthereumMMGenTX): fmt_keys = ('from','token_to','amt','nonce') fee_is_approximate = True - def __init__(self,*args,**kwargs): - if not kwargs.get('offline'): - from mmgen.tw import TrackingWallet - self.decimals = TrackingWallet().get_param('decimals') - from .contract import Token - self.token_obj = Token(g.token,self.decimals) - EthereumMMGenTX.__init__(self,*args,**kwargs) - def update_change_output(self,change_amt): if self.outputs[0].is_chg: self.update_output_amt(0,self.inputs[0].amt) # token transaction, so check both eth and token balances # TODO: add test with insufficient funds - def precheck_sufficient_funds(self,inputs_sum,sel_unspent): - eth_bal = self.twuo.wallet.get_eth_balance(sel_unspent[0].addr) + async def precheck_sufficient_funds(self,inputs_sum,sel_unspent): + eth_bal = await self.tw.get_eth_balance(sel_unspent[0].addr) if eth_bal == 0: # we don't know the fee yet msg('This account has no ether to pay for the transaction fee!') return False - return super().precheck_sufficient_funds(inputs_sum,sel_unspent) + return await super().precheck_sufficient_funds(inputs_sum,sel_unspent) def final_inputs_ok_msg(self,change_amt): token_bal = ( ETHAmt('0') if self.outputs[0].is_chg else @@ -454,49 +461,30 @@ class EthereumTokenMMGenTX(EthereumMMGenTX): m = "Transaction leaves ≈{} {} and {} {} in the sender's account" return m.format( change_amt.hl(), g.coin, token_bal.hl(), g.dcoin ) - def get_change_amt(self): # here we know the fee - eth_bal = self.twuo.wallet.get_eth_balance(self.inputs[0].addr) + async def get_change_amt(self): # here we know the fee + eth_bal = await self.tw.get_eth_balance(self.inputs[0].addr) return eth_bal - self.fee - def resolve_g_token_from_tx_file(self): - g.dcoin = self.dcoin - if is_hex_str(self.hex): return # for txsend we can leave g.token uninitialized - d = json.loads(self.hex) - if g.token.upper() == self.dcoin: - g.token = d['token_addr'] - elif g.token != d['token_addr']: - m1 = "'{p}': invalid --token parameter for {t} {n} token transaction file\n" - m2 = "Please use '--token={t}'" - die(1,(m1+m2).format(p=g.token,t=self.dcoin,n=capfirst(g.proto.name))) + def resolve_g_token_from_txfile(self): + pass - def make_txobj(self): # called by create_raw() - super().make_txobj() - t = self.token_obj + async def make_txobj(self): # called by create_raw() + await super().make_txobj() + t = Token(self.tw.token,self.tw.decimals) o = self.txobj o['token_addr'] = t.addr - o['decimals'] = t.decimals() + o['decimals'] = t.decimals o['token_to'] = o['to'] o['data'] = t.create_data(o['token_to'],o['amt']) - def check_txfile_hex_data(self): - d = super().check_txfile_hex_data() + def parse_txfile_hex_data(self): + d = EthereumMMGenTX.parse_txfile_hex_data(self) o = self.txobj - - if self.check_sigs(): # online, from rlp and wallet - o['token_addr'] = TokenAddr(o['to']) - o['decimals'] = self.decimals - else: # offline, from json - o['token_addr'] = TokenAddr(d['token_addr']) - o['decimals'] = Int(d['decimals']) - - from .contract import Token - t = self.token_obj = Token(o['token_addr'],o['decimals']) - - if self.check_sigs(): # online, from rlp - 'amt' was eth amt, now token amt - o['amt'] = t.transferdata2amt(o['data']) - else: # offline, from json - 'amt' is token amt - o['data'] = t.create_data(o['to'],o['amt']) - + assert self.tw.token == o['to'] + o['token_addr'] = TokenAddr(o['to']) + o['decimals'] = self.tw.decimals + t = Token(o['token_addr'],o['decimals']) + o['amt'] = t.transferdata2amt(o['data']) o['token_to'] = type(t).transferdata2sendaddr(o['data']) def format_view_body(self,*args,**kwargs): @@ -505,25 +493,47 @@ class EthereumTokenMMGenTX(EthereumMMGenTX): c=blue('(' + g.dcoin + ')'), r=super().format_view_body(*args,**kwargs)) - def do_sign(self,wif,tx_num_str): +class EthereumTokenMMGenTxForSigning(EthereumTokenMMGenTX,EthereumMMGenTxForSigning): + + def resolve_g_token_from_txfile(self): + d = json.loads(self.hex) + if g.token.upper() == self.dcoin: + g.token = d['token_addr'] + elif g.token != d['token_addr']: + m1 = "'{p}': invalid --token parameter for {t} {n} token transaction file\n" + m2 = "Please use '--token={t}'" + die(1,(m1+m2).format(p=g.token,t=self.dcoin,n=capfirst(g.proto.name))) + + def parse_txfile_hex_data(self): + d = EthereumMMGenTxForSigning.parse_txfile_hex_data(self) o = self.txobj - t = self.token_obj + o['token_addr'] = TokenAddr(d['token_addr']) + o['decimals'] = Int(d['decimals']) + t = Token(o['token_addr'],o['decimals']) + o['data'] = t.create_data(o['to'],o['amt']) + o['token_to'] = type(t).transferdata2sendaddr(o['data']) + + async def do_sign(self,wif,tx_num_str): + o = self.txobj + t = Token(o['token_addr'],o['decimals']) tx_in = t.make_tx_in(o['from'],o['to'],o['amt'],self.start_gas,o['gasPrice'],nonce=o['nonce']) - (self.hex,self.coin_txid) = t.txsign(tx_in,wif,o['from'],chain_id=o['chainId']) + (self.hex,self.coin_txid) = await t.txsign(tx_in,wif,o['from'],chain_id=o['chainId']) assert self.check_sigs(),'Signature check failed' -class EthereumMMGenBumpTX(EthereumMMGenTX,MMGenBumpTX): +class EthereumMMGenBumpTX(MMGenBumpTX,EthereumMMGenTxForSigning): - def choose_output(self): pass - - def set_min_fee(self): - self.min_fee = ETHAmt(self.fee * Decimal('1.101')) + @property + def min_fee(self): + return ETHAmt(self.fee * Decimal('1.101')) def update_fee(self,foo,fee): self.fee = fee - def get_nonce(self): + async def get_nonce(self): return self.txobj['nonce'] -class EthereumTokenMMGenBumpTX(EthereumTokenMMGenTX,EthereumMMGenBumpTX): pass -class EthereumMMGenSplitTX(MMGenSplitTX): pass +class EthereumTokenMMGenBumpTX(EthereumMMGenBumpTX,EthereumTokenMMGenTxForSigning): + pass + +class EthereumMMGenSplitTX(MMGenSplitTX): + pass diff --git a/mmgen/daemon.py b/mmgen/daemon.py index 1f13f3b5..2ab3f77e 100755 --- a/mmgen/daemon.py +++ b/mmgen/daemon.py @@ -125,7 +125,7 @@ class Daemon(MMGenObject): self.wait_for_state('stopped') os.makedirs(self.datadir,exist_ok=True) - if self.cfg_file: + if self.cfg_file and not 'keep_cfg_file' in self.flags: open('{}/{}'.format(self.datadir,self.cfg_file),'w').write(self.cfg_file_hdr) if self.use_pidfile and os.path.exists(self.pidfile): @@ -221,7 +221,7 @@ class MoneroWalletDaemon(Daemon): exec_fn_mswin = 'monero-wallet-rpc.exe' ps_pid_mswin = True - def __init__(self,wallet_dir,test_suite=False): + def __init__(self,wallet_dir,test_suite=False,host=None,user=None,passwd=None): self.platform = g.platform self.wallet_dir = wallet_dir if test_suite: @@ -237,7 +237,13 @@ class MoneroWalletDaemon(Daemon): if self.platform == 'win': self.use_pidfile = False - if not g.monero_wallet_rpc_password: + self.host = host or g.monero_wallet_rpc_host + self.user = user or g.monero_wallet_rpc_user + self.passwd = passwd or g.monero_wallet_rpc_password + + assert self.host + assert self.user + if not self.passwd: die(1, 'You must set your Monero wallet RPC password.\n' + 'This can be done on the command line, with the --monero-wallet-rpc-password\n' + @@ -252,7 +258,7 @@ class MoneroWalletDaemon(Daemon): '--rpc-bind-port={}'.format(self.rpc_port), '--wallet-dir='+self.wallet_dir, '--log-file='+self.logfile, - '--rpc-login={}:{}'.format(g.monero_wallet_rpc_user,g.monero_wallet_rpc_password) ] + '--rpc-login={}:{}'.format(self.user,self.passwd) ] if self.platform == 'linux': cmd += ['--pidfile={}'.format(self.pidfile)] cmd += [] if 'no_daemonize' in self.flags else ['--detach'] @@ -260,15 +266,16 @@ class MoneroWalletDaemon(Daemon): @property def state(self): - if not self.test_socket(g.monero_wallet_rpc_host,self.rpc_port): + return 'ready' if self.test_socket('localhost',self.rpc_port) else 'stopped' + if not self.test_socket(self.host,self.rpc_port): return 'stopped' - from .rpc import MoneroWalletRPCConnection + from .rpc import MoneroWalletRPCClient try: - MoneroWalletRPCConnection( - g.monero_wallet_rpc_host, + MoneroWalletRPCClient( + self.host, self.rpc_port, - g.monero_wallet_rpc_user, - g.monero_wallet_rpc_password).get_version() + self.user, + self.passwd).call('get_version') return 'ready' except: return 'stopped' @@ -280,7 +287,7 @@ class MoneroWalletDaemon(Daemon): class CoinDaemon(Daemon): cfg_file_hdr = '' subclasses_must_implement = ('state','stop_cmd') - avail_flags = ('no_daemonize',) + avail_flags = ('no_daemonize','keep_cfg_file') network_ids = ('btc','btc_tn','btc_rt','bch','bch_tn','bch_rt','ltc','ltc_tn','ltc_rt','xmr','eth','etc') @@ -466,6 +473,7 @@ class MoneroDaemon(CoinDaemon): exec_fn_mswin = 'monerod.exe' ps_pid_mswin = True new_console_mswin = True + host = 'localhost' # FIXME def subclass_init(self): if self.platform == 'win': @@ -488,7 +496,7 @@ class MoneroDaemon(CoinDaemon): @property def state(self): - if not self.test_socket(g.monero_wallet_rpc_host,self.rpc_port): + if not self.test_socket(self.host,self.rpc_port): return 'stopped' cp = self.run_cmd( [self.coind_exec] @@ -532,16 +540,23 @@ class EthereumDaemon(CoinDaemon): @property def state(self): - from .rpc import EthereumRPCConnection + return 'ready' if self.test_socket('localhost',self.rpc_port) else 'stopped' + + # the following code does not work + from mmgen.protocol import init_coin + init_coin('eth') + + async def do(): + print(g.rpc) + ret = await g.rpc.call('eth_chainId') + print(ret) + return ('stopped','ready')[ret == '0x11'] + try: - conn = EthereumRPCConnection('localhost',self.rpc_port,socket_timeout=0.2) - except: + return run_session(do()) # socket exception is not propagated + except:# SocketError: return 'stopped' - ret = conn.eth_chainId(on_fail='return') - - return ('stopped','ready')[ret == '0x11'] - @property def stop_cmd(self): return ['kill','-Wf',self.pid] if self.platform == 'win' else ['kill',self.pid] diff --git a/mmgen/exception.py b/mmgen/exception.py index 8587949f..d6fb4a5c 100755 --- a/mmgen/exception.py +++ b/mmgen/exception.py @@ -33,6 +33,7 @@ class FileNotFound(Exception): mmcode = 1 class InvalidPasswdFormat(Exception): mmcode = 1 class CfgFileParseError(Exception): mmcode = 1 class UserOptError(Exception): mmcode = 1 +class NoLEDSupport(Exception): mmcode = 1 # 2: yellow hl, message only class InvalidTokenAddress(Exception): mmcode = 2 diff --git a/mmgen/globalvars.py b/mmgen/globalvars.py index 8fea2d35..42023e3c 100755 --- a/mmgen/globalvars.py +++ b/mmgen/globalvars.py @@ -47,7 +47,7 @@ class g(object): # Constants: version = '0.12.099' - release_date = 'March 2020' + release_date = 'May 2020' proj_name = 'MMGen' proj_url = 'https://github.com/mmgen/mmgen' @@ -95,7 +95,7 @@ class g(object): accept_defaults = False use_internal_keccak_module = False - chain = None # set by first call to rpc_init() + chain = None chains = ('mainnet','testnet','regtest') # rpc: @@ -107,7 +107,8 @@ class g(object): monero_wallet_rpc_user = 'monero' monero_wallet_rpc_password = '' rpc_fail_on_command = '' - rpc = None # global RPC handle + rpc = None # global RPC handle + aiohttp_rpc_queue_len = 16 use_cached_balances = False # regtest: @@ -155,7 +156,7 @@ class g(object): # 'long' opts - opt sets global var common_opts = ( 'color','no_license','testnet', - 'rpc_host','rpc_port','rpc_user','rpc_password', + 'rpc_host','rpc_port','rpc_user','rpc_password','rpc_backend','aiohttp_rpc_queue_len', 'monero_wallet_rpc_host','monero_wallet_rpc_user','monero_wallet_rpc_password', 'daemon_data_dir','force_256_color','regtest','coin','bob','alice', 'accept_defaults','token' @@ -210,6 +211,7 @@ class g(object): 'MMGEN_TESTNET', 'MMGEN_REGTEST', 'MMGEN_TRACEBACK', + 'MMGEN_RPC_BACKEND', 'MMGEN_USE_STANDALONE_SCRYPT_MODULE', 'MMGEN_DISABLE_COLOR', @@ -223,12 +225,15 @@ class g(object): 'comment_file', 'contract_data', ) - # Auto-typechecked and auto-set opts - incompatible with global_sets_opt and opt_sets_global + # Auto-typechecked and auto-set opts. These have no corresponding value in g. # First value in list is the default ov = namedtuple('autoset_opt_info',['type','choices']) autoset_opts = { - 'fee_estimate_mode': ov('nocase_pfx', ('conservative','economical')), + 'fee_estimate_mode': ov('nocase_pfx', ['conservative','economical']), + 'rpc_backend': ov('nocase_pfx', ['auto','httplib','curl','aiohttp','requests']), } + if platform == 'win': + autoset_opts['rpc_backend'].choices.remove('aiohttp') min_screen_width = 80 minconf = 1 diff --git a/mmgen/main_addrimport.py b/mmgen/main_addrimport.py index f740ce2f..0ebd4eed 100755 --- a/mmgen/main_addrimport.py +++ b/mmgen/main_addrimport.py @@ -71,119 +71,126 @@ The --batch and --rescan options cannot be used together. } } -cmd_args = opts.init(opts_data) +def parse_cmd_args(cmd_args): -def import_mmgen_list(infile): - al = (AddrList,KeyAddrList)[bool(opt.keyaddr_file)](infile) - if al.al_id.mmtype in ('S','B'): - from .tx import segwit_is_active - if not segwit_is_active(): - rdie(2,'Segwit is not active on this chain. Cannot import Segwit addresses') - return al + def import_mmgen_list(infile): + al = (AddrList,KeyAddrList)[bool(opt.keyaddr_file)](infile) + if al.al_id.mmtype in ('S','B'): + from .tx import segwit_is_active + if not segwit_is_active(): + rdie(2,'Segwit is not active on this chain. Cannot import Segwit addresses') + return al -if len(cmd_args) == 1: - infile = cmd_args[0] - check_infile(infile) - if opt.addrlist: - al = AddrList(addrlist=get_lines_from_file( - infile, - 'non-{pnm} addresses'.format(pnm=g.proj_name), - trim_comments=True)) + if len(cmd_args) == 1: + infile = cmd_args[0] + check_infile(infile) + if opt.addrlist: + al = AddrList(addrlist=get_lines_from_file( + infile, + 'non-{pnm} addresses'.format(pnm=g.proj_name), + trim_comments=True)) + else: + al = import_mmgen_list(infile) + elif len(cmd_args) == 0 and opt.address: + al = AddrList(addrlist=[opt.address]) + infile = 'command line' else: - al = import_mmgen_list(infile) -elif len(cmd_args) == 0 and opt.address: - al = AddrList(addrlist=[opt.address]) - infile = 'command line' -else: - die(1,ai_msgs('bad_args')) + die(1,ai_msgs('bad_args')) -m = ' from Seed ID {}'.format(al.al_id.sid) if hasattr(al.al_id,'sid') else '' -qmsg('OK. {} addresses{}'.format(al.num_addrs,m)) + return al,infile -err_msg = None +def check_opts(tw): + batch = bool(opt.batch) + rescan = bool(opt.rescan) -from .tw import TrackingWallet -tw = TrackingWallet(mode='w') + if rescan and not 'rescan' in tw.caps: + msg("'--rescan' ignored: not supported by {}".format(type(tw).__name__)) + rescan = False -if g.token: - if not is_coin_addr(g.token): - m = "When importing addresses for a new token, the token must be specified by address, not symbol." - raise InvalidTokenAddress('{!r}: invalid token address\n{}'.format(m)) - sym = tw.addr2sym(g.token) # check for presence in wallet or blockchain; raises exception on failure + if rescan and not opt.quiet: + confirm_or_raise(ai_msgs('rescan'),'continue',expect='YES') -if opt.rescan and not 'rescan' in tw.caps: - msg("'--rescan' ignored: not supported by {}".format(type(tw).__name__)) - opt.rescan = False + if batch and not 'batch' in tw.caps: + msg("'--batch' ignored: not supported by {}".format(type(tw).__name__)) + batch = False -if opt.rescan and not opt.quiet: - confirm_or_raise(ai_msgs('rescan'),'continue',expect='YES') + return batch,rescan -if opt.batch and not 'batch' in tw.caps: - msg("'--batch' ignored: not supported by {}".format(type(tw).__name__)) - opt.batch = False - -def import_address(addr,label,rescan): - try: tw.import_address(addr,label,rescan) +async def import_addr(tw,addr,label,rescan,msg_fmt,msg_args): + try: + task = asyncio.ensure_future(tw.import_address(addr,label,rescan)) # Python 3.7+: create_task() + if rescan: + start = time.time() + while True: + if task.done(): + break + msg_r(('\r{} '+msg_fmt).format(secs_to_hms(int(time.time()-start)),*msg_args)) + await asyncio.sleep(0.5) + await task + msg('\nOK') + else: + await task + qmsg(msg_fmt.format(*msg_args) + ' - OK') except Exception as e: - global err_msg - err_msg = e.args[0] - if g.token and not tw.get_token_param(g.token,'symbol'): - tw.set_token_param(g.token,'symbol',sym) - tw.set_token_param(g.token,'decimals',tw.token_obj.decimals()) + die(2,'\nImport of address {!r} failed: {!r}'.format(addr,e.args[0])) -w_n_of_m = len(str(al.num_addrs)) * 2 + 2 -w_mmid = 1 if opt.addrlist or opt.address else len(str(max(al.idxs()))) + 13 -msg_fmt = '{{:{}}} {{:34}} {{:{}}}'.format(w_n_of_m,w_mmid) +def make_args_list(tw,al,batch,rescan): -if opt.rescan: import threading + fs = '{:%s} {:34} {:%s}' % ( + len(str(al.num_addrs)) * 2 + 2, + 1 if opt.addrlist or opt.address else len(str(max(al.idxs()))) + 13 ) -fs = 'Importing {} address{} from {}{}' -bm =' (batch mode)' if opt.batch else '' -msg(fs.format(len(al.data),suf(al.data,'es'),infile,bm)) + for num,e in enumerate(al.data,1): + if e.idx: + label = '{}:{}'.format(al.al_id,e.idx) + (' ' + e.label if e.label else '') + add_msg = label + else: + label = '{}:{}'.format(g.proto.base_coin.lower(),e.addr) + add_msg = 'non-'+g.proj_name -if not al.data[0].addr.is_for_chain(g.chain): - die(2,'Address{} not compatible with {} chain!'.format((' list','')[bool(opt.address)],g.chain)) + if batch: + yield (e.addr,TwLabel(label),False) + else: + msg_args = ( f'{num}/{al.num_addrs}:', e.addr, '('+add_msg+')' ) + yield (tw,e.addr,TwLabel(label),rescan,fs,msg_args) -for n,e in enumerate(al.data): - if e.idx: - label = '{}:{}'.format(al.al_id,e.idx) - if e.label: label += ' ' + e.label - m = label +async def main(): + al,infile = parse_cmd_args(cmd_args) + + qmsg( + f'OK. {al.num_addrs} addresses' + + (f' from Seed ID {al.al_id.sid}' if hasattr(al.al_id,'sid') else '') ) + + msg( + f'Importing {len(al.data)} address{suf(al.data,"es")} from {infile}' + + (' (batch mode)' if opt.batch else '') ) + + if not al.data[0].addr.is_for_chain(g.chain): + die(2,f'Address{(" list","")[bool(opt.address)]} incompatible with {g.chain} chain!') + + from .tw import TrackingWallet + tw = await TrackingWallet(mode='i') + + batch,rescan = check_opts(tw) + + if g.token: + await tw.import_token() + + args_list = make_args_list(tw,al,batch,rescan) + + if batch: + ret = await tw.batch_import_address(list(args_list)) + msg(f'OK: {len(ret)} addresses imported') + elif rescan: + for arg_list in args_list: + await import_addr(*arg_list) else: - label = '{}:{}'.format(g.proto.base_coin.lower(),e.addr) - m = 'non-'+g.proj_name + tasks = [import_addr(*arg_list) for arg_list in args_list] + await asyncio.gather(*tasks) + msg('OK') - label = TwLabel(label) + del tw - if opt.batch: - if n == 0: arg_list = [] - arg_list.append((e.addr,label,False)) - continue - - msg_data = ('{}/{}:'.format(n+1,al.num_addrs),e.addr,'({})'.format(m)) - - if opt.rescan: - t = threading.Thread(target=import_address,args=[e.addr,label,True]) - t.daemon = True - t.start() - start = int(time.time()) - while True: - if t.is_alive(): - elapsed = int(time.time()-start) - msg_r(('\r{} '+msg_fmt).format(secs_to_hms(elapsed),*msg_data)) - time.sleep(0.5) - else: - if err_msg: die(2,'\nImport failed: {!r}'.format(err_msg)) - msg('\nOK') - break - else: - import_address(e.addr,label,False) - msg_r('\r'+msg_fmt.format(*msg_data)) - if err_msg: die(2,'\nImport failed: {!r}'.format(err_msg)) - msg(' - OK') - -if opt.batch: - ret = tw.batch_import_address(arg_list) - msg('OK: {} addresses imported'.format(len(ret))) - -del tw +cmd_args = opts.init(opts_data) +import asyncio +run_session(main()) diff --git a/mmgen/main_autosign.py b/mmgen/main_autosign.py index 24748461..412de947 100755 --- a/mmgen/main_autosign.py +++ b/mmgen/main_autosign.py @@ -112,18 +112,19 @@ cmd_args = opts.init(opts_data,add_opts=['mmgen_keys_from_file','in_fmt']) exit_if_mswin('autosigning') import mmgen.tx -import mmgen.altcoins.eth.tx from .txsign import txsign from .protocol import CoinProtocol,init_coin +from .rpc import rpc_init + if g.test_suite: from .daemon import CoinDaemon if opt.mountpoint: - mountpoint = opt.mountpoint # TODO: make global + mountpoint = opt.mountpoint opt.outdir = tx_dir = os.path.join(mountpoint,'tx') -def check_daemons_running(): +async def check_daemons_running(): if opt.coin: die(1,'--coin option not supported with this command. Use --coins instead') if opt.coins: @@ -140,7 +141,7 @@ def check_daemons_running(): g.rpc_port = CoinDaemon(get_network_id(coin,g.testnet),test_suite=True).rpc_port vmsg(f'Checking {coin} daemon') try: - rpc_init(reinit=True) + await rpc_init() except SystemExit as e: if e.code != 0: ydie(1,f'{coin} daemon not running or not listening on port {g.proto.rpc_port}') @@ -174,7 +175,7 @@ def do_umount(): msg(f'Unmounting {mountpoint}') run(['umount',mountpoint],check=True) -def sign_tx_file(txfile,signed_txs): +async def sign_tx_file(txfile,signed_txs): try: init_coin('BTC',testnet=False) tmp_tx = mmgen.tx.MMGenTX(txfile,metadata_only=True) @@ -193,15 +194,15 @@ def sign_tx_file(txfile,signed_txs): g.token = tmp_tx.dcoin g.dcoin = tmp_tx.dcoin or g.coin - tx = mmgen.tx.MMGenTX(txfile,offline=True) + tx = mmgen.tx.MMGenTxForSigning(txfile) if g.proto.sign_mode == 'daemon': if g.test_suite: g.proto.daemon_data_dir = 'test/daemons/' + g.coin.lower() g.rpc_port = CoinDaemon(get_network_id(g.coin,g.testnet),test_suite=True).rpc_port - rpc_init(reinit=True) + await rpc_init() - if txsign(tx,wfs,None,None): + if await txsign(tx,wfs,None,None): tx.write_to_file(ask_write=False) signed_txs.append(tx) return True @@ -215,7 +216,7 @@ def sign_tx_file(txfile,signed_txs): except: return False -def sign(): +async def sign(): dirlist = os.listdir(tx_dir) raw,signed = [set(f[:-6] for f in dirlist if f.endswith(ext)) for ext in ('.rawtx','.sigtx')] unsigned = [os.path.join(tx_dir,f+'.rawtx') for f in raw - signed] @@ -223,7 +224,7 @@ def sign(): if unsigned: signed_txs,fails = [],[] for txfile in unsigned: - ret = sign_tx_file(txfile,signed_txs) + ret = await sign_tx_file(txfile,signed_txs) if not ret: fails.append(txfile) qmsg('') @@ -296,23 +297,23 @@ def print_summary(signed_txs): else: msg('No non-MMGen outputs') -def do_sign(): +async def do_sign(): if not opt.stealth_led: - set_led('busy') + led.set('busy') do_mount() key_ok = decrypt_wallets() if key_ok: if opt.stealth_led: - set_led('busy') - ret = sign() + led.set('busy') + ret = await sign() do_umount() - set_led(('standby','off','error')[(not ret)*2 or bool(opt.stealth_led)]) + led.set(('standby','off','error')[(not ret)*2 or bool(opt.stealth_led)]) return ret else: msg('Password is incorrect!') do_umount() if not opt.stealth_led: - set_led('error') + led.set('error') return False def wipe_existing_key(): @@ -374,35 +375,6 @@ def setup(): ss_out = Wallet(ss=ss_in) ss_out.write_to_file(desc='autosign wallet',outdir=wallet_dir) -def ev_sleep(secs): - ev.wait(secs) - return ev.isSet() - -def do_led(on,off): - if not on: - open(status_ctl,'w').write('0\n') - while True: - if ev_sleep(3600): return - - while True: - for s_time,val in ((on,255),(off,0)): - open(status_ctl,'w').write('{}\n'.format(val)) - if ev_sleep(s_time): return - -def set_led(cmd): - if not opt.led: return - vmsg("Setting LED state to '{}'".format(cmd)) - timings = { - 'off': ( 0, 0 ), - 'standby': ( 2.2, 0.2 ), - 'busy': ( 0.06, 0.06 ), - 'error': ( 0.5, 0.5 )}[cmd] - global led_thread - if led_thread: - ev.set(); led_thread.join(); ev.clear() - led_thread = threading.Thread(target=do_led,name='LED loop',args=timings) - led_thread.start() - def get_insert_status(): if opt.no_insert_check: return True @@ -410,15 +382,21 @@ def get_insert_status(): except: return False else: return True -def do_loop(): +def check_wipe_present(): + try: + run(['wipe','-v'],stdout=DEVNULL,stderr=DEVNULL,check=True) + except: + die(2,"The 'wipe' utility must be installed before running this program") + +async def do_loop(): n,prev_status = 0,False if not opt.stealth_led: - set_led('standby') + led.set('standby') while True: status = get_insert_status() if status and not prev_status: msg('Device insertion detected') - do_sign() + await do_sign() prev_status = status if not n % 10: msg_r('\r{}\rWaiting'.format(' '*17)) @@ -427,54 +405,6 @@ def do_loop(): msg_r('.') n += 1 -def check_access(fn,desc='status LED control',init_val=None): - try: - b = open(fn).read().strip() - open(fn,'w').write('{}\n'.format(init_val or b)) - return True - except: - m1 = "You do not have access to the {} file\n".format(desc) - m2 = "To allow access, run 'sudo chmod 0666 {}'".format(fn) - msg(m1+m2) - return False - -def check_wipe_present(): - try: - run(['wipe','-v'],stdout=DEVNULL,stderr=DEVNULL,check=True) - except: - die(2,"The 'wipe' utility must be installed before running this program") - -def init_led(): - sc = { - 'opi': '/sys/class/leds/orangepi:red:status/brightness', - 'rpi': '/sys/class/leds/led0/brightness' - } - tc = { - 'rpi': '/sys/class/leds/led0/trigger', # mmc,none - } - for k in ('opi','rpi'): - try: os.stat(sc[k]) - except: pass - else: - board = k - break - else: - die(2,'Control files not found! LED option not supported') - - status_ctl = sc[board] - trigger_ctl = tc[board] if board in tc else None - - if not check_access(status_ctl) or ( - trigger_ctl and not check_access(trigger_ctl,desc='LED trigger',init_val='none') - ): - sys.exit(1) - - if trigger_ctl: - open(trigger_ctl,'w').write('none\n') - - return status_ctl,trigger_ctl - -# main() if len(cmd_args) not in (0,1): opts.usage() @@ -489,32 +419,29 @@ if len(cmd_args) == 1: check_wipe_present() wfs = get_wallet_files() -check_daemons_running() - -def at_exit(exit_val,nl=False): - if nl: msg('') - msg('Cleaning up...') - if opt.led: - set_led('off') - ev.set() - led_thread.join() - if trigger_ctl: - open(trigger_ctl,'w').write('mmc0\n') +def at_exit(exit_val,message='\nCleaning up...'): + if message: + msg(message) + led.stop() sys.exit(exit_val) -def handler(a,b): at_exit(1,nl=True) +def handler(a,b): + at_exit(1) signal.signal(signal.SIGTERM,handler) signal.signal(signal.SIGINT,handler) -if opt.led: - import threading - status_ctl,trigger_ctl = init_led() - ev = threading.Event() - led_thread = None +from .led import LEDControl +led = LEDControl(enabled=opt.led,simulate=g.test_suite and not os.getenv('MMGEN_TEST_SUITE_AUTOSIGN_LIVE')) +led.set('off') -if len(cmd_args) == 0: - ret = do_sign() - at_exit(int(not ret)) -elif cmd_args[0] == 'wait': - do_loop() +async def main(): + await check_daemons_running() + + if len(cmd_args) == 0: + ret = await do_sign() + at_exit(int(not ret),message='') + elif cmd_args[0] == 'wait': + await do_loop() + +run_session(main(),do_rpc_init=False) diff --git a/mmgen/main_split.py b/mmgen/main_split.py index 8058f61d..059e867e 100755 --- a/mmgen/main_split.py +++ b/mmgen/main_split.py @@ -20,6 +20,7 @@ """ mmgen-split: Split funds after a replayable chain fork using a timelocked transaction + UNMAINTAINED """ import time @@ -115,28 +116,27 @@ if opt.tx_fees: opt.tx_fee = opt.tx_fees.split(',')[idx] opts.opt_is_tx_fee('foo',opt.tx_fee,'transaction fee') # raises exception on error -rpc_init(reinit=True) - tx1 = MMGenSplitTX() opt.no_blank = True -gmsg("Creating timelocked transaction for long chain ({})".format(g.coin)) -locktime = int(opt.locktime or 0) or g.rpc.getblockcount() -tx1.create(mmids[0],locktime) +async def main(): + gmsg("Creating timelocked transaction for long chain ({})".format(g.coin)) + locktime = int(opt.locktime or 0) or await g.rpc.call('getblockcount') + tx1.create(mmids[0],locktime) -tx1.format() -tx1.create_fn() + tx1.format() + tx1.create_fn() -gmsg("\nCreating transaction for short chain ({})".format(opt.other_coin)) + gmsg("\nCreating transaction for short chain ({})".format(opt.other_coin)) -init_coin(opt.other_coin) + init_coin(opt.other_coin) -tx2 = MMGenSplitTX() -tx2.inputs = tx1.inputs -tx2.inputs.convert_coin() + tx2 = MMGenSplitTX() + tx2.inputs = tx1.inputs + tx2.inputs.convert_coin() -tx2.create_split(mmids[1]) + tx2.create_split(mmids[1]) -for tx,desc in ((tx1,'Long chain (timelocked)'),(tx2,'Short chain')): - tx.desc = desc + ' transaction' - tx.write_to_file(ask_write=False,ask_overwrite=not opt.yes,ask_write_default_yes=False) + for tx,desc in ((tx1,'Long chain (timelocked)'),(tx2,'Short chain')): + tx.desc = desc + ' transaction' + tx.write_to_file(ask_write=False,ask_overwrite=not opt.yes,ask_write_default_yes=False) diff --git a/mmgen/main_tool.py b/mmgen/main_tool.py index d2599202..6aa925d5 100755 --- a/mmgen/main_tool.py +++ b/mmgen/main_tool.py @@ -110,4 +110,7 @@ args,kwargs = tool._process_args(cmd,cmd_args) ret = tool.MMGenToolCmds.call(cmd,*args,**kwargs) +if type(ret).__name__ == 'coroutine': + ret = run_session(ret) + tool._process_result(ret,pager='pager' in kwargs and kwargs['pager'],print_result=True) diff --git a/mmgen/main_txbump.py b/mmgen/main_txbump.py index 8061a56a..65eabd2f 100755 --- a/mmgen/main_txbump.py +++ b/mmgen/main_txbump.py @@ -96,8 +96,6 @@ column below: cmd_args = opts.init(opts_data) -rpc_init() - tx_file = cmd_args.pop(0) check_infile(tx_file) @@ -109,61 +107,62 @@ seed_files = get_seed_files(opt,cmd_args) if (cmd_args or opt.send) else None kal = get_keyaddrlist(opt) kl = get_keylist(opt) -tx = MMGenBumpTX(filename=tx_file,send=(seed_files or kl or kal)) +sign_and_send = bool(seed_files or kl or kal) do_license_msg() silent = opt.yes and opt.tx_fee != None and opt.output_to_reduce != None -if not silent: - msg(green('ORIGINAL TRANSACTION')) - msg(tx.format_view(terse=True)) +async def main(): -tx.set_min_fee() + from .tw import TrackingWallet + tx = MMGenBumpTX(filename=tx_file,send=sign_and_send,tw=await TrackingWallet() if g.token else None) -tx.check_bumpable() + if not silent: + msg(green('ORIGINAL TRANSACTION')) + msg(tx.format_view(terse=True)) -msg('Creating new transaction') + tx.check_bumpable() # needs cached networkinfo['relayfee'] -op_idx = tx.choose_output() + msg('Creating new transaction') -if not silent: - msg('Minimum fee for new transaction: {} {}'.format(tx.min_fee.hl(),g.coin)) + op_idx = tx.choose_output() -fee = tx.get_usr_fee_interactive(tx_fee=opt.tx_fee,desc='User-selected') + if not silent: + msg('Minimum fee for new transaction: {} {}'.format(tx.min_fee.hl(),g.coin)) -tx.update_fee(op_idx,fee) + fee = tx.get_usr_fee_interactive(tx_fee=opt.tx_fee,desc='User-selected') -d = tx.get_fee_from_tx() -assert d == fee and d <= g.proto.max_tx_fee + tx.update_fee(op_idx,fee) -if g.proto.base_proto == 'Bitcoin': - tx.outputs.sort_bip69() # output amts have changed, so re-sort + d = tx.get_fee_from_tx() + assert d == fee and d <= g.proto.max_tx_fee -if not opt.yes: - tx.add_comment() # edits an existing comment + if g.proto.base_proto == 'Bitcoin': + tx.outputs.sort_bip69() # output amts have changed, so re-sort -from .tw import TwUnspentOutputs -tx.twuo = TwUnspentOutputs(minconf=opt.minconf) + if not opt.yes: + tx.add_comment() # edits an existing comment -tx.create_raw() # creates tx.hex, tx.txid -tx.add_timestamp() -tx.add_blockcount() + await tx.create_raw() # creates tx.hex, tx.txid -qmsg('Fee successfully increased') + tx.add_timestamp() + tx.add_blockcount() -if not silent: - msg(green('\nREPLACEMENT TRANSACTION:')) - msg_r(tx.format_view(terse=True)) + qmsg('Fee successfully increased') -del tx.twuo.wallet + if not silent: + msg(green('\nREPLACEMENT TRANSACTION:')) + msg_r(tx.format_view(terse=True)) -if seed_files or kl or kal: - if txsign(tx,seed_files,kl,kal): - tx.write_to_file(ask_write=False) - tx.send(exit_on_fail=True) - tx.write_to_file(ask_write=False) + if sign_and_send: + if await txsign(tx,seed_files,kl,kal): + tx.write_to_file(ask_write=False) + await tx.send(exit_on_fail=True) + tx.write_to_file(ask_write=False) + else: + die(2,'Transaction could not be signed') else: - die(2,'Transaction could not be signed') -else: - tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=False,ask_overwrite=not opt.yes) + tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=False,ask_overwrite=not opt.yes) + +run_session(main()) diff --git a/mmgen/main_txcreate.py b/mmgen/main_txcreate.py index 8badc068..ee36c141 100755 --- a/mmgen/main_txcreate.py +++ b/mmgen/main_txcreate.py @@ -78,9 +78,11 @@ cmd_args = opts.init(opts_data) g.use_cached_balances = opt.cached_balances -rpc_init() +async def main(): + from .tx import MMGenTX + from .tw import TrackingWallet + tx = MMGenTX(tw=await TrackingWallet() if g.token else None) + await tx.create(cmd_args,int(opt.locktime or 0),do_info=opt.info) + tx.write_to_file(ask_write=not opt.yes,ask_overwrite=not opt.yes,ask_write_default_yes=False) -from .tx import MMGenTX -tx = MMGenTX() -tx.create(cmd_args,int(opt.locktime or 0),do_info=opt.info) -tx.write_to_file(ask_write=not opt.yes,ask_overwrite=not opt.yes,ask_write_default_yes=False) +run_session(main()) diff --git a/mmgen/main_txdo.py b/mmgen/main_txdo.py index cff07604..b653144e 100755 --- a/mmgen/main_txdo.py +++ b/mmgen/main_txdo.py @@ -115,8 +115,6 @@ cmd_args = opts.init(opts_data) g.use_cached_balances = opt.cached_balances -rpc_init() - from .tx import * from .txsign import * @@ -124,16 +122,23 @@ seed_files = get_seed_files(opt,cmd_args) kal = get_keyaddrlist(opt) kl = get_keylist(opt) -if kl and kal: kl.remove_dup_keys(kal) +if kl and kal: + kl.remove_dup_keys(kal) -tx = MMGenTX(caller='txdo') +async def main(): + from .tw import TrackingWallet + tx1 = MMGenTX(caller='txdo',tw=await TrackingWallet() if g.token else None) -tx.create(cmd_args,int(opt.locktime or 0)) + await tx1.create(cmd_args,int(opt.locktime or 0)) -if txsign(tx,seed_files,kl,kal): - tx.write_to_file(ask_write=False) - tx.send(exit_on_fail=True) - tx.write_to_file(ask_overwrite=False,ask_write=False) - tx.print_contract_addr() -else: - die(2,'Transaction could not be signed') + tx2 = MMGenTxForSigning(data=tx1.__dict__) + + if await txsign(tx2,seed_files,kl,kal): + tx2.write_to_file(ask_write=False) + await tx2.send(exit_on_fail=True) + tx2.write_to_file(ask_overwrite=False,ask_write=False) + tx2.print_contract_addr() + else: + die(2,'Transaction could not be signed') + +run_session(main()) diff --git a/mmgen/main_txsend.py b/mmgen/main_txsend.py index 3d06bd74..7ecb7931 100755 --- a/mmgen/main_txsend.py +++ b/mmgen/main_txsend.py @@ -40,8 +40,6 @@ opts_data = { cmd_args = opts.init(opts_data) -rpc_init() - if len(cmd_args) == 1: infile = cmd_args[0]; check_infile(infile) else: @@ -52,22 +50,33 @@ if not opt.status: from .tx import * -tx = MMGenTX(infile,quiet_open=True) # sig check performed here -vmsg("Signed transaction file '{}' is valid".format(infile)) +async def main(): -if not tx.marked_signed(): - die(1,'Transaction is not signed!') + from .tw import TrackingWallet + tx = MMGenTX(infile,quiet_open=True,tw=await TrackingWallet() if g.token else None) -if opt.status: - if tx.coin_txid: qmsg('{} txid: {}'.format(g.coin,tx.coin_txid.hl())) - tx.get_status(status=True) - sys.exit(0) + if g.token: + from .tw import TrackingWallet + tx.tw = await TrackingWallet() -if not opt.yes: - tx.view_with_prompt('View transaction data?') - if tx.add_comment(): # edits an existing comment, returns true if changed - tx.write_to_file(ask_write_default_yes=True) + vmsg("Signed transaction file '{}' is valid".format(infile)) -tx.send(exit_on_fail=True) -tx.write_to_file(ask_overwrite=False,ask_write=False) -tx.print_contract_addr() + if not tx.marked_signed(): + die(1,'Transaction is not signed!') + + if opt.status: + if tx.coin_txid: + qmsg('{} txid: {}'.format(g.coin,tx.coin_txid.hl())) + await tx.get_status(status=True) + sys.exit(0) + + if not opt.yes: + tx.view_with_prompt('View transaction data?') + if tx.add_comment(): # edits an existing comment, returns true if changed + tx.write_to_file(ask_write_default_yes=True) + + await tx.send(exit_on_fail=True) + tx.write_to_file(ask_overwrite=False,ask_write=False) + tx.print_contract_addr() + +run_session(main()) diff --git a/mmgen/main_txsign.py b/mmgen/main_txsign.py index b67b389e..1d5b2676 100755 --- a/mmgen/main_txsign.py +++ b/mmgen/main_txsign.py @@ -97,9 +97,6 @@ if not infiles: for i in infiles: check_infile(i) -if g.proto.sign_mode == 'daemon': - rpc_init() - if not opt.info and not opt.terse_info: do_license_msg(immed=True) @@ -107,39 +104,51 @@ from .txsign import * tx_files = get_tx_files(opt,infiles) seed_files = get_seed_files(opt,infiles) - kal = get_keyaddrlist(opt) kl = get_keylist(opt) -if kl and kal: kl.remove_dup_keys(kal) -tx_num_str,bad_tx_count = '',0 -for tx_num,tx_file in enumerate(tx_files,1): - if len(tx_files) > 1: - msg('\nTransaction #{} of {}:'.format(tx_num,len(tx_files))) - tx_num_str = ' #{}'.format(tx_num) - tx = MMGenTX(tx_file,offline=True) +if kl and kal: + kl.remove_dup_keys(kal) - if tx.marked_signed(): - msg('Transaction is already signed!'); continue +async def main(): + bad_tx_count = 0 + tx_num_disp = '' + for tx_num,tx_file in enumerate(tx_files,1): + if len(tx_files) > 1: + msg('\nTransaction #{} of {}:'.format(tx_num,len(tx_files))) + tx_num_disp = f' #{tx_num}' - vmsg("Successfully opened transaction file '{}'".format(tx_file)) + tx = MMGenTxForSigning(tx_file) - if opt.tx_id: - msg(tx.txid); continue + if tx.marked_signed(): + msg('Transaction is already signed!') + continue - if opt.info or opt.terse_info: - tx.view(pause=False,terse=opt.terse_info); continue + vmsg(f'Successfully opened transaction file {tx_file!r}') - if not opt.yes: - tx.view_with_prompt('View data for transaction{}?'.format(tx_num_str)) + if opt.tx_id: + msg(tx.txid) + continue + + if opt.info or opt.terse_info: + tx.view(pause=False,terse=opt.terse_info) + continue - if txsign(tx,seed_files,kl,kal,tx_num_str): if not opt.yes: - tx.add_comment() # edits an existing comment - tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=True,add_desc=tx_num_str) - else: - ymsg('Transaction could not be signed') - bad_tx_count += 1 + tx.view_with_prompt(f'View data for transaction{tx_num_disp}?') -if bad_tx_count: - ydie(2,'{} transaction{} could not be signed'.format(bad_tx_count,suf(bad_tx_count))) + if await txsign(tx,seed_files,kl,kal,tx_num_disp): + if not opt.yes: + tx.add_comment() # edits an existing comment + tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=True,add_desc=tx_num_disp) + else: + ymsg('Transaction could not be signed') + bad_tx_count += 1 + + if bad_tx_count: + ydie(2,f'{bad_tx_count} transaction{suf(bad_tx_count)} could not be signed') + +run_session( + main(), + do_rpc_init = g.proto.sign_mode == 'daemon' +) diff --git a/mmgen/obj.py b/mmgen/obj.py index 5c33328a..853489d9 100755 --- a/mmgen/obj.py +++ b/mmgen/obj.py @@ -28,6 +28,12 @@ from .exception import * from .globalvars import * from .color import * +class aInitMeta(type): + async def __call__(cls,*args,**kwargs): + instance = super().__call__(*args,**kwargs) + await instance.__ainit__(*args,**kwargs) + return instance + def is_mmgen_seed_id(s): return SeedID(sid=s,on_fail='silent') def is_mmgen_idx(s): return AddrIdx(s,on_fail='silent') def is_mmgen_id(s): return MMGenID(s,on_fail='silent') diff --git a/mmgen/opts.py b/mmgen/opts.py index c7db833f..0e059afc 100755 --- a/mmgen/opts.py +++ b/mmgen/opts.py @@ -110,7 +110,7 @@ def override_globals_from_cfg_file(ucfg): else: die(2,'{!r}: unrecognized option in {!r}, line {}'.format(d.name,ucfg.fn,d.lineno)) -def override_globals_from_env(): +def override_globals_and_set_opts_from_env(opt): for name in g.env_opts: if name == 'MMGEN_DEBUG_ALL': continue @@ -118,7 +118,13 @@ def override_globals_from_env(): val = os.getenv(name) # os.getenv() returns None if env var is unset if val: # exclude empty string values; string value of '0' or 'false' sets variable to False gname = name[(6,14)[disable]:].lower() - setattr(g,gname,set_for_type(val,getattr(g,gname),name,disable)) + if hasattr(g,gname): + setattr(g,gname,set_for_type(val,getattr(g,gname),name,disable)) + elif hasattr(opt,gname): + if getattr(opt,gname) is None: # env must not override cmdline! + setattr(opt,gname,val) + else: + raise ValueError(f'Name {gname} not present in globals or opts') def common_opts_code(s): from .protocol import CoinProtocol @@ -167,6 +173,8 @@ common_opts_data = { --, --rpc-port=p Communicate with {dn} listening on port 'p' --, --rpc-user=user Override 'rpc_user' in mmgen.cfg --, --rpc-password=pass Override 'rpc_password' in mmgen.cfg +--, --rpc-backend=s Override 'rpc_backend' in mmgen.cfg +--, --aiohttp-rpc-queue-len=N Override 'aiohttp_rpc_queue_len' in mmgen.cfg --, --monero-wallet-rpc-host=host Override 'monero_wallet_rpc_host' in mmgen.cfg --, --monero-wallet-rpc-user=user Override 'monero_wallet_rpc_user' in mmgen.cfg --, --monero-wallet-rpc-password=pass Override 'monero_wallet_rpc_password' in mmgen.cfg @@ -232,7 +240,7 @@ def init(opts_data,add_opts=[],opt_filter=None,parse_only=False): cfg_file('sample') # check for changes in system template file override_globals_from_cfg_file(cfg_file('usr')) - override_globals_from_env() + override_globals_and_set_opts_from_env(opt) # Set globals from opts, setting type from original global value # Do here, before opts are set from globals below @@ -240,7 +248,7 @@ def init(opts_data,add_opts=[],opt_filter=None,parse_only=False): for k in (g.common_opts + g.opt_sets_global): if hasattr(opt,k): val = getattr(opt,k) - if val != None: + if val != None and hasattr(g,k): setattr(g,k,set_for_type(val,getattr(g,k),'--'+k)) g.coin = g.coin.upper() # allow user to use lowercase @@ -337,7 +345,7 @@ def opt_is_tx_fee(key,val,desc): # 'key' must remain a placeholder return from .tx import MMGenTX - tx = MMGenTX(offline=True) + tx = MMGenTX() # Size of 224 is just a ball-park figure to eliminate the most extreme cases at startup # This check will be performed again once we know the true size ret = tx.process_fee_spec(val,224,on_fail='return') @@ -466,7 +474,8 @@ def check_usr_opts(usr_opts): # Raises an exception if any check fails opt_compares(val,'<=',g.max_urandchars,desc) def chk_tx_fee(key,val,desc): - opt_is_tx_fee(key,val,desc) + pass +# opt_is_tx_fee(key,val,desc) # TODO: move this check elsewhere def chk_tx_confs(key,val,desc): opt_is_int(val,desc) diff --git a/mmgen/rpc.py b/mmgen/rpc.py index cbc7c6f4..8daa520b 100755 --- a/mmgen/rpc.py +++ b/mmgen/rpc.py @@ -20,217 +20,383 @@ rpc.py: Cryptocoin RPC library for the MMGen suite """ -import http.client,base64,json +import base64,json,asyncio from decimal import Decimal - from .common import * +from .obj import aInitMeta + +rpc_credentials_msg = lambda: '\n'+fmt(f""" + Error: no {g.proto.name.capitalize()} RPC authentication method found + + RPC credentials must be supplied using one of the following methods: + + A) If daemon is local and running as same user as you: + + - no credentials required, or matching rpcuser/rpcpassword and + rpc_user/rpc_password values in {g.proto.name}.conf and mmgen.cfg + + B) If daemon is running remotely or as different user: + + - matching credentials in {g.proto.name}.conf and mmgen.cfg as described above + + The --rpc-user/--rpc-password options may be supplied on the MMGen command line. + They override the corresponding values in mmgen.cfg. Set them to an empty string + to use cookie authentication with a local server when the options are set + in mmgen.cfg. + + For better security, rpcauth should be used in {g.proto.name}.conf instead of + rpcuser/rpcpassword. + +""",strip_char='\t') def dmsg_rpc(fs,data=None,is_json=False): if g.debug_rpc: msg(fs if data == None else fs.format(pp_fmt(json.loads(data) if is_json else data))) -class RPCConnection(MMGenObject): +class json_encoder(json.JSONEncoder): + def default(self,obj): + if isinstance(obj,Decimal): + return str(obj) + else: + return json.JSONEncoder.default(self,obj) - auth = True - db_fs = ' host [{h}] port [{p}] user [{u}] passwd [{pw}] auth_cookie [{c}]\n' - http_hdrs = { 'Content-Type': 'application/json' } +class RPCBackends: - def __init__(self,host=None,port=None,user=None,passwd=None,auth_cookie=None,socket_timeout=1): + class aiohttp: + + def __init__(self,caller): + self.caller = caller + self.session = g.session + self.url = caller.url + self.timeout = caller.timeout + if caller.auth_type == 'basic': + import aiohttp + self.auth = aiohttp.BasicAuth(*caller.auth,encoding='UTF-8') + else: + self.auth = None + + async def run(self,payload,timeout=None): + dmsg_rpc('\n RPC PAYLOAD data (aiohttp) ==>\n{}\n',payload) + async with self.session.post( + url = self.url, + auth = self.auth, + data = json.dumps(payload,cls=json_encoder), + timeout = timeout or self.timeout, + ) as res: + return (await res.text(),res.status) + + class requests: + + def __init__(self,caller): + self.url = caller.url + self.timeout = caller.timeout + import requests,urllib3 + urllib3.disable_warnings() + self.session = requests.Session() + self.session.headers = caller.http_hdrs + if caller.auth_type: + auth = 'HTTP' + caller.auth_type.capitalize() + 'Auth' + self.session.auth = getattr(requests.auth,auth)(*caller.auth) + + async def run(self,payload,timeout=None): + dmsg_rpc('\n RPC PAYLOAD data (requests) ==>\n{}\n',payload) + res = self.session.post( + url = self.url, + data = json.dumps(payload,cls=json_encoder), + timeout = timeout or self.timeout, + verify = False ) + return (res.content,res.status_code) + + class httplib: + + def __init__(self,caller): + import http.client + self.session = http.client.HTTPConnection(caller.host,caller.port,caller.timeout) + self.http_hdrs = caller.http_hdrs + self.host = caller.host + self.port = caller.port + if caller.auth_type == 'basic': + auth_str = f'{caller.auth.user}:{caller.auth.passwd}' + auth_str_b64 = 'Basic ' + base64.b64encode(auth_str.encode()).decode() + self.http_hdrs.update({ 'Host': self.host, 'Authorization': auth_str_b64 }) + fs = ' RPC AUTHORIZATION data ==> raw: [{}]\n{:>31}enc: [{}]\n' + dmsg_rpc(fs.format(auth_str,'',auth_str_b64)) + + async def run(self,payload,timeout=None): + dmsg_rpc('\n RPC PAYLOAD data (httplib) ==>\n{}\n',payload) + if timeout: + import http.client + s = http.client.HTTPConnection(self.host,self.port,timeout) + else: + s = self.session + try: + s.request( + method = 'POST', + url = '/', + body = json.dumps(payload,cls=json_encoder), + headers = self.http_hdrs ) + r = s.getresponse() # => http.client.HTTPResponse instance + except Exception as e: + raise RPCFailure(str(e)) + return (r.read(),r.status) + + class curl: + + def __init__(self,caller): + + def gen(): + for k,v in caller.http_hdrs.items(): + for s in ('--header',f'{k}: {v}'): + yield s + if caller.auth_type: + """ + Authentication with curl is insecure, as it exposes the user's credentials + via the command line. Use for testing only. + """ + for s in ('--user',f'{caller.auth.user}:{caller.auth.passwd}'): + yield s + if caller.auth_type == 'digest': + yield '--digest' + + self.url = caller.url + self.exec_opts = list(gen()) + ['--silent'] + self.arg_max = 8192 # set way below system ARG_MAX, just to be safe + self.timeout = caller.timeout + + async def run(self,payload,timeout=None): + data = json.dumps(payload,cls=json_encoder) + if len(data) > self.arg_max: + return self.httplib(payload,timeout=timeout) + dmsg_rpc('\n RPC PAYLOAD data (curl) ==>\n{}\n',payload) + exec_cmd = [ + 'curl', + '--proxy', '', + '--connect-timeout', str(timeout or self.timeout), + '--request', 'POST', + '--write-out', '%{http_code}', + '--data-binary', data + ] + self.exec_opts + [self.url] + + dmsg_rpc(' RPC curl exec data ==>\n{}\n',exec_cmd) + + from subprocess import run,PIPE + res = run(exec_cmd,stdout=PIPE,check=True).stdout.decode() + # res = run(exec_cmd,stdout=PIPE,check=True,text='UTF-8').stdout # Python 3.7+ + return (res[:-3],int(res[-3:])) + +from collections import namedtuple +auth_data = namedtuple('rpc_auth_data',['user','passwd']) + +class RPCClient(MMGenObject): + + has_auth_cookie = False + url_fs = 'http://{}:{}' + + def __init__(self,host,port): dmsg_rpc('=== {}.__init__() debug ==='.format(type(self).__name__)) - dmsg_rpc(self.db_fs.format(h=host,p=port,u=user,pw=passwd,c=auth_cookie)) + dmsg_rpc(f' cls [{type(self).__name__}] host [{host}] port [{port}]\n') import socket try: - socket.create_connection((host,port),timeout=socket_timeout).close() + socket.create_connection((host,port),timeout=1).close() except: raise SocketError('Unable to connect to {}:{}'.format(host,port)) - if user and passwd: # user/pass overrides cookie - pass - elif auth_cookie: - user,passwd = auth_cookie.split(':') - elif self.auth: - msg('Error: no {} RPC authentication method found'.format(g.proto.name.capitalize())) - if passwd: die(1,"'rpcuser' entry not found in {}.conf or mmgen.cfg".format(g.proto.name)) - elif user: die(1,"'rpcpassword' entry not found in {}.conf or mmgen.cfg".format(g.proto.name)) - else: - m1 = 'Either provide rpcuser/rpcpassword in {pn}.conf or mmgen.cfg\n' - m2 = '(or, alternatively, copy the authentication cookie to the {pnu}\n' - m3 = 'data dir if {pnm} and {dn} are running as different users)' - die(1,(m1+m2+m3).format( - pn=g.proto.name, - pnu=g.proto.name.capitalize(), - dn=g.proto.daemon_name, - pnm=g.proj_name)) - - if self.auth: - fs = ' RPC AUTHORIZATION data ==> raw: [{}]\n{:>31}enc: [{}]\n' - auth_str = f'{user}:{passwd}' - auth_str_b64 = 'Basic ' + base64.b64encode(auth_str.encode()).decode() - dmsg_rpc(fs.format(auth_str,'',auth_str_b64)) - self.http_hdrs.update({ 'Host': host, 'Authorization': auth_str_b64 }) - + self.http_hdrs = { 'Content-Type': 'application/json' } + self.url = self.url_fs.format(host,port) self.host = host self.port = port - self.user = user - self.passwd = passwd + self.timeout = g.http_timeout + self.auth = None - for method in self.rpcmethods: - exec('{c}.{m} = lambda self,*args,**kwargs: self.request("{m}",*args,**kwargs)'.format( - c=type(self).__name__,m=method)) + def set_backend(self,backend=None): + bn = backend or opt.rpc_backend + if bn == 'auto': + self.backend = {'linux':RPCBackends.httplib,'win':RPCBackends.curl}[g.platform](self) + else: + self.backend = getattr(RPCBackends,bn)(self) - def calls(self,method,args_list): + def set_auth(self): """ - Perform a list of RPC calls, returning results in a list + MMGen's credentials override coin daemon's + """ + if g.rpc_user: + user,passwd = (g.rpc_user,g.rpc_password) + else: + user,passwd = get_coin_daemon_cfg_options(('rpcuser','rpcpassword')).values() + if user and passwd: + self.auth = auth_data(user,passwd) + return + + if self.has_auth_cookie: + cookie = self.get_daemon_auth_cookie() + if cookie: + self.auth = auth_data(*cookie.split(':')) + return + + die(1,rpc_credentials_msg()) + + # positional params are passed to the daemon, kwargs to the backend + # 'timeout' is currently the only supported kwarg + + async def call(self,method,*params,**kwargs): + """ + default call: call with param list unrolled, exactly as with cli + """ + if method == g.rpc_fail_on_command: + method = 'badcommand_' + method + return await self.process_http_resp(self.backend.run( + payload = {'id': 1, 'jsonrpc': '2.0', 'method': method, 'params': params }, + **kwargs + )) + + async def batch_call(self,method,param_list,**kwargs): + """ + Make a single call with a list of tuples as first argument + For RPC calls that return a list of results + """ + return await self.process_http_resp(self.backend.run( + payload = [{ + 'id': n, + 'jsonrpc': '2.0', + 'method': method, + 'params': params } for n,params in enumerate(param_list,1) ], + **kwargs + ),batch=True) + + async def gathered_call(self,method,args_list,**kwargs): + """ + Perform multiple RPC calls, returning results in a list Can be called two ways: 1) method = methodname, args_list = [args_tuple1, args_tuple2,...] 2) method = None, args_list = [(methodname1,args_tuple1), (methodname2,args_tuple2), ...] """ - cmd_list = args_list if method == None else tuple(zip([method] * len(args_list), args_list)) - if True: - return [self.request(method,*params) for method,params in cmd_list] - - # Normal mode: call with arg list unrolled, exactly as with cli - # Batch mode: call with list of arg lists as first argument - # kwargs are for local use and are not passed to server - - # By default, raises RPCFailure exception with an error msg on all errors and exceptions - # on_fail is one of 'raise' (default), 'return' or 'silent' - # With on_fail='return', returns 'rpcfail',(resp_object,(die_args)) - def request(self,cmd,*args,**kwargs): - - if g.debug: - print_stack_trace('RPC REQUEST {}\n args: {!r}\n kwargs: {!r}'.format(cmd,args,kwargs)) - - if g.rpc_fail_on_command == cmd: - cmd = 'badcommand_' + cmd - - cf = { 'timeout':g.http_timeout, 'batch':False, 'on_fail':'raise' } - - if cf['on_fail'] not in ('raise','return','silent'): - raise ValueError("request(): {}: illegal value for 'on_fail'".format(cf['on_fail'])) - - for k in cf: - if k in kwargs and kwargs[k]: cf[k] = kwargs[k] - - if cf['batch']: - p = [{'method':cmd,'params':r,'id':n,'jsonrpc':'2.0'} for n,r in enumerate(args[0],1)] - else: - p = {'method':cmd,'params':args,'id':1,'jsonrpc':'2.0'} - - dmsg_rpc('=== request() debug ===') - dmsg_rpc(' RPC POST data ==>\n{}\n',p) - - ca_type = self.coin_amt_type if hasattr(self,'coin_amt_type') else str - from .obj import HexStr - class MyJSONEncoder(json.JSONEncoder): - def default(self,obj): - if isinstance(obj,g.proto.coin_amt): - return ca_type(obj) - elif isinstance(obj,HexStr): - return obj - else: - return json.JSONEncoder.default(self,obj) - - data = json.dumps(p,cls=MyJSONEncoder) - - if g.platform == 'win' and len(data) < 4096: # set way below ARG_MAX, just to be safe - return self.do_request_curl(data,cf) - else: - return self.do_request_httplib(data,cf) - - def do_request_httplib(self,data,cf): - - def do_fail(*args): # args[0] is either None or HTTPResponse object - if cf['on_fail'] in ('return','silent'): return 'rpcfail',args - - try: s = '{}'.format(args[2]) - except: s = repr(args[2]) - - if s == '' and args[0] != None: - from http import HTTPStatus - hs = HTTPStatus(args[0].code) - s = '{} {}'.format(hs.value,hs.name) - - raise RPCFailure(s) - - hc = http.client.HTTPConnection(self.host,self.port,cf['timeout']) - try: - hc.request('POST','/',data,self.http_hdrs) - except Exception as e: - m = '{}\nUnable to connect to {} at {}:{}' - return do_fail(None,2,m.format(e.args[0],g.proto.daemon_name,self.host,self.port)) - - try: - r = hc.getresponse() # returns HTTPResponse instance - except Exception: - m = 'Unable to connect to {} at {}:{} (but port is bound?)' - return do_fail(None,2,m.format(g.proto.daemon_name,self.host,self.port)) - - dmsg_rpc(' RPC GETRESPONSE data ==>\n{}\n',r.__dict__) - - if r.status != 200: - if cf['on_fail'] not in ('silent','raise'): - msg_r(yellow('{} RPC Error: '.format(g.proto.daemon_name.capitalize()))) - msg(red('{} {}'.format(r.status,r.reason))) - e1 = r.read().decode() - try: - e3 = json.loads(e1)['error'] - e2 = '{} (code {})'.format(e3['message'],e3['code']) - except: - e2 = str(e1) - return do_fail(r,1,e2) - - r2 = r.read().decode() - - dmsg_rpc(' RPC REPLY data ==>\n{}\n',r2,is_json=True) - - if not r2: - return do_fail(r,2,'Empty reply') - - r3 = json.loads(r2,parse_float=Decimal) + cur_pos = 0 + chunk_size = 1024 ret = [] - for resp in r3 if cf['batch'] else [r3]: - if 'error' in resp and resp['error'] != None: - return do_fail(r,1,'{} returned an error: {}'.format( - g.proto.daemon_name.capitalize(),resp['error'])) - elif 'result' not in resp: - return do_fail(r,1, 'Missing JSON-RPC result\n' + repr(resps)) + while cur_pos < len(cmd_list): + tasks = [self.process_http_resp(self.backend.run( + payload = {'id': n, 'jsonrpc': '2.0', 'method': method, 'params': params }, + **kwargs + )) for n,(method,params) in enumerate(cmd_list[cur_pos:chunk_size+cur_pos],1)] + ret.extend(await asyncio.gather(*tasks)) + cur_pos += chunk_size + + return ret + + async def process_http_resp(self,coro,batch=False): + text,status = await coro + if status == 200: + dmsg_rpc(' RPC RESPONSE data ==>\n{}\n',text,is_json=True) + if batch: + return [r['result'] for r in json.loads(text,parse_float=Decimal,encoding='UTF-8')] else: - ret.append(resp['result']) + try: + return json.loads(text,parse_float=Decimal,encoding='UTF-8')['result'] + except: + raise RPCFailure(json.loads(text)['error']['message']) + else: + import http + s = http.HTTPStatus(status) + m = '' + if text: + try: m = ': ' + json.loads(text)['error']['message'] + except: + try: m = f': {text.decode()}' + except: m = f': {text}' + raise RPCFailure(f'{s.value} {s.name}{m}') - return ret if cf['batch'] else ret[0] - def do_request_curl(self,data,cf): - from subprocess import run,PIPE - exec_cmd = ['curl', '--proxy', '', '--silent','--request', 'POST', '--data-binary', data] - for k,v in self.http_hdrs.items(): - exec_cmd += ['--header', '{}: {}'.format(k,v)] - if self.auth: - exec_cmd += ['--user', self.user + ':' + self.passwd] - exec_cmd += ['http://{}:{}/'.format(self.host,self.port)] +class BitcoinRPCClient(RPCClient,metaclass=aInitMeta): - cp = run(exec_cmd,stdout=PIPE,check=True) - res = json.loads(cp.stdout,parse_float=Decimal) - dmsg_rpc(' RPC RESULT data ==>\n{}\n',res) + auth_type = 'basic' + has_auth_cookie = True - def do_fail(s): - if cf['on_fail'] in ('return','silent'): - return ('rpcfail',s) - raise RPCFailure(s) + def __init__(self,*args,**kwargs): pass - for resp in ([res],res)[cf['batch']]: - if 'error' in resp and resp['error'] != None: - return do_fail('{} returned an error: {}'.format(g.proto.daemon_name,resp['error'])) - elif 'result' not in resp: - return do_fail('Missing JSON-RPC result\n{!r}'.format(resp)) + async def __ainit__(self,backend=None): - return [r['result'] for r in res] if cf['batch'] else res['result'] + async def check_chainfork_mismatch(block0): + try: + assert block0 == g.proto.block0,'Incorrect Genesis block for {}'.format(g.proto.__name__) + for fork in g.proto.forks: + if fork.height == None or self.blockcount < fork.height: + break + if fork.hash != await self.call('getblockhash',fork.height): + die(3,f'Bad block hash at fork block {fork.height}. Is this the {fork.name} chain?') + except Exception as e: + die(2,"{}\n'{c}' requested, but this is not the {c} chain!".format(e.args[0],c=g.coin)) + + def check_chaintype_mismatch(): + try: + if g.regtest: assert g.chain == 'regtest','--regtest option selected, but chain is not regtest' + if g.testnet: assert g.chain != 'mainnet','--testnet option selected, but chain is mainnet' + if not g.testnet: assert g.chain == 'mainnet','mainnet selected, but chain is not mainnet' + except Exception as e: + die(1,'{}\nChain is {}!'.format(e.args[0],g.chain)) + + user,passwd = get_coin_daemon_cfg_options(('rpcuser','rpcpassword')).values() + + super().__init__( + host = g.rpc_host or 'localhost', + port = g.rpc_port or g.proto.rpc_port) + + self.set_auth() # set_auth() requires cookie, so must be called after __init__() tests socket + self.set_backend(backend) # backend requires self.auth + + if g.bob or g.alice: + from .regtest import MMGenRegtest + MMGenRegtest(g.coin).switch_user(('alice','bob')[g.bob],quiet=True) + + self.cached = {} + ( + self.cached['networkinfo'], + self.blockcount, + self.cached['blockchaininfo'], + block0 + ) = await self.gathered_call(None, ( + ('getnetworkinfo',()), + ('getblockcount',()), + ('getblockchaininfo',()), + ('getblockhash',(0,)), + )) + self.daemon_version = self.cached['networkinfo']['version'] + g.chain = self.cached['blockchaininfo']['chain'] + + tip = await self.call('getblockhash',self.blockcount) + self.cur_date = (await self.call('getblockheader',tip))['time'] + if g.chain != 'regtest': + g.chain += 'net' + assert g.chain in g.chains + check_chaintype_mismatch() + + if g.chain == 'mainnet': # skip this for testnet, as Genesis block may change + await check_chainfork_mismatch(block0) + + self.caps = ('full_node',) + for func,cap in ( + ('setlabel','label_api'), + ('signrawtransactionwithkey','sign_with_key') ): + if len((await self.call('help',func)).split('\n')) > 3: + self.caps += (cap,) + + # TODO: these belong in protocol.py + @classmethod + def get_daemon_auth_cookie_fn(cls): + cdir = os.path.join( + g.proto.daemon_data_dir, + g.proto.daemon_data_subdir ) + return os.path.join(cdir,'.cookie') + + @classmethod + def get_daemon_auth_cookie(cls): + fn = cls.get_daemon_auth_cookie_fn() + return get_lines_from_file(fn,'')[0] if file_is_readable(fn) else '' rpcmethods = ( 'backupwallet', @@ -268,12 +434,39 @@ class RPCConnection(MMGenObject): 'walletpassphrase', ) -class EthereumRPCConnection(RPCConnection): +class EthereumRPCClient(RPCClient,metaclass=aInitMeta): - auth = False - db_fs = ' host [{h}] port [{p}]\n' - _blockcount = None - _cur_date = None + auth_type = None + + def __init__(self,*args,**kwargs): pass + + async def __ainit__(self,backend=None): + + super().__init__( + host = g.rpc_host or 'localhost', + port = g.rpc_port or g.proto.rpc_port ) + + self.set_backend(backend) + + self.blockcount = int(await self.call('eth_blockNumber'),16) + + vi,bh,ch,nk = await self.gathered_call(None, ( + ('parity_versionInfo',()), + ('parity_getBlockHeaderByNumber',()), + ('parity_chain',()), + ('parity_nodeKind',()), + )) + + self.daemon_version = vi['version'] + self.cur_date = int(bh['timestamp'],16) + g.chain = ch.replace(' ','_') + self.caps = ('full_node',) if nk['capability'] == 'full' else () + + try: + await self.call('eth_chainId') + self.caps += ('eth_chainId',) + except RPCFailure: + pass rpcmethods = ( 'eth_accounts', @@ -314,20 +507,25 @@ class EthereumRPCConnection(RPCConnection): 'parity_versionInfo', ) - # blockcount and cur_date require network RPC calls, so evaluate lazily - @property - def blockcount(self): - if self._blockcount == None: - self._blockcount = int(self.eth_blockNumber(),16) - return self._blockcount +class MoneroWalletRPCClient(RPCClient): - @property - def cur_date(self): - if self._cur_date == None: - self._cur_date = int(self.parity_getBlockHeaderByNumber(hex(self.blockcount))['timestamp'],16) - return self._cur_date + auth_type = 'digest' + url_fs = 'http://{}:{}/json_rpc' -class MoneroWalletRPCConnection(RPCConnection): + def __init__(self,host,port,user,passwd): + super().__init__(host,port) + self.auth = auth_data(user,passwd) + self.set_backend('requests') + if False: # insecure, for debugging only + self.backend = RPCBackends.curl(self) + self.backend.exec_opts.remove('--silent') + self.backend.exec_opts.extend(['--insecure','--verbose']) + + async def call(self,method,*params,**kwargs): + assert params == (), f'{type(self).__name__}.call() accepts keyword arguments only' + return await self.process_http_resp(self.backend.run( + payload = {'id': 0, 'jsonrpc': '2.0', 'method': method, 'params': kwargs }, + )) rpcmethods = ( 'get_version', @@ -340,159 +538,16 @@ class MoneroWalletRPCConnection(RPCConnection): 'refresh', # start_height ) - def request(self,cmd,*args,**kwargs): - if args != (): - m = '{}.request() accepts only keyword args\nCmd: {!r}' - raise ValueError(m.format(type(self).__name__,cmd)) - import requests - import urllib3 - urllib3.disable_warnings() - ret = requests.post( - url = 'https://{}:{}/json_rpc'.format(self.host,self.port), - json = { - 'jsonrpc': '2.0', - 'id': '0', - 'method': cmd, - 'params': kwargs, - }, - auth = requests.auth.HTTPDigestAuth(self.user,self.passwd), - headers = self.http_hdrs, - verify = False ) +async def rpc_init(proto=None,backend=None): - res = json.loads(ret._content) - if 'error' in res: - raise RPCFailure(repr(res['error'])) - return(res['result']) + proto = proto or g.proto - def request_curltest(self,cmd,*args,**kwargs): - "insecure, for testing only" - from subprocess import run,PIPE - data = { - 'jsonrpc': '2.0', - 'id': '0', - 'method': cmd, - 'params': kwargs, - } - exec_cmd = [ - 'curl', '--proxy', '', '--verbose','--insecure', '--request', 'POST', - '--digest', '--user', '{}:{}'.format(g.monero_wallet_rpc_user,g.monero_wallet_rpc_password), - '--header', 'Content-Type: application/json', - '--data', json.dumps(data), - 'https://{}:{}/json_rpc'.format(self.host,self.port) ] + if not 'rpc' in proto.mmcaps: + die(1,'Coin daemon operations not supported for {}!'.format(proto.__name__)) - cp = run(exec_cmd,stdout=PIPE,check=True) + g.rpc = await { + 'bitcoind': BitcoinRPCClient, + 'parity': EthereumRPCClient, + }[proto.daemon_family](backend=backend) - res = json.loads(cp.stdout) - if 'error' in res: - raise RPCFailure(repr(res['error'])) - return(res['result']) - -def rpc_error(ret): - return type(ret) is tuple and ret and ret[0] == 'rpcfail' - -def rpc_errmsg(ret): - try: - return ret[1][2] - except: - return repr(ret) - -def init_daemon_parity(): - - def resolve_token_arg(token_arg): - from .obj import CoinAddr - from .altcoins.eth.tw import EthereumTrackingWallet - from .altcoins.eth.contract import Token - - tw = EthereumTrackingWallet(no_rpc=True) - - try: addr = CoinAddr(token_arg,on_fail='raise') - except: addr = tw.sym2addr(token_arg) - - if not addr: - m = "'{}': unrecognized token symbol" - raise UnrecognizedTokenSymbol(m.format(token_arg)) - - sym = tw.addr2sym(addr) # throws exception on failure - vmsg('ERC20 token resolved: {} ({})'.format(addr,sym)) - - return addr,sym - - conn = EthereumRPCConnection( - g.rpc_host or 'localhost', - g.rpc_port or g.proto.rpc_port) - conn.daemon_version = conn.parity_versionInfo()['version'] # fail immediately if daemon is geth - conn.coin_amt_type = str - g.chain = conn.parity_chain().replace(' ','_') - - conn.caps = () - try: - conn.request('eth_chainId') - conn.caps += ('eth_chainId',) - except RPCFailure: - pass - - if conn.request('parity_nodeKind')['capability'] == 'full': - conn.caps += ('full_node',) - - if g.token: - g.rpc = conn # set g.rpc so rpc_init() will return immediately - (g.token,g.dcoin) = resolve_token_arg(g.token) - - return conn - -def init_daemon_bitcoind(): - - def check_chainfork_mismatch(conn): - block0 = conn.getblockhash(0) - latest = conn.blockcount - try: - assert block0 == g.proto.block0,'Incorrect Genesis block for {}'.format(g.proto.__name__) - for fork in g.proto.forks: - if fork[0] == None or latest < fork[0]: break - assert conn.getblockhash(fork[0]) == fork[1], ( - 'Bad block hash at fork block {}. Is this the {} chain?'.format(fork[0],fork[2].upper())) - except Exception as e: - die(2,"{}\n'{c}' requested, but this is not the {c} chain!".format(e.args[0],c=g.coin)) - - def check_chaintype_mismatch(): - try: - if g.regtest: assert g.chain == 'regtest','--regtest option selected, but chain is not regtest' - if g.testnet: assert g.chain != 'mainnet','--testnet option selected, but chain is mainnet' - if not g.testnet: assert g.chain == 'mainnet','mainnet selected, but chain is not mainnet' - except Exception as e: - die(1,'{}\nChain is {}!'.format(e.args[0],g.chain)) - - cfg = get_daemon_cfg_options(('rpcuser','rpcpassword')) - - conn = RPCConnection( - g.rpc_host or 'localhost', - g.rpc_port or g.proto.rpc_port, - g.rpc_user or cfg['rpcuser'], # MMGen's rpcuser,rpcpassword override coin daemon's - g.rpc_password or cfg['rpcpassword'], - auth_cookie=get_coin_daemon_auth_cookie()) - - if g.bob or g.alice: - from .regtest import MMGenRegtest - MMGenRegtest(g.coin).switch_user(('alice','bob')[g.bob],quiet=True) - conn.daemon_version = int(conn.getnetworkinfo()['version']) - conn.blockcount = conn.getblockcount() - conn.cur_date = conn.getblockheader(conn.getblockhash(conn.blockcount))['time'] - conn.coin_amt_type = (float,str)[conn.daemon_version>=120000] - g.chain = conn.getblockchaininfo()['chain'] - if g.chain != 'regtest': g.chain += 'net' - assert g.chain in g.chains - check_chaintype_mismatch() - - if g.chain == 'mainnet': # skip this for testnet, as Genesis block may change - check_chainfork_mismatch(conn) - - conn.caps = ('full_node',) - for func,cap in ( - ('setlabel','label_api'), - ('signrawtransactionwithkey','sign_with_key') ): - if len(conn.request('help',func).split('\n')) > 3: - conn.caps += (cap,) - return conn - -def init_daemon(name): - return globals()['init_daemon_'+name]() + return g.rpc diff --git a/mmgen/tool.py b/mmgen/tool.py index 9b98c40c..901353e4 100755 --- a/mmgen/tool.py +++ b/mmgen/tool.py @@ -650,14 +650,15 @@ class MMGenToolCmdFile(MMGenToolCmds): file_sort = kwargs.get('filesort') or 'mtime' from .filename import MMGenFileList - from .tx import MMGenTX + from .tx import MMGenTX,MMGenTxForSigning flist = MMGenFileList(infiles,ftype=MMGenTX) flist.sort_by_age(key=file_sort) # in-place sort - sep = '—'*77+'\n' - return sep.join( - [MMGenTX(fn,offline=True).format_view(terse=terse,sort=tx_sort) for fn in flist.names()] - ).rstrip() + def gen(): + for fn in flist.names(): + yield (MMGenTxForSigning,MMGenTX)[fn.endswith('.sigtx')](fn).format_view(terse=terse,sort=tx_sort) + + return ('—'*77+'\n').join(gen()).rstrip() class MMGenToolCmdFileCrypt(MMGenToolCmds): """ @@ -841,12 +842,12 @@ from .tw import TwAddrList,TwUnspentOutputs class MMGenToolCmdRPC(MMGenToolCmds): "tracking wallet commands using the JSON-RPC interface" - def getbalance(self,minconf=1,quiet=False,pager=False): + async def getbalance(self,minconf=1,quiet=False,pager=False): "list confirmed/unconfirmed, spendable/unspendable balances in tracking wallet" from .tw import TwGetBalance - return TwGetBalance(minconf,quiet).format() + return (await TwGetBalance(minconf,quiet)).format() - def listaddress(self, + async def listaddress(self, mmgen_addr:str, minconf = 1, pager = False, @@ -855,7 +856,7 @@ class MMGenToolCmdRPC(MMGenToolCmds): age_fmt: _options_annot_str(TwAddrList.age_fmts) = 'confs', ): "list the specified MMGen address and its balance" - return self.listaddresses( mmgen_addrs = mmgen_addr, + return await self.listaddresses( mmgen_addrs = mmgen_addr, minconf = minconf, pager = pager, showempty = showempty, @@ -863,7 +864,7 @@ class MMGenToolCmdRPC(MMGenToolCmds): age_fmt = age_fmt, ) - def listaddresses( self, + async def listaddresses( self, mmgen_addrs:'(range or list)' = '', minconf = 1, showempty = False, @@ -890,13 +891,12 @@ class MMGenToolCmdRPC(MMGenToolCmds): die(1,m.format(mmgen_addrs)) usr_addr_list = [MMGenID('{}:{}'.format(a[0],i)) for i in AddrIdxList(a[1])] - rpc_init() - al = TwAddrList(usr_addr_list,minconf,showempty,showbtcaddrs,all_labels) + al = await TwAddrList(usr_addr_list,minconf,showempty,showbtcaddrs,all_labels) if not al: die(0,('No tracked addresses with balances!','No tracked addresses!')[showempty]) - return al.format(showbtcaddrs,sort,show_age,age_fmt or 'confs') + return await al.format(showbtcaddrs,sort,show_age,age_fmt or 'confs') - def twview( self, + async def twview( self, pager = False, reverse = False, wide = False, @@ -906,9 +906,8 @@ class MMGenToolCmdRPC(MMGenToolCmds): show_mmid = True, wide_show_confs = True): "view tracking wallet" - rpc_init() - twuo = TwUnspentOutputs(minconf=minconf) - twuo.do_sort(sort,reverse=reverse) + twuo = await TwUnspentOutputs(minconf=minconf) + await twuo.get_unspent_data(reverse_sort=reverse) twuo.age_fmt = age_fmt twuo.show_mmid = show_mmid if wide: @@ -916,25 +915,23 @@ class MMGenToolCmdRPC(MMGenToolCmds): else: ret = twuo.format_for_display() del twuo.wallet - return ret + return await ret - def add_label(self,mmgen_or_coin_addr:str,label:str): + async def add_label(self,mmgen_or_coin_addr:str,label:str): "add descriptive label for address in tracking wallet" - rpc_init() from .tw import TrackingWallet - TrackingWallet(mode='w').add_label(mmgen_or_coin_addr,label,on_fail='raise') + await (await TrackingWallet(mode='w')).add_label(mmgen_or_coin_addr,label,on_fail='raise') return True - def remove_label(self,mmgen_or_coin_addr:str): + async def remove_label(self,mmgen_or_coin_addr:str): "remove descriptive label for address in tracking wallet" - self.add_label(mmgen_or_coin_addr,'') + await self.add_label(mmgen_or_coin_addr,'') return True - def remove_address(self,mmgen_or_coin_addr:str): + async def remove_address(self,mmgen_or_coin_addr:str): "remove an address from tracking wallet" from .tw import TrackingWallet - tw = TrackingWallet(mode='w') - ret = tw.remove_address(mmgen_or_coin_addr) # returns None on failure + ret = await (await TrackingWallet(mode='w')).remove_address(mmgen_or_coin_addr) # returns None on failure if ret: msg("Address '{}' deleted from tracking wallet".format(ret)) return ret @@ -988,7 +985,7 @@ class MMGenToolCmdMonero(MMGenToolCmds): if monerod_args: self.monerod_args = monerod_args - def create(n,d,fn,c,m): + async def create(n,d,fn,c,m): try: os.stat(fn) except: pass else: @@ -997,7 +994,8 @@ class MMGenToolCmdMonero(MMGenToolCmds): gmsg(m) from .baseconv import baseconv - ret = c.restore_deterministic_wallet( + ret = await c.call( + 'restore_deterministic_wallet', filename = os.path.basename(fn), password = d.wallet_passwd, seed = baseconv.fromhex(d.sec,'xmrseed',tostr=True), @@ -1007,7 +1005,7 @@ class MMGenToolCmdMonero(MMGenToolCmds): pp_msg(ret) if opt.debug else msg(' Address: {}'.format(ret['address'])) return True - def sync(n,d,fn,c,m): + async def sync(n,d,fn,c,m): try: os.stat(fn) except: @@ -1021,11 +1019,14 @@ class MMGenToolCmdMonero(MMGenToolCmds): t_start = time.time() msg_r(' Opening wallet...') - c.open_wallet(filename=os.path.basename(fn),password=d.wallet_passwd) + await c.call( + 'open_wallet', + filename=os.path.basename(fn), + password=d.wallet_passwd ) msg('done') msg_r(' Getting wallet height...') - wallet_height = c.get_height()['height'] + wallet_height = (await c.call('get_height'))['height'] msg('\r Wallet height: {} '.format(wallet_height)) behind = chain_height - wallet_height @@ -1033,7 +1034,7 @@ class MMGenToolCmdMonero(MMGenToolCmds): m = ' Wallet is {} blocks behind chain tip. Please be patient. Syncing...' msg_r(m.format(behind)) - ret = c.refresh() + ret = await c.call('refresh') if behind > 1000: msg('done') @@ -1043,7 +1044,7 @@ class MMGenToolCmdMonero(MMGenToolCmds): t_elapsed = int(time.time() - t_start) - ret = c.get_balance() # account_index=0, address_indices=[0,1] + ret = await c.call('get_balance') # account_index=0, address_indices=[0,1] from .obj import XMRAmt bals[fn] = tuple([XMRAmt(ret[k],from_unit='min_coin_unit') for k in ('balance','unlocked_balance')]) @@ -1053,16 +1054,14 @@ class MMGenToolCmdMonero(MMGenToolCmds): else: msg(' Balance: {} Unlocked balance: {}'.format(*[b.hl() for b in bals[fn]])) - msg(' Wallet height: {}'.format(c.get_height()['height'])) + msg(' Wallet height: {}'.format((await c.call('get_height'))['height'])) msg(' Sync time: {:02}:{:02}'.format(t_elapsed//60,t_elapsed%60)) - c.close_wallet() + await c.call('close_wallet') return True - def process_wallets(): - m = { 'create': ('Creat','Generat',create,False), - 'sync': ('Sync', 'Sync', sync, True) } - opt.accept_defaults = opt.accept_defaults or m[op][3] + async def process_wallets(op): + opt.accept_defaults = opt.accept_defaults or op.accept_defaults from .protocol import init_coin init_coin('xmr') from .addr import AddrList @@ -1070,18 +1069,18 @@ class MMGenToolCmdMonero(MMGenToolCmds): data = [d for d in al.data if addrs == '' or d.idx in AddrIdxList(addrs)] dl = len(data) assert dl,"No addresses in addrfile within range '{}'".format(addrs) - gmsg('\n{}ing {} wallet{}'.format(m[op][0],dl,suf(dl))) + gmsg('\n{}ing {} wallet{}'.format(op.desc,dl,suf(dl))) from .daemon import MoneroWalletDaemon wd = MoneroWalletDaemon(opt.outdir or '.',test_suite=g.test_suite) wd.restart() - from .rpc import MoneroWalletRPCConnection - c = MoneroWalletRPCConnection( - g.monero_wallet_rpc_host, - wd.rpc_port, - g.monero_wallet_rpc_user, - g.monero_wallet_rpc_password) + from .rpc import MoneroWalletRPCClient + c = MoneroWalletRPCClient( + host = g.monero_wallet_rpc_host, + port = wd.rpc_port, + user = g.monero_wallet_rpc_user, + passwd = g.monero_wallet_rpc_password) wallets_processed = 0 for n,d in enumerate(data): # [d.sec,d.wallet_passwd,d.viewkey,d.addr] @@ -1091,13 +1090,13 @@ class MMGenToolCmdMonero(MMGenToolCmds): d.idx, '-α' if g.debug_utf8 else '')) - info = '\n{}ing wallet {}/{} ({})'.format(m[op][1],n+1,dl,fn) - wallets_processed += m[op][2](n,d,fn,c,info) + info = '\n{}ing wallet {}/{} ({})'.format(op.action,n+1,dl,fn) + wallets_processed += await op.func(n,d,fn,c,info) wd.stop() - gmsg('\n{} wallet{} {}ed'.format(wallets_processed,suf(wallets_processed),m[op][0].lower())) + gmsg('\n{} wallet{} {}ed'.format(wallets_processed,suf(wallets_processed),op.desc.lower())) - if wallets_processed and op == 'sync': + if wallets_processed and op.name == 'sync': col1_w = max(map(len,bals)) + 1 fs = '{:%s} {} {}' % col1_w msg('\n'+fs.format('Wallet','Balance ','Unlocked Balance ')) @@ -1114,8 +1113,13 @@ class MMGenToolCmdMonero(MMGenToolCmds): bals = {} # locked,unlocked + from collections import namedtuple + wo = namedtuple('mwo',['name','desc','action','func','accept_defaults']) + op = { # reusing name! + 'create': wo('create', 'Creat', 'Generat', create, False), + 'sync': wo('sync', 'Sync', 'Sync', sync, True) }[op] try: - process_wallets() + run_session(process_wallets(op),do_rpc_init=False) except KeyboardInterrupt: rdie(1,'\nUser interrupt\n') except EOFError: diff --git a/mmgen/tw.py b/mmgen/tw.py index 7b11bb39..01d06829 100755 --- a/mmgen/tw.py +++ b/mmgen/tw.py @@ -40,21 +40,21 @@ _date_formatter = { 'date_time': lambda secs: '{}-{:02}-{:02} {:02}:{:02}'.format(*time.gmtime(secs)[:5]), } -def _set_dates(us): +async def _set_dates(us): if us and us[0].date is None: # 'blocktime' differs from 'time', is same as getblockheader['time'] - dates = [o['blocktime'] for o in g.rpc.calls('gettransaction',[(o.txid,) for o in us])] + dates = [o['blocktime'] for o in await g.rpc.gathered_call('gettransaction',[(o.txid,) for o in us])] for o,date in zip(us,dates): o.date = date if os.getenv('MMGEN_BOGUS_WALLET_DATA'): # 1831006505 (09 Jan 2028) = projected time of block 1000000 _date_formatter['days'] = lambda date: (1831006505 - date) // 86400 - def _set_dates(us): + async def _set_dates(us): for o in us: o.date = 1831006505 - int(9.7 * 60 * (o.confs - 1)) -class TwUnspentOutputs(MMGenObject): +class TwUnspentOutputs(MMGenObject,metaclass=aInitMeta): def __new__(cls,*args,**kwargs): return MMGenObject.__new__(altcoin_subclass(cls,'tw','TwUnspentOutputs')) @@ -104,7 +104,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program. """.strip().format(g.proj_name.lower()) } - def __init__(self,minconf=1,addrs=[]): + async def __ainit__(self,minconf=1,addrs=[]): self.unspent = self.MMGenTwOutputList() self.fmt_display = '' self.fmt_print = '' @@ -117,9 +117,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program. self.sort_key = 'age' self.disp_prec = self.get_display_precision() - self.wallet = TrackingWallet('w') - self.get_unspent_data() - self.do_sort() + self.wallet = await TrackingWallet(mode='w') @property def age_fmt(self): @@ -138,7 +136,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program. def total(self): return sum(i.amt for i in self.unspent) - def get_unspent_rpc(self): + async def get_unspent_rpc(self): # bitcoin-cli help listunspent: # Arguments: # 1. minconf (numeric, optional, default=1) The minimum confirmations to filter @@ -149,17 +147,20 @@ watch-only wallet using '{}-addrimport' and then re-run this program. # for now, self.addrs is just an empty list for Bitcoin and friends add_args = (9999999,self.addrs) if self.addrs else () - return g.rpc.listunspent(self.minconf,*add_args) + return await g.rpc.call('listunspent',self.minconf,*add_args) - def get_unspent_data(self): + async def get_unspent_data(self,sort_key=None,reverse_sort=False): if g.bogus_wallet_data: # for debugging purposes only us_rpc = eval(get_data_from_file(g.bogus_wallet_data)) # testing, so ok else: - us_rpc = self.get_unspent_rpc() + us_rpc = await self.get_unspent_rpc() + + if not us_rpc: + die(0,self.wmsg['no_spendable_outputs']) - if not us_rpc: die(0,self.wmsg['no_spendable_outputs']) tr_rpc = [] lbl_id = ('account','label')['label_api' in g.rpc.caps] + for o in us_rpc: if not lbl_id in o: continue # coinbase outputs have no account field @@ -183,6 +184,8 @@ watch-only wallet using '{}-addrimport' and then re-run this program. if not self.unspent: die(1,'No tracked {}s in tracking wallet!'.format(self.item_desc)) + self.do_sort(key=sort_key,reverse=reverse_sort) + def do_sort(self,key=None,reverse=False): sort_funcs = { 'addr': lambda i: i.addr, @@ -214,10 +217,10 @@ watch-only wallet using '{}-addrimport' and then re-run this program. m2 = 'Please resize your screen to at least {} characters and hit ENTER ' my_raw_input((m1+m2).format(g.min_screen_width)) - def format_for_display(self): + async def format_for_display(self): unsp = self.unspent if self.age_fmt in self.age_fmts_date_dependent: - _set_dates(unsp) + await _set_dates(unsp) self.set_term_columns() # allow for 7-digit confirmation nums @@ -291,9 +294,9 @@ watch-only wallet using '{}-addrimport' and then re-run this program. self.fmt_display = '\n'.join(out) + '\n' return self.fmt_display - def format_for_printing(self,color=False,show_confs=True): + async def format_for_printing(self,color=False,show_confs=True): if self.age_fmt in self.age_fmts_date_dependent: - _set_dates(self.unspent) + await _set_dates(self.unspent) addr_w = max(len(i.addr) for i in self.unspent) mmid_w = max(len(('',i.twmmid)[i.twmmid.type=='mmgen']) for i in self.unspent) or 12 # DEADBEEF:S:1 amt_w = g.proto.coin_amt.max_prec + 5 @@ -374,14 +377,14 @@ watch-only wallet using '{}-addrimport' and then re-run this program. if keypress_confirm(fs.format(self.item_desc,n)): return n - def view_and_sort(self,tx): + async def view_and_sort(self,tx): from .term import get_char prompt = self.prompt.strip() + '\b' no_output,oneshot_msg = False,None while True: msg_r('' if no_output else '\n\n' if opt.no_blank else CUR_HOME+ERASE_ALL) reply = get_char( - '' if no_output else self.format_for_display()+'\n'+(oneshot_msg or '')+prompt, + '' if no_output else await self.format_for_display()+'\n'+(oneshot_msg or '')+prompt, immed_chars=''.join(self.key_mappings.keys()) ) no_output = False @@ -409,17 +412,15 @@ watch-only wallet using '{}-addrimport' and then re-run this program. idx = self.get_idx_from_user(action) if idx: e = self.unspent[idx-1] - bal = self.wallet.get_balance(e.addr,force_rpc=True) - self.get_unspent_data() - self.do_sort() + bal = await self.wallet.get_balance(e.addr,force_rpc=True) + await self.get_unspent_data() oneshot_msg = yellow('{} balance for account #{} refreshed\n\n'.format(g.dcoin,idx)) elif action == 'a_lbl_add': idx,lbl = self.get_idx_from_user(action) if idx: e = self.unspent[idx-1] - if self.wallet.add_label(e.twmmid,lbl,addr=e.addr): - self.get_unspent_data() - self.do_sort() + if await self.wallet.add_label(e.twmmid,lbl,addr=e.addr): + await self.get_unspent_data() a = 'added to' if lbl else 'removed from' oneshot_msg = yellow("Label {} {} #{}\n\n".format(a,self.item_desc,idx)) else: @@ -428,9 +429,8 @@ watch-only wallet using '{}-addrimport' and then re-run this program. idx = self.get_idx_from_user(action) if idx: e = self.unspent[idx-1] - if self.wallet.remove_address(e.addr): - self.get_unspent_data() - self.do_sort() + if await self.wallet.remove_address(e.addr): + await self.get_unspent_data() oneshot_msg = yellow("{} #{} removed\n\n".format(capfirst(self.item_desc),idx)) else: oneshot_msg = red('Address could not be removed\n\n') @@ -439,13 +439,13 @@ watch-only wallet using '{}-addrimport' and then re-run this program. ','.join(self.sort_info(include_group=False)).lower()) msg('') try: - write_data_to_file(of,self.format_for_printing(),desc='{} listing'.format(self.desc)) + write_data_to_file(of,await self.format_for_printing(),desc='{} listing'.format(self.desc)) except UserNonConfirmation as e: oneshot_msg = red("File '{}' not overwritten by user request\n\n".format(of)) else: oneshot_msg = yellow("Data written to '{}'\n\n".format(of)) elif action in ('a_view','a_view_wide'): - do_pager(self.fmt_display if action == 'a_view' else self.format_for_printing(color=True)) + do_pager(self.fmt_display if action == 'a_view' else await self.format_for_printing(color=True)) if g.platform == 'linux' and oneshot_msg == None: msg_r(CUR_RIGHT(len(prompt.split('\n')[-1])-2)) no_output = True @@ -458,7 +458,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program. else: return _date_formatter[age_fmt](o.date) -class TwAddrList(MMGenDict): +class TwAddrList(MMGenDict,metaclass=aInitMeta): has_age = True age_fmts = TwUnspentOutputs.age_fmts age_disp = TwUnspentOutputs.age_disp @@ -466,7 +466,10 @@ class TwAddrList(MMGenDict): def __new__(cls,*args,**kwargs): return MMGenDict.__new__(altcoin_subclass(cls,'tw','TwAddrList'),*args,**kwargs) - def __init__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None): + def __init__(self,*args,**kwargs): + pass + + async def __ainit__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None): def check_dup_mmid(acct_labels): mmid_prev,err = None,False @@ -490,10 +493,9 @@ class TwAddrList(MMGenDict): if err: rdie(3,'Tracking wallet is corrupted!') self.total = g.proto.coin_amt('0') - rpc_init() lbl_id = ('account','label')['label_api' in g.rpc.caps] - for d in g.rpc.listunspent(0): + for d in await g.rpc.call('listunspent',0): if not lbl_id in d: continue # skip coinbase outputs with missing account if d['confirmations'] < minconf: continue label = get_tw_label(d[lbl_id]) @@ -520,11 +522,12 @@ class TwAddrList(MMGenDict): # for compatibility with old mmids, must use raw RPC rather than native data for matching # args: minconf,watchonly, MUST use keys() so we get list, not dict if 'label_api' in g.rpc.caps: - acct_list = g.rpc.listlabels() - acct_addrs = [list(a.keys()) for a in g.rpc.getaddressesbylabel([[k] for k in acct_list],batch=True)] + acct_list = await g.rpc.call('listlabels') + aa = await g.rpc.batch_call('getaddressesbylabel',[(k,) for k in acct_list]) + acct_addrs = [list(a.keys()) for a in aa] else: - acct_list = list(g.rpc.listaccounts(0,True).keys()) # raw list, no 'L' - acct_addrs = g.rpc.getaddressesbyaccount([[a] for a in acct_list],batch=True) # use raw list here + acct_list = list((await g.rpc.call('listaccounts',0,True)).keys()) # raw list, no 'L' + acct_addrs = await g.rpc.batch_call('getaddressesbyaccount',[(a,) for a in acct_list]) # use raw list here acct_labels = MMGenList([get_tw_label(a) for a in acct_list]) check_dup_mmid(acct_labels) assert len(acct_list) == len(acct_addrs),( @@ -545,7 +548,7 @@ class TwAddrList(MMGenDict): def coinaddr_list(self): return [self[k]['addr'] for k in self] - def format(self,showbtcaddrs,sort,show_age,age_fmt): + async def format(self,showbtcaddrs,sort,show_age,age_fmt): if not self.has_age: show_age = False if age_fmt not in self.age_fmts: @@ -580,7 +583,7 @@ class TwAddrList(MMGenDict): al_id_save = None mmids = sorted(self,key=sort_algo,reverse=bool(sort and 'reverse' in sort)) if show_age: - _set_dates([o for o in mmids if hasattr(o,'confs')]) + await _set_dates([o for o in mmids if hasattr(o,'confs')]) for mmid in mmids: if mmid.type == 'mmgen': if al_id_save and al_id_save != mmid.obj.al_id: @@ -603,22 +606,27 @@ class TwAddrList(MMGenDict): return '\n'.join(out + ['\nTOTAL: {} {}'.format(self.total.hl(color=True),g.dcoin)]) -class TrackingWallet(MMGenObject): +class TrackingWallet(MMGenObject,metaclass=aInitMeta): caps = ('rescan','batch') data_key = 'addresses' use_tw_file = False aggressive_sync = False + importing = False def __new__(cls,*args,**kwargs): return MMGenObject.__new__(altcoin_subclass(cls,'tw','TrackingWallet')) - def __init__(self,mode='r',no_rpc=False): + async def __ainit__(self,mode='r'): + + assert mode in ('r','w','i'), "{!r}: wallet mode must be 'r','w' or 'i'".format(mode) + if mode == 'i': + self.importing = True + mode = 'w' if g.debug: print_stack_trace('TW INIT {!r} {!r}'.format(mode,self)) - assert mode in ('r','w'),"{!r}: wallet mode must be 'r' or 'w'".format(self) self.mode = mode self.desc = self.base_desc = '{} tracking wallet'.format(capfirst(g.proto.name)) @@ -632,7 +640,6 @@ class TrackingWallet(MMGenObject): raise WalletFileError(m.format(self.data['coin'],g.coin)) self.conv_types(self.data[self.data_key]) - self.rpc_init() self.cur_balances = {} # cache balances to prevent repeated lookups per program invocation def init_empty(self): @@ -697,12 +704,9 @@ class TrackingWallet(MMGenObject): @staticmethod def conv_types(ad): for k,v in ad.items(): - if k in ('params','coin'): continue - v['mmid'] = TwMMGenID(v['mmid'],on_fail='raise') - v['comment'] = TwComment(v['comment'],on_fail='raise') - - def rpc_init(self): - rpc_init() + if k not in ('params','coin'): + v['mmid'] = TwMMGenID(v['mmid'],on_fail='raise') + v['comment'] = TwComment(v['comment'],on_fail='raise') @property def data_root(self): @@ -728,14 +732,14 @@ class TrackingWallet(MMGenObject): if addr in data_root and 'balance' in data_root[addr]: return g.proto.coin_amt(data_root[addr]['balance']) - def get_balance(self,addr,force_rpc=False): + async def get_balance(self,addr,force_rpc=False): ret = None if force_rpc else self.get_cached_balance(addr,self.cur_balances,self.data_root) if ret == None: - ret = self.rpc_get_balance(addr) + ret = await self.rpc_get_balance(addr) self.cache_balance(addr,ret,self.cur_balances,self.data_root) return ret - def rpc_get_balance(self,addr): + async def rpc_get_balance(self,addr): raise NotImplementedError('not implemented') @property @@ -752,12 +756,12 @@ class TrackingWallet(MMGenObject): return dict((x['mmid'],{'addr':x['addr'],'comment':x['comment']}) for x in self.sorted_list) @write_mode - def import_address(self,addr,label,rescan): - return g.rpc.importaddress(addr,label,rescan,timeout=(False,3600)[rescan]) + async def import_address(self,addr,label,rescan): + return await g.rpc.call('importaddress',addr,label,rescan,timeout=(False,3600)[rescan]) @write_mode def batch_import_address(self,arg_list): - return g.rpc.importaddress(arg_list,batch=True) + return g.rpc.batch_call('importaddress',arg_list) def force_write(self): mode_save = self.mode @@ -789,24 +793,30 @@ class TrackingWallet(MMGenObject): elif g.debug: msg('Data is unchanged\n') - def is_in_wallet(self,addr): - return addr in TwAddrList([],0,True,True,True,wallet=self).coinaddr_list() + async def is_in_wallet(self,addr): + return addr in (await TwAddrList([],0,True,True,True,wallet=self)).coinaddr_list() @write_mode - def set_label(self,coinaddr,lbl): + async def set_label(self,coinaddr,lbl): # bitcoin-abc 'setlabel' RPC is broken, so use old 'importaddress' method to set label # broken behavior: new label is set OK, but old label gets attached to another address if 'label_api' in g.rpc.caps and g.coin != 'BCH': - return g.rpc.setlabel(coinaddr,lbl,on_fail='return') + args = ('setlabel',coinaddr,lbl) else: # NOTE: this works because importaddress() removes the old account before # associating the new account with the address. # RPC args: addr,label,rescan[=true],p2sh[=none] - return g.rpc.importaddress(coinaddr,lbl,False,on_fail='return') + args = ('importaddress',coinaddr,lbl,False) + + try: + return await g.rpc.call(*args) + except Exception as e: + rmsg(e.args[0]) + return False # returns on failure @write_mode - def add_label(self,arg1,label='',addr=None,silent=False,on_fail='return'): + async def add_label(self,arg1,label='',addr=None,silent=False,on_fail='return'): from .tx import is_mmgen_id,is_coin_addr mmaddr,coinaddr = None,None if is_coin_addr(addr or arg1): @@ -815,14 +825,14 @@ class TrackingWallet(MMGenObject): mmaddr = TwMMGenID(arg1) if mmaddr and not coinaddr: - from .addr import AddrData - coinaddr = AddrData(source='tw').mmaddr2coinaddr(mmaddr) + from .addr import TwAddrData + coinaddr = (await TwAddrData()).mmaddr2coinaddr(mmaddr) try: if not is_mmgen_id(arg1): assert coinaddr,"Invalid coin address for this chain: {}".format(arg1) assert coinaddr,"{pn} address '{ma}' not found in tracking wallet" - assert self.is_in_wallet(coinaddr),"Address '{ca}' not found in tracking wallet" + assert await self.is_in_wallet(coinaddr),"Address '{ca}' not found in tracking wallet" except Exception as e: msg(e.args[0].format(pn=g.proj_name,ma=mmaddr,ca=coinaddr)) return False @@ -830,8 +840,8 @@ class TrackingWallet(MMGenObject): # Allow for the possibility that BTC addr of MMGen addr was entered. # Do reverse lookup, so that MMGen addr will not be marked as non-MMGen. if not mmaddr: - from .addr import AddrData - mmaddr = AddrData(source='tw').coinaddr2mmaddr(coinaddr) + from .addr import TwAddrData + mmaddr = (await TwAddrData()).coinaddr2mmaddr(coinaddr) if not mmaddr: mmaddr = '{}:{}'.format(g.proto.base_coin.lower(),coinaddr) @@ -844,11 +854,7 @@ class TrackingWallet(MMGenObject): lbl = TwLabel(mmaddr + ('',' '+cmt)[bool(cmt)],on_fail=on_fail) - ret = self.set_label(coinaddr,lbl) - - from .rpc import rpc_error,rpc_errmsg - if rpc_error(ret): - msg('From {}: {}'.format(g.proto.daemon_name,rpc_errmsg(ret))) + if await self.set_label(coinaddr,lbl) == False: if not silent: msg('Label could not be {}'.format(('removed','added')[bool(label)])) return False @@ -861,32 +867,31 @@ class TrackingWallet(MMGenObject): return True @write_mode - def remove_label(self,mmaddr): - self.add_label(mmaddr,'') + async def remove_label(self,mmaddr): + await self.add_label(mmaddr,'') @write_mode - def remove_address(self,addr): + async def remove_address(self,addr): raise NotImplementedError('address removal not implemented for coin {}'.format(g.coin)) -class TwGetBalance(MMGenObject): +class TwGetBalance(MMGenObject,metaclass=aInitMeta): fs = '{w:13} {u:<16} {p:<16} {c}\n' def __new__(cls,*args,**kwargs): return MMGenObject.__new__(altcoin_subclass(cls,'tw','TwGetBalance')) - def __init__(self,minconf,quiet): + async def __ainit__(self,minconf,quiet): - rpc_init() self.minconf = minconf self.quiet = quiet self.data = {k:[g.proto.coin_amt('0')] * 4 for k in ('TOTAL','Non-MMGen','Non-wallet')} - self.create_data() + await self.create_data() - def create_data(self): + async def create_data(self): # 0: unconfirmed, 1: below minconf, 2: confirmed, 3: spendable (privkey in wallet) lbl_id = ('account','label')['label_api' in g.rpc.caps] - for d in g.rpc.listunspent(0): + for d in await g.rpc.call('listunspent',0): lbl = get_tw_label(d[lbl_id]) if lbl: if lbl.mmid.type == 'mmgen': diff --git a/mmgen/tx.py b/mmgen/tx.py index 3e881a44..51ad734c 100755 --- a/mmgen/tx.py +++ b/mmgen/tx.py @@ -82,8 +82,7 @@ def mmaddr2coinaddr(mmaddr,ad_w,ad_f): return CoinAddr(coin_addr) def segwit_is_active(exit_on_error=False): - rpc_init() - d = g.rpc.getblockchaininfo() + d = g.rpc.cached['blockchaininfo'] if d['chain'] == 'regtest': return True if ( 'bip9_softforks' in d @@ -281,6 +280,7 @@ class MMGenTX(MMGenObject): sig_ext = 'sigtx' txid_ext = 'txid' desc = 'transaction' + hexdata_type = 'hex' fee_fail_fs = 'Network fee estimation for {c} confirmations failed ({t})' no_chg_msg = 'Warning: Change address will be deleted as transaction produces no change' rel_fee_desc = 'satoshis per byte' @@ -308,7 +308,11 @@ inputs must be supplied to '{pnl}-txsign' in a file with the '--keys-from-file' option. Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_name.lower()) - def __init__(self,filename=None,metadata_only=False,caller=None,quiet_open=False,offline=False): + def __init__(self,filename=None,metadata_only=False,caller=None,quiet_open=False,data=None,tw=None): + if data: + assert type(data) is dict, type(data) + self.__dict__ = data + return self.inputs = MMGenTxInputList() self.outputs = MMGenTxOutputList() self.send_amt = g.proto.coin_amt('0') # total amt minus change @@ -327,6 +331,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam self.dcoin = None self.caller = caller self.locktime = None + self.tw = tw if filename: self.parse_tx_file(filename,metadata_only=metadata_only,quiet_open=quiet_open) @@ -400,12 +405,12 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam def update_txid(self): self.txid = MMGenTxID(make_chksum_6(bytes.fromhex(self.hex)).upper()) - def create_raw(self): + async def create_raw(self): i = [{'txid':e.txid,'vout':e.vout} for e in self.inputs] if self.inputs[0].sequence: i[0]['sequence'] = self.inputs[0].sequence o = {e.addr:e.amt for e in self.outputs} - self.hex = HexStr(g.rpc.createrawtransaction(i,o)) + self.hex = HexStr(await g.rpc.call('createrawtransaction',i,o)) self.update_txid() def print_contract_addr(self): pass @@ -436,9 +441,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam def has_segwit_inputs(self): return any(i.mmid and i.mmid.mmtype in ('S','B') for i in self.inputs) - def compare_size_and_estimated_size(self): + def compare_size_and_estimated_size(self,tx_decoded): est_vsize = self.estimate_size() - d = g.rpc.decoderawtransaction(self.hex) + d = tx_decoded vsize = d['vsize'] if 'vsize' in d else d['size'] vmsg('\nVsize: {} (true) {} (estimated)'.format(vsize,est_vsize)) m1 = 'Estimated transaction vsize is {:1.2f} times the true vsize\n' @@ -522,8 +527,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam return int(ret * float(opt.vsize_adj)) if hasattr(opt,'vsize_adj') and opt.vsize_adj else ret # coin-specific fee routines - def get_relay_fee(self): - kb_fee = g.proto.coin_amt(g.rpc.getnetworkinfo()['relayfee']) + @property + def relay_fee(self): + kb_fee = g.proto.coin_amt(g.rpc.cached['networkinfo']['relayfee']) ret = kb_fee * self.estimate_size() // 1024 vmsg('Relay fee: {} {c}/kB, for transaction: {} {c}'.format(kb_fee,ret,c=g.coin)) return ret @@ -533,9 +539,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam unit = getattr(g.proto.coin_amt,to_unit or 'min_coin_unit') return int(abs_fee // unit // self.estimate_size()) - def get_rel_fee_from_network(self): + async def get_rel_fee_from_network(self): try: - ret = g.rpc.estimatesmartfee(opt.tx_confs,opt.fee_estimate_mode.upper()) + ret = await g.rpc.call('estimatesmartfee',opt.tx_confs,opt.fee_estimate_mode.upper()) fee_per_kb = ret['feerate'] if 'feerate' in ret else -2 fe_type = 'estimatesmartfee' except: @@ -577,9 +583,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam m = '{} {c}: {} fee too large (maximum fee: {} {c})' msg(m.format(abs_fee,desc,g.proto.max_tx_fee,c=g.coin)) return False - elif abs_fee < self.get_relay_fee(): + elif abs_fee < self.relay_fee: m = '{} {c}: {} fee too small (below relay fee of {} {c})' - msg(m.format(str(abs_fee),desc,str(self.get_relay_fee()),c=g.coin)) + msg(m.format(str(abs_fee),desc,str(self.relay_fee),c=g.coin)) return False else: return abs_fee @@ -626,14 +632,14 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam tx_fee = my_raw_input(self.usr_fee_prompt) desc = 'User-selected' - def get_fee_from_user(self,have_estimate_fail=[]): + async def get_fee_from_user(self,have_estimate_fail=[]): if opt.tx_fee: desc = 'User-selected' start_fee = opt.tx_fee else: desc = 'Network-estimated (mode: {})'.format(opt.fee_estimate_mode.upper()) - fee_per_kb,fe_type = self.get_rel_fee_from_network() + fee_per_kb,fe_type = await self.get_rel_fee_from_network() if fee_per_kb < 0: if not have_estimate_fail: @@ -677,11 +683,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam assert isinstance(val,int),'locktime value not an integer' self.hex = self.hex[:-8] + bytes.fromhex('{:08x}'.format(val))[::-1].hex() - def get_blockcount(self): - return int(g.rpc.getblockcount()) - def add_blockcount(self): - self.blockcount = self.get_blockcount() + self.blockcount = g.rpc.blockcount def format(self): self.inputs.check_coin_mismatch() @@ -718,75 +721,6 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam def get_non_mmaddrs(self,desc): return {i.addr for i in getattr(self,desc) if not i.mmid} - def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception - - if self.marked_signed(): - msg('Transaction is already signed!') - return False - - if not self.check_correct_chain(on_fail='return'): - return False - - if (self.has_segwit_inputs() or self.has_segwit_outputs()) and not g.proto.cap('segwit'): - ymsg("TX has Segwit inputs or outputs, but {} doesn't support Segwit!".format(g.coin)) - return False - - self.check_pubkey_scripts() - - qmsg('Passing {} key{} to {}'.format(len(keys),suf(keys),g.proto.daemon_name)) - - if self.has_segwit_inputs(): - from .addr import KeyGenerator,AddrGenerator - kg = KeyGenerator('std') - ag = AddrGenerator('segwit') - keydict = MMGenDict([(d.addr,d.sec) for d in keys]) - - sig_data = [] - for d in self.inputs: - e = {k:getattr(d,k) for k in ('txid','vout','scriptPubKey','amt')} - e['amount'] = e['amt'] - del e['amt'] - if d.mmid and d.mmid.mmtype == 'S': - e['redeemScript'] = ag.to_segwit_redeem_script(kg.to_pubhex(keydict[d.addr])) - sig_data.append(e) - - msg_r('Signing transaction{}...'.format(tx_num_str)) - wifs = [d.sec.wif for d in keys] - - try: - ret = g.rpc.signrawtransactionwithkey(self.hex,wifs,sig_data,g.proto.sighash_type) \ - if 'sign_with_key' in g.rpc.caps else \ - g.rpc.signrawtransaction(self.hex,sig_data,wifs,g.proto.sighash_type) - except Exception as e: - msg(yellow('This is not the BCH chain.\nRe-run the script without the --coin=bch option.' - if 'Invalid sighash param' in e.args[0] else e.args[0])) - return False - - if not ret['complete']: - msg('failed\n{} returned the following errors:'.format(g.proto.daemon_name.capitalize())) - msg(repr(ret['errors'])) - return False - - try: - self.hex = HexStr(ret['hex']) - self.compare_size_and_estimated_size() - dt = DeserializedTX(self.hex) - self.check_hex_tx_matches_mmgen_tx(dt) - self.coin_txid = CoinTxID(dt['txid'],on_fail='raise') - self.check_sigs(dt) - if not self.coin_txid == g.rpc.decoderawtransaction(ret['hex'])['txid']: - raise BadMMGenTxID('txid mismatch (after signing)') - msg('OK') - return True - except Exception as e: - try: m = '{}'.format(e.args[0]) - except: m = repr(e.args[0]) - msg('\n'+yellow(m)) - if g.traceback: - import traceback - ymsg('\n'+''.join(traceback.format_exception(*sys.exc_info()))) - return False - def mark_raw(self): self.desc = 'transaction' self.ext = self.raw_ext @@ -874,38 +808,45 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam def has_segwit_outputs(self): return any(o.mmid and o.mmid.mmtype in ('S','B') for o in self.outputs) - def get_status(self,status=False): + async def get_status(self,status=False): class r(object): pass - def is_in_wallet(): - ret = g.rpc.gettransaction(self.coin_txid,on_fail='silent') + async def is_in_wallet(): + try: ret = await g.rpc.call('gettransaction',self.coin_txid) + except: return False if 'confirmations' in ret and ret['confirmations'] > 0: r.confs = ret['confirmations'] return True else: return False - def is_in_utxos(): - return 'txid' in g.rpc.getrawtransaction(self.coin_txid,True,on_fail='silent') + async def is_in_utxos(): + try: return 'txid' in await g.rpc.call('getrawtransaction',self.coin_txid,True) + except: return False - def is_in_mempool(): - return 'height' in g.rpc.getmempoolentry(self.coin_txid,on_fail='silent') + async def is_in_mempool(): + try: return 'height' in await g.rpc.call('getmempoolentry',self.coin_txid) + except: return False - def is_replaced(): - if is_in_mempool(): return False - ret = g.rpc.gettransaction(self.coin_txid,on_fail='silent') - - if not 'bip125-replaceable' in ret or not 'confirmations' in ret or ret['confirmations'] > 0: + async def is_replaced(): + if await is_in_mempool(): return False + try: + ret = await g.rpc.call('gettransaction',self.coin_txid) + except: + return False + else: + if 'bip125-replaceable' in ret and 'confirmations' in ret and ret['confirmations'] <= 0: + r.replacing_confs = -ret['confirmations'] + r.replacing_txs = ret['walletconflicts'] + return True + else: + return False - r.replacing_confs = -ret['confirmations'] - r.replacing_txs = ret['walletconflicts'] - return True - - if is_in_mempool(): + if await is_in_mempool(): if status: - d = g.rpc.gettransaction(self.coin_txid,on_fail='silent') + d = await g.rpc.call('gettransaction',self.coin_txid) brs = 'bip125-replaceable' rep = '{}replaceable'.format(('NOT ','')[brs in d and d[brs]=='yes']) t = d['timereceived'] @@ -917,22 +858,23 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam msg('TX status: in mempool, {}\n{}'.format(rep,b)) else: msg('Warning: transaction is in mempool!') - elif is_in_wallet(): + elif await is_in_wallet(): die(0,'Transaction has {} confirmation{}'.format(r.confs,suf(r.confs))) - elif is_in_utxos(): + elif await is_in_utxos(): die(2,red('ERROR: transaction is in the blockchain (but not in the tracking wallet)!')) - elif is_replaced(): - m1 = 'Transaction has been replaced' - m2 = 'Replacement transaction is in mempool' - rc = r.replacing_confs - if rc: - m2 = 'Replacement transaction has {} confirmation{}'.format(rc,suf(rc)) - msg('{}\n{}'.format(m1,m2)) + elif await is_replaced(): + msg('Transaction has been replaced\nReplacement transaction ' + ( + f'has {r.replacing_confs} confirmation{suf(r.replacing_confs)}' + if r.replacing_confs else + 'is in mempool' )) if not opt.quiet: msg('Replacing transactions:') - d = ((t,g.rpc.getmempoolentry(t,on_fail='silent')) for t in r.replacing_txs) - for txid,mp_entry in d: - msg(' {}{}'.format(txid,' in mempool' if ('height' in mp_entry) else '')) + d = [] + for txid in r.replacing_txs: + try: d.append(await g.rpc.call('getmempoolentry',txid)) + except: d.append({}) + for txid,mp_entry in zip(r.replacing_txs,d): + msg(f' {txid}' + ('',' in mempool')['height' in mp_entry]) die(0,'') def confirm_send(self): @@ -942,8 +884,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam confirm_or_raise(m1,m2,m3) msg('Sending transaction') - def send(self,prompt_user=True,exit_on_fail=False): - + async def send(self,prompt_user=True,exit_on_fail=False): if not self.marked_signed(): die(1,'Transaction is not signed!') @@ -961,15 +902,21 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam die(2,'Transaction fee ({}) greater than {} max_tx_fee ({} {})!'.format( self.get_fee_from_tx(),g.proto.name.capitalize(),g.proto.max_tx_fee,g.coin)) - self.get_status() + await self.get_status() - if prompt_user: self.confirm_send() + if prompt_user: + self.confirm_send() - ret = None if g.bogus_send else g.rpc.sendrawtransaction(self.hex,on_fail='return') + if g.bogus_send: + ret = None + else: + try: + ret = await g.rpc.call('sendrawtransaction',self.hex) + except Exception as e: + ret = False - from .rpc import rpc_error,rpc_errmsg - if rpc_error(ret): - errmsg = rpc_errmsg(ret) + if ret == False: + errmsg = e if 'Signature must use SIGHASH_FORKID' in errmsg: m = 'The Aug. 1 2017 UAHF has activated on this chain.' m += "\nRe-run the script with the --coin=bch option." @@ -1061,7 +1008,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam msg('') # def is_replaceable_from_rpc(self): -# dec_tx = g.rpc.decoderawtransaction(self.hex) +# dec_tx = await g.rpc.call('decoderawtransaction',self.hex) # return None < dec_tx['vin'][0]['sequence'] <= g.max_int - 2 def is_replaceable(self): @@ -1138,8 +1085,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam blockcount = None if g.proto.base_coin != 'ETH': try: - rpc_init() - blockcount = self.get_blockcount() + blockcount = g.rpc.blockcount except: pass @@ -1187,6 +1133,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam def check_txfile_hex_data(self): self.hex = HexStr(self.hex,on_fail='raise') + def parse_txfile_hex_data(self): + pass + def parse_tx_file(self,infile,metadata_only=False,quiet_open=False): def eval_io_data(raw_data,desc): @@ -1271,6 +1220,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam desc = 'transaction file hex data' self.check_txfile_hex_data() + desc = f'transaction file {self.hexdata_type} data' + self.parse_txfile_hex_data() # the following ops will all fail if g.coin doesn't match self.coin desc = 'coin type in metadata' assert self.coin == g.coin,self.coin @@ -1286,7 +1237,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam self.chain = 'mainnet' if self.dcoin: - self.resolve_g_token_from_tx_file() + self.resolve_g_token_from_txfile() + g.dcoin = self.dcoin def process_cmd_arg(self,arg,ad_f,ad_w): @@ -1320,8 +1272,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam if not self.outputs: die(2,'At least one output must be specified on the command line') - def get_outputs_from_cmdline(self,cmd_args): - from .addr import AddrList,AddrData + async def get_outputs_from_cmdline(self,cmd_args): + from .addr import AddrList,AddrData,TwAddrData addrfiles = [a for a in cmd_args if get_extension(a) == AddrList.ext] cmd_args = set(cmd_args) - set(addrfiles) @@ -1330,7 +1282,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam check_infile(a) ad_f.add(AddrList(a)) - ad_w = AddrData(source='tw',wallet=self.twuo.wallet) + ad_w = await TwAddrData(wallet=self.tw) self.process_cmd_args(cmd_args,ad_f,ad_w) @@ -1349,7 +1301,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam msg('Unspent output number must be <= {}'.format(len(unspent))) # we don't know fee yet, so perform preliminary check with fee == 0 - def precheck_sufficient_funds(self,inputs_sum,sel_unspent): + async def precheck_sufficient_funds(self,inputs_sum,sel_unspent): if self.twuo.total < self.send_amt: msg(self.msg_wallet_low_coin.format(self.send_amt-inputs_sum,g.dcoin)) return False @@ -1358,7 +1310,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam return False return True - def get_change_amt(self): + async def get_change_amt(self): return self.sum_inputs() - self.send_amt - self.fee def warn_insufficient_chg(self,change_amt): @@ -1394,11 +1346,11 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam return set(sel_nums) # silently discard duplicates - def get_cmdline_input_addrs(self): + async def get_cmdline_input_addrs(self): # Bitcoin full node, call doesn't go to the network, so just call listunspent with addrs=[] return [] - def get_inputs_from_user(self): + async def get_inputs_from_user(self): while True: us_f = self.select_unspent_cmdline if opt.inputs else self.select_unspent @@ -1408,7 +1360,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam sel_unspent = self.twuo.MMGenTwOutputList([self.twuo.unspent[i-1] for i in sel_nums]) inputs_sum = sum(s.amt for s in sel_unspent) - if not self.precheck_sufficient_funds(inputs_sum,sel_unspent): + if not await self.precheck_sufficient_funds(inputs_sum,sel_unspent): continue non_mmaddrs = [i for i in sel_unspent if i.twmmid.type == 'non-mmgen'] @@ -1420,9 +1372,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam self.copy_inputs_from_tw(sel_unspent) # makes self.inputs - self.fee = self.get_fee_from_user() + self.fee = await self.get_fee_from_user() - change_amt = self.get_change_amt() + change_amt = await self.get_change_amt() if change_amt >= 0: p = self.final_inputs_ok_msg(change_amt) @@ -1439,27 +1391,34 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam if not self.send_amt: self.send_amt = change_amt - def create(self,cmd_args,locktime,do_info=False): + async def set_token_params(self): + pass + + async def create(self,cmd_args,locktime,do_info=False): assert isinstance(locktime,int),'locktime must be of type int' - if opt.comment_file: self.add_comment(opt.comment_file) - - twuo_addrs = self.get_cmdline_input_addrs() - from .tw import TwUnspentOutputs - self.twuo = TwUnspentOutputs(minconf=opt.minconf,addrs=twuo_addrs) + + if opt.comment_file: + self.add_comment(opt.comment_file) + + twuo_addrs = await self.get_cmdline_input_addrs() + + self.twuo = await TwUnspentOutputs(minconf=opt.minconf,addrs=twuo_addrs) + await self.twuo.get_unspent_data() if not do_info: - self.get_outputs_from_cmdline(cmd_args) + await self.get_outputs_from_cmdline(cmd_args) do_license_msg() if not opt.inputs: - self.twuo.view_and_sort(self) + await self.twuo.view_and_sort(self) self.twuo.display_total() if do_info: + del self.twuo.wallet sys.exit(0) self.send_amt = self.sum_outputs() @@ -1468,7 +1427,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam ('Unknown','{} {}'.format(self.send_amt.hl(),g.dcoin))[bool(self.send_amt)] )) - change_amt = self.get_inputs_from_user() + change_amt = await self.get_inputs_from_user() self.update_change_output(change_amt) self.update_send_amt(change_amt) @@ -1482,7 +1441,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam if not opt.yes: self.add_comment() # edits an existing comment - self.create_raw() # creates self.hex, self.txid + + await self.create_raw() # creates self.hex, self.txid if g.proto.base_proto == 'Bitcoin' and locktime: msg('Setting nlocktime to {}!'.format(strfmt_locktime(locktime))) @@ -1501,9 +1461,85 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam if not opt.yes: self.view_with_prompt('View decoded transaction?') - del self.twuo + del self.twuo.wallet -class MMGenBumpTX(MMGenTX): +class MMGenTxForSigning(MMGenTX): + + hexdata_type = 'json' + + def __new__(cls,*args,**kwargs): + return MMGenObject.__new__(altcoin_subclass(cls,'tx','MMGenTxForSigning')) + + async def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception + + if self.marked_signed(): + msg('Transaction is already signed!') + return False + + if not self.check_correct_chain(on_fail='return'): + return False + + if (self.has_segwit_inputs() or self.has_segwit_outputs()) and not g.proto.cap('segwit'): + ymsg("TX has Segwit inputs or outputs, but {} doesn't support Segwit!".format(g.coin)) + return False + + self.check_pubkey_scripts() + + qmsg('Passing {} key{} to {}'.format(len(keys),suf(keys),g.proto.daemon_name)) + + if self.has_segwit_inputs(): + from .addr import KeyGenerator,AddrGenerator + kg = KeyGenerator('std') + ag = AddrGenerator('segwit') + keydict = MMGenDict([(d.addr,d.sec) for d in keys]) + + sig_data = [] + for d in self.inputs: + e = {k:getattr(d,k) for k in ('txid','vout','scriptPubKey','amt')} + e['amount'] = e['amt'] + del e['amt'] + if d.mmid and d.mmid.mmtype == 'S': + e['redeemScript'] = ag.to_segwit_redeem_script(kg.to_pubhex(keydict[d.addr])) + sig_data.append(e) + + msg_r('Signing transaction{}...'.format(tx_num_str)) + wifs = [d.sec.wif for d in keys] + + try: + args = ( + ('signrawtransaction',self.hex,sig_data,wifs,g.proto.sighash_type), + ('signrawtransactionwithkey',self.hex,wifs,sig_data,g.proto.sighash_type) + )['sign_with_key' in g.rpc.caps] + ret = await g.rpc.call(*args) + except Exception as e: + msg(yellow(( + e.args[0], + 'This is not the BCH chain.\nRe-run the script without the --coin=bch option.' + )['Invalid sighash param' in e.args[0]])) + return False + + try: + self.hex = HexStr(ret['hex']) + tx_decoded = await g.rpc.call('decoderawtransaction',ret['hex']) + self.compare_size_and_estimated_size(tx_decoded) + dt = DeserializedTX(self.hex) + self.check_hex_tx_matches_mmgen_tx(dt) + self.coin_txid = CoinTxID(dt['txid'],on_fail='raise') + self.check_sigs(dt) + if not self.coin_txid == tx_decoded['txid']: + raise BadMMGenTxID('txid mismatch (after signing)') + msg('OK') + return True + except Exception as e: + try: m = '{}'.format(e.args[0]) + except: m = repr(e.args[0]) + msg('\n'+yellow(m)) + if g.traceback: + import traceback + ymsg('\n'+''.join(traceback.format_exception(*sys.exc_info()))) + return False + +class MMGenBumpTX(MMGenTxForSigning): def __new__(cls,*args,**kwargs): return MMGenTX.__new__(altcoin_subclass(cls,'tx','MMGenBumpTX'),*args,**kwargs) @@ -1511,9 +1547,8 @@ class MMGenBumpTX(MMGenTX): min_fee = None bump_output_idx = None - def __init__(self,filename,send=False): - - super().__init__(filename) + def __init__(self,filename,send=False,tw=None): + super().__init__(filename,tw=tw) if not self.is_replaceable(): die(1,"Transaction '{}' is not replaceable".format(self.txid)) @@ -1576,8 +1611,9 @@ class MMGenBumpTX(MMGenTX): self.bump_output_idx = idx return idx - def set_min_fee(self): - self.min_fee = self.sum_inputs() - self.sum_outputs() + self.get_relay_fee() + @property + def min_fee(self): + return self.sum_inputs() - self.sum_outputs() + self.relay_fee def update_fee(self,op_idx,fee): amt = self.sum_inputs() - self.sum_outputs(exclude=op_idx) - fee @@ -1598,10 +1634,10 @@ class MMGenBumpTX(MMGenTX): # NOT MAINTAINED class MMGenSplitTX(MMGenTX): - def get_outputs_from_cmdline(self,mmid): # TODO: check that addr is empty + async def get_outputs_from_cmdline(self,mmid): # TODO: check that addr is empty - from .addr import AddrData - ad_w = AddrData(source='tw') + from .addr import TwAddrData + ad_w = await TwAddrData() if is_mmgen_id(mmid): coin_addr = mmaddr2coinaddr(mmid,ad_w,None) if is_mmgen_id(mmid) else CoinAddr(mmid) @@ -1620,17 +1656,12 @@ class MMGenSplitTX(MMGenTX): g.rpc_host = opt.rpc_host2 if opt.tx_fees: opt.tx_fee = opt.tx_fees.split(',')[1] - try: - rpc_init(reinit=True) - except: - ymsg('Connect to {} daemon failed. Network fee estimation unavailable'.format(g.coin)) - return self.get_usr_fee_interactive(opt.tx_fee,'User-selected') return super().get_fee_from_user() - def create_split(self,mmid): + async def create_split(self,mmid): self.outputs = self.MMGenTxOutputList() - self.get_outputs_from_cmdline(mmid) + await self.get_outputs_from_cmdline(mmid) while True: change_amt = self.sum_inputs() - self.get_split_fee_from_user() @@ -1647,7 +1678,8 @@ class MMGenSplitTX(MMGenTX): if not opt.yes: self.add_comment() # edits an existing comment - self.create_raw() # creates self.hex, self.txid + + await self.create_raw() # creates self.hex, self.txid self.add_timestamp() self.add_blockcount() # TODO diff --git a/mmgen/txsign.py b/mmgen/txsign.py index 12ac7a37..6e93986b 100755 --- a/mmgen/txsign.py +++ b/mmgen/txsign.py @@ -139,7 +139,7 @@ def get_keylist(opt): return kal return None -def txsign(tx,seed_files,kl,kal,tx_num_str=''): +async def txsign(tx,seed_files,kl,kal,tx_num_str=''): keys = MMGenList() # list of AddrListEntry objects non_mm_addrs = tx.get_non_mmaddrs('inputs') @@ -169,4 +169,4 @@ def txsign(tx,seed_files,kl,kal,tx_num_str=''): if extra_sids: msg('Unused Seed ID{}: {}'.format(suf(extra_sids),' '.join(extra_sids))) - return tx.sign(tx_num_str,keys) # returns True or False + return await tx.sign(tx_num_str,keys) # returns True or False diff --git a/mmgen/util.py b/mmgen/util.py index b2c4dfa4..d6efcab4 100755 --- a/mmgen/util.py +++ b/mmgen/util.py @@ -818,35 +818,32 @@ def do_license_msg(immed=False): msg_r('\r') msg('') -def get_daemon_cfg_options(cfg_keys): - +# TODO: these belong in protocol.py +def get_coin_daemon_cfg_fn(): # Use dirname() to remove 'bob' or 'alice' component cfg_dir = os.path.dirname(g.data_dir) if g.regtest else g.proto.daemon_data_dir - cfg_file = os.path.join(cfg_dir,g.proto.name+'.conf' ) + return os.path.join(cfg_dir,g.proto.name+'.conf' ) +def get_coin_daemon_cfg_options(req_keys): + + fn = get_coin_daemon_cfg_fn() try: - lines = get_lines_from_file(cfg_file,'',silent=not opt.verbose) - kv_pairs = [l.split('=') for l in lines] - cfg = {k:v for k,v in kv_pairs if k in cfg_keys} + lines = get_lines_from_file(fn,'',silent=not opt.verbose) except: - vmsg("Warning: '{}' does not exist or is unreadable".format(cfg_file)) - cfg = {} + vmsg(f'Warning: {fn!r} does not exist or is unreadable') + return dict((k,None) for k in req_keys) - for k in set(cfg_keys) - set(cfg.keys()): cfg[k] = '' + def gen(): + for key in req_keys: + val = None + for l in lines: + if l.startswith(key): + res = l.split('=',1) + if len(res) == 2 and not ' ' in res[1].strip(): + val = res[1].strip() + yield (key,val) - return cfg - -def get_coin_daemon_auth_cookie(): - f = os.path.join(g.proto.daemon_data_dir,g.proto.daemon_data_subdir,'.cookie') - return get_lines_from_file(f,'')[0] if file_is_readable(f) else '' - -def rpc_init(reinit=False): - if not 'rpc' in g.proto.mmcaps: - die(1,'Coin daemon operations not supported for coin {}!'.format(g.coin)) - if g.rpc != None and not reinit: return g.rpc - from .rpc import init_daemon - g.rpc = init_daemon(g.proto.daemon_family) - return g.rpc + return dict(gen()) def format_par(s,indent=0,width=80,as_list=False): words,lines = s.split(),[] @@ -886,3 +883,28 @@ def get_network_id(coin=None,testnet=None): if coin == None: assert testnet == None if coin != None: assert testnet != None return (coin or g.coin).lower() + ('','_tn')[testnet or g.testnet] + +def run_session(callback,do_rpc_init=True,backend=None): + backend = backend or opt.rpc_backend + import asyncio + async def do(): + if backend == 'aiohttp': + import aiohttp + async with aiohttp.ClientSession( + headers = { 'Content-Type': 'application/json' }, + connector = aiohttp.TCPConnector(limit_per_host=g.aiohttp_rpc_queue_len), + ) as g.session: + if do_rpc_init: + from .rpc import rpc_init + await rpc_init(backend=backend) + ret = await callback + g.session = None + return ret + else: + if do_rpc_init: + from .rpc import rpc_init + await rpc_init(backend=backend) + return await callback + + # return asyncio.run(do()) # Python 3.7+ + return asyncio.get_event_loop().run_until_complete(do()) diff --git a/setup.py b/setup.py index 75cd42bc..beb5c2f8 100755 --- a/setup.py +++ b/setup.py @@ -114,6 +114,7 @@ setup( 'mmgen.filename', 'mmgen.globalvars', 'mmgen.keccak', + 'mmgen.led', 'mmgen.license', 'mmgen.mn_electrum', 'mmgen.mn_entry', diff --git a/test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.sigtx b/test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.sigtx new file mode 100644 index 00000000..989d1695 --- /dev/null +++ b/test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.sigtx @@ -0,0 +1,7 @@ +e9feb9 +ETH KOVAN B472BD 23.45495 20180725_111111 7654321 +f86d808509502f900082520894b7d06382a998817a16ba487a99636bf8cae29d9d89014580b8c15e8c60008077a057b7b73b9102fda68922eba0fbb70171425f1f27bad944838d3ded819eb03a63a00acb20276cbb30209923f959c63595b1b35663cbb30d186c6a5a2dbccb3e07fa +[{'confs': 0, 'label': '', 'mmid': '98831F3A:E:1', 'amt': '123.456', 'addr': '97ccc3a117b3696340c42561361054b1c9c793d5'}] +[{'mmid': '98831F3A:E:2', 'amt': '23.45495', 'addr': '2a6db46c87407e6d28fcb97d3bd0f5cf4aafca46'}] +qRHzrPVpZFYxnQvk3atLzUtp41bZupJ2UQNnKe3ZnmqFsEngS6vaCCvesKKy9khzVq6y2RqarVBcZLnjtXxMpbAcdEtyBWiBYmZdoU8SN4uAbroHT1c7gEbmUNVKKdqHD86ZRRqDNpdh1ztmLiMAy3ibM83puwJHNpGGHgUGjZ1RSEgyVKCs2rZ9wXN8rBMibDDPYo1LgtAst2FkB36Mgf4Vf7ekoRAdiRNGd5YZ3RXAVsSdnZcyn4rdeQDMDkCq7JJDoB25eNEuXQutZFUcf2fEfxkMbW1sXJDNFQq +0f277d20bf3793f94521a809943a659478bdfa6836a399f0568a93aeb4ce5184 diff --git a/test/ref/ethereum/tracking-wallet-v2.json b/test/ref/ethereum/tracking-wallet-v2.json index 6a7289fb..77b02bd3 100644 --- a/test/ref/ethereum/tracking-wallet-v2.json +++ b/test/ref/ethereum/tracking-wallet-v2.json @@ -1 +1 @@ -{"coin":"ETH","accounts":{"e704b6cfd9f0edb2e6cfbd0c913438d37ede7b35":{"mmid":"98831F3A:E:1","comment":""},"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef":{"mmid":"eth:deadbeefdeadbeefdeadbeefdeadbeefdeadbeef","comment":""}},"tokens":{"d5f051401ca478b34c80d0b5a119e437dc6d9df5":{"6e0fbe42e1343309b3ccb9068dbad6132f86c96f":{"mmid":"98831F3A:E:12","comment":""},"b72268fa55a57fe838a745b27c90f0edb415af61":{"mmid":"98831F3A:E:13","comment":""}},"3dd0864668c36d27b53a98137764c99f9fd5b7b2":{"ca2b705adf43151ce61e01c653424e9790e9eb84":{"mmid":"98831F3A:E:22","comment":""},"7227a8fd728b74208255c6a8a09cb9fb66b1230c":{"mmid":"98831F3A:E:23","comment":""}}}} +{"coin":"ETH","accounts":{"e704b6cfd9f0edb2e6cfbd0c913438d37ede7b35":{"mmid":"98831F3A:E:1","comment":""},"b7d06382a998817a16ba487a99636bf8cae29d9d":{"mmid":"98831F3A:E:2","comment":""},"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef":{"mmid":"eth:deadbeefdeadbeefdeadbeefdeadbeefdeadbeef","comment":""}},"tokens":{"d5f051401ca478b34c80d0b5a119e437dc6d9df5":{"6e0fbe42e1343309b3ccb9068dbad6132f86c96f":{"mmid":"98831F3A:E:12","comment":""},"b72268fa55a57fe838a745b27c90f0edb415af61":{"mmid":"98831F3A:E:13","comment":""}},"3dd0864668c36d27b53a98137764c99f9fd5b7b2":{"ca2b705adf43151ce61e01c653424e9790e9eb84":{"mmid":"98831F3A:E:22","comment":""},"7227a8fd728b74208255c6a8a09cb9fb66b1230c":{"mmid":"98831F3A:E:23","comment":""}}}} diff --git a/test/test.py b/test/test.py index 3d3b4cc3..6a9c5a91 100755 --- a/test/test.py +++ b/test/test.py @@ -492,6 +492,7 @@ class CmdGroupMgr(object): cmd_groups_extra = { 'autosign_btc': ('TestSuiteAutosignBTC',{'modname':'autosign'}), 'autosign_live': ('TestSuiteAutosignLive',{'modname':'autosign'}), + 'autosign_live_simulate': ('TestSuiteAutosignLiveSimulate',{'modname':'autosign'}), 'create_ref_tx': ('TestSuiteRefTX',{'modname':'misc','full_data':True}), } @@ -867,7 +868,10 @@ class TestSuiteRunner(object): if k in cfg: setattr(self.ts,k,cfg[k]) - self.process_retval(cmd,getattr(self.ts,cmd)(*arg_list)) # run the test + ret = getattr(self.ts,cmd)(*arg_list) # run the test + if type(ret).__name__ == 'coroutine': + ret = run_session(ret) + self.process_retval(cmd,ret) if opt.profile: omsg('\r\033[50C{:.4f}'.format(time.time() - start)) diff --git a/test/test_py_d/ts_autosign.py b/test/test_py_d/ts_autosign.py index 42ee5150..0d42b193 100755 --- a/test/test_py_d/ts_autosign.py +++ b/test/test_py_d/ts_autosign.py @@ -43,21 +43,26 @@ class TestSuiteAutosign(TestSuiteBase): def autosign_live(self): return self.autosign_btc(live=True) - def autosign_btc(self,live=False): + def autosign_live_simulate(self): + return self.autosign_btc(live=True,simulate=True) + + def autosign_btc(self,live=False,simulate=False): return self.autosign( coins=['btc'], daemon_coins=['btc'], txfiles=['btc'], txcount=3, - live=live) + live=live, + simulate=simulate ) - # tests everything except device detection, mount/unmount + # tests everything except mount/unmount def autosign( self, coins=['btc','bch','ltc','eth'], daemon_coins=['btc','bch','ltc'], txfiles=['btc','bch','ltc','eth','mm1','etc'], txcount=12, - live=False): + live=False, + simulate=False): if self.skip_for_win(): return 'skip' @@ -79,13 +84,16 @@ class TestSuiteAutosign(TestSuiteBase): wf = t.written_to_file('Autosign wallet') t.ok() - def copy_files(mountpoint,remove_signed_only=False,include_bad_tx=True): - fdata_in = (('btc',''), - ('bch',''), - ('ltc','litecoin'), - ('eth','ethereum'), - ('mm1','ethereum'), - ('etc','ethereum_classic')) + def copy_files( + mountpoint, + remove_signed_only=False, + include_bad_tx=True, + fdata_in = (('btc',''), + ('bch',''), + ('ltc','litecoin'), + ('eth','ethereum'), + ('mm1','ethereum'), + ('etc','ethereum_classic')) ): fdata = [e for e in fdata_in if e[0] in txfiles] from .ts_ref import TestSuiteRef tfns = [TestSuiteRef.sources['ref_tx_file'][c][1] for c,d in fdata] + \ @@ -124,8 +132,11 @@ class TestSuiteAutosign(TestSuiteBase): omsg_r(blue('\nRemove removable device and then hit ENTER ')) input() - if gen_wallet: make_wallet(opts) - else: do_mount() + if gen_wallet: + if not opt.skip_deps: + make_wallet(opts) + else: + do_mount() copy_files(mountpoint,include_bad_tx=not led_opts) @@ -150,7 +161,7 @@ class TestSuiteAutosign(TestSuiteBase): do_unmount() omsg(green(m1)) - t = self.spawn('mmgen-autosign',opts+led_opts+['wait'],extra_desc=desc) + t = self.spawn('mmgen-autosign',opts+led_opts+['--quiet','--no-summary','wait'],extra_desc=desc) if not opt.exact_output: omsg('') do_loop() do_mount() # race condition due to device insertion detection @@ -160,6 +171,8 @@ class TestSuiteAutosign(TestSuiteBase): imsg(purple('\nKilling wait loop!')) t.kill(2) # 2 = SIGINT t.req_exit_val = 1 + if simulate and led_opts: + t.expect("Stopping LED") return t def do_autosign(opts,mountpoint): @@ -179,15 +192,44 @@ class TestSuiteAutosign(TestSuiteBase): t.ok() copy_files(mountpoint,remove_signed_only=True) - t = self.spawn('mmgen-autosign',opts+['wait'],extra_desc='(sign)') + t = self.spawn('mmgen-autosign',opts+['--quiet','wait'],extra_desc='(sign)') t.expect('{} transactions signed'.format(txcount)) t.expect('2 transactions failed to sign') t.expect('Waiting') t.kill(2) t.req_exit_val = 1 imsg('') + t.ok() + + copy_files(mountpoint,include_bad_tx=True,fdata_in=(('btc',''),)) + t = self.spawn( + 'mmgen-autosign', + opts + ['--quiet','--stealth-led','wait'], + extra_desc='(sign - --quiet --stealth-led)' ) + t.expect('2 transactions failed to sign') + t.expect('Waiting') + t.kill(2) + t.req_exit_val = 1 + imsg('') + t.ok() + + copy_files(mountpoint,include_bad_tx=False,fdata_in=(('btc',''),)) + t = self.spawn( + 'mmgen-autosign', + opts + ['--quiet','--led'], + extra_desc='(sign - --quiet --led)' ) + t.read() + imsg('') + t.ok() + return t + # begin execution + + if simulate and not opt.exact_output: + rmsg('This command must be run with --exact-output enabled!') + return False + network_ids = [c+'_tn' for c in daemon_coins] + daemon_coins start_test_daemons(*network_ids) @@ -205,27 +247,27 @@ class TestSuiteAutosign(TestSuiteBase): ydie(1,"Directory '{}' does not exist! Exiting".format(mountpoint)) opts = ['--coins='+','.join(coins)] - led_files = { 'opi': ('/sys/class/leds/orangepi:red:status/brightness',), - 'rpi': ('/sys/class/leds/led0/brightness','/sys/class/leds/led0/trigger') } - for k in ('opi','rpi'): - if os.path.exists(led_files[k][0]): - led_support = k - break + from mmgen.led import LEDControl + + if simulate: + LEDControl.create_dummy_control_files() + + try: + cf = LEDControl(enabled=True,simulate=simulate) + except: + ret = "'no LED support detected'" else: - led_support = None - - if led_support: - for fn in (led_files[led_support]): - run(['sudo','chmod','0666',fn],check=True) + for fn in (cf.board.status,cf.board.trigger): + if fn: + run(['sudo','chmod','0666',fn],check=True) + os.environ['MMGEN_TEST_SUITE_AUTOSIGN_LIVE'] = '1' omsg(purple('Running autosign test with no LED')) do_autosign_live(opts,mountpoint) omsg(purple("Running autosign test with '--led'")) do_autosign_live(opts,mountpoint,led_opts=['--led'],gen_wallet=False) omsg(purple("Running autosign test with '--stealth-led'")) ret = do_autosign_live(opts,mountpoint,led_opts=['--stealth-led'],gen_wallet=False) - else: - ret = do_autosign_live(opts,mountpoint) else: mountpoint = self.tmpdir opts = ['--no-insert-check','--mountpoint='+mountpoint,'--coins='+','.join(coins)] @@ -247,3 +289,9 @@ class TestSuiteAutosignLive(TestSuiteAutosignBTC): cmd_group = ( ('autosign_live', 'transaction autosigning (BTC,ETH,ETC - test device insertion/removal + LED)'), ) + +class TestSuiteAutosignLiveSimulate(TestSuiteAutosignBTC): + 'live autosigning operations with device insertion/removal and LED check in simulated environment' + cmd_group = ( + ('autosign_live_simulate', 'transaction autosigning (BTC,ETH,ETC - test device insertion/removal + simulated LED)'), + ) diff --git a/test/test_py_d/ts_ethdev.py b/test/test_py_d/ts_ethdev.py index ab2da725..b2de0f9d 100755 --- a/test/test_py_d/ts_ethdev.py +++ b/test/test_py_d/ts_ethdev.py @@ -211,7 +211,8 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): ('token_addrgen', 'generating token addresses'), ('token_addrimport_badaddr1','importing token addresses (no token address)'), ('token_addrimport_badaddr2','importing token addresses (bad token address)'), - ('token_addrimport', 'importing token addresses'), + ('token_addrimport', 'importing token addresses'), + ('token_addrimport_batch','importing token addresses (dummy batch mode)'), ('bal7', 'the {} balance'.format(g.coin)), ('token_bal1', 'the {} balance and token balance'.format(g.coin)), @@ -351,7 +352,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): def addrimport(self,ext='21-23]{}.addrs',expect='9/9',add_args=[],bad_input=False): ext = ext.format('-α' if g.debug_utf8 else '') fn = self.get_file_with_ext(ext,no_dot=True,delete=False) - t = self.spawn('mmgen-addrimport', self.eth_args[1:] + add_args + [fn]) + t = self.spawn('mmgen-addrimport', self.eth_args[1:-1] + add_args + [fn]) if bad_input: t.read() return t @@ -507,9 +508,11 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): t = self.spawn('mmgen-tool', self.eth_args + ['--token=mm1','twview','wide=1']) for b in token_bals[n]: addr,_amt1,_amt2,adj = b if len(b) == 4 else b + (False,) - if adj and g.coin == 'ETC': _amt2 = str(Decimal(_amt2) + Decimal(adj[1]) * self.bal_corr) + if adj and g.coin == 'ETC': + _amt2 = str(Decimal(_amt2) + Decimal(adj[1]) * self.bal_corr) pat = r'{}\s+{}\s+{}\s'.format(addr,_amt1.replace('.',r'\.'),_amt2.replace('.',r'\.')) t.expect(pat,regex=True) + t.expect('Total MM1:') t.read() return t @@ -577,12 +580,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): token_data = { 'name':'MMGen Token 2', 'symbol':'MM2', 'supply':10**18, 'decimals':10 } return self.token_compile(token_data) - def _rpc_init(self): - g.proto.rpc_port = self.rpc_port - rpc_init() - - def token_deploy(self,num,key,gas,mmgen_cmd='txdo',tx_fee='8G'): - self._rpc_init() + async def token_deploy(self,num,key,gas,mmgen_cmd='txdo',tx_fee='8G'): keyfile = joinpath(self.tmpdir,parity_key_fn) fn = joinpath(self.tmpdir,'mm'+str(num),key+'.bin') os.environ['MMGEN_BOGUS_SEND'] = '' @@ -609,62 +607,63 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): bogus_send=False) addr = t.expect_getend('Contract address: ') from mmgen.altcoins.eth.tx import EthereumMMGenTX as etx - assert etx.get_exec_status(txid,True) != 0,( + assert (await etx.get_exec_status(txid,True)) != 0,( "Contract '{}:{}' failed to execute. Aborting".format(num,key)) if key == 'Token': self.write_to_tmpfile('token_addr{}'.format(num),addr+'\n') imsg('\nToken MM{} deployed!'.format(num)) return t - def token_deploy1a(self): return self.token_deploy(num=1,key='SafeMath',gas=200000) - def token_deploy1b(self): return self.token_deploy(num=1,key='Owned',gas=250000) - def token_deploy1c(self): return self.token_deploy(num=1,key='Token',gas=1100000,tx_fee='7G') + async def token_deploy1a(self): return await self.token_deploy(num=1,key='SafeMath',gas=200000) + async def token_deploy1b(self): return await self.token_deploy(num=1,key='Owned',gas=250000) + async def token_deploy1c(self): return await self.token_deploy(num=1,key='Token',gas=1100000,tx_fee='7G') def tx_status2(self): return self.tx_status(ext=g.coin+'[0,7000]{}.sigtx',expect_str='successfully executed') def bal6(self): return self.bal5() - def token_deploy2a(self): return self.token_deploy(num=2,key='SafeMath',gas=200000) - def token_deploy2b(self): return self.token_deploy(num=2,key='Owned',gas=250000) - def token_deploy2c(self): return self.token_deploy(num=2,key='Token',gas=1100000) + async def token_deploy2a(self): return await self.token_deploy(num=2,key='SafeMath',gas=200000) + async def token_deploy2b(self): return await self.token_deploy(num=2,key='Owned',gas=250000) + async def token_deploy2c(self): return await self.token_deploy(num=2,key='Token',gas=1100000) - def contract_deploy(self): # test create,sign,send - return self.token_deploy(num=2,key='SafeMath',gas=1100000,mmgen_cmd='txcreate') + async def contract_deploy(self): # test create,sign,send + return await self.token_deploy(num=2,key='SafeMath',gas=1100000,mmgen_cmd='txcreate') - def token_transfer_ops(self,op,amt=1000): + async def token_transfer_ops(self,op,amt=1000): self.spawn('',msg_only=True) sid = dfl_sid from mmgen.tool import MMGenToolCmdWallet usr_mmaddrs = ['{}:E:{}'.format(sid,i) for i in (11,21)] usr_addrs = [MMGenToolCmdWallet().gen_addr(addr,dfl_words_file) for addr in usr_mmaddrs] - self._rpc_init() - from mmgen.altcoins.eth.contract import Token + from mmgen.altcoins.eth.contract import TokenResolve from mmgen.altcoins.eth.tx import EthereumMMGenTX as etx - def do_transfer(): + async def do_transfer(): for i in range(2): - tk = Token(self.read_from_tmpfile('token_addr{}'.format(i+1)).strip()) - imsg_r('\n'+tk.info()) - imsg('dev token balance (pre-send): {}'.format(tk.balance(dfl_addr))) + tk = await TokenResolve(self.read_from_tmpfile('token_addr{}'.format(i+1)).strip()) + imsg_r('\n' + await tk.info()) + imsg('dev token balance (pre-send): {}'.format(await tk.get_balance(dfl_addr))) imsg('Sending {} {} to address {} ({})'.format(amt,g.coin,usr_addrs[i],usr_mmaddrs[i])) from mmgen.obj import ETHAmt - txid = tk.transfer( dfl_addr, usr_addrs[i], amt, dfl_privkey, + txid = await tk.transfer( dfl_addr, usr_addrs[i], amt, dfl_privkey, start_gas = ETHAmt(60000,'wei'), gasPrice = ETHAmt(8,'Gwei') ) - assert etx.get_exec_status(txid,True) != 0,'Transfer of token funds failed. Aborting' + assert (await etx.get_exec_status(txid,True)) != 0,'Transfer of token funds failed. Aborting' - def show_bals(): + async def show_bals(): for i in range(2): - tk = Token(self.read_from_tmpfile('token_addr{}'.format(i+1)).strip()) - imsg('Token: {}'.format(tk.symbol())) - imsg('dev token balance: {}'.format(tk.balance(dfl_addr))) + tk = await TokenResolve(self.read_from_tmpfile(f'token_addr{i+1}').strip()) + imsg('Token: {}'.format(await tk.get_symbol())) + imsg('dev token balance: {}'.format(await tk.get_balance(dfl_addr))) imsg('usr token balance: {} ({} {})'.format( - tk.balance(usr_addrs[i]),usr_mmaddrs[i],usr_addrs[i])) + await tk.get_balance(usr_addrs[i]),usr_mmaddrs[i],usr_addrs[i])) silence() - if op == 'show_bals': show_bals() - elif op == 'do_transfer': do_transfer() + if op == 'show_bals': + await show_bals() + elif op == 'do_transfer': + await do_transfer() end_silence() return 'ok' @@ -689,15 +688,18 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): t.req_exit_val = 2 return t - def token_addrimport(self): + def token_addrimport(self,extra_args=[],expect='3/3'): for n,r in ('1','11-13'),('2','21-23'): tk_addr = self.read_from_tmpfile('token_addr'+n).strip() - t = self.addrimport(ext='['+r+']{}.addrs',expect='3/3',add_args=['--token='+tk_addr]) + t = self.addrimport(ext='['+r+']{}.addrs',expect=expect,add_args=['--token='+tk_addr]+extra_args) t.p.wait() ok_msg() t.skip_ok = True return t + def token_addrimport_batch(self): + return self.token_addrimport(extra_args=['--batch'],expect='OK: 3') + def bal7(self): return self.bal5() def token_bal1(self): return self.token_bal(n='1') diff --git a/test/test_py_d/ts_main.py b/test/test_py_d/ts_main.py index a1830856..bc002f7d 100755 --- a/test/test_py_d/ts_main.py +++ b/test/test_py_d/ts_main.py @@ -146,8 +146,10 @@ class TestSuiteMain(TestSuiteBase,TestSuiteShared): def __init__(self,trunner,cfgs,spawn): if g.coin.lower() not in self.networks: return - rpc_init() - self.lbl_id = ('account','label')['label_api' in g.rpc.caps] + from mmgen.rpc import rpc_init + g.regtest = False # rpc_init hack + self.rpc = run_session(rpc_init()) + self.lbl_id = ('account','label')['label_api' in self.rpc.caps] if g.coin in ('BTC','BCH','LTC'): self.tx_fee = {'btc':'0.0001','bch':'0.001','ltc':'0.01'}[g.coin.lower()] self.txbump_fee = {'btc':'123s','bch':'567s','ltc':'12345s'}[g.coin.lower()] diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index f21d1753..0467b0b4 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -182,6 +182,7 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared): ('bob_import_addr', 'importing non-MMGen address with --rescan'), ('bob_bal4', "Bob's balance (after import with rescan)"), ('bob_import_list', 'importing flat address list'), + ('bob_import_list_rescan', 'importing flat address list with --rescan'), ('bob_split2', "splitting Bob's funds"), ('bob_0conf0_getbalance', "Bob's balance (unconfirmed, minconf=0)"), ('bob_0conf1_getbalance', "Bob's balance (unconfirmed, minconf=1)"), @@ -735,11 +736,15 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared): def bob_import_addr(self): addr = self.read_from_tmpfile('non-mmgen.addrs').split()[0] - return self.user_import('bob',['--rescan','--address='+addr]) + return self.user_import('bob',['--quiet','--address='+addr]) def bob_import_list(self): addrfile = joinpath(self.tmpdir,'non-mmgen.addrs') - return self.user_import('bob',['--addrlist',addrfile]) + return self.user_import('bob',['--quiet','--addrlist',addrfile]) + + def bob_import_list_rescan(self): + addrfile = joinpath(self.tmpdir,'non-mmgen.addrs') + return self.user_import('bob',['--quiet','--rescan','--addrlist',addrfile]) def bob_split2(self): addrs = self.read_from_tmpfile('non-mmgen.addrs').split() diff --git a/test/tooltest2.py b/test/tooltest2.py index 28f9050e..a7eb6f0f 100755 --- a/test/tooltest2.py +++ b/test/tooltest2.py @@ -759,7 +759,10 @@ tests = { 'ltc_testnet': [ ( ['test/ref/litecoin/A5A1E0-LTC[1454.64322,1453,tl=1320969600].testnet.rawtx'], None ), ], 'eth_mainnet': [ ( ['test/ref/ethereum/88FEFD-ETH[23.45495,40000].rawtx'], None ), ], - 'eth_testnet': [ ( ['test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.rawtx'], None ), ], + 'eth_testnet': [ ( [ + 'test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.rawtx', + 'test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.sigtx' + ], None ), ], 'mm1_mainnet': [ ( ['test/ref/ethereum/5881D2-MM1[1.23456,50000].rawtx'], None ), ], 'mm1_testnet': [ ( ['test/ref/ethereum/6BDB25-MM1[1.23456,50000].testnet.rawtx'], None ), ], 'etc_mainnet': [ ( ['test/ref/ethereum_classic/ED3848-ETC[1.2345,40000].rawtx'], None ), ], diff --git a/test/unit_tests_d/ut_rpc.py b/test/unit_tests_d/ut_rpc.py new file mode 100755 index 00000000..bbd9a9aa --- /dev/null +++ b/test/unit_tests_d/ut_rpc.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +test.unit_tests_d.ut_rpc: RPC unit test for the MMGen suite +""" + +from mmgen.common import * +from mmgen.exception import * + +from mmgen.protocol import init_coin,EthereumProtocol +from mmgen.rpc import MoneroWalletRPCClient +from mmgen.daemon import CoinDaemon,MoneroWalletDaemon + +class unit_tests: + + def btc(self,name,ut): + + async def run_test(): + c = g.rpc + qmsg(' Testing backend {!r}'.format(type(c.backend).__name__)) + addrs = ( + ('bc1qvmqas4maw7lg9clqu6kqu9zq9cluvlln5hw97q','test address #1'), # deadbeef * 8 + ('bc1qe50rj25cldtskw5huxam335kyshtqtlrf4pt9x','test address #2'), # deadbeef * 7 + deadbeee + ) + + await c.batch_call('importaddress',addrs,timeout=120) + ret = await c.batch_call('getaddressesbylabel',[(l,) for a,l in addrs]) + assert list(ret[0].keys())[0] == addrs[0][0] + + bh = (await c.call('getblockchaininfo',timeout=300))['bestblockhash'] + await c.gathered_call('getblock',((bh,),(bh,1)),timeout=300) + await c.gathered_call(None,(('getblock',(bh,)),('getblock',(bh,1))),timeout=300) + + + d = CoinDaemon('btc',test_suite=True) + d.remove_datadir() + d.start() + g.proto.daemon_data_dir = d.datadir # used by BitcoinRPCClient.set_auth() to find the cookie + g.rpc_port = d.rpc_port + + for backend in g.autoset_opts['rpc_backend'].choices: + run_session(run_test(),backend=backend) + + d.stop() + + if g.platform != 'win': + + qmsg(f'\n Testing authentication with credentials from bitcoin.conf:') + d.remove_datadir() + os.makedirs(d.datadir) + + cf = os.path.join(d.datadir,'bitcoin.conf') + open(cf,'a').write('\nrpcuser = ut_rpc\nrpcpassword = ut_rpc_passw0rd\n') + + d.add_flag('keep_cfg_file') + d.start() + + async def do(): + assert g.rpc.auth.user == 'ut_rpc', 'user is not ut_rpc!' + + run_session(do()) + d.stop() + + qmsg(' OK') + return True + + def eth(self,name,ut): + ed = CoinDaemon('eth',test_suite=True) + ed.start() + init_coin('eth') + g.rpc_port = CoinDaemon('eth',test_suite=True).rpc_port + + async def run_test(): + qmsg(' Testing backend {!r}'.format(type(g.rpc.backend).__name__)) + ret = await g.rpc.call('parity_versionInfo',timeout=300) + #print(ret) + + for backend in g.autoset_opts['rpc_backend'].choices: + run_session(run_test(),backend=backend) + + ed.stop() + return True + + def xmr_wallet(self,name,ut): + + async def run(): + md = CoinDaemon('xmr',test_suite=True) + md.start() + + g.monero_wallet_rpc_password = 'passwOrd' + mwd = MoneroWalletDaemon(wallet_dir='test/trash',test_suite=True) + mwd.start() + + c = MoneroWalletRPCClient( + host = g.monero_wallet_rpc_host, + port = mwd.rpc_port, + user = g.monero_wallet_rpc_user, + passwd = g.monero_wallet_rpc_password) + + await c.call('get_version') + + gmsg('OK') + mwd.wait = False + mwd.stop() + md.wait = False + md.stop() + + run_session(run(),do_rpc_init=False) + return True diff --git a/test/unit_tests_d/ut_tx_deserialize.py b/test/unit_tests_d/ut_tx_deserialize.py index 1e6d6922..cc1a7210 100755 --- a/test/unit_tests_d/ut_tx_deserialize.py +++ b/test/unit_tests_d/ut_tx_deserialize.py @@ -3,9 +3,14 @@ test/unit_tests_d/ut_tx_deserialize: TX deserialization unit test for the MMGen suite """ -import os +import os,json + from mmgen.common import * from ..include.common import * +from mmgen.protocol import init_coin +from mmgen.tx import MMGenTX,DeserializedTX +from mmgen.rpc import rpc_init +from mmgen.daemon import CoinDaemon class unit_test(object): @@ -16,7 +21,7 @@ class unit_test(object): def run_test(self,name,ut): - def test_tx(txhex,desc,n): + async def test_tx(txhex,desc,n): def has_nonstandard_outputs(outputs): for o in outputs: @@ -25,7 +30,7 @@ class unit_test(object): return True return False - d = g.rpc.decoderawtransaction(txhex) + d = await g.rpc.call('decoderawtransaction',txhex) if has_nonstandard_outputs(d['vout']): return False @@ -86,7 +91,7 @@ class unit_test(object): Msg_r('Testing transactions from {!r}'.format(fn)) if not opt.quiet: Msg('') - def test_core_vectors(): + async def test_core_vectors(): self._get_core_repo_root() fn_b = 'src/test/data/tx_valid.json' fn = os.path.join(self.core_repo_root,fn_b) @@ -95,38 +100,33 @@ class unit_test(object): n = 1 for e in data: if type(e[0]) == list: - test_tx(e[1],desc,n) + await rpc_init() + await test_tx(e[1],desc,n) n += 1 else: desc = e[0] Msg('OK') - def test_mmgen_txs(): + async def test_mmgen_txs(): fns = ( ('btc',False,'test/ref/0B8D5A[15.31789,14,tl=1320969600].rawtx'), ('btc',True,'test/ref/0C7115[15.86255,14,tl=1320969600].testnet.rawtx'), # ('bch',False,'test/ref/460D4D-BCH[10.19764,tl=1320969600].rawtx') ) - from mmgen.protocol import init_coin - from mmgen.tx import MMGenTX - from mmgen.daemon import CoinDaemon print_info('test/ref/*rawtx','MMGen reference transactions') for n,(coin,tn,fn) in enumerate(fns): init_coin(coin,tn) g.proto.daemon_data_dir = 'test/daemons/' + g.coin.lower() g.rpc_port = CoinDaemon(coin + ('','_tn')[tn],test_suite=True).rpc_port - rpc_init(reinit=True) - test_tx(MMGenTX(fn).hex,fn,n+1) + await rpc_init() + await test_tx(MMGenTX(fn).hex,fn,n+1) init_coin('btc',False) g.rpc_port = CoinDaemon('btc',test_suite=True).rpc_port - rpc_init(reinit=True) + await rpc_init() Msg('OK') - from mmgen.tx import DeserializedTX - import json - start_test_daemons('btc','btc_tn') # ,'bch') - test_mmgen_txs() - test_core_vectors() + run_session(test_mmgen_txs(),do_rpc_init=False) + run_session(test_core_vectors(),do_rpc_init=False) stop_test_daemons('btc','btc_tn') # ,'bch') return True