diff --git a/data_files/mmgen.cfg b/data_files/mmgen.cfg index 3dd5a9f4..89365b43 100644 --- a/data_files/mmgen.cfg +++ b/data_files/mmgen.cfg @@ -35,6 +35,17 @@ # Uncomment to override 'rpcpassword' from coin daemon config file: # rpc_password mypassword +# Choose the backend to use for JSON-RPC connections. Valid choices are +# 'httplib', 'requests', 'curl', 'aiohttp' (Linux only) or 'auto' (defaults +# to curl for Windows/MSYS2 and httplib for Linux): +# rpc_backend auto + +# Increase to allow aiohttp to make more simultaneous RPC connections to the +# daemon. Must be no greater than the 'rpcworkqueue' value in effect on the +# currently running bitcoind (DEFAULT_HTTP_WORKQUEUE = 16). Values over 32 +# may produce little benefit or even reduce performance: +# aiohttp_rpc_queue_len 16 + # Uncomment to set the coin daemon datadir: # daemon_data_dir /path/to/datadir diff --git a/mmgen/addr.py b/mmgen/addr.py index 03fa2734..3e22ddfb 100755 --- a/mmgen/addr.py +++ b/mmgen/addr.py @@ -1023,10 +1023,8 @@ re-import your addresses. def __new__(cls,*args,**kwargs): return MMGenObject.__new__(altcoin_subclass(cls,'tw','AddrData')) - def __init__(self,source=None,wallet=None): + def __init__(self,*args,**kwargs): self.al_ids = {} - if source == 'tw': - self.add_tw_data(wallet) def seed_ids(self): return list(self.al_ids.keys()) @@ -1048,30 +1046,34 @@ re-import your addresses. return (list(d.values())[0][0]) if d else None @classmethod - def get_tw_data(cls,wallet=None): + async def get_tw_data(cls,wallet=None): vmsg('Getting address data from tracking wallet') if 'label_api' in g.rpc.caps: - accts = g.rpc.listlabels() - alists = [list(a.keys()) for a in g.rpc.getaddressesbylabel([[k] for k in accts],batch=True)] + accts = await g.rpc.call('listlabels') + ll = await g.rpc.batch_call('getaddressesbylabel',[(k,) for k in accts]) + alists = [list(a.keys()) for a in ll] else: - accts = g.rpc.listaccounts(0,True) - alists = g.rpc.getaddressesbyaccount([[k] for k in accts],batch=True) + accts = await g.rpc.call('listaccounts',0,True) + alists = await g.rpc.batch_call('getaddressesbyaccount',[(k,) for k in accts]) return list(zip(accts,alists)) - def add_tw_data(self,wallet): - d,out,i = self.get_tw_data(wallet),{},0 - for acct,addr_array in d: + async def add_tw_data(self,wallet): + + twd = await type(self).get_tw_data(wallet) + out,i = {},0 + for acct,addr_array in twd: l = TwLabel(acct,on_fail='silent') if l and l.mmid.type == 'mmgen': obj = l.mmid.obj - i += 1 if len(addr_array) != 1: die(2,self.msgs['too_many_acct_addresses'].format(acct)) al_id = AddrListID(SeedID(sid=obj.sid),MMGenAddrType(obj.mmtype)) if al_id not in out: out[al_id] = [] out[al_id].append(AddrListEntry(idx=obj.idx,addr=addr_array[0],label=l.comment)) - vmsg('{n} {pnm} addresses found, {m} accounts total'.format(n=i,pnm=pnm,m=len(d))) + i += 1 + + vmsg('{n} {pnm} addresses found, {m} accounts total'.format(n=i,pnm=pnm,m=len(twd))) for al_id in out: self.add(AddrList(al_id=al_id,adata=AddrListList(sorted(out[al_id],key=lambda a: a.idx)))) @@ -1087,3 +1089,15 @@ re-import your addresses. for al_id in self.al_ids: d.update(self.al_ids[al_id].make_reverse_dict(coinaddrs)) return d + +class TwAddrData(AddrData,metaclass=aInitMeta): + + def __new__(cls,*args,**kwargs): + return MMGenObject.__new__(altcoin_subclass(cls,'tw','TwAddrData')) + + def __init__(self,*args,**kwargs): + pass + + async def __ainit__(self,wallet=None): + self.al_ids = {} + await self.add_tw_data(wallet) diff --git a/mmgen/altcoins/eth/contract.py b/mmgen/altcoins/eth/contract.py index 1f89e235..ae7f3f03 100755 --- a/mmgen/altcoins/eth/contract.py +++ b/mmgen/altcoins/eth/contract.py @@ -25,7 +25,7 @@ from . import rlp from mmgen.globalvars import g from mmgen.common import * -from mmgen.obj import MMGenObject,CoinAddr,TokenAddr,CoinTxID,ETHAmt +from mmgen.obj import MMGenObject,CoinAddr,TokenAddr,CoinTxID,ETHAmt,aInitMeta from mmgen.util import msg try: @@ -39,21 +39,7 @@ def parse_abi(s): def create_method_id(sig): return keccak_256(sig.encode()).hexdigest()[:8] -class Token(MMGenObject): # ERC20 - - _decimals = None - - # Test that token is in the blockchain by calling constructor w/o decimals arg - def __init__(self,addr,decimals=None): - self.addr = TokenAddr(addr) - if decimals: - self._decimals = decimals - else: - rpc_init() - self.decimals() - if not self._decimals: - raise TokenNotInBlockchain("Token '{}' not in blockchain".format(addr)) - self.base_unit = Decimal('10') ** -self._decimals +class TokenBase(MMGenObject): # ERC20 @staticmethod def transferdata2sendaddr(data): # online @@ -62,53 +48,50 @@ class Token(MMGenObject): # ERC20 def transferdata2amt(self,data): # online return ETHAmt(int(parse_abi(data)[-1],16) * self.base_unit) - def do_call(self,method_sig,method_args='',toUnit=False): + async def do_call(self,method_sig,method_args='',toUnit=False): data = create_method_id(method_sig) + method_args if g.debug: msg('ETH_CALL {}: {}'.format(method_sig,'\n '.join(parse_abi(data)))) - ret = g.rpc.eth_call({ 'to': '0x'+self.addr, 'data': '0x'+data }) + ret = await g.rpc.call('eth_call',{ 'to': '0x'+self.addr, 'data': '0x'+data }) if toUnit: return int(ret,16) * self.base_unit else: return ret - def balance(self,acct_addr): - return ETHAmt(self.do_call('balanceOf(address)',acct_addr.rjust(64,'0'),toUnit=True)) + async def get_balance(self,acct_addr): + return ETHAmt(await self.do_call('balanceOf(address)',acct_addr.rjust(64,'0'),toUnit=True)) def strip(self,s): return ''.join([chr(b) for b in s if 32 <= b <= 127]).strip() - # TODO: make these properties - def decimals(self): - if self._decimals == None: - res = self.do_call('decimals()') - try: - assert res[:2] == '0x' - self._decimals = int(res[2:],16) - except: - msg("RPC call to decimals() failed (returned '{}')".format(res)) - return None - return self._decimals + async def get_name(self): + return self.strip(bytes.fromhex((await self.do_call('name()'))[2:])) - def name(self): - return self.strip(bytes.fromhex(self.do_call('name()')[2:])) + async def get_symbol(self): + return self.strip(bytes.fromhex((await self.do_call('symbol()'))[2:])) - def symbol(self): - return self.strip(bytes.fromhex(self.do_call('symbol()')[2:])) + async def get_decimals(self): + ret = await self.do_call('decimals()') + try: + assert ret[:2] == '0x' + return int(ret,16) + except: + msg("RPC call to decimals() failed (returned '{}')".format(ret)) + return None - def total_supply(self): - return self.do_call('totalSupply()',toUnit=True) + async def get_total_supply(self): + return await self.do_call('totalSupply()',toUnit=True) - def info(self): + async def info(self): fs = '{:15}{}\n' * 5 return fs.format('token address:', self.addr, - 'token symbol:', self.symbol(), - 'token name:', self.name(), - 'decimals:', self.decimals(), - 'total supply:', self.total_supply()) + 'token symbol:', await self.get_symbol(), + 'token name:', await self.get_name(), + 'decimals:', self.decimals, + 'total supply:', await self.get_total_supply()) - def code(self): - return g.rpc.eth_getCode('0x'+self.addr)[2:] + async def code(self): + return (await g.rpc.call('eth_getCode','0x'+self.addr))[2:] def create_data(self,to_addr,amt,method_sig='transfer(address,uint256)',from_addr=None): from_arg = from_addr.rjust(64,'0') if from_addr else '' @@ -126,13 +109,13 @@ class Token(MMGenObject): # ERC20 'nonce': nonce, 'data': bytes.fromhex(data) } - def txsign(self,tx_in,key,from_addr,chain_id=None): + async def txsign(self,tx_in,key,from_addr,chain_id=None): from .pyethereum.transactions import Transaction if chain_id is None: chain_id_method = ('parity_chainId','eth_chainId')['eth_chainId' in g.rpc.caps] - chain_id = int(g.rpc.request(chain_id_method),16) + chain_id = int(await g.rpc.call(chain_id_method),16) tx = Transaction(**tx_in).sign(key,chain_id) hex_tx = rlp.encode(tx).hex() coin_txid = CoinTxID(tx.hash.hex()) @@ -147,18 +130,38 @@ class Token(MMGenObject): # ERC20 # The following are used for token deployment only: - def txsend(self,hex_tx): - return g.rpc.eth_sendRawTransaction('0x'+hex_tx).replace('0x','',1) + async def txsend(self,hex_tx): + return (await g.rpc.call('eth_sendRawTransaction','0x'+hex_tx)).replace('0x','',1) - def transfer( self,from_addr,to_addr,amt,key,start_gas,gasPrice, + async def transfer( self,from_addr,to_addr,amt,key,start_gas,gasPrice, method_sig='transfer(address,uint256)', from_addr2=None, return_data=False): tx_in = self.make_tx_in( from_addr,to_addr,amt, start_gas,gasPrice, - nonce = int(g.rpc.parity_nextNonce('0x'+from_addr),16), + nonce = int(await g.rpc.call('parity_nextNonce','0x'+from_addr),16), method_sig = method_sig, from_addr2 = from_addr2 ) - (hex_tx,coin_txid) = self.txsign(tx_in,key,from_addr) - return self.txsend(hex_tx) + (hex_tx,coin_txid) = await self.txsign(tx_in,key,from_addr) + return await self.txsend(hex_tx) + +class Token(TokenBase): + + def __init__(self,addr,decimals): + self.addr = TokenAddr(addr) + assert isinstance(decimals,int),f'decimals param must be int instance, not {type(decimals)}' + self.decimals = decimals + self.base_unit = Decimal('10') ** -self.decimals + +class TokenResolve(TokenBase,metaclass=aInitMeta): + + def __init__(self,addr): + return super().__init__() + + async def __ainit__(self,addr): + self.addr = TokenAddr(addr) + decimals = await self.get_decimals() # requires self.addr! + if not decimals: + raise TokenNotInBlockchain(f'Token {addr!r} not in blockchain') + Token.__init__(self,addr,decimals) diff --git a/mmgen/altcoins/eth/tw.py b/mmgen/altcoins/eth/tw.py index 3bc2c1aa..28daa69b 100755 --- a/mmgen/altcoins/eth/tw.py +++ b/mmgen/altcoins/eth/tw.py @@ -22,27 +22,17 @@ altcoins.eth.tw: Ethereum tracking wallet and related classes for the MMGen suit from mmgen.common import * from mmgen.obj import ETHAmt,TwLabel,is_coin_addr,is_mmgen_id,MMGenListItem,ListItemAttr,ImmutableAttr -from mmgen.tw import TrackingWallet,TwAddrList,TwUnspentOutputs -from mmgen.addr import AddrData -from .contract import Token +from mmgen.tw import TrackingWallet,TwAddrList,TwUnspentOutputs,TwGetBalance +from mmgen.addr import AddrData,TwAddrData +from .contract import Token,TokenResolve class EthereumTrackingWallet(TrackingWallet): - caps = () + caps = ('batch',) data_key = 'accounts' use_tw_file = True - def __init__(self,mode='r',no_rpc=False): - TrackingWallet.__init__(self,mode=mode) - - for v in self.data['tokens'].values(): - self.conv_types(v) - - if g.token and not is_coin_addr(g.token): - ret = self.sym2addr(g.token,no_rpc=no_rpc) - if ret: g.token = ret - - def is_in_wallet(self,addr): + async def is_in_wallet(self,addr): return addr in self.data_root def init_empty(self): @@ -84,14 +74,17 @@ class EthereumTrackingWallet(TrackingWallet): self.force_write() msg('{} upgraded successfully!'.format(self.desc)) - # Don't call rpc_init() for Ethereum, because it may create a wallet instance - def rpc_init(self): pass - - def rpc_get_balance(self,addr): - return ETHAmt(int(g.rpc.eth_getBalance('0x'+addr),16),'wei') + async def rpc_get_balance(self,addr): + return ETHAmt(int(await g.rpc.call('eth_getBalance','0x'+addr),16),'wei') @write_mode - def import_address(self,addr,label,foo): + async def batch_import_address(self,args_list): + for arg_list in args_list: + await self.import_address(*arg_list) + return args_list + + @write_mode + async def import_address(self,addr,label,foo): r = self.data_root if addr in r: if not r[addr]['mmid'] and label.mmid: @@ -101,7 +94,7 @@ class EthereumTrackingWallet(TrackingWallet): r[addr] = { 'mmid': label.mmid, 'comment': label.comment } @write_mode - def remove_address(self,addr): + async def remove_address(self,addr): r = self.data_root if is_coin_addr(addr): @@ -109,7 +102,7 @@ class EthereumTrackingWallet(TrackingWallet): elif is_mmgen_id(addr): have_match = lambda k: r[k]['mmid'] == addr else: - die(1,"'{}' is not an Ethereum address or MMGen ID".format(addr)) + die(1,f'{addr!r} is not an Ethereum address or MMGen ID') for k in r: if have_match(k): @@ -119,46 +112,30 @@ class EthereumTrackingWallet(TrackingWallet): self.write() return ret else: - m = "Address '{}' not found in '{}' section of tracking wallet" - msg(m.format(addr,self.data_root_desc)) + msg(f'Address {addr!r} not found in {self.data_root_desc!r} section of tracking wallet') return None @write_mode - def set_label(self,coinaddr,lbl): + async def set_label(self,coinaddr,lbl): for addr,d in list(self.data_root.items()): if addr == coinaddr: d['comment'] = lbl.comment self.write() return None - else: # emulate on_fail='return' of RPC library - m = "Address '{}' not found in '{}' section of tracking wallet" - return ('rpcfail',(None,2,m.format(coinaddr,self.data_root_desc))) - - def addr2sym(self,req_addr): + else: + msg(f'Address {coinaddr!r} not found in {self.data_root_desc!r} section of tracking wallet') + return False + async def addr2sym(self,req_addr): for addr in self.data['tokens']: if addr == req_addr: - ret = self.data['tokens'][addr]['params'].get('symbol') - if ret: return ret - else: break - - self.token_obj = Token(req_addr) - ret = self.token_obj.symbol().upper() - self.force_set_token_param(req_addr,'symbol',ret) - return ret - - def sym2addr(self,sym,no_rpc=False): - - for addr in self.data['tokens']: - if self.data['tokens'][addr]['params'].get('symbol') == sym.upper(): - return addr - - if no_rpc: + return self.data['tokens'][addr]['params']['symbol'] + else: return None + async def sym2addr(self,sym): for addr in self.data['tokens']: - if Token(addr).symbol().upper() == sym.upper(): - self.force_set_token_param(addr,'symbol',sym.upper()) + if self.data['tokens'][addr]['params']['symbol'] == sym.upper(): return addr else: return None @@ -168,17 +145,6 @@ class EthereumTrackingWallet(TrackingWallet): return self.data['tokens'][token]['params'].get(param) return None - def force_set_token_param(self,*args,**kwargs): - mode_save = self.mode - self.mode = 'w' - self.set_token_param(*args,**kwargs) - self.mode = mode_save - - @write_mode - def set_token_param(self,token,param,val): - if token in self.data['tokens']: - self.data['tokens'][token]['params'][param] = val - class EthereumTokenTrackingWallet(EthereumTrackingWallet): desc = 'Ethereum token tracking wallet' @@ -186,29 +152,32 @@ class EthereumTokenTrackingWallet(EthereumTrackingWallet): symbol = None cur_eth_balances = {} - def __init__(self,mode='r',no_rpc=False): - EthereumTrackingWallet.__init__(self,mode=mode,no_rpc=no_rpc) + async def __ainit__(self,mode='r'): + await super().__ainit__(mode=mode) - self.desc = 'Ethereum token tracking wallet' + for v in self.data['tokens'].values(): + self.conv_types(v) if not is_coin_addr(g.token): - raise UnrecognizedTokenSymbol('Specified token {!r} could not be resolved!'.format(g.token)) + g.token = await self.sym2addr(g.token) # returns None on failure - if mode == 'r' and not g.token in self.data['tokens']: + if not is_coin_addr(g.token): + if self.importing: + m = 'When importing addresses for a new token, the token must be specified by address, not symbol.' + raise InvalidTokenAddress(f'{g.token!r}: invalid token address\n{m}') + else: + raise UnrecognizedTokenSymbol(f'Specified token {g.token!r} could not be resolved!') + + if g.token in self.data['tokens']: + self.decimals = self.data['tokens'][g.token]['params']['decimals'] + self.symbol = self.data['tokens'][g.token]['params']['symbol'] + elif not self.importing: raise TokenNotInWallet('Specified token {!r} not in wallet!'.format(g.token)) self.token = g.token + g.dcoin = self.symbol - if self.token in self.data['tokens']: - for k in ('decimals','symbol'): - setattr(self,k,self.get_param(k)) - if getattr(self,k) == None: - setattr(self,k,getattr(Token(self.token,self.decimals),k)()) - if getattr(self,k) != None: - self.set_param(k,getattr(self,k)) - self.write() - - def is_in_wallet(self,addr): + async def is_in_wallet(self,addr): return addr in self.data['tokens'][self.token] @property @@ -217,43 +186,39 @@ class EthereumTokenTrackingWallet(EthereumTrackingWallet): @property def data_root_desc(self): - return 'token ' + Token(self.token,self.decimals).symbol() + return 'token ' + self.get_param('symbol') - @write_mode - def add_token(self,token): - msg("Adding token '{}' to tracking wallet.".format(token)) - self.data['tokens'][token] = { 'params': {} } + async def rpc_get_balance(self,addr): + return await Token(self.token,self.decimals).get_balance(addr) - @write_mode - def import_address(self,*args,**kwargs): - if self.token not in self.data['tokens']: - self.add_token(self.token) - EthereumTrackingWallet.import_address(self,*args,**kwargs) - - def rpc_get_balance(self,addr): - return Token(self.token,self.decimals).balance(addr) - - def get_eth_balance(self,addr,force_rpc=False): + async def get_eth_balance(self,addr,force_rpc=False): cache = self.cur_eth_balances - data_root = self.data['accounts'] - ret = None if force_rpc else self.get_cached_balance(addr,cache,data_root) + r = self.data['accounts'] + ret = None if force_rpc else self.get_cached_balance(addr,cache,r) if ret == None: - ret = EthereumTrackingWallet.rpc_get_balance(self,addr) - self.cache_balance(addr,ret,cache,data_root) + ret = await super().rpc_get_balance(addr) + self.cache_balance(addr,ret,cache,r) return ret - def force_set_param(self,*args,**kwargs): - mode_save = self.mode - self.mode = 'w' - self.set_param(*args,**kwargs) - self.mode = mode_save + def get_param(self,param): + return self.data['tokens'][self.token]['params'][param] @write_mode - def set_param(self,param,val): - self.data['tokens'][self.token]['params'][param] = val - - def get_param(self,param): - return self.data['tokens'][self.token]['params'].get(param) + async def import_token(tw): + """ + Token 'symbol' and 'decimals' values are resolved from the network by the system just + once, upon token import. Thereafter, token address, symbol and decimals are resolved + either from the tracking wallet (online operations) or transaction file (when signing). + """ + if not g.token in tw.data['tokens']: + t = await TokenResolve(g.token) + tw.token = g.token + tw.data['tokens'][tw.token] = { + 'params': { + 'symbol': await t.get_symbol(), + 'decimals': t.decimals + } + } # No unspent outputs with Ethereum, but naming must be consistent class EthereumTwUnspentOutputs(TwUnspentOutputs): @@ -277,23 +242,23 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, 'q':'a_quit','p':'a_print','v':'a_view','w':'a_view_wide', 'l':'a_lbl_add','D':'a_addr_delete','R':'a_balance_refresh' } - def __init__(self,*args,**kwargs): + async def __ainit__(self,*args,**kwargs): if g.use_cached_balances: self.hdr_fmt += '\n' + yellow('WARNING: Using cached balances. These may be out of date!') - TwUnspentOutputs.__init__(self,*args,**kwargs) + await TwUnspentOutputs.__ainit__(self,*args,**kwargs) def do_sort(self,key=None,reverse=False): if key == 'txid': return super().do_sort(key=key,reverse=reverse) - def get_unspent_rpc(self): + async def get_unspent_rpc(self): wl = self.wallet.sorted_list if self.addrs: wl = [d for d in wl if d['addr'] in self.addrs] return [{ 'account': TwLabel(d['mmid']+' '+d['comment'],on_fail='raise'), 'address': d['addr'], - 'amount': self.wallet.get_balance(d['addr']), + 'amount': await self.wallet.get_balance(d['addr']), 'confirmations': 0, # TODO } for d in wl] @@ -311,6 +276,9 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, def age_disp(self,o,age_fmt): # TODO return None + def age_disp(self,o,age_fmt): # TODO + return None + class EthereumTokenTwUnspentOutputs(EthereumTwUnspentOutputs): disp_type = 'token' @@ -320,18 +288,18 @@ class EthereumTokenTwUnspentOutputs(EthereumTwUnspentOutputs): def get_display_precision(self): return 10 # truncate precision for narrow display - def get_unspent_data(self): - super().get_unspent_data() + async def get_unspent_data(self,*args,**kwargs): + await super().get_unspent_data(*args,**kwargs) for e in self.unspent: - e.amt2 = self.wallet.get_eth_balance(e.addr) + e.amt2 = await self.wallet.get_eth_balance(e.addr) class EthereumTwAddrList(TwAddrList): has_age = False - def __init__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None): + async def __ainit__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None): - self.wallet = wallet or TrackingWallet(mode='w') + self.wallet = wallet or await TrackingWallet(mode='w') tw_dict = self.wallet.mmid_ordered_dict self.total = g.proto.coin_amt('0') @@ -341,7 +309,7 @@ class EthereumTwAddrList(TwAddrList): label = TwLabel(mmid+' '+d['comment'],on_fail='raise') if usr_addr_list and (label.mmid not in usr_addr_list): continue - bal = self.wallet.get_balance(d['addr']) + bal = await self.wallet.get_balance(d['addr']) if bal == 0 and not showempty: if not label.comment or not all_labels: continue @@ -352,18 +320,20 @@ class EthereumTwAddrList(TwAddrList): self[label.mmid]['amt'] += bal self.total += bal -class EthereumTokenTwAddrList(EthereumTwAddrList): pass + del self.wallet + +class EthereumTokenTwAddrList(EthereumTwAddrList): + pass -from mmgen.tw import TwGetBalance class EthereumTwGetBalance(TwGetBalance): fs = '{w:13} {c}\n' # TODO - for now, just suppress display of meaningless data - def __init__(self,*args,**kwargs): - self.wallet = TrackingWallet(mode='w') - TwGetBalance.__init__(self,*args,**kwargs) + async def __ainit__(self,*args,**kwargs): + self.wallet = await TrackingWallet(mode='w') + await TwGetBalance.__ainit__(self,*args,**kwargs) - def create_data(self): + async def create_data(self): data = self.wallet.mmid_ordered_dict for d in data: if d.type == 'mmgen': @@ -374,20 +344,24 @@ class EthereumTwGetBalance(TwGetBalance): key = 'Non-MMGen' conf_level = 2 # TODO - amt = self.wallet.get_balance(data[d]['addr']) + amt = await self.wallet.get_balance(data[d]['addr']) self.data['TOTAL'][conf_level] += amt self.data[key][conf_level] += amt -class EthereumTokenTwGetBalance(EthereumTwGetBalance): pass + del self.wallet -class EthereumAddrData(AddrData): +class EthereumTwAddrData(TwAddrData): @classmethod - def get_tw_data(cls,wallet=None): + async def get_tw_data(cls,wallet=None): vmsg('Getting address data from tracking wallet') - tw = (wallet or TrackingWallet()).mmid_ordered_dict + tw = (wallet or await TrackingWallet()).mmid_ordered_dict # emulate the output of RPC 'listaccounts' and 'getaddressesbyaccount' return [(mmid+' '+d['comment'],[d['addr']]) for mmid,d in list(tw.items())] +class EthereumTokenTwGetBalance(EthereumTwGetBalance): pass +class EthereumTokenTwAddrData(EthereumTwAddrData): pass + +class EthereumAddrData(AddrData): pass class EthereumTokenAddrData(EthereumAddrData): pass diff --git a/mmgen/altcoins/eth/tx.py b/mmgen/altcoins/eth/tx.py index 480e0b90..04d88f17 100755 --- a/mmgen/altcoins/eth/tx.py +++ b/mmgen/altcoins/eth/tx.py @@ -24,7 +24,9 @@ import json from mmgen.common import * from mmgen.obj import * -from mmgen.tx import MMGenTX,MMGenBumpTX,MMGenSplitTX +from mmgen.tx import MMGenTX,MMGenBumpTX,MMGenSplitTX,MMGenTxForSigning +from mmgen.tw import TrackingWallet +from .contract import Token class EthereumMMGenTX(MMGenTX): desc = 'Ethereum transaction' @@ -49,7 +51,7 @@ class EthereumMMGenTX(MMGenTX): usr_contract_data = HexStr('') def __init__(self,*args,**kwargs): - super().__init__(*args,**kwargs) + MMGenTX.__init__(self,*args,**kwargs) if hasattr(opt,'tx_gas') and opt.tx_gas: self.tx_gas = self.start_gas = ETHAmt(int(opt.tx_gas),'wei') if hasattr(opt,'contract_data') and opt.contract_data: @@ -58,9 +60,12 @@ class EthereumMMGenTX(MMGenTX): self.usr_contract_data = HexStr(open(opt.contract_data).read().strip()) self.disable_fee_check = True + def check_txfile_hex_data(self): + pass + @classmethod - def get_exec_status(cls,txid,silent=False): - d = g.rpc.eth_getTransactionReceipt('0x'+txid) + async def get_exec_status(cls,txid,silent=False): + d = await g.rpc.call('eth_getTransactionReceipt','0x'+txid) if not silent: if 'contractAddress' in d and d['contractAddress']: msg('Contract address: {}'.format(d['contractAddress'].replace('0x',''))) @@ -84,46 +89,35 @@ class EthereumMMGenTX(MMGenTX): return True return False - # hex data if signed, json if unsigned - def check_txfile_hex_data(self): - if self.check_sigs(): - from .pyethereum.transactions import Transaction - from . import rlp - etx = rlp.decode(bytes.fromhex(self.hex),Transaction) - d = etx.to_dict() # ==> hex values have '0x' prefix, 0 is '0x' - for k in ('sender','to','data'): - if k in d: d[k] = d[k].replace('0x','',1) - o = { 'from': CoinAddr(d['sender']), - 'to': CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is token address - 'amt': ETHAmt(d['value'],'wei'), - 'gasPrice': ETHAmt(d['gasprice'],'wei'), - 'startGas': ETHAmt(d['startgas'],'wei'), - 'nonce': ETHNonce(d['nonce']), - 'data': HexStr(d['data']) } - if o['data'] and not o['to']: # token- or contract-creating transaction - o['token_addr'] = TokenAddr(etx.creates.hex()) # NB: could be a non-token contract address - self.disable_fee_check = True - txid = CoinTxID(etx.hash.hex()) - assert txid == self.coin_txid,"txid in tx.hex doesn't match value in MMGen transaction file" - else: - d = json.loads(self.hex) - o = { 'from': CoinAddr(d['from']), - 'to': CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is sendto address - 'amt': ETHAmt(d['amt']), - 'gasPrice': ETHAmt(d['gasPrice']), - 'startGas': ETHAmt(d['startGas']), - 'nonce': ETHNonce(d['nonce']), - 'chainId': Int(d['chainId']), - 'data': HexStr(d['data']) } + def parse_txfile_hex_data(self): + from .pyethereum.transactions import Transaction + from . import rlp + etx = rlp.decode(bytes.fromhex(self.hex),Transaction) + d = etx.to_dict() # ==> hex values have '0x' prefix, 0 is '0x' + for k in ('sender','to','data'): + if k in d: d[k] = d[k].replace('0x','',1) + o = { + 'from': CoinAddr(d['sender']), + 'to': CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is token address + 'amt': ETHAmt(d['value'],'wei'), + 'gasPrice': ETHAmt(d['gasprice'],'wei'), + 'startGas': ETHAmt(d['startgas'],'wei'), + 'nonce': ETHNonce(d['nonce']), + 'data': HexStr(d['data']) } + if o['data'] and not o['to']: # token- or contract-creating transaction + o['token_addr'] = TokenAddr(etx.creates.hex()) # NB: could be a non-token contract address + self.disable_fee_check = True + txid = CoinTxID(etx.hash.hex()) + assert txid == self.coin_txid,"txid in tx.hex doesn't match value in MMGen transaction file" self.tx_gas = o['startGas'] # approximate, but better than nothing self.fee = self.fee_rel2abs(o['gasPrice'].toWei()) self.txobj = o return d # 'token_addr','decimals' required by Token subclass - def get_nonce(self): - return ETHNonce(int(g.rpc.parity_nextNonce('0x'+self.inputs[0].addr),16)) + async def get_nonce(self): + return ETHNonce(int(await g.rpc.call('parity_nextNonce','0x'+self.inputs[0].addr),16)) - def make_txobj(self): # called by create_raw() + async def make_txobj(self): # called by create_raw() chain_id_method = ('parity_chainId','eth_chainId')['eth_chainId' in g.rpc.caps] self.txobj = { 'from': self.inputs[0].addr, @@ -131,20 +125,20 @@ class EthereumMMGenTX(MMGenTX): 'amt': self.outputs[0].amt if self.outputs else ETHAmt('0'), 'gasPrice': self.usr_rel_fee or self.fee_abs2rel(self.fee,to_unit='eth'), 'startGas': self.start_gas, - 'nonce': self.get_nonce(), - 'chainId': Int(g.rpc.request(chain_id_method),16), + 'nonce': await self.get_nonce(), + 'chainId': Int(await g.rpc.call(chain_id_method),16), 'data': self.usr_contract_data, } # Instead of serializing tx data as with BTC, just create a JSON dump. # This complicates things but means we avoid using the rlp library to deserialize the data, # thus removing an attack vector - def create_raw(self): + async def create_raw(self): assert len(self.inputs) == 1,'Transaction has more than one input!' o_num = len(self.outputs) o_ok = 0 if self.usr_contract_data else 1 assert o_num == o_ok,'Transaction has {} output{} (should have {})'.format(o_num,suf(o_num),o_ok) - self.make_txobj() + await self.make_txobj() odict = { k: str(v) for k,v in self.txobj.items() if k != 'token_to' } self.hex = json.dumps(odict) self.update_txid() @@ -156,9 +150,6 @@ class EthereumMMGenTX(MMGenTX): assert not is_hex_str(self.hex),'update_txid() must be called only when self.hex is not hex data' self.txid = MMGenTxID(make_chksum_6(self.hex).upper()) - def get_blockcount(self): - return Int(g.rpc.eth_blockNumber(),16) - def process_cmd_args(self,cmd_args,ad_f,ad_w): lc = len(cmd_args) if lc == 0 and self.usr_contract_data and not 'Token' in type(self).__name__: @@ -185,7 +176,9 @@ class EthereumMMGenTX(MMGenTX): return [int(reply)] # coin-specific fee routines: - def get_relay_fee(self): return ETHAmt('0') # TODO + @property + def relay_fee(self): + return ETHAmt('0') # TODO # given absolute fee in ETH, return gas price in Gwei using tx_gas def fee_abs2rel(self,abs_fee,to_unit='Gwei'): @@ -194,8 +187,8 @@ class EthereumMMGenTX(MMGenTX): return ret if to_unit == 'eth' else ret.to_unit(to_unit,show_decimal=True) # get rel_fee (gas price) from network, return in native wei - def get_rel_fee_from_network(self): - return Int(g.rpc.eth_gasPrice(),16),'eth_gasPrice' # ==> rel_fee,fe_type + async def get_rel_fee_from_network(self): + return Int(await g.rpc.call('eth_gasPrice'),16),'eth_gasPrice' # ==> rel_fee,fe_type # given rel fee and units, return absolute fee using tx_gas def convert_fee_spec(self,foo,units,amt,unit): @@ -264,7 +257,7 @@ class EthereumMMGenTX(MMGenTX): def format_view_rel_fee(self,terse): return '' def format_view_verbose_footer(self): return '' # TODO - def resolve_g_token_from_tx_file(self): + def resolve_g_token_from_txfile(self): die(2,"The '--token' option must be specified for token transaction files") def final_inputs_ok_msg(self,change_amt): @@ -272,7 +265,126 @@ class EthereumMMGenTX(MMGenTX): chg = '0' if (self.outputs and self.outputs[0].is_chg) else change_amt return m.format(ETHAmt(chg).hl(),g.coin) - def do_sign(self,wif,tx_num_str): + async def get_status(self,status=False): + + class r(object): pass + + async def is_in_mempool(): + if not 'full_node' in g.rpc.caps: + return False + return '0x'+self.coin_txid in [x['hash'] for x in await g.rpc.call('parity_pendingTransactions')] + + async def is_in_wallet(): + d = await g.rpc.call('eth_getTransactionReceipt','0x'+self.coin_txid) + if d and 'blockNumber' in d and d['blockNumber'] is not None: + r.confs = 1 + int(await g.rpc.call('eth_blockNumber'),16) - int(d['blockNumber'],16) + r.exec_status = int(d['status'],16) + return True + return False + + if await is_in_mempool(): + msg('Transaction is in mempool' if status else 'Warning: transaction is in mempool!') + return + + if status: + if await is_in_wallet(): + if self.txobj['data']: + cd = capfirst(self.contract_desc) + if r.exec_status == 0: + msg('{} failed to execute!'.format(cd)) + else: + msg('{} successfully executed with status {}'.format(cd,r.exec_status)) + die(0,'Transaction has {} confirmation{}'.format(r.confs,suf(r.confs))) + die(1,'Transaction is neither in mempool nor blockchain!') + + async def send(self,prompt_user=True,exit_on_fail=False): + + if not self.marked_signed(): + die(1,'Transaction is not signed!') + + self.check_correct_chain(on_fail='die') + + fee = self.fee_rel2abs(self.txobj['gasPrice'].toWei()) + + if not self.disable_fee_check and (fee > g.proto.max_tx_fee): + die(2,'Transaction fee ({}) greater than {} max_tx_fee ({} {})!'.format( + fee,g.proto.name.capitalize(),g.proto.max_tx_fee,g.coin)) + + await self.get_status() + + if prompt_user: + self.confirm_send() + + if g.bogus_send: + ret = None + else: + try: + ret = await g.rpc.call('eth_sendRawTransaction','0x'+self.hex) + except: + raise + ret = False + + if ret == False: + msg(red('Send of MMGen transaction {} failed'.format(self.txid))) + if exit_on_fail: + sys.exit(1) + return False + else: + if g.bogus_send: + m = 'BOGUS transaction NOT sent: {}' + else: + m = 'Transaction sent: {}' + assert ret == '0x'+self.coin_txid,'txid mismatch (after sending)' + self.desc = 'sent transaction' + msg(m.format(self.coin_txid.hl())) + self.add_timestamp() + self.add_blockcount() + return True + + async def get_cmdline_input_addrs(self): + ret = [] + if opt.inputs: + r = (await TrackingWallet()).data_root # must create new instance here + m = 'Address {!r} not in tracking wallet' + for i in opt.inputs.split(','): + if is_mmgen_id(i): + for addr in r: + if r[addr]['mmid'] == i: + ret.append(addr) + break + else: + raise UserAddressNotInWallet(m.format(i)) + elif is_coin_addr(i): + if not i in r: + raise UserAddressNotInWallet(m.format(i)) + ret.append(i) + else: + die(1,"'{}': not an MMGen ID or coin address".format(i)) + return ret + + def print_contract_addr(self): + if 'token_addr' in self.txobj: + msg('Contract address: {}'.format(self.txobj['token_addr'].hl())) + +class EthereumMMGenTxForSigning(EthereumMMGenTX,MMGenTxForSigning): + + def parse_txfile_hex_data(self): + d = json.loads(self.hex) + o = { + 'from': CoinAddr(d['from']), + 'to': CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is sendto address + 'amt': ETHAmt(d['amt']), + 'gasPrice': ETHAmt(d['gasPrice']), + 'startGas': ETHAmt(d['startGas']), + 'nonce': ETHNonce(d['nonce']), + 'chainId': Int(d['chainId']), + 'data': HexStr(d['data']) } + self.tx_gas = o['startGas'] # approximate, but better than nothing + self.fee = self.fee_rel2abs(o['gasPrice'].toWei()) + self.txobj = o + return d # 'token_addr','decimals' required by Token subclass + + async def do_sign(self,wif,tx_num_str): o = self.txobj o_conv = { 'to': bytes.fromhex(o['to']), @@ -299,7 +411,7 @@ class EthereumMMGenTX(MMGenTX): assert self.check_sigs(),'Signature check failed' - def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception + async def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception if self.marked_signed(): msg('Transaction is already signed!') @@ -311,7 +423,7 @@ class EthereumMMGenTX(MMGenTX): msg_r('Signing transaction{}...'.format(tx_num_str)) try: - self.do_sign(keys[0].sec.wif,tx_num_str) + await self.do_sign(keys[0].sec.wif,tx_num_str) msg('OK') return True except Exception as e: @@ -322,103 +434,6 @@ class EthereumMMGenTX(MMGenTX): ymsg('\n'+''.join(traceback.format_exception(*sys.exc_info()))) return False - def get_status(self,status=False): - - class r(object): pass - - def is_in_mempool(): - if not 'full_node' in g.rpc.caps: - return False - return '0x'+self.coin_txid in [x['hash'] for x in g.rpc.parity_pendingTransactions()] - - def is_in_wallet(): - d = g.rpc.eth_getTransactionReceipt('0x'+self.coin_txid) - if d and 'blockNumber' in d and d['blockNumber'] is not None: - r.confs = 1 + int(g.rpc.eth_blockNumber(),16) - int(d['blockNumber'],16) - r.exec_status = int(d['status'],16) - return True - return False - - if is_in_mempool(): - msg('Transaction is in mempool' if status else 'Warning: transaction is in mempool!') - return - - if status: - if is_in_wallet(): - if self.txobj['data']: - cd = capfirst(self.contract_desc) - if r.exec_status == 0: - msg('{} failed to execute!'.format(cd)) - else: - msg('{} successfully executed with status {}'.format(cd,r.exec_status)) - die(0,'Transaction has {} confirmation{}'.format(r.confs,suf(r.confs))) - die(1,'Transaction is neither in mempool nor blockchain!') - - def send(self,prompt_user=True,exit_on_fail=False): - - if not self.marked_signed(): - die(1,'Transaction is not signed!') - - self.check_correct_chain(on_fail='die') - - fee = self.fee_rel2abs(self.txobj['gasPrice'].toWei()) - - if not self.disable_fee_check and (fee > g.proto.max_tx_fee): - die(2,'Transaction fee ({}) greater than {} max_tx_fee ({} {})!'.format( - fee,g.proto.name.capitalize(),g.proto.max_tx_fee,g.coin)) - - self.get_status() - - if prompt_user: - self.confirm_send() - - ret = None if g.bogus_send else g.rpc.eth_sendRawTransaction('0x'+self.hex,on_fail='return') - - from mmgen.rpc import rpc_error,rpc_errmsg - if rpc_error(ret): - msg(yellow(rpc_errmsg(ret))) - msg(red('Send of MMGen transaction {} failed'.format(self.txid))) - if exit_on_fail: - sys.exit(1) - return False - else: - if g.bogus_send: - m = 'BOGUS transaction NOT sent: {}' - else: - m = 'Transaction sent: {}' - assert ret == '0x'+self.coin_txid,'txid mismatch (after sending)' - self.desc = 'sent transaction' - msg(m.format(self.coin_txid.hl())) - self.add_timestamp() - self.add_blockcount() - return True - - def get_cmdline_input_addrs(self): - ret = [] - if opt.inputs: - from mmgen.tw import TrackingWallet - r = TrackingWallet().data_root # must create new instance here - m = 'Address {!r} not in tracking wallet' - for i in opt.inputs.split(','): - if is_mmgen_id(i): - for addr in r: - if r[addr]['mmid'] == i: - ret.append(addr) - break - else: - raise UserAddressNotInWallet(m.format(i)) - elif is_coin_addr(i): - if not i in r: - raise UserAddressNotInWallet(m.format(i)) - ret.append(i) - else: - die(1,"'{}': not an MMGen ID or coin address".format(i)) - return ret - - def print_contract_addr(self): - if 'token_addr' in self.txobj: - msg('Contract address: {}'.format(self.txobj['token_addr'].hl())) - class EthereumTokenMMGenTX(EthereumMMGenTX): desc = 'Ethereum token transaction' contract_desc = 'token contract' @@ -427,26 +442,18 @@ class EthereumTokenMMGenTX(EthereumMMGenTX): fmt_keys = ('from','token_to','amt','nonce') fee_is_approximate = True - def __init__(self,*args,**kwargs): - if not kwargs.get('offline'): - from mmgen.tw import TrackingWallet - self.decimals = TrackingWallet().get_param('decimals') - from .contract import Token - self.token_obj = Token(g.token,self.decimals) - EthereumMMGenTX.__init__(self,*args,**kwargs) - def update_change_output(self,change_amt): if self.outputs[0].is_chg: self.update_output_amt(0,self.inputs[0].amt) # token transaction, so check both eth and token balances # TODO: add test with insufficient funds - def precheck_sufficient_funds(self,inputs_sum,sel_unspent): - eth_bal = self.twuo.wallet.get_eth_balance(sel_unspent[0].addr) + async def precheck_sufficient_funds(self,inputs_sum,sel_unspent): + eth_bal = await self.tw.get_eth_balance(sel_unspent[0].addr) if eth_bal == 0: # we don't know the fee yet msg('This account has no ether to pay for the transaction fee!') return False - return super().precheck_sufficient_funds(inputs_sum,sel_unspent) + return await super().precheck_sufficient_funds(inputs_sum,sel_unspent) def final_inputs_ok_msg(self,change_amt): token_bal = ( ETHAmt('0') if self.outputs[0].is_chg else @@ -454,49 +461,30 @@ class EthereumTokenMMGenTX(EthereumMMGenTX): m = "Transaction leaves ≈{} {} and {} {} in the sender's account" return m.format( change_amt.hl(), g.coin, token_bal.hl(), g.dcoin ) - def get_change_amt(self): # here we know the fee - eth_bal = self.twuo.wallet.get_eth_balance(self.inputs[0].addr) + async def get_change_amt(self): # here we know the fee + eth_bal = await self.tw.get_eth_balance(self.inputs[0].addr) return eth_bal - self.fee - def resolve_g_token_from_tx_file(self): - g.dcoin = self.dcoin - if is_hex_str(self.hex): return # for txsend we can leave g.token uninitialized - d = json.loads(self.hex) - if g.token.upper() == self.dcoin: - g.token = d['token_addr'] - elif g.token != d['token_addr']: - m1 = "'{p}': invalid --token parameter for {t} {n} token transaction file\n" - m2 = "Please use '--token={t}'" - die(1,(m1+m2).format(p=g.token,t=self.dcoin,n=capfirst(g.proto.name))) + def resolve_g_token_from_txfile(self): + pass - def make_txobj(self): # called by create_raw() - super().make_txobj() - t = self.token_obj + async def make_txobj(self): # called by create_raw() + await super().make_txobj() + t = Token(self.tw.token,self.tw.decimals) o = self.txobj o['token_addr'] = t.addr - o['decimals'] = t.decimals() + o['decimals'] = t.decimals o['token_to'] = o['to'] o['data'] = t.create_data(o['token_to'],o['amt']) - def check_txfile_hex_data(self): - d = super().check_txfile_hex_data() + def parse_txfile_hex_data(self): + d = EthereumMMGenTX.parse_txfile_hex_data(self) o = self.txobj - - if self.check_sigs(): # online, from rlp and wallet - o['token_addr'] = TokenAddr(o['to']) - o['decimals'] = self.decimals - else: # offline, from json - o['token_addr'] = TokenAddr(d['token_addr']) - o['decimals'] = Int(d['decimals']) - - from .contract import Token - t = self.token_obj = Token(o['token_addr'],o['decimals']) - - if self.check_sigs(): # online, from rlp - 'amt' was eth amt, now token amt - o['amt'] = t.transferdata2amt(o['data']) - else: # offline, from json - 'amt' is token amt - o['data'] = t.create_data(o['to'],o['amt']) - + assert self.tw.token == o['to'] + o['token_addr'] = TokenAddr(o['to']) + o['decimals'] = self.tw.decimals + t = Token(o['token_addr'],o['decimals']) + o['amt'] = t.transferdata2amt(o['data']) o['token_to'] = type(t).transferdata2sendaddr(o['data']) def format_view_body(self,*args,**kwargs): @@ -505,25 +493,47 @@ class EthereumTokenMMGenTX(EthereumMMGenTX): c=blue('(' + g.dcoin + ')'), r=super().format_view_body(*args,**kwargs)) - def do_sign(self,wif,tx_num_str): +class EthereumTokenMMGenTxForSigning(EthereumTokenMMGenTX,EthereumMMGenTxForSigning): + + def resolve_g_token_from_txfile(self): + d = json.loads(self.hex) + if g.token.upper() == self.dcoin: + g.token = d['token_addr'] + elif g.token != d['token_addr']: + m1 = "'{p}': invalid --token parameter for {t} {n} token transaction file\n" + m2 = "Please use '--token={t}'" + die(1,(m1+m2).format(p=g.token,t=self.dcoin,n=capfirst(g.proto.name))) + + def parse_txfile_hex_data(self): + d = EthereumMMGenTxForSigning.parse_txfile_hex_data(self) o = self.txobj - t = self.token_obj + o['token_addr'] = TokenAddr(d['token_addr']) + o['decimals'] = Int(d['decimals']) + t = Token(o['token_addr'],o['decimals']) + o['data'] = t.create_data(o['to'],o['amt']) + o['token_to'] = type(t).transferdata2sendaddr(o['data']) + + async def do_sign(self,wif,tx_num_str): + o = self.txobj + t = Token(o['token_addr'],o['decimals']) tx_in = t.make_tx_in(o['from'],o['to'],o['amt'],self.start_gas,o['gasPrice'],nonce=o['nonce']) - (self.hex,self.coin_txid) = t.txsign(tx_in,wif,o['from'],chain_id=o['chainId']) + (self.hex,self.coin_txid) = await t.txsign(tx_in,wif,o['from'],chain_id=o['chainId']) assert self.check_sigs(),'Signature check failed' -class EthereumMMGenBumpTX(EthereumMMGenTX,MMGenBumpTX): +class EthereumMMGenBumpTX(MMGenBumpTX,EthereumMMGenTxForSigning): - def choose_output(self): pass - - def set_min_fee(self): - self.min_fee = ETHAmt(self.fee * Decimal('1.101')) + @property + def min_fee(self): + return ETHAmt(self.fee * Decimal('1.101')) def update_fee(self,foo,fee): self.fee = fee - def get_nonce(self): + async def get_nonce(self): return self.txobj['nonce'] -class EthereumTokenMMGenBumpTX(EthereumTokenMMGenTX,EthereumMMGenBumpTX): pass -class EthereumMMGenSplitTX(MMGenSplitTX): pass +class EthereumTokenMMGenBumpTX(EthereumMMGenBumpTX,EthereumTokenMMGenTxForSigning): + pass + +class EthereumMMGenSplitTX(MMGenSplitTX): + pass diff --git a/mmgen/daemon.py b/mmgen/daemon.py index 1f13f3b5..2ab3f77e 100755 --- a/mmgen/daemon.py +++ b/mmgen/daemon.py @@ -125,7 +125,7 @@ class Daemon(MMGenObject): self.wait_for_state('stopped') os.makedirs(self.datadir,exist_ok=True) - if self.cfg_file: + if self.cfg_file and not 'keep_cfg_file' in self.flags: open('{}/{}'.format(self.datadir,self.cfg_file),'w').write(self.cfg_file_hdr) if self.use_pidfile and os.path.exists(self.pidfile): @@ -221,7 +221,7 @@ class MoneroWalletDaemon(Daemon): exec_fn_mswin = 'monero-wallet-rpc.exe' ps_pid_mswin = True - def __init__(self,wallet_dir,test_suite=False): + def __init__(self,wallet_dir,test_suite=False,host=None,user=None,passwd=None): self.platform = g.platform self.wallet_dir = wallet_dir if test_suite: @@ -237,7 +237,13 @@ class MoneroWalletDaemon(Daemon): if self.platform == 'win': self.use_pidfile = False - if not g.monero_wallet_rpc_password: + self.host = host or g.monero_wallet_rpc_host + self.user = user or g.monero_wallet_rpc_user + self.passwd = passwd or g.monero_wallet_rpc_password + + assert self.host + assert self.user + if not self.passwd: die(1, 'You must set your Monero wallet RPC password.\n' + 'This can be done on the command line, with the --monero-wallet-rpc-password\n' + @@ -252,7 +258,7 @@ class MoneroWalletDaemon(Daemon): '--rpc-bind-port={}'.format(self.rpc_port), '--wallet-dir='+self.wallet_dir, '--log-file='+self.logfile, - '--rpc-login={}:{}'.format(g.monero_wallet_rpc_user,g.monero_wallet_rpc_password) ] + '--rpc-login={}:{}'.format(self.user,self.passwd) ] if self.platform == 'linux': cmd += ['--pidfile={}'.format(self.pidfile)] cmd += [] if 'no_daemonize' in self.flags else ['--detach'] @@ -260,15 +266,16 @@ class MoneroWalletDaemon(Daemon): @property def state(self): - if not self.test_socket(g.monero_wallet_rpc_host,self.rpc_port): + return 'ready' if self.test_socket('localhost',self.rpc_port) else 'stopped' + if not self.test_socket(self.host,self.rpc_port): return 'stopped' - from .rpc import MoneroWalletRPCConnection + from .rpc import MoneroWalletRPCClient try: - MoneroWalletRPCConnection( - g.monero_wallet_rpc_host, + MoneroWalletRPCClient( + self.host, self.rpc_port, - g.monero_wallet_rpc_user, - g.monero_wallet_rpc_password).get_version() + self.user, + self.passwd).call('get_version') return 'ready' except: return 'stopped' @@ -280,7 +287,7 @@ class MoneroWalletDaemon(Daemon): class CoinDaemon(Daemon): cfg_file_hdr = '' subclasses_must_implement = ('state','stop_cmd') - avail_flags = ('no_daemonize',) + avail_flags = ('no_daemonize','keep_cfg_file') network_ids = ('btc','btc_tn','btc_rt','bch','bch_tn','bch_rt','ltc','ltc_tn','ltc_rt','xmr','eth','etc') @@ -466,6 +473,7 @@ class MoneroDaemon(CoinDaemon): exec_fn_mswin = 'monerod.exe' ps_pid_mswin = True new_console_mswin = True + host = 'localhost' # FIXME def subclass_init(self): if self.platform == 'win': @@ -488,7 +496,7 @@ class MoneroDaemon(CoinDaemon): @property def state(self): - if not self.test_socket(g.monero_wallet_rpc_host,self.rpc_port): + if not self.test_socket(self.host,self.rpc_port): return 'stopped' cp = self.run_cmd( [self.coind_exec] @@ -532,16 +540,23 @@ class EthereumDaemon(CoinDaemon): @property def state(self): - from .rpc import EthereumRPCConnection + return 'ready' if self.test_socket('localhost',self.rpc_port) else 'stopped' + + # the following code does not work + from mmgen.protocol import init_coin + init_coin('eth') + + async def do(): + print(g.rpc) + ret = await g.rpc.call('eth_chainId') + print(ret) + return ('stopped','ready')[ret == '0x11'] + try: - conn = EthereumRPCConnection('localhost',self.rpc_port,socket_timeout=0.2) - except: + return run_session(do()) # socket exception is not propagated + except:# SocketError: return 'stopped' - ret = conn.eth_chainId(on_fail='return') - - return ('stopped','ready')[ret == '0x11'] - @property def stop_cmd(self): return ['kill','-Wf',self.pid] if self.platform == 'win' else ['kill',self.pid] diff --git a/mmgen/exception.py b/mmgen/exception.py index 8587949f..d6fb4a5c 100755 --- a/mmgen/exception.py +++ b/mmgen/exception.py @@ -33,6 +33,7 @@ class FileNotFound(Exception): mmcode = 1 class InvalidPasswdFormat(Exception): mmcode = 1 class CfgFileParseError(Exception): mmcode = 1 class UserOptError(Exception): mmcode = 1 +class NoLEDSupport(Exception): mmcode = 1 # 2: yellow hl, message only class InvalidTokenAddress(Exception): mmcode = 2 diff --git a/mmgen/globalvars.py b/mmgen/globalvars.py index 8fea2d35..42023e3c 100755 --- a/mmgen/globalvars.py +++ b/mmgen/globalvars.py @@ -47,7 +47,7 @@ class g(object): # Constants: version = '0.12.099' - release_date = 'March 2020' + release_date = 'May 2020' proj_name = 'MMGen' proj_url = 'https://github.com/mmgen/mmgen' @@ -95,7 +95,7 @@ class g(object): accept_defaults = False use_internal_keccak_module = False - chain = None # set by first call to rpc_init() + chain = None chains = ('mainnet','testnet','regtest') # rpc: @@ -107,7 +107,8 @@ class g(object): monero_wallet_rpc_user = 'monero' monero_wallet_rpc_password = '' rpc_fail_on_command = '' - rpc = None # global RPC handle + rpc = None # global RPC handle + aiohttp_rpc_queue_len = 16 use_cached_balances = False # regtest: @@ -155,7 +156,7 @@ class g(object): # 'long' opts - opt sets global var common_opts = ( 'color','no_license','testnet', - 'rpc_host','rpc_port','rpc_user','rpc_password', + 'rpc_host','rpc_port','rpc_user','rpc_password','rpc_backend','aiohttp_rpc_queue_len', 'monero_wallet_rpc_host','monero_wallet_rpc_user','monero_wallet_rpc_password', 'daemon_data_dir','force_256_color','regtest','coin','bob','alice', 'accept_defaults','token' @@ -210,6 +211,7 @@ class g(object): 'MMGEN_TESTNET', 'MMGEN_REGTEST', 'MMGEN_TRACEBACK', + 'MMGEN_RPC_BACKEND', 'MMGEN_USE_STANDALONE_SCRYPT_MODULE', 'MMGEN_DISABLE_COLOR', @@ -223,12 +225,15 @@ class g(object): 'comment_file', 'contract_data', ) - # Auto-typechecked and auto-set opts - incompatible with global_sets_opt and opt_sets_global + # Auto-typechecked and auto-set opts. These have no corresponding value in g. # First value in list is the default ov = namedtuple('autoset_opt_info',['type','choices']) autoset_opts = { - 'fee_estimate_mode': ov('nocase_pfx', ('conservative','economical')), + 'fee_estimate_mode': ov('nocase_pfx', ['conservative','economical']), + 'rpc_backend': ov('nocase_pfx', ['auto','httplib','curl','aiohttp','requests']), } + if platform == 'win': + autoset_opts['rpc_backend'].choices.remove('aiohttp') min_screen_width = 80 minconf = 1 diff --git a/mmgen/main_addrimport.py b/mmgen/main_addrimport.py index f740ce2f..0ebd4eed 100755 --- a/mmgen/main_addrimport.py +++ b/mmgen/main_addrimport.py @@ -71,119 +71,126 @@ The --batch and --rescan options cannot be used together. } } -cmd_args = opts.init(opts_data) +def parse_cmd_args(cmd_args): -def import_mmgen_list(infile): - al = (AddrList,KeyAddrList)[bool(opt.keyaddr_file)](infile) - if al.al_id.mmtype in ('S','B'): - from .tx import segwit_is_active - if not segwit_is_active(): - rdie(2,'Segwit is not active on this chain. Cannot import Segwit addresses') - return al + def import_mmgen_list(infile): + al = (AddrList,KeyAddrList)[bool(opt.keyaddr_file)](infile) + if al.al_id.mmtype in ('S','B'): + from .tx import segwit_is_active + if not segwit_is_active(): + rdie(2,'Segwit is not active on this chain. Cannot import Segwit addresses') + return al -if len(cmd_args) == 1: - infile = cmd_args[0] - check_infile(infile) - if opt.addrlist: - al = AddrList(addrlist=get_lines_from_file( - infile, - 'non-{pnm} addresses'.format(pnm=g.proj_name), - trim_comments=True)) + if len(cmd_args) == 1: + infile = cmd_args[0] + check_infile(infile) + if opt.addrlist: + al = AddrList(addrlist=get_lines_from_file( + infile, + 'non-{pnm} addresses'.format(pnm=g.proj_name), + trim_comments=True)) + else: + al = import_mmgen_list(infile) + elif len(cmd_args) == 0 and opt.address: + al = AddrList(addrlist=[opt.address]) + infile = 'command line' else: - al = import_mmgen_list(infile) -elif len(cmd_args) == 0 and opt.address: - al = AddrList(addrlist=[opt.address]) - infile = 'command line' -else: - die(1,ai_msgs('bad_args')) + die(1,ai_msgs('bad_args')) -m = ' from Seed ID {}'.format(al.al_id.sid) if hasattr(al.al_id,'sid') else '' -qmsg('OK. {} addresses{}'.format(al.num_addrs,m)) + return al,infile -err_msg = None +def check_opts(tw): + batch = bool(opt.batch) + rescan = bool(opt.rescan) -from .tw import TrackingWallet -tw = TrackingWallet(mode='w') + if rescan and not 'rescan' in tw.caps: + msg("'--rescan' ignored: not supported by {}".format(type(tw).__name__)) + rescan = False -if g.token: - if not is_coin_addr(g.token): - m = "When importing addresses for a new token, the token must be specified by address, not symbol." - raise InvalidTokenAddress('{!r}: invalid token address\n{}'.format(m)) - sym = tw.addr2sym(g.token) # check for presence in wallet or blockchain; raises exception on failure + if rescan and not opt.quiet: + confirm_or_raise(ai_msgs('rescan'),'continue',expect='YES') -if opt.rescan and not 'rescan' in tw.caps: - msg("'--rescan' ignored: not supported by {}".format(type(tw).__name__)) - opt.rescan = False + if batch and not 'batch' in tw.caps: + msg("'--batch' ignored: not supported by {}".format(type(tw).__name__)) + batch = False -if opt.rescan and not opt.quiet: - confirm_or_raise(ai_msgs('rescan'),'continue',expect='YES') + return batch,rescan -if opt.batch and not 'batch' in tw.caps: - msg("'--batch' ignored: not supported by {}".format(type(tw).__name__)) - opt.batch = False - -def import_address(addr,label,rescan): - try: tw.import_address(addr,label,rescan) +async def import_addr(tw,addr,label,rescan,msg_fmt,msg_args): + try: + task = asyncio.ensure_future(tw.import_address(addr,label,rescan)) # Python 3.7+: create_task() + if rescan: + start = time.time() + while True: + if task.done(): + break + msg_r(('\r{} '+msg_fmt).format(secs_to_hms(int(time.time()-start)),*msg_args)) + await asyncio.sleep(0.5) + await task + msg('\nOK') + else: + await task + qmsg(msg_fmt.format(*msg_args) + ' - OK') except Exception as e: - global err_msg - err_msg = e.args[0] - if g.token and not tw.get_token_param(g.token,'symbol'): - tw.set_token_param(g.token,'symbol',sym) - tw.set_token_param(g.token,'decimals',tw.token_obj.decimals()) + die(2,'\nImport of address {!r} failed: {!r}'.format(addr,e.args[0])) -w_n_of_m = len(str(al.num_addrs)) * 2 + 2 -w_mmid = 1 if opt.addrlist or opt.address else len(str(max(al.idxs()))) + 13 -msg_fmt = '{{:{}}} {{:34}} {{:{}}}'.format(w_n_of_m,w_mmid) +def make_args_list(tw,al,batch,rescan): -if opt.rescan: import threading + fs = '{:%s} {:34} {:%s}' % ( + len(str(al.num_addrs)) * 2 + 2, + 1 if opt.addrlist or opt.address else len(str(max(al.idxs()))) + 13 ) -fs = 'Importing {} address{} from {}{}' -bm =' (batch mode)' if opt.batch else '' -msg(fs.format(len(al.data),suf(al.data,'es'),infile,bm)) + for num,e in enumerate(al.data,1): + if e.idx: + label = '{}:{}'.format(al.al_id,e.idx) + (' ' + e.label if e.label else '') + add_msg = label + else: + label = '{}:{}'.format(g.proto.base_coin.lower(),e.addr) + add_msg = 'non-'+g.proj_name -if not al.data[0].addr.is_for_chain(g.chain): - die(2,'Address{} not compatible with {} chain!'.format((' list','')[bool(opt.address)],g.chain)) + if batch: + yield (e.addr,TwLabel(label),False) + else: + msg_args = ( f'{num}/{al.num_addrs}:', e.addr, '('+add_msg+')' ) + yield (tw,e.addr,TwLabel(label),rescan,fs,msg_args) -for n,e in enumerate(al.data): - if e.idx: - label = '{}:{}'.format(al.al_id,e.idx) - if e.label: label += ' ' + e.label - m = label +async def main(): + al,infile = parse_cmd_args(cmd_args) + + qmsg( + f'OK. {al.num_addrs} addresses' + + (f' from Seed ID {al.al_id.sid}' if hasattr(al.al_id,'sid') else '') ) + + msg( + f'Importing {len(al.data)} address{suf(al.data,"es")} from {infile}' + + (' (batch mode)' if opt.batch else '') ) + + if not al.data[0].addr.is_for_chain(g.chain): + die(2,f'Address{(" list","")[bool(opt.address)]} incompatible with {g.chain} chain!') + + from .tw import TrackingWallet + tw = await TrackingWallet(mode='i') + + batch,rescan = check_opts(tw) + + if g.token: + await tw.import_token() + + args_list = make_args_list(tw,al,batch,rescan) + + if batch: + ret = await tw.batch_import_address(list(args_list)) + msg(f'OK: {len(ret)} addresses imported') + elif rescan: + for arg_list in args_list: + await import_addr(*arg_list) else: - label = '{}:{}'.format(g.proto.base_coin.lower(),e.addr) - m = 'non-'+g.proj_name + tasks = [import_addr(*arg_list) for arg_list in args_list] + await asyncio.gather(*tasks) + msg('OK') - label = TwLabel(label) + del tw - if opt.batch: - if n == 0: arg_list = [] - arg_list.append((e.addr,label,False)) - continue - - msg_data = ('{}/{}:'.format(n+1,al.num_addrs),e.addr,'({})'.format(m)) - - if opt.rescan: - t = threading.Thread(target=import_address,args=[e.addr,label,True]) - t.daemon = True - t.start() - start = int(time.time()) - while True: - if t.is_alive(): - elapsed = int(time.time()-start) - msg_r(('\r{} '+msg_fmt).format(secs_to_hms(elapsed),*msg_data)) - time.sleep(0.5) - else: - if err_msg: die(2,'\nImport failed: {!r}'.format(err_msg)) - msg('\nOK') - break - else: - import_address(e.addr,label,False) - msg_r('\r'+msg_fmt.format(*msg_data)) - if err_msg: die(2,'\nImport failed: {!r}'.format(err_msg)) - msg(' - OK') - -if opt.batch: - ret = tw.batch_import_address(arg_list) - msg('OK: {} addresses imported'.format(len(ret))) - -del tw +cmd_args = opts.init(opts_data) +import asyncio +run_session(main()) diff --git a/mmgen/main_autosign.py b/mmgen/main_autosign.py index 24748461..412de947 100755 --- a/mmgen/main_autosign.py +++ b/mmgen/main_autosign.py @@ -112,18 +112,19 @@ cmd_args = opts.init(opts_data,add_opts=['mmgen_keys_from_file','in_fmt']) exit_if_mswin('autosigning') import mmgen.tx -import mmgen.altcoins.eth.tx from .txsign import txsign from .protocol import CoinProtocol,init_coin +from .rpc import rpc_init + if g.test_suite: from .daemon import CoinDaemon if opt.mountpoint: - mountpoint = opt.mountpoint # TODO: make global + mountpoint = opt.mountpoint opt.outdir = tx_dir = os.path.join(mountpoint,'tx') -def check_daemons_running(): +async def check_daemons_running(): if opt.coin: die(1,'--coin option not supported with this command. Use --coins instead') if opt.coins: @@ -140,7 +141,7 @@ def check_daemons_running(): g.rpc_port = CoinDaemon(get_network_id(coin,g.testnet),test_suite=True).rpc_port vmsg(f'Checking {coin} daemon') try: - rpc_init(reinit=True) + await rpc_init() except SystemExit as e: if e.code != 0: ydie(1,f'{coin} daemon not running or not listening on port {g.proto.rpc_port}') @@ -174,7 +175,7 @@ def do_umount(): msg(f'Unmounting {mountpoint}') run(['umount',mountpoint],check=True) -def sign_tx_file(txfile,signed_txs): +async def sign_tx_file(txfile,signed_txs): try: init_coin('BTC',testnet=False) tmp_tx = mmgen.tx.MMGenTX(txfile,metadata_only=True) @@ -193,15 +194,15 @@ def sign_tx_file(txfile,signed_txs): g.token = tmp_tx.dcoin g.dcoin = tmp_tx.dcoin or g.coin - tx = mmgen.tx.MMGenTX(txfile,offline=True) + tx = mmgen.tx.MMGenTxForSigning(txfile) if g.proto.sign_mode == 'daemon': if g.test_suite: g.proto.daemon_data_dir = 'test/daemons/' + g.coin.lower() g.rpc_port = CoinDaemon(get_network_id(g.coin,g.testnet),test_suite=True).rpc_port - rpc_init(reinit=True) + await rpc_init() - if txsign(tx,wfs,None,None): + if await txsign(tx,wfs,None,None): tx.write_to_file(ask_write=False) signed_txs.append(tx) return True @@ -215,7 +216,7 @@ def sign_tx_file(txfile,signed_txs): except: return False -def sign(): +async def sign(): dirlist = os.listdir(tx_dir) raw,signed = [set(f[:-6] for f in dirlist if f.endswith(ext)) for ext in ('.rawtx','.sigtx')] unsigned = [os.path.join(tx_dir,f+'.rawtx') for f in raw - signed] @@ -223,7 +224,7 @@ def sign(): if unsigned: signed_txs,fails = [],[] for txfile in unsigned: - ret = sign_tx_file(txfile,signed_txs) + ret = await sign_tx_file(txfile,signed_txs) if not ret: fails.append(txfile) qmsg('') @@ -296,23 +297,23 @@ def print_summary(signed_txs): else: msg('No non-MMGen outputs') -def do_sign(): +async def do_sign(): if not opt.stealth_led: - set_led('busy') + led.set('busy') do_mount() key_ok = decrypt_wallets() if key_ok: if opt.stealth_led: - set_led('busy') - ret = sign() + led.set('busy') + ret = await sign() do_umount() - set_led(('standby','off','error')[(not ret)*2 or bool(opt.stealth_led)]) + led.set(('standby','off','error')[(not ret)*2 or bool(opt.stealth_led)]) return ret else: msg('Password is incorrect!') do_umount() if not opt.stealth_led: - set_led('error') + led.set('error') return False def wipe_existing_key(): @@ -374,35 +375,6 @@ def setup(): ss_out = Wallet(ss=ss_in) ss_out.write_to_file(desc='autosign wallet',outdir=wallet_dir) -def ev_sleep(secs): - ev.wait(secs) - return ev.isSet() - -def do_led(on,off): - if not on: - open(status_ctl,'w').write('0\n') - while True: - if ev_sleep(3600): return - - while True: - for s_time,val in ((on,255),(off,0)): - open(status_ctl,'w').write('{}\n'.format(val)) - if ev_sleep(s_time): return - -def set_led(cmd): - if not opt.led: return - vmsg("Setting LED state to '{}'".format(cmd)) - timings = { - 'off': ( 0, 0 ), - 'standby': ( 2.2, 0.2 ), - 'busy': ( 0.06, 0.06 ), - 'error': ( 0.5, 0.5 )}[cmd] - global led_thread - if led_thread: - ev.set(); led_thread.join(); ev.clear() - led_thread = threading.Thread(target=do_led,name='LED loop',args=timings) - led_thread.start() - def get_insert_status(): if opt.no_insert_check: return True @@ -410,15 +382,21 @@ def get_insert_status(): except: return False else: return True -def do_loop(): +def check_wipe_present(): + try: + run(['wipe','-v'],stdout=DEVNULL,stderr=DEVNULL,check=True) + except: + die(2,"The 'wipe' utility must be installed before running this program") + +async def do_loop(): n,prev_status = 0,False if not opt.stealth_led: - set_led('standby') + led.set('standby') while True: status = get_insert_status() if status and not prev_status: msg('Device insertion detected') - do_sign() + await do_sign() prev_status = status if not n % 10: msg_r('\r{}\rWaiting'.format(' '*17)) @@ -427,54 +405,6 @@ def do_loop(): msg_r('.') n += 1 -def check_access(fn,desc='status LED control',init_val=None): - try: - b = open(fn).read().strip() - open(fn,'w').write('{}\n'.format(init_val or b)) - return True - except: - m1 = "You do not have access to the {} file\n".format(desc) - m2 = "To allow access, run 'sudo chmod 0666 {}'".format(fn) - msg(m1+m2) - return False - -def check_wipe_present(): - try: - run(['wipe','-v'],stdout=DEVNULL,stderr=DEVNULL,check=True) - except: - die(2,"The 'wipe' utility must be installed before running this program") - -def init_led(): - sc = { - 'opi': '/sys/class/leds/orangepi:red:status/brightness', - 'rpi': '/sys/class/leds/led0/brightness' - } - tc = { - 'rpi': '/sys/class/leds/led0/trigger', # mmc,none - } - for k in ('opi','rpi'): - try: os.stat(sc[k]) - except: pass - else: - board = k - break - else: - die(2,'Control files not found! LED option not supported') - - status_ctl = sc[board] - trigger_ctl = tc[board] if board in tc else None - - if not check_access(status_ctl) or ( - trigger_ctl and not check_access(trigger_ctl,desc='LED trigger',init_val='none') - ): - sys.exit(1) - - if trigger_ctl: - open(trigger_ctl,'w').write('none\n') - - return status_ctl,trigger_ctl - -# main() if len(cmd_args) not in (0,1): opts.usage() @@ -489,32 +419,29 @@ if len(cmd_args) == 1: check_wipe_present() wfs = get_wallet_files() -check_daemons_running() - -def at_exit(exit_val,nl=False): - if nl: msg('') - msg('Cleaning up...') - if opt.led: - set_led('off') - ev.set() - led_thread.join() - if trigger_ctl: - open(trigger_ctl,'w').write('mmc0\n') +def at_exit(exit_val,message='\nCleaning up...'): + if message: + msg(message) + led.stop() sys.exit(exit_val) -def handler(a,b): at_exit(1,nl=True) +def handler(a,b): + at_exit(1) signal.signal(signal.SIGTERM,handler) signal.signal(signal.SIGINT,handler) -if opt.led: - import threading - status_ctl,trigger_ctl = init_led() - ev = threading.Event() - led_thread = None +from .led import LEDControl +led = LEDControl(enabled=opt.led,simulate=g.test_suite and not os.getenv('MMGEN_TEST_SUITE_AUTOSIGN_LIVE')) +led.set('off') -if len(cmd_args) == 0: - ret = do_sign() - at_exit(int(not ret)) -elif cmd_args[0] == 'wait': - do_loop() +async def main(): + await check_daemons_running() + + if len(cmd_args) == 0: + ret = await do_sign() + at_exit(int(not ret),message='') + elif cmd_args[0] == 'wait': + await do_loop() + +run_session(main(),do_rpc_init=False) diff --git a/mmgen/main_split.py b/mmgen/main_split.py index 8058f61d..059e867e 100755 --- a/mmgen/main_split.py +++ b/mmgen/main_split.py @@ -20,6 +20,7 @@ """ mmgen-split: Split funds after a replayable chain fork using a timelocked transaction + UNMAINTAINED """ import time @@ -115,28 +116,27 @@ if opt.tx_fees: opt.tx_fee = opt.tx_fees.split(',')[idx] opts.opt_is_tx_fee('foo',opt.tx_fee,'transaction fee') # raises exception on error -rpc_init(reinit=True) - tx1 = MMGenSplitTX() opt.no_blank = True -gmsg("Creating timelocked transaction for long chain ({})".format(g.coin)) -locktime = int(opt.locktime or 0) or g.rpc.getblockcount() -tx1.create(mmids[0],locktime) +async def main(): + gmsg("Creating timelocked transaction for long chain ({})".format(g.coin)) + locktime = int(opt.locktime or 0) or await g.rpc.call('getblockcount') + tx1.create(mmids[0],locktime) -tx1.format() -tx1.create_fn() + tx1.format() + tx1.create_fn() -gmsg("\nCreating transaction for short chain ({})".format(opt.other_coin)) + gmsg("\nCreating transaction for short chain ({})".format(opt.other_coin)) -init_coin(opt.other_coin) + init_coin(opt.other_coin) -tx2 = MMGenSplitTX() -tx2.inputs = tx1.inputs -tx2.inputs.convert_coin() + tx2 = MMGenSplitTX() + tx2.inputs = tx1.inputs + tx2.inputs.convert_coin() -tx2.create_split(mmids[1]) + tx2.create_split(mmids[1]) -for tx,desc in ((tx1,'Long chain (timelocked)'),(tx2,'Short chain')): - tx.desc = desc + ' transaction' - tx.write_to_file(ask_write=False,ask_overwrite=not opt.yes,ask_write_default_yes=False) + for tx,desc in ((tx1,'Long chain (timelocked)'),(tx2,'Short chain')): + tx.desc = desc + ' transaction' + tx.write_to_file(ask_write=False,ask_overwrite=not opt.yes,ask_write_default_yes=False) diff --git a/mmgen/main_tool.py b/mmgen/main_tool.py index d2599202..6aa925d5 100755 --- a/mmgen/main_tool.py +++ b/mmgen/main_tool.py @@ -110,4 +110,7 @@ args,kwargs = tool._process_args(cmd,cmd_args) ret = tool.MMGenToolCmds.call(cmd,*args,**kwargs) +if type(ret).__name__ == 'coroutine': + ret = run_session(ret) + tool._process_result(ret,pager='pager' in kwargs and kwargs['pager'],print_result=True) diff --git a/mmgen/main_txbump.py b/mmgen/main_txbump.py index 8061a56a..65eabd2f 100755 --- a/mmgen/main_txbump.py +++ b/mmgen/main_txbump.py @@ -96,8 +96,6 @@ column below: cmd_args = opts.init(opts_data) -rpc_init() - tx_file = cmd_args.pop(0) check_infile(tx_file) @@ -109,61 +107,62 @@ seed_files = get_seed_files(opt,cmd_args) if (cmd_args or opt.send) else None kal = get_keyaddrlist(opt) kl = get_keylist(opt) -tx = MMGenBumpTX(filename=tx_file,send=(seed_files or kl or kal)) +sign_and_send = bool(seed_files or kl or kal) do_license_msg() silent = opt.yes and opt.tx_fee != None and opt.output_to_reduce != None -if not silent: - msg(green('ORIGINAL TRANSACTION')) - msg(tx.format_view(terse=True)) +async def main(): -tx.set_min_fee() + from .tw import TrackingWallet + tx = MMGenBumpTX(filename=tx_file,send=sign_and_send,tw=await TrackingWallet() if g.token else None) -tx.check_bumpable() + if not silent: + msg(green('ORIGINAL TRANSACTION')) + msg(tx.format_view(terse=True)) -msg('Creating new transaction') + tx.check_bumpable() # needs cached networkinfo['relayfee'] -op_idx = tx.choose_output() + msg('Creating new transaction') -if not silent: - msg('Minimum fee for new transaction: {} {}'.format(tx.min_fee.hl(),g.coin)) + op_idx = tx.choose_output() -fee = tx.get_usr_fee_interactive(tx_fee=opt.tx_fee,desc='User-selected') + if not silent: + msg('Minimum fee for new transaction: {} {}'.format(tx.min_fee.hl(),g.coin)) -tx.update_fee(op_idx,fee) + fee = tx.get_usr_fee_interactive(tx_fee=opt.tx_fee,desc='User-selected') -d = tx.get_fee_from_tx() -assert d == fee and d <= g.proto.max_tx_fee + tx.update_fee(op_idx,fee) -if g.proto.base_proto == 'Bitcoin': - tx.outputs.sort_bip69() # output amts have changed, so re-sort + d = tx.get_fee_from_tx() + assert d == fee and d <= g.proto.max_tx_fee -if not opt.yes: - tx.add_comment() # edits an existing comment + if g.proto.base_proto == 'Bitcoin': + tx.outputs.sort_bip69() # output amts have changed, so re-sort -from .tw import TwUnspentOutputs -tx.twuo = TwUnspentOutputs(minconf=opt.minconf) + if not opt.yes: + tx.add_comment() # edits an existing comment -tx.create_raw() # creates tx.hex, tx.txid -tx.add_timestamp() -tx.add_blockcount() + await tx.create_raw() # creates tx.hex, tx.txid -qmsg('Fee successfully increased') + tx.add_timestamp() + tx.add_blockcount() -if not silent: - msg(green('\nREPLACEMENT TRANSACTION:')) - msg_r(tx.format_view(terse=True)) + qmsg('Fee successfully increased') -del tx.twuo.wallet + if not silent: + msg(green('\nREPLACEMENT TRANSACTION:')) + msg_r(tx.format_view(terse=True)) -if seed_files or kl or kal: - if txsign(tx,seed_files,kl,kal): - tx.write_to_file(ask_write=False) - tx.send(exit_on_fail=True) - tx.write_to_file(ask_write=False) + if sign_and_send: + if await txsign(tx,seed_files,kl,kal): + tx.write_to_file(ask_write=False) + await tx.send(exit_on_fail=True) + tx.write_to_file(ask_write=False) + else: + die(2,'Transaction could not be signed') else: - die(2,'Transaction could not be signed') -else: - tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=False,ask_overwrite=not opt.yes) + tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=False,ask_overwrite=not opt.yes) + +run_session(main()) diff --git a/mmgen/main_txcreate.py b/mmgen/main_txcreate.py index 8badc068..ee36c141 100755 --- a/mmgen/main_txcreate.py +++ b/mmgen/main_txcreate.py @@ -78,9 +78,11 @@ cmd_args = opts.init(opts_data) g.use_cached_balances = opt.cached_balances -rpc_init() +async def main(): + from .tx import MMGenTX + from .tw import TrackingWallet + tx = MMGenTX(tw=await TrackingWallet() if g.token else None) + await tx.create(cmd_args,int(opt.locktime or 0),do_info=opt.info) + tx.write_to_file(ask_write=not opt.yes,ask_overwrite=not opt.yes,ask_write_default_yes=False) -from .tx import MMGenTX -tx = MMGenTX() -tx.create(cmd_args,int(opt.locktime or 0),do_info=opt.info) -tx.write_to_file(ask_write=not opt.yes,ask_overwrite=not opt.yes,ask_write_default_yes=False) +run_session(main()) diff --git a/mmgen/main_txdo.py b/mmgen/main_txdo.py index cff07604..b653144e 100755 --- a/mmgen/main_txdo.py +++ b/mmgen/main_txdo.py @@ -115,8 +115,6 @@ cmd_args = opts.init(opts_data) g.use_cached_balances = opt.cached_balances -rpc_init() - from .tx import * from .txsign import * @@ -124,16 +122,23 @@ seed_files = get_seed_files(opt,cmd_args) kal = get_keyaddrlist(opt) kl = get_keylist(opt) -if kl and kal: kl.remove_dup_keys(kal) +if kl and kal: + kl.remove_dup_keys(kal) -tx = MMGenTX(caller='txdo') +async def main(): + from .tw import TrackingWallet + tx1 = MMGenTX(caller='txdo',tw=await TrackingWallet() if g.token else None) -tx.create(cmd_args,int(opt.locktime or 0)) + await tx1.create(cmd_args,int(opt.locktime or 0)) -if txsign(tx,seed_files,kl,kal): - tx.write_to_file(ask_write=False) - tx.send(exit_on_fail=True) - tx.write_to_file(ask_overwrite=False,ask_write=False) - tx.print_contract_addr() -else: - die(2,'Transaction could not be signed') + tx2 = MMGenTxForSigning(data=tx1.__dict__) + + if await txsign(tx2,seed_files,kl,kal): + tx2.write_to_file(ask_write=False) + await tx2.send(exit_on_fail=True) + tx2.write_to_file(ask_overwrite=False,ask_write=False) + tx2.print_contract_addr() + else: + die(2,'Transaction could not be signed') + +run_session(main()) diff --git a/mmgen/main_txsend.py b/mmgen/main_txsend.py index 3d06bd74..7ecb7931 100755 --- a/mmgen/main_txsend.py +++ b/mmgen/main_txsend.py @@ -40,8 +40,6 @@ opts_data = { cmd_args = opts.init(opts_data) -rpc_init() - if len(cmd_args) == 1: infile = cmd_args[0]; check_infile(infile) else: @@ -52,22 +50,33 @@ if not opt.status: from .tx import * -tx = MMGenTX(infile,quiet_open=True) # sig check performed here -vmsg("Signed transaction file '{}' is valid".format(infile)) +async def main(): -if not tx.marked_signed(): - die(1,'Transaction is not signed!') + from .tw import TrackingWallet + tx = MMGenTX(infile,quiet_open=True,tw=await TrackingWallet() if g.token else None) -if opt.status: - if tx.coin_txid: qmsg('{} txid: {}'.format(g.coin,tx.coin_txid.hl())) - tx.get_status(status=True) - sys.exit(0) + if g.token: + from .tw import TrackingWallet + tx.tw = await TrackingWallet() -if not opt.yes: - tx.view_with_prompt('View transaction data?') - if tx.add_comment(): # edits an existing comment, returns true if changed - tx.write_to_file(ask_write_default_yes=True) + vmsg("Signed transaction file '{}' is valid".format(infile)) -tx.send(exit_on_fail=True) -tx.write_to_file(ask_overwrite=False,ask_write=False) -tx.print_contract_addr() + if not tx.marked_signed(): + die(1,'Transaction is not signed!') + + if opt.status: + if tx.coin_txid: + qmsg('{} txid: {}'.format(g.coin,tx.coin_txid.hl())) + await tx.get_status(status=True) + sys.exit(0) + + if not opt.yes: + tx.view_with_prompt('View transaction data?') + if tx.add_comment(): # edits an existing comment, returns true if changed + tx.write_to_file(ask_write_default_yes=True) + + await tx.send(exit_on_fail=True) + tx.write_to_file(ask_overwrite=False,ask_write=False) + tx.print_contract_addr() + +run_session(main()) diff --git a/mmgen/main_txsign.py b/mmgen/main_txsign.py index b67b389e..1d5b2676 100755 --- a/mmgen/main_txsign.py +++ b/mmgen/main_txsign.py @@ -97,9 +97,6 @@ if not infiles: for i in infiles: check_infile(i) -if g.proto.sign_mode == 'daemon': - rpc_init() - if not opt.info and not opt.terse_info: do_license_msg(immed=True) @@ -107,39 +104,51 @@ from .txsign import * tx_files = get_tx_files(opt,infiles) seed_files = get_seed_files(opt,infiles) - kal = get_keyaddrlist(opt) kl = get_keylist(opt) -if kl and kal: kl.remove_dup_keys(kal) -tx_num_str,bad_tx_count = '',0 -for tx_num,tx_file in enumerate(tx_files,1): - if len(tx_files) > 1: - msg('\nTransaction #{} of {}:'.format(tx_num,len(tx_files))) - tx_num_str = ' #{}'.format(tx_num) - tx = MMGenTX(tx_file,offline=True) +if kl and kal: + kl.remove_dup_keys(kal) - if tx.marked_signed(): - msg('Transaction is already signed!'); continue +async def main(): + bad_tx_count = 0 + tx_num_disp = '' + for tx_num,tx_file in enumerate(tx_files,1): + if len(tx_files) > 1: + msg('\nTransaction #{} of {}:'.format(tx_num,len(tx_files))) + tx_num_disp = f' #{tx_num}' - vmsg("Successfully opened transaction file '{}'".format(tx_file)) + tx = MMGenTxForSigning(tx_file) - if opt.tx_id: - msg(tx.txid); continue + if tx.marked_signed(): + msg('Transaction is already signed!') + continue - if opt.info or opt.terse_info: - tx.view(pause=False,terse=opt.terse_info); continue + vmsg(f'Successfully opened transaction file {tx_file!r}') - if not opt.yes: - tx.view_with_prompt('View data for transaction{}?'.format(tx_num_str)) + if opt.tx_id: + msg(tx.txid) + continue + + if opt.info or opt.terse_info: + tx.view(pause=False,terse=opt.terse_info) + continue - if txsign(tx,seed_files,kl,kal,tx_num_str): if not opt.yes: - tx.add_comment() # edits an existing comment - tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=True,add_desc=tx_num_str) - else: - ymsg('Transaction could not be signed') - bad_tx_count += 1 + tx.view_with_prompt(f'View data for transaction{tx_num_disp}?') -if bad_tx_count: - ydie(2,'{} transaction{} could not be signed'.format(bad_tx_count,suf(bad_tx_count))) + if await txsign(tx,seed_files,kl,kal,tx_num_disp): + if not opt.yes: + tx.add_comment() # edits an existing comment + tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=True,add_desc=tx_num_disp) + else: + ymsg('Transaction could not be signed') + bad_tx_count += 1 + + if bad_tx_count: + ydie(2,f'{bad_tx_count} transaction{suf(bad_tx_count)} could not be signed') + +run_session( + main(), + do_rpc_init = g.proto.sign_mode == 'daemon' +) diff --git a/mmgen/obj.py b/mmgen/obj.py index 5c33328a..853489d9 100755 --- a/mmgen/obj.py +++ b/mmgen/obj.py @@ -28,6 +28,12 @@ from .exception import * from .globalvars import * from .color import * +class aInitMeta(type): + async def __call__(cls,*args,**kwargs): + instance = super().__call__(*args,**kwargs) + await instance.__ainit__(*args,**kwargs) + return instance + def is_mmgen_seed_id(s): return SeedID(sid=s,on_fail='silent') def is_mmgen_idx(s): return AddrIdx(s,on_fail='silent') def is_mmgen_id(s): return MMGenID(s,on_fail='silent') diff --git a/mmgen/opts.py b/mmgen/opts.py index c7db833f..0e059afc 100755 --- a/mmgen/opts.py +++ b/mmgen/opts.py @@ -110,7 +110,7 @@ def override_globals_from_cfg_file(ucfg): else: die(2,'{!r}: unrecognized option in {!r}, line {}'.format(d.name,ucfg.fn,d.lineno)) -def override_globals_from_env(): +def override_globals_and_set_opts_from_env(opt): for name in g.env_opts: if name == 'MMGEN_DEBUG_ALL': continue @@ -118,7 +118,13 @@ def override_globals_from_env(): val = os.getenv(name) # os.getenv() returns None if env var is unset if val: # exclude empty string values; string value of '0' or 'false' sets variable to False gname = name[(6,14)[disable]:].lower() - setattr(g,gname,set_for_type(val,getattr(g,gname),name,disable)) + if hasattr(g,gname): + setattr(g,gname,set_for_type(val,getattr(g,gname),name,disable)) + elif hasattr(opt,gname): + if getattr(opt,gname) is None: # env must not override cmdline! + setattr(opt,gname,val) + else: + raise ValueError(f'Name {gname} not present in globals or opts') def common_opts_code(s): from .protocol import CoinProtocol @@ -167,6 +173,8 @@ common_opts_data = { --, --rpc-port=p Communicate with {dn} listening on port 'p' --, --rpc-user=user Override 'rpc_user' in mmgen.cfg --, --rpc-password=pass Override 'rpc_password' in mmgen.cfg +--, --rpc-backend=s Override 'rpc_backend' in mmgen.cfg +--, --aiohttp-rpc-queue-len=N Override 'aiohttp_rpc_queue_len' in mmgen.cfg --, --monero-wallet-rpc-host=host Override 'monero_wallet_rpc_host' in mmgen.cfg --, --monero-wallet-rpc-user=user Override 'monero_wallet_rpc_user' in mmgen.cfg --, --monero-wallet-rpc-password=pass Override 'monero_wallet_rpc_password' in mmgen.cfg @@ -232,7 +240,7 @@ def init(opts_data,add_opts=[],opt_filter=None,parse_only=False): cfg_file('sample') # check for changes in system template file override_globals_from_cfg_file(cfg_file('usr')) - override_globals_from_env() + override_globals_and_set_opts_from_env(opt) # Set globals from opts, setting type from original global value # Do here, before opts are set from globals below @@ -240,7 +248,7 @@ def init(opts_data,add_opts=[],opt_filter=None,parse_only=False): for k in (g.common_opts + g.opt_sets_global): if hasattr(opt,k): val = getattr(opt,k) - if val != None: + if val != None and hasattr(g,k): setattr(g,k,set_for_type(val,getattr(g,k),'--'+k)) g.coin = g.coin.upper() # allow user to use lowercase @@ -337,7 +345,7 @@ def opt_is_tx_fee(key,val,desc): # 'key' must remain a placeholder return from .tx import MMGenTX - tx = MMGenTX(offline=True) + tx = MMGenTX() # Size of 224 is just a ball-park figure to eliminate the most extreme cases at startup # This check will be performed again once we know the true size ret = tx.process_fee_spec(val,224,on_fail='return') @@ -466,7 +474,8 @@ def check_usr_opts(usr_opts): # Raises an exception if any check fails opt_compares(val,'<=',g.max_urandchars,desc) def chk_tx_fee(key,val,desc): - opt_is_tx_fee(key,val,desc) + pass +# opt_is_tx_fee(key,val,desc) # TODO: move this check elsewhere def chk_tx_confs(key,val,desc): opt_is_int(val,desc) diff --git a/mmgen/rpc.py b/mmgen/rpc.py index cbc7c6f4..8daa520b 100755 --- a/mmgen/rpc.py +++ b/mmgen/rpc.py @@ -20,217 +20,383 @@ rpc.py: Cryptocoin RPC library for the MMGen suite """ -import http.client,base64,json +import base64,json,asyncio from decimal import Decimal - from .common import * +from .obj import aInitMeta + +rpc_credentials_msg = lambda: '\n'+fmt(f""" + Error: no {g.proto.name.capitalize()} RPC authentication method found + + RPC credentials must be supplied using one of the following methods: + + A) If daemon is local and running as same user as you: + + - no credentials required, or matching rpcuser/rpcpassword and + rpc_user/rpc_password values in {g.proto.name}.conf and mmgen.cfg + + B) If daemon is running remotely or as different user: + + - matching credentials in {g.proto.name}.conf and mmgen.cfg as described above + + The --rpc-user/--rpc-password options may be supplied on the MMGen command line. + They override the corresponding values in mmgen.cfg. Set them to an empty string + to use cookie authentication with a local server when the options are set + in mmgen.cfg. + + For better security, rpcauth should be used in {g.proto.name}.conf instead of + rpcuser/rpcpassword. + +""",strip_char='\t') def dmsg_rpc(fs,data=None,is_json=False): if g.debug_rpc: msg(fs if data == None else fs.format(pp_fmt(json.loads(data) if is_json else data))) -class RPCConnection(MMGenObject): +class json_encoder(json.JSONEncoder): + def default(self,obj): + if isinstance(obj,Decimal): + return str(obj) + else: + return json.JSONEncoder.default(self,obj) - auth = True - db_fs = ' host [{h}] port [{p}] user [{u}] passwd [{pw}] auth_cookie [{c}]\n' - http_hdrs = { 'Content-Type': 'application/json' } +class RPCBackends: - def __init__(self,host=None,port=None,user=None,passwd=None,auth_cookie=None,socket_timeout=1): + class aiohttp: + + def __init__(self,caller): + self.caller = caller + self.session = g.session + self.url = caller.url + self.timeout = caller.timeout + if caller.auth_type == 'basic': + import aiohttp + self.auth = aiohttp.BasicAuth(*caller.auth,encoding='UTF-8') + else: + self.auth = None + + async def run(self,payload,timeout=None): + dmsg_rpc('\n RPC PAYLOAD data (aiohttp) ==>\n{}\n',payload) + async with self.session.post( + url = self.url, + auth = self.auth, + data = json.dumps(payload,cls=json_encoder), + timeout = timeout or self.timeout, + ) as res: + return (await res.text(),res.status) + + class requests: + + def __init__(self,caller): + self.url = caller.url + self.timeout = caller.timeout + import requests,urllib3 + urllib3.disable_warnings() + self.session = requests.Session() + self.session.headers = caller.http_hdrs + if caller.auth_type: + auth = 'HTTP' + caller.auth_type.capitalize() + 'Auth' + self.session.auth = getattr(requests.auth,auth)(*caller.auth) + + async def run(self,payload,timeout=None): + dmsg_rpc('\n RPC PAYLOAD data (requests) ==>\n{}\n',payload) + res = self.session.post( + url = self.url, + data = json.dumps(payload,cls=json_encoder), + timeout = timeout or self.timeout, + verify = False ) + return (res.content,res.status_code) + + class httplib: + + def __init__(self,caller): + import http.client + self.session = http.client.HTTPConnection(caller.host,caller.port,caller.timeout) + self.http_hdrs = caller.http_hdrs + self.host = caller.host + self.port = caller.port + if caller.auth_type == 'basic': + auth_str = f'{caller.auth.user}:{caller.auth.passwd}' + auth_str_b64 = 'Basic ' + base64.b64encode(auth_str.encode()).decode() + self.http_hdrs.update({ 'Host': self.host, 'Authorization': auth_str_b64 }) + fs = ' RPC AUTHORIZATION data ==> raw: [{}]\n{:>31}enc: [{}]\n' + dmsg_rpc(fs.format(auth_str,'',auth_str_b64)) + + async def run(self,payload,timeout=None): + dmsg_rpc('\n RPC PAYLOAD data (httplib) ==>\n{}\n',payload) + if timeout: + import http.client + s = http.client.HTTPConnection(self.host,self.port,timeout) + else: + s = self.session + try: + s.request( + method = 'POST', + url = '/', + body = json.dumps(payload,cls=json_encoder), + headers = self.http_hdrs ) + r = s.getresponse() # => http.client.HTTPResponse instance + except Exception as e: + raise RPCFailure(str(e)) + return (r.read(),r.status) + + class curl: + + def __init__(self,caller): + + def gen(): + for k,v in caller.http_hdrs.items(): + for s in ('--header',f'{k}: {v}'): + yield s + if caller.auth_type: + """ + Authentication with curl is insecure, as it exposes the user's credentials + via the command line. Use for testing only. + """ + for s in ('--user',f'{caller.auth.user}:{caller.auth.passwd}'): + yield s + if caller.auth_type == 'digest': + yield '--digest' + + self.url = caller.url + self.exec_opts = list(gen()) + ['--silent'] + self.arg_max = 8192 # set way below system ARG_MAX, just to be safe + self.timeout = caller.timeout + + async def run(self,payload,timeout=None): + data = json.dumps(payload,cls=json_encoder) + if len(data) > self.arg_max: + return self.httplib(payload,timeout=timeout) + dmsg_rpc('\n RPC PAYLOAD data (curl) ==>\n{}\n',payload) + exec_cmd = [ + 'curl', + '--proxy', '', + '--connect-timeout', str(timeout or self.timeout), + '--request', 'POST', + '--write-out', '%{http_code}', + '--data-binary', data + ] + self.exec_opts + [self.url] + + dmsg_rpc(' RPC curl exec data ==>\n{}\n',exec_cmd) + + from subprocess import run,PIPE + res = run(exec_cmd,stdout=PIPE,check=True).stdout.decode() + # res = run(exec_cmd,stdout=PIPE,check=True,text='UTF-8').stdout # Python 3.7+ + return (res[:-3],int(res[-3:])) + +from collections import namedtuple +auth_data = namedtuple('rpc_auth_data',['user','passwd']) + +class RPCClient(MMGenObject): + + has_auth_cookie = False + url_fs = 'http://{}:{}' + + def __init__(self,host,port): dmsg_rpc('=== {}.__init__() debug ==='.format(type(self).__name__)) - dmsg_rpc(self.db_fs.format(h=host,p=port,u=user,pw=passwd,c=auth_cookie)) + dmsg_rpc(f' cls [{type(self).__name__}] host [{host}] port [{port}]\n') import socket try: - socket.create_connection((host,port),timeout=socket_timeout).close() + socket.create_connection((host,port),timeout=1).close() except: raise SocketError('Unable to connect to {}:{}'.format(host,port)) - if user and passwd: # user/pass overrides cookie - pass - elif auth_cookie: - user,passwd = auth_cookie.split(':') - elif self.auth: - msg('Error: no {} RPC authentication method found'.format(g.proto.name.capitalize())) - if passwd: die(1,"'rpcuser' entry not found in {}.conf or mmgen.cfg".format(g.proto.name)) - elif user: die(1,"'rpcpassword' entry not found in {}.conf or mmgen.cfg".format(g.proto.name)) - else: - m1 = 'Either provide rpcuser/rpcpassword in {pn}.conf or mmgen.cfg\n' - m2 = '(or, alternatively, copy the authentication cookie to the {pnu}\n' - m3 = 'data dir if {pnm} and {dn} are running as different users)' - die(1,(m1+m2+m3).format( - pn=g.proto.name, - pnu=g.proto.name.capitalize(), - dn=g.proto.daemon_name, - pnm=g.proj_name)) - - if self.auth: - fs = ' RPC AUTHORIZATION data ==> raw: [{}]\n{:>31}enc: [{}]\n' - auth_str = f'{user}:{passwd}' - auth_str_b64 = 'Basic ' + base64.b64encode(auth_str.encode()).decode() - dmsg_rpc(fs.format(auth_str,'',auth_str_b64)) - self.http_hdrs.update({ 'Host': host, 'Authorization': auth_str_b64 }) - + self.http_hdrs = { 'Content-Type': 'application/json' } + self.url = self.url_fs.format(host,port) self.host = host self.port = port - self.user = user - self.passwd = passwd + self.timeout = g.http_timeout + self.auth = None - for method in self.rpcmethods: - exec('{c}.{m} = lambda self,*args,**kwargs: self.request("{m}",*args,**kwargs)'.format( - c=type(self).__name__,m=method)) + def set_backend(self,backend=None): + bn = backend or opt.rpc_backend + if bn == 'auto': + self.backend = {'linux':RPCBackends.httplib,'win':RPCBackends.curl}[g.platform](self) + else: + self.backend = getattr(RPCBackends,bn)(self) - def calls(self,method,args_list): + def set_auth(self): """ - Perform a list of RPC calls, returning results in a list + MMGen's credentials override coin daemon's + """ + if g.rpc_user: + user,passwd = (g.rpc_user,g.rpc_password) + else: + user,passwd = get_coin_daemon_cfg_options(('rpcuser','rpcpassword')).values() + if user and passwd: + self.auth = auth_data(user,passwd) + return + + if self.has_auth_cookie: + cookie = self.get_daemon_auth_cookie() + if cookie: + self.auth = auth_data(*cookie.split(':')) + return + + die(1,rpc_credentials_msg()) + + # positional params are passed to the daemon, kwargs to the backend + # 'timeout' is currently the only supported kwarg + + async def call(self,method,*params,**kwargs): + """ + default call: call with param list unrolled, exactly as with cli + """ + if method == g.rpc_fail_on_command: + method = 'badcommand_' + method + return await self.process_http_resp(self.backend.run( + payload = {'id': 1, 'jsonrpc': '2.0', 'method': method, 'params': params }, + **kwargs + )) + + async def batch_call(self,method,param_list,**kwargs): + """ + Make a single call with a list of tuples as first argument + For RPC calls that return a list of results + """ + return await self.process_http_resp(self.backend.run( + payload = [{ + 'id': n, + 'jsonrpc': '2.0', + 'method': method, + 'params': params } for n,params in enumerate(param_list,1) ], + **kwargs + ),batch=True) + + async def gathered_call(self,method,args_list,**kwargs): + """ + Perform multiple RPC calls, returning results in a list Can be called two ways: 1) method = methodname, args_list = [args_tuple1, args_tuple2,...] 2) method = None, args_list = [(methodname1,args_tuple1), (methodname2,args_tuple2), ...] """ - cmd_list = args_list if method == None else tuple(zip([method] * len(args_list), args_list)) - if True: - return [self.request(method,*params) for method,params in cmd_list] - - # Normal mode: call with arg list unrolled, exactly as with cli - # Batch mode: call with list of arg lists as first argument - # kwargs are for local use and are not passed to server - - # By default, raises RPCFailure exception with an error msg on all errors and exceptions - # on_fail is one of 'raise' (default), 'return' or 'silent' - # With on_fail='return', returns 'rpcfail',(resp_object,(die_args)) - def request(self,cmd,*args,**kwargs): - - if g.debug: - print_stack_trace('RPC REQUEST {}\n args: {!r}\n kwargs: {!r}'.format(cmd,args,kwargs)) - - if g.rpc_fail_on_command == cmd: - cmd = 'badcommand_' + cmd - - cf = { 'timeout':g.http_timeout, 'batch':False, 'on_fail':'raise' } - - if cf['on_fail'] not in ('raise','return','silent'): - raise ValueError("request(): {}: illegal value for 'on_fail'".format(cf['on_fail'])) - - for k in cf: - if k in kwargs and kwargs[k]: cf[k] = kwargs[k] - - if cf['batch']: - p = [{'method':cmd,'params':r,'id':n,'jsonrpc':'2.0'} for n,r in enumerate(args[0],1)] - else: - p = {'method':cmd,'params':args,'id':1,'jsonrpc':'2.0'} - - dmsg_rpc('=== request() debug ===') - dmsg_rpc(' RPC POST data ==>\n{}\n',p) - - ca_type = self.coin_amt_type if hasattr(self,'coin_amt_type') else str - from .obj import HexStr - class MyJSONEncoder(json.JSONEncoder): - def default(self,obj): - if isinstance(obj,g.proto.coin_amt): - return ca_type(obj) - elif isinstance(obj,HexStr): - return obj - else: - return json.JSONEncoder.default(self,obj) - - data = json.dumps(p,cls=MyJSONEncoder) - - if g.platform == 'win' and len(data) < 4096: # set way below ARG_MAX, just to be safe - return self.do_request_curl(data,cf) - else: - return self.do_request_httplib(data,cf) - - def do_request_httplib(self,data,cf): - - def do_fail(*args): # args[0] is either None or HTTPResponse object - if cf['on_fail'] in ('return','silent'): return 'rpcfail',args - - try: s = '{}'.format(args[2]) - except: s = repr(args[2]) - - if s == '' and args[0] != None: - from http import HTTPStatus - hs = HTTPStatus(args[0].code) - s = '{} {}'.format(hs.value,hs.name) - - raise RPCFailure(s) - - hc = http.client.HTTPConnection(self.host,self.port,cf['timeout']) - try: - hc.request('POST','/',data,self.http_hdrs) - except Exception as e: - m = '{}\nUnable to connect to {} at {}:{}' - return do_fail(None,2,m.format(e.args[0],g.proto.daemon_name,self.host,self.port)) - - try: - r = hc.getresponse() # returns HTTPResponse instance - except Exception: - m = 'Unable to connect to {} at {}:{} (but port is bound?)' - return do_fail(None,2,m.format(g.proto.daemon_name,self.host,self.port)) - - dmsg_rpc(' RPC GETRESPONSE data ==>\n{}\n',r.__dict__) - - if r.status != 200: - if cf['on_fail'] not in ('silent','raise'): - msg_r(yellow('{} RPC Error: '.format(g.proto.daemon_name.capitalize()))) - msg(red('{} {}'.format(r.status,r.reason))) - e1 = r.read().decode() - try: - e3 = json.loads(e1)['error'] - e2 = '{} (code {})'.format(e3['message'],e3['code']) - except: - e2 = str(e1) - return do_fail(r,1,e2) - - r2 = r.read().decode() - - dmsg_rpc(' RPC REPLY data ==>\n{}\n',r2,is_json=True) - - if not r2: - return do_fail(r,2,'Empty reply') - - r3 = json.loads(r2,parse_float=Decimal) + cur_pos = 0 + chunk_size = 1024 ret = [] - for resp in r3 if cf['batch'] else [r3]: - if 'error' in resp and resp['error'] != None: - return do_fail(r,1,'{} returned an error: {}'.format( - g.proto.daemon_name.capitalize(),resp['error'])) - elif 'result' not in resp: - return do_fail(r,1, 'Missing JSON-RPC result\n' + repr(resps)) + while cur_pos < len(cmd_list): + tasks = [self.process_http_resp(self.backend.run( + payload = {'id': n, 'jsonrpc': '2.0', 'method': method, 'params': params }, + **kwargs + )) for n,(method,params) in enumerate(cmd_list[cur_pos:chunk_size+cur_pos],1)] + ret.extend(await asyncio.gather(*tasks)) + cur_pos += chunk_size + + return ret + + async def process_http_resp(self,coro,batch=False): + text,status = await coro + if status == 200: + dmsg_rpc(' RPC RESPONSE data ==>\n{}\n',text,is_json=True) + if batch: + return [r['result'] for r in json.loads(text,parse_float=Decimal,encoding='UTF-8')] else: - ret.append(resp['result']) + try: + return json.loads(text,parse_float=Decimal,encoding='UTF-8')['result'] + except: + raise RPCFailure(json.loads(text)['error']['message']) + else: + import http + s = http.HTTPStatus(status) + m = '' + if text: + try: m = ': ' + json.loads(text)['error']['message'] + except: + try: m = f': {text.decode()}' + except: m = f': {text}' + raise RPCFailure(f'{s.value} {s.name}{m}') - return ret if cf['batch'] else ret[0] - def do_request_curl(self,data,cf): - from subprocess import run,PIPE - exec_cmd = ['curl', '--proxy', '', '--silent','--request', 'POST', '--data-binary', data] - for k,v in self.http_hdrs.items(): - exec_cmd += ['--header', '{}: {}'.format(k,v)] - if self.auth: - exec_cmd += ['--user', self.user + ':' + self.passwd] - exec_cmd += ['http://{}:{}/'.format(self.host,self.port)] +class BitcoinRPCClient(RPCClient,metaclass=aInitMeta): - cp = run(exec_cmd,stdout=PIPE,check=True) - res = json.loads(cp.stdout,parse_float=Decimal) - dmsg_rpc(' RPC RESULT data ==>\n{}\n',res) + auth_type = 'basic' + has_auth_cookie = True - def do_fail(s): - if cf['on_fail'] in ('return','silent'): - return ('rpcfail',s) - raise RPCFailure(s) + def __init__(self,*args,**kwargs): pass - for resp in ([res],res)[cf['batch']]: - if 'error' in resp and resp['error'] != None: - return do_fail('{} returned an error: {}'.format(g.proto.daemon_name,resp['error'])) - elif 'result' not in resp: - return do_fail('Missing JSON-RPC result\n{!r}'.format(resp)) + async def __ainit__(self,backend=None): - return [r['result'] for r in res] if cf['batch'] else res['result'] + async def check_chainfork_mismatch(block0): + try: + assert block0 == g.proto.block0,'Incorrect Genesis block for {}'.format(g.proto.__name__) + for fork in g.proto.forks: + if fork.height == None or self.blockcount < fork.height: + break + if fork.hash != await self.call('getblockhash',fork.height): + die(3,f'Bad block hash at fork block {fork.height}. Is this the {fork.name} chain?') + except Exception as e: + die(2,"{}\n'{c}' requested, but this is not the {c} chain!".format(e.args[0],c=g.coin)) + + def check_chaintype_mismatch(): + try: + if g.regtest: assert g.chain == 'regtest','--regtest option selected, but chain is not regtest' + if g.testnet: assert g.chain != 'mainnet','--testnet option selected, but chain is mainnet' + if not g.testnet: assert g.chain == 'mainnet','mainnet selected, but chain is not mainnet' + except Exception as e: + die(1,'{}\nChain is {}!'.format(e.args[0],g.chain)) + + user,passwd = get_coin_daemon_cfg_options(('rpcuser','rpcpassword')).values() + + super().__init__( + host = g.rpc_host or 'localhost', + port = g.rpc_port or g.proto.rpc_port) + + self.set_auth() # set_auth() requires cookie, so must be called after __init__() tests socket + self.set_backend(backend) # backend requires self.auth + + if g.bob or g.alice: + from .regtest import MMGenRegtest + MMGenRegtest(g.coin).switch_user(('alice','bob')[g.bob],quiet=True) + + self.cached = {} + ( + self.cached['networkinfo'], + self.blockcount, + self.cached['blockchaininfo'], + block0 + ) = await self.gathered_call(None, ( + ('getnetworkinfo',()), + ('getblockcount',()), + ('getblockchaininfo',()), + ('getblockhash',(0,)), + )) + self.daemon_version = self.cached['networkinfo']['version'] + g.chain = self.cached['blockchaininfo']['chain'] + + tip = await self.call('getblockhash',self.blockcount) + self.cur_date = (await self.call('getblockheader',tip))['time'] + if g.chain != 'regtest': + g.chain += 'net' + assert g.chain in g.chains + check_chaintype_mismatch() + + if g.chain == 'mainnet': # skip this for testnet, as Genesis block may change + await check_chainfork_mismatch(block0) + + self.caps = ('full_node',) + for func,cap in ( + ('setlabel','label_api'), + ('signrawtransactionwithkey','sign_with_key') ): + if len((await self.call('help',func)).split('\n')) > 3: + self.caps += (cap,) + + # TODO: these belong in protocol.py + @classmethod + def get_daemon_auth_cookie_fn(cls): + cdir = os.path.join( + g.proto.daemon_data_dir, + g.proto.daemon_data_subdir ) + return os.path.join(cdir,'.cookie') + + @classmethod + def get_daemon_auth_cookie(cls): + fn = cls.get_daemon_auth_cookie_fn() + return get_lines_from_file(fn,'')[0] if file_is_readable(fn) else '' rpcmethods = ( 'backupwallet', @@ -268,12 +434,39 @@ class RPCConnection(MMGenObject): 'walletpassphrase', ) -class EthereumRPCConnection(RPCConnection): +class EthereumRPCClient(RPCClient,metaclass=aInitMeta): - auth = False - db_fs = ' host [{h}] port [{p}]\n' - _blockcount = None - _cur_date = None + auth_type = None + + def __init__(self,*args,**kwargs): pass + + async def __ainit__(self,backend=None): + + super().__init__( + host = g.rpc_host or 'localhost', + port = g.rpc_port or g.proto.rpc_port ) + + self.set_backend(backend) + + self.blockcount = int(await self.call('eth_blockNumber'),16) + + vi,bh,ch,nk = await self.gathered_call(None, ( + ('parity_versionInfo',()), + ('parity_getBlockHeaderByNumber',()), + ('parity_chain',()), + ('parity_nodeKind',()), + )) + + self.daemon_version = vi['version'] + self.cur_date = int(bh['timestamp'],16) + g.chain = ch.replace(' ','_') + self.caps = ('full_node',) if nk['capability'] == 'full' else () + + try: + await self.call('eth_chainId') + self.caps += ('eth_chainId',) + except RPCFailure: + pass rpcmethods = ( 'eth_accounts', @@ -314,20 +507,25 @@ class EthereumRPCConnection(RPCConnection): 'parity_versionInfo', ) - # blockcount and cur_date require network RPC calls, so evaluate lazily - @property - def blockcount(self): - if self._blockcount == None: - self._blockcount = int(self.eth_blockNumber(),16) - return self._blockcount +class MoneroWalletRPCClient(RPCClient): - @property - def cur_date(self): - if self._cur_date == None: - self._cur_date = int(self.parity_getBlockHeaderByNumber(hex(self.blockcount))['timestamp'],16) - return self._cur_date + auth_type = 'digest' + url_fs = 'http://{}:{}/json_rpc' -class MoneroWalletRPCConnection(RPCConnection): + def __init__(self,host,port,user,passwd): + super().__init__(host,port) + self.auth = auth_data(user,passwd) + self.set_backend('requests') + if False: # insecure, for debugging only + self.backend = RPCBackends.curl(self) + self.backend.exec_opts.remove('--silent') + self.backend.exec_opts.extend(['--insecure','--verbose']) + + async def call(self,method,*params,**kwargs): + assert params == (), f'{type(self).__name__}.call() accepts keyword arguments only' + return await self.process_http_resp(self.backend.run( + payload = {'id': 0, 'jsonrpc': '2.0', 'method': method, 'params': kwargs }, + )) rpcmethods = ( 'get_version', @@ -340,159 +538,16 @@ class MoneroWalletRPCConnection(RPCConnection): 'refresh', # start_height ) - def request(self,cmd,*args,**kwargs): - if args != (): - m = '{}.request() accepts only keyword args\nCmd: {!r}' - raise ValueError(m.format(type(self).__name__,cmd)) - import requests - import urllib3 - urllib3.disable_warnings() - ret = requests.post( - url = 'https://{}:{}/json_rpc'.format(self.host,self.port), - json = { - 'jsonrpc': '2.0', - 'id': '0', - 'method': cmd, - 'params': kwargs, - }, - auth = requests.auth.HTTPDigestAuth(self.user,self.passwd), - headers = self.http_hdrs, - verify = False ) +async def rpc_init(proto=None,backend=None): - res = json.loads(ret._content) - if 'error' in res: - raise RPCFailure(repr(res['error'])) - return(res['result']) + proto = proto or g.proto - def request_curltest(self,cmd,*args,**kwargs): - "insecure, for testing only" - from subprocess import run,PIPE - data = { - 'jsonrpc': '2.0', - 'id': '0', - 'method': cmd, - 'params': kwargs, - } - exec_cmd = [ - 'curl', '--proxy', '', '--verbose','--insecure', '--request', 'POST', - '--digest', '--user', '{}:{}'.format(g.monero_wallet_rpc_user,g.monero_wallet_rpc_password), - '--header', 'Content-Type: application/json', - '--data', json.dumps(data), - 'https://{}:{}/json_rpc'.format(self.host,self.port) ] + if not 'rpc' in proto.mmcaps: + die(1,'Coin daemon operations not supported for {}!'.format(proto.__name__)) - cp = run(exec_cmd,stdout=PIPE,check=True) + g.rpc = await { + 'bitcoind': BitcoinRPCClient, + 'parity': EthereumRPCClient, + }[proto.daemon_family](backend=backend) - res = json.loads(cp.stdout) - if 'error' in res: - raise RPCFailure(repr(res['error'])) - return(res['result']) - -def rpc_error(ret): - return type(ret) is tuple and ret and ret[0] == 'rpcfail' - -def rpc_errmsg(ret): - try: - return ret[1][2] - except: - return repr(ret) - -def init_daemon_parity(): - - def resolve_token_arg(token_arg): - from .obj import CoinAddr - from .altcoins.eth.tw import EthereumTrackingWallet - from .altcoins.eth.contract import Token - - tw = EthereumTrackingWallet(no_rpc=True) - - try: addr = CoinAddr(token_arg,on_fail='raise') - except: addr = tw.sym2addr(token_arg) - - if not addr: - m = "'{}': unrecognized token symbol" - raise UnrecognizedTokenSymbol(m.format(token_arg)) - - sym = tw.addr2sym(addr) # throws exception on failure - vmsg('ERC20 token resolved: {} ({})'.format(addr,sym)) - - return addr,sym - - conn = EthereumRPCConnection( - g.rpc_host or 'localhost', - g.rpc_port or g.proto.rpc_port) - conn.daemon_version = conn.parity_versionInfo()['version'] # fail immediately if daemon is geth - conn.coin_amt_type = str - g.chain = conn.parity_chain().replace(' ','_') - - conn.caps = () - try: - conn.request('eth_chainId') - conn.caps += ('eth_chainId',) - except RPCFailure: - pass - - if conn.request('parity_nodeKind')['capability'] == 'full': - conn.caps += ('full_node',) - - if g.token: - g.rpc = conn # set g.rpc so rpc_init() will return immediately - (g.token,g.dcoin) = resolve_token_arg(g.token) - - return conn - -def init_daemon_bitcoind(): - - def check_chainfork_mismatch(conn): - block0 = conn.getblockhash(0) - latest = conn.blockcount - try: - assert block0 == g.proto.block0,'Incorrect Genesis block for {}'.format(g.proto.__name__) - for fork in g.proto.forks: - if fork[0] == None or latest < fork[0]: break - assert conn.getblockhash(fork[0]) == fork[1], ( - 'Bad block hash at fork block {}. Is this the {} chain?'.format(fork[0],fork[2].upper())) - except Exception as e: - die(2,"{}\n'{c}' requested, but this is not the {c} chain!".format(e.args[0],c=g.coin)) - - def check_chaintype_mismatch(): - try: - if g.regtest: assert g.chain == 'regtest','--regtest option selected, but chain is not regtest' - if g.testnet: assert g.chain != 'mainnet','--testnet option selected, but chain is mainnet' - if not g.testnet: assert g.chain == 'mainnet','mainnet selected, but chain is not mainnet' - except Exception as e: - die(1,'{}\nChain is {}!'.format(e.args[0],g.chain)) - - cfg = get_daemon_cfg_options(('rpcuser','rpcpassword')) - - conn = RPCConnection( - g.rpc_host or 'localhost', - g.rpc_port or g.proto.rpc_port, - g.rpc_user or cfg['rpcuser'], # MMGen's rpcuser,rpcpassword override coin daemon's - g.rpc_password or cfg['rpcpassword'], - auth_cookie=get_coin_daemon_auth_cookie()) - - if g.bob or g.alice: - from .regtest import MMGenRegtest - MMGenRegtest(g.coin).switch_user(('alice','bob')[g.bob],quiet=True) - conn.daemon_version = int(conn.getnetworkinfo()['version']) - conn.blockcount = conn.getblockcount() - conn.cur_date = conn.getblockheader(conn.getblockhash(conn.blockcount))['time'] - conn.coin_amt_type = (float,str)[conn.daemon_version>=120000] - g.chain = conn.getblockchaininfo()['chain'] - if g.chain != 'regtest': g.chain += 'net' - assert g.chain in g.chains - check_chaintype_mismatch() - - if g.chain == 'mainnet': # skip this for testnet, as Genesis block may change - check_chainfork_mismatch(conn) - - conn.caps = ('full_node',) - for func,cap in ( - ('setlabel','label_api'), - ('signrawtransactionwithkey','sign_with_key') ): - if len(conn.request('help',func).split('\n')) > 3: - conn.caps += (cap,) - return conn - -def init_daemon(name): - return globals()['init_daemon_'+name]() + return g.rpc diff --git a/mmgen/tool.py b/mmgen/tool.py index 9b98c40c..901353e4 100755 --- a/mmgen/tool.py +++ b/mmgen/tool.py @@ -650,14 +650,15 @@ class MMGenToolCmdFile(MMGenToolCmds): file_sort = kwargs.get('filesort') or 'mtime' from .filename import MMGenFileList - from .tx import MMGenTX + from .tx import MMGenTX,MMGenTxForSigning flist = MMGenFileList(infiles,ftype=MMGenTX) flist.sort_by_age(key=file_sort) # in-place sort - sep = '—'*77+'\n' - return sep.join( - [MMGenTX(fn,offline=True).format_view(terse=terse,sort=tx_sort) for fn in flist.names()] - ).rstrip() + def gen(): + for fn in flist.names(): + yield (MMGenTxForSigning,MMGenTX)[fn.endswith('.sigtx')](fn).format_view(terse=terse,sort=tx_sort) + + return ('—'*77+'\n').join(gen()).rstrip() class MMGenToolCmdFileCrypt(MMGenToolCmds): """ @@ -841,12 +842,12 @@ from .tw import TwAddrList,TwUnspentOutputs class MMGenToolCmdRPC(MMGenToolCmds): "tracking wallet commands using the JSON-RPC interface" - def getbalance(self,minconf=1,quiet=False,pager=False): + async def getbalance(self,minconf=1,quiet=False,pager=False): "list confirmed/unconfirmed, spendable/unspendable balances in tracking wallet" from .tw import TwGetBalance - return TwGetBalance(minconf,quiet).format() + return (await TwGetBalance(minconf,quiet)).format() - def listaddress(self, + async def listaddress(self, mmgen_addr:str, minconf = 1, pager = False, @@ -855,7 +856,7 @@ class MMGenToolCmdRPC(MMGenToolCmds): age_fmt: _options_annot_str(TwAddrList.age_fmts) = 'confs', ): "list the specified MMGen address and its balance" - return self.listaddresses( mmgen_addrs = mmgen_addr, + return await self.listaddresses( mmgen_addrs = mmgen_addr, minconf = minconf, pager = pager, showempty = showempty, @@ -863,7 +864,7 @@ class MMGenToolCmdRPC(MMGenToolCmds): age_fmt = age_fmt, ) - def listaddresses( self, + async def listaddresses( self, mmgen_addrs:'(range or list)' = '', minconf = 1, showempty = False, @@ -890,13 +891,12 @@ class MMGenToolCmdRPC(MMGenToolCmds): die(1,m.format(mmgen_addrs)) usr_addr_list = [MMGenID('{}:{}'.format(a[0],i)) for i in AddrIdxList(a[1])] - rpc_init() - al = TwAddrList(usr_addr_list,minconf,showempty,showbtcaddrs,all_labels) + al = await TwAddrList(usr_addr_list,minconf,showempty,showbtcaddrs,all_labels) if not al: die(0,('No tracked addresses with balances!','No tracked addresses!')[showempty]) - return al.format(showbtcaddrs,sort,show_age,age_fmt or 'confs') + return await al.format(showbtcaddrs,sort,show_age,age_fmt or 'confs') - def twview( self, + async def twview( self, pager = False, reverse = False, wide = False, @@ -906,9 +906,8 @@ class MMGenToolCmdRPC(MMGenToolCmds): show_mmid = True, wide_show_confs = True): "view tracking wallet" - rpc_init() - twuo = TwUnspentOutputs(minconf=minconf) - twuo.do_sort(sort,reverse=reverse) + twuo = await TwUnspentOutputs(minconf=minconf) + await twuo.get_unspent_data(reverse_sort=reverse) twuo.age_fmt = age_fmt twuo.show_mmid = show_mmid if wide: @@ -916,25 +915,23 @@ class MMGenToolCmdRPC(MMGenToolCmds): else: ret = twuo.format_for_display() del twuo.wallet - return ret + return await ret - def add_label(self,mmgen_or_coin_addr:str,label:str): + async def add_label(self,mmgen_or_coin_addr:str,label:str): "add descriptive label for address in tracking wallet" - rpc_init() from .tw import TrackingWallet - TrackingWallet(mode='w').add_label(mmgen_or_coin_addr,label,on_fail='raise') + await (await TrackingWallet(mode='w')).add_label(mmgen_or_coin_addr,label,on_fail='raise') return True - def remove_label(self,mmgen_or_coin_addr:str): + async def remove_label(self,mmgen_or_coin_addr:str): "remove descriptive label for address in tracking wallet" - self.add_label(mmgen_or_coin_addr,'') + await self.add_label(mmgen_or_coin_addr,'') return True - def remove_address(self,mmgen_or_coin_addr:str): + async def remove_address(self,mmgen_or_coin_addr:str): "remove an address from tracking wallet" from .tw import TrackingWallet - tw = TrackingWallet(mode='w') - ret = tw.remove_address(mmgen_or_coin_addr) # returns None on failure + ret = await (await TrackingWallet(mode='w')).remove_address(mmgen_or_coin_addr) # returns None on failure if ret: msg("Address '{}' deleted from tracking wallet".format(ret)) return ret @@ -988,7 +985,7 @@ class MMGenToolCmdMonero(MMGenToolCmds): if monerod_args: self.monerod_args = monerod_args - def create(n,d,fn,c,m): + async def create(n,d,fn,c,m): try: os.stat(fn) except: pass else: @@ -997,7 +994,8 @@ class MMGenToolCmdMonero(MMGenToolCmds): gmsg(m) from .baseconv import baseconv - ret = c.restore_deterministic_wallet( + ret = await c.call( + 'restore_deterministic_wallet', filename = os.path.basename(fn), password = d.wallet_passwd, seed = baseconv.fromhex(d.sec,'xmrseed',tostr=True), @@ -1007,7 +1005,7 @@ class MMGenToolCmdMonero(MMGenToolCmds): pp_msg(ret) if opt.debug else msg(' Address: {}'.format(ret['address'])) return True - def sync(n,d,fn,c,m): + async def sync(n,d,fn,c,m): try: os.stat(fn) except: @@ -1021,11 +1019,14 @@ class MMGenToolCmdMonero(MMGenToolCmds): t_start = time.time() msg_r(' Opening wallet...') - c.open_wallet(filename=os.path.basename(fn),password=d.wallet_passwd) + await c.call( + 'open_wallet', + filename=os.path.basename(fn), + password=d.wallet_passwd ) msg('done') msg_r(' Getting wallet height...') - wallet_height = c.get_height()['height'] + wallet_height = (await c.call('get_height'))['height'] msg('\r Wallet height: {} '.format(wallet_height)) behind = chain_height - wallet_height @@ -1033,7 +1034,7 @@ class MMGenToolCmdMonero(MMGenToolCmds): m = ' Wallet is {} blocks behind chain tip. Please be patient. Syncing...' msg_r(m.format(behind)) - ret = c.refresh() + ret = await c.call('refresh') if behind > 1000: msg('done') @@ -1043,7 +1044,7 @@ class MMGenToolCmdMonero(MMGenToolCmds): t_elapsed = int(time.time() - t_start) - ret = c.get_balance() # account_index=0, address_indices=[0,1] + ret = await c.call('get_balance') # account_index=0, address_indices=[0,1] from .obj import XMRAmt bals[fn] = tuple([XMRAmt(ret[k],from_unit='min_coin_unit') for k in ('balance','unlocked_balance')]) @@ -1053,16 +1054,14 @@ class MMGenToolCmdMonero(MMGenToolCmds): else: msg(' Balance: {} Unlocked balance: {}'.format(*[b.hl() for b in bals[fn]])) - msg(' Wallet height: {}'.format(c.get_height()['height'])) + msg(' Wallet height: {}'.format((await c.call('get_height'))['height'])) msg(' Sync time: {:02}:{:02}'.format(t_elapsed//60,t_elapsed%60)) - c.close_wallet() + await c.call('close_wallet') return True - def process_wallets(): - m = { 'create': ('Creat','Generat',create,False), - 'sync': ('Sync', 'Sync', sync, True) } - opt.accept_defaults = opt.accept_defaults or m[op][3] + async def process_wallets(op): + opt.accept_defaults = opt.accept_defaults or op.accept_defaults from .protocol import init_coin init_coin('xmr') from .addr import AddrList @@ -1070,18 +1069,18 @@ class MMGenToolCmdMonero(MMGenToolCmds): data = [d for d in al.data if addrs == '' or d.idx in AddrIdxList(addrs)] dl = len(data) assert dl,"No addresses in addrfile within range '{}'".format(addrs) - gmsg('\n{}ing {} wallet{}'.format(m[op][0],dl,suf(dl))) + gmsg('\n{}ing {} wallet{}'.format(op.desc,dl,suf(dl))) from .daemon import MoneroWalletDaemon wd = MoneroWalletDaemon(opt.outdir or '.',test_suite=g.test_suite) wd.restart() - from .rpc import MoneroWalletRPCConnection - c = MoneroWalletRPCConnection( - g.monero_wallet_rpc_host, - wd.rpc_port, - g.monero_wallet_rpc_user, - g.monero_wallet_rpc_password) + from .rpc import MoneroWalletRPCClient + c = MoneroWalletRPCClient( + host = g.monero_wallet_rpc_host, + port = wd.rpc_port, + user = g.monero_wallet_rpc_user, + passwd = g.monero_wallet_rpc_password) wallets_processed = 0 for n,d in enumerate(data): # [d.sec,d.wallet_passwd,d.viewkey,d.addr] @@ -1091,13 +1090,13 @@ class MMGenToolCmdMonero(MMGenToolCmds): d.idx, '-α' if g.debug_utf8 else '')) - info = '\n{}ing wallet {}/{} ({})'.format(m[op][1],n+1,dl,fn) - wallets_processed += m[op][2](n,d,fn,c,info) + info = '\n{}ing wallet {}/{} ({})'.format(op.action,n+1,dl,fn) + wallets_processed += await op.func(n,d,fn,c,info) wd.stop() - gmsg('\n{} wallet{} {}ed'.format(wallets_processed,suf(wallets_processed),m[op][0].lower())) + gmsg('\n{} wallet{} {}ed'.format(wallets_processed,suf(wallets_processed),op.desc.lower())) - if wallets_processed and op == 'sync': + if wallets_processed and op.name == 'sync': col1_w = max(map(len,bals)) + 1 fs = '{:%s} {} {}' % col1_w msg('\n'+fs.format('Wallet','Balance ','Unlocked Balance ')) @@ -1114,8 +1113,13 @@ class MMGenToolCmdMonero(MMGenToolCmds): bals = {} # locked,unlocked + from collections import namedtuple + wo = namedtuple('mwo',['name','desc','action','func','accept_defaults']) + op = { # reusing name! + 'create': wo('create', 'Creat', 'Generat', create, False), + 'sync': wo('sync', 'Sync', 'Sync', sync, True) }[op] try: - process_wallets() + run_session(process_wallets(op),do_rpc_init=False) except KeyboardInterrupt: rdie(1,'\nUser interrupt\n') except EOFError: diff --git a/mmgen/tw.py b/mmgen/tw.py index 7b11bb39..01d06829 100755 --- a/mmgen/tw.py +++ b/mmgen/tw.py @@ -40,21 +40,21 @@ _date_formatter = { 'date_time': lambda secs: '{}-{:02}-{:02} {:02}:{:02}'.format(*time.gmtime(secs)[:5]), } -def _set_dates(us): +async def _set_dates(us): if us and us[0].date is None: # 'blocktime' differs from 'time', is same as getblockheader['time'] - dates = [o['blocktime'] for o in g.rpc.calls('gettransaction',[(o.txid,) for o in us])] + dates = [o['blocktime'] for o in await g.rpc.gathered_call('gettransaction',[(o.txid,) for o in us])] for o,date in zip(us,dates): o.date = date if os.getenv('MMGEN_BOGUS_WALLET_DATA'): # 1831006505 (09 Jan 2028) = projected time of block 1000000 _date_formatter['days'] = lambda date: (1831006505 - date) // 86400 - def _set_dates(us): + async def _set_dates(us): for o in us: o.date = 1831006505 - int(9.7 * 60 * (o.confs - 1)) -class TwUnspentOutputs(MMGenObject): +class TwUnspentOutputs(MMGenObject,metaclass=aInitMeta): def __new__(cls,*args,**kwargs): return MMGenObject.__new__(altcoin_subclass(cls,'tw','TwUnspentOutputs')) @@ -104,7 +104,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program. """.strip().format(g.proj_name.lower()) } - def __init__(self,minconf=1,addrs=[]): + async def __ainit__(self,minconf=1,addrs=[]): self.unspent = self.MMGenTwOutputList() self.fmt_display = '' self.fmt_print = '' @@ -117,9 +117,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program. self.sort_key = 'age' self.disp_prec = self.get_display_precision() - self.wallet = TrackingWallet('w') - self.get_unspent_data() - self.do_sort() + self.wallet = await TrackingWallet(mode='w') @property def age_fmt(self): @@ -138,7 +136,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program. def total(self): return sum(i.amt for i in self.unspent) - def get_unspent_rpc(self): + async def get_unspent_rpc(self): # bitcoin-cli help listunspent: # Arguments: # 1. minconf (numeric, optional, default=1) The minimum confirmations to filter @@ -149,17 +147,20 @@ watch-only wallet using '{}-addrimport' and then re-run this program. # for now, self.addrs is just an empty list for Bitcoin and friends add_args = (9999999,self.addrs) if self.addrs else () - return g.rpc.listunspent(self.minconf,*add_args) + return await g.rpc.call('listunspent',self.minconf,*add_args) - def get_unspent_data(self): + async def get_unspent_data(self,sort_key=None,reverse_sort=False): if g.bogus_wallet_data: # for debugging purposes only us_rpc = eval(get_data_from_file(g.bogus_wallet_data)) # testing, so ok else: - us_rpc = self.get_unspent_rpc() + us_rpc = await self.get_unspent_rpc() + + if not us_rpc: + die(0,self.wmsg['no_spendable_outputs']) - if not us_rpc: die(0,self.wmsg['no_spendable_outputs']) tr_rpc = [] lbl_id = ('account','label')['label_api' in g.rpc.caps] + for o in us_rpc: if not lbl_id in o: continue # coinbase outputs have no account field @@ -183,6 +184,8 @@ watch-only wallet using '{}-addrimport' and then re-run this program. if not self.unspent: die(1,'No tracked {}s in tracking wallet!'.format(self.item_desc)) + self.do_sort(key=sort_key,reverse=reverse_sort) + def do_sort(self,key=None,reverse=False): sort_funcs = { 'addr': lambda i: i.addr, @@ -214,10 +217,10 @@ watch-only wallet using '{}-addrimport' and then re-run this program. m2 = 'Please resize your screen to at least {} characters and hit ENTER ' my_raw_input((m1+m2).format(g.min_screen_width)) - def format_for_display(self): + async def format_for_display(self): unsp = self.unspent if self.age_fmt in self.age_fmts_date_dependent: - _set_dates(unsp) + await _set_dates(unsp) self.set_term_columns() # allow for 7-digit confirmation nums @@ -291,9 +294,9 @@ watch-only wallet using '{}-addrimport' and then re-run this program. self.fmt_display = '\n'.join(out) + '\n' return self.fmt_display - def format_for_printing(self,color=False,show_confs=True): + async def format_for_printing(self,color=False,show_confs=True): if self.age_fmt in self.age_fmts_date_dependent: - _set_dates(self.unspent) + await _set_dates(self.unspent) addr_w = max(len(i.addr) for i in self.unspent) mmid_w = max(len(('',i.twmmid)[i.twmmid.type=='mmgen']) for i in self.unspent) or 12 # DEADBEEF:S:1 amt_w = g.proto.coin_amt.max_prec + 5 @@ -374,14 +377,14 @@ watch-only wallet using '{}-addrimport' and then re-run this program. if keypress_confirm(fs.format(self.item_desc,n)): return n - def view_and_sort(self,tx): + async def view_and_sort(self,tx): from .term import get_char prompt = self.prompt.strip() + '\b' no_output,oneshot_msg = False,None while True: msg_r('' if no_output else '\n\n' if opt.no_blank else CUR_HOME+ERASE_ALL) reply = get_char( - '' if no_output else self.format_for_display()+'\n'+(oneshot_msg or '')+prompt, + '' if no_output else await self.format_for_display()+'\n'+(oneshot_msg or '')+prompt, immed_chars=''.join(self.key_mappings.keys()) ) no_output = False @@ -409,17 +412,15 @@ watch-only wallet using '{}-addrimport' and then re-run this program. idx = self.get_idx_from_user(action) if idx: e = self.unspent[idx-1] - bal = self.wallet.get_balance(e.addr,force_rpc=True) - self.get_unspent_data() - self.do_sort() + bal = await self.wallet.get_balance(e.addr,force_rpc=True) + await self.get_unspent_data() oneshot_msg = yellow('{} balance for account #{} refreshed\n\n'.format(g.dcoin,idx)) elif action == 'a_lbl_add': idx,lbl = self.get_idx_from_user(action) if idx: e = self.unspent[idx-1] - if self.wallet.add_label(e.twmmid,lbl,addr=e.addr): - self.get_unspent_data() - self.do_sort() + if await self.wallet.add_label(e.twmmid,lbl,addr=e.addr): + await self.get_unspent_data() a = 'added to' if lbl else 'removed from' oneshot_msg = yellow("Label {} {} #{}\n\n".format(a,self.item_desc,idx)) else: @@ -428,9 +429,8 @@ watch-only wallet using '{}-addrimport' and then re-run this program. idx = self.get_idx_from_user(action) if idx: e = self.unspent[idx-1] - if self.wallet.remove_address(e.addr): - self.get_unspent_data() - self.do_sort() + if await self.wallet.remove_address(e.addr): + await self.get_unspent_data() oneshot_msg = yellow("{} #{} removed\n\n".format(capfirst(self.item_desc),idx)) else: oneshot_msg = red('Address could not be removed\n\n') @@ -439,13 +439,13 @@ watch-only wallet using '{}-addrimport' and then re-run this program. ','.join(self.sort_info(include_group=False)).lower()) msg('') try: - write_data_to_file(of,self.format_for_printing(),desc='{} listing'.format(self.desc)) + write_data_to_file(of,await self.format_for_printing(),desc='{} listing'.format(self.desc)) except UserNonConfirmation as e: oneshot_msg = red("File '{}' not overwritten by user request\n\n".format(of)) else: oneshot_msg = yellow("Data written to '{}'\n\n".format(of)) elif action in ('a_view','a_view_wide'): - do_pager(self.fmt_display if action == 'a_view' else self.format_for_printing(color=True)) + do_pager(self.fmt_display if action == 'a_view' else await self.format_for_printing(color=True)) if g.platform == 'linux' and oneshot_msg == None: msg_r(CUR_RIGHT(len(prompt.split('\n')[-1])-2)) no_output = True @@ -458,7 +458,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program. else: return _date_formatter[age_fmt](o.date) -class TwAddrList(MMGenDict): +class TwAddrList(MMGenDict,metaclass=aInitMeta): has_age = True age_fmts = TwUnspentOutputs.age_fmts age_disp = TwUnspentOutputs.age_disp @@ -466,7 +466,10 @@ class TwAddrList(MMGenDict): def __new__(cls,*args,**kwargs): return MMGenDict.__new__(altcoin_subclass(cls,'tw','TwAddrList'),*args,**kwargs) - def __init__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None): + def __init__(self,*args,**kwargs): + pass + + async def __ainit__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None): def check_dup_mmid(acct_labels): mmid_prev,err = None,False @@ -490,10 +493,9 @@ class TwAddrList(MMGenDict): if err: rdie(3,'Tracking wallet is corrupted!') self.total = g.proto.coin_amt('0') - rpc_init() lbl_id = ('account','label')['label_api' in g.rpc.caps] - for d in g.rpc.listunspent(0): + for d in await g.rpc.call('listunspent',0): if not lbl_id in d: continue # skip coinbase outputs with missing account if d['confirmations'] < minconf: continue label = get_tw_label(d[lbl_id]) @@ -520,11 +522,12 @@ class TwAddrList(MMGenDict): # for compatibility with old mmids, must use raw RPC rather than native data for matching # args: minconf,watchonly, MUST use keys() so we get list, not dict if 'label_api' in g.rpc.caps: - acct_list = g.rpc.listlabels() - acct_addrs = [list(a.keys()) for a in g.rpc.getaddressesbylabel([[k] for k in acct_list],batch=True)] + acct_list = await g.rpc.call('listlabels') + aa = await g.rpc.batch_call('getaddressesbylabel',[(k,) for k in acct_list]) + acct_addrs = [list(a.keys()) for a in aa] else: - acct_list = list(g.rpc.listaccounts(0,True).keys()) # raw list, no 'L' - acct_addrs = g.rpc.getaddressesbyaccount([[a] for a in acct_list],batch=True) # use raw list here + acct_list = list((await g.rpc.call('listaccounts',0,True)).keys()) # raw list, no 'L' + acct_addrs = await g.rpc.batch_call('getaddressesbyaccount',[(a,) for a in acct_list]) # use raw list here acct_labels = MMGenList([get_tw_label(a) for a in acct_list]) check_dup_mmid(acct_labels) assert len(acct_list) == len(acct_addrs),( @@ -545,7 +548,7 @@ class TwAddrList(MMGenDict): def coinaddr_list(self): return [self[k]['addr'] for k in self] - def format(self,showbtcaddrs,sort,show_age,age_fmt): + async def format(self,showbtcaddrs,sort,show_age,age_fmt): if not self.has_age: show_age = False if age_fmt not in self.age_fmts: @@ -580,7 +583,7 @@ class TwAddrList(MMGenDict): al_id_save = None mmids = sorted(self,key=sort_algo,reverse=bool(sort and 'reverse' in sort)) if show_age: - _set_dates([o for o in mmids if hasattr(o,'confs')]) + await _set_dates([o for o in mmids if hasattr(o,'confs')]) for mmid in mmids: if mmid.type == 'mmgen': if al_id_save and al_id_save != mmid.obj.al_id: @@ -603,22 +606,27 @@ class TwAddrList(MMGenDict): return '\n'.join(out + ['\nTOTAL: {} {}'.format(self.total.hl(color=True),g.dcoin)]) -class TrackingWallet(MMGenObject): +class TrackingWallet(MMGenObject,metaclass=aInitMeta): caps = ('rescan','batch') data_key = 'addresses' use_tw_file = False aggressive_sync = False + importing = False def __new__(cls,*args,**kwargs): return MMGenObject.__new__(altcoin_subclass(cls,'tw','TrackingWallet')) - def __init__(self,mode='r',no_rpc=False): + async def __ainit__(self,mode='r'): + + assert mode in ('r','w','i'), "{!r}: wallet mode must be 'r','w' or 'i'".format(mode) + if mode == 'i': + self.importing = True + mode = 'w' if g.debug: print_stack_trace('TW INIT {!r} {!r}'.format(mode,self)) - assert mode in ('r','w'),"{!r}: wallet mode must be 'r' or 'w'".format(self) self.mode = mode self.desc = self.base_desc = '{} tracking wallet'.format(capfirst(g.proto.name)) @@ -632,7 +640,6 @@ class TrackingWallet(MMGenObject): raise WalletFileError(m.format(self.data['coin'],g.coin)) self.conv_types(self.data[self.data_key]) - self.rpc_init() self.cur_balances = {} # cache balances to prevent repeated lookups per program invocation def init_empty(self): @@ -697,12 +704,9 @@ class TrackingWallet(MMGenObject): @staticmethod def conv_types(ad): for k,v in ad.items(): - if k in ('params','coin'): continue - v['mmid'] = TwMMGenID(v['mmid'],on_fail='raise') - v['comment'] = TwComment(v['comment'],on_fail='raise') - - def rpc_init(self): - rpc_init() + if k not in ('params','coin'): + v['mmid'] = TwMMGenID(v['mmid'],on_fail='raise') + v['comment'] = TwComment(v['comment'],on_fail='raise') @property def data_root(self): @@ -728,14 +732,14 @@ class TrackingWallet(MMGenObject): if addr in data_root and 'balance' in data_root[addr]: return g.proto.coin_amt(data_root[addr]['balance']) - def get_balance(self,addr,force_rpc=False): + async def get_balance(self,addr,force_rpc=False): ret = None if force_rpc else self.get_cached_balance(addr,self.cur_balances,self.data_root) if ret == None: - ret = self.rpc_get_balance(addr) + ret = await self.rpc_get_balance(addr) self.cache_balance(addr,ret,self.cur_balances,self.data_root) return ret - def rpc_get_balance(self,addr): + async def rpc_get_balance(self,addr): raise NotImplementedError('not implemented') @property @@ -752,12 +756,12 @@ class TrackingWallet(MMGenObject): return dict((x['mmid'],{'addr':x['addr'],'comment':x['comment']}) for x in self.sorted_list) @write_mode - def import_address(self,addr,label,rescan): - return g.rpc.importaddress(addr,label,rescan,timeout=(False,3600)[rescan]) + async def import_address(self,addr,label,rescan): + return await g.rpc.call('importaddress',addr,label,rescan,timeout=(False,3600)[rescan]) @write_mode def batch_import_address(self,arg_list): - return g.rpc.importaddress(arg_list,batch=True) + return g.rpc.batch_call('importaddress',arg_list) def force_write(self): mode_save = self.mode @@ -789,24 +793,30 @@ class TrackingWallet(MMGenObject): elif g.debug: msg('Data is unchanged\n') - def is_in_wallet(self,addr): - return addr in TwAddrList([],0,True,True,True,wallet=self).coinaddr_list() + async def is_in_wallet(self,addr): + return addr in (await TwAddrList([],0,True,True,True,wallet=self)).coinaddr_list() @write_mode - def set_label(self,coinaddr,lbl): + async def set_label(self,coinaddr,lbl): # bitcoin-abc 'setlabel' RPC is broken, so use old 'importaddress' method to set label # broken behavior: new label is set OK, but old label gets attached to another address if 'label_api' in g.rpc.caps and g.coin != 'BCH': - return g.rpc.setlabel(coinaddr,lbl,on_fail='return') + args = ('setlabel',coinaddr,lbl) else: # NOTE: this works because importaddress() removes the old account before # associating the new account with the address. # RPC args: addr,label,rescan[=true],p2sh[=none] - return g.rpc.importaddress(coinaddr,lbl,False,on_fail='return') + args = ('importaddress',coinaddr,lbl,False) + + try: + return await g.rpc.call(*args) + except Exception as e: + rmsg(e.args[0]) + return False # returns on failure @write_mode - def add_label(self,arg1,label='',addr=None,silent=False,on_fail='return'): + async def add_label(self,arg1,label='',addr=None,silent=False,on_fail='return'): from .tx import is_mmgen_id,is_coin_addr mmaddr,coinaddr = None,None if is_coin_addr(addr or arg1): @@ -815,14 +825,14 @@ class TrackingWallet(MMGenObject): mmaddr = TwMMGenID(arg1) if mmaddr and not coinaddr: - from .addr import AddrData - coinaddr = AddrData(source='tw').mmaddr2coinaddr(mmaddr) + from .addr import TwAddrData + coinaddr = (await TwAddrData()).mmaddr2coinaddr(mmaddr) try: if not is_mmgen_id(arg1): assert coinaddr,"Invalid coin address for this chain: {}".format(arg1) assert coinaddr,"{pn} address '{ma}' not found in tracking wallet" - assert self.is_in_wallet(coinaddr),"Address '{ca}' not found in tracking wallet" + assert await self.is_in_wallet(coinaddr),"Address '{ca}' not found in tracking wallet" except Exception as e: msg(e.args[0].format(pn=g.proj_name,ma=mmaddr,ca=coinaddr)) return False @@ -830,8 +840,8 @@ class TrackingWallet(MMGenObject): # Allow for the possibility that BTC addr of MMGen addr was entered. # Do reverse lookup, so that MMGen addr will not be marked as non-MMGen. if not mmaddr: - from .addr import AddrData - mmaddr = AddrData(source='tw').coinaddr2mmaddr(coinaddr) + from .addr import TwAddrData + mmaddr = (await TwAddrData()).coinaddr2mmaddr(coinaddr) if not mmaddr: mmaddr = '{}:{}'.format(g.proto.base_coin.lower(),coinaddr) @@ -844,11 +854,7 @@ class TrackingWallet(MMGenObject): lbl = TwLabel(mmaddr + ('',' '+cmt)[bool(cmt)],on_fail=on_fail) - ret = self.set_label(coinaddr,lbl) - - from .rpc import rpc_error,rpc_errmsg - if rpc_error(ret): - msg('From {}: {}'.format(g.proto.daemon_name,rpc_errmsg(ret))) + if await self.set_label(coinaddr,lbl) == False: if not silent: msg('Label could not be {}'.format(('removed','added')[bool(label)])) return False @@ -861,32 +867,31 @@ class TrackingWallet(MMGenObject): return True @write_mode - def remove_label(self,mmaddr): - self.add_label(mmaddr,'') + async def remove_label(self,mmaddr): + await self.add_label(mmaddr,'') @write_mode - def remove_address(self,addr): + async def remove_address(self,addr): raise NotImplementedError('address removal not implemented for coin {}'.format(g.coin)) -class TwGetBalance(MMGenObject): +class TwGetBalance(MMGenObject,metaclass=aInitMeta): fs = '{w:13} {u:<16} {p:<16} {c}\n' def __new__(cls,*args,**kwargs): return MMGenObject.__new__(altcoin_subclass(cls,'tw','TwGetBalance')) - def __init__(self,minconf,quiet): + async def __ainit__(self,minconf,quiet): - rpc_init() self.minconf = minconf self.quiet = quiet self.data = {k:[g.proto.coin_amt('0')] * 4 for k in ('TOTAL','Non-MMGen','Non-wallet')} - self.create_data() + await self.create_data() - def create_data(self): + async def create_data(self): # 0: unconfirmed, 1: below minconf, 2: confirmed, 3: spendable (privkey in wallet) lbl_id = ('account','label')['label_api' in g.rpc.caps] - for d in g.rpc.listunspent(0): + for d in await g.rpc.call('listunspent',0): lbl = get_tw_label(d[lbl_id]) if lbl: if lbl.mmid.type == 'mmgen': diff --git a/mmgen/tx.py b/mmgen/tx.py index 3e881a44..51ad734c 100755 --- a/mmgen/tx.py +++ b/mmgen/tx.py @@ -82,8 +82,7 @@ def mmaddr2coinaddr(mmaddr,ad_w,ad_f): return CoinAddr(coin_addr) def segwit_is_active(exit_on_error=False): - rpc_init() - d = g.rpc.getblockchaininfo() + d = g.rpc.cached['blockchaininfo'] if d['chain'] == 'regtest': return True if ( 'bip9_softforks' in d @@ -281,6 +280,7 @@ class MMGenTX(MMGenObject): sig_ext = 'sigtx' txid_ext = 'txid' desc = 'transaction' + hexdata_type = 'hex' fee_fail_fs = 'Network fee estimation for {c} confirmations failed ({t})' no_chg_msg = 'Warning: Change address will be deleted as transaction produces no change' rel_fee_desc = 'satoshis per byte' @@ -308,7 +308,11 @@ inputs must be supplied to '{pnl}-txsign' in a file with the '--keys-from-file' option. Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_name.lower()) - def __init__(self,filename=None,metadata_only=False,caller=None,quiet_open=False,offline=False): + def __init__(self,filename=None,metadata_only=False,caller=None,quiet_open=False,data=None,tw=None): + if data: + assert type(data) is dict, type(data) + self.__dict__ = data + return self.inputs = MMGenTxInputList() self.outputs = MMGenTxOutputList() self.send_amt = g.proto.coin_amt('0') # total amt minus change @@ -327,6 +331,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam self.dcoin = None self.caller = caller self.locktime = None + self.tw = tw if filename: self.parse_tx_file(filename,metadata_only=metadata_only,quiet_open=quiet_open) @@ -400,12 +405,12 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam def update_txid(self): self.txid = MMGenTxID(make_chksum_6(bytes.fromhex(self.hex)).upper()) - def create_raw(self): + async def create_raw(self): i = [{'txid':e.txid,'vout':e.vout} for e in self.inputs] if self.inputs[0].sequence: i[0]['sequence'] = self.inputs[0].sequence o = {e.addr:e.amt for e in self.outputs} - self.hex = HexStr(g.rpc.createrawtransaction(i,o)) + self.hex = HexStr(await g.rpc.call('createrawtransaction',i,o)) self.update_txid() def print_contract_addr(self): pass @@ -436,9 +441,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam def has_segwit_inputs(self): return any(i.mmid and i.mmid.mmtype in ('S','B') for i in self.inputs) - def compare_size_and_estimated_size(self): + def compare_size_and_estimated_size(self,tx_decoded): est_vsize = self.estimate_size() - d = g.rpc.decoderawtransaction(self.hex) + d = tx_decoded vsize = d['vsize'] if 'vsize' in d else d['size'] vmsg('\nVsize: {} (true) {} (estimated)'.format(vsize,est_vsize)) m1 = 'Estimated transaction vsize is {:1.2f} times the true vsize\n' @@ -522,8 +527,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam return int(ret * float(opt.vsize_adj)) if hasattr(opt,'vsize_adj') and opt.vsize_adj else ret # coin-specific fee routines - def get_relay_fee(self): - kb_fee = g.proto.coin_amt(g.rpc.getnetworkinfo()['relayfee']) + @property + def relay_fee(self): + kb_fee = g.proto.coin_amt(g.rpc.cached['networkinfo']['relayfee']) ret = kb_fee * self.estimate_size() // 1024 vmsg('Relay fee: {} {c}/kB, for transaction: {} {c}'.format(kb_fee,ret,c=g.coin)) return ret @@ -533,9 +539,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam unit = getattr(g.proto.coin_amt,to_unit or 'min_coin_unit') return int(abs_fee // unit // self.estimate_size()) - def get_rel_fee_from_network(self): + async def get_rel_fee_from_network(self): try: - ret = g.rpc.estimatesmartfee(opt.tx_confs,opt.fee_estimate_mode.upper()) + ret = await g.rpc.call('estimatesmartfee',opt.tx_confs,opt.fee_estimate_mode.upper()) fee_per_kb = ret['feerate'] if 'feerate' in ret else -2 fe_type = 'estimatesmartfee' except: @@ -577,9 +583,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam m = '{} {c}: {} fee too large (maximum fee: {} {c})' msg(m.format(abs_fee,desc,g.proto.max_tx_fee,c=g.coin)) return False - elif abs_fee < self.get_relay_fee(): + elif abs_fee < self.relay_fee: m = '{} {c}: {} fee too small (below relay fee of {} {c})' - msg(m.format(str(abs_fee),desc,str(self.get_relay_fee()),c=g.coin)) + msg(m.format(str(abs_fee),desc,str(self.relay_fee),c=g.coin)) return False else: return abs_fee @@ -626,14 +632,14 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam tx_fee = my_raw_input(self.usr_fee_prompt) desc = 'User-selected' - def get_fee_from_user(self,have_estimate_fail=[]): + async def get_fee_from_user(self,have_estimate_fail=[]): if opt.tx_fee: desc = 'User-selected' start_fee = opt.tx_fee else: desc = 'Network-estimated (mode: {})'.format(opt.fee_estimate_mode.upper()) - fee_per_kb,fe_type = self.get_rel_fee_from_network() + fee_per_kb,fe_type = await self.get_rel_fee_from_network() if fee_per_kb < 0: if not have_estimate_fail: @@ -677,11 +683,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam assert isinstance(val,int),'locktime value not an integer' self.hex = self.hex[:-8] + bytes.fromhex('{:08x}'.format(val))[::-1].hex() - def get_blockcount(self): - return int(g.rpc.getblockcount()) - def add_blockcount(self): - self.blockcount = self.get_blockcount() + self.blockcount = g.rpc.blockcount def format(self): self.inputs.check_coin_mismatch() @@ -718,75 +721,6 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam def get_non_mmaddrs(self,desc): return {i.addr for i in getattr(self,desc) if not i.mmid} - def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception - - if self.marked_signed(): - msg('Transaction is already signed!') - return False - - if not self.check_correct_chain(on_fail='return'): - return False - - if (self.has_segwit_inputs() or self.has_segwit_outputs()) and not g.proto.cap('segwit'): - ymsg("TX has Segwit inputs or outputs, but {} doesn't support Segwit!".format(g.coin)) - return False - - self.check_pubkey_scripts() - - qmsg('Passing {} key{} to {}'.format(len(keys),suf(keys),g.proto.daemon_name)) - - if self.has_segwit_inputs(): - from .addr import KeyGenerator,AddrGenerator - kg = KeyGenerator('std') - ag = AddrGenerator('segwit') - keydict = MMGenDict([(d.addr,d.sec) for d in keys]) - - sig_data = [] - for d in self.inputs: - e = {k:getattr(d,k) for k in ('txid','vout','scriptPubKey','amt')} - e['amount'] = e['amt'] - del e['amt'] - if d.mmid and d.mmid.mmtype == 'S': - e['redeemScript'] = ag.to_segwit_redeem_script(kg.to_pubhex(keydict[d.addr])) - sig_data.append(e) - - msg_r('Signing transaction{}...'.format(tx_num_str)) - wifs = [d.sec.wif for d in keys] - - try: - ret = g.rpc.signrawtransactionwithkey(self.hex,wifs,sig_data,g.proto.sighash_type) \ - if 'sign_with_key' in g.rpc.caps else \ - g.rpc.signrawtransaction(self.hex,sig_data,wifs,g.proto.sighash_type) - except Exception as e: - msg(yellow('This is not the BCH chain.\nRe-run the script without the --coin=bch option.' - if 'Invalid sighash param' in e.args[0] else e.args[0])) - return False - - if not ret['complete']: - msg('failed\n{} returned the following errors:'.format(g.proto.daemon_name.capitalize())) - msg(repr(ret['errors'])) - return False - - try: - self.hex = HexStr(ret['hex']) - self.compare_size_and_estimated_size() - dt = DeserializedTX(self.hex) - self.check_hex_tx_matches_mmgen_tx(dt) - self.coin_txid = CoinTxID(dt['txid'],on_fail='raise') - self.check_sigs(dt) - if not self.coin_txid == g.rpc.decoderawtransaction(ret['hex'])['txid']: - raise BadMMGenTxID('txid mismatch (after signing)') - msg('OK') - return True - except Exception as e: - try: m = '{}'.format(e.args[0]) - except: m = repr(e.args[0]) - msg('\n'+yellow(m)) - if g.traceback: - import traceback - ymsg('\n'+''.join(traceback.format_exception(*sys.exc_info()))) - return False - def mark_raw(self): self.desc = 'transaction' self.ext = self.raw_ext @@ -874,38 +808,45 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam def has_segwit_outputs(self): return any(o.mmid and o.mmid.mmtype in ('S','B') for o in self.outputs) - def get_status(self,status=False): + async def get_status(self,status=False): class r(object): pass - def is_in_wallet(): - ret = g.rpc.gettransaction(self.coin_txid,on_fail='silent') + async def is_in_wallet(): + try: ret = await g.rpc.call('gettransaction',self.coin_txid) + except: return False if 'confirmations' in ret and ret['confirmations'] > 0: r.confs = ret['confirmations'] return True else: return False - def is_in_utxos(): - return 'txid' in g.rpc.getrawtransaction(self.coin_txid,True,on_fail='silent') + async def is_in_utxos(): + try: return 'txid' in await g.rpc.call('getrawtransaction',self.coin_txid,True) + except: return False - def is_in_mempool(): - return 'height' in g.rpc.getmempoolentry(self.coin_txid,on_fail='silent') + async def is_in_mempool(): + try: return 'height' in await g.rpc.call('getmempoolentry',self.coin_txid) + except: return False - def is_replaced(): - if is_in_mempool(): return False - ret = g.rpc.gettransaction(self.coin_txid,on_fail='silent') - - if not 'bip125-replaceable' in ret or not 'confirmations' in ret or ret['confirmations'] > 0: + async def is_replaced(): + if await is_in_mempool(): return False + try: + ret = await g.rpc.call('gettransaction',self.coin_txid) + except: + return False + else: + if 'bip125-replaceable' in ret and 'confirmations' in ret and ret['confirmations'] <= 0: + r.replacing_confs = -ret['confirmations'] + r.replacing_txs = ret['walletconflicts'] + return True + else: + return False - r.replacing_confs = -ret['confirmations'] - r.replacing_txs = ret['walletconflicts'] - return True - - if is_in_mempool(): + if await is_in_mempool(): if status: - d = g.rpc.gettransaction(self.coin_txid,on_fail='silent') + d = await g.rpc.call('gettransaction',self.coin_txid) brs = 'bip125-replaceable' rep = '{}replaceable'.format(('NOT ','')[brs in d and d[brs]=='yes']) t = d['timereceived'] @@ -917,22 +858,23 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam msg('TX status: in mempool, {}\n{}'.format(rep,b)) else: msg('Warning: transaction is in mempool!') - elif is_in_wallet(): + elif await is_in_wallet(): die(0,'Transaction has {} confirmation{}'.format(r.confs,suf(r.confs))) - elif is_in_utxos(): + elif await is_in_utxos(): die(2,red('ERROR: transaction is in the blockchain (but not in the tracking wallet)!')) - elif is_replaced(): - m1 = 'Transaction has been replaced' - m2 = 'Replacement transaction is in mempool' - rc = r.replacing_confs - if rc: - m2 = 'Replacement transaction has {} confirmation{}'.format(rc,suf(rc)) - msg('{}\n{}'.format(m1,m2)) + elif await is_replaced(): + msg('Transaction has been replaced\nReplacement transaction ' + ( + f'has {r.replacing_confs} confirmation{suf(r.replacing_confs)}' + if r.replacing_confs else + 'is in mempool' )) if not opt.quiet: msg('Replacing transactions:') - d = ((t,g.rpc.getmempoolentry(t,on_fail='silent')) for t in r.replacing_txs) - for txid,mp_entry in d: - msg(' {}{}'.format(txid,' in mempool' if ('height' in mp_entry) else '')) + d = [] + for txid in r.replacing_txs: + try: d.append(await g.rpc.call('getmempoolentry',txid)) + except: d.append({}) + for txid,mp_entry in zip(r.replacing_txs,d): + msg(f' {txid}' + ('',' in mempool')['height' in mp_entry]) die(0,'') def confirm_send(self): @@ -942,8 +884,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam confirm_or_raise(m1,m2,m3) msg('Sending transaction') - def send(self,prompt_user=True,exit_on_fail=False): - + async def send(self,prompt_user=True,exit_on_fail=False): if not self.marked_signed(): die(1,'Transaction is not signed!') @@ -961,15 +902,21 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam die(2,'Transaction fee ({}) greater than {} max_tx_fee ({} {})!'.format( self.get_fee_from_tx(),g.proto.name.capitalize(),g.proto.max_tx_fee,g.coin)) - self.get_status() + await self.get_status() - if prompt_user: self.confirm_send() + if prompt_user: + self.confirm_send() - ret = None if g.bogus_send else g.rpc.sendrawtransaction(self.hex,on_fail='return') + if g.bogus_send: + ret = None + else: + try: + ret = await g.rpc.call('sendrawtransaction',self.hex) + except Exception as e: + ret = False - from .rpc import rpc_error,rpc_errmsg - if rpc_error(ret): - errmsg = rpc_errmsg(ret) + if ret == False: + errmsg = e if 'Signature must use SIGHASH_FORKID' in errmsg: m = 'The Aug. 1 2017 UAHF has activated on this chain.' m += "\nRe-run the script with the --coin=bch option." @@ -1061,7 +1008,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam msg('') # def is_replaceable_from_rpc(self): -# dec_tx = g.rpc.decoderawtransaction(self.hex) +# dec_tx = await g.rpc.call('decoderawtransaction',self.hex) # return None < dec_tx['vin'][0]['sequence'] <= g.max_int - 2 def is_replaceable(self): @@ -1138,8 +1085,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam blockcount = None if g.proto.base_coin != 'ETH': try: - rpc_init() - blockcount = self.get_blockcount() + blockcount = g.rpc.blockcount except: pass @@ -1187,6 +1133,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam def check_txfile_hex_data(self): self.hex = HexStr(self.hex,on_fail='raise') + def parse_txfile_hex_data(self): + pass + def parse_tx_file(self,infile,metadata_only=False,quiet_open=False): def eval_io_data(raw_data,desc): @@ -1271,6 +1220,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam desc = 'transaction file hex data' self.check_txfile_hex_data() + desc = f'transaction file {self.hexdata_type} data' + self.parse_txfile_hex_data() # the following ops will all fail if g.coin doesn't match self.coin desc = 'coin type in metadata' assert self.coin == g.coin,self.coin @@ -1286,7 +1237,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam self.chain = 'mainnet' if self.dcoin: - self.resolve_g_token_from_tx_file() + self.resolve_g_token_from_txfile() + g.dcoin = self.dcoin def process_cmd_arg(self,arg,ad_f,ad_w): @@ -1320,8 +1272,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam if not self.outputs: die(2,'At least one output must be specified on the command line') - def get_outputs_from_cmdline(self,cmd_args): - from .addr import AddrList,AddrData + async def get_outputs_from_cmdline(self,cmd_args): + from .addr import AddrList,AddrData,TwAddrData addrfiles = [a for a in cmd_args if get_extension(a) == AddrList.ext] cmd_args = set(cmd_args) - set(addrfiles) @@ -1330,7 +1282,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam check_infile(a) ad_f.add(AddrList(a)) - ad_w = AddrData(source='tw',wallet=self.twuo.wallet) + ad_w = await TwAddrData(wallet=self.tw) self.process_cmd_args(cmd_args,ad_f,ad_w) @@ -1349,7 +1301,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam msg('Unspent output number must be <= {}'.format(len(unspent))) # we don't know fee yet, so perform preliminary check with fee == 0 - def precheck_sufficient_funds(self,inputs_sum,sel_unspent): + async def precheck_sufficient_funds(self,inputs_sum,sel_unspent): if self.twuo.total < self.send_amt: msg(self.msg_wallet_low_coin.format(self.send_amt-inputs_sum,g.dcoin)) return False @@ -1358,7 +1310,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam return False return True - def get_change_amt(self): + async def get_change_amt(self): return self.sum_inputs() - self.send_amt - self.fee def warn_insufficient_chg(self,change_amt): @@ -1394,11 +1346,11 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam return set(sel_nums) # silently discard duplicates - def get_cmdline_input_addrs(self): + async def get_cmdline_input_addrs(self): # Bitcoin full node, call doesn't go to the network, so just call listunspent with addrs=[] return [] - def get_inputs_from_user(self): + async def get_inputs_from_user(self): while True: us_f = self.select_unspent_cmdline if opt.inputs else self.select_unspent @@ -1408,7 +1360,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam sel_unspent = self.twuo.MMGenTwOutputList([self.twuo.unspent[i-1] for i in sel_nums]) inputs_sum = sum(s.amt for s in sel_unspent) - if not self.precheck_sufficient_funds(inputs_sum,sel_unspent): + if not await self.precheck_sufficient_funds(inputs_sum,sel_unspent): continue non_mmaddrs = [i for i in sel_unspent if i.twmmid.type == 'non-mmgen'] @@ -1420,9 +1372,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam self.copy_inputs_from_tw(sel_unspent) # makes self.inputs - self.fee = self.get_fee_from_user() + self.fee = await self.get_fee_from_user() - change_amt = self.get_change_amt() + change_amt = await self.get_change_amt() if change_amt >= 0: p = self.final_inputs_ok_msg(change_amt) @@ -1439,27 +1391,34 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam if not self.send_amt: self.send_amt = change_amt - def create(self,cmd_args,locktime,do_info=False): + async def set_token_params(self): + pass + + async def create(self,cmd_args,locktime,do_info=False): assert isinstance(locktime,int),'locktime must be of type int' - if opt.comment_file: self.add_comment(opt.comment_file) - - twuo_addrs = self.get_cmdline_input_addrs() - from .tw import TwUnspentOutputs - self.twuo = TwUnspentOutputs(minconf=opt.minconf,addrs=twuo_addrs) + + if opt.comment_file: + self.add_comment(opt.comment_file) + + twuo_addrs = await self.get_cmdline_input_addrs() + + self.twuo = await TwUnspentOutputs(minconf=opt.minconf,addrs=twuo_addrs) + await self.twuo.get_unspent_data() if not do_info: - self.get_outputs_from_cmdline(cmd_args) + await self.get_outputs_from_cmdline(cmd_args) do_license_msg() if not opt.inputs: - self.twuo.view_and_sort(self) + await self.twuo.view_and_sort(self) self.twuo.display_total() if do_info: + del self.twuo.wallet sys.exit(0) self.send_amt = self.sum_outputs() @@ -1468,7 +1427,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam ('Unknown','{} {}'.format(self.send_amt.hl(),g.dcoin))[bool(self.send_amt)] )) - change_amt = self.get_inputs_from_user() + change_amt = await self.get_inputs_from_user() self.update_change_output(change_amt) self.update_send_amt(change_amt) @@ -1482,7 +1441,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam if not opt.yes: self.add_comment() # edits an existing comment - self.create_raw() # creates self.hex, self.txid + + await self.create_raw() # creates self.hex, self.txid if g.proto.base_proto == 'Bitcoin' and locktime: msg('Setting nlocktime to {}!'.format(strfmt_locktime(locktime))) @@ -1501,9 +1461,85 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam if not opt.yes: self.view_with_prompt('View decoded transaction?') - del self.twuo + del self.twuo.wallet -class MMGenBumpTX(MMGenTX): +class MMGenTxForSigning(MMGenTX): + + hexdata_type = 'json' + + def __new__(cls,*args,**kwargs): + return MMGenObject.__new__(altcoin_subclass(cls,'tx','MMGenTxForSigning')) + + async def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception + + if self.marked_signed(): + msg('Transaction is already signed!') + return False + + if not self.check_correct_chain(on_fail='return'): + return False + + if (self.has_segwit_inputs() or self.has_segwit_outputs()) and not g.proto.cap('segwit'): + ymsg("TX has Segwit inputs or outputs, but {} doesn't support Segwit!".format(g.coin)) + return False + + self.check_pubkey_scripts() + + qmsg('Passing {} key{} to {}'.format(len(keys),suf(keys),g.proto.daemon_name)) + + if self.has_segwit_inputs(): + from .addr import KeyGenerator,AddrGenerator + kg = KeyGenerator('std') + ag = AddrGenerator('segwit') + keydict = MMGenDict([(d.addr,d.sec) for d in keys]) + + sig_data = [] + for d in self.inputs: + e = {k:getattr(d,k) for k in ('txid','vout','scriptPubKey','amt')} + e['amount'] = e['amt'] + del e['amt'] + if d.mmid and d.mmid.mmtype == 'S': + e['redeemScript'] = ag.to_segwit_redeem_script(kg.to_pubhex(keydict[d.addr])) + sig_data.append(e) + + msg_r('Signing transaction{}...'.format(tx_num_str)) + wifs = [d.sec.wif for d in keys] + + try: + args = ( + ('signrawtransaction',self.hex,sig_data,wifs,g.proto.sighash_type), + ('signrawtransactionwithkey',self.hex,wifs,sig_data,g.proto.sighash_type) + )['sign_with_key' in g.rpc.caps] + ret = await g.rpc.call(*args) + except Exception as e: + msg(yellow(( + e.args[0], + 'This is not the BCH chain.\nRe-run the script without the --coin=bch option.' + )['Invalid sighash param' in e.args[0]])) + return False + + try: + self.hex = HexStr(ret['hex']) + tx_decoded = await g.rpc.call('decoderawtransaction',ret['hex']) + self.compare_size_and_estimated_size(tx_decoded) + dt = DeserializedTX(self.hex) + self.check_hex_tx_matches_mmgen_tx(dt) + self.coin_txid = CoinTxID(dt['txid'],on_fail='raise') + self.check_sigs(dt) + if not self.coin_txid == tx_decoded['txid']: + raise BadMMGenTxID('txid mismatch (after signing)') + msg('OK') + return True + except Exception as e: + try: m = '{}'.format(e.args[0]) + except: m = repr(e.args[0]) + msg('\n'+yellow(m)) + if g.traceback: + import traceback + ymsg('\n'+''.join(traceback.format_exception(*sys.exc_info()))) + return False + +class MMGenBumpTX(MMGenTxForSigning): def __new__(cls,*args,**kwargs): return MMGenTX.__new__(altcoin_subclass(cls,'tx','MMGenBumpTX'),*args,**kwargs) @@ -1511,9 +1547,8 @@ class MMGenBumpTX(MMGenTX): min_fee = None bump_output_idx = None - def __init__(self,filename,send=False): - - super().__init__(filename) + def __init__(self,filename,send=False,tw=None): + super().__init__(filename,tw=tw) if not self.is_replaceable(): die(1,"Transaction '{}' is not replaceable".format(self.txid)) @@ -1576,8 +1611,9 @@ class MMGenBumpTX(MMGenTX): self.bump_output_idx = idx return idx - def set_min_fee(self): - self.min_fee = self.sum_inputs() - self.sum_outputs() + self.get_relay_fee() + @property + def min_fee(self): + return self.sum_inputs() - self.sum_outputs() + self.relay_fee def update_fee(self,op_idx,fee): amt = self.sum_inputs() - self.sum_outputs(exclude=op_idx) - fee @@ -1598,10 +1634,10 @@ class MMGenBumpTX(MMGenTX): # NOT MAINTAINED class MMGenSplitTX(MMGenTX): - def get_outputs_from_cmdline(self,mmid): # TODO: check that addr is empty + async def get_outputs_from_cmdline(self,mmid): # TODO: check that addr is empty - from .addr import AddrData - ad_w = AddrData(source='tw') + from .addr import TwAddrData + ad_w = await TwAddrData() if is_mmgen_id(mmid): coin_addr = mmaddr2coinaddr(mmid,ad_w,None) if is_mmgen_id(mmid) else CoinAddr(mmid) @@ -1620,17 +1656,12 @@ class MMGenSplitTX(MMGenTX): g.rpc_host = opt.rpc_host2 if opt.tx_fees: opt.tx_fee = opt.tx_fees.split(',')[1] - try: - rpc_init(reinit=True) - except: - ymsg('Connect to {} daemon failed. Network fee estimation unavailable'.format(g.coin)) - return self.get_usr_fee_interactive(opt.tx_fee,'User-selected') return super().get_fee_from_user() - def create_split(self,mmid): + async def create_split(self,mmid): self.outputs = self.MMGenTxOutputList() - self.get_outputs_from_cmdline(mmid) + await self.get_outputs_from_cmdline(mmid) while True: change_amt = self.sum_inputs() - self.get_split_fee_from_user() @@ -1647,7 +1678,8 @@ class MMGenSplitTX(MMGenTX): if not opt.yes: self.add_comment() # edits an existing comment - self.create_raw() # creates self.hex, self.txid + + await self.create_raw() # creates self.hex, self.txid self.add_timestamp() self.add_blockcount() # TODO diff --git a/mmgen/txsign.py b/mmgen/txsign.py index 12ac7a37..6e93986b 100755 --- a/mmgen/txsign.py +++ b/mmgen/txsign.py @@ -139,7 +139,7 @@ def get_keylist(opt): return kal return None -def txsign(tx,seed_files,kl,kal,tx_num_str=''): +async def txsign(tx,seed_files,kl,kal,tx_num_str=''): keys = MMGenList() # list of AddrListEntry objects non_mm_addrs = tx.get_non_mmaddrs('inputs') @@ -169,4 +169,4 @@ def txsign(tx,seed_files,kl,kal,tx_num_str=''): if extra_sids: msg('Unused Seed ID{}: {}'.format(suf(extra_sids),' '.join(extra_sids))) - return tx.sign(tx_num_str,keys) # returns True or False + return await tx.sign(tx_num_str,keys) # returns True or False diff --git a/mmgen/util.py b/mmgen/util.py index b2c4dfa4..d6efcab4 100755 --- a/mmgen/util.py +++ b/mmgen/util.py @@ -818,35 +818,32 @@ def do_license_msg(immed=False): msg_r('\r') msg('') -def get_daemon_cfg_options(cfg_keys): - +# TODO: these belong in protocol.py +def get_coin_daemon_cfg_fn(): # Use dirname() to remove 'bob' or 'alice' component cfg_dir = os.path.dirname(g.data_dir) if g.regtest else g.proto.daemon_data_dir - cfg_file = os.path.join(cfg_dir,g.proto.name+'.conf' ) + return os.path.join(cfg_dir,g.proto.name+'.conf' ) +def get_coin_daemon_cfg_options(req_keys): + + fn = get_coin_daemon_cfg_fn() try: - lines = get_lines_from_file(cfg_file,'',silent=not opt.verbose) - kv_pairs = [l.split('=') for l in lines] - cfg = {k:v for k,v in kv_pairs if k in cfg_keys} + lines = get_lines_from_file(fn,'',silent=not opt.verbose) except: - vmsg("Warning: '{}' does not exist or is unreadable".format(cfg_file)) - cfg = {} + vmsg(f'Warning: {fn!r} does not exist or is unreadable') + return dict((k,None) for k in req_keys) - for k in set(cfg_keys) - set(cfg.keys()): cfg[k] = '' + def gen(): + for key in req_keys: + val = None + for l in lines: + if l.startswith(key): + res = l.split('=',1) + if len(res) == 2 and not ' ' in res[1].strip(): + val = res[1].strip() + yield (key,val) - return cfg - -def get_coin_daemon_auth_cookie(): - f = os.path.join(g.proto.daemon_data_dir,g.proto.daemon_data_subdir,'.cookie') - return get_lines_from_file(f,'')[0] if file_is_readable(f) else '' - -def rpc_init(reinit=False): - if not 'rpc' in g.proto.mmcaps: - die(1,'Coin daemon operations not supported for coin {}!'.format(g.coin)) - if g.rpc != None and not reinit: return g.rpc - from .rpc import init_daemon - g.rpc = init_daemon(g.proto.daemon_family) - return g.rpc + return dict(gen()) def format_par(s,indent=0,width=80,as_list=False): words,lines = s.split(),[] @@ -886,3 +883,28 @@ def get_network_id(coin=None,testnet=None): if coin == None: assert testnet == None if coin != None: assert testnet != None return (coin or g.coin).lower() + ('','_tn')[testnet or g.testnet] + +def run_session(callback,do_rpc_init=True,backend=None): + backend = backend or opt.rpc_backend + import asyncio + async def do(): + if backend == 'aiohttp': + import aiohttp + async with aiohttp.ClientSession( + headers = { 'Content-Type': 'application/json' }, + connector = aiohttp.TCPConnector(limit_per_host=g.aiohttp_rpc_queue_len), + ) as g.session: + if do_rpc_init: + from .rpc import rpc_init + await rpc_init(backend=backend) + ret = await callback + g.session = None + return ret + else: + if do_rpc_init: + from .rpc import rpc_init + await rpc_init(backend=backend) + return await callback + + # return asyncio.run(do()) # Python 3.7+ + return asyncio.get_event_loop().run_until_complete(do()) diff --git a/setup.py b/setup.py index 75cd42bc..beb5c2f8 100755 --- a/setup.py +++ b/setup.py @@ -114,6 +114,7 @@ setup( 'mmgen.filename', 'mmgen.globalvars', 'mmgen.keccak', + 'mmgen.led', 'mmgen.license', 'mmgen.mn_electrum', 'mmgen.mn_entry', diff --git a/test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.sigtx b/test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.sigtx new file mode 100644 index 00000000..989d1695 --- /dev/null +++ b/test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.sigtx @@ -0,0 +1,7 @@ +e9feb9 +ETH KOVAN B472BD 23.45495 20180725_111111 7654321 +f86d808509502f900082520894b7d06382a998817a16ba487a99636bf8cae29d9d89014580b8c15e8c60008077a057b7b73b9102fda68922eba0fbb70171425f1f27bad944838d3ded819eb03a63a00acb20276cbb30209923f959c63595b1b35663cbb30d186c6a5a2dbccb3e07fa +[{'confs': 0, 'label': '', 'mmid': '98831F3A:E:1', 'amt': '123.456', 'addr': '97ccc3a117b3696340c42561361054b1c9c793d5'}] +[{'mmid': '98831F3A:E:2', 'amt': '23.45495', 'addr': '2a6db46c87407e6d28fcb97d3bd0f5cf4aafca46'}] +qRHzrPVpZFYxnQvk3atLzUtp41bZupJ2UQNnKe3ZnmqFsEngS6vaCCvesKKy9khzVq6y2RqarVBcZLnjtXxMpbAcdEtyBWiBYmZdoU8SN4uAbroHT1c7gEbmUNVKKdqHD86ZRRqDNpdh1ztmLiMAy3ibM83puwJHNpGGHgUGjZ1RSEgyVKCs2rZ9wXN8rBMibDDPYo1LgtAst2FkB36Mgf4Vf7ekoRAdiRNGd5YZ3RXAVsSdnZcyn4rdeQDMDkCq7JJDoB25eNEuXQutZFUcf2fEfxkMbW1sXJDNFQq +0f277d20bf3793f94521a809943a659478bdfa6836a399f0568a93aeb4ce5184 diff --git a/test/ref/ethereum/tracking-wallet-v2.json b/test/ref/ethereum/tracking-wallet-v2.json index 6a7289fb..77b02bd3 100644 --- a/test/ref/ethereum/tracking-wallet-v2.json +++ b/test/ref/ethereum/tracking-wallet-v2.json @@ -1 +1 @@ -{"coin":"ETH","accounts":{"e704b6cfd9f0edb2e6cfbd0c913438d37ede7b35":{"mmid":"98831F3A:E:1","comment":""},"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef":{"mmid":"eth:deadbeefdeadbeefdeadbeefdeadbeefdeadbeef","comment":""}},"tokens":{"d5f051401ca478b34c80d0b5a119e437dc6d9df5":{"6e0fbe42e1343309b3ccb9068dbad6132f86c96f":{"mmid":"98831F3A:E:12","comment":""},"b72268fa55a57fe838a745b27c90f0edb415af61":{"mmid":"98831F3A:E:13","comment":""}},"3dd0864668c36d27b53a98137764c99f9fd5b7b2":{"ca2b705adf43151ce61e01c653424e9790e9eb84":{"mmid":"98831F3A:E:22","comment":""},"7227a8fd728b74208255c6a8a09cb9fb66b1230c":{"mmid":"98831F3A:E:23","comment":""}}}} +{"coin":"ETH","accounts":{"e704b6cfd9f0edb2e6cfbd0c913438d37ede7b35":{"mmid":"98831F3A:E:1","comment":""},"b7d06382a998817a16ba487a99636bf8cae29d9d":{"mmid":"98831F3A:E:2","comment":""},"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef":{"mmid":"eth:deadbeefdeadbeefdeadbeefdeadbeefdeadbeef","comment":""}},"tokens":{"d5f051401ca478b34c80d0b5a119e437dc6d9df5":{"6e0fbe42e1343309b3ccb9068dbad6132f86c96f":{"mmid":"98831F3A:E:12","comment":""},"b72268fa55a57fe838a745b27c90f0edb415af61":{"mmid":"98831F3A:E:13","comment":""}},"3dd0864668c36d27b53a98137764c99f9fd5b7b2":{"ca2b705adf43151ce61e01c653424e9790e9eb84":{"mmid":"98831F3A:E:22","comment":""},"7227a8fd728b74208255c6a8a09cb9fb66b1230c":{"mmid":"98831F3A:E:23","comment":""}}}} diff --git a/test/test.py b/test/test.py index 3d3b4cc3..6a9c5a91 100755 --- a/test/test.py +++ b/test/test.py @@ -492,6 +492,7 @@ class CmdGroupMgr(object): cmd_groups_extra = { 'autosign_btc': ('TestSuiteAutosignBTC',{'modname':'autosign'}), 'autosign_live': ('TestSuiteAutosignLive',{'modname':'autosign'}), + 'autosign_live_simulate': ('TestSuiteAutosignLiveSimulate',{'modname':'autosign'}), 'create_ref_tx': ('TestSuiteRefTX',{'modname':'misc','full_data':True}), } @@ -867,7 +868,10 @@ class TestSuiteRunner(object): if k in cfg: setattr(self.ts,k,cfg[k]) - self.process_retval(cmd,getattr(self.ts,cmd)(*arg_list)) # run the test + ret = getattr(self.ts,cmd)(*arg_list) # run the test + if type(ret).__name__ == 'coroutine': + ret = run_session(ret) + self.process_retval(cmd,ret) if opt.profile: omsg('\r\033[50C{:.4f}'.format(time.time() - start)) diff --git a/test/test_py_d/ts_autosign.py b/test/test_py_d/ts_autosign.py index 42ee5150..0d42b193 100755 --- a/test/test_py_d/ts_autosign.py +++ b/test/test_py_d/ts_autosign.py @@ -43,21 +43,26 @@ class TestSuiteAutosign(TestSuiteBase): def autosign_live(self): return self.autosign_btc(live=True) - def autosign_btc(self,live=False): + def autosign_live_simulate(self): + return self.autosign_btc(live=True,simulate=True) + + def autosign_btc(self,live=False,simulate=False): return self.autosign( coins=['btc'], daemon_coins=['btc'], txfiles=['btc'], txcount=3, - live=live) + live=live, + simulate=simulate ) - # tests everything except device detection, mount/unmount + # tests everything except mount/unmount def autosign( self, coins=['btc','bch','ltc','eth'], daemon_coins=['btc','bch','ltc'], txfiles=['btc','bch','ltc','eth','mm1','etc'], txcount=12, - live=False): + live=False, + simulate=False): if self.skip_for_win(): return 'skip' @@ -79,13 +84,16 @@ class TestSuiteAutosign(TestSuiteBase): wf = t.written_to_file('Autosign wallet') t.ok() - def copy_files(mountpoint,remove_signed_only=False,include_bad_tx=True): - fdata_in = (('btc',''), - ('bch',''), - ('ltc','litecoin'), - ('eth','ethereum'), - ('mm1','ethereum'), - ('etc','ethereum_classic')) + def copy_files( + mountpoint, + remove_signed_only=False, + include_bad_tx=True, + fdata_in = (('btc',''), + ('bch',''), + ('ltc','litecoin'), + ('eth','ethereum'), + ('mm1','ethereum'), + ('etc','ethereum_classic')) ): fdata = [e for e in fdata_in if e[0] in txfiles] from .ts_ref import TestSuiteRef tfns = [TestSuiteRef.sources['ref_tx_file'][c][1] for c,d in fdata] + \ @@ -124,8 +132,11 @@ class TestSuiteAutosign(TestSuiteBase): omsg_r(blue('\nRemove removable device and then hit ENTER ')) input() - if gen_wallet: make_wallet(opts) - else: do_mount() + if gen_wallet: + if not opt.skip_deps: + make_wallet(opts) + else: + do_mount() copy_files(mountpoint,include_bad_tx=not led_opts) @@ -150,7 +161,7 @@ class TestSuiteAutosign(TestSuiteBase): do_unmount() omsg(green(m1)) - t = self.spawn('mmgen-autosign',opts+led_opts+['wait'],extra_desc=desc) + t = self.spawn('mmgen-autosign',opts+led_opts+['--quiet','--no-summary','wait'],extra_desc=desc) if not opt.exact_output: omsg('') do_loop() do_mount() # race condition due to device insertion detection @@ -160,6 +171,8 @@ class TestSuiteAutosign(TestSuiteBase): imsg(purple('\nKilling wait loop!')) t.kill(2) # 2 = SIGINT t.req_exit_val = 1 + if simulate and led_opts: + t.expect("Stopping LED") return t def do_autosign(opts,mountpoint): @@ -179,15 +192,44 @@ class TestSuiteAutosign(TestSuiteBase): t.ok() copy_files(mountpoint,remove_signed_only=True) - t = self.spawn('mmgen-autosign',opts+['wait'],extra_desc='(sign)') + t = self.spawn('mmgen-autosign',opts+['--quiet','wait'],extra_desc='(sign)') t.expect('{} transactions signed'.format(txcount)) t.expect('2 transactions failed to sign') t.expect('Waiting') t.kill(2) t.req_exit_val = 1 imsg('') + t.ok() + + copy_files(mountpoint,include_bad_tx=True,fdata_in=(('btc',''),)) + t = self.spawn( + 'mmgen-autosign', + opts + ['--quiet','--stealth-led','wait'], + extra_desc='(sign - --quiet --stealth-led)' ) + t.expect('2 transactions failed to sign') + t.expect('Waiting') + t.kill(2) + t.req_exit_val = 1 + imsg('') + t.ok() + + copy_files(mountpoint,include_bad_tx=False,fdata_in=(('btc',''),)) + t = self.spawn( + 'mmgen-autosign', + opts + ['--quiet','--led'], + extra_desc='(sign - --quiet --led)' ) + t.read() + imsg('') + t.ok() + return t + # begin execution + + if simulate and not opt.exact_output: + rmsg('This command must be run with --exact-output enabled!') + return False + network_ids = [c+'_tn' for c in daemon_coins] + daemon_coins start_test_daemons(*network_ids) @@ -205,27 +247,27 @@ class TestSuiteAutosign(TestSuiteBase): ydie(1,"Directory '{}' does not exist! Exiting".format(mountpoint)) opts = ['--coins='+','.join(coins)] - led_files = { 'opi': ('/sys/class/leds/orangepi:red:status/brightness',), - 'rpi': ('/sys/class/leds/led0/brightness','/sys/class/leds/led0/trigger') } - for k in ('opi','rpi'): - if os.path.exists(led_files[k][0]): - led_support = k - break + from mmgen.led import LEDControl + + if simulate: + LEDControl.create_dummy_control_files() + + try: + cf = LEDControl(enabled=True,simulate=simulate) + except: + ret = "'no LED support detected'" else: - led_support = None - - if led_support: - for fn in (led_files[led_support]): - run(['sudo','chmod','0666',fn],check=True) + for fn in (cf.board.status,cf.board.trigger): + if fn: + run(['sudo','chmod','0666',fn],check=True) + os.environ['MMGEN_TEST_SUITE_AUTOSIGN_LIVE'] = '1' omsg(purple('Running autosign test with no LED')) do_autosign_live(opts,mountpoint) omsg(purple("Running autosign test with '--led'")) do_autosign_live(opts,mountpoint,led_opts=['--led'],gen_wallet=False) omsg(purple("Running autosign test with '--stealth-led'")) ret = do_autosign_live(opts,mountpoint,led_opts=['--stealth-led'],gen_wallet=False) - else: - ret = do_autosign_live(opts,mountpoint) else: mountpoint = self.tmpdir opts = ['--no-insert-check','--mountpoint='+mountpoint,'--coins='+','.join(coins)] @@ -247,3 +289,9 @@ class TestSuiteAutosignLive(TestSuiteAutosignBTC): cmd_group = ( ('autosign_live', 'transaction autosigning (BTC,ETH,ETC - test device insertion/removal + LED)'), ) + +class TestSuiteAutosignLiveSimulate(TestSuiteAutosignBTC): + 'live autosigning operations with device insertion/removal and LED check in simulated environment' + cmd_group = ( + ('autosign_live_simulate', 'transaction autosigning (BTC,ETH,ETC - test device insertion/removal + simulated LED)'), + ) diff --git a/test/test_py_d/ts_ethdev.py b/test/test_py_d/ts_ethdev.py index ab2da725..b2de0f9d 100755 --- a/test/test_py_d/ts_ethdev.py +++ b/test/test_py_d/ts_ethdev.py @@ -211,7 +211,8 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): ('token_addrgen', 'generating token addresses'), ('token_addrimport_badaddr1','importing token addresses (no token address)'), ('token_addrimport_badaddr2','importing token addresses (bad token address)'), - ('token_addrimport', 'importing token addresses'), + ('token_addrimport', 'importing token addresses'), + ('token_addrimport_batch','importing token addresses (dummy batch mode)'), ('bal7', 'the {} balance'.format(g.coin)), ('token_bal1', 'the {} balance and token balance'.format(g.coin)), @@ -351,7 +352,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): def addrimport(self,ext='21-23]{}.addrs',expect='9/9',add_args=[],bad_input=False): ext = ext.format('-α' if g.debug_utf8 else '') fn = self.get_file_with_ext(ext,no_dot=True,delete=False) - t = self.spawn('mmgen-addrimport', self.eth_args[1:] + add_args + [fn]) + t = self.spawn('mmgen-addrimport', self.eth_args[1:-1] + add_args + [fn]) if bad_input: t.read() return t @@ -507,9 +508,11 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): t = self.spawn('mmgen-tool', self.eth_args + ['--token=mm1','twview','wide=1']) for b in token_bals[n]: addr,_amt1,_amt2,adj = b if len(b) == 4 else b + (False,) - if adj and g.coin == 'ETC': _amt2 = str(Decimal(_amt2) + Decimal(adj[1]) * self.bal_corr) + if adj and g.coin == 'ETC': + _amt2 = str(Decimal(_amt2) + Decimal(adj[1]) * self.bal_corr) pat = r'{}\s+{}\s+{}\s'.format(addr,_amt1.replace('.',r'\.'),_amt2.replace('.',r'\.')) t.expect(pat,regex=True) + t.expect('Total MM1:') t.read() return t @@ -577,12 +580,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): token_data = { 'name':'MMGen Token 2', 'symbol':'MM2', 'supply':10**18, 'decimals':10 } return self.token_compile(token_data) - def _rpc_init(self): - g.proto.rpc_port = self.rpc_port - rpc_init() - - def token_deploy(self,num,key,gas,mmgen_cmd='txdo',tx_fee='8G'): - self._rpc_init() + async def token_deploy(self,num,key,gas,mmgen_cmd='txdo',tx_fee='8G'): keyfile = joinpath(self.tmpdir,parity_key_fn) fn = joinpath(self.tmpdir,'mm'+str(num),key+'.bin') os.environ['MMGEN_BOGUS_SEND'] = '' @@ -609,62 +607,63 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): bogus_send=False) addr = t.expect_getend('Contract address: ') from mmgen.altcoins.eth.tx import EthereumMMGenTX as etx - assert etx.get_exec_status(txid,True) != 0,( + assert (await etx.get_exec_status(txid,True)) != 0,( "Contract '{}:{}' failed to execute. Aborting".format(num,key)) if key == 'Token': self.write_to_tmpfile('token_addr{}'.format(num),addr+'\n') imsg('\nToken MM{} deployed!'.format(num)) return t - def token_deploy1a(self): return self.token_deploy(num=1,key='SafeMath',gas=200000) - def token_deploy1b(self): return self.token_deploy(num=1,key='Owned',gas=250000) - def token_deploy1c(self): return self.token_deploy(num=1,key='Token',gas=1100000,tx_fee='7G') + async def token_deploy1a(self): return await self.token_deploy(num=1,key='SafeMath',gas=200000) + async def token_deploy1b(self): return await self.token_deploy(num=1,key='Owned',gas=250000) + async def token_deploy1c(self): return await self.token_deploy(num=1,key='Token',gas=1100000,tx_fee='7G') def tx_status2(self): return self.tx_status(ext=g.coin+'[0,7000]{}.sigtx',expect_str='successfully executed') def bal6(self): return self.bal5() - def token_deploy2a(self): return self.token_deploy(num=2,key='SafeMath',gas=200000) - def token_deploy2b(self): return self.token_deploy(num=2,key='Owned',gas=250000) - def token_deploy2c(self): return self.token_deploy(num=2,key='Token',gas=1100000) + async def token_deploy2a(self): return await self.token_deploy(num=2,key='SafeMath',gas=200000) + async def token_deploy2b(self): return await self.token_deploy(num=2,key='Owned',gas=250000) + async def token_deploy2c(self): return await self.token_deploy(num=2,key='Token',gas=1100000) - def contract_deploy(self): # test create,sign,send - return self.token_deploy(num=2,key='SafeMath',gas=1100000,mmgen_cmd='txcreate') + async def contract_deploy(self): # test create,sign,send + return await self.token_deploy(num=2,key='SafeMath',gas=1100000,mmgen_cmd='txcreate') - def token_transfer_ops(self,op,amt=1000): + async def token_transfer_ops(self,op,amt=1000): self.spawn('',msg_only=True) sid = dfl_sid from mmgen.tool import MMGenToolCmdWallet usr_mmaddrs = ['{}:E:{}'.format(sid,i) for i in (11,21)] usr_addrs = [MMGenToolCmdWallet().gen_addr(addr,dfl_words_file) for addr in usr_mmaddrs] - self._rpc_init() - from mmgen.altcoins.eth.contract import Token + from mmgen.altcoins.eth.contract import TokenResolve from mmgen.altcoins.eth.tx import EthereumMMGenTX as etx - def do_transfer(): + async def do_transfer(): for i in range(2): - tk = Token(self.read_from_tmpfile('token_addr{}'.format(i+1)).strip()) - imsg_r('\n'+tk.info()) - imsg('dev token balance (pre-send): {}'.format(tk.balance(dfl_addr))) + tk = await TokenResolve(self.read_from_tmpfile('token_addr{}'.format(i+1)).strip()) + imsg_r('\n' + await tk.info()) + imsg('dev token balance (pre-send): {}'.format(await tk.get_balance(dfl_addr))) imsg('Sending {} {} to address {} ({})'.format(amt,g.coin,usr_addrs[i],usr_mmaddrs[i])) from mmgen.obj import ETHAmt - txid = tk.transfer( dfl_addr, usr_addrs[i], amt, dfl_privkey, + txid = await tk.transfer( dfl_addr, usr_addrs[i], amt, dfl_privkey, start_gas = ETHAmt(60000,'wei'), gasPrice = ETHAmt(8,'Gwei') ) - assert etx.get_exec_status(txid,True) != 0,'Transfer of token funds failed. Aborting' + assert (await etx.get_exec_status(txid,True)) != 0,'Transfer of token funds failed. Aborting' - def show_bals(): + async def show_bals(): for i in range(2): - tk = Token(self.read_from_tmpfile('token_addr{}'.format(i+1)).strip()) - imsg('Token: {}'.format(tk.symbol())) - imsg('dev token balance: {}'.format(tk.balance(dfl_addr))) + tk = await TokenResolve(self.read_from_tmpfile(f'token_addr{i+1}').strip()) + imsg('Token: {}'.format(await tk.get_symbol())) + imsg('dev token balance: {}'.format(await tk.get_balance(dfl_addr))) imsg('usr token balance: {} ({} {})'.format( - tk.balance(usr_addrs[i]),usr_mmaddrs[i],usr_addrs[i])) + await tk.get_balance(usr_addrs[i]),usr_mmaddrs[i],usr_addrs[i])) silence() - if op == 'show_bals': show_bals() - elif op == 'do_transfer': do_transfer() + if op == 'show_bals': + await show_bals() + elif op == 'do_transfer': + await do_transfer() end_silence() return 'ok' @@ -689,15 +688,18 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): t.req_exit_val = 2 return t - def token_addrimport(self): + def token_addrimport(self,extra_args=[],expect='3/3'): for n,r in ('1','11-13'),('2','21-23'): tk_addr = self.read_from_tmpfile('token_addr'+n).strip() - t = self.addrimport(ext='['+r+']{}.addrs',expect='3/3',add_args=['--token='+tk_addr]) + t = self.addrimport(ext='['+r+']{}.addrs',expect=expect,add_args=['--token='+tk_addr]+extra_args) t.p.wait() ok_msg() t.skip_ok = True return t + def token_addrimport_batch(self): + return self.token_addrimport(extra_args=['--batch'],expect='OK: 3') + def bal7(self): return self.bal5() def token_bal1(self): return self.token_bal(n='1') diff --git a/test/test_py_d/ts_main.py b/test/test_py_d/ts_main.py index a1830856..bc002f7d 100755 --- a/test/test_py_d/ts_main.py +++ b/test/test_py_d/ts_main.py @@ -146,8 +146,10 @@ class TestSuiteMain(TestSuiteBase,TestSuiteShared): def __init__(self,trunner,cfgs,spawn): if g.coin.lower() not in self.networks: return - rpc_init() - self.lbl_id = ('account','label')['label_api' in g.rpc.caps] + from mmgen.rpc import rpc_init + g.regtest = False # rpc_init hack + self.rpc = run_session(rpc_init()) + self.lbl_id = ('account','label')['label_api' in self.rpc.caps] if g.coin in ('BTC','BCH','LTC'): self.tx_fee = {'btc':'0.0001','bch':'0.001','ltc':'0.01'}[g.coin.lower()] self.txbump_fee = {'btc':'123s','bch':'567s','ltc':'12345s'}[g.coin.lower()] diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index f21d1753..0467b0b4 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -182,6 +182,7 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared): ('bob_import_addr', 'importing non-MMGen address with --rescan'), ('bob_bal4', "Bob's balance (after import with rescan)"), ('bob_import_list', 'importing flat address list'), + ('bob_import_list_rescan', 'importing flat address list with --rescan'), ('bob_split2', "splitting Bob's funds"), ('bob_0conf0_getbalance', "Bob's balance (unconfirmed, minconf=0)"), ('bob_0conf1_getbalance', "Bob's balance (unconfirmed, minconf=1)"), @@ -735,11 +736,15 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared): def bob_import_addr(self): addr = self.read_from_tmpfile('non-mmgen.addrs').split()[0] - return self.user_import('bob',['--rescan','--address='+addr]) + return self.user_import('bob',['--quiet','--address='+addr]) def bob_import_list(self): addrfile = joinpath(self.tmpdir,'non-mmgen.addrs') - return self.user_import('bob',['--addrlist',addrfile]) + return self.user_import('bob',['--quiet','--addrlist',addrfile]) + + def bob_import_list_rescan(self): + addrfile = joinpath(self.tmpdir,'non-mmgen.addrs') + return self.user_import('bob',['--quiet','--rescan','--addrlist',addrfile]) def bob_split2(self): addrs = self.read_from_tmpfile('non-mmgen.addrs').split() diff --git a/test/tooltest2.py b/test/tooltest2.py index 28f9050e..a7eb6f0f 100755 --- a/test/tooltest2.py +++ b/test/tooltest2.py @@ -759,7 +759,10 @@ tests = { 'ltc_testnet': [ ( ['test/ref/litecoin/A5A1E0-LTC[1454.64322,1453,tl=1320969600].testnet.rawtx'], None ), ], 'eth_mainnet': [ ( ['test/ref/ethereum/88FEFD-ETH[23.45495,40000].rawtx'], None ), ], - 'eth_testnet': [ ( ['test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.rawtx'], None ), ], + 'eth_testnet': [ ( [ + 'test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.rawtx', + 'test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.sigtx' + ], None ), ], 'mm1_mainnet': [ ( ['test/ref/ethereum/5881D2-MM1[1.23456,50000].rawtx'], None ), ], 'mm1_testnet': [ ( ['test/ref/ethereum/6BDB25-MM1[1.23456,50000].testnet.rawtx'], None ), ], 'etc_mainnet': [ ( ['test/ref/ethereum_classic/ED3848-ETC[1.2345,40000].rawtx'], None ), ], diff --git a/test/unit_tests_d/ut_rpc.py b/test/unit_tests_d/ut_rpc.py new file mode 100755 index 00000000..bbd9a9aa --- /dev/null +++ b/test/unit_tests_d/ut_rpc.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +test.unit_tests_d.ut_rpc: RPC unit test for the MMGen suite +""" + +from mmgen.common import * +from mmgen.exception import * + +from mmgen.protocol import init_coin,EthereumProtocol +from mmgen.rpc import MoneroWalletRPCClient +from mmgen.daemon import CoinDaemon,MoneroWalletDaemon + +class unit_tests: + + def btc(self,name,ut): + + async def run_test(): + c = g.rpc + qmsg(' Testing backend {!r}'.format(type(c.backend).__name__)) + addrs = ( + ('bc1qvmqas4maw7lg9clqu6kqu9zq9cluvlln5hw97q','test address #1'), # deadbeef * 8 + ('bc1qe50rj25cldtskw5huxam335kyshtqtlrf4pt9x','test address #2'), # deadbeef * 7 + deadbeee + ) + + await c.batch_call('importaddress',addrs,timeout=120) + ret = await c.batch_call('getaddressesbylabel',[(l,) for a,l in addrs]) + assert list(ret[0].keys())[0] == addrs[0][0] + + bh = (await c.call('getblockchaininfo',timeout=300))['bestblockhash'] + await c.gathered_call('getblock',((bh,),(bh,1)),timeout=300) + await c.gathered_call(None,(('getblock',(bh,)),('getblock',(bh,1))),timeout=300) + + + d = CoinDaemon('btc',test_suite=True) + d.remove_datadir() + d.start() + g.proto.daemon_data_dir = d.datadir # used by BitcoinRPCClient.set_auth() to find the cookie + g.rpc_port = d.rpc_port + + for backend in g.autoset_opts['rpc_backend'].choices: + run_session(run_test(),backend=backend) + + d.stop() + + if g.platform != 'win': + + qmsg(f'\n Testing authentication with credentials from bitcoin.conf:') + d.remove_datadir() + os.makedirs(d.datadir) + + cf = os.path.join(d.datadir,'bitcoin.conf') + open(cf,'a').write('\nrpcuser = ut_rpc\nrpcpassword = ut_rpc_passw0rd\n') + + d.add_flag('keep_cfg_file') + d.start() + + async def do(): + assert g.rpc.auth.user == 'ut_rpc', 'user is not ut_rpc!' + + run_session(do()) + d.stop() + + qmsg(' OK') + return True + + def eth(self,name,ut): + ed = CoinDaemon('eth',test_suite=True) + ed.start() + init_coin('eth') + g.rpc_port = CoinDaemon('eth',test_suite=True).rpc_port + + async def run_test(): + qmsg(' Testing backend {!r}'.format(type(g.rpc.backend).__name__)) + ret = await g.rpc.call('parity_versionInfo',timeout=300) + #print(ret) + + for backend in g.autoset_opts['rpc_backend'].choices: + run_session(run_test(),backend=backend) + + ed.stop() + return True + + def xmr_wallet(self,name,ut): + + async def run(): + md = CoinDaemon('xmr',test_suite=True) + md.start() + + g.monero_wallet_rpc_password = 'passwOrd' + mwd = MoneroWalletDaemon(wallet_dir='test/trash',test_suite=True) + mwd.start() + + c = MoneroWalletRPCClient( + host = g.monero_wallet_rpc_host, + port = mwd.rpc_port, + user = g.monero_wallet_rpc_user, + passwd = g.monero_wallet_rpc_password) + + await c.call('get_version') + + gmsg('OK') + mwd.wait = False + mwd.stop() + md.wait = False + md.stop() + + run_session(run(),do_rpc_init=False) + return True diff --git a/test/unit_tests_d/ut_tx_deserialize.py b/test/unit_tests_d/ut_tx_deserialize.py index 1e6d6922..cc1a7210 100755 --- a/test/unit_tests_d/ut_tx_deserialize.py +++ b/test/unit_tests_d/ut_tx_deserialize.py @@ -3,9 +3,14 @@ test/unit_tests_d/ut_tx_deserialize: TX deserialization unit test for the MMGen suite """ -import os +import os,json + from mmgen.common import * from ..include.common import * +from mmgen.protocol import init_coin +from mmgen.tx import MMGenTX,DeserializedTX +from mmgen.rpc import rpc_init +from mmgen.daemon import CoinDaemon class unit_test(object): @@ -16,7 +21,7 @@ class unit_test(object): def run_test(self,name,ut): - def test_tx(txhex,desc,n): + async def test_tx(txhex,desc,n): def has_nonstandard_outputs(outputs): for o in outputs: @@ -25,7 +30,7 @@ class unit_test(object): return True return False - d = g.rpc.decoderawtransaction(txhex) + d = await g.rpc.call('decoderawtransaction',txhex) if has_nonstandard_outputs(d['vout']): return False @@ -86,7 +91,7 @@ class unit_test(object): Msg_r('Testing transactions from {!r}'.format(fn)) if not opt.quiet: Msg('') - def test_core_vectors(): + async def test_core_vectors(): self._get_core_repo_root() fn_b = 'src/test/data/tx_valid.json' fn = os.path.join(self.core_repo_root,fn_b) @@ -95,38 +100,33 @@ class unit_test(object): n = 1 for e in data: if type(e[0]) == list: - test_tx(e[1],desc,n) + await rpc_init() + await test_tx(e[1],desc,n) n += 1 else: desc = e[0] Msg('OK') - def test_mmgen_txs(): + async def test_mmgen_txs(): fns = ( ('btc',False,'test/ref/0B8D5A[15.31789,14,tl=1320969600].rawtx'), ('btc',True,'test/ref/0C7115[15.86255,14,tl=1320969600].testnet.rawtx'), # ('bch',False,'test/ref/460D4D-BCH[10.19764,tl=1320969600].rawtx') ) - from mmgen.protocol import init_coin - from mmgen.tx import MMGenTX - from mmgen.daemon import CoinDaemon print_info('test/ref/*rawtx','MMGen reference transactions') for n,(coin,tn,fn) in enumerate(fns): init_coin(coin,tn) g.proto.daemon_data_dir = 'test/daemons/' + g.coin.lower() g.rpc_port = CoinDaemon(coin + ('','_tn')[tn],test_suite=True).rpc_port - rpc_init(reinit=True) - test_tx(MMGenTX(fn).hex,fn,n+1) + await rpc_init() + await test_tx(MMGenTX(fn).hex,fn,n+1) init_coin('btc',False) g.rpc_port = CoinDaemon('btc',test_suite=True).rpc_port - rpc_init(reinit=True) + await rpc_init() Msg('OK') - from mmgen.tx import DeserializedTX - import json - start_test_daemons('btc','btc_tn') # ,'bch') - test_mmgen_txs() - test_core_vectors() + run_session(test_mmgen_txs(),do_rpc_init=False) + run_session(test_core_vectors(),do_rpc_init=False) stop_test_daemons('btc','btc_tn') # ,'bch') return True