asyncio/aiohttp support

Asynchronous HTTP significantly speeds up operations involving multiple
JSON-RPC calls to the server, such as tracking wallet views for wallets
with a large number of outputs.

This patch adds base-level asyncio infrastructure plus aiohttp support to all
applicable MMGen commands.

The aiohttp package is not currently supported by MSYS2, so Windows users will
have to choose one of the other backends ('curl' is the default).

Tested on: Linux, Armbian, Windows; Python 3.6, 3.7, 3.8

New user features:

    - configurable RPC backends via the 'rpc_backend' option.  Supported
      options are 'aiohttp' (Linux-only), 'httplib', 'requests' and 'curl'

    - configurable RPC queue size via the 'aiohttp_rpc_queue_len' option

The patch also includes a rewrite/redesign of large parts of the MMGen code
base, most importantly:

    - rpc.py - full rewrite of RPC library, new RPCBackends class
    - main_addrimport.py - full rewrite
    - main_autosign.py - LED code now handled by new LEDControl class
    - eth/tw.py, eth/tx.py - reworked logic for resolving token symbols and
      addresses
    - eth/tx.py - separate classes for signed and unsigned transactions

Testing:

    # Set a backend (choose one):
    $ export MMGEN_RPC_BACKEND='aiohttp' # Linux-only
    $ export MMGEN_RPC_BACKEND='curl'    # Windows
    $ export MMGEN_RPC_BACKEND='httplib' # compare performance with 'aiohttp'

    # Bitcoin:
    $ test/unit_tests.py rpc btc
    $ test/test.py main regtest autosign

    # Ethereum:
    $ test/unit_tests.py rpc eth
    $ test/tooltest2.py --coin=eth --testnet=1 txview
    $ test/test.py --coin=eth ethdev

    # Monero wallet:
    $ test/unit_tests.py rpc xmr_wallet
    $ test/test-release.sh -F xmr

data_files/mmgen.cfg

@@ -35,6 +35,17 @@
 # Uncomment to override 'rpcpassword' from coin daemon config file:
 # rpc_password mypassword
 
+# Choose the backend to use for JSON-RPC connections.  Valid choices are
+# 'httplib', 'requests', 'curl', 'aiohttp' (Linux only) or 'auto' (defaults
+# to curl for Windows/MSYS2 and httplib for Linux):
+# rpc_backend auto
+
+# Increase to allow aiohttp to make more simultaneous RPC connections to the
+# daemon.  Must be no greater than the 'rpcworkqueue' value in effect on the
+# currently running bitcoind (DEFAULT_HTTP_WORKQUEUE = 16).  Values over 32
+# may produce little benefit or even reduce performance:
+# aiohttp_rpc_queue_len 16
+
 # Uncomment to set the coin daemon datadir:
 # daemon_data_dir /path/to/datadir
 

mmgen/addr.py

@@ -1023,10 +1023,8 @@ re-import your addresses.
 	def __new__(cls,*args,**kwargs):
 		return MMGenObject.__new__(altcoin_subclass(cls,'tw','AddrData'))
 
-	def __init__(self,source=None,wallet=None):
+	def __init__(self,*args,**kwargs):
 		self.al_ids = {}
-		if source == 'tw':
-			self.add_tw_data(wallet)
 
 	def seed_ids(self):
 		return list(self.al_ids.keys())
@@ -1048,30 +1046,34 @@ re-import your addresses.
 		return (list(d.values())[0][0]) if d else None
 
 	@classmethod
-	def get_tw_data(cls,wallet=None):
+	async def get_tw_data(cls,wallet=None):
 		vmsg('Getting address data from tracking wallet')
 		if 'label_api' in g.rpc.caps:
-			accts = g.rpc.listlabels()
-			alists = [list(a.keys()) for a in g.rpc.getaddressesbylabel([[k] for k in accts],batch=True)]
+			accts = await g.rpc.call('listlabels')
+			ll = await g.rpc.batch_call('getaddressesbylabel',[(k,) for k in accts])
+			alists = [list(a.keys()) for a in ll]
 		else:
-			accts = g.rpc.listaccounts(0,True)
-			alists = g.rpc.getaddressesbyaccount([[k] for k in accts],batch=True)
+			accts = await g.rpc.call('listaccounts',0,True)
+			alists = await g.rpc.batch_call('getaddressesbyaccount',[(k,) for k in accts])
 		return list(zip(accts,alists))
 
-	def add_tw_data(self,wallet):
-		d,out,i = self.get_tw_data(wallet),{},0
-		for acct,addr_array in d:
+	async def add_tw_data(self,wallet):
+
+		twd = await type(self).get_tw_data(wallet)
+		out,i = {},0
+		for acct,addr_array in twd:
 			l = TwLabel(acct,on_fail='silent')
 			if l and l.mmid.type == 'mmgen':
 				obj = l.mmid.obj
-				i += 1
 				if len(addr_array) != 1:
 					die(2,self.msgs['too_many_acct_addresses'].format(acct))
 				al_id = AddrListID(SeedID(sid=obj.sid),MMGenAddrType(obj.mmtype))
 				if al_id not in out:
 					out[al_id] = []
 				out[al_id].append(AddrListEntry(idx=obj.idx,addr=addr_array[0],label=l.comment))
-		vmsg('{n} {pnm} addresses found, {m} accounts total'.format(n=i,pnm=pnm,m=len(d)))
+				i += 1
+
+		vmsg('{n} {pnm} addresses found, {m} accounts total'.format(n=i,pnm=pnm,m=len(twd)))
 		for al_id in out:
 			self.add(AddrList(al_id=al_id,adata=AddrListList(sorted(out[al_id],key=lambda a: a.idx))))
 
@@ -1087,3 +1089,15 @@ re-import your addresses.
 		for al_id in self.al_ids:
 			d.update(self.al_ids[al_id].make_reverse_dict(coinaddrs))
 		return d
+
+class TwAddrData(AddrData,metaclass=aInitMeta):
+
+	def __new__(cls,*args,**kwargs):
+		return MMGenObject.__new__(altcoin_subclass(cls,'tw','TwAddrData'))
+
+	def __init__(self,*args,**kwargs):
+		pass
+
+	async def __ainit__(self,wallet=None):
+		self.al_ids = {}
+		await self.add_tw_data(wallet)

mmgen/altcoins/eth/contract.py

@@ -25,7 +25,7 @@ from . import rlp
 
 from mmgen.globalvars import g
 from mmgen.common import *
-from mmgen.obj import MMGenObject,CoinAddr,TokenAddr,CoinTxID,ETHAmt
+from mmgen.obj import MMGenObject,CoinAddr,TokenAddr,CoinTxID,ETHAmt,aInitMeta
 from mmgen.util import msg
 
 try:
@@ -39,21 +39,7 @@ def parse_abi(s):
 
 def create_method_id(sig): return keccak_256(sig.encode()).hexdigest()[:8]
 
-class Token(MMGenObject): # ERC20
-
-	_decimals = None
-
-	# Test that token is in the blockchain by calling constructor w/o decimals arg
-	def __init__(self,addr,decimals=None):
-		self.addr = TokenAddr(addr)
-		if decimals:
-			self._decimals = decimals
-		else:
-			rpc_init()
-			self.decimals()
-		if not self._decimals:
-			raise TokenNotInBlockchain("Token '{}' not in blockchain".format(addr))
-		self.base_unit = Decimal('10') ** -self._decimals
+class TokenBase(MMGenObject): # ERC20
 
 	@staticmethod
 	def transferdata2sendaddr(data): # online
@@ -62,53 +48,50 @@ class Token(MMGenObject): # ERC20
 	def transferdata2amt(self,data): # online
 		return ETHAmt(int(parse_abi(data)[-1],16) * self.base_unit)
 
-	def do_call(self,method_sig,method_args='',toUnit=False):
+	async def do_call(self,method_sig,method_args='',toUnit=False):
 		data = create_method_id(method_sig) + method_args
 		if g.debug:
 			msg('ETH_CALL {}:  {}'.format(method_sig,'\n  '.join(parse_abi(data))))
-		ret = g.rpc.eth_call({ 'to': '0x'+self.addr, 'data': '0x'+data })
+		ret = await g.rpc.call('eth_call',{ 'to': '0x'+self.addr, 'data': '0x'+data })
 		if toUnit:
 			return int(ret,16) * self.base_unit
 		else:
 			return ret
 
-	def balance(self,acct_addr):
-		return ETHAmt(self.do_call('balanceOf(address)',acct_addr.rjust(64,'0'),toUnit=True))
+	async def get_balance(self,acct_addr):
+		return ETHAmt(await self.do_call('balanceOf(address)',acct_addr.rjust(64,'0'),toUnit=True))
 
 	def strip(self,s):
 		return ''.join([chr(b) for b in s if 32 <= b <= 127]).strip()
 
-	# TODO: make these properties
-	def decimals(self):
-		if self._decimals == None:
-			res = self.do_call('decimals()')
-			try:
-				assert res[:2] == '0x'
-				self._decimals = int(res[2:],16)
-			except:
-				msg("RPC call to decimals() failed (returned '{}')".format(res))
-				return None
-		return self._decimals
+	async def get_name(self):
+		return self.strip(bytes.fromhex((await self.do_call('name()'))[2:]))
 
-	def name(self):
-		return self.strip(bytes.fromhex(self.do_call('name()')[2:]))
+	async def get_symbol(self):
+		return self.strip(bytes.fromhex((await self.do_call('symbol()'))[2:]))
 
-	def symbol(self):
-		return self.strip(bytes.fromhex(self.do_call('symbol()')[2:]))
+	async def get_decimals(self):
+		ret = await self.do_call('decimals()')
+		try:
+			assert ret[:2] == '0x'
+			return int(ret,16)
+		except:
+			msg("RPC call to decimals() failed (returned '{}')".format(ret))
+			return None
 
-	def total_supply(self):
-		return self.do_call('totalSupply()',toUnit=True)
+	async def get_total_supply(self):
+		return await self.do_call('totalSupply()',toUnit=True)
 
-	def info(self):
+	async def info(self):
 		fs = '{:15}{}\n' * 5
 		return fs.format('token address:', self.addr,
-						'token symbol:',   self.symbol(),
-						'token name:',     self.name(),
-						'decimals:',       self.decimals(),
-						'total supply:',   self.total_supply())
+						'token symbol:',   await self.get_symbol(),
+						'token name:',     await self.get_name(),
+						'decimals:',       self.decimals,
+						'total supply:',   await self.get_total_supply())
 
-	def code(self):
-		return g.rpc.eth_getCode('0x'+self.addr)[2:]
+	async def code(self):
+		return (await g.rpc.call('eth_getCode','0x'+self.addr))[2:]
 
 	def create_data(self,to_addr,amt,method_sig='transfer(address,uint256)',from_addr=None):
 		from_arg = from_addr.rjust(64,'0') if from_addr else ''
@@ -126,13 +109,13 @@ class Token(MMGenObject): # ERC20
 				'nonce':    nonce,
 				'data':     bytes.fromhex(data) }
 
-	def txsign(self,tx_in,key,from_addr,chain_id=None):
+	async def txsign(self,tx_in,key,from_addr,chain_id=None):
 
 		from .pyethereum.transactions import Transaction
 
 		if chain_id is None:
 			chain_id_method = ('parity_chainId','eth_chainId')['eth_chainId' in g.rpc.caps]
-			chain_id = int(g.rpc.request(chain_id_method),16)
+			chain_id = int(await g.rpc.call(chain_id_method),16)
 		tx = Transaction(**tx_in).sign(key,chain_id)
 		hex_tx = rlp.encode(tx).hex()
 		coin_txid = CoinTxID(tx.hash.hex())
@@ -147,18 +130,38 @@ class Token(MMGenObject): # ERC20
 
 # The following are used for token deployment only:
 
-	def txsend(self,hex_tx):
-		return g.rpc.eth_sendRawTransaction('0x'+hex_tx).replace('0x','',1)
+	async def txsend(self,hex_tx):
+		return (await g.rpc.call('eth_sendRawTransaction','0x'+hex_tx)).replace('0x','',1)
 
-	def transfer(   self,from_addr,to_addr,amt,key,start_gas,gasPrice,
+	async def transfer(   self,from_addr,to_addr,amt,key,start_gas,gasPrice,
 					method_sig='transfer(address,uint256)',
 					from_addr2=None,
 					return_data=False):
 		tx_in = self.make_tx_in(
 					from_addr,to_addr,amt,
 					start_gas,gasPrice,
-					nonce = int(g.rpc.parity_nextNonce('0x'+from_addr),16),
+					nonce = int(await g.rpc.call('parity_nextNonce','0x'+from_addr),16),
 					method_sig = method_sig,
 					from_addr2 = from_addr2 )
-		(hex_tx,coin_txid) = self.txsign(tx_in,key,from_addr)
-		return self.txsend(hex_tx)
+		(hex_tx,coin_txid) = await self.txsign(tx_in,key,from_addr)
+		return await self.txsend(hex_tx)
+
+class Token(TokenBase):
+
+	def __init__(self,addr,decimals):
+		self.addr = TokenAddr(addr)
+		assert isinstance(decimals,int),f'decimals param must be int instance, not {type(decimals)}'
+		self.decimals = decimals
+		self.base_unit = Decimal('10') ** -self.decimals
+
+class TokenResolve(TokenBase,metaclass=aInitMeta):
+
+	def __init__(self,addr):
+		return super().__init__()
+
+	async def __ainit__(self,addr):
+		self.addr = TokenAddr(addr)
+		decimals = await self.get_decimals() # requires self.addr!
+		if not decimals:
+			raise TokenNotInBlockchain(f'Token {addr!r} not in blockchain')
+		Token.__init__(self,addr,decimals)

mmgen/altcoins/eth/tw.py

@@ -22,27 +22,17 @@ altcoins.eth.tw: Ethereum tracking wallet and related classes for the MMGen suit
 
 from mmgen.common import *
 from mmgen.obj import ETHAmt,TwLabel,is_coin_addr,is_mmgen_id,MMGenListItem,ListItemAttr,ImmutableAttr
-from mmgen.tw import TrackingWallet,TwAddrList,TwUnspentOutputs
-from mmgen.addr import AddrData
-from .contract import Token
+from mmgen.tw import TrackingWallet,TwAddrList,TwUnspentOutputs,TwGetBalance
+from mmgen.addr import AddrData,TwAddrData
+from .contract import Token,TokenResolve
 
 class EthereumTrackingWallet(TrackingWallet):
 
-	caps = ()
+	caps = ('batch',)
 	data_key = 'accounts'
 	use_tw_file = True
 
-	def __init__(self,mode='r',no_rpc=False):
-		TrackingWallet.__init__(self,mode=mode)
-
-		for v in self.data['tokens'].values():
-			self.conv_types(v)
-
-		if g.token and not is_coin_addr(g.token):
-			ret = self.sym2addr(g.token,no_rpc=no_rpc)
-			if ret: g.token = ret
-
-	def is_in_wallet(self,addr):
+	async def is_in_wallet(self,addr):
 		return addr in self.data_root
 
 	def init_empty(self):
@@ -84,14 +74,17 @@ class EthereumTrackingWallet(TrackingWallet):
 			self.force_write()
 			msg('{} upgraded successfully!'.format(self.desc))
 
-	# Don't call rpc_init() for Ethereum, because it may create a wallet instance
-	def rpc_init(self): pass
+	async def rpc_get_balance(self,addr):
+		return ETHAmt(int(await g.rpc.call('eth_getBalance','0x'+addr),16),'wei')
 
-	def rpc_get_balance(self,addr):
-		return ETHAmt(int(g.rpc.eth_getBalance('0x'+addr),16),'wei')
+	@write_mode
+	async def batch_import_address(self,args_list):
+		for arg_list in args_list:
+			await self.import_address(*arg_list)
+		return args_list
 
 	@write_mode
-	def import_address(self,addr,label,foo):
+	async def import_address(self,addr,label,foo):
 		r = self.data_root
 		if addr in r:
 			if not r[addr]['mmid'] and label.mmid:
@@ -101,7 +94,7 @@ class EthereumTrackingWallet(TrackingWallet):
 		r[addr] = { 'mmid': label.mmid, 'comment': label.comment }
 
 	@write_mode
-	def remove_address(self,addr):
+	async def remove_address(self,addr):
 		r = self.data_root
 
 		if is_coin_addr(addr):
@@ -109,7 +102,7 @@ class EthereumTrackingWallet(TrackingWallet):
 		elif is_mmgen_id(addr):
 			have_match = lambda k: r[k]['mmid'] == addr
 		else:
-			die(1,"'{}' is not an Ethereum address or MMGen ID".format(addr))
+			die(1,f'{addr!r} is not an Ethereum address or MMGen ID')
 
 		for k in r:
 			if have_match(k):
@@ -119,46 +112,30 @@ class EthereumTrackingWallet(TrackingWallet):
 				self.write()
 				return ret
 		else:
-			m = "Address '{}' not found in '{}' section of tracking wallet"
-			msg(m.format(addr,self.data_root_desc))
+			msg(f'Address {addr!r} not found in {self.data_root_desc!r} section of tracking wallet')
 			return None
 
 	@write_mode
-	def set_label(self,coinaddr,lbl):
+	async def set_label(self,coinaddr,lbl):
 		for addr,d in list(self.data_root.items()):
 			if addr == coinaddr:
 				d['comment'] = lbl.comment
 				self.write()
 				return None
-		else: # emulate on_fail='return' of RPC library
-			m = "Address '{}' not found in '{}' section of tracking wallet"
-			return ('rpcfail',(None,2,m.format(coinaddr,self.data_root_desc)))
-
-	def addr2sym(self,req_addr):
+		else:
+			msg(f'Address {coinaddr!r} not found in {self.data_root_desc!r} section of tracking wallet')
+			return False
 
+	async def addr2sym(self,req_addr):
 		for addr in self.data['tokens']:
 			if addr == req_addr:
-				ret = self.data['tokens'][addr]['params'].get('symbol')
-				if ret: return ret
-				else: break
-
-		self.token_obj = Token(req_addr)
-		ret = self.token_obj.symbol().upper()
-		self.force_set_token_param(req_addr,'symbol',ret)
-		return ret
-
-	def sym2addr(self,sym,no_rpc=False):
-
-		for addr in self.data['tokens']:
-			if self.data['tokens'][addr]['params'].get('symbol') == sym.upper():
-				return addr
-
-		if no_rpc:
+				return self.data['tokens'][addr]['params']['symbol']
+		else:
 			return None
 
+	async def sym2addr(self,sym):
 		for addr in self.data['tokens']:
-			if Token(addr).symbol().upper() == sym.upper():
-				self.force_set_token_param(addr,'symbol',sym.upper())
+			if self.data['tokens'][addr]['params']['symbol'] == sym.upper():
 				return addr
 		else:
 			return None
@@ -168,17 +145,6 @@ class EthereumTrackingWallet(TrackingWallet):
 			return self.data['tokens'][token]['params'].get(param)
 		return None
 
-	def force_set_token_param(self,*args,**kwargs):
-		mode_save = self.mode
-		self.mode = 'w'
-		self.set_token_param(*args,**kwargs)
-		self.mode = mode_save
-
-	@write_mode
-	def set_token_param(self,token,param,val):
-		if token in self.data['tokens']:
-			self.data['tokens'][token]['params'][param] = val
-
 class EthereumTokenTrackingWallet(EthereumTrackingWallet):
 
 	desc = 'Ethereum token tracking wallet'
@@ -186,29 +152,32 @@ class EthereumTokenTrackingWallet(EthereumTrackingWallet):
 	symbol = None
 	cur_eth_balances = {}
 
-	def __init__(self,mode='r',no_rpc=False):
-		EthereumTrackingWallet.__init__(self,mode=mode,no_rpc=no_rpc)
+	async def __ainit__(self,mode='r'):
+		await super().__ainit__(mode=mode)
 
-		self.desc = 'Ethereum token tracking wallet'
+		for v in self.data['tokens'].values():
+			self.conv_types(v)
 
 		if not is_coin_addr(g.token):
-			raise UnrecognizedTokenSymbol('Specified token {!r} could not be resolved!'.format(g.token))
+			g.token = await self.sym2addr(g.token) # returns None on failure
+
+		if not is_coin_addr(g.token):
+			if self.importing:
+				m = 'When importing addresses for a new token, the token must be specified by address, not symbol.'
+				raise InvalidTokenAddress(f'{g.token!r}: invalid token address\n{m}')
+			else:
+				raise UnrecognizedTokenSymbol(f'Specified token {g.token!r} could not be resolved!')
 
-		if mode == 'r' and not g.token in self.data['tokens']:
+		if g.token in self.data['tokens']:
+			self.decimals = self.data['tokens'][g.token]['params']['decimals']
+			self.symbol = self.data['tokens'][g.token]['params']['symbol']
+		elif not self.importing:
 			raise TokenNotInWallet('Specified token {!r} not in wallet!'.format(g.token))
 
 		self.token = g.token
+		g.dcoin = self.symbol
 
-		if self.token in self.data['tokens']:
-			for k in ('decimals','symbol'):
-				setattr(self,k,self.get_param(k))
-				if getattr(self,k) == None:
-					setattr(self,k,getattr(Token(self.token,self.decimals),k)())
-					if getattr(self,k) != None:
-						self.set_param(k,getattr(self,k))
-						self.write()
-
-	def is_in_wallet(self,addr):
+	async def is_in_wallet(self,addr):
 		return addr in self.data['tokens'][self.token]
 
 	@property
@@ -217,43 +186,39 @@ class EthereumTokenTrackingWallet(EthereumTrackingWallet):
 
 	@property
 	def data_root_desc(self):
-		return 'token ' + Token(self.token,self.decimals).symbol()
-
-	@write_mode
-	def add_token(self,token):
-		msg("Adding token '{}' to tracking wallet.".format(token))
-		self.data['tokens'][token] = { 'params': {} }
-
-	@write_mode
-	def import_address(self,*args,**kwargs):
-		if self.token not in self.data['tokens']:
-			self.add_token(self.token)
-		EthereumTrackingWallet.import_address(self,*args,**kwargs)
+		return 'token ' + self.get_param('symbol')
 
-	def rpc_get_balance(self,addr):
-		return Token(self.token,self.decimals).balance(addr)
+	async def rpc_get_balance(self,addr):
+		return await Token(self.token,self.decimals).get_balance(addr)
 
-	def get_eth_balance(self,addr,force_rpc=False):
+	async def get_eth_balance(self,addr,force_rpc=False):
 		cache = self.cur_eth_balances
-		data_root = self.data['accounts']
-		ret = None if force_rpc else self.get_cached_balance(addr,cache,data_root)
+		r = self.data['accounts']
+		ret = None if force_rpc else self.get_cached_balance(addr,cache,r)
 		if ret == None:
-			ret = EthereumTrackingWallet.rpc_get_balance(self,addr)
-			self.cache_balance(addr,ret,cache,data_root)
+			ret = await super().rpc_get_balance(addr)
+			self.cache_balance(addr,ret,cache,r)
 		return ret
 
-	def force_set_param(self,*args,**kwargs):
-		mode_save = self.mode
-		self.mode = 'w'
-		self.set_param(*args,**kwargs)
-		self.mode = mode_save
+	def get_param(self,param):
+		return self.data['tokens'][self.token]['params'][param]
 
 	@write_mode
-	def set_param(self,param,val):
-		self.data['tokens'][self.token]['params'][param] = val
-
-	def get_param(self,param):
-		return self.data['tokens'][self.token]['params'].get(param)
+	async def import_token(tw):
+		"""
+		Token 'symbol' and 'decimals' values are resolved from the network by the system just
+		once, upon token import.  Thereafter, token address, symbol and decimals are resolved
+		either from the tracking wallet (online operations) or transaction file (when signing).
+		"""
+		if not g.token in tw.data['tokens']:
+			t = await TokenResolve(g.token)
+			tw.token = g.token
+			tw.data['tokens'][tw.token] = {
+				'params': {
+					'symbol': await t.get_symbol(),
+					'decimals': t.decimals
+				}
+			}
 
 # No unspent outputs with Ethereum, but naming must be consistent
 class EthereumTwUnspentOutputs(TwUnspentOutputs):
@@ -277,23 +242,23 @@ Actions:         [q]uit view, [p]rint to file, pager [v]iew, [w]ide view,
 		'q':'a_quit','p':'a_print','v':'a_view','w':'a_view_wide',
 		'l':'a_lbl_add','D':'a_addr_delete','R':'a_balance_refresh' }
 
-	def __init__(self,*args,**kwargs):
+	async def __ainit__(self,*args,**kwargs):
 		if g.use_cached_balances:
 			self.hdr_fmt += '\n' + yellow('WARNING: Using cached balances. These may be out of date!')
-		TwUnspentOutputs.__init__(self,*args,**kwargs)
+		await TwUnspentOutputs.__ainit__(self,*args,**kwargs)
 
 	def do_sort(self,key=None,reverse=False):
 		if key == 'txid': return
 		super().do_sort(key=key,reverse=reverse)
 
-	def get_unspent_rpc(self):
+	async def get_unspent_rpc(self):
 		wl = self.wallet.sorted_list
 		if self.addrs:
 			wl = [d for d in wl if d['addr'] in self.addrs]
 		return [{
 				'account': TwLabel(d['mmid']+' '+d['comment'],on_fail='raise'),
 				'address': d['addr'],
-				'amount': self.wallet.get_balance(d['addr']),
+				'amount': await self.wallet.get_balance(d['addr']),
 				'confirmations': 0, # TODO
 				} for d in wl]
 
@@ -311,6 +276,9 @@ Actions:         [q]uit view, [p]rint to file, pager [v]iew, [w]ide view,
 	def age_disp(self,o,age_fmt): # TODO
 		return None
 
+	def age_disp(self,o,age_fmt): # TODO
+		return None
+
 class EthereumTokenTwUnspentOutputs(EthereumTwUnspentOutputs):
 
 	disp_type = 'token'
@@ -320,18 +288,18 @@ class EthereumTokenTwUnspentOutputs(EthereumTwUnspentOutputs):
 	def get_display_precision(self):
 		return 10 # truncate precision for narrow display
 
-	def get_unspent_data(self):
-		super().get_unspent_data()
+	async def get_unspent_data(self,*args,**kwargs):
+		await super().get_unspent_data(*args,**kwargs)
 		for e in self.unspent:
-			e.amt2 = self.wallet.get_eth_balance(e.addr)
+			e.amt2 = await self.wallet.get_eth_balance(e.addr)
 
 class EthereumTwAddrList(TwAddrList):
 
 	has_age = False
 
-	def __init__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None):
+	async def __ainit__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None):
 
-		self.wallet = wallet or TrackingWallet(mode='w')
+		self.wallet = wallet or await TrackingWallet(mode='w')
 		tw_dict = self.wallet.mmid_ordered_dict
 		self.total = g.proto.coin_amt('0')
 
@@ -341,7 +309,7 @@ class EthereumTwAddrList(TwAddrList):
 			label = TwLabel(mmid+' '+d['comment'],on_fail='raise')
 			if usr_addr_list and (label.mmid not in usr_addr_list):
 				continue
-			bal = self.wallet.get_balance(d['addr'])
+			bal = await self.wallet.get_balance(d['addr'])
 			if bal == 0 and not showempty:
 				if not label.comment or not all_labels:
 					continue
@@ -352,18 +320,20 @@ class EthereumTwAddrList(TwAddrList):
 			self[label.mmid]['amt'] += bal
 			self.total += bal
 
-class EthereumTokenTwAddrList(EthereumTwAddrList): pass
+		del self.wallet
+
+class EthereumTokenTwAddrList(EthereumTwAddrList):
+	pass
 
-from mmgen.tw import TwGetBalance
 class EthereumTwGetBalance(TwGetBalance):
 
 	fs = '{w:13} {c}\n' # TODO - for now, just suppress display of meaningless data
 
-	def __init__(self,*args,**kwargs):
-		self.wallet = TrackingWallet(mode='w')
-		TwGetBalance.__init__(self,*args,**kwargs)
+	async def __ainit__(self,*args,**kwargs):
+		self.wallet = await TrackingWallet(mode='w')
+		await TwGetBalance.__ainit__(self,*args,**kwargs)
 
-	def create_data(self):
+	async def create_data(self):
 		data = self.wallet.mmid_ordered_dict
 		for d in data:
 			if d.type == 'mmgen':
@@ -374,20 +344,24 @@ class EthereumTwGetBalance(TwGetBalance):
 				key = 'Non-MMGen'
 
 			conf_level = 2 # TODO
-			amt = self.wallet.get_balance(data[d]['addr'])
+			amt = await self.wallet.get_balance(data[d]['addr'])
 
 			self.data['TOTAL'][conf_level] += amt
 			self.data[key][conf_level] += amt
 
-class EthereumTokenTwGetBalance(EthereumTwGetBalance): pass
+		del self.wallet
 
-class EthereumAddrData(AddrData):
+class EthereumTwAddrData(TwAddrData):
 
 	@classmethod
-	def get_tw_data(cls,wallet=None):
+	async def get_tw_data(cls,wallet=None):
 		vmsg('Getting address data from tracking wallet')
-		tw = (wallet or TrackingWallet()).mmid_ordered_dict
+		tw = (wallet or await TrackingWallet()).mmid_ordered_dict
 		# emulate the output of RPC 'listaccounts' and 'getaddressesbyaccount'
 		return [(mmid+' '+d['comment'],[d['addr']]) for mmid,d in list(tw.items())]
 
+class EthereumTokenTwGetBalance(EthereumTwGetBalance): pass
+class EthereumTokenTwAddrData(EthereumTwAddrData): pass
+
+class EthereumAddrData(AddrData): pass
 class EthereumTokenAddrData(EthereumAddrData): pass

mmgen/altcoins/eth/tx.py

@@ -24,7 +24,9 @@ import json
 from mmgen.common import *
 from mmgen.obj import *
 
-from mmgen.tx import MMGenTX,MMGenBumpTX,MMGenSplitTX
+from mmgen.tx import MMGenTX,MMGenBumpTX,MMGenSplitTX,MMGenTxForSigning
+from mmgen.tw import TrackingWallet
+from .contract import Token
 
 class EthereumMMGenTX(MMGenTX):
 	desc = 'Ethereum transaction'
@@ -49,7 +51,7 @@ class EthereumMMGenTX(MMGenTX):
 	usr_contract_data = HexStr('')
 
 	def __init__(self,*args,**kwargs):
-		super().__init__(*args,**kwargs)
+		MMGenTX.__init__(self,*args,**kwargs)
 		if hasattr(opt,'tx_gas') and opt.tx_gas:
 			self.tx_gas = self.start_gas = ETHAmt(int(opt.tx_gas),'wei')
 		if hasattr(opt,'contract_data') and opt.contract_data:
@@ -58,9 +60,12 @@ class EthereumMMGenTX(MMGenTX):
 			self.usr_contract_data = HexStr(open(opt.contract_data).read().strip())
 			self.disable_fee_check = True
 
+	def check_txfile_hex_data(self):
+		pass
+
 	@classmethod
-	def get_exec_status(cls,txid,silent=False):
-		d = g.rpc.eth_getTransactionReceipt('0x'+txid)
+	async def get_exec_status(cls,txid,silent=False):
+		d = await g.rpc.call('eth_getTransactionReceipt','0x'+txid)
 		if not silent:
 			if 'contractAddress' in d and d['contractAddress']:
 				msg('Contract address: {}'.format(d['contractAddress'].replace('0x','')))
@@ -84,46 +89,35 @@ class EthereumMMGenTX(MMGenTX):
 			return True
 		return False
 
-	# hex data if signed, json if unsigned
-	def check_txfile_hex_data(self):
-		if self.check_sigs():
-			from .pyethereum.transactions import Transaction
-			from . import rlp
-			etx = rlp.decode(bytes.fromhex(self.hex),Transaction)
-			d = etx.to_dict() # ==> hex values have '0x' prefix, 0 is '0x'
-			for k in ('sender','to','data'):
-				if k in d: d[k] = d[k].replace('0x','',1)
-			o = {   'from':     CoinAddr(d['sender']),
-					'to':       CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is token address
-					'amt':      ETHAmt(d['value'],'wei'),
-					'gasPrice': ETHAmt(d['gasprice'],'wei'),
-					'startGas': ETHAmt(d['startgas'],'wei'),
-					'nonce':    ETHNonce(d['nonce']),
-					'data':     HexStr(d['data']) }
-			if o['data'] and not o['to']: # token- or contract-creating transaction
-				o['token_addr'] = TokenAddr(etx.creates.hex()) # NB: could be a non-token contract address
-				self.disable_fee_check = True
-			txid = CoinTxID(etx.hash.hex())
-			assert txid == self.coin_txid,"txid in tx.hex doesn't match value in MMGen transaction file"
-		else:
-			d = json.loads(self.hex)
-			o = {   'from':     CoinAddr(d['from']),
-					'to':       CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is sendto address
-					'amt':      ETHAmt(d['amt']),
-					'gasPrice': ETHAmt(d['gasPrice']),
-					'startGas': ETHAmt(d['startGas']),
-					'nonce':    ETHNonce(d['nonce']),
-					'chainId':  Int(d['chainId']),
-					'data':     HexStr(d['data']) }
+	def parse_txfile_hex_data(self):
+		from .pyethereum.transactions import Transaction
+		from . import rlp
+		etx = rlp.decode(bytes.fromhex(self.hex),Transaction)
+		d = etx.to_dict() # ==> hex values have '0x' prefix, 0 is '0x'
+		for k in ('sender','to','data'):
+			if k in d: d[k] = d[k].replace('0x','',1)
+		o = {
+			'from':     CoinAddr(d['sender']),
+			'to':       CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is token address
+			'amt':      ETHAmt(d['value'],'wei'),
+			'gasPrice': ETHAmt(d['gasprice'],'wei'),
+			'startGas': ETHAmt(d['startgas'],'wei'),
+			'nonce':    ETHNonce(d['nonce']),
+			'data':     HexStr(d['data']) }
+		if o['data'] and not o['to']: # token- or contract-creating transaction
+			o['token_addr'] = TokenAddr(etx.creates.hex()) # NB: could be a non-token contract address
+			self.disable_fee_check = True
+		txid = CoinTxID(etx.hash.hex())
+		assert txid == self.coin_txid,"txid in tx.hex doesn't match value in MMGen transaction file"
 		self.tx_gas = o['startGas'] # approximate, but better than nothing
 		self.fee = self.fee_rel2abs(o['gasPrice'].toWei())
 		self.txobj = o
 		return d # 'token_addr','decimals' required by Token subclass
 
-	def get_nonce(self):
-		return ETHNonce(int(g.rpc.parity_nextNonce('0x'+self.inputs[0].addr),16))
+	async def get_nonce(self):
+		return ETHNonce(int(await g.rpc.call('parity_nextNonce','0x'+self.inputs[0].addr),16))
 
-	def make_txobj(self): # called by create_raw()
+	async def make_txobj(self): # called by create_raw()
 		chain_id_method = ('parity_chainId','eth_chainId')['eth_chainId' in g.rpc.caps]
 		self.txobj = {
 			'from': self.inputs[0].addr,
@@ -131,20 +125,20 @@ class EthereumMMGenTX(MMGenTX):
 			'amt':  self.outputs[0].amt if self.outputs else ETHAmt('0'),
 			'gasPrice': self.usr_rel_fee or self.fee_abs2rel(self.fee,to_unit='eth'),
 			'startGas': self.start_gas,
-			'nonce': self.get_nonce(),
-			'chainId': Int(g.rpc.request(chain_id_method),16),
+			'nonce': await self.get_nonce(),
+			'chainId': Int(await g.rpc.call(chain_id_method),16),
 			'data':  self.usr_contract_data,
 		}
 
 	# Instead of serializing tx data as with BTC, just create a JSON dump.
 	# This complicates things but means we avoid using the rlp library to deserialize the data,
 	# thus removing an attack vector
-	def create_raw(self):
+	async def create_raw(self):
 		assert len(self.inputs) == 1,'Transaction has more than one input!'
 		o_num = len(self.outputs)
 		o_ok = 0 if self.usr_contract_data else 1
 		assert o_num == o_ok,'Transaction has {} output{} (should have {})'.format(o_num,suf(o_num),o_ok)
-		self.make_txobj()
+		await self.make_txobj()
 		odict = { k: str(v) for k,v in self.txobj.items() if k != 'token_to' }
 		self.hex = json.dumps(odict)
 		self.update_txid()
@@ -156,9 +150,6 @@ class EthereumMMGenTX(MMGenTX):
 		assert not is_hex_str(self.hex),'update_txid() must be called only when self.hex is not hex data'
 		self.txid = MMGenTxID(make_chksum_6(self.hex).upper())
 
-	def get_blockcount(self):
-		return Int(g.rpc.eth_blockNumber(),16)
-
 	def process_cmd_args(self,cmd_args,ad_f,ad_w):
 		lc = len(cmd_args)
 		if lc == 0 and self.usr_contract_data and not 'Token' in type(self).__name__:
@@ -185,7 +176,9 @@ class EthereumMMGenTX(MMGenTX):
 					return [int(reply)]
 
 	# coin-specific fee routines:
-	def get_relay_fee(self): return ETHAmt('0') # TODO
+	@property
+	def relay_fee(self):
+		return ETHAmt('0') # TODO
 
 	# given absolute fee in ETH, return gas price in Gwei using tx_gas
 	def fee_abs2rel(self,abs_fee,to_unit='Gwei'):
@@ -194,8 +187,8 @@ class EthereumMMGenTX(MMGenTX):
 		return ret if to_unit == 'eth' else ret.to_unit(to_unit,show_decimal=True)
 
 	# get rel_fee (gas price) from network, return in native wei
-	def get_rel_fee_from_network(self):
-		return Int(g.rpc.eth_gasPrice(),16),'eth_gasPrice' # ==> rel_fee,fe_type
+	async def get_rel_fee_from_network(self):
+		return Int(await g.rpc.call('eth_gasPrice'),16),'eth_gasPrice' # ==> rel_fee,fe_type
 
 	# given rel fee and units, return absolute fee using tx_gas
 	def convert_fee_spec(self,foo,units,amt,unit):
@@ -264,7 +257,7 @@ class EthereumMMGenTX(MMGenTX):
 	def format_view_rel_fee(self,terse): return ''
 	def format_view_verbose_footer(self): return '' # TODO
 
-	def resolve_g_token_from_tx_file(self):
+	def resolve_g_token_from_txfile(self):
 		die(2,"The '--token' option must be specified for token transaction files")
 
 	def final_inputs_ok_msg(self,change_amt):
@@ -272,79 +265,29 @@ class EthereumMMGenTX(MMGenTX):
 		chg = '0' if (self.outputs and self.outputs[0].is_chg) else change_amt
 		return m.format(ETHAmt(chg).hl(),g.coin)
 
-	def do_sign(self,wif,tx_num_str):
-		o = self.txobj
-		o_conv = {
-			'to':       bytes.fromhex(o['to']),
-			'startgas': o['startGas'].toWei(),
-			'gasprice': o['gasPrice'].toWei(),
-			'value':    o['amt'].toWei() if o['amt'] else 0,
-			'nonce':    o['nonce'],
-			'data':     bytes.fromhex(o['data']) }
-
-		from .pyethereum.transactions import Transaction
-		etx = Transaction(**o_conv).sign(wif,o['chainId'])
-		assert etx.sender.hex() == o['from'],(
-			'Sender address recovered from signature does not match true sender')
-
-		from . import rlp
-		self.hex = rlp.encode(etx).hex()
-		self.coin_txid = CoinTxID(etx.hash.hex())
-
-		if o['data']:
-			if o['to']:
-				assert self.txobj['token_addr'] == TokenAddr(etx.creates.hex()),'Token address mismatch'
-			else: # token- or contract-creating transaction
-				self.txobj['token_addr'] = TokenAddr(etx.creates.hex())
-
-		assert self.check_sigs(),'Signature check failed'
-
-	def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception
-
-		if self.marked_signed():
-			msg('Transaction is already signed!')
-			return False
-
-		if not self.check_correct_chain(on_fail='return'):
-			return False
-
-		msg_r('Signing transaction{}...'.format(tx_num_str))
-
-		try:
-			self.do_sign(keys[0].sec.wif,tx_num_str)
-			msg('OK')
-			return True
-		except Exception as e:
-			m = "{!r}: transaction signing failed!"
-			msg(m.format(e.args[0]))
-			if g.traceback:
-				import traceback
-				ymsg('\n'+''.join(traceback.format_exception(*sys.exc_info())))
-			return False
-
-	def get_status(self,status=False):
+	async def get_status(self,status=False):
 
 		class r(object): pass
 
-		def is_in_mempool():
+		async def is_in_mempool():
 			if not 'full_node' in g.rpc.caps:
 				return False
-			return '0x'+self.coin_txid in [x['hash'] for x in g.rpc.parity_pendingTransactions()]
+			return '0x'+self.coin_txid in [x['hash'] for x in await g.rpc.call('parity_pendingTransactions')]
 
-		def is_in_wallet():
-			d = g.rpc.eth_getTransactionReceipt('0x'+self.coin_txid)
+		async def is_in_wallet():
+			d = await g.rpc.call('eth_getTransactionReceipt','0x'+self.coin_txid)
 			if d and 'blockNumber' in d and d['blockNumber'] is not None:
-				r.confs = 1 + int(g.rpc.eth_blockNumber(),16) - int(d['blockNumber'],16)
+				r.confs = 1 + int(await g.rpc.call('eth_blockNumber'),16) - int(d['blockNumber'],16)
 				r.exec_status = int(d['status'],16)
 				return True
 			return False
 
-		if is_in_mempool():
+		if await is_in_mempool():
 			msg('Transaction is in mempool' if status else 'Warning: transaction is in mempool!')
 			return
 
 		if status:
-			if is_in_wallet():
+			if await is_in_wallet():
 				if self.txobj['data']:
 					cd = capfirst(self.contract_desc)
 					if r.exec_status == 0:
@@ -354,7 +297,7 @@ class EthereumMMGenTX(MMGenTX):
 				die(0,'Transaction has {} confirmation{}'.format(r.confs,suf(r.confs)))
 			die(1,'Transaction is neither in mempool nor blockchain!')
 
-	def send(self,prompt_user=True,exit_on_fail=False):
+	async def send(self,prompt_user=True,exit_on_fail=False):
 
 		if not self.marked_signed():
 			die(1,'Transaction is not signed!')
@@ -367,16 +310,21 @@ class EthereumMMGenTX(MMGenTX):
 			die(2,'Transaction fee ({}) greater than {} max_tx_fee ({} {})!'.format(
 				fee,g.proto.name.capitalize(),g.proto.max_tx_fee,g.coin))
 
-		self.get_status()
+		await self.get_status()
 
 		if prompt_user:
 			self.confirm_send()
 
-		ret = None if g.bogus_send else g.rpc.eth_sendRawTransaction('0x'+self.hex,on_fail='return')
+		if g.bogus_send:
+			ret = None
+		else:
+			try:
+				ret = await g.rpc.call('eth_sendRawTransaction','0x'+self.hex)
+			except:
+				raise
+				ret = False
 
-		from mmgen.rpc import rpc_error,rpc_errmsg
-		if rpc_error(ret):
-			msg(yellow(rpc_errmsg(ret)))
+		if ret == False:
 			msg(red('Send of MMGen transaction {} failed'.format(self.txid)))
 			if exit_on_fail:
 				sys.exit(1)
@@ -393,11 +341,10 @@ class EthereumMMGenTX(MMGenTX):
 			self.add_blockcount()
 			return True
 
-	def get_cmdline_input_addrs(self):
+	async def get_cmdline_input_addrs(self):
 		ret = []
 		if opt.inputs:
-			from mmgen.tw import TrackingWallet
-			r = TrackingWallet().data_root # must create new instance here
+			r = (await TrackingWallet()).data_root # must create new instance here
 			m = 'Address {!r} not in tracking wallet'
 			for i in opt.inputs.split(','):
 				if is_mmgen_id(i):
@@ -419,6 +366,74 @@ class EthereumMMGenTX(MMGenTX):
 		if 'token_addr' in self.txobj:
 			msg('Contract address: {}'.format(self.txobj['token_addr'].hl()))
 
+class EthereumMMGenTxForSigning(EthereumMMGenTX,MMGenTxForSigning):
+
+	def parse_txfile_hex_data(self):
+		d = json.loads(self.hex)
+		o = {
+			'from':     CoinAddr(d['from']),
+			'to':       CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is sendto address
+			'amt':      ETHAmt(d['amt']),
+			'gasPrice': ETHAmt(d['gasPrice']),
+			'startGas': ETHAmt(d['startGas']),
+			'nonce':    ETHNonce(d['nonce']),
+			'chainId':  Int(d['chainId']),
+			'data':     HexStr(d['data']) }
+		self.tx_gas = o['startGas'] # approximate, but better than nothing
+		self.fee = self.fee_rel2abs(o['gasPrice'].toWei())
+		self.txobj = o
+		return d # 'token_addr','decimals' required by Token subclass
+
+	async def do_sign(self,wif,tx_num_str):
+		o = self.txobj
+		o_conv = {
+			'to':       bytes.fromhex(o['to']),
+			'startgas': o['startGas'].toWei(),
+			'gasprice': o['gasPrice'].toWei(),
+			'value':    o['amt'].toWei() if o['amt'] else 0,
+			'nonce':    o['nonce'],
+			'data':     bytes.fromhex(o['data']) }
+
+		from .pyethereum.transactions import Transaction
+		etx = Transaction(**o_conv).sign(wif,o['chainId'])
+		assert etx.sender.hex() == o['from'],(
+			'Sender address recovered from signature does not match true sender')
+
+		from . import rlp
+		self.hex = rlp.encode(etx).hex()
+		self.coin_txid = CoinTxID(etx.hash.hex())
+
+		if o['data']:
+			if o['to']:
+				assert self.txobj['token_addr'] == TokenAddr(etx.creates.hex()),'Token address mismatch'
+			else: # token- or contract-creating transaction
+				self.txobj['token_addr'] = TokenAddr(etx.creates.hex())
+
+		assert self.check_sigs(),'Signature check failed'
+
+	async def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception
+
+		if self.marked_signed():
+			msg('Transaction is already signed!')
+			return False
+
+		if not self.check_correct_chain(on_fail='return'):
+			return False
+
+		msg_r('Signing transaction{}...'.format(tx_num_str))
+
+		try:
+			await self.do_sign(keys[0].sec.wif,tx_num_str)
+			msg('OK')
+			return True
+		except Exception as e:
+			m = "{!r}: transaction signing failed!"
+			msg(m.format(e.args[0]))
+			if g.traceback:
+				import traceback
+				ymsg('\n'+''.join(traceback.format_exception(*sys.exc_info())))
+			return False
+
 class EthereumTokenMMGenTX(EthereumMMGenTX):
 	desc = 'Ethereum token transaction'
 	contract_desc = 'token contract'
@@ -427,26 +442,18 @@ class EthereumTokenMMGenTX(EthereumMMGenTX):
 	fmt_keys = ('from','token_to','amt','nonce')
 	fee_is_approximate = True
 
-	def __init__(self,*args,**kwargs):
-		if not kwargs.get('offline'):
-			from mmgen.tw import TrackingWallet
-			self.decimals = TrackingWallet().get_param('decimals')
-			from .contract import Token
-			self.token_obj = Token(g.token,self.decimals)
-		EthereumMMGenTX.__init__(self,*args,**kwargs)
-
 	def update_change_output(self,change_amt):
 		if self.outputs[0].is_chg:
 			self.update_output_amt(0,self.inputs[0].amt)
 
 	# token transaction, so check both eth and token balances
 	# TODO: add test with insufficient funds
-	def precheck_sufficient_funds(self,inputs_sum,sel_unspent):
-		eth_bal = self.twuo.wallet.get_eth_balance(sel_unspent[0].addr)
+	async def precheck_sufficient_funds(self,inputs_sum,sel_unspent):
+		eth_bal = await self.tw.get_eth_balance(sel_unspent[0].addr)
 		if eth_bal == 0: # we don't know the fee yet
 			msg('This account has no ether to pay for the transaction fee!')
 			return False
-		return super().precheck_sufficient_funds(inputs_sum,sel_unspent)
+		return await super().precheck_sufficient_funds(inputs_sum,sel_unspent)
 
 	def final_inputs_ok_msg(self,change_amt):
 		token_bal   = ( ETHAmt('0') if self.outputs[0].is_chg else
@@ -454,49 +461,30 @@ class EthereumTokenMMGenTX(EthereumMMGenTX):
 		m = "Transaction leaves ≈{} {} and {} {} in the sender's account"
 		return m.format( change_amt.hl(), g.coin, token_bal.hl(), g.dcoin )
 
-	def get_change_amt(self): # here we know the fee
-		eth_bal = self.twuo.wallet.get_eth_balance(self.inputs[0].addr)
+	async def get_change_amt(self): # here we know the fee
+		eth_bal = await self.tw.get_eth_balance(self.inputs[0].addr)
 		return eth_bal - self.fee
 
-	def resolve_g_token_from_tx_file(self):
-		g.dcoin = self.dcoin
-		if is_hex_str(self.hex): return # for txsend we can leave g.token uninitialized
-		d = json.loads(self.hex)
-		if g.token.upper() == self.dcoin:
-			g.token = d['token_addr']
-		elif g.token != d['token_addr']:
-			m1 = "'{p}': invalid --token parameter for {t} {n} token transaction file\n"
-			m2 = "Please use '--token={t}'"
-			die(1,(m1+m2).format(p=g.token,t=self.dcoin,n=capfirst(g.proto.name)))
+	def resolve_g_token_from_txfile(self):
+		pass
 
-	def make_txobj(self): # called by create_raw()
-		super().make_txobj()
-		t = self.token_obj
+	async def make_txobj(self): # called by create_raw()
+		await super().make_txobj()
+		t = Token(self.tw.token,self.tw.decimals)
 		o = self.txobj
 		o['token_addr'] = t.addr
-		o['decimals'] = t.decimals()
+		o['decimals'] = t.decimals
 		o['token_to'] = o['to']
 		o['data'] = t.create_data(o['token_to'],o['amt'])
 
-	def check_txfile_hex_data(self):
-		d = super().check_txfile_hex_data()
+	def parse_txfile_hex_data(self):
+		d = EthereumMMGenTX.parse_txfile_hex_data(self)
 		o = self.txobj
-
-		if self.check_sigs(): # online, from rlp and wallet
-			o['token_addr'] = TokenAddr(o['to'])
-			o['decimals'] = self.decimals
-		else:                 # offline, from json
-			o['token_addr'] = TokenAddr(d['token_addr'])
-			o['decimals'] = Int(d['decimals'])
-
-		from .contract import Token
-		t = self.token_obj = Token(o['token_addr'],o['decimals'])
-
-		if self.check_sigs(): # online, from rlp - 'amt' was eth amt, now token amt
-			o['amt'] = t.transferdata2amt(o['data'])
-		else:                 # offline, from json - 'amt' is token amt
-			o['data'] = t.create_data(o['to'],o['amt'])
-
+		assert self.tw.token == o['to']
+		o['token_addr'] = TokenAddr(o['to'])
+		o['decimals']   = self.tw.decimals
+		t = Token(o['token_addr'],o['decimals'])
+		o['amt'] = t.transferdata2amt(o['data'])
 		o['token_to'] = type(t).transferdata2sendaddr(o['data'])
 
 	def format_view_body(self,*args,**kwargs):
@@ -505,25 +493,47 @@ class EthereumTokenMMGenTX(EthereumMMGenTX):
 			c=blue('(' + g.dcoin + ')'),
 			r=super().format_view_body(*args,**kwargs))
 
-	def do_sign(self,wif,tx_num_str):
+class EthereumTokenMMGenTxForSigning(EthereumTokenMMGenTX,EthereumMMGenTxForSigning):
+
+	def resolve_g_token_from_txfile(self):
+		d = json.loads(self.hex)
+		if g.token.upper() == self.dcoin:
+			g.token = d['token_addr']
+		elif g.token != d['token_addr']:
+			m1 = "'{p}': invalid --token parameter for {t} {n} token transaction file\n"
+			m2 = "Please use '--token={t}'"
+			die(1,(m1+m2).format(p=g.token,t=self.dcoin,n=capfirst(g.proto.name)))
+
+	def parse_txfile_hex_data(self):
+		d = EthereumMMGenTxForSigning.parse_txfile_hex_data(self)
+		o = self.txobj
+		o['token_addr'] = TokenAddr(d['token_addr'])
+		o['decimals'] = Int(d['decimals'])
+		t = Token(o['token_addr'],o['decimals'])
+		o['data'] = t.create_data(o['to'],o['amt'])
+		o['token_to'] = type(t).transferdata2sendaddr(o['data'])
+
+	async def do_sign(self,wif,tx_num_str):
 		o = self.txobj
-		t = self.token_obj
+		t = Token(o['token_addr'],o['decimals'])
 		tx_in = t.make_tx_in(o['from'],o['to'],o['amt'],self.start_gas,o['gasPrice'],nonce=o['nonce'])
-		(self.hex,self.coin_txid) = t.txsign(tx_in,wif,o['from'],chain_id=o['chainId'])
+		(self.hex,self.coin_txid) = await t.txsign(tx_in,wif,o['from'],chain_id=o['chainId'])
 		assert self.check_sigs(),'Signature check failed'
 
-class EthereumMMGenBumpTX(EthereumMMGenTX,MMGenBumpTX):
+class EthereumMMGenBumpTX(MMGenBumpTX,EthereumMMGenTxForSigning):
 
-	def choose_output(self): pass
-
-	def set_min_fee(self):
-		self.min_fee = ETHAmt(self.fee * Decimal('1.101'))
+	@property
+	def min_fee(self):
+		return ETHAmt(self.fee * Decimal('1.101'))
 
 	def update_fee(self,foo,fee):
 		self.fee = fee
 
-	def get_nonce(self):
+	async def get_nonce(self):
 		return self.txobj['nonce']
 
-class EthereumTokenMMGenBumpTX(EthereumTokenMMGenTX,EthereumMMGenBumpTX): pass
-class EthereumMMGenSplitTX(MMGenSplitTX): pass
+class EthereumTokenMMGenBumpTX(EthereumMMGenBumpTX,EthereumTokenMMGenTxForSigning):
+	pass
+
+class EthereumMMGenSplitTX(MMGenSplitTX):
+	pass

mmgen/daemon.py

@@ -125,7 +125,7 @@ class Daemon(MMGenObject):
 		self.wait_for_state('stopped')
 
 		os.makedirs(self.datadir,exist_ok=True)
-		if self.cfg_file:
+		if self.cfg_file and not 'keep_cfg_file' in self.flags:
 			open('{}/{}'.format(self.datadir,self.cfg_file),'w').write(self.cfg_file_hdr)
 
 		if self.use_pidfile and os.path.exists(self.pidfile):
@@ -221,7 +221,7 @@ class MoneroWalletDaemon(Daemon):
 	exec_fn_mswin = 'monero-wallet-rpc.exe'
 	ps_pid_mswin = True
 
-	def __init__(self,wallet_dir,test_suite=False):
+	def __init__(self,wallet_dir,test_suite=False,host=None,user=None,passwd=None):
 		self.platform = g.platform
 		self.wallet_dir = wallet_dir
 		if test_suite:
@@ -237,7 +237,13 @@ class MoneroWalletDaemon(Daemon):
 		if self.platform == 'win':
 			self.use_pidfile = False
 
-		if not g.monero_wallet_rpc_password:
+		self.host = host or g.monero_wallet_rpc_host
+		self.user = user or g.monero_wallet_rpc_user
+		self.passwd = passwd or g.monero_wallet_rpc_password
+
+		assert self.host
+		assert self.user
+		if not self.passwd:
 			die(1,
 				'You must set your Monero wallet RPC password.\n' +
 				'This can be done on the command line, with the --monero-wallet-rpc-password\n' +
@@ -252,7 +258,7 @@ class MoneroWalletDaemon(Daemon):
 			'--rpc-bind-port={}'.format(self.rpc_port),
 			'--wallet-dir='+self.wallet_dir,
 			'--log-file='+self.logfile,
-			'--rpc-login={}:{}'.format(g.monero_wallet_rpc_user,g.monero_wallet_rpc_password) ]
+			'--rpc-login={}:{}'.format(self.user,self.passwd) ]
 		if self.platform == 'linux':
 			cmd += ['--pidfile={}'.format(self.pidfile)]
 			cmd += [] if 'no_daemonize' in self.flags else ['--detach']
@@ -260,15 +266,16 @@ class MoneroWalletDaemon(Daemon):
 
 	@property
 	def state(self):
-		if not self.test_socket(g.monero_wallet_rpc_host,self.rpc_port):
+		return 'ready' if self.test_socket('localhost',self.rpc_port) else 'stopped'
+		if not self.test_socket(self.host,self.rpc_port):
 			return 'stopped'
-		from .rpc import MoneroWalletRPCConnection
+		from .rpc import MoneroWalletRPCClient
 		try:
-			MoneroWalletRPCConnection(
-				g.monero_wallet_rpc_host,
+			MoneroWalletRPCClient(
+				self.host,
 				self.rpc_port,
-				g.monero_wallet_rpc_user,
-				g.monero_wallet_rpc_password).get_version()
+				self.user,
+				self.passwd).call('get_version')
 			return 'ready'
 		except:
 			return 'stopped'
@@ -280,7 +287,7 @@ class MoneroWalletDaemon(Daemon):
 class CoinDaemon(Daemon):
 	cfg_file_hdr = ''
 	subclasses_must_implement = ('state','stop_cmd')
-	avail_flags = ('no_daemonize',)
+	avail_flags = ('no_daemonize','keep_cfg_file')
 
 	network_ids = ('btc','btc_tn','btc_rt','bch','bch_tn','bch_rt','ltc','ltc_tn','ltc_rt','xmr','eth','etc')
 
@@ -466,6 +473,7 @@ class MoneroDaemon(CoinDaemon):
 	exec_fn_mswin = 'monerod.exe'
 	ps_pid_mswin = True
 	new_console_mswin = True
+	host = 'localhost' # FIXME
 
 	def subclass_init(self):
 		if self.platform == 'win':
@@ -488,7 +496,7 @@ class MoneroDaemon(CoinDaemon):
 
 	@property
 	def state(self):
-		if not self.test_socket(g.monero_wallet_rpc_host,self.rpc_port):
+		if not self.test_socket(self.host,self.rpc_port):
 			return 'stopped'
 		cp = self.run_cmd(
 			[self.coind_exec]
@@ -532,15 +540,22 @@ class EthereumDaemon(CoinDaemon):
 
 	@property
 	def state(self):
-		from .rpc import EthereumRPCConnection
-		try:
-			conn = EthereumRPCConnection('localhost',self.rpc_port,socket_timeout=0.2)
-		except:
-			return 'stopped'
+		return 'ready' if self.test_socket('localhost',self.rpc_port) else 'stopped'
 
-		ret = conn.eth_chainId(on_fail='return')
+		# the following code does not work
+		from mmgen.protocol import init_coin
+		init_coin('eth')
 
-		return ('stopped','ready')[ret == '0x11']
+		async def do():
+			print(g.rpc)
+			ret = await g.rpc.call('eth_chainId')
+			print(ret)
+			return ('stopped','ready')[ret == '0x11']
+
+		try:
+			return run_session(do()) # socket exception is not propagated
+		except:# SocketError:
+			return 'stopped'
 
 	@property
 	def stop_cmd(self):

mmgen/exception.py

@@ -33,6 +33,7 @@ class FileNotFound(Exception):            mmcode = 1
 class InvalidPasswdFormat(Exception):     mmcode = 1
 class CfgFileParseError(Exception):       mmcode = 1
 class UserOptError(Exception):            mmcode = 1
+class NoLEDSupport(Exception):            mmcode = 1
 
 # 2: yellow hl, message only
 class InvalidTokenAddress(Exception):     mmcode = 2

mmgen/globalvars.py

@@ -47,7 +47,7 @@ class g(object):
 	# Constants:
 
 	version      = '0.12.099'
-	release_date = 'March 2020'
+	release_date = 'May 2020'
 
 	proj_name = 'MMGen'
 	proj_url  = 'https://github.com/mmgen/mmgen'
@@ -95,7 +95,7 @@ class g(object):
 	accept_defaults      = False
 	use_internal_keccak_module = False
 
-	chain                = None # set by first call to rpc_init()
+	chain                = None
 	chains               = ('mainnet','testnet','regtest')
 
 	# rpc:
@@ -107,7 +107,8 @@ class g(object):
 	monero_wallet_rpc_user = 'monero'
 	monero_wallet_rpc_password = ''
 	rpc_fail_on_command  = ''
-	rpc                 = None # global RPC handle
+	rpc                  = None # global RPC handle
+	aiohttp_rpc_queue_len = 16
 	use_cached_balances  = False
 
 	# regtest:
@@ -155,7 +156,7 @@ class g(object):
 	# 'long' opts - opt sets global var
 	common_opts = (
 		'color','no_license','testnet',
-		'rpc_host','rpc_port','rpc_user','rpc_password',
+		'rpc_host','rpc_port','rpc_user','rpc_password','rpc_backend','aiohttp_rpc_queue_len',
 		'monero_wallet_rpc_host','monero_wallet_rpc_user','monero_wallet_rpc_password',
 		'daemon_data_dir','force_256_color','regtest','coin','bob','alice',
 		'accept_defaults','token'
@@ -210,6 +211,7 @@ class g(object):
 		'MMGEN_TESTNET',
 		'MMGEN_REGTEST',
 		'MMGEN_TRACEBACK',
+		'MMGEN_RPC_BACKEND',
 		'MMGEN_USE_STANDALONE_SCRYPT_MODULE',
 
 		'MMGEN_DISABLE_COLOR',
@@ -223,12 +225,15 @@ class g(object):
 		'comment_file',
 		'contract_data',
 	)
-	# Auto-typechecked and auto-set opts - incompatible with global_sets_opt and opt_sets_global
+	# Auto-typechecked and auto-set opts.  These have no corresponding value in g.
 	# First value in list is the default
 	ov = namedtuple('autoset_opt_info',['type','choices'])
 	autoset_opts = {
-		'fee_estimate_mode': ov('nocase_pfx', ('conservative','economical')),
+		'fee_estimate_mode': ov('nocase_pfx', ['conservative','economical']),
+		'rpc_backend':       ov('nocase_pfx', ['auto','httplib','curl','aiohttp','requests']),
 	}
+	if platform == 'win':
+		autoset_opts['rpc_backend'].choices.remove('aiohttp')
 
 	min_screen_width = 80
 	minconf = 1

mmgen/main_addrimport.py

@@ -71,119 +71,126 @@ The --batch and --rescan options cannot be used together.
 	}
 }
 
-cmd_args = opts.init(opts_data)
-
-def import_mmgen_list(infile):
-	al = (AddrList,KeyAddrList)[bool(opt.keyaddr_file)](infile)
-	if al.al_id.mmtype in ('S','B'):
-		from .tx import segwit_is_active
-		if not segwit_is_active():
-			rdie(2,'Segwit is not active on this chain. Cannot import Segwit addresses')
-	return al
-
-if len(cmd_args) == 1:
-	infile = cmd_args[0]
-	check_infile(infile)
-	if opt.addrlist:
-		al = AddrList(addrlist=get_lines_from_file(
-			infile,
-			'non-{pnm} addresses'.format(pnm=g.proj_name),
-			trim_comments=True))
+def parse_cmd_args(cmd_args):
+
+	def import_mmgen_list(infile):
+		al = (AddrList,KeyAddrList)[bool(opt.keyaddr_file)](infile)
+		if al.al_id.mmtype in ('S','B'):
+			from .tx import segwit_is_active
+			if not segwit_is_active():
+				rdie(2,'Segwit is not active on this chain. Cannot import Segwit addresses')
+		return al
+
+	if len(cmd_args) == 1:
+		infile = cmd_args[0]
+		check_infile(infile)
+		if opt.addrlist:
+			al = AddrList(addrlist=get_lines_from_file(
+				infile,
+				'non-{pnm} addresses'.format(pnm=g.proj_name),
+				trim_comments=True))
+		else:
+			al = import_mmgen_list(infile)
+	elif len(cmd_args) == 0 and opt.address:
+		al = AddrList(addrlist=[opt.address])
+		infile = 'command line'
 	else:
-		al = import_mmgen_list(infile)
-elif len(cmd_args) == 0 and opt.address:
-	al = AddrList(addrlist=[opt.address])
-	infile = 'command line'
-else:
-	die(1,ai_msgs('bad_args'))
+		die(1,ai_msgs('bad_args'))
+
+	return al,infile
+
+def check_opts(tw):
+	batch = bool(opt.batch)
+	rescan = bool(opt.rescan)
+
+	if rescan and not 'rescan' in tw.caps:
+		msg("'--rescan' ignored: not supported by {}".format(type(tw).__name__))
+		rescan = False
+
+	if rescan and not opt.quiet:
+		confirm_or_raise(ai_msgs('rescan'),'continue',expect='YES')
+
+	if batch and not 'batch' in tw.caps:
+		msg("'--batch' ignored: not supported by {}".format(type(tw).__name__))
+		batch = False
+
+	return batch,rescan
+
+async def import_addr(tw,addr,label,rescan,msg_fmt,msg_args):
+	try:
+		task = asyncio.ensure_future(tw.import_address(addr,label,rescan)) # Python 3.7+: create_task()
+		if rescan:
+			start = time.time()
+			while True:
+				if task.done():
+					break
+				msg_r(('\r{} '+msg_fmt).format(secs_to_hms(int(time.time()-start)),*msg_args))
+				await asyncio.sleep(0.5)
+			await task
+			msg('\nOK')
+		else:
+			await task
+			qmsg(msg_fmt.format(*msg_args) + ' - OK')
+	except Exception as e:
+		die(2,'\nImport of address {!r} failed: {!r}'.format(addr,e.args[0]))
 
-m = ' from Seed ID {}'.format(al.al_id.sid) if hasattr(al.al_id,'sid') else ''
-qmsg('OK. {} addresses{}'.format(al.num_addrs,m))
+def make_args_list(tw,al,batch,rescan):
 
-err_msg = None
+	fs = '{:%s} {:34} {:%s}' % (
+		len(str(al.num_addrs)) * 2 + 2,
+		1 if opt.addrlist or opt.address else len(str(max(al.idxs()))) + 13 )
 
-from .tw import TrackingWallet
-tw = TrackingWallet(mode='w')
+	for num,e in enumerate(al.data,1):
+		if e.idx:
+			label = '{}:{}'.format(al.al_id,e.idx) + (' ' + e.label if e.label else '')
+			add_msg = label
+		else:
+			label = '{}:{}'.format(g.proto.base_coin.lower(),e.addr)
+			add_msg = 'non-'+g.proj_name
 
-if g.token:
-	if not is_coin_addr(g.token):
-		m = "When importing addresses for a new token, the token must be specified by address, not symbol."
-		raise InvalidTokenAddress('{!r}: invalid token address\n{}'.format(m))
-	sym = tw.addr2sym(g.token) # check for presence in wallet or blockchain; raises exception on failure
+		if batch:
+			yield (e.addr,TwLabel(label),False)
+		else:
+			msg_args = ( f'{num}/{al.num_addrs}:', e.addr, '('+add_msg+')' )
+			yield (tw,e.addr,TwLabel(label),rescan,fs,msg_args)
 
-if opt.rescan and not 'rescan' in tw.caps:
-	msg("'--rescan' ignored: not supported by {}".format(type(tw).__name__))
-	opt.rescan = False
+async def main():
+	al,infile = parse_cmd_args(cmd_args)
 
-if opt.rescan and not opt.quiet:
-	confirm_or_raise(ai_msgs('rescan'),'continue',expect='YES')
+	qmsg(
+		f'OK. {al.num_addrs} addresses'
+		+ (f' from Seed ID {al.al_id.sid}' if hasattr(al.al_id,'sid') else '') )
 
-if opt.batch and not 'batch' in tw.caps:
-	msg("'--batch' ignored: not supported by {}".format(type(tw).__name__))
-	opt.batch = False
+	msg(
+		f'Importing {len(al.data)} address{suf(al.data,"es")} from {infile}'
+		+ (' (batch mode)' if opt.batch else '') )
 
-def import_address(addr,label,rescan):
-	try: tw.import_address(addr,label,rescan)
-	except Exception as e:
-		global err_msg
-		err_msg = e.args[0]
-	if g.token and not tw.get_token_param(g.token,'symbol'):
-		tw.set_token_param(g.token,'symbol',sym)
-		tw.set_token_param(g.token,'decimals',tw.token_obj.decimals())
-
-w_n_of_m = len(str(al.num_addrs)) * 2 + 2
-w_mmid = 1 if opt.addrlist or opt.address else len(str(max(al.idxs()))) + 13
-msg_fmt = '{{:{}}} {{:34}} {{:{}}}'.format(w_n_of_m,w_mmid)
-
-if opt.rescan: import threading
-
-fs = 'Importing {} address{} from {}{}'
-bm =' (batch mode)' if opt.batch else ''
-msg(fs.format(len(al.data),suf(al.data,'es'),infile,bm))
-
-if not al.data[0].addr.is_for_chain(g.chain):
-	die(2,'Address{} not compatible with {} chain!'.format((' list','')[bool(opt.address)],g.chain))
-
-for n,e in enumerate(al.data):
-	if e.idx:
-		label = '{}:{}'.format(al.al_id,e.idx)
-		if e.label: label += ' ' + e.label
-		m = label
-	else:
-		label = '{}:{}'.format(g.proto.base_coin.lower(),e.addr)
-		m = 'non-'+g.proj_name
-
-	label = TwLabel(label)
-
-	if opt.batch:
-		if n == 0: arg_list = []
-		arg_list.append((e.addr,label,False))
-		continue
-
-	msg_data = ('{}/{}:'.format(n+1,al.num_addrs),e.addr,'({})'.format(m))
-
-	if opt.rescan:
-		t = threading.Thread(target=import_address,args=[e.addr,label,True])
-		t.daemon = True
-		t.start()
-		start = int(time.time())
-		while True:
-			if t.is_alive():
-				elapsed = int(time.time()-start)
-				msg_r(('\r{} '+msg_fmt).format(secs_to_hms(elapsed),*msg_data))
-				time.sleep(0.5)
-			else:
-				if err_msg: die(2,'\nImport failed: {!r}'.format(err_msg))
-				msg('\nOK')
-				break
+	if not al.data[0].addr.is_for_chain(g.chain):
+		die(2,f'Address{(" list","")[bool(opt.address)]} incompatible with {g.chain} chain!')
+
+	from .tw import TrackingWallet
+	tw = await TrackingWallet(mode='i')
+
+	batch,rescan = check_opts(tw)
+
+	if g.token:
+		await tw.import_token()
+
+	args_list = make_args_list(tw,al,batch,rescan)
+
+	if batch:
+		ret = await tw.batch_import_address(list(args_list))
+		msg(f'OK: {len(ret)} addresses imported')
+	elif rescan:
+		for arg_list in args_list:
+			await import_addr(*arg_list)
 	else:
-		import_address(e.addr,label,False)
-		msg_r('\r'+msg_fmt.format(*msg_data))
-		if err_msg: die(2,'\nImport failed: {!r}'.format(err_msg))
-		msg(' - OK')
+		tasks = [import_addr(*arg_list) for arg_list in args_list]
+		await asyncio.gather(*tasks)
+		msg('OK')
 
-if opt.batch:
-	ret = tw.batch_import_address(arg_list)
-	msg('OK: {} addresses imported'.format(len(ret)))
+	del tw
 
-del tw
+cmd_args = opts.init(opts_data)
+import asyncio
+run_session(main())

mmgen/main_autosign.py

@@ -112,18 +112,19 @@ cmd_args = opts.init(opts_data,add_opts=['mmgen_keys_from_file','in_fmt'])
 exit_if_mswin('autosigning')
 
 import mmgen.tx
-import mmgen.altcoins.eth.tx
 from .txsign import txsign
 from .protocol import CoinProtocol,init_coin
+from .rpc import rpc_init
+
 if g.test_suite:
 	from .daemon import CoinDaemon
 
 if opt.mountpoint:
-	mountpoint = opt.mountpoint # TODO: make global
+	mountpoint = opt.mountpoint
 
 opt.outdir = tx_dir = os.path.join(mountpoint,'tx')
 
-def check_daemons_running():
+async def check_daemons_running():
 	if opt.coin:
 		die(1,'--coin option not supported with this command.  Use --coins instead')
 	if opt.coins:
@@ -140,7 +141,7 @@ def check_daemons_running():
 				g.rpc_port = CoinDaemon(get_network_id(coin,g.testnet),test_suite=True).rpc_port
 			vmsg(f'Checking {coin} daemon')
 			try:
-				rpc_init(reinit=True)
+				await rpc_init()
 			except SystemExit as e:
 				if e.code != 0:
 					ydie(1,f'{coin} daemon not running or not listening on port {g.proto.rpc_port}')
@@ -174,7 +175,7 @@ def do_umount():
 		msg(f'Unmounting {mountpoint}')
 		run(['umount',mountpoint],check=True)
 
-def sign_tx_file(txfile,signed_txs):
+async def sign_tx_file(txfile,signed_txs):
 	try:
 		init_coin('BTC',testnet=False)
 		tmp_tx = mmgen.tx.MMGenTX(txfile,metadata_only=True)
@@ -193,15 +194,15 @@ def sign_tx_file(txfile,signed_txs):
 		g.token = tmp_tx.dcoin
 		g.dcoin = tmp_tx.dcoin or g.coin
 
-		tx = mmgen.tx.MMGenTX(txfile,offline=True)
+		tx = mmgen.tx.MMGenTxForSigning(txfile)
 
 		if g.proto.sign_mode == 'daemon':
 			if g.test_suite:
 				g.proto.daemon_data_dir = 'test/daemons/' + g.coin.lower()
 				g.rpc_port = CoinDaemon(get_network_id(g.coin,g.testnet),test_suite=True).rpc_port
-			rpc_init(reinit=True)
+			await rpc_init()
 
-		if txsign(tx,wfs,None,None):
+		if await txsign(tx,wfs,None,None):
 			tx.write_to_file(ask_write=False)
 			signed_txs.append(tx)
 			return True
@@ -215,7 +216,7 @@ def sign_tx_file(txfile,signed_txs):
 	except:
 		return False
 
-def sign():
+async def sign():
 	dirlist  = os.listdir(tx_dir)
 	raw,signed = [set(f[:-6] for f in dirlist if f.endswith(ext)) for ext in ('.rawtx','.sigtx')]
 	unsigned = [os.path.join(tx_dir,f+'.rawtx') for f in raw - signed]
@@ -223,7 +224,7 @@ def sign():
 	if unsigned:
 		signed_txs,fails = [],[]
 		for txfile in unsigned:
-			ret = sign_tx_file(txfile,signed_txs)
+			ret = await sign_tx_file(txfile,signed_txs)
 			if not ret:
 				fails.append(txfile)
 			qmsg('')
@@ -296,23 +297,23 @@ def print_summary(signed_txs):
 	else:
 		msg('No non-MMGen outputs')
 
-def do_sign():
+async def do_sign():
 	if not opt.stealth_led:
-		set_led('busy')
+		led.set('busy')
 	do_mount()
 	key_ok = decrypt_wallets()
 	if key_ok:
 		if opt.stealth_led:
-			set_led('busy')
-		ret = sign()
+			led.set('busy')
+		ret = await sign()
 		do_umount()
-		set_led(('standby','off','error')[(not ret)*2 or bool(opt.stealth_led)])
+		led.set(('standby','off','error')[(not ret)*2 or bool(opt.stealth_led)])
 		return ret
 	else:
 		msg('Password is incorrect!')
 		do_umount()
 		if not opt.stealth_led:
-			set_led('error')
+			led.set('error')
 		return False
 
 def wipe_existing_key():
@@ -374,35 +375,6 @@ def setup():
 	ss_out = Wallet(ss=ss_in)
 	ss_out.write_to_file(desc='autosign wallet',outdir=wallet_dir)
 
-def ev_sleep(secs):
-	ev.wait(secs)
-	return ev.isSet()
-
-def do_led(on,off):
-	if not on:
-		open(status_ctl,'w').write('0\n')
-		while True:
-			if ev_sleep(3600): return
-
-	while True:
-		for s_time,val in ((on,255),(off,0)):
-			open(status_ctl,'w').write('{}\n'.format(val))
-			if ev_sleep(s_time): return
-
-def set_led(cmd):
-	if not opt.led: return
-	vmsg("Setting LED state to '{}'".format(cmd))
-	timings = {
-		'off':     ( 0, 0 ),
-		'standby': ( 2.2, 0.2 ),
-		'busy':    ( 0.06, 0.06 ),
-		'error':   ( 0.5, 0.5 )}[cmd]
-	global led_thread
-	if led_thread:
-		ev.set(); led_thread.join(); ev.clear()
-	led_thread = threading.Thread(target=do_led,name='LED loop',args=timings)
-	led_thread.start()
-
 def get_insert_status():
 	if opt.no_insert_check:
 		return True
@@ -410,15 +382,21 @@ def get_insert_status():
 	except: return False
 	else: return True
 
-def do_loop():
+def check_wipe_present():
+	try:
+		run(['wipe','-v'],stdout=DEVNULL,stderr=DEVNULL,check=True)
+	except:
+		die(2,"The 'wipe' utility must be installed before running this program")
+
+async def do_loop():
 	n,prev_status = 0,False
 	if not opt.stealth_led:
-		set_led('standby')
+		led.set('standby')
 	while True:
 		status = get_insert_status()
 		if status and not prev_status:
 			msg('Device insertion detected')
-			do_sign()
+			await do_sign()
 		prev_status = status
 		if not n % 10:
 			msg_r('\r{}\rWaiting'.format(' '*17))
@@ -427,54 +405,6 @@ def do_loop():
 		msg_r('.')
 		n += 1
 
-def check_access(fn,desc='status LED control',init_val=None):
-	try:
-		b = open(fn).read().strip()
-		open(fn,'w').write('{}\n'.format(init_val or b))
-		return True
-	except:
-		m1 = "You do not have access to the {} file\n".format(desc)
-		m2 = "To allow access, run 'sudo chmod 0666 {}'".format(fn)
-		msg(m1+m2)
-		return False
-
-def check_wipe_present():
-	try:
-		run(['wipe','-v'],stdout=DEVNULL,stderr=DEVNULL,check=True)
-	except:
-		die(2,"The 'wipe' utility must be installed before running this program")
-
-def init_led():
-	sc = {
-		'opi': '/sys/class/leds/orangepi:red:status/brightness',
-		'rpi': '/sys/class/leds/led0/brightness'
-	}
-	tc = {
-		'rpi': '/sys/class/leds/led0/trigger', # mmc,none
-	}
-	for k in ('opi','rpi'):
-		try: os.stat(sc[k])
-		except: pass
-		else:
-			board = k
-			break
-	else:
-		die(2,'Control files not found!  LED option not supported')
-
-	status_ctl  = sc[board]
-	trigger_ctl = tc[board] if board in tc else None
-
-	if not check_access(status_ctl) or (
-			trigger_ctl and not check_access(trigger_ctl,desc='LED trigger',init_val='none')
-		):
-		sys.exit(1)
-
-	if trigger_ctl:
-		open(trigger_ctl,'w').write('none\n')
-
-	return status_ctl,trigger_ctl
-
-# main()
 if len(cmd_args) not in (0,1):
 	opts.usage()
 
@@ -489,32 +419,29 @@ if len(cmd_args) == 1:
 check_wipe_present()
 wfs = get_wallet_files()
 
-check_daemons_running()
-
-def at_exit(exit_val,nl=False):
-	if nl: msg('')
-	msg('Cleaning up...')
-	if opt.led:
-		set_led('off')
-		ev.set()
-		led_thread.join()
-		if trigger_ctl:
-			open(trigger_ctl,'w').write('mmc0\n')
+def at_exit(exit_val,message='\nCleaning up...'):
+	if message:
+		msg(message)
+	led.stop()
 	sys.exit(exit_val)
 
-def handler(a,b): at_exit(1,nl=True)
+def handler(a,b):
+	at_exit(1)
 
 signal.signal(signal.SIGTERM,handler)
 signal.signal(signal.SIGINT,handler)
 
-if opt.led:
-	import threading
-	status_ctl,trigger_ctl = init_led()
-	ev = threading.Event()
-	led_thread = None
-
-if len(cmd_args) == 0:
-	ret = do_sign()
-	at_exit(int(not ret))
-elif cmd_args[0] == 'wait':
-	do_loop()
+from .led import LEDControl
+led = LEDControl(enabled=opt.led,simulate=g.test_suite and not os.getenv('MMGEN_TEST_SUITE_AUTOSIGN_LIVE'))
+led.set('off')
+
+async def main():
+	await check_daemons_running()
+
+	if len(cmd_args) == 0:
+		ret = await do_sign()
+		at_exit(int(not ret),message='')
+	elif cmd_args[0] == 'wait':
+		await do_loop()
+
+run_session(main(),do_rpc_init=False)

mmgen/main_split.py

@@ -20,6 +20,7 @@
 
 """
 mmgen-split: Split funds after a replayable chain fork using a timelocked transaction
+             UNMAINTAINED
 """
 
 import time
@@ -115,28 +116,27 @@ if opt.tx_fees:
 		opt.tx_fee = opt.tx_fees.split(',')[idx]
 		opts.opt_is_tx_fee('foo',opt.tx_fee,'transaction fee') # raises exception on error
 
-rpc_init(reinit=True)
-
 tx1 = MMGenSplitTX()
 opt.no_blank = True
 
-gmsg("Creating timelocked transaction for long chain ({})".format(g.coin))
-locktime = int(opt.locktime or 0) or g.rpc.getblockcount()
-tx1.create(mmids[0],locktime)
+async def main():
+	gmsg("Creating timelocked transaction for long chain ({})".format(g.coin))
+	locktime = int(opt.locktime or 0) or await g.rpc.call('getblockcount')
+	tx1.create(mmids[0],locktime)
 
-tx1.format()
-tx1.create_fn()
+	tx1.format()
+	tx1.create_fn()
 
-gmsg("\nCreating transaction for short chain ({})".format(opt.other_coin))
+	gmsg("\nCreating transaction for short chain ({})".format(opt.other_coin))
 
-init_coin(opt.other_coin)
+	init_coin(opt.other_coin)
 
-tx2 = MMGenSplitTX()
-tx2.inputs = tx1.inputs
-tx2.inputs.convert_coin()
+	tx2 = MMGenSplitTX()
+	tx2.inputs = tx1.inputs
+	tx2.inputs.convert_coin()
 
-tx2.create_split(mmids[1])
+	tx2.create_split(mmids[1])
 
-for tx,desc in ((tx1,'Long chain (timelocked)'),(tx2,'Short chain')):
-	tx.desc = desc + ' transaction'
-	tx.write_to_file(ask_write=False,ask_overwrite=not opt.yes,ask_write_default_yes=False)
+	for tx,desc in ((tx1,'Long chain (timelocked)'),(tx2,'Short chain')):
+		tx.desc = desc + ' transaction'
+		tx.write_to_file(ask_write=False,ask_overwrite=not opt.yes,ask_write_default_yes=False)

mmgen/main_tool.py

@@ -110,4 +110,7 @@ args,kwargs = tool._process_args(cmd,cmd_args)
 
 ret = tool.MMGenToolCmds.call(cmd,*args,**kwargs)
 
+if type(ret).__name__ == 'coroutine':
+	ret = run_session(ret)
+
 tool._process_result(ret,pager='pager' in kwargs and kwargs['pager'],print_result=True)

mmgen/main_txbump.py

@@ -96,8 +96,6 @@ column below:
 
 cmd_args = opts.init(opts_data)
 
-rpc_init()
-
 tx_file = cmd_args.pop(0)
 check_infile(tx_file)
 
@@ -109,61 +107,62 @@ seed_files = get_seed_files(opt,cmd_args) if (cmd_args or opt.send) else None
 kal = get_keyaddrlist(opt)
 kl = get_keylist(opt)
 
-tx = MMGenBumpTX(filename=tx_file,send=(seed_files or kl or kal))
+sign_and_send = bool(seed_files or kl or kal)
 
 do_license_msg()
 
 silent = opt.yes and opt.tx_fee != None and opt.output_to_reduce != None
 
-if not silent:
-	msg(green('ORIGINAL TRANSACTION'))
-	msg(tx.format_view(terse=True))
+async def main():
 
-tx.set_min_fee()
+	from .tw import TrackingWallet
+	tx = MMGenBumpTX(filename=tx_file,send=sign_and_send,tw=await TrackingWallet() if g.token else None)
 
-tx.check_bumpable()
+	if not silent:
+		msg(green('ORIGINAL TRANSACTION'))
+		msg(tx.format_view(terse=True))
 
-msg('Creating new transaction')
+	tx.check_bumpable() # needs cached networkinfo['relayfee']
 
-op_idx = tx.choose_output()
+	msg('Creating new transaction')
 
-if not silent:
-	msg('Minimum fee for new transaction: {} {}'.format(tx.min_fee.hl(),g.coin))
+	op_idx = tx.choose_output()
 
-fee = tx.get_usr_fee_interactive(tx_fee=opt.tx_fee,desc='User-selected')
+	if not silent:
+		msg('Minimum fee for new transaction: {} {}'.format(tx.min_fee.hl(),g.coin))
 
-tx.update_fee(op_idx,fee)
+	fee = tx.get_usr_fee_interactive(tx_fee=opt.tx_fee,desc='User-selected')
 
-d = tx.get_fee_from_tx()
-assert d == fee and d <= g.proto.max_tx_fee
+	tx.update_fee(op_idx,fee)
 
-if g.proto.base_proto == 'Bitcoin':
-	tx.outputs.sort_bip69() # output amts have changed, so re-sort
+	d = tx.get_fee_from_tx()
+	assert d == fee and d <= g.proto.max_tx_fee
 
-if not opt.yes:
-	tx.add_comment()   # edits an existing comment
+	if g.proto.base_proto == 'Bitcoin':
+		tx.outputs.sort_bip69() # output amts have changed, so re-sort
 
-from .tw import TwUnspentOutputs
-tx.twuo = TwUnspentOutputs(minconf=opt.minconf)
+	if not opt.yes:
+		tx.add_comment()   # edits an existing comment
 
-tx.create_raw()        # creates tx.hex, tx.txid
-tx.add_timestamp()
-tx.add_blockcount()
+	await tx.create_raw() # creates tx.hex, tx.txid
 
-qmsg('Fee successfully increased')
+	tx.add_timestamp()
+	tx.add_blockcount()
 
-if not silent:
-	msg(green('\nREPLACEMENT TRANSACTION:'))
-	msg_r(tx.format_view(terse=True))
+	qmsg('Fee successfully increased')
 
-del tx.twuo.wallet
+	if not silent:
+		msg(green('\nREPLACEMENT TRANSACTION:'))
+		msg_r(tx.format_view(terse=True))
 
-if seed_files or kl or kal:
-	if txsign(tx,seed_files,kl,kal):
-		tx.write_to_file(ask_write=False)
-		tx.send(exit_on_fail=True)
-		tx.write_to_file(ask_write=False)
+	if sign_and_send:
+		if await txsign(tx,seed_files,kl,kal):
+			tx.write_to_file(ask_write=False)
+			await tx.send(exit_on_fail=True)
+			tx.write_to_file(ask_write=False)
+		else:
+			die(2,'Transaction could not be signed')
 	else:
-		die(2,'Transaction could not be signed')
-else:
-	tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=False,ask_overwrite=not opt.yes)
+		tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=False,ask_overwrite=not opt.yes)
+
+run_session(main())

mmgen/main_txcreate.py

@@ -78,9 +78,11 @@ cmd_args = opts.init(opts_data)
 
 g.use_cached_balances = opt.cached_balances
 
-rpc_init()
+async def main():
+	from .tx import MMGenTX
+	from .tw import TrackingWallet
+	tx = MMGenTX(tw=await TrackingWallet() if g.token else None)
+	await tx.create(cmd_args,int(opt.locktime or 0),do_info=opt.info)
+	tx.write_to_file(ask_write=not opt.yes,ask_overwrite=not opt.yes,ask_write_default_yes=False)
 
-from .tx import MMGenTX
-tx = MMGenTX()
-tx.create(cmd_args,int(opt.locktime or 0),do_info=opt.info)
-tx.write_to_file(ask_write=not opt.yes,ask_overwrite=not opt.yes,ask_write_default_yes=False)
+run_session(main())

mmgen/main_txdo.py

@@ -115,8 +115,6 @@ cmd_args = opts.init(opts_data)
 
 g.use_cached_balances = opt.cached_balances
 
-rpc_init()
-
 from .tx import *
 from .txsign import *
 
@@ -124,16 +122,23 @@ seed_files = get_seed_files(opt,cmd_args)
 
 kal = get_keyaddrlist(opt)
 kl = get_keylist(opt)
-if kl and kal: kl.remove_dup_keys(kal)
+if kl and kal:
+	kl.remove_dup_keys(kal)
+
+async def main():
+	from .tw import TrackingWallet
+	tx1 = MMGenTX(caller='txdo',tw=await TrackingWallet() if g.token else None)
+
+	await tx1.create(cmd_args,int(opt.locktime or 0))
 
-tx = MMGenTX(caller='txdo')
+	tx2 = MMGenTxForSigning(data=tx1.__dict__)
 
-tx.create(cmd_args,int(opt.locktime or 0))
+	if await txsign(tx2,seed_files,kl,kal):
+		tx2.write_to_file(ask_write=False)
+		await tx2.send(exit_on_fail=True)
+		tx2.write_to_file(ask_overwrite=False,ask_write=False)
+		tx2.print_contract_addr()
+	else:
+		die(2,'Transaction could not be signed')
 
-if txsign(tx,seed_files,kl,kal):
-	tx.write_to_file(ask_write=False)
-	tx.send(exit_on_fail=True)
-	tx.write_to_file(ask_overwrite=False,ask_write=False)
-	tx.print_contract_addr()
-else:
-	die(2,'Transaction could not be signed')
+run_session(main())

mmgen/main_txsend.py

@@ -40,8 +40,6 @@ opts_data = {
 
 cmd_args = opts.init(opts_data)
 
-rpc_init()
-
 if len(cmd_args) == 1:
 	infile = cmd_args[0]; check_infile(infile)
 else:
@@ -52,22 +50,33 @@ if not opt.status:
 
 from .tx import *
 
-tx = MMGenTX(infile,quiet_open=True) # sig check performed here
-vmsg("Signed transaction file '{}' is valid".format(infile))
+async def main():
+
+	from .tw import TrackingWallet
+	tx = MMGenTX(infile,quiet_open=True,tw=await TrackingWallet() if g.token else None)
+
+	if g.token:
+		from .tw import TrackingWallet
+		tx.tw = await TrackingWallet()
+
+	vmsg("Signed transaction file '{}' is valid".format(infile))
+
+	if not tx.marked_signed():
+		die(1,'Transaction is not signed!')
 
-if not tx.marked_signed():
-	die(1,'Transaction is not signed!')
+	if opt.status:
+		if tx.coin_txid:
+			qmsg('{} txid: {}'.format(g.coin,tx.coin_txid.hl()))
+		await tx.get_status(status=True)
+		sys.exit(0)
 
-if opt.status:
-	if tx.coin_txid: qmsg('{} txid: {}'.format(g.coin,tx.coin_txid.hl()))
-	tx.get_status(status=True)
-	sys.exit(0)
+	if not opt.yes:
+		tx.view_with_prompt('View transaction data?')
+		if tx.add_comment(): # edits an existing comment, returns true if changed
+			tx.write_to_file(ask_write_default_yes=True)
 
-if not opt.yes:
-	tx.view_with_prompt('View transaction data?')
-	if tx.add_comment(): # edits an existing comment, returns true if changed
-		tx.write_to_file(ask_write_default_yes=True)
+	await tx.send(exit_on_fail=True)
+	tx.write_to_file(ask_overwrite=False,ask_write=False)
+	tx.print_contract_addr()
 
-tx.send(exit_on_fail=True)
-tx.write_to_file(ask_overwrite=False,ask_write=False)
-tx.print_contract_addr()
+run_session(main())

mmgen/main_txsign.py

@@ -97,9 +97,6 @@ if not infiles:
 for i in infiles:
 	check_infile(i)
 
-if g.proto.sign_mode == 'daemon':
-	rpc_init()
-
 if not opt.info and not opt.terse_info:
 	do_license_msg(immed=True)
 
@@ -107,39 +104,51 @@ from .txsign import *
 
 tx_files   = get_tx_files(opt,infiles)
 seed_files = get_seed_files(opt,infiles)
-
 kal        = get_keyaddrlist(opt)
 kl         = get_keylist(opt)
-if kl and kal: kl.remove_dup_keys(kal)
 
-tx_num_str,bad_tx_count = '',0
-for tx_num,tx_file in enumerate(tx_files,1):
-	if len(tx_files) > 1:
-		msg('\nTransaction #{} of {}:'.format(tx_num,len(tx_files)))
-		tx_num_str = ' #{}'.format(tx_num)
-	tx = MMGenTX(tx_file,offline=True)
+if kl and kal:
+	kl.remove_dup_keys(kal)
+
+async def main():
+	bad_tx_count = 0
+	tx_num_disp = ''
+	for tx_num,tx_file in enumerate(tx_files,1):
+		if len(tx_files) > 1:
+			msg('\nTransaction #{} of {}:'.format(tx_num,len(tx_files)))
+			tx_num_disp = f' #{tx_num}'
 
-	if tx.marked_signed():
-		msg('Transaction is already signed!'); continue
+		tx = MMGenTxForSigning(tx_file)
 
-	vmsg("Successfully opened transaction file '{}'".format(tx_file))
+		if tx.marked_signed():
+			msg('Transaction is already signed!')
+			continue
 
-	if opt.tx_id:
-		msg(tx.txid); continue
+		vmsg(f'Successfully opened transaction file {tx_file!r}')
 
-	if opt.info or opt.terse_info:
-		tx.view(pause=False,terse=opt.terse_info); continue
+		if opt.tx_id:
+			msg(tx.txid)
+			continue
 
-	if not opt.yes:
-		tx.view_with_prompt('View data for transaction{}?'.format(tx_num_str))
+		if opt.info or opt.terse_info:
+			tx.view(pause=False,terse=opt.terse_info)
+			continue
 
-	if txsign(tx,seed_files,kl,kal,tx_num_str):
 		if not opt.yes:
-			tx.add_comment() # edits an existing comment
-		tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=True,add_desc=tx_num_str)
-	else:
-		ymsg('Transaction could not be signed')
-		bad_tx_count += 1
-
-if bad_tx_count:
-	ydie(2,'{} transaction{} could not be signed'.format(bad_tx_count,suf(bad_tx_count)))
+			tx.view_with_prompt(f'View data for transaction{tx_num_disp}?')
+
+		if await txsign(tx,seed_files,kl,kal,tx_num_disp):
+			if not opt.yes:
+				tx.add_comment() # edits an existing comment
+			tx.write_to_file(ask_write=not opt.yes,ask_write_default_yes=True,add_desc=tx_num_disp)
+		else:
+			ymsg('Transaction could not be signed')
+			bad_tx_count += 1
+
+	if bad_tx_count:
+		ydie(2,f'{bad_tx_count} transaction{suf(bad_tx_count)} could not be signed')
+
+run_session(
+	main(),
+	do_rpc_init = g.proto.sign_mode == 'daemon'
+)

mmgen/obj.py

@@ -28,6 +28,12 @@ from .exception import *
 from .globalvars import *
 from .color import *
 
+class aInitMeta(type):
+	async def __call__(cls,*args,**kwargs):
+		instance = super().__call__(*args,**kwargs)
+		await instance.__ainit__(*args,**kwargs)
+		return instance
+
 def is_mmgen_seed_id(s): return SeedID(sid=s,on_fail='silent')
 def is_mmgen_idx(s):     return AddrIdx(s,on_fail='silent')
 def is_mmgen_id(s):      return MMGenID(s,on_fail='silent')

mmgen/opts.py

@@ -110,7 +110,7 @@ def override_globals_from_cfg_file(ucfg):
 		else:
 			die(2,'{!r}: unrecognized option in {!r}, line {}'.format(d.name,ucfg.fn,d.lineno))
 
-def override_globals_from_env():
+def override_globals_and_set_opts_from_env(opt):
 	for name in g.env_opts:
 		if name == 'MMGEN_DEBUG_ALL':
 			continue
@@ -118,7 +118,13 @@ def override_globals_from_env():
 		val = os.getenv(name) # os.getenv() returns None if env var is unset
 		if val: # exclude empty string values; string value of '0' or 'false' sets variable to False
 			gname = name[(6,14)[disable]:].lower()
-			setattr(g,gname,set_for_type(val,getattr(g,gname),name,disable))
+			if hasattr(g,gname):
+				setattr(g,gname,set_for_type(val,getattr(g,gname),name,disable))
+			elif hasattr(opt,gname):
+				if getattr(opt,gname) is None: # env must not override cmdline!
+					setattr(opt,gname,val)
+			else:
+				raise ValueError(f'Name {gname} not present in globals or opts')
 
 def common_opts_code(s):
 	from .protocol import CoinProtocol
@@ -167,6 +173,8 @@ common_opts_data = {
 --, --rpc-port=p          Communicate with {dn} listening on port 'p'
 --, --rpc-user=user       Override 'rpc_user' in mmgen.cfg
 --, --rpc-password=pass   Override 'rpc_password' in mmgen.cfg
+--, --rpc-backend=s       Override 'rpc_backend' in mmgen.cfg
+--, --aiohttp-rpc-queue-len=N Override 'aiohttp_rpc_queue_len' in mmgen.cfg
 --, --monero-wallet-rpc-host=host Override 'monero_wallet_rpc_host' in mmgen.cfg
 --, --monero-wallet-rpc-user=user Override 'monero_wallet_rpc_user' in mmgen.cfg
 --, --monero-wallet-rpc-password=pass Override 'monero_wallet_rpc_password' in mmgen.cfg
@@ -232,7 +240,7 @@ def init(opts_data,add_opts=[],opt_filter=None,parse_only=False):
 		cfg_file('sample') # check for changes in system template file
 		override_globals_from_cfg_file(cfg_file('usr'))
 
-	override_globals_from_env()
+	override_globals_and_set_opts_from_env(opt)
 
 	# Set globals from opts, setting type from original global value
 	# Do here, before opts are set from globals below
@@ -240,7 +248,7 @@ def init(opts_data,add_opts=[],opt_filter=None,parse_only=False):
 	for k in (g.common_opts + g.opt_sets_global):
 		if hasattr(opt,k):
 			val = getattr(opt,k)
-			if val != None:
+			if val != None and hasattr(g,k):
 				setattr(g,k,set_for_type(val,getattr(g,k),'--'+k))
 
 	g.coin = g.coin.upper() # allow user to use lowercase
@@ -337,7 +345,7 @@ def opt_is_tx_fee(key,val,desc): # 'key' must remain a placeholder
 		return
 
 	from .tx import MMGenTX
-	tx = MMGenTX(offline=True)
+	tx = MMGenTX()
 	# Size of 224 is just a ball-park figure to eliminate the most extreme cases at startup
 	# This check will be performed again once we know the true size
 	ret = tx.process_fee_spec(val,224,on_fail='return')
@@ -466,7 +474,8 @@ def check_usr_opts(usr_opts): # Raises an exception if any check fails
 		opt_compares(val,'<=',g.max_urandchars,desc)
 
 	def chk_tx_fee(key,val,desc):
-		opt_is_tx_fee(key,val,desc)
+		pass
+#		opt_is_tx_fee(key,val,desc) # TODO: move this check elsewhere
 
 	def chk_tx_confs(key,val,desc):
 		opt_is_int(val,desc)

mmgen/rpc.py

@@ -20,217 +20,383 @@
 rpc.py:  Cryptocoin RPC library for the MMGen suite
 """
 
-import http.client,base64,json
+import base64,json,asyncio
 from decimal import Decimal
-
 from .common import *
+from .obj import aInitMeta
+
+rpc_credentials_msg = lambda: '\n'+fmt(f"""
+	Error: no {g.proto.name.capitalize()} RPC authentication method found
+
+	RPC credentials must be supplied using one of the following methods:
+
+	A) If daemon is local and running as same user as you:
+
+	   - no credentials required, or matching rpcuser/rpcpassword and
+	     rpc_user/rpc_password values in {g.proto.name}.conf and mmgen.cfg
+
+	B) If daemon is running remotely or as different user:
+
+	   - matching credentials in {g.proto.name}.conf and mmgen.cfg as described above
+
+	The --rpc-user/--rpc-password options may be supplied on the MMGen command line.
+	They override the corresponding values in mmgen.cfg. Set them to an empty string
+	to use cookie authentication with a local server when the options are set
+	in mmgen.cfg.
+
+	For better security, rpcauth should be used in {g.proto.name}.conf instead of
+	rpcuser/rpcpassword.
+
+""",strip_char='\t')
 
 def dmsg_rpc(fs,data=None,is_json=False):
 	if g.debug_rpc:
 		msg(fs if data == None else fs.format(pp_fmt(json.loads(data) if is_json else data)))
 
-class RPCConnection(MMGenObject):
+class json_encoder(json.JSONEncoder):
+	def default(self,obj):
+		if isinstance(obj,Decimal):
+			return str(obj)
+		else:
+			return json.JSONEncoder.default(self,obj)
+
+class RPCBackends:
 
-	auth = True
-	db_fs = '    host [{h}] port [{p}] user [{u}] passwd [{pw}] auth_cookie [{c}]\n'
-	http_hdrs = { 'Content-Type': 'application/json' }
+	class aiohttp:
 
-	def __init__(self,host=None,port=None,user=None,passwd=None,auth_cookie=None,socket_timeout=1):
+		def __init__(self,caller):
+			self.caller = caller
+			self.session = g.session
+			self.url = caller.url
+			self.timeout = caller.timeout
+			if caller.auth_type == 'basic':
+				import aiohttp
+				self.auth = aiohttp.BasicAuth(*caller.auth,encoding='UTF-8')
+			else:
+				self.auth = None
+
+		async def run(self,payload,timeout=None):
+			dmsg_rpc('\n    RPC PAYLOAD data (aiohttp) ==>\n{}\n',payload)
+			async with self.session.post(
+				url     = self.url,
+				auth    = self.auth,
+				data    = json.dumps(payload,cls=json_encoder),
+				timeout = timeout or self.timeout,
+			) as res:
+				return (await res.text(),res.status)
+
+	class requests:
+
+		def __init__(self,caller):
+			self.url = caller.url
+			self.timeout = caller.timeout
+			import requests,urllib3
+			urllib3.disable_warnings()
+			self.session = requests.Session()
+			self.session.headers = caller.http_hdrs
+			if caller.auth_type:
+				auth = 'HTTP' + caller.auth_type.capitalize() + 'Auth'
+				self.session.auth = getattr(requests.auth,auth)(*caller.auth)
+
+		async def run(self,payload,timeout=None):
+			dmsg_rpc('\n    RPC PAYLOAD data (requests) ==>\n{}\n',payload)
+			res = self.session.post(
+				url = self.url,
+				data = json.dumps(payload,cls=json_encoder),
+				timeout = timeout or self.timeout,
+				verify = False )
+			return (res.content,res.status_code)
+
+	class httplib:
+
+		def __init__(self,caller):
+			import http.client
+			self.session = http.client.HTTPConnection(caller.host,caller.port,caller.timeout)
+			self.http_hdrs = caller.http_hdrs
+			self.host = caller.host
+			self.port = caller.port
+			if caller.auth_type == 'basic':
+				auth_str = f'{caller.auth.user}:{caller.auth.passwd}'
+				auth_str_b64 = 'Basic ' + base64.b64encode(auth_str.encode()).decode()
+				self.http_hdrs.update({ 'Host': self.host, 'Authorization': auth_str_b64 })
+				fs = '    RPC AUTHORIZATION data ==> raw: [{}]\n{:>31}enc: [{}]\n'
+				dmsg_rpc(fs.format(auth_str,'',auth_str_b64))
+
+		async def run(self,payload,timeout=None):
+			dmsg_rpc('\n    RPC PAYLOAD data (httplib) ==>\n{}\n',payload)
+			if timeout:
+				import http.client
+				s = http.client.HTTPConnection(self.host,self.port,timeout)
+			else:
+				s = self.session
+			try:
+				s.request(
+					method  = 'POST',
+					url     = '/',
+					body    = json.dumps(payload,cls=json_encoder),
+					headers = self.http_hdrs )
+				r = s.getresponse() # => http.client.HTTPResponse instance
+			except Exception as e:
+				raise RPCFailure(str(e))
+			return (r.read(),r.status)
+
+	class curl:
+
+		def __init__(self,caller):
+
+			def gen():
+				for k,v in caller.http_hdrs.items():
+					for s in ('--header',f'{k}: {v}'):
+						yield s
+				if caller.auth_type:
+					"""
+					Authentication with curl is insecure, as it exposes the user's credentials
+					via the command line.  Use for testing only.
+					"""
+					for s in ('--user',f'{caller.auth.user}:{caller.auth.passwd}'):
+						yield s
+				if caller.auth_type == 'digest':
+					yield '--digest'
+
+			self.url = caller.url
+			self.exec_opts = list(gen()) + ['--silent']
+			self.arg_max = 8192 # set way below system ARG_MAX, just to be safe
+			self.timeout = caller.timeout
+
+		async def run(self,payload,timeout=None):
+			data = json.dumps(payload,cls=json_encoder)
+			if len(data) > self.arg_max:
+				return self.httplib(payload,timeout=timeout)
+			dmsg_rpc('\n    RPC PAYLOAD data (curl) ==>\n{}\n',payload)
+			exec_cmd = [
+				'curl',
+				'--proxy', '',
+				'--connect-timeout', str(timeout or self.timeout),
+				'--request', 'POST',
+				'--write-out', '%{http_code}',
+				'--data-binary', data
+				] + self.exec_opts + [self.url]
+
+			dmsg_rpc('    RPC curl exec data ==>\n{}\n',exec_cmd)
+
+			from subprocess import run,PIPE
+			res = run(exec_cmd,stdout=PIPE,check=True).stdout.decode()
+			# res = run(exec_cmd,stdout=PIPE,check=True,text='UTF-8').stdout # Python 3.7+
+			return (res[:-3],int(res[-3:]))
+
+from collections import namedtuple
+auth_data = namedtuple('rpc_auth_data',['user','passwd'])
+
+class RPCClient(MMGenObject):
+
+	has_auth_cookie = False
+	url_fs = 'http://{}:{}'
+
+	def __init__(self,host,port):
 
 		dmsg_rpc('=== {}.__init__() debug ==='.format(type(self).__name__))
-		dmsg_rpc(self.db_fs.format(h=host,p=port,u=user,pw=passwd,c=auth_cookie))
+		dmsg_rpc(f'    cls [{type(self).__name__}] host [{host}] port [{port}]\n')
 
 		import socket
 		try:
-			socket.create_connection((host,port),timeout=socket_timeout).close()
+			socket.create_connection((host,port),timeout=1).close()
 		except:
 			raise SocketError('Unable to connect to {}:{}'.format(host,port))
 
-		if user and passwd: # user/pass overrides cookie
-			pass
-		elif auth_cookie:
-			user,passwd = auth_cookie.split(':')
-		elif self.auth:
-			msg('Error: no {} RPC authentication method found'.format(g.proto.name.capitalize()))
-			if passwd: die(1,"'rpcuser' entry not found in {}.conf or mmgen.cfg".format(g.proto.name))
-			elif user: die(1,"'rpcpassword' entry not found in {}.conf or mmgen.cfg".format(g.proto.name))
-			else:
-				m1 = 'Either provide rpcuser/rpcpassword in {pn}.conf or mmgen.cfg\n'
-				m2 = '(or, alternatively, copy the authentication cookie to the {pnu}\n'
-				m3 = 'data dir if {pnm} and {dn} are running as different users)'
-				die(1,(m1+m2+m3).format(
-					pn=g.proto.name,
-					pnu=g.proto.name.capitalize(),
-					dn=g.proto.daemon_name,
-					pnm=g.proj_name))
-
-		if self.auth:
-			fs = '    RPC AUTHORIZATION data ==> raw: [{}]\n{:>31}enc: [{}]\n'
-			auth_str = f'{user}:{passwd}'
-			auth_str_b64 = 'Basic ' + base64.b64encode(auth_str.encode()).decode()
-			dmsg_rpc(fs.format(auth_str,'',auth_str_b64))
-			self.http_hdrs.update({ 'Host': host, 'Authorization': auth_str_b64 })
-
+		self.http_hdrs = { 'Content-Type': 'application/json' }
+		self.url = self.url_fs.format(host,port)
 		self.host = host
 		self.port = port
-		self.user = user
-		self.passwd = passwd
+		self.timeout = g.http_timeout
+		self.auth = None
 
-		for method in self.rpcmethods:
-			exec('{c}.{m} = lambda self,*args,**kwargs: self.request("{m}",*args,**kwargs)'.format(
-						c=type(self).__name__,m=method))
+	def set_backend(self,backend=None):
+		bn = backend or opt.rpc_backend
+		if bn == 'auto':
+			self.backend = {'linux':RPCBackends.httplib,'win':RPCBackends.curl}[g.platform](self)
+		else:
+			self.backend = getattr(RPCBackends,bn)(self)
 
-	def calls(self,method,args_list):
+	def set_auth(self):
 		"""
-		Perform a list of RPC calls, returning results in a list
-
-		Can be called two ways:
-		  1) method = methodname, args_list = [args_tuple1, args_tuple2,...]
-		  2) method = None, args_list = [(methodname1,args_tuple1), (methodname2,args_tuple2), ...]
+		MMGen's credentials override coin daemon's
 		"""
-
-		cmd_list = args_list if method == None else tuple(zip([method] * len(args_list), args_list))
-
-		if True:
-			return [self.request(method,*params) for method,params in cmd_list]
-
-	# Normal mode: call with arg list unrolled, exactly as with cli
-	# Batch mode:  call with list of arg lists as first argument
-	# kwargs are for local use and are not passed to server
-
-	# By default, raises RPCFailure exception with an error msg on all errors and exceptions
-	# on_fail is one of 'raise' (default), 'return' or 'silent'
-	# With on_fail='return', returns 'rpcfail',(resp_object,(die_args))
-	def request(self,cmd,*args,**kwargs):
-
-		if g.debug:
-			print_stack_trace('RPC REQUEST {}\n  args: {!r}\n  kwargs: {!r}'.format(cmd,args,kwargs))
-
-		if g.rpc_fail_on_command == cmd:
-			cmd = 'badcommand_' + cmd
-
-		cf = { 'timeout':g.http_timeout, 'batch':False, 'on_fail':'raise' }
-
-		if cf['on_fail'] not in ('raise','return','silent'):
-			raise ValueError("request(): {}: illegal value for 'on_fail'".format(cf['on_fail']))
-
-		for k in cf:
-			if k in kwargs and kwargs[k]: cf[k] = kwargs[k]
-
-		if cf['batch']:
-			p = [{'method':cmd,'params':r,'id':n,'jsonrpc':'2.0'} for n,r in enumerate(args[0],1)]
-		else:
-			p = {'method':cmd,'params':args,'id':1,'jsonrpc':'2.0'}
-
-		dmsg_rpc('=== request() debug ===')
-		dmsg_rpc('    RPC POST data ==>\n{}\n',p)
-
-		ca_type = self.coin_amt_type if hasattr(self,'coin_amt_type') else str
-		from .obj import HexStr
-		class MyJSONEncoder(json.JSONEncoder):
-			def default(self,obj):
-				if isinstance(obj,g.proto.coin_amt):
-					return ca_type(obj)
-				elif isinstance(obj,HexStr):
-					return obj
-				else:
-					return json.JSONEncoder.default(self,obj)
-
-		data = json.dumps(p,cls=MyJSONEncoder)
-
-		if g.platform == 'win' and len(data) < 4096: # set way below ARG_MAX, just to be safe
-			return self.do_request_curl(data,cf)
+		if g.rpc_user:
+			user,passwd = (g.rpc_user,g.rpc_password)
 		else:
-			return self.do_request_httplib(data,cf)
-
-	def do_request_httplib(self,data,cf):
-
-		def do_fail(*args): # args[0] is either None or HTTPResponse object
-			if cf['on_fail'] in ('return','silent'): return 'rpcfail',args
+			user,passwd = get_coin_daemon_cfg_options(('rpcuser','rpcpassword')).values()
 
-			try:    s = '{}'.format(args[2])
-			except: s = repr(args[2])
+		if user and passwd:
+			self.auth = auth_data(user,passwd)
+			return
 
-			if s == '' and args[0] != None:
-				from http import HTTPStatus
-				hs = HTTPStatus(args[0].code)
-				s = '{} {}'.format(hs.value,hs.name)
+		if self.has_auth_cookie:
+			cookie = self.get_daemon_auth_cookie()
+			if cookie:
+				self.auth = auth_data(*cookie.split(':'))
+				return
 
-			raise RPCFailure(s)
+		die(1,rpc_credentials_msg())
 
-		hc = http.client.HTTPConnection(self.host,self.port,cf['timeout'])
-		try:
-			hc.request('POST','/',data,self.http_hdrs)
-		except Exception as e:
-			m = '{}\nUnable to connect to {} at {}:{}'
-			return do_fail(None,2,m.format(e.args[0],g.proto.daemon_name,self.host,self.port))
-
-		try:
-			r = hc.getresponse() # returns HTTPResponse instance
-		except Exception:
-			m = 'Unable to connect to {} at {}:{} (but port is bound?)'
-			return do_fail(None,2,m.format(g.proto.daemon_name,self.host,self.port))
-
-		dmsg_rpc('    RPC GETRESPONSE data ==>\n{}\n',r.__dict__)
-
-		if r.status != 200:
-			if cf['on_fail'] not in ('silent','raise'):
-				msg_r(yellow('{} RPC Error: '.format(g.proto.daemon_name.capitalize())))
-				msg(red('{} {}'.format(r.status,r.reason)))
-			e1 = r.read().decode()
-			try:
-				e3 = json.loads(e1)['error']
-				e2 = '{} (code {})'.format(e3['message'],e3['code'])
-			except:
-				e2 = str(e1)
-			return do_fail(r,1,e2)
-
-		r2 = r.read().decode()
+	# positional params are passed to the daemon, kwargs to the backend
+	# 'timeout' is currently the only supported kwarg
 
-		dmsg_rpc('    RPC REPLY data ==>\n{}\n',r2,is_json=True)
+	async def call(self,method,*params,**kwargs):
+		"""
+		default call: call with param list unrolled, exactly as with cli
+		"""
+		if method == g.rpc_fail_on_command:
+			method = 'badcommand_' + method
+		return await self.process_http_resp(self.backend.run(
+			payload = {'id': 1, 'jsonrpc': '2.0', 'method': method, 'params': params },
+			**kwargs
+		))
+
+	async def batch_call(self,method,param_list,**kwargs):
+		"""
+		Make a single call with a list of tuples as first argument
+		For RPC calls that return a list of results
+		"""
+		return await self.process_http_resp(self.backend.run(
+			payload = [{
+				'id': n,
+				'jsonrpc': '2.0',
+				'method': method,
+				'params': params } for n,params in enumerate(param_list,1) ],
+			**kwargs
+		),batch=True)
 
-		if not r2:
-			return do_fail(r,2,'Empty reply')
+	async def gathered_call(self,method,args_list,**kwargs):
+		"""
+		Perform multiple RPC calls, returning results in a list
+		Can be called two ways:
+		  1) method = methodname, args_list = [args_tuple1, args_tuple2,...]
+		  2) method = None, args_list = [(methodname1,args_tuple1), (methodname2,args_tuple2), ...]
+		"""
+		cmd_list = args_list if method == None else tuple(zip([method] * len(args_list), args_list))
 
-		r3 = json.loads(r2,parse_float=Decimal)
+		cur_pos = 0
+		chunk_size = 1024
 		ret = []
 
-		for resp in r3 if cf['batch'] else [r3]:
-			if 'error' in resp and resp['error'] != None:
-				return do_fail(r,1,'{} returned an error: {}'.format(
-					g.proto.daemon_name.capitalize(),resp['error']))
-			elif 'result' not in resp:
-				return do_fail(r,1, 'Missing JSON-RPC result\n' + repr(resps))
+		while cur_pos < len(cmd_list):
+			tasks = [self.process_http_resp(self.backend.run(
+						payload = {'id': n, 'jsonrpc': '2.0', 'method': method, 'params': params },
+						**kwargs
+					)) for n,(method,params)  in enumerate(cmd_list[cur_pos:chunk_size+cur_pos],1)]
+			ret.extend(await asyncio.gather(*tasks))
+			cur_pos += chunk_size
+
+		return ret
+
+	async def process_http_resp(self,coro,batch=False):
+		text,status = await coro
+		if status == 200:
+			dmsg_rpc('    RPC RESPONSE data ==>\n{}\n',text,is_json=True)
+			if batch:
+				return [r['result'] for r in json.loads(text,parse_float=Decimal,encoding='UTF-8')]
 			else:
-				ret.append(resp['result'])
+				try:
+					return json.loads(text,parse_float=Decimal,encoding='UTF-8')['result']
+				except:
+					raise RPCFailure(json.loads(text)['error']['message'])
+		else:
+			import http
+			s = http.HTTPStatus(status)
+			m = ''
+			if text:
+				try: m = ': ' + json.loads(text)['error']['message']
+				except:
+					try: m = f': {text.decode()}'
+					except: m = f': {text}'
+			raise RPCFailure(f'{s.value} {s.name}{m}')
 
-		return ret if cf['batch'] else ret[0]
 
-	def do_request_curl(self,data,cf):
-		from subprocess import run,PIPE
-		exec_cmd = ['curl', '--proxy', '', '--silent','--request', 'POST', '--data-binary', data]
-		for k,v in self.http_hdrs.items():
-			exec_cmd += ['--header', '{}: {}'.format(k,v)]
-		if self.auth:
-			exec_cmd += ['--user', self.user + ':' + self.passwd]
-		exec_cmd += ['http://{}:{}/'.format(self.host,self.port)]
+class BitcoinRPCClient(RPCClient,metaclass=aInitMeta):
 
-		cp = run(exec_cmd,stdout=PIPE,check=True)
-		res = json.loads(cp.stdout,parse_float=Decimal)
-		dmsg_rpc('    RPC RESULT data ==>\n{}\n',res)
+	auth_type = 'basic'
+	has_auth_cookie = True
 
-		def do_fail(s):
-			if cf['on_fail'] in ('return','silent'):
-				return ('rpcfail',s)
-			raise RPCFailure(s)
+	def __init__(self,*args,**kwargs): pass
 
-		for resp in ([res],res)[cf['batch']]:
-			if 'error' in resp and resp['error'] != None:
-				return do_fail('{} returned an error: {}'.format(g.proto.daemon_name,resp['error']))
-			elif 'result' not in resp:
-				return do_fail('Missing JSON-RPC result\n{!r}'.format(resp))
+	async def __ainit__(self,backend=None):
 
-		return [r['result'] for r in res] if cf['batch'] else res['result']
+		async def check_chainfork_mismatch(block0):
+			try:
+				assert block0 == g.proto.block0,'Incorrect Genesis block for {}'.format(g.proto.__name__)
+				for fork in g.proto.forks:
+					if fork.height == None or self.blockcount < fork.height:
+						break
+					if fork.hash != await self.call('getblockhash',fork.height):
+						die(3,f'Bad block hash at fork block {fork.height}. Is this the {fork.name} chain?')
+			except Exception as e:
+				die(2,"{}\n'{c}' requested, but this is not the {c} chain!".format(e.args[0],c=g.coin))
+
+		def check_chaintype_mismatch():
+			try:
+				if g.regtest: assert g.chain == 'regtest','--regtest option selected, but chain is not regtest'
+				if g.testnet: assert g.chain != 'mainnet','--testnet option selected, but chain is mainnet'
+				if not g.testnet: assert g.chain == 'mainnet','mainnet selected, but chain is not mainnet'
+			except Exception as e:
+				die(1,'{}\nChain is {}!'.format(e.args[0],g.chain))
+
+		user,passwd = get_coin_daemon_cfg_options(('rpcuser','rpcpassword')).values()
+
+		super().__init__(
+			host = g.rpc_host or 'localhost',
+			port = g.rpc_port or g.proto.rpc_port)
+
+		self.set_auth() # set_auth() requires cookie, so must be called after __init__() tests socket
+		self.set_backend(backend) # backend requires self.auth
+
+		if g.bob or g.alice:
+			from .regtest import MMGenRegtest
+			MMGenRegtest(g.coin).switch_user(('alice','bob')[g.bob],quiet=True)
+
+		self.cached = {}
+		(
+			self.cached['networkinfo'],
+			self.blockcount,
+			self.cached['blockchaininfo'],
+			block0
+		) = await self.gathered_call(None, (
+				('getnetworkinfo',()),
+				('getblockcount',()),
+				('getblockchaininfo',()),
+				('getblockhash',(0,)),
+			))
+		self.daemon_version = self.cached['networkinfo']['version']
+		g.chain = self.cached['blockchaininfo']['chain']
+
+		tip = await self.call('getblockhash',self.blockcount)
+		self.cur_date = (await self.call('getblockheader',tip))['time']
+		if g.chain != 'regtest':
+			g.chain += 'net'
+		assert g.chain in g.chains
+		check_chaintype_mismatch()
+
+		if g.chain == 'mainnet': # skip this for testnet, as Genesis block may change
+			await check_chainfork_mismatch(block0)
+
+		self.caps = ('full_node',)
+		for func,cap in (
+			('setlabel','label_api'),
+			('signrawtransactionwithkey','sign_with_key') ):
+			if len((await self.call('help',func)).split('\n')) > 3:
+				self.caps += (cap,)
+
+	# TODO: these belong in protocol.py
+	@classmethod
+	def get_daemon_auth_cookie_fn(cls):
+		cdir = os.path.join(
+			g.proto.daemon_data_dir,
+			g.proto.daemon_data_subdir )
+		return os.path.join(cdir,'.cookie')
+
+	@classmethod
+	def get_daemon_auth_cookie(cls):
+		fn = cls.get_daemon_auth_cookie_fn()
+		return get_lines_from_file(fn,'')[0] if file_is_readable(fn) else ''
 
 	rpcmethods = (
 		'backupwallet',
@@ -268,12 +434,39 @@ class RPCConnection(MMGenObject):
 		'walletpassphrase',
 	)
 
-class EthereumRPCConnection(RPCConnection):
+class EthereumRPCClient(RPCClient,metaclass=aInitMeta):
 
-	auth = False
-	db_fs = '    host [{h}] port [{p}]\n'
-	_blockcount = None
-	_cur_date = None
+	auth_type = None
+
+	def __init__(self,*args,**kwargs): pass
+
+	async def __ainit__(self,backend=None):
+
+		super().__init__(
+			host = g.rpc_host or 'localhost',
+			port = g.rpc_port or g.proto.rpc_port )
+
+		self.set_backend(backend)
+
+		self.blockcount = int(await self.call('eth_blockNumber'),16)
+
+		vi,bh,ch,nk = await self.gathered_call(None, (
+				('parity_versionInfo',()),
+				('parity_getBlockHeaderByNumber',()),
+				('parity_chain',()),
+				('parity_nodeKind',()),
+			))
+
+		self.daemon_version = vi['version']
+		self.cur_date = int(bh['timestamp'],16)
+		g.chain = ch.replace(' ','_')
+		self.caps = ('full_node',) if nk['capability'] == 'full' else ()
+
+		try:
+			await self.call('eth_chainId')
+			self.caps += ('eth_chainId',)
+		except RPCFailure:
+			pass
 
 	rpcmethods = (
 		'eth_accounts',
@@ -314,20 +507,25 @@ class EthereumRPCConnection(RPCConnection):
 		'parity_versionInfo',
 	)
 
-	# blockcount and cur_date require network RPC calls, so evaluate lazily
-	@property
-	def blockcount(self):
-		if self._blockcount == None:
-			self._blockcount = int(self.eth_blockNumber(),16)
-		return self._blockcount
+class MoneroWalletRPCClient(RPCClient):
+
+	auth_type = 'digest'
+	url_fs = 'http://{}:{}/json_rpc'
 
-	@property
-	def cur_date(self):
-		if self._cur_date == None:
-			self._cur_date = int(self.parity_getBlockHeaderByNumber(hex(self.blockcount))['timestamp'],16)
-		return self._cur_date
+	def __init__(self,host,port,user,passwd):
+		super().__init__(host,port)
+		self.auth = auth_data(user,passwd)
+		self.set_backend('requests')
+		if False: # insecure, for debugging only
+			self.backend = RPCBackends.curl(self)
+			self.backend.exec_opts.remove('--silent')
+			self.backend.exec_opts.extend(['--insecure','--verbose'])
 
-class MoneroWalletRPCConnection(RPCConnection):
+	async def call(self,method,*params,**kwargs):
+		assert params == (), f'{type(self).__name__}.call() accepts keyword arguments only'
+		return await self.process_http_resp(self.backend.run(
+			payload = {'id': 0, 'jsonrpc': '2.0', 'method': method, 'params': kwargs },
+		))
 
 	rpcmethods = (
 		'get_version',
@@ -340,159 +538,16 @@ class MoneroWalletRPCConnection(RPCConnection):
 		'refresh',       # start_height
 	)
 
-	def request(self,cmd,*args,**kwargs):
-		if args != ():
-			m = '{}.request() accepts only keyword args\nCmd: {!r}'
-			raise ValueError(m.format(type(self).__name__,cmd))
-		import requests
-		import urllib3
-		urllib3.disable_warnings()
-		ret = requests.post(
-			url = 'https://{}:{}/json_rpc'.format(self.host,self.port),
-			json = {
-				'jsonrpc': '2.0',
-				'id': '0',
-				'method': cmd,
-				'params': kwargs,
-			},
-			auth = requests.auth.HTTPDigestAuth(self.user,self.passwd),
-			headers = self.http_hdrs,
-			verify = False )
-
-		res = json.loads(ret._content)
-		if 'error' in res:
-			raise RPCFailure(repr(res['error']))
-		return(res['result'])
-
-	def request_curltest(self,cmd,*args,**kwargs):
-		"insecure, for testing only"
-		from subprocess import run,PIPE
-		data = {
-			'jsonrpc': '2.0',
-			'id': '0',
-			'method': cmd,
-			'params': kwargs,
-		}
-		exec_cmd = [
-			'curl', '--proxy', '', '--verbose','--insecure', '--request', 'POST',
-			'--digest', '--user', '{}:{}'.format(g.monero_wallet_rpc_user,g.monero_wallet_rpc_password),
-			'--header', 'Content-Type: application/json',
-			'--data', json.dumps(data),
-			'https://{}:{}/json_rpc'.format(self.host,self.port) ]
-
-		cp = run(exec_cmd,stdout=PIPE,check=True)
-
-		res = json.loads(cp.stdout)
-		if 'error' in res:
-			raise RPCFailure(repr(res['error']))
-		return(res['result'])
-
-def rpc_error(ret):
-	return type(ret) is tuple and ret and ret[0] == 'rpcfail'
-
-def rpc_errmsg(ret):
-	try:
-		return ret[1][2]
-	except:
-		return repr(ret)
-
-def init_daemon_parity():
-
-	def resolve_token_arg(token_arg):
-		from .obj import CoinAddr
-		from .altcoins.eth.tw import EthereumTrackingWallet
-		from .altcoins.eth.contract import Token
-
-		tw = EthereumTrackingWallet(no_rpc=True)
-
-		try:    addr = CoinAddr(token_arg,on_fail='raise')
-		except: addr = tw.sym2addr(token_arg)
-
-		if not addr:
-			m = "'{}': unrecognized token symbol"
-			raise UnrecognizedTokenSymbol(m.format(token_arg))
-
-		sym = tw.addr2sym(addr) # throws exception on failure
-		vmsg('ERC20 token resolved: {} ({})'.format(addr,sym))
-
-		return addr,sym
-
-	conn = EthereumRPCConnection(
-				g.rpc_host or 'localhost',
-				g.rpc_port or g.proto.rpc_port)
-	conn.daemon_version = conn.parity_versionInfo()['version'] # fail immediately if daemon is geth
-	conn.coin_amt_type = str
-	g.chain = conn.parity_chain().replace(' ','_')
-
-	conn.caps = ()
-	try:
-		conn.request('eth_chainId')
-		conn.caps += ('eth_chainId',)
-	except RPCFailure:
-		pass
-
-	if conn.request('parity_nodeKind')['capability'] == 'full':
-		conn.caps += ('full_node',)
-
-	if g.token:
-		g.rpc = conn # set g.rpc so rpc_init() will return immediately
-		(g.token,g.dcoin) = resolve_token_arg(g.token)
-
-	return conn
-
-def init_daemon_bitcoind():
-
-	def check_chainfork_mismatch(conn):
-		block0 = conn.getblockhash(0)
-		latest = conn.blockcount
-		try:
-			assert block0 == g.proto.block0,'Incorrect Genesis block for {}'.format(g.proto.__name__)
-			for fork in g.proto.forks:
-				if fork[0] == None or latest < fork[0]: break
-				assert conn.getblockhash(fork[0]) == fork[1], (
-					'Bad block hash at fork block {}. Is this the {} chain?'.format(fork[0],fork[2].upper()))
-		except Exception as e:
-			die(2,"{}\n'{c}' requested, but this is not the {c} chain!".format(e.args[0],c=g.coin))
-
-	def check_chaintype_mismatch():
-		try:
-			if g.regtest: assert g.chain == 'regtest','--regtest option selected, but chain is not regtest'
-			if g.testnet: assert g.chain != 'mainnet','--testnet option selected, but chain is mainnet'
-			if not g.testnet: assert g.chain == 'mainnet','mainnet selected, but chain is not mainnet'
-		except Exception as e:
-			die(1,'{}\nChain is {}!'.format(e.args[0],g.chain))
-
-	cfg = get_daemon_cfg_options(('rpcuser','rpcpassword'))
-
-	conn = RPCConnection(
-				g.rpc_host or 'localhost',
-				g.rpc_port or g.proto.rpc_port,
-				g.rpc_user or cfg['rpcuser'], # MMGen's rpcuser,rpcpassword override coin daemon's
-				g.rpc_password or cfg['rpcpassword'],
-				auth_cookie=get_coin_daemon_auth_cookie())
-
-	if g.bob or g.alice:
-		from .regtest import MMGenRegtest
-		MMGenRegtest(g.coin).switch_user(('alice','bob')[g.bob],quiet=True)
-	conn.daemon_version = int(conn.getnetworkinfo()['version'])
-	conn.blockcount = conn.getblockcount()
-	conn.cur_date = conn.getblockheader(conn.getblockhash(conn.blockcount))['time']
-	conn.coin_amt_type = (float,str)[conn.daemon_version>=120000]
-	g.chain = conn.getblockchaininfo()['chain']
-	if g.chain != 'regtest': g.chain += 'net'
-	assert g.chain in g.chains
-	check_chaintype_mismatch()
-
-	if g.chain == 'mainnet': # skip this for testnet, as Genesis block may change
-		check_chainfork_mismatch(conn)
-
-	conn.caps = ('full_node',)
-	for func,cap in (
-		('setlabel','label_api'),
-		('signrawtransactionwithkey','sign_with_key') ):
-		if len(conn.request('help',func).split('\n')) > 3:
-			conn.caps += (cap,)
-	return conn
-
-def init_daemon(name):
-	return globals()['init_daemon_'+name]()
+async def rpc_init(proto=None,backend=None):
+
+	proto = proto or g.proto
+
+	if not 'rpc' in proto.mmcaps:
+		die(1,'Coin daemon operations not supported for {}!'.format(proto.__name__))
+
+	g.rpc = await {
+		'bitcoind': BitcoinRPCClient,
+		'parity':   EthereumRPCClient,
+	}[proto.daemon_family](backend=backend)
+
+	return g.rpc

mmgen/tool.py

@@ -650,14 +650,15 @@ class MMGenToolCmdFile(MMGenToolCmds):
 		file_sort = kwargs.get('filesort') or 'mtime'
 
 		from .filename import MMGenFileList
-		from .tx import MMGenTX
+		from .tx import MMGenTX,MMGenTxForSigning
 		flist = MMGenFileList(infiles,ftype=MMGenTX)
 		flist.sort_by_age(key=file_sort) # in-place sort
 
-		sep = '—'*77+'\n'
-		return sep.join(
-			[MMGenTX(fn,offline=True).format_view(terse=terse,sort=tx_sort) for fn in flist.names()]
-		).rstrip()
+		def gen():
+			for fn in flist.names():
+				yield (MMGenTxForSigning,MMGenTX)[fn.endswith('.sigtx')](fn).format_view(terse=terse,sort=tx_sort)
+
+		return ('—'*77+'\n').join(gen()).rstrip()
 
 class MMGenToolCmdFileCrypt(MMGenToolCmds):
 	"""
@@ -841,12 +842,12 @@ from .tw import TwAddrList,TwUnspentOutputs
 class MMGenToolCmdRPC(MMGenToolCmds):
 	"tracking wallet commands using the JSON-RPC interface"
 
-	def getbalance(self,minconf=1,quiet=False,pager=False):
+	async def getbalance(self,minconf=1,quiet=False,pager=False):
 		"list confirmed/unconfirmed, spendable/unspendable balances in tracking wallet"
 		from .tw import TwGetBalance
-		return TwGetBalance(minconf,quiet).format()
+		return (await TwGetBalance(minconf,quiet)).format()
 
-	def listaddress(self,
+	async def listaddress(self,
 					mmgen_addr:str,
 					minconf = 1,
 					pager = False,
@@ -855,7 +856,7 @@ class MMGenToolCmdRPC(MMGenToolCmds):
 					age_fmt: _options_annot_str(TwAddrList.age_fmts) = 'confs',
 					):
 		"list the specified MMGen address and its balance"
-		return self.listaddresses(  mmgen_addrs = mmgen_addr,
+		return await self.listaddresses(  mmgen_addrs = mmgen_addr,
 									minconf = minconf,
 									pager = pager,
 									showempty = showempty,
@@ -863,7 +864,7 @@ class MMGenToolCmdRPC(MMGenToolCmds):
 									age_fmt = age_fmt,
 								)
 
-	def listaddresses(  self,
+	async def listaddresses(  self,
 						mmgen_addrs:'(range or list)' = '',
 						minconf = 1,
 						showempty = False,
@@ -890,13 +891,12 @@ class MMGenToolCmdRPC(MMGenToolCmds):
 				die(1,m.format(mmgen_addrs))
 			usr_addr_list = [MMGenID('{}:{}'.format(a[0],i)) for i in AddrIdxList(a[1])]
 
-		rpc_init()
-		al = TwAddrList(usr_addr_list,minconf,showempty,showbtcaddrs,all_labels)
+		al = await TwAddrList(usr_addr_list,minconf,showempty,showbtcaddrs,all_labels)
 		if not al:
 			die(0,('No tracked addresses with balances!','No tracked addresses!')[showempty])
-		return al.format(showbtcaddrs,sort,show_age,age_fmt or 'confs')
+		return await al.format(showbtcaddrs,sort,show_age,age_fmt or 'confs')
 
-	def twview( self,
+	async def twview( self,
 				pager = False,
 				reverse = False,
 				wide = False,
@@ -906,9 +906,8 @@ class MMGenToolCmdRPC(MMGenToolCmds):
 				show_mmid = True,
 				wide_show_confs = True):
 		"view tracking wallet"
-		rpc_init()
-		twuo = TwUnspentOutputs(minconf=minconf)
-		twuo.do_sort(sort,reverse=reverse)
+		twuo = await TwUnspentOutputs(minconf=minconf)
+		await twuo.get_unspent_data(reverse_sort=reverse)
 		twuo.age_fmt = age_fmt
 		twuo.show_mmid = show_mmid
 		if wide:
@@ -916,25 +915,23 @@ class MMGenToolCmdRPC(MMGenToolCmds):
 		else:
 			ret = twuo.format_for_display()
 		del twuo.wallet
-		return ret
+		return await ret
 
-	def add_label(self,mmgen_or_coin_addr:str,label:str):
+	async def add_label(self,mmgen_or_coin_addr:str,label:str):
 		"add descriptive label for address in tracking wallet"
-		rpc_init()
 		from .tw import TrackingWallet
-		TrackingWallet(mode='w').add_label(mmgen_or_coin_addr,label,on_fail='raise')
+		await (await TrackingWallet(mode='w')).add_label(mmgen_or_coin_addr,label,on_fail='raise')
 		return True
 
-	def remove_label(self,mmgen_or_coin_addr:str):
+	async def remove_label(self,mmgen_or_coin_addr:str):
 		"remove descriptive label for address in tracking wallet"
-		self.add_label(mmgen_or_coin_addr,'')
+		await self.add_label(mmgen_or_coin_addr,'')
 		return True
 
-	def remove_address(self,mmgen_or_coin_addr:str):
+	async def remove_address(self,mmgen_or_coin_addr:str):
 		"remove an address from tracking wallet"
 		from .tw import TrackingWallet
-		tw = TrackingWallet(mode='w')
-		ret = tw.remove_address(mmgen_or_coin_addr) # returns None on failure
+		ret = await (await TrackingWallet(mode='w')).remove_address(mmgen_or_coin_addr) # returns None on failure
 		if ret:
 			msg("Address '{}' deleted from tracking wallet".format(ret))
 		return ret
@@ -988,7 +985,7 @@ class MMGenToolCmdMonero(MMGenToolCmds):
 		if monerod_args:
 			self.monerod_args = monerod_args
 
-		def create(n,d,fn,c,m):
+		async def create(n,d,fn,c,m):
 			try: os.stat(fn)
 			except: pass
 			else:
@@ -997,7 +994,8 @@ class MMGenToolCmdMonero(MMGenToolCmds):
 			gmsg(m)
 
 			from .baseconv import baseconv
-			ret = c.restore_deterministic_wallet(
+			ret = await c.call(
+				'restore_deterministic_wallet',
 				filename  = os.path.basename(fn),
 				password  = d.wallet_passwd,
 				seed      = baseconv.fromhex(d.sec,'xmrseed',tostr=True),
@@ -1007,7 +1005,7 @@ class MMGenToolCmdMonero(MMGenToolCmds):
 			pp_msg(ret) if opt.debug else msg('  Address: {}'.format(ret['address']))
 			return True
 
-		def sync(n,d,fn,c,m):
+		async def sync(n,d,fn,c,m):
 			try:
 				os.stat(fn)
 			except:
@@ -1021,11 +1019,14 @@ class MMGenToolCmdMonero(MMGenToolCmds):
 			t_start = time.time()
 
 			msg_r('  Opening wallet...')
-			c.open_wallet(filename=os.path.basename(fn),password=d.wallet_passwd)
+			await c.call(
+				'open_wallet',
+				filename=os.path.basename(fn),
+				password=d.wallet_passwd )
 			msg('done')
 
 			msg_r('  Getting wallet height...')
-			wallet_height = c.get_height()['height']
+			wallet_height = (await c.call('get_height'))['height']
 			msg('\r  Wallet height: {}        '.format(wallet_height))
 
 			behind = chain_height - wallet_height
@@ -1033,7 +1034,7 @@ class MMGenToolCmdMonero(MMGenToolCmds):
 				m = '  Wallet is {} blocks behind chain tip.  Please be patient.  Syncing...'
 				msg_r(m.format(behind))
 
-			ret = c.refresh()
+			ret = await c.call('refresh')
 
 			if behind > 1000:
 				msg('done')
@@ -1043,7 +1044,7 @@ class MMGenToolCmdMonero(MMGenToolCmds):
 
 			t_elapsed = int(time.time() - t_start)
 
-			ret = c.get_balance() # account_index=0, address_indices=[0,1]
+			ret = await c.call('get_balance') # account_index=0, address_indices=[0,1]
 
 			from .obj import XMRAmt
 			bals[fn] = tuple([XMRAmt(ret[k],from_unit='min_coin_unit') for k in ('balance','unlocked_balance')])
@@ -1053,16 +1054,14 @@ class MMGenToolCmdMonero(MMGenToolCmds):
 			else:
 				msg('  Balance: {} Unlocked balance: {}'.format(*[b.hl() for b in bals[fn]]))
 
-			msg('  Wallet height: {}'.format(c.get_height()['height']))
+			msg('  Wallet height: {}'.format((await c.call('get_height'))['height']))
 			msg('  Sync time: {:02}:{:02}'.format(t_elapsed//60,t_elapsed%60))
 
-			c.close_wallet()
+			await c.call('close_wallet')
 			return True
 
-		def process_wallets():
-			m =   { 'create': ('Creat','Generat',create,False),
-					'sync':   ('Sync', 'Sync',   sync,  True) }
-			opt.accept_defaults = opt.accept_defaults or m[op][3]
+		async def process_wallets(op):
+			opt.accept_defaults = opt.accept_defaults or op.accept_defaults
 			from .protocol import init_coin
 			init_coin('xmr')
 			from .addr import AddrList
@@ -1070,18 +1069,18 @@ class MMGenToolCmdMonero(MMGenToolCmds):
 			data = [d for d in al.data if addrs == '' or d.idx in AddrIdxList(addrs)]
 			dl = len(data)
 			assert dl,"No addresses in addrfile within range '{}'".format(addrs)
-			gmsg('\n{}ing {} wallet{}'.format(m[op][0],dl,suf(dl)))
+			gmsg('\n{}ing {} wallet{}'.format(op.desc,dl,suf(dl)))
 
 			from .daemon import MoneroWalletDaemon
 			wd = MoneroWalletDaemon(opt.outdir or '.',test_suite=g.test_suite)
 			wd.restart()
 
-			from .rpc import MoneroWalletRPCConnection
-			c = MoneroWalletRPCConnection(
-				g.monero_wallet_rpc_host,
-				wd.rpc_port,
-				g.monero_wallet_rpc_user,
-				g.monero_wallet_rpc_password)
+			from .rpc import MoneroWalletRPCClient
+			c = MoneroWalletRPCClient(
+				host = g.monero_wallet_rpc_host,
+				port = wd.rpc_port,
+				user = g.monero_wallet_rpc_user,
+				passwd = g.monero_wallet_rpc_password)
 
 			wallets_processed = 0
 			for n,d in enumerate(data): # [d.sec,d.wallet_passwd,d.viewkey,d.addr]
@@ -1091,13 +1090,13 @@ class MMGenToolCmdMonero(MMGenToolCmds):
 						d.idx,
 						'-α' if g.debug_utf8 else ''))
 
-				info = '\n{}ing wallet {}/{} ({})'.format(m[op][1],n+1,dl,fn)
-				wallets_processed += m[op][2](n,d,fn,c,info)
+				info = '\n{}ing wallet {}/{} ({})'.format(op.action,n+1,dl,fn)
+				wallets_processed += await op.func(n,d,fn,c,info)
 
 			wd.stop()
-			gmsg('\n{} wallet{} {}ed'.format(wallets_processed,suf(wallets_processed),m[op][0].lower()))
+			gmsg('\n{} wallet{} {}ed'.format(wallets_processed,suf(wallets_processed),op.desc.lower()))
 
-			if wallets_processed and op == 'sync':
+			if wallets_processed and op.name == 'sync':
 				col1_w = max(map(len,bals)) + 1
 				fs = '{:%s} {} {}' % col1_w
 				msg('\n'+fs.format('Wallet','Balance           ','Unlocked Balance  '))
@@ -1114,8 +1113,13 @@ class MMGenToolCmdMonero(MMGenToolCmds):
 
 		bals = {} # locked,unlocked
 
+		from collections import namedtuple
+		wo = namedtuple('mwo',['name','desc','action','func','accept_defaults'])
+		op = { # reusing name!
+			'create': wo('create', 'Creat', 'Generat', create, False),
+			'sync':   wo('sync',   'Sync',  'Sync',    sync,   True) }[op]
 		try:
-			process_wallets()
+			run_session(process_wallets(op),do_rpc_init=False)
 		except KeyboardInterrupt:
 			rdie(1,'\nUser interrupt\n')
 		except EOFError:

mmgen/tw.py

@@ -40,21 +40,21 @@ _date_formatter = {
 	'date_time': lambda secs: '{}-{:02}-{:02} {:02}:{:02}'.format(*time.gmtime(secs)[:5]),
 }
 
-def _set_dates(us):
+async def _set_dates(us):
 	if us and us[0].date is None:
 		# 'blocktime' differs from 'time', is same as getblockheader['time']
-		dates = [o['blocktime'] for o in g.rpc.calls('gettransaction',[(o.txid,) for o in us])]
+		dates = [o['blocktime'] for o in await g.rpc.gathered_call('gettransaction',[(o.txid,) for o in us])]
 		for o,date in zip(us,dates):
 			o.date = date
 
 if os.getenv('MMGEN_BOGUS_WALLET_DATA'):
 	# 1831006505 (09 Jan 2028) = projected time of block 1000000
 	_date_formatter['days'] = lambda date: (1831006505 - date) // 86400
-	def _set_dates(us):
+	async def _set_dates(us):
 		for o in us:
 			o.date = 1831006505 - int(9.7 * 60 * (o.confs - 1))
 
-class TwUnspentOutputs(MMGenObject):
+class TwUnspentOutputs(MMGenObject,metaclass=aInitMeta):
 
 	def __new__(cls,*args,**kwargs):
 		return MMGenObject.__new__(altcoin_subclass(cls,'tw','TwUnspentOutputs'))
@@ -104,7 +104,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 """.strip().format(g.proj_name.lower())
 	}
 
-	def __init__(self,minconf=1,addrs=[]):
+	async def __ainit__(self,minconf=1,addrs=[]):
 		self.unspent      = self.MMGenTwOutputList()
 		self.fmt_display  = ''
 		self.fmt_print    = ''
@@ -117,9 +117,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 		self.sort_key     = 'age'
 		self.disp_prec    = self.get_display_precision()
 
-		self.wallet = TrackingWallet('w')
-		self.get_unspent_data()
-		self.do_sort()
+		self.wallet = await TrackingWallet(mode='w')
 
 	@property
 	def age_fmt(self):
@@ -138,7 +136,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 	def total(self):
 		return sum(i.amt for i in self.unspent)
 
-	def get_unspent_rpc(self):
+	async def get_unspent_rpc(self):
 		# bitcoin-cli help listunspent:
 		# Arguments:
 		# 1. minconf        (numeric, optional, default=1) The minimum confirmations to filter
@@ -149,17 +147,20 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 
 		# for now, self.addrs is just an empty list for Bitcoin and friends
 		add_args = (9999999,self.addrs) if self.addrs else ()
-		return g.rpc.listunspent(self.minconf,*add_args)
+		return await g.rpc.call('listunspent',self.minconf,*add_args)
 
-	def get_unspent_data(self):
+	async def get_unspent_data(self,sort_key=None,reverse_sort=False):
 		if g.bogus_wallet_data: # for debugging purposes only
 			us_rpc = eval(get_data_from_file(g.bogus_wallet_data)) # testing, so ok
 		else:
-			us_rpc = self.get_unspent_rpc()
+			us_rpc = await self.get_unspent_rpc()
+
+		if not us_rpc:
+			die(0,self.wmsg['no_spendable_outputs'])
 
-		if not us_rpc: die(0,self.wmsg['no_spendable_outputs'])
 		tr_rpc = []
 		lbl_id = ('account','label')['label_api' in g.rpc.caps]
+
 		for o in us_rpc:
 			if not lbl_id in o:
 				continue # coinbase outputs have no account field
@@ -183,6 +184,8 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 		if not self.unspent:
 			die(1,'No tracked {}s in tracking wallet!'.format(self.item_desc))
 
+		self.do_sort(key=sort_key,reverse=reverse_sort)
+
 	def do_sort(self,key=None,reverse=False):
 		sort_funcs = {
 			'addr':  lambda i: i.addr,
@@ -214,10 +217,10 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 			m2 = 'Please resize your screen to at least {} characters and hit ENTER '
 			my_raw_input((m1+m2).format(g.min_screen_width))
 
-	def format_for_display(self):
+	async def format_for_display(self):
 		unsp = self.unspent
 		if self.age_fmt in self.age_fmts_date_dependent:
-			_set_dates(unsp)
+			await _set_dates(unsp)
 		self.set_term_columns()
 
 		# allow for 7-digit confirmation nums
@@ -291,9 +294,9 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 		self.fmt_display = '\n'.join(out) + '\n'
 		return self.fmt_display
 
-	def format_for_printing(self,color=False,show_confs=True):
+	async def format_for_printing(self,color=False,show_confs=True):
 		if self.age_fmt in self.age_fmts_date_dependent:
-			_set_dates(self.unspent)
+			await _set_dates(self.unspent)
 		addr_w = max(len(i.addr) for i in self.unspent)
 		mmid_w = max(len(('',i.twmmid)[i.twmmid.type=='mmgen']) for i in self.unspent) or 12 # DEADBEEF:S:1
 		amt_w = g.proto.coin_amt.max_prec + 5
@@ -374,14 +377,14 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 					if keypress_confirm(fs.format(self.item_desc,n)):
 						return n
 
-	def view_and_sort(self,tx):
+	async def view_and_sort(self,tx):
 		from .term import get_char
 		prompt = self.prompt.strip() + '\b'
 		no_output,oneshot_msg = False,None
 		while True:
 			msg_r('' if no_output else '\n\n' if opt.no_blank else CUR_HOME+ERASE_ALL)
 			reply = get_char(
-				'' if no_output else self.format_for_display()+'\n'+(oneshot_msg or '')+prompt,
+				'' if no_output else await self.format_for_display()+'\n'+(oneshot_msg or '')+prompt,
 				immed_chars=''.join(self.key_mappings.keys())
 			)
 			no_output = False
@@ -409,17 +412,15 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 				idx = self.get_idx_from_user(action)
 				if idx:
 					e = self.unspent[idx-1]
-					bal = self.wallet.get_balance(e.addr,force_rpc=True)
-					self.get_unspent_data()
-					self.do_sort()
+					bal = await self.wallet.get_balance(e.addr,force_rpc=True)
+					await self.get_unspent_data()
 					oneshot_msg = yellow('{} balance for account #{} refreshed\n\n'.format(g.dcoin,idx))
 			elif action == 'a_lbl_add':
 				idx,lbl = self.get_idx_from_user(action)
 				if idx:
 					e = self.unspent[idx-1]
-					if self.wallet.add_label(e.twmmid,lbl,addr=e.addr):
-						self.get_unspent_data()
-						self.do_sort()
+					if await self.wallet.add_label(e.twmmid,lbl,addr=e.addr):
+						await self.get_unspent_data()
 						a = 'added to' if lbl else 'removed from'
 						oneshot_msg = yellow("Label {} {} #{}\n\n".format(a,self.item_desc,idx))
 					else:
@@ -428,9 +429,8 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 				idx = self.get_idx_from_user(action)
 				if idx:
 					e = self.unspent[idx-1]
-					if self.wallet.remove_address(e.addr):
-						self.get_unspent_data()
-						self.do_sort()
+					if await self.wallet.remove_address(e.addr):
+						await self.get_unspent_data()
 						oneshot_msg = yellow("{} #{} removed\n\n".format(capfirst(self.item_desc),idx))
 					else:
 						oneshot_msg = red('Address could not be removed\n\n')
@@ -439,13 +439,13 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 										','.join(self.sort_info(include_group=False)).lower())
 				msg('')
 				try:
-					write_data_to_file(of,self.format_for_printing(),desc='{} listing'.format(self.desc))
+					write_data_to_file(of,await self.format_for_printing(),desc='{} listing'.format(self.desc))
 				except UserNonConfirmation as e:
 					oneshot_msg = red("File '{}' not overwritten by user request\n\n".format(of))
 				else:
 					oneshot_msg = yellow("Data written to '{}'\n\n".format(of))
 			elif action in ('a_view','a_view_wide'):
-				do_pager(self.fmt_display if action == 'a_view' else self.format_for_printing(color=True))
+				do_pager(self.fmt_display if action == 'a_view' else await self.format_for_printing(color=True))
 				if g.platform == 'linux' and oneshot_msg == None:
 					msg_r(CUR_RIGHT(len(prompt.split('\n')[-1])-2))
 					no_output = True
@@ -458,7 +458,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 		else:
 			return _date_formatter[age_fmt](o.date)
 
-class TwAddrList(MMGenDict):
+class TwAddrList(MMGenDict,metaclass=aInitMeta):
 	has_age = True
 	age_fmts = TwUnspentOutputs.age_fmts
 	age_disp = TwUnspentOutputs.age_disp
@@ -466,7 +466,10 @@ class TwAddrList(MMGenDict):
 	def __new__(cls,*args,**kwargs):
 		return MMGenDict.__new__(altcoin_subclass(cls,'tw','TwAddrList'),*args,**kwargs)
 
-	def __init__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None):
+	def __init__(self,*args,**kwargs):
+		pass
+
+	async def __ainit__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None):
 
 		def check_dup_mmid(acct_labels):
 			mmid_prev,err = None,False
@@ -490,10 +493,9 @@ class TwAddrList(MMGenDict):
 			if err: rdie(3,'Tracking wallet is corrupted!')
 
 		self.total = g.proto.coin_amt('0')
-		rpc_init()
 
 		lbl_id = ('account','label')['label_api' in g.rpc.caps]
-		for d in g.rpc.listunspent(0):
+		for d in await g.rpc.call('listunspent',0):
 			if not lbl_id in d: continue  # skip coinbase outputs with missing account
 			if d['confirmations'] < minconf: continue
 			label = get_tw_label(d[lbl_id])
@@ -520,11 +522,12 @@ class TwAddrList(MMGenDict):
 			# for compatibility with old mmids, must use raw RPC rather than native data for matching
 			# args: minconf,watchonly, MUST use keys() so we get list, not dict
 			if 'label_api' in g.rpc.caps:
-				acct_list = g.rpc.listlabels()
-				acct_addrs = [list(a.keys()) for a in g.rpc.getaddressesbylabel([[k] for k in acct_list],batch=True)]
+				acct_list = await g.rpc.call('listlabels')
+				aa = await g.rpc.batch_call('getaddressesbylabel',[(k,) for k in acct_list])
+				acct_addrs = [list(a.keys()) for a in aa]
 			else:
-				acct_list = list(g.rpc.listaccounts(0,True).keys()) # raw list, no 'L'
-				acct_addrs = g.rpc.getaddressesbyaccount([[a] for a in acct_list],batch=True) # use raw list here
+				acct_list = list((await g.rpc.call('listaccounts',0,True)).keys()) # raw list, no 'L'
+				acct_addrs = await g.rpc.batch_call('getaddressesbyaccount',[(a,) for a in acct_list]) # use raw list here
 			acct_labels = MMGenList([get_tw_label(a) for a in acct_list])
 			check_dup_mmid(acct_labels)
 			assert len(acct_list) == len(acct_addrs),(
@@ -545,7 +548,7 @@ class TwAddrList(MMGenDict):
 
 	def coinaddr_list(self): return [self[k]['addr'] for k in self]
 
-	def format(self,showbtcaddrs,sort,show_age,age_fmt):
+	async def format(self,showbtcaddrs,sort,show_age,age_fmt):
 		if not self.has_age:
 			show_age = False
 		if age_fmt not in self.age_fmts:
@@ -580,7 +583,7 @@ class TwAddrList(MMGenDict):
 		al_id_save = None
 		mmids = sorted(self,key=sort_algo,reverse=bool(sort and 'reverse' in sort))
 		if show_age:
-			_set_dates([o for o in mmids if hasattr(o,'confs')])
+			await _set_dates([o for o in mmids if hasattr(o,'confs')])
 		for mmid in mmids:
 			if mmid.type == 'mmgen':
 				if al_id_save and al_id_save != mmid.obj.al_id:
@@ -603,22 +606,27 @@ class TwAddrList(MMGenDict):
 
 		return '\n'.join(out + ['\nTOTAL: {} {}'.format(self.total.hl(color=True),g.dcoin)])
 
-class TrackingWallet(MMGenObject):
+class TrackingWallet(MMGenObject,metaclass=aInitMeta):
 
 	caps = ('rescan','batch')
 	data_key = 'addresses'
 	use_tw_file = False
 	aggressive_sync = False
+	importing = False
 
 	def __new__(cls,*args,**kwargs):
 		return MMGenObject.__new__(altcoin_subclass(cls,'tw','TrackingWallet'))
 
-	def __init__(self,mode='r',no_rpc=False):
+	async def __ainit__(self,mode='r'):
+
+		assert mode in ('r','w','i'), "{!r}: wallet mode must be 'r','w' or 'i'".format(mode)
+		if mode == 'i':
+			self.importing = True
+			mode = 'w'
 
 		if g.debug:
 			print_stack_trace('TW INIT {!r} {!r}'.format(mode,self))
 
-		assert mode in ('r','w'),"{!r}: wallet mode must be 'r' or 'w'".format(self)
 		self.mode = mode
 		self.desc = self.base_desc = '{} tracking wallet'.format(capfirst(g.proto.name))
 
@@ -632,7 +640,6 @@ class TrackingWallet(MMGenObject):
 			raise WalletFileError(m.format(self.data['coin'],g.coin))
 
 		self.conv_types(self.data[self.data_key])
-		self.rpc_init()
 		self.cur_balances = {} # cache balances to prevent repeated lookups per program invocation
 
 	def init_empty(self):
@@ -697,12 +704,9 @@ class TrackingWallet(MMGenObject):
 	@staticmethod
 	def conv_types(ad):
 		for k,v in ad.items():
-			if k in ('params','coin'): continue
-			v['mmid'] = TwMMGenID(v['mmid'],on_fail='raise')
-			v['comment'] = TwComment(v['comment'],on_fail='raise')
-
-	def rpc_init(self):
-		rpc_init()
+			if k not in ('params','coin'):
+				v['mmid'] = TwMMGenID(v['mmid'],on_fail='raise')
+				v['comment'] = TwComment(v['comment'],on_fail='raise')
 
 	@property
 	def data_root(self):
@@ -728,14 +732,14 @@ class TrackingWallet(MMGenObject):
 		if addr in data_root and 'balance' in data_root[addr]:
 			return g.proto.coin_amt(data_root[addr]['balance'])
 
-	def get_balance(self,addr,force_rpc=False):
+	async def get_balance(self,addr,force_rpc=False):
 		ret = None if force_rpc else self.get_cached_balance(addr,self.cur_balances,self.data_root)
 		if ret == None:
-			ret = self.rpc_get_balance(addr)
+			ret = await self.rpc_get_balance(addr)
 			self.cache_balance(addr,ret,self.cur_balances,self.data_root)
 		return ret
 
-	def rpc_get_balance(self,addr):
+	async def rpc_get_balance(self,addr):
 		raise NotImplementedError('not implemented')
 
 	@property
@@ -752,12 +756,12 @@ class TrackingWallet(MMGenObject):
 		return dict((x['mmid'],{'addr':x['addr'],'comment':x['comment']}) for x in self.sorted_list)
 
 	@write_mode
-	def import_address(self,addr,label,rescan):
-		return g.rpc.importaddress(addr,label,rescan,timeout=(False,3600)[rescan])
+	async def import_address(self,addr,label,rescan):
+		return await g.rpc.call('importaddress',addr,label,rescan,timeout=(False,3600)[rescan])
 
 	@write_mode
 	def batch_import_address(self,arg_list):
-		return g.rpc.importaddress(arg_list,batch=True)
+		return g.rpc.batch_call('importaddress',arg_list)
 
 	def force_write(self):
 		mode_save = self.mode
@@ -789,24 +793,30 @@ class TrackingWallet(MMGenObject):
 		elif g.debug:
 			msg('Data is unchanged\n')
 
-	def is_in_wallet(self,addr):
-		return addr in TwAddrList([],0,True,True,True,wallet=self).coinaddr_list()
+	async def is_in_wallet(self,addr):
+		return addr in (await TwAddrList([],0,True,True,True,wallet=self)).coinaddr_list()
 
 	@write_mode
-	def set_label(self,coinaddr,lbl):
+	async def set_label(self,coinaddr,lbl):
 		# bitcoin-abc 'setlabel' RPC is broken, so use old 'importaddress' method to set label
 		# broken behavior: new label is set OK, but old label gets attached to another address
 		if 'label_api' in g.rpc.caps and g.coin != 'BCH':
-			return g.rpc.setlabel(coinaddr,lbl,on_fail='return')
+			args = ('setlabel',coinaddr,lbl)
 		else:
 			# NOTE: this works because importaddress() removes the old account before
 			# associating the new account with the address.
 			# RPC args: addr,label,rescan[=true],p2sh[=none]
-			return g.rpc.importaddress(coinaddr,lbl,False,on_fail='return')
+			args = ('importaddress',coinaddr,lbl,False)
+
+		try:
+			return await g.rpc.call(*args)
+		except Exception as e:
+			rmsg(e.args[0])
+			return False
 
 	# returns on failure
 	@write_mode
-	def add_label(self,arg1,label='',addr=None,silent=False,on_fail='return'):
+	async def add_label(self,arg1,label='',addr=None,silent=False,on_fail='return'):
 		from .tx import is_mmgen_id,is_coin_addr
 		mmaddr,coinaddr = None,None
 		if is_coin_addr(addr or arg1):
@@ -815,14 +825,14 @@ class TrackingWallet(MMGenObject):
 			mmaddr = TwMMGenID(arg1)
 
 		if mmaddr and not coinaddr:
-			from .addr import AddrData
-			coinaddr = AddrData(source='tw').mmaddr2coinaddr(mmaddr)
+			from .addr import TwAddrData
+			coinaddr = (await TwAddrData()).mmaddr2coinaddr(mmaddr)
 
 		try:
 			if not is_mmgen_id(arg1):
 				assert coinaddr,"Invalid coin address for this chain: {}".format(arg1)
 			assert coinaddr,"{pn} address '{ma}' not found in tracking wallet"
-			assert self.is_in_wallet(coinaddr),"Address '{ca}' not found in tracking wallet"
+			assert await self.is_in_wallet(coinaddr),"Address '{ca}' not found in tracking wallet"
 		except Exception as e:
 			msg(e.args[0].format(pn=g.proj_name,ma=mmaddr,ca=coinaddr))
 			return False
@@ -830,8 +840,8 @@ class TrackingWallet(MMGenObject):
 		# Allow for the possibility that BTC addr of MMGen addr was entered.
 		# Do reverse lookup, so that MMGen addr will not be marked as non-MMGen.
 		if not mmaddr:
-			from .addr import AddrData
-			mmaddr = AddrData(source='tw').coinaddr2mmaddr(coinaddr)
+			from .addr import TwAddrData
+			mmaddr = (await TwAddrData()).coinaddr2mmaddr(coinaddr)
 
 		if not mmaddr:
 			mmaddr = '{}:{}'.format(g.proto.base_coin.lower(),coinaddr)
@@ -844,11 +854,7 @@ class TrackingWallet(MMGenObject):
 
 		lbl = TwLabel(mmaddr + ('',' '+cmt)[bool(cmt)],on_fail=on_fail)
 
-		ret = self.set_label(coinaddr,lbl)
-
-		from .rpc import rpc_error,rpc_errmsg
-		if rpc_error(ret):
-			msg('From {}: {}'.format(g.proto.daemon_name,rpc_errmsg(ret)))
+		if await self.set_label(coinaddr,lbl) == False:
 			if not silent:
 				msg('Label could not be {}'.format(('removed','added')[bool(label)]))
 			return False
@@ -861,32 +867,31 @@ class TrackingWallet(MMGenObject):
 			return True
 
 	@write_mode
-	def remove_label(self,mmaddr):
-		self.add_label(mmaddr,'')
+	async def remove_label(self,mmaddr):
+		await self.add_label(mmaddr,'')
 
 	@write_mode
-	def remove_address(self,addr):
+	async def remove_address(self,addr):
 		raise NotImplementedError('address removal not implemented for coin {}'.format(g.coin))
 
-class TwGetBalance(MMGenObject):
+class TwGetBalance(MMGenObject,metaclass=aInitMeta):
 
 	fs = '{w:13} {u:<16} {p:<16} {c}\n'
 
 	def __new__(cls,*args,**kwargs):
 		return MMGenObject.__new__(altcoin_subclass(cls,'tw','TwGetBalance'))
 
-	def __init__(self,minconf,quiet):
+	async def __ainit__(self,minconf,quiet):
 
-		rpc_init()
 		self.minconf = minconf
 		self.quiet = quiet
 		self.data = {k:[g.proto.coin_amt('0')] * 4 for k in ('TOTAL','Non-MMGen','Non-wallet')}
-		self.create_data()
+		await self.create_data()
 
-	def create_data(self):
+	async def create_data(self):
 		# 0: unconfirmed, 1: below minconf, 2: confirmed, 3: spendable (privkey in wallet)
 		lbl_id = ('account','label')['label_api' in g.rpc.caps]
-		for d in g.rpc.listunspent(0):
+		for d in await g.rpc.call('listunspent',0):
 			lbl = get_tw_label(d[lbl_id])
 			if lbl:
 				if lbl.mmid.type == 'mmgen':

mmgen/tx.py

@@ -82,8 +82,7 @@ def mmaddr2coinaddr(mmaddr,ad_w,ad_f):
 	return CoinAddr(coin_addr)
 
 def segwit_is_active(exit_on_error=False):
-	rpc_init()
-	d = g.rpc.getblockchaininfo()
+	d = g.rpc.cached['blockchaininfo']
 	if d['chain'] == 'regtest':
 		return True
 	if (    'bip9_softforks' in d
@@ -281,6 +280,7 @@ class MMGenTX(MMGenObject):
 	sig_ext  = 'sigtx'
 	txid_ext = 'txid'
 	desc     = 'transaction'
+	hexdata_type = 'hex'
 	fee_fail_fs = 'Network fee estimation for {c} confirmations failed ({t})'
 	no_chg_msg = 'Warning: Change address will be deleted as transaction produces no change'
 	rel_fee_desc = 'satoshis per byte'
@@ -308,7 +308,11 @@ inputs must be supplied to '{pnl}-txsign' in a file with the '--keys-from-file'
 option.
 Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_name.lower())
 
-	def __init__(self,filename=None,metadata_only=False,caller=None,quiet_open=False,offline=False):
+	def __init__(self,filename=None,metadata_only=False,caller=None,quiet_open=False,data=None,tw=None):
+		if data:
+			assert type(data) is dict, type(data)
+			self.__dict__ = data
+			return
 		self.inputs      = MMGenTxInputList()
 		self.outputs     = MMGenTxOutputList()
 		self.send_amt    = g.proto.coin_amt('0')  # total amt minus change
@@ -327,6 +331,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		self.dcoin       = None
 		self.caller      = caller
 		self.locktime    = None
+		self.tw          = tw
 
 		if filename:
 			self.parse_tx_file(filename,metadata_only=metadata_only,quiet_open=quiet_open)
@@ -400,12 +405,12 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 	def update_txid(self):
 		self.txid = MMGenTxID(make_chksum_6(bytes.fromhex(self.hex)).upper())
 
-	def create_raw(self):
+	async def create_raw(self):
 		i = [{'txid':e.txid,'vout':e.vout} for e in self.inputs]
 		if self.inputs[0].sequence:
 			i[0]['sequence'] = self.inputs[0].sequence
 		o = {e.addr:e.amt for e in self.outputs}
-		self.hex = HexStr(g.rpc.createrawtransaction(i,o))
+		self.hex = HexStr(await g.rpc.call('createrawtransaction',i,o))
 		self.update_txid()
 
 	def print_contract_addr(self): pass
@@ -436,9 +441,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 	def has_segwit_inputs(self):
 		return any(i.mmid and i.mmid.mmtype in ('S','B') for i in self.inputs)
 
-	def compare_size_and_estimated_size(self):
+	def compare_size_and_estimated_size(self,tx_decoded):
 		est_vsize = self.estimate_size()
-		d = g.rpc.decoderawtransaction(self.hex)
+		d = tx_decoded
 		vsize = d['vsize'] if 'vsize' in d else d['size']
 		vmsg('\nVsize: {} (true) {} (estimated)'.format(vsize,est_vsize))
 		m1 = 'Estimated transaction vsize is {:1.2f} times the true vsize\n'
@@ -522,8 +527,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		return int(ret * float(opt.vsize_adj)) if hasattr(opt,'vsize_adj') and opt.vsize_adj else ret
 
 	# coin-specific fee routines
-	def get_relay_fee(self):
-		kb_fee = g.proto.coin_amt(g.rpc.getnetworkinfo()['relayfee'])
+	@property
+	def relay_fee(self):
+		kb_fee = g.proto.coin_amt(g.rpc.cached['networkinfo']['relayfee'])
 		ret = kb_fee * self.estimate_size() // 1024
 		vmsg('Relay fee: {} {c}/kB, for transaction: {} {c}'.format(kb_fee,ret,c=g.coin))
 		return ret
@@ -533,9 +539,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		unit = getattr(g.proto.coin_amt,to_unit or 'min_coin_unit')
 		return int(abs_fee // unit // self.estimate_size())
 
-	def get_rel_fee_from_network(self):
+	async def get_rel_fee_from_network(self):
 		try:
-			ret = g.rpc.estimatesmartfee(opt.tx_confs,opt.fee_estimate_mode.upper())
+			ret = await g.rpc.call('estimatesmartfee',opt.tx_confs,opt.fee_estimate_mode.upper())
 			fee_per_kb = ret['feerate'] if 'feerate' in ret else -2
 			fe_type = 'estimatesmartfee'
 		except:
@@ -577,9 +583,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 			m = '{} {c}: {} fee too large (maximum fee: {} {c})'
 			msg(m.format(abs_fee,desc,g.proto.max_tx_fee,c=g.coin))
 			return False
-		elif abs_fee < self.get_relay_fee():
+		elif abs_fee < self.relay_fee:
 			m = '{} {c}: {} fee too small (below relay fee of {} {c})'
-			msg(m.format(str(abs_fee),desc,str(self.get_relay_fee()),c=g.coin))
+			msg(m.format(str(abs_fee),desc,str(self.relay_fee),c=g.coin))
 			return False
 		else:
 			return abs_fee
@@ -626,14 +632,14 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 			tx_fee = my_raw_input(self.usr_fee_prompt)
 			desc = 'User-selected'
 
-	def get_fee_from_user(self,have_estimate_fail=[]):
+	async def get_fee_from_user(self,have_estimate_fail=[]):
 
 		if opt.tx_fee:
 			desc = 'User-selected'
 			start_fee = opt.tx_fee
 		else:
 			desc = 'Network-estimated (mode: {})'.format(opt.fee_estimate_mode.upper())
-			fee_per_kb,fe_type = self.get_rel_fee_from_network()
+			fee_per_kb,fe_type = await self.get_rel_fee_from_network()
 
 			if fee_per_kb < 0:
 				if not have_estimate_fail:
@@ -677,11 +683,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		assert isinstance(val,int),'locktime value not an integer'
 		self.hex = self.hex[:-8] + bytes.fromhex('{:08x}'.format(val))[::-1].hex()
 
-	def get_blockcount(self):
-		return int(g.rpc.getblockcount())
-
 	def add_blockcount(self):
-		self.blockcount = self.get_blockcount()
+		self.blockcount = g.rpc.blockcount
 
 	def format(self):
 		self.inputs.check_coin_mismatch()
@@ -718,75 +721,6 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 	def get_non_mmaddrs(self,desc):
 		return {i.addr for i in getattr(self,desc) if not i.mmid}
 
-	def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception
-
-		if self.marked_signed():
-			msg('Transaction is already signed!')
-			return False
-
-		if not self.check_correct_chain(on_fail='return'):
-			return False
-
-		if (self.has_segwit_inputs() or self.has_segwit_outputs()) and not g.proto.cap('segwit'):
-			ymsg("TX has Segwit inputs or outputs, but {} doesn't support Segwit!".format(g.coin))
-			return False
-
-		self.check_pubkey_scripts()
-
-		qmsg('Passing {} key{} to {}'.format(len(keys),suf(keys),g.proto.daemon_name))
-
-		if self.has_segwit_inputs():
-			from .addr import KeyGenerator,AddrGenerator
-			kg = KeyGenerator('std')
-			ag = AddrGenerator('segwit')
-			keydict = MMGenDict([(d.addr,d.sec) for d in keys])
-
-		sig_data = []
-		for d in self.inputs:
-			e = {k:getattr(d,k) for k in ('txid','vout','scriptPubKey','amt')}
-			e['amount'] = e['amt']
-			del e['amt']
-			if d.mmid and d.mmid.mmtype == 'S':
-				e['redeemScript'] = ag.to_segwit_redeem_script(kg.to_pubhex(keydict[d.addr]))
-			sig_data.append(e)
-
-		msg_r('Signing transaction{}...'.format(tx_num_str))
-		wifs = [d.sec.wif for d in keys]
-
-		try:
-			ret = g.rpc.signrawtransactionwithkey(self.hex,wifs,sig_data,g.proto.sighash_type) \
-				if 'sign_with_key' in g.rpc.caps else \
-					g.rpc.signrawtransaction(self.hex,sig_data,wifs,g.proto.sighash_type)
-		except Exception as e:
-			msg(yellow('This is not the BCH chain.\nRe-run the script without the --coin=bch option.'
-				if 'Invalid sighash param' in e.args[0] else e.args[0]))
-			return False
-
-		if not ret['complete']:
-			msg('failed\n{} returned the following errors:'.format(g.proto.daemon_name.capitalize()))
-			msg(repr(ret['errors']))
-			return False
-
-		try:
-			self.hex = HexStr(ret['hex'])
-			self.compare_size_and_estimated_size()
-			dt = DeserializedTX(self.hex)
-			self.check_hex_tx_matches_mmgen_tx(dt)
-			self.coin_txid = CoinTxID(dt['txid'],on_fail='raise')
-			self.check_sigs(dt)
-			if not self.coin_txid == g.rpc.decoderawtransaction(ret['hex'])['txid']:
-				raise BadMMGenTxID('txid mismatch (after signing)')
-			msg('OK')
-			return True
-		except Exception as e:
-			try: m = '{}'.format(e.args[0])
-			except: m = repr(e.args[0])
-			msg('\n'+yellow(m))
-			if g.traceback:
-				import traceback
-				ymsg('\n'+''.join(traceback.format_exception(*sys.exc_info())))
-			return False
-
 	def mark_raw(self):
 		self.desc = 'transaction'
 		self.ext = self.raw_ext
@@ -874,38 +808,45 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 	def has_segwit_outputs(self):
 		return any(o.mmid and o.mmid.mmtype in ('S','B') for o in self.outputs)
 
-	def get_status(self,status=False):
+	async def get_status(self,status=False):
 
 		class r(object): pass
 
-		def is_in_wallet():
-			ret = g.rpc.gettransaction(self.coin_txid,on_fail='silent')
+		async def is_in_wallet():
+			try: ret = await g.rpc.call('gettransaction',self.coin_txid)
+			except: return False
 			if 'confirmations' in ret and ret['confirmations'] > 0:
 				r.confs = ret['confirmations']
 				return True
 			else:
 				return False
 
-		def is_in_utxos():
-			return 'txid' in g.rpc.getrawtransaction(self.coin_txid,True,on_fail='silent')
-
-		def is_in_mempool():
-			return 'height' in g.rpc.getmempoolentry(self.coin_txid,on_fail='silent')
+		async def is_in_utxos():
+			try: return 'txid' in await g.rpc.call('getrawtransaction',self.coin_txid,True)
+			except: return False
 
-		def is_replaced():
-			if is_in_mempool(): return False
-			ret = g.rpc.gettransaction(self.coin_txid,on_fail='silent')
+		async def is_in_mempool():
+			try: return 'height' in await g.rpc.call('getmempoolentry',self.coin_txid)
+			except: return False
 
-			if not 'bip125-replaceable' in ret or not 'confirmations' in ret or ret['confirmations'] > 0:
+		async def is_replaced():
+			if await is_in_mempool():
 				return False
+			try:
+				ret = await g.rpc.call('gettransaction',self.coin_txid)
+			except:
+				return False
+			else:
+				if 'bip125-replaceable' in ret and 'confirmations' in ret and ret['confirmations'] <= 0:
+					r.replacing_confs = -ret['confirmations']
+					r.replacing_txs = ret['walletconflicts']
+					return True
+				else:
+					return False
 
-			r.replacing_confs = -ret['confirmations']
-			r.replacing_txs = ret['walletconflicts']
-			return True
-
-		if is_in_mempool():
+		if await is_in_mempool():
 			if status:
-				d = g.rpc.gettransaction(self.coin_txid,on_fail='silent')
+				d = await g.rpc.call('gettransaction',self.coin_txid)
 				brs = 'bip125-replaceable'
 				rep = '{}replaceable'.format(('NOT ','')[brs in d and d[brs]=='yes'])
 				t = d['timereceived']
@@ -917,22 +858,23 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 					msg('TX status: in mempool, {}\n{}'.format(rep,b))
 			else:
 				msg('Warning: transaction is in mempool!')
-		elif is_in_wallet():
+		elif await is_in_wallet():
 			die(0,'Transaction has {} confirmation{}'.format(r.confs,suf(r.confs)))
-		elif is_in_utxos():
+		elif await is_in_utxos():
 			die(2,red('ERROR: transaction is in the blockchain (but not in the tracking wallet)!'))
-		elif is_replaced():
-			m1 = 'Transaction has been replaced'
-			m2 = 'Replacement transaction is in mempool'
-			rc = r.replacing_confs
-			if rc:
-				m2 = 'Replacement transaction has {} confirmation{}'.format(rc,suf(rc))
-			msg('{}\n{}'.format(m1,m2))
+		elif await is_replaced():
+			msg('Transaction has been replaced\nReplacement transaction ' + (
+					f'has {r.replacing_confs} confirmation{suf(r.replacing_confs)}'
+				if r.replacing_confs else
+					'is in mempool' ))
 			if not opt.quiet:
 				msg('Replacing transactions:')
-				d = ((t,g.rpc.getmempoolentry(t,on_fail='silent')) for t in r.replacing_txs)
-				for txid,mp_entry in d:
-					msg('  {}{}'.format(txid,' in mempool' if ('height' in mp_entry) else ''))
+				d = []
+				for txid in r.replacing_txs:
+					try:    d.append(await g.rpc.call('getmempoolentry',txid))
+					except: d.append({})
+				for txid,mp_entry in zip(r.replacing_txs,d):
+					msg(f'  {txid}' + ('',' in mempool')['height' in mp_entry])
 			die(0,'')
 
 	def confirm_send(self):
@@ -942,8 +884,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		confirm_or_raise(m1,m2,m3)
 		msg('Sending transaction')
 
-	def send(self,prompt_user=True,exit_on_fail=False):
-
+	async def send(self,prompt_user=True,exit_on_fail=False):
 		if not self.marked_signed():
 			die(1,'Transaction is not signed!')
 
@@ -961,15 +902,21 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 			die(2,'Transaction fee ({}) greater than {} max_tx_fee ({} {})!'.format(
 				self.get_fee_from_tx(),g.proto.name.capitalize(),g.proto.max_tx_fee,g.coin))
 
-		self.get_status()
+		await self.get_status()
 
-		if prompt_user: self.confirm_send()
+		if prompt_user:
+			self.confirm_send()
 
-		ret = None if g.bogus_send else g.rpc.sendrawtransaction(self.hex,on_fail='return')
+		if g.bogus_send:
+			ret = None
+		else:
+			try:
+				ret = await g.rpc.call('sendrawtransaction',self.hex)
+			except Exception as e:
+				ret = False
 
-		from .rpc import rpc_error,rpc_errmsg
-		if rpc_error(ret):
-			errmsg = rpc_errmsg(ret)
+		if ret == False:
+			errmsg = e
 			if 'Signature must use SIGHASH_FORKID' in errmsg:
 				m  = 'The Aug. 1 2017 UAHF has activated on this chain.'
 				m += "\nRe-run the script with the --coin=bch option."
@@ -1061,7 +1008,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 				msg('')
 
 #	def is_replaceable_from_rpc(self):
-#		dec_tx = g.rpc.decoderawtransaction(self.hex)
+#		dec_tx = await g.rpc.call('decoderawtransaction',self.hex)
 #		return None < dec_tx['vin'][0]['sequence'] <= g.max_int - 2
 
 	def is_replaceable(self):
@@ -1138,8 +1085,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		blockcount = None
 		if g.proto.base_coin != 'ETH':
 			try:
-				rpc_init()
-				blockcount = self.get_blockcount()
+				blockcount = g.rpc.blockcount
 			except:
 				pass
 
@@ -1187,6 +1133,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 	def check_txfile_hex_data(self):
 		self.hex = HexStr(self.hex,on_fail='raise')
 
+	def parse_txfile_hex_data(self):
+		pass
+
 	def parse_tx_file(self,infile,metadata_only=False,quiet_open=False):
 
 		def eval_io_data(raw_data,desc):
@@ -1271,6 +1220,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 
 			desc = 'transaction file hex data'
 			self.check_txfile_hex_data()
+			desc = f'transaction file {self.hexdata_type} data'
+			self.parse_txfile_hex_data()
 			# the following ops will all fail if g.coin doesn't match self.coin
 			desc = 'coin type in metadata'
 			assert self.coin == g.coin,self.coin
@@ -1286,7 +1237,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 			self.chain = 'mainnet'
 
 		if self.dcoin:
-			self.resolve_g_token_from_tx_file()
+			self.resolve_g_token_from_txfile()
+			g.dcoin = self.dcoin
 
 	def process_cmd_arg(self,arg,ad_f,ad_w):
 
@@ -1320,8 +1272,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		if not self.outputs:
 			die(2,'At least one output must be specified on the command line')
 
-	def get_outputs_from_cmdline(self,cmd_args):
-		from .addr import AddrList,AddrData
+	async def get_outputs_from_cmdline(self,cmd_args):
+		from .addr import AddrList,AddrData,TwAddrData
 		addrfiles = [a for a in cmd_args if get_extension(a) == AddrList.ext]
 		cmd_args = set(cmd_args) - set(addrfiles)
 
@@ -1330,7 +1282,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 			check_infile(a)
 			ad_f.add(AddrList(a))
 
-		ad_w = AddrData(source='tw',wallet=self.twuo.wallet)
+		ad_w = await TwAddrData(wallet=self.tw)
 
 		self.process_cmd_args(cmd_args,ad_f,ad_w)
 
@@ -1349,7 +1301,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 					msg('Unspent output number must be <= {}'.format(len(unspent)))
 
 	# we don't know fee yet, so perform preliminary check with fee == 0
-	def precheck_sufficient_funds(self,inputs_sum,sel_unspent):
+	async def precheck_sufficient_funds(self,inputs_sum,sel_unspent):
 		if self.twuo.total < self.send_amt:
 			msg(self.msg_wallet_low_coin.format(self.send_amt-inputs_sum,g.dcoin))
 			return False
@@ -1358,7 +1310,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 			return False
 		return True
 
-	def get_change_amt(self):
+	async def get_change_amt(self):
 		return self.sum_inputs() - self.send_amt - self.fee
 
 	def warn_insufficient_chg(self,change_amt):
@@ -1394,11 +1346,11 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 
 		return set(sel_nums) # silently discard duplicates
 
-	def get_cmdline_input_addrs(self):
+	async def get_cmdline_input_addrs(self):
 		# Bitcoin full node, call doesn't go to the network, so just call listunspent with addrs=[]
 		return []
 
-	def get_inputs_from_user(self):
+	async def get_inputs_from_user(self):
 
 		while True:
 			us_f = self.select_unspent_cmdline if opt.inputs else self.select_unspent
@@ -1408,7 +1360,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 			sel_unspent = self.twuo.MMGenTwOutputList([self.twuo.unspent[i-1] for i in sel_nums])
 
 			inputs_sum = sum(s.amt for s in sel_unspent)
-			if not self.precheck_sufficient_funds(inputs_sum,sel_unspent):
+			if not await self.precheck_sufficient_funds(inputs_sum,sel_unspent):
 				continue
 
 			non_mmaddrs = [i for i in sel_unspent if i.twmmid.type == 'non-mmgen']
@@ -1420,9 +1372,9 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 
 			self.copy_inputs_from_tw(sel_unspent)  # makes self.inputs
 
-			self.fee = self.get_fee_from_user()
+			self.fee = await self.get_fee_from_user()
 
-			change_amt = self.get_change_amt()
+			change_amt = await self.get_change_amt()
 
 			if change_amt >= 0:
 				p = self.final_inputs_ok_msg(change_amt)
@@ -1439,27 +1391,34 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		if not self.send_amt:
 			self.send_amt = change_amt
 
-	def create(self,cmd_args,locktime,do_info=False):
+	async def set_token_params(self):
+		pass
+
+	async def create(self,cmd_args,locktime,do_info=False):
 		assert isinstance(locktime,int),'locktime must be of type int'
 
-		if opt.comment_file: self.add_comment(opt.comment_file)
+		from .tw import TwUnspentOutputs
 
-		twuo_addrs = self.get_cmdline_input_addrs()
+		if opt.comment_file:
+			self.add_comment(opt.comment_file)
 
-		from .tw import TwUnspentOutputs
-		self.twuo = TwUnspentOutputs(minconf=opt.minconf,addrs=twuo_addrs)
+		twuo_addrs = await self.get_cmdline_input_addrs()
+
+		self.twuo = await TwUnspentOutputs(minconf=opt.minconf,addrs=twuo_addrs)
+		await self.twuo.get_unspent_data()
 
 		if not do_info:
-			self.get_outputs_from_cmdline(cmd_args)
+			await self.get_outputs_from_cmdline(cmd_args)
 
 		do_license_msg()
 
 		if not opt.inputs:
-			self.twuo.view_and_sort(self)
+			await self.twuo.view_and_sort(self)
 
 		self.twuo.display_total()
 
 		if do_info:
+			del self.twuo.wallet
 			sys.exit(0)
 
 		self.send_amt = self.sum_outputs()
@@ -1468,7 +1427,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 			('Unknown','{} {}'.format(self.send_amt.hl(),g.dcoin))[bool(self.send_amt)]
 		))
 
-		change_amt = self.get_inputs_from_user()
+		change_amt = await self.get_inputs_from_user()
 
 		self.update_change_output(change_amt)
 		self.update_send_amt(change_amt)
@@ -1482,7 +1441,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 
 		if not opt.yes:
 			self.add_comment()  # edits an existing comment
-		self.create_raw()       # creates self.hex, self.txid
+
+		await self.create_raw()       # creates self.hex, self.txid
 
 		if g.proto.base_proto == 'Bitcoin' and locktime:
 			msg('Setting nlocktime to {}!'.format(strfmt_locktime(locktime)))
@@ -1501,9 +1461,85 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		if not opt.yes:
 			self.view_with_prompt('View decoded transaction?')
 
-		del self.twuo
+		del self.twuo.wallet
+
+class MMGenTxForSigning(MMGenTX):
+
+	hexdata_type = 'json'
+
+	def __new__(cls,*args,**kwargs):
+		return MMGenObject.__new__(altcoin_subclass(cls,'tx','MMGenTxForSigning'))
+
+	async def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception
+
+		if self.marked_signed():
+			msg('Transaction is already signed!')
+			return False
+
+		if not self.check_correct_chain(on_fail='return'):
+			return False
+
+		if (self.has_segwit_inputs() or self.has_segwit_outputs()) and not g.proto.cap('segwit'):
+			ymsg("TX has Segwit inputs or outputs, but {} doesn't support Segwit!".format(g.coin))
+			return False
+
+		self.check_pubkey_scripts()
+
+		qmsg('Passing {} key{} to {}'.format(len(keys),suf(keys),g.proto.daemon_name))
+
+		if self.has_segwit_inputs():
+			from .addr import KeyGenerator,AddrGenerator
+			kg = KeyGenerator('std')
+			ag = AddrGenerator('segwit')
+			keydict = MMGenDict([(d.addr,d.sec) for d in keys])
+
+		sig_data = []
+		for d in self.inputs:
+			e = {k:getattr(d,k) for k in ('txid','vout','scriptPubKey','amt')}
+			e['amount'] = e['amt']
+			del e['amt']
+			if d.mmid and d.mmid.mmtype == 'S':
+				e['redeemScript'] = ag.to_segwit_redeem_script(kg.to_pubhex(keydict[d.addr]))
+			sig_data.append(e)
+
+		msg_r('Signing transaction{}...'.format(tx_num_str))
+		wifs = [d.sec.wif for d in keys]
 
-class MMGenBumpTX(MMGenTX):
+		try:
+			args = (
+				('signrawtransaction',self.hex,sig_data,wifs,g.proto.sighash_type),
+				('signrawtransactionwithkey',self.hex,wifs,sig_data,g.proto.sighash_type)
+			)['sign_with_key' in g.rpc.caps]
+			ret = await g.rpc.call(*args)
+		except Exception as e:
+			msg(yellow((
+				e.args[0],
+				'This is not the BCH chain.\nRe-run the script without the --coin=bch option.'
+			)['Invalid sighash param' in e.args[0]]))
+			return False
+
+		try:
+			self.hex = HexStr(ret['hex'])
+			tx_decoded = await g.rpc.call('decoderawtransaction',ret['hex'])
+			self.compare_size_and_estimated_size(tx_decoded)
+			dt = DeserializedTX(self.hex)
+			self.check_hex_tx_matches_mmgen_tx(dt)
+			self.coin_txid = CoinTxID(dt['txid'],on_fail='raise')
+			self.check_sigs(dt)
+			if not self.coin_txid == tx_decoded['txid']:
+				raise BadMMGenTxID('txid mismatch (after signing)')
+			msg('OK')
+			return True
+		except Exception as e:
+			try: m = '{}'.format(e.args[0])
+			except: m = repr(e.args[0])
+			msg('\n'+yellow(m))
+			if g.traceback:
+				import traceback
+				ymsg('\n'+''.join(traceback.format_exception(*sys.exc_info())))
+			return False
+
+class MMGenBumpTX(MMGenTxForSigning):
 
 	def __new__(cls,*args,**kwargs):
 		return MMGenTX.__new__(altcoin_subclass(cls,'tx','MMGenBumpTX'),*args,**kwargs)
@@ -1511,9 +1547,8 @@ class MMGenBumpTX(MMGenTX):
 	min_fee = None
 	bump_output_idx = None
 
-	def __init__(self,filename,send=False):
-
-		super().__init__(filename)
+	def __init__(self,filename,send=False,tw=None):
+		super().__init__(filename,tw=tw)
 
 		if not self.is_replaceable():
 			die(1,"Transaction '{}' is not replaceable".format(self.txid))
@@ -1576,8 +1611,9 @@ class MMGenBumpTX(MMGenTX):
 							self.bump_output_idx = idx
 							return idx
 
-	def set_min_fee(self):
-		self.min_fee = self.sum_inputs() - self.sum_outputs() + self.get_relay_fee()
+	@property
+	def min_fee(self):
+		return self.sum_inputs() - self.sum_outputs() + self.relay_fee
 
 	def update_fee(self,op_idx,fee):
 		amt = self.sum_inputs() - self.sum_outputs(exclude=op_idx) - fee
@@ -1598,10 +1634,10 @@ class MMGenBumpTX(MMGenTX):
 # NOT MAINTAINED
 class MMGenSplitTX(MMGenTX):
 
-	def get_outputs_from_cmdline(self,mmid): # TODO: check that addr is empty
+	async def get_outputs_from_cmdline(self,mmid): # TODO: check that addr is empty
 
-		from .addr import AddrData
-		ad_w = AddrData(source='tw')
+		from .addr import TwAddrData
+		ad_w = await TwAddrData()
 
 		if is_mmgen_id(mmid):
 			coin_addr = mmaddr2coinaddr(mmid,ad_w,None) if is_mmgen_id(mmid) else CoinAddr(mmid)
@@ -1620,17 +1656,12 @@ class MMGenSplitTX(MMGenTX):
 			g.rpc_host = opt.rpc_host2
 		if opt.tx_fees:
 			opt.tx_fee = opt.tx_fees.split(',')[1]
-		try:
-			rpc_init(reinit=True)
-		except:
-			ymsg('Connect to {} daemon failed.  Network fee estimation unavailable'.format(g.coin))
-			return self.get_usr_fee_interactive(opt.tx_fee,'User-selected')
 		return super().get_fee_from_user()
 
-	def create_split(self,mmid):
+	async def create_split(self,mmid):
 
 		self.outputs = self.MMGenTxOutputList()
-		self.get_outputs_from_cmdline(mmid)
+		await self.get_outputs_from_cmdline(mmid)
 
 		while True:
 			change_amt = self.sum_inputs() - self.get_split_fee_from_user()
@@ -1647,7 +1678,8 @@ class MMGenSplitTX(MMGenTX):
 
 		if not opt.yes:
 			self.add_comment()  # edits an existing comment
-		self.create_raw()       # creates self.hex, self.txid
+
+		await self.create_raw()       # creates self.hex, self.txid
 
 		self.add_timestamp()
 		self.add_blockcount() # TODO

mmgen/txsign.py

@@ -139,7 +139,7 @@ def get_keylist(opt):
 		return kal
 	return None
 
-def txsign(tx,seed_files,kl,kal,tx_num_str=''):
+async def txsign(tx,seed_files,kl,kal,tx_num_str=''):
 
 	keys = MMGenList() # list of AddrListEntry objects
 	non_mm_addrs = tx.get_non_mmaddrs('inputs')
@@ -169,4 +169,4 @@ def txsign(tx,seed_files,kl,kal,tx_num_str=''):
 	if extra_sids:
 		msg('Unused Seed ID{}: {}'.format(suf(extra_sids),' '.join(extra_sids)))
 
-	return tx.sign(tx_num_str,keys) # returns True or False
+	return await tx.sign(tx_num_str,keys) # returns True or False

mmgen/util.py

@@ -818,35 +818,32 @@ def do_license_msg(immed=False):
 			msg_r('\r')
 	msg('')
 
-def get_daemon_cfg_options(cfg_keys):
-
+# TODO: these belong in protocol.py
+def get_coin_daemon_cfg_fn():
 	# Use dirname() to remove 'bob' or 'alice' component
 	cfg_dir = os.path.dirname(g.data_dir) if g.regtest else g.proto.daemon_data_dir
-	cfg_file = os.path.join(cfg_dir,g.proto.name+'.conf' )
+	return os.path.join(cfg_dir,g.proto.name+'.conf' )
+
+def get_coin_daemon_cfg_options(req_keys):
 
+	fn = get_coin_daemon_cfg_fn()
 	try:
-		lines = get_lines_from_file(cfg_file,'',silent=not opt.verbose)
-		kv_pairs = [l.split('=') for l in lines]
-		cfg = {k:v for k,v in kv_pairs if k in cfg_keys}
+		lines = get_lines_from_file(fn,'',silent=not opt.verbose)
 	except:
-		vmsg("Warning: '{}' does not exist or is unreadable".format(cfg_file))
-		cfg = {}
-
-	for k in set(cfg_keys) - set(cfg.keys()): cfg[k] = ''
-
-	return cfg
+		vmsg(f'Warning: {fn!r} does not exist or is unreadable')
+		return dict((k,None) for k in req_keys)
 
-def get_coin_daemon_auth_cookie():
-	f = os.path.join(g.proto.daemon_data_dir,g.proto.daemon_data_subdir,'.cookie')
-	return get_lines_from_file(f,'')[0] if file_is_readable(f) else ''
+	def gen():
+		for key in req_keys:
+			val = None
+			for l in lines:
+				if l.startswith(key):
+					res = l.split('=',1)
+					if len(res) == 2 and not ' ' in res[1].strip():
+						val = res[1].strip()
+			yield (key,val)
 
-def rpc_init(reinit=False):
-	if not 'rpc' in g.proto.mmcaps:
-		die(1,'Coin daemon operations not supported for coin {}!'.format(g.coin))
-	if g.rpc != None and not reinit: return g.rpc
-	from .rpc import init_daemon
-	g.rpc = init_daemon(g.proto.daemon_family)
-	return g.rpc
+	return dict(gen())
 
 def format_par(s,indent=0,width=80,as_list=False):
 	words,lines = s.split(),[]
@@ -886,3 +883,28 @@ def get_network_id(coin=None,testnet=None):
 	if coin == None: assert testnet == None
 	if coin != None: assert testnet != None
 	return (coin or g.coin).lower() + ('','_tn')[testnet or g.testnet]
+
+def run_session(callback,do_rpc_init=True,backend=None):
+	backend = backend or opt.rpc_backend
+	import asyncio
+	async def do():
+		if backend == 'aiohttp':
+			import aiohttp
+			async with aiohttp.ClientSession(
+				headers = { 'Content-Type': 'application/json' },
+				connector = aiohttp.TCPConnector(limit_per_host=g.aiohttp_rpc_queue_len),
+			) as g.session:
+				if do_rpc_init:
+					from .rpc import rpc_init
+					await rpc_init(backend=backend)
+				ret = await callback
+			g.session = None
+			return ret
+		else:
+			if do_rpc_init:
+				from .rpc import rpc_init
+				await rpc_init(backend=backend)
+			return await callback
+
+	# return asyncio.run(do()) # Python 3.7+
+	return asyncio.get_event_loop().run_until_complete(do())

setup.py

@@ -114,6 +114,7 @@ setup(
 			'mmgen.filename',
 			'mmgen.globalvars',
 			'mmgen.keccak',
+			'mmgen.led',
 			'mmgen.license',
 			'mmgen.mn_electrum',
 			'mmgen.mn_entry',

test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.sigtx

@@ -0,0 +1,7 @@
+e9feb9
+ETH KOVAN B472BD 23.45495 20180725_111111 7654321
+f86d808509502f900082520894b7d06382a998817a16ba487a99636bf8cae29d9d89014580b8c15e8c60008077a057b7b73b9102fda68922eba0fbb70171425f1f27bad944838d3ded819eb03a63a00acb20276cbb30209923f959c63595b1b35663cbb30d186c6a5a2dbccb3e07fa
+[{'confs': 0, 'label': '', 'mmid': '98831F3A:E:1', 'amt': '123.456', 'addr': '97ccc3a117b3696340c42561361054b1c9c793d5'}]
+[{'mmid': '98831F3A:E:2', 'amt': '23.45495', 'addr': '2a6db46c87407e6d28fcb97d3bd0f5cf4aafca46'}]
+qRHzrPVpZFYxnQvk3atLzUtp41bZupJ2UQNnKe3ZnmqFsEngS6vaCCvesKKy9khzVq6y2RqarVBcZLnjtXxMpbAcdEtyBWiBYmZdoU8SN4uAbroHT1c7gEbmUNVKKdqHD86ZRRqDNpdh1ztmLiMAy3ibM83puwJHNpGGHgUGjZ1RSEgyVKCs2rZ9wXN8rBMibDDPYo1LgtAst2FkB36Mgf4Vf7ekoRAdiRNGd5YZ3RXAVsSdnZcyn4rdeQDMDkCq7JJDoB25eNEuXQutZFUcf2fEfxkMbW1sXJDNFQq
+0f277d20bf3793f94521a809943a659478bdfa6836a399f0568a93aeb4ce5184

test/ref/ethereum/tracking-wallet-v2.json

@@ -1 +1 @@
-{"coin":"ETH","accounts":{"e704b6cfd9f0edb2e6cfbd0c913438d37ede7b35":{"mmid":"98831F3A:E:1","comment":""},"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef":{"mmid":"eth:deadbeefdeadbeefdeadbeefdeadbeefdeadbeef","comment":""}},"tokens":{"d5f051401ca478b34c80d0b5a119e437dc6d9df5":{"6e0fbe42e1343309b3ccb9068dbad6132f86c96f":{"mmid":"98831F3A:E:12","comment":""},"b72268fa55a57fe838a745b27c90f0edb415af61":{"mmid":"98831F3A:E:13","comment":""}},"3dd0864668c36d27b53a98137764c99f9fd5b7b2":{"ca2b705adf43151ce61e01c653424e9790e9eb84":{"mmid":"98831F3A:E:22","comment":""},"7227a8fd728b74208255c6a8a09cb9fb66b1230c":{"mmid":"98831F3A:E:23","comment":""}}}}
+{"coin":"ETH","accounts":{"e704b6cfd9f0edb2e6cfbd0c913438d37ede7b35":{"mmid":"98831F3A:E:1","comment":""},"b7d06382a998817a16ba487a99636bf8cae29d9d":{"mmid":"98831F3A:E:2","comment":""},"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef":{"mmid":"eth:deadbeefdeadbeefdeadbeefdeadbeefdeadbeef","comment":""}},"tokens":{"d5f051401ca478b34c80d0b5a119e437dc6d9df5":{"6e0fbe42e1343309b3ccb9068dbad6132f86c96f":{"mmid":"98831F3A:E:12","comment":""},"b72268fa55a57fe838a745b27c90f0edb415af61":{"mmid":"98831F3A:E:13","comment":""}},"3dd0864668c36d27b53a98137764c99f9fd5b7b2":{"ca2b705adf43151ce61e01c653424e9790e9eb84":{"mmid":"98831F3A:E:22","comment":""},"7227a8fd728b74208255c6a8a09cb9fb66b1230c":{"mmid":"98831F3A:E:23","comment":""}}}}

test/test.py

@@ -492,6 +492,7 @@ class CmdGroupMgr(object):
 	cmd_groups_extra = {
 		'autosign_btc':     ('TestSuiteAutosignBTC',{'modname':'autosign'}),
 		'autosign_live':    ('TestSuiteAutosignLive',{'modname':'autosign'}),
+		'autosign_live_simulate': ('TestSuiteAutosignLiveSimulate',{'modname':'autosign'}),
 		'create_ref_tx':    ('TestSuiteRefTX',{'modname':'misc','full_data':True}),
 	}
 
@@ -867,7 +868,10 @@ class TestSuiteRunner(object):
 			if k in cfg:
 				setattr(self.ts,k,cfg[k])
 
-		self.process_retval(cmd,getattr(self.ts,cmd)(*arg_list)) # run the test
+		ret = getattr(self.ts,cmd)(*arg_list) # run the test
+		if type(ret).__name__ == 'coroutine':
+			ret = run_session(ret)
+		self.process_retval(cmd,ret)
 
 		if opt.profile:
 			omsg('\r\033[50C{:.4f}'.format(time.time() - start))

test/test_py_d/ts_autosign.py

@@ -43,21 +43,26 @@ class TestSuiteAutosign(TestSuiteBase):
 	def autosign_live(self):
 		return self.autosign_btc(live=True)
 
-	def autosign_btc(self,live=False):
+	def autosign_live_simulate(self):
+		return self.autosign_btc(live=True,simulate=True)
+
+	def autosign_btc(self,live=False,simulate=False):
 		return self.autosign(
 					coins=['btc'],
 					daemon_coins=['btc'],
 					txfiles=['btc'],
 					txcount=3,
-					live=live)
+					live=live,
+					simulate=simulate )
 
-	# tests everything except device detection, mount/unmount
+	# tests everything except mount/unmount
 	def autosign(   self,
 					coins=['btc','bch','ltc','eth'],
 					daemon_coins=['btc','bch','ltc'],
 					txfiles=['btc','bch','ltc','eth','mm1','etc'],
 					txcount=12,
-					live=False):
+					live=False,
+					simulate=False):
 
 		if self.skip_for_win(): return 'skip'
 
@@ -79,13 +84,16 @@ class TestSuiteAutosign(TestSuiteBase):
 			wf = t.written_to_file('Autosign wallet')
 			t.ok()
 
-		def copy_files(mountpoint,remove_signed_only=False,include_bad_tx=True):
-			fdata_in = (('btc',''),
-						('bch',''),
-						('ltc','litecoin'),
-						('eth','ethereum'),
-						('mm1','ethereum'),
-						('etc','ethereum_classic'))
+		def copy_files(
+				mountpoint,
+				remove_signed_only=False,
+				include_bad_tx=True,
+				fdata_in = (('btc',''),
+							('bch',''),
+							('ltc','litecoin'),
+							('eth','ethereum'),
+							('mm1','ethereum'),
+							('etc','ethereum_classic')) ):
 			fdata = [e for e in fdata_in if e[0] in txfiles]
 			from .ts_ref import TestSuiteRef
 			tfns  = [TestSuiteRef.sources['ref_tx_file'][c][1] for c,d in fdata] + \
@@ -124,8 +132,11 @@ class TestSuiteAutosign(TestSuiteBase):
 				omsg_r(blue('\nRemove removable device and then hit ENTER '))
 				input()
 
-			if gen_wallet: make_wallet(opts)
-			else: do_mount()
+			if gen_wallet:
+				if not opt.skip_deps:
+					make_wallet(opts)
+			else:
+				do_mount()
 
 			copy_files(mountpoint,include_bad_tx=not led_opts)
 
@@ -150,7 +161,7 @@ class TestSuiteAutosign(TestSuiteBase):
 
 			do_unmount()
 			omsg(green(m1))
-			t = self.spawn('mmgen-autosign',opts+led_opts+['wait'],extra_desc=desc)
+			t = self.spawn('mmgen-autosign',opts+led_opts+['--quiet','--no-summary','wait'],extra_desc=desc)
 			if not opt.exact_output: omsg('')
 			do_loop()
 			do_mount() # race condition due to device insertion detection
@@ -160,6 +171,8 @@ class TestSuiteAutosign(TestSuiteBase):
 			imsg(purple('\nKilling wait loop!'))
 			t.kill(2) # 2 = SIGINT
 			t.req_exit_val = 1
+			if simulate and led_opts:
+				t.expect("Stopping LED")
 			return t
 
 		def do_autosign(opts,mountpoint):
@@ -179,15 +192,44 @@ class TestSuiteAutosign(TestSuiteBase):
 			t.ok()
 
 			copy_files(mountpoint,remove_signed_only=True)
-			t = self.spawn('mmgen-autosign',opts+['wait'],extra_desc='(sign)')
+			t = self.spawn('mmgen-autosign',opts+['--quiet','wait'],extra_desc='(sign)')
 			t.expect('{} transactions signed'.format(txcount))
 			t.expect('2 transactions failed to sign')
 			t.expect('Waiting')
 			t.kill(2)
 			t.req_exit_val = 1
 			imsg('')
+			t.ok()
+
+			copy_files(mountpoint,include_bad_tx=True,fdata_in=(('btc',''),))
+			t = self.spawn(
+				'mmgen-autosign',
+				opts + ['--quiet','--stealth-led','wait'],
+				extra_desc='(sign - --quiet --stealth-led)' )
+			t.expect('2 transactions failed to sign')
+			t.expect('Waiting')
+			t.kill(2)
+			t.req_exit_val = 1
+			imsg('')
+			t.ok()
+
+			copy_files(mountpoint,include_bad_tx=False,fdata_in=(('btc',''),))
+			t = self.spawn(
+				'mmgen-autosign',
+				opts + ['--quiet','--led'],
+				extra_desc='(sign - --quiet --led)' )
+			t.read()
+			imsg('')
+			t.ok()
+
 			return t
 
+		# begin execution
+
+		if simulate and not opt.exact_output:
+			rmsg('This command must be run with --exact-output enabled!')
+			return False
+
 		network_ids = [c+'_tn' for c in daemon_coins] + daemon_coins
 		start_test_daemons(*network_ids)
 
@@ -205,27 +247,27 @@ class TestSuiteAutosign(TestSuiteBase):
 				ydie(1,"Directory '{}' does not exist!  Exiting".format(mountpoint))
 
 			opts = ['--coins='+','.join(coins)]
-			led_files = {   'opi': ('/sys/class/leds/orangepi:red:status/brightness',),
-							'rpi': ('/sys/class/leds/led0/brightness','/sys/class/leds/led0/trigger') }
 
-			for k in ('opi','rpi'):
-				if os.path.exists(led_files[k][0]):
-					led_support = k
-					break
-			else:
-				led_support = None
+			from mmgen.led import LEDControl
+
+			if simulate:
+				LEDControl.create_dummy_control_files()
 
-			if led_support:
-				for fn in (led_files[led_support]):
-					run(['sudo','chmod','0666',fn],check=True)
+			try:
+				cf = LEDControl(enabled=True,simulate=simulate)
+			except:
+				ret = "'no LED support detected'"
+			else:
+				for fn in (cf.board.status,cf.board.trigger):
+					if fn:
+						run(['sudo','chmod','0666',fn],check=True)
+				os.environ['MMGEN_TEST_SUITE_AUTOSIGN_LIVE'] = '1'
 				omsg(purple('Running autosign test with no LED'))
 				do_autosign_live(opts,mountpoint)
 				omsg(purple("Running autosign test with '--led'"))
 				do_autosign_live(opts,mountpoint,led_opts=['--led'],gen_wallet=False)
 				omsg(purple("Running autosign test with '--stealth-led'"))
 				ret = do_autosign_live(opts,mountpoint,led_opts=['--stealth-led'],gen_wallet=False)
-			else:
-				ret = do_autosign_live(opts,mountpoint)
 		else:
 			mountpoint = self.tmpdir
 			opts = ['--no-insert-check','--mountpoint='+mountpoint,'--coins='+','.join(coins)]
@@ -247,3 +289,9 @@ class TestSuiteAutosignLive(TestSuiteAutosignBTC):
 	cmd_group = (
 		('autosign_live', 'transaction autosigning (BTC,ETH,ETC - test device insertion/removal + LED)'),
 	)
+
+class TestSuiteAutosignLiveSimulate(TestSuiteAutosignBTC):
+	'live autosigning operations with device insertion/removal and LED check in simulated environment'
+	cmd_group = (
+		('autosign_live_simulate', 'transaction autosigning (BTC,ETH,ETC - test device insertion/removal + simulated LED)'),
+	)

test/test_py_d/ts_ethdev.py

@@ -211,7 +211,8 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		('token_addrgen',        'generating token addresses'),
 		('token_addrimport_badaddr1','importing token addresses (no token address)'),
 		('token_addrimport_badaddr2','importing token addresses (bad token address)'),
-		('token_addrimport',    'importing token addresses'),
+		('token_addrimport',     'importing token addresses'),
+		('token_addrimport_batch','importing token addresses (dummy batch mode)'),
 
 		('bal7',                'the {} balance'.format(g.coin)),
 		('token_bal1',          'the {} balance and token balance'.format(g.coin)),
@@ -351,7 +352,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 	def addrimport(self,ext='21-23]{}.addrs',expect='9/9',add_args=[],bad_input=False):
 		ext = ext.format('-α' if g.debug_utf8 else '')
 		fn = self.get_file_with_ext(ext,no_dot=True,delete=False)
-		t = self.spawn('mmgen-addrimport', self.eth_args[1:] + add_args + [fn])
+		t = self.spawn('mmgen-addrimport', self.eth_args[1:-1] + add_args + [fn])
 		if bad_input:
 			t.read()
 			return t
@@ -507,9 +508,11 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		t = self.spawn('mmgen-tool', self.eth_args + ['--token=mm1','twview','wide=1'])
 		for b in token_bals[n]:
 			addr,_amt1,_amt2,adj = b if len(b) == 4 else b + (False,)
-			if adj and g.coin == 'ETC': _amt2 = str(Decimal(_amt2) + Decimal(adj[1]) * self.bal_corr)
+			if adj and g.coin == 'ETC':
+				_amt2 = str(Decimal(_amt2) + Decimal(adj[1]) * self.bal_corr)
 			pat = r'{}\s+{}\s+{}\s'.format(addr,_amt1.replace('.',r'\.'),_amt2.replace('.',r'\.'))
 			t.expect(pat,regex=True)
+		t.expect('Total MM1:')
 		t.read()
 		return t
 
@@ -577,12 +580,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		token_data = { 'name':'MMGen Token 2', 'symbol':'MM2', 'supply':10**18, 'decimals':10 }
 		return self.token_compile(token_data)
 
-	def _rpc_init(self):
-		g.proto.rpc_port = self.rpc_port
-		rpc_init()
-
-	def token_deploy(self,num,key,gas,mmgen_cmd='txdo',tx_fee='8G'):
-		self._rpc_init()
+	async def token_deploy(self,num,key,gas,mmgen_cmd='txdo',tx_fee='8G'):
 		keyfile = joinpath(self.tmpdir,parity_key_fn)
 		fn = joinpath(self.tmpdir,'mm'+str(num),key+'.bin')
 		os.environ['MMGEN_BOGUS_SEND'] = ''
@@ -609,62 +607,63 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 			bogus_send=False)
 		addr = t.expect_getend('Contract address: ')
 		from mmgen.altcoins.eth.tx import EthereumMMGenTX as etx
-		assert etx.get_exec_status(txid,True) != 0,(
+		assert (await etx.get_exec_status(txid,True)) != 0,(
 			"Contract '{}:{}' failed to execute. Aborting".format(num,key))
 		if key == 'Token':
 			self.write_to_tmpfile('token_addr{}'.format(num),addr+'\n')
 			imsg('\nToken MM{} deployed!'.format(num))
 		return t
 
-	def token_deploy1a(self): return self.token_deploy(num=1,key='SafeMath',gas=200000)
-	def token_deploy1b(self): return self.token_deploy(num=1,key='Owned',gas=250000)
-	def token_deploy1c(self): return self.token_deploy(num=1,key='Token',gas=1100000,tx_fee='7G')
+	async def token_deploy1a(self): return await self.token_deploy(num=1,key='SafeMath',gas=200000)
+	async def token_deploy1b(self): return await self.token_deploy(num=1,key='Owned',gas=250000)
+	async def token_deploy1c(self): return await self.token_deploy(num=1,key='Token',gas=1100000,tx_fee='7G')
 
 	def tx_status2(self):
 		return self.tx_status(ext=g.coin+'[0,7000]{}.sigtx',expect_str='successfully executed')
 
 	def bal6(self): return self.bal5()
 
-	def token_deploy2a(self): return self.token_deploy(num=2,key='SafeMath',gas=200000)
-	def token_deploy2b(self): return self.token_deploy(num=2,key='Owned',gas=250000)
-	def token_deploy2c(self): return self.token_deploy(num=2,key='Token',gas=1100000)
+	async def token_deploy2a(self): return await self.token_deploy(num=2,key='SafeMath',gas=200000)
+	async def token_deploy2b(self): return await self.token_deploy(num=2,key='Owned',gas=250000)
+	async def token_deploy2c(self): return await self.token_deploy(num=2,key='Token',gas=1100000)
 
-	def contract_deploy(self): # test create,sign,send
-		return self.token_deploy(num=2,key='SafeMath',gas=1100000,mmgen_cmd='txcreate')
+	async def contract_deploy(self): # test create,sign,send
+		return await self.token_deploy(num=2,key='SafeMath',gas=1100000,mmgen_cmd='txcreate')
 
-	def token_transfer_ops(self,op,amt=1000):
+	async def token_transfer_ops(self,op,amt=1000):
 		self.spawn('',msg_only=True)
 		sid = dfl_sid
 		from mmgen.tool import MMGenToolCmdWallet
 		usr_mmaddrs = ['{}:E:{}'.format(sid,i) for i in (11,21)]
 		usr_addrs = [MMGenToolCmdWallet().gen_addr(addr,dfl_words_file) for addr in usr_mmaddrs]
-		self._rpc_init()
 
-		from mmgen.altcoins.eth.contract import Token
+		from mmgen.altcoins.eth.contract import TokenResolve
 		from mmgen.altcoins.eth.tx import EthereumMMGenTX as etx
-		def do_transfer():
+		async def do_transfer():
 			for i in range(2):
-				tk = Token(self.read_from_tmpfile('token_addr{}'.format(i+1)).strip())
-				imsg_r('\n'+tk.info())
-				imsg('dev token balance (pre-send): {}'.format(tk.balance(dfl_addr)))
+				tk = await TokenResolve(self.read_from_tmpfile('token_addr{}'.format(i+1)).strip())
+				imsg_r('\n' + await tk.info())
+				imsg('dev token balance (pre-send): {}'.format(await tk.get_balance(dfl_addr)))
 				imsg('Sending {} {} to address {} ({})'.format(amt,g.coin,usr_addrs[i],usr_mmaddrs[i]))
 				from mmgen.obj import ETHAmt
-				txid = tk.transfer( dfl_addr, usr_addrs[i], amt, dfl_privkey,
+				txid = await tk.transfer( dfl_addr, usr_addrs[i], amt, dfl_privkey,
 									start_gas = ETHAmt(60000,'wei'),
 									gasPrice  = ETHAmt(8,'Gwei') )
-				assert etx.get_exec_status(txid,True) != 0,'Transfer of token funds failed. Aborting'
+				assert (await etx.get_exec_status(txid,True)) != 0,'Transfer of token funds failed. Aborting'
 
-		def show_bals():
+		async def show_bals():
 			for i in range(2):
-				tk = Token(self.read_from_tmpfile('token_addr{}'.format(i+1)).strip())
-				imsg('Token: {}'.format(tk.symbol()))
-				imsg('dev token balance: {}'.format(tk.balance(dfl_addr)))
+				tk = await TokenResolve(self.read_from_tmpfile(f'token_addr{i+1}').strip())
+				imsg('Token: {}'.format(await tk.get_symbol()))
+				imsg('dev token balance: {}'.format(await tk.get_balance(dfl_addr)))
 				imsg('usr token balance: {} ({} {})'.format(
-						tk.balance(usr_addrs[i]),usr_mmaddrs[i],usr_addrs[i]))
+						await tk.get_balance(usr_addrs[i]),usr_mmaddrs[i],usr_addrs[i]))
 
 		silence()
-		if op == 'show_bals': show_bals()
-		elif op == 'do_transfer': do_transfer()
+		if op == 'show_bals':
+			await show_bals()
+		elif op == 'do_transfer':
+			await do_transfer()
 		end_silence()
 		return 'ok'
 
@@ -689,15 +688,18 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		t.req_exit_val = 2
 		return t
 
-	def token_addrimport(self):
+	def token_addrimport(self,extra_args=[],expect='3/3'):
 		for n,r in ('1','11-13'),('2','21-23'):
 			tk_addr = self.read_from_tmpfile('token_addr'+n).strip()
-			t = self.addrimport(ext='['+r+']{}.addrs',expect='3/3',add_args=['--token='+tk_addr])
+			t = self.addrimport(ext='['+r+']{}.addrs',expect=expect,add_args=['--token='+tk_addr]+extra_args)
 			t.p.wait()
 			ok_msg()
 		t.skip_ok = True
 		return t
 
+	def token_addrimport_batch(self):
+		return self.token_addrimport(extra_args=['--batch'],expect='OK: 3')
+
 	def bal7(self):       return self.bal5()
 	def token_bal1(self): return self.token_bal(n='1')
 

test/test_py_d/ts_main.py

@@ -146,8 +146,10 @@ class TestSuiteMain(TestSuiteBase,TestSuiteShared):
 	def __init__(self,trunner,cfgs,spawn):
 		if g.coin.lower() not in self.networks:
 			return
-		rpc_init()
-		self.lbl_id = ('account','label')['label_api' in g.rpc.caps]
+		from mmgen.rpc import rpc_init
+		g.regtest = False # rpc_init hack
+		self.rpc = run_session(rpc_init())
+		self.lbl_id = ('account','label')['label_api' in self.rpc.caps]
 		if g.coin in ('BTC','BCH','LTC'):
 			self.tx_fee     = {'btc':'0.0001','bch':'0.001','ltc':'0.01'}[g.coin.lower()]
 			self.txbump_fee = {'btc':'123s','bch':'567s','ltc':'12345s'}[g.coin.lower()]

test/test_py_d/ts_regtest.py

@@ -182,6 +182,7 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 		('bob_import_addr',          'importing non-MMGen address with --rescan'),
 		('bob_bal4',                 "Bob's balance (after import with rescan)"),
 		('bob_import_list',          'importing flat address list'),
+		('bob_import_list_rescan',   'importing flat address list with --rescan'),
 		('bob_split2',               "splitting Bob's funds"),
 		('bob_0conf0_getbalance',    "Bob's balance (unconfirmed, minconf=0)"),
 		('bob_0conf1_getbalance',    "Bob's balance (unconfirmed, minconf=1)"),
@@ -735,11 +736,15 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 
 	def bob_import_addr(self):
 		addr = self.read_from_tmpfile('non-mmgen.addrs').split()[0]
-		return self.user_import('bob',['--rescan','--address='+addr])
+		return self.user_import('bob',['--quiet','--address='+addr])
 
 	def bob_import_list(self):
 		addrfile = joinpath(self.tmpdir,'non-mmgen.addrs')
-		return self.user_import('bob',['--addrlist',addrfile])
+		return self.user_import('bob',['--quiet','--addrlist',addrfile])
+
+	def bob_import_list_rescan(self):
+		addrfile = joinpath(self.tmpdir,'non-mmgen.addrs')
+		return self.user_import('bob',['--quiet','--rescan','--addrlist',addrfile])
 
 	def bob_split2(self):
 		addrs = self.read_from_tmpfile('non-mmgen.addrs').split()

test/tooltest2.py

@@ -759,7 +759,10 @@ tests = {
 			'ltc_testnet': [ ( ['test/ref/litecoin/A5A1E0-LTC[1454.64322,1453,tl=1320969600].testnet.rawtx'],
 									None ), ],
 			'eth_mainnet': [ ( ['test/ref/ethereum/88FEFD-ETH[23.45495,40000].rawtx'], None ), ],
-			'eth_testnet': [ ( ['test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.rawtx'], None ), ],
+			'eth_testnet': [ ( [
+				'test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.rawtx',
+				'test/ref/ethereum/B472BD-ETH[23.45495,40000].testnet.sigtx'
+				], None ), ],
 			'mm1_mainnet': [ ( ['test/ref/ethereum/5881D2-MM1[1.23456,50000].rawtx'], None ), ],
 			'mm1_testnet': [ ( ['test/ref/ethereum/6BDB25-MM1[1.23456,50000].testnet.rawtx'], None ), ],
 			'etc_mainnet': [ ( ['test/ref/ethereum_classic/ED3848-ETC[1.2345,40000].rawtx'], None ), ],

test/unit_tests_d/ut_rpc.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""
+test.unit_tests_d.ut_rpc: RPC unit test for the MMGen suite
+"""
+
+from mmgen.common import *
+from mmgen.exception import *
+
+from mmgen.protocol import init_coin,EthereumProtocol
+from mmgen.rpc import MoneroWalletRPCClient
+from mmgen.daemon import CoinDaemon,MoneroWalletDaemon
+
+class unit_tests:
+
+	def btc(self,name,ut):
+
+		async def run_test():
+			c = g.rpc
+			qmsg('  Testing backend {!r}'.format(type(c.backend).__name__))
+			addrs = (
+				('bc1qvmqas4maw7lg9clqu6kqu9zq9cluvlln5hw97q','test address #1'), # deadbeef * 8
+				('bc1qe50rj25cldtskw5huxam335kyshtqtlrf4pt9x','test address #2'), # deadbeef * 7 + deadbeee
+			)
+
+			await c.batch_call('importaddress',addrs,timeout=120)
+			ret = await c.batch_call('getaddressesbylabel',[(l,) for a,l in addrs])
+			assert list(ret[0].keys())[0] == addrs[0][0]
+
+			bh = (await c.call('getblockchaininfo',timeout=300))['bestblockhash']
+			await c.gathered_call('getblock',((bh,),(bh,1)),timeout=300)
+			await c.gathered_call(None,(('getblock',(bh,)),('getblock',(bh,1))),timeout=300)
+
+
+		d = CoinDaemon('btc',test_suite=True)
+		d.remove_datadir()
+		d.start()
+		g.proto.daemon_data_dir = d.datadir # used by BitcoinRPCClient.set_auth() to find the cookie
+		g.rpc_port = d.rpc_port
+
+		for backend in g.autoset_opts['rpc_backend'].choices:
+			run_session(run_test(),backend=backend)
+
+		d.stop()
+
+		if g.platform != 'win':
+
+			qmsg(f'\n  Testing authentication with credentials from bitcoin.conf:')
+			d.remove_datadir()
+			os.makedirs(d.datadir)
+
+			cf = os.path.join(d.datadir,'bitcoin.conf')
+			open(cf,'a').write('\nrpcuser = ut_rpc\nrpcpassword = ut_rpc_passw0rd\n')
+
+			d.add_flag('keep_cfg_file')
+			d.start()
+
+			async def do():
+				assert g.rpc.auth.user == 'ut_rpc', 'user is not ut_rpc!'
+
+			run_session(do())
+			d.stop()
+
+		qmsg('  OK')
+		return True
+
+	def eth(self,name,ut):
+		ed = CoinDaemon('eth',test_suite=True)
+		ed.start()
+		init_coin('eth')
+		g.rpc_port = CoinDaemon('eth',test_suite=True).rpc_port
+
+		async def run_test():
+			qmsg('  Testing backend {!r}'.format(type(g.rpc.backend).__name__))
+			ret = await g.rpc.call('parity_versionInfo',timeout=300)
+			#print(ret)
+
+		for backend in g.autoset_opts['rpc_backend'].choices:
+			run_session(run_test(),backend=backend)
+
+		ed.stop()
+		return True
+
+	def xmr_wallet(self,name,ut):
+
+		async def run():
+			md = CoinDaemon('xmr',test_suite=True)
+			md.start()
+
+			g.monero_wallet_rpc_password = 'passwOrd'
+			mwd = MoneroWalletDaemon(wallet_dir='test/trash',test_suite=True)
+			mwd.start()
+
+			c = MoneroWalletRPCClient(
+				host = g.monero_wallet_rpc_host,
+				port = mwd.rpc_port,
+				user = g.monero_wallet_rpc_user,
+				passwd = g.monero_wallet_rpc_password)
+
+			await c.call('get_version')
+
+			gmsg('OK')
+			mwd.wait = False
+			mwd.stop()
+			md.wait = False
+			md.stop()
+
+		run_session(run(),do_rpc_init=False)
+		return True

test/unit_tests_d/ut_tx_deserialize.py

@@ -3,9 +3,14 @@
 test/unit_tests_d/ut_tx_deserialize: TX deserialization unit test for the MMGen suite
 """
 
-import os
+import os,json
+
 from mmgen.common import *
 from ..include.common import *
+from mmgen.protocol import init_coin
+from mmgen.tx import MMGenTX,DeserializedTX
+from mmgen.rpc import rpc_init
+from mmgen.daemon import CoinDaemon
 
 class unit_test(object):
 
@@ -16,7 +21,7 @@ class unit_test(object):
 
 	def run_test(self,name,ut):
 
-		def test_tx(txhex,desc,n):
+		async def test_tx(txhex,desc,n):
 
 			def has_nonstandard_outputs(outputs):
 				for o in outputs:
@@ -25,7 +30,7 @@ class unit_test(object):
 						return True
 				return False
 
-			d = g.rpc.decoderawtransaction(txhex)
+			d = await g.rpc.call('decoderawtransaction',txhex)
 
 			if has_nonstandard_outputs(d['vout']): return False
 
@@ -86,7 +91,7 @@ class unit_test(object):
 				Msg_r('Testing transactions from {!r}'.format(fn))
 				if not opt.quiet: Msg('')
 
-		def test_core_vectors():
+		async def test_core_vectors():
 			self._get_core_repo_root()
 			fn_b = 'src/test/data/tx_valid.json'
 			fn = os.path.join(self.core_repo_root,fn_b)
@@ -95,38 +100,33 @@ class unit_test(object):
 			n = 1
 			for e in data:
 				if type(e[0]) == list:
-					test_tx(e[1],desc,n)
+					await rpc_init()
+					await test_tx(e[1],desc,n)
 					n += 1
 				else:
 					desc = e[0]
 			Msg('OK')
 
-		def test_mmgen_txs():
+		async def test_mmgen_txs():
 			fns = ( ('btc',False,'test/ref/0B8D5A[15.31789,14,tl=1320969600].rawtx'),
 					('btc',True,'test/ref/0C7115[15.86255,14,tl=1320969600].testnet.rawtx'),
 				#	('bch',False,'test/ref/460D4D-BCH[10.19764,tl=1320969600].rawtx')
 				)
-			from mmgen.protocol import init_coin
-			from mmgen.tx import MMGenTX
-			from mmgen.daemon import CoinDaemon
 			print_info('test/ref/*rawtx','MMGen reference transactions')
 			for n,(coin,tn,fn) in enumerate(fns):
 				init_coin(coin,tn)
 				g.proto.daemon_data_dir = 'test/daemons/' + g.coin.lower()
 				g.rpc_port = CoinDaemon(coin + ('','_tn')[tn],test_suite=True).rpc_port
-				rpc_init(reinit=True)
-				test_tx(MMGenTX(fn).hex,fn,n+1)
+				await rpc_init()
+				await test_tx(MMGenTX(fn).hex,fn,n+1)
 			init_coin('btc',False)
 			g.rpc_port = CoinDaemon('btc',test_suite=True).rpc_port
-			rpc_init(reinit=True)
+			await rpc_init()
 			Msg('OK')
 
-		from mmgen.tx import DeserializedTX
-		import json
-
 		start_test_daemons('btc','btc_tn') # ,'bch')
-		test_mmgen_txs()
-		test_core_vectors()
+		run_session(test_mmgen_txs(),do_rpc_init=False)
+		run_session(test_core_vectors(),do_rpc_init=False)
 		stop_test_daemons('btc','btc_tn') # ,'bch')
 
 		return True