TrackingWallet: balance caching, Parity light client optimizations

- activated for Ethereum only, but framework exists for all coins
- both session caching and persistent caching in the wallet are supported
- network-destined RPC calls are never repeated in a given invocation
- RPC balance lookups can be suppressed entirely with --cached-balances

mmgen/addr.py

@@ -898,10 +898,10 @@ re-import your addresses.
 	def __new__(cls,*args,**kwargs):
 		return MMGenObject.__new__(altcoin_subclass(cls,'tw','AddrData'))
 
-	def __init__(self,source=None):
+	def __init__(self,source=None,wallet=None):
 		self.al_ids = {}
 		if source == 'tw':
-			self.add_tw_data()
+			self.add_tw_data(wallet)
 
 	def seed_ids(self):
 		return list(self.al_ids.keys())
@@ -923,7 +923,7 @@ re-import your addresses.
 		return (list(d.values())[0][0]) if d else None
 
 	@classmethod
-	def get_tw_data(cls):
+	def get_tw_data(cls,wallet=None):
 		vmsg('Getting address data from tracking wallet')
 		if 'label_api' in g.rpch.caps:
 			accts = g.rpch.listlabels()
@@ -933,8 +933,8 @@ re-import your addresses.
 			alists = g.rpch.getaddressesbyaccount([[k] for k in accts],batch=True)
 		return list(zip(accts,alists))
 
-	def add_tw_data(self):
-		d,out,i = self.get_tw_data(),{},0
+	def add_tw_data(self,wallet):
+		d,out,i = self.get_tw_data(wallet),{},0
 		for acct,addr_array in d:
 			l = TwLabel(acct,on_fail='silent')
 			if l and l.mmid.type == 'mmgen':

mmgen/altcoins/eth/contract.py

@@ -41,14 +41,19 @@ def create_method_id(sig): return keccak_256(sig.encode()).hexdigest()[:8]
 
 class Token(MMGenObject): # ERC20
 
+	_decimals = None
+
 	# Test that token is in the blockchain by calling constructor w/o decimals arg
 	def __init__(self,addr,decimals=None):
 		self.addr = TokenAddr(addr)
-		if decimals is None:
-			decimals = self.decimals()
-			if not decimals:
-				raise TokenNotInBlockchain("Token '{}' not in blockchain".format(addr))
-		self.base_unit = Decimal('10') ** -decimals
+		if decimals:
+			self._decimals = decimals
+		else:
+			rpc_init()
+			self.decimals()
+		if not self._decimals:
+			raise TokenNotInBlockchain("Token '{}' not in blockchain".format(addr))
+		self.base_unit = Decimal('10') ** -self._decimals
 
 	@staticmethod
 	def transferdata2sendaddr(data): # online
@@ -70,14 +75,17 @@ class Token(MMGenObject): # ERC20
 	def strip(self,s):
 		return ''.join([chr(b) for b in s if 32 <= b <= 127]).strip()
 
+	# TODO: make these properties
 	def decimals(self):
-			ret = self.do_call('decimals()')
+		if self._decimals == None:
+			res = self.do_call('decimals()')
 			try:
-				a,b = ret[:2],ret[2:]
-				assert a == '0x' and is_hex_str_lc(b)
+				assert res[:2] == '0x'
+				self._decimals = int(res[2:],16)
 			except:
-				"RPC call to decimals() failed (returned '{}')".format(ret)
-			return int(b,16) if b else None
+				msg("RPC call to decimals() failed (returned '{}')".format(res))
+				return None
+		return self._decimals
 
 	def name(self):
 		return self.strip(bytes.fromhex(self.do_call('name()')[2:]))
@@ -105,10 +113,8 @@ class Token(MMGenObject): # ERC20
 		amt_arg = '{:064x}'.format(int(amt//self.base_unit))
 		return create_method_id(method_sig) + from_arg + to_arg + amt_arg
 
-	def txcreate(   self,from_addr,to_addr,amt,start_gas,gasPrice,nonce=None,
+	def make_tx_in( self,from_addr,to_addr,amt,start_gas,gasPrice,nonce,
 					method_sig='transfer(address,uint256)',from_addr2=None):
-		if nonce is None:
-			nonce = int(g.rpch.parity_nextNonce('0x'+from_addr),16)
 		data = self.create_data(to_addr,amt,method_sig=method_sig,from_addr=from_addr2)
 		return {'to':       bytes.fromhex(self.addr),
 				'startgas': start_gas.toWei(),
@@ -145,10 +151,11 @@ class Token(MMGenObject): # ERC20
 					method_sig='transfer(address,uint256)',
 					from_addr2=None,
 					return_data=False):
-		tx_in = self.txcreate(  from_addr,to_addr,amt,
-								start_gas,gasPrice,
-								nonce=None,
-								method_sig=method_sig,
-								from_addr2=from_addr2)
+		tx_in = self.make_tx_in(
+					from_addr,to_addr,amt,
+					start_gas,gasPrice,
+					nonce = int(g.rpch.parity_nextNonce('0x'+from_addr),16),
+					method_sig = method_sig,
+					from_addr2 = from_addr2 )
 		(hex_tx,coin_txid) = self.txsign(tx_in,key,from_addr)
 		return self.txsend(hex_tx)

mmgen/altcoins/eth/tw.py

@@ -20,157 +20,238 @@
 altcoins.eth.tw: Ethereum tracking wallet and related classes for the MMGen suite
 """
 
-import json
 from mmgen.common import *
-from mmgen.obj import ETHAmt,TwMMGenID,TwComment,TwLabel
+from mmgen.obj import ETHAmt,TwLabel,is_coin_addr,is_mmgen_id
 from mmgen.tw import TrackingWallet,TwAddrList,TwUnspentOutputs
 from mmgen.addr import AddrData
 from .contract import Token
 
 class EthereumTrackingWallet(TrackingWallet):
 
-	desc = 'Ethereum tracking wallet'
 	caps = ()
+	data_key = 'accounts'
+	use_tw_file = True
 
-	data_dir = os.path.join(g.altcoin_data_dir,g.coin.lower(),g.proto.data_subdir)
-	tw_file = os.path.join(data_dir,'tracking-wallet.json')
-
-	def __init__(self,mode='r'):
+	def __init__(self,mode='r',no_rpc=False):
 		TrackingWallet.__init__(self,mode=mode)
-		check_or_create_dir(self.data_dir)
-		try:
-			self.orig_data = get_data_from_file(self.tw_file,quiet=True)
-			self.data = json.loads(self.orig_data)
-		except:
-			try: os.stat(self.tw_file)
-			except:
-				self.orig_data = ''
-				self.data = {'coin':g.coin,'accounts':{},'tokens':{}}
-			else: die(2,"File '{}' exists but does not contain valid json data".format(self.tw_file))
-		else:
-			self.upgrade_wallet_maybe()
-			m = 'Tracking wallet coin ({}) does not match current coin ({})!'
-			assert self.data['coin'] == g.coin,m.format(self.data['coin'],g.coin)
-			if not 'tokens' in self.data:
-				self.data['tokens'] = {}
-			def conv_types(ad):
-				for v in ad.values():
-					v['mmid'] = TwMMGenID(v['mmid'],on_fail='raise')
-					v['comment'] = TwComment(v['comment'],on_fail='raise')
-			conv_types(self.data['accounts'])
-			for v in self.data['tokens'].values():
-				conv_types(v)
+
+		for v in self.data['tokens'].values():
+			self.conv_types(v)
+
+		if g.token and not is_coin_addr(g.token):
+			ret = self.sym2addr(g.token,no_rpc=no_rpc)
+			if ret: g.token = ret
+
+	def is_in_wallet(self,addr):
+		return addr in self.data_root
+
+	def init_empty(self):
+		self.data = { 'coin': g.coin, 'accounts': {}, 'tokens': {} }
 
 	def upgrade_wallet_maybe(self):
+
+		upgraded = False
+
 		if not 'accounts' in self.data or not 'coin' in self.data:
-			ymsg('Upgrading {}!'.format(self.desc))
+			ymsg('Upgrading {} (v1->v2: accounts field added)'.format(self.desc))
 			if not 'accounts' in self.data:
 				self.data = {}
+				import json
 				self.data['accounts'] = json.loads(self.orig_data)
 			if not 'coin' in self.data:
 				self.data['coin'] = g.coin
-			mode_save = self.mode
-			self.mode = 'w'
-			self.write()
-			self.mode = mode_save
-			self.orig_data = json.dumps(self.data)
+			upgraded = True
+
+		def have_token_params_fields():
+			for k in self.data['tokens']:
+				if 'params' in self.data['tokens'][k]:
+					return True
+
+		def add_token_params_fields():
+			for k in self.data['tokens']:
+				self.data['tokens'][k]['params'] = {}
+
+		if not 'tokens' in self.data:
+			self.data['tokens'] = {}
+			upgraded = True
+
+		if self.data['tokens'] and not have_token_params_fields():
+			ymsg('Upgrading {} (v2->v3: token params fields added)'.format(self.desc))
+			add_token_params_fields()
+			upgraded = True
+
+		if upgraded:
+			self.force_write()
 			msg('{} upgraded successfully!'.format(self.desc))
 
-	def data_root(self): return self.data['accounts']
-	def data_root_desc(self): return 'accounts'
+	# Don't call rpc_init() for Ethereum, because it may create a wallet instance
+	def rpc_init(self): pass
+
+	def rpc_get_balance(self,addr):
+		return ETHAmt(int(g.rpch.eth_getBalance('0x'+addr),16),'wei')
 
 	@write_mode
 	def import_address(self,addr,label,foo):
-		ad = self.data_root()
-		if addr in ad:
-			if not ad[addr]['mmid'] and label.mmid:
+		r = self.data_root
+		if addr in r:
+			if not r[addr]['mmid'] and label.mmid:
 				msg("Warning: MMGen ID '{}' was missing in tracking wallet!".format(label.mmid))
-			elif ad[addr]['mmid'] != label.mmid:
+			elif r[addr]['mmid'] != label.mmid:
 				die(3,"MMGen ID '{}' does not match tracking wallet!".format(label.mmid))
-		ad[addr] = { 'mmid': label.mmid, 'comment': label.comment }
-
-	@write_mode
-	def write(self): # use 'check_data' to check wallet hasn't been altered by another program
-		write_data_to_file( self.tw_file,
-							json.dumps(self.data),'Ethereum tracking wallet data',
-							ask_overwrite=False,ignore_opt_outdir=True,quiet=True,
-							check_data=True,cmp_data=self.orig_data)
-
-	@write_mode
-	def delete_all(self):
-		self.data = {}
-		self.write()
+		r[addr] = { 'mmid': label.mmid, 'comment': label.comment }
 
 	@write_mode
 	def remove_address(self,addr):
-		root = self.data_root()
+		r = self.data_root
 
-		from mmgen.obj import is_coin_addr,is_mmgen_id
 		if is_coin_addr(addr):
 			have_match = lambda k: k == addr
 		elif is_mmgen_id(addr):
-			have_match = lambda k: root[k]['mmid'] == addr
+			have_match = lambda k: r[k]['mmid'] == addr
 		else:
 			die(1,"'{}' is not an Ethereum address or MMGen ID".format(addr))
 
-		for k in root:
+		for k in r:
 			if have_match(k):
 				# return the addr resolved to mmid if possible
-				ret = root[k]['mmid'] if is_mmgen_id(root[k]['mmid']) else addr
-				del root[k]
+				ret = r[k]['mmid'] if is_mmgen_id(r[k]['mmid']) else addr
+				del r[k]
 				self.write()
 				return ret
 		else:
 			m = "Address '{}' not found in '{}' section of tracking wallet"
-			msg(m.format(addr,self.data_root_desc()))
+			msg(m.format(addr,self.data_root_desc))
 			return None
 
-	def is_in_wallet(self,addr):
-		return addr in self.data_root()
-
-	def sorted_list(self):
-		return sorted(
-			[{'addr':x[0],'mmid':x[1]['mmid'],'comment':x[1]['comment']} for x in list(self.data_root().items())],
-			key=lambda x: x['mmid'].sort_key+x['addr'] )
-
-	def mmid_ordered_dict(self):
-		from collections import OrderedDict
-		return OrderedDict([(x['mmid'],{'addr':x['addr'],'comment':x['comment']}) for x in self.sorted_list()])
-
 	@write_mode
 	def set_label(self,coinaddr,lbl):
-		for addr,d in list(self.data_root().items()):
+		for addr,d in list(self.data_root.items()):
 			if addr == coinaddr:
 				d['comment'] = lbl.comment
 				self.write()
 				return None
 		else: # emulate on_fail='return' of RPC library
 			m = "Address '{}' not found in '{}' section of tracking wallet"
-			return ('rpcfail',(None,2,m.format(coinaddr,self.data_root_desc())))
+			return ('rpcfail',(None,2,m.format(coinaddr,self.data_root_desc)))
+
+	def addr2sym(self,req_addr):
+
+		for addr in self.data['tokens']:
+			if addr == req_addr:
+				ret = self.data['tokens'][addr]['params'].get('symbol')
+				if ret: return ret
+				else: break
+
+		self.token_obj = Token(req_addr)
+		ret = self.token_obj.symbol().upper()
+		self.force_set_token_param(req_addr,'symbol',ret)
+		return ret
+
+	def sym2addr(self,sym,no_rpc=False):
+
+		for addr in self.data['tokens']:
+			if self.data['tokens'][addr]['params'].get('symbol') == sym.upper():
+				return addr
+
+		if no_rpc: return None
+
+		for addr in self.data['tokens']:
+			if Token(addr).symbol().upper() == sym.upper():
+				self.force_set_token_param(addr,'symbol',sym.upper())
+				return addr
+
+		return None
+
+	def get_token_param(self,token,param):
+		if token in self.data['tokens']:
+			return self.data['tokens'][token]['params'].get(param)
+		return None
+
+	def force_set_token_param(self,*args,**kwargs):
+		mode_save = self.mode
+		self.mode = 'w'
+		self.set_token_param(*args,**kwargs)
+		self.mode = mode_save
+
+	@write_mode
+	def set_token_param(self,token,param,val):
+		if token in self.data['tokens']:
+			self.data['tokens'][token]['params'][param] = val
 
 class EthereumTokenTrackingWallet(EthereumTrackingWallet):
 
-	def token_is_in_wallet(self,addr):
-		return addr in self.data['tokens']
+	decimals = None
+	symbol = None
+	cur_eth_balances = {}
 
+	def __init__(self,mode='r',no_rpc=False):
+		EthereumTrackingWallet.__init__(self,mode=mode,no_rpc=no_rpc)
+
+		self.desc = 'Ethereum token tracking wallet'
+
+		if not is_coin_addr(g.token):
+			raise UnrecognizedTokenSymbol('Specified token {!r} could not be resolved!'.format(g.token))
+
+		if mode == 'r' and not g.token in self.data['tokens']:
+			raise TokenNotInWallet('Specified token {!r} not in wallet!'.format(g.token))
+
+		self.token = g.token
+
+		if self.token in self.data['tokens']:
+			for k in ('decimals','symbol'):
+				setattr(self,k,self.get_param(k))
+				if getattr(self,k) == None:
+					setattr(self,k,getattr(Token(self.token,self.decimals),k)())
+					if getattr(self,k) != None:
+						self.set_param(k,getattr(self,k))
+						self.write()
+
+	def is_in_wallet(self,addr):
+		return addr in self.data['tokens'][self.token]
+
+	@property
+	def data_root(self):
+		return self.data['tokens'][self.token]
+
+	@property
 	def data_root_desc(self):
-		return 'token ' + Token(g.token).symbol()
+		return 'token ' + Token(self.token,self.decimals).symbol()
 
 	@write_mode
 	def add_token(self,token):
 		msg("Adding token '{}' to tracking wallet.".format(token))
-		self.data['tokens'][token] = {}
+		self.data['tokens'][token] = { 'params': {} }
 
-	def data_root(self): # create the token data root if necessary
-		if g.token not in self.data['tokens']:
-			self.add_token(g.token)
-		return self.data['tokens'][g.token]
+	@write_mode
+	def import_address(self,*args,**kwargs):
+		if self.token not in self.data['tokens']:
+			self.add_token(self.token)
+		EthereumTrackingWallet.import_address(self,*args,**kwargs)
 
-	def sym2addr(self,sym): # online
-		for addr in self.data['tokens']:
-			if Token(addr).symbol().upper() == sym.upper():
-				return addr
-		return None
+	def rpc_get_balance(self,addr):
+		return Token(self.token,self.decimals).balance(addr)
+
+	def get_eth_balance(self,addr,force_rpc=False):
+		cache = self.cur_eth_balances
+		data_root = self.data['accounts']
+		ret = None if force_rpc else self.get_cached_balance(addr,cache,data_root)
+		if ret == None:
+			ret = EthereumTrackingWallet.rpc_get_balance(self,addr)
+			self.cache_balance(addr,ret,cache,data_root)
+		return ret
+
+	def force_set_param(self,*args,**kwargs):
+		mode_save = self.mode
+		self.mode = 'w'
+		self.set_param(*args,**kwargs)
+		self.mode = mode_save
+
+	@write_mode
+	def set_param(self,param,val):
+		self.data['tokens'][self.token]['params'][param] = val
+
+	def get_param(self,param):
+		return self.data['tokens'][self.token]['params'].get(param)
 
 # No unspent outputs with Ethereum, but naming must be consistent
 class EthereumTwUnspentOutputs(TwUnspentOutputs):
@@ -186,29 +267,33 @@ class EthereumTwUnspentOutputs(TwUnspentOutputs):
 Sort options:    [a]mount, a[d]dress, [r]everse, [M]mgen addr
 Display options: show [m]mgen addr, r[e]draw screen
 Actions:         [q]uit view, [p]rint to file, pager [v]iew, [w]ide view,
-                 add [l]abel, [R]emove address:
+                 add [l]abel, [D]elete address, [R]efresh balance:
 """
 	key_mappings = {
 		'a':'s_amt','d':'s_addr','r':'d_reverse','M':'s_twmmid',
 		'm':'d_mmid','e':'d_redraw',
 		'q':'a_quit','p':'a_print','v':'a_view','w':'a_view_wide',
-		'l':'a_lbl_add','R':'a_addr_delete' }
+		'l':'a_lbl_add','D':'a_addr_delete','R':'a_balance_refresh' }
+
+	def __init__(self,*args,**kwargs):
+		if g.use_cached_balances:
+			self.hdr_fmt += '\n' + yellow('WARNING: Using cached balances. These may be out of date!')
+		TwUnspentOutputs.__init__(self,*args,**kwargs)
 
 	def do_sort(self,key=None,reverse=False):
 		if key == 'txid': return
 		super(EthereumTwUnspentOutputs,self).do_sort(key=key,reverse=reverse)
 
-	def get_addr_bal(self,addr):
-		return ETHAmt(int(g.rpch.eth_getBalance('0x'+addr),16),'wei')
-
 	def get_unspent_rpc(self):
-		rpc_init()
+		wl = self.wallet.sorted_list
+		if self.addrs:
+			wl = [d for d in wl if d['addr'] in self.addrs]
 		return [{
 				'account': TwLabel(d['mmid']+' '+d['comment'],on_fail='raise'),
 				'address': d['addr'],
-				'amount': self.get_addr_bal(d['addr']),
+				'amount': self.wallet.get_balance(d['addr']),
 				'confirmations': 0, # TODO
-				} for d in TrackingWallet().sorted_list()]
+				} for d in wl]
 
 class EthereumTokenTwUnspentOutputs(EthereumTwUnspentOutputs):
 
@@ -218,22 +303,18 @@ class EthereumTokenTwUnspentOutputs(EthereumTwUnspentOutputs):
 
 	def get_display_precision(self): return 10 # truncate precision for narrow display
 
-	def get_addr_bal(self,addr):
-		return Token(g.token).balance(addr)
-
+	# NB: two wallet instances open simultaneously on the same data:
 	def get_unspent_data(self):
 		super(type(self),self).get_unspent_data()
 		for e in self.unspent:
-			e.amt2 = ETHAmt(int(g.rpch.eth_getBalance('0x'+e.addr),16),'wei')
+			e.amt2 = self.wallet.get_eth_balance(e.addr)
 
 class EthereumTwAddrList(TwAddrList):
 
-	def __init__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels):
+	def __init__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None):
 
-		rpc_init()
-		if g.token: self.token = Token(g.token)
-
-		tw_dict = TrackingWallet().mmid_ordered_dict()
+		self.wallet = wallet or TrackingWallet(mode='w')
+		tw_dict = self.wallet.mmid_ordered_dict
 		self.total = g.proto.coin_amt('0')
 
 		from mmgen.obj import CoinAddr
@@ -241,7 +322,7 @@ class EthereumTwAddrList(TwAddrList):
 #			if d['confirmations'] < minconf: continue # cannot get confirmations for eth account
 			label = TwLabel(mmid+' '+d['comment'],on_fail='raise')
 			if usr_addr_list and (label.mmid not in usr_addr_list): continue
-			bal = self.get_addr_balance(d['addr'])
+			bal = self.wallet.get_balance(d['addr'])
 			if bal == 0 and not showempty:
 				if not label.comment: continue
 				if not all_labels: continue
@@ -252,21 +333,19 @@ class EthereumTwAddrList(TwAddrList):
 			self[label.mmid]['amt'] += bal
 			self.total += bal
 
-	def get_addr_balance(self,addr):
-		return ETHAmt(int(g.rpch.eth_getBalance('0x'+addr),16),'wei')
-
-class EthereumTokenTwAddrList(EthereumTwAddrList):
-
-	def get_addr_balance(self,addr):
-		return self.token.balance(addr)
+class EthereumTokenTwAddrList(EthereumTwAddrList): pass
 
 from mmgen.tw import TwGetBalance
 class EthereumTwGetBalance(TwGetBalance):
 
 	fs = '{w:13} {c}\n' # TODO - for now, just suppress display of meaningless data
 
+	def __init__(self,*args,**kwargs):
+		self.wallet = TrackingWallet(mode='w')
+		TwGetBalance.__init__(self,*args,**kwargs)
+
 	def create_data(self):
-		data = TrackingWallet().mmid_ordered_dict()
+		data = self.wallet.mmid_ordered_dict
 		for d in data:
 			if d.type == 'mmgen':
 				key = d.obj.sid
@@ -276,25 +355,19 @@ class EthereumTwGetBalance(TwGetBalance):
 				key = 'Non-MMGen'
 
 			conf_level = 2 # TODO
-			amt = self.get_addr_balance(data[d]['addr'])
+			amt = self.wallet.get_balance(data[d]['addr'])
 
 			self.data['TOTAL'][conf_level] += amt
 			self.data[key][conf_level] += amt
 
-	def get_addr_balance(self,addr):
-		return ETHAmt(int(g.rpch.eth_getBalance('0x'+addr),16),'wei')
-
-class EthereumTokenTwGetBalance(EthereumTwGetBalance):
-
-	def get_addr_balance(self,addr):
-		return Token(g.token).balance(addr)
+class EthereumTokenTwGetBalance(EthereumTwGetBalance): pass
 
 class EthereumAddrData(AddrData):
 
 	@classmethod
-	def get_tw_data(cls):
+	def get_tw_data(cls,wallet=None):
 		vmsg('Getting address data from tracking wallet')
-		tw = TrackingWallet().mmid_ordered_dict()
+		tw = (wallet or TrackingWallet()).mmid_ordered_dict
 		# emulate the output of RPC 'listaccounts' and 'getaddressesbyaccount'
 		return [(mmid+' '+d['comment'],[d['addr']]) for mmid,d in list(tw.items())]
 

mmgen/altcoins/eth/tx.py

@@ -27,7 +27,8 @@ from mmgen.obj import *
 from mmgen.tx import MMGenTX,MMGenBumpTX,MMGenSplitTX
 
 class EthereumMMGenTX(MMGenTX):
-	desc   = 'Ethereum transaction'
+	desc = 'Ethereum transaction'
+	contract_desc = 'contract'
 	tx_gas = ETHAmt(21000,'wei')    # an approximate number, used for fee estimation purposes
 	start_gas = ETHAmt(21000,'wei') # the actual startgas amt used in the transaction
 									# for simple sends with no data, tx_gas = start_gas = 21000
@@ -39,6 +40,7 @@ class EthereumMMGenTX(MMGenTX):
 	txview_hdr_fs_short = 'TX {i} ({a} {c}) UTC={t} Sig={s} Locktime={l}\n'
 	txview_ftr_fs = 'Total in account: {i} {d}\nTotal to spend:   {o} {d}\nTX fee:           {a} {c}{r}\n'
 	txview_ftr_fs_short = 'In {i} {d} - Out {o} {d}\nFee {a} {c}{r}\n'
+	fmt_keys = ('from','to','amt','nonce')
 	usr_fee_prompt = 'Enter transaction fee or gas price: '
 	fn_fee_unit = 'Mwei'
 	usr_rel_fee = None # not in MMGenTX
@@ -56,10 +58,6 @@ class EthereumMMGenTX(MMGenTX):
 			self.usr_contract_data = HexStr(open(opt.contract_data).read().strip())
 			self.disable_fee_check = True
 
-	@classmethod
-	def get_receipt(cls,txid):
-		return g.rpch.eth_getTransactionReceipt('0x'+txid)
-
 	@classmethod
 	def get_exec_status(cls,txid,silent=False):
 		d = g.rpch.eth_getTransactionReceipt('0x'+txid)
@@ -74,8 +72,7 @@ class EthereumMMGenTX(MMGenTX):
 		return self.fee
 
 	def check_fee(self):
-		if self.disable_fee_check: return
-		assert self.fee <= g.proto.max_tx_fee
+		assert self.disable_fee_check or (self.fee <= g.proto.max_tx_fee)
 
 	def get_hex_locktime(self): return None # TODO
 
@@ -87,32 +84,31 @@ class EthereumMMGenTX(MMGenTX):
 			return True
 		return False
 
-	# hex data if signed, json if unsigned: see create_raw()
+	# hex data if signed, json if unsigned
 	def check_txfile_hex_data(self):
 		if self.check_sigs():
-
 			from .pyethereum.transactions import Transaction
-
 			from . import rlp
 			etx = rlp.decode(bytes.fromhex(self.hex),Transaction)
 			d = etx.to_dict() # ==> hex values have '0x' prefix, 0 is '0x'
 			for k in ('sender','to','data'):
 				if k in d: d[k] = d[k].replace('0x','',1)
 			o = {   'from':     CoinAddr(d['sender']),
-					'to':       CoinAddr(d['to']) if d['to'] else Str(''),
+					'to':       CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is token address
 					'amt':      ETHAmt(d['value'],'wei'),
 					'gasPrice': ETHAmt(d['gasprice'],'wei'),
 					'startGas': ETHAmt(d['startgas'],'wei'),
 					'nonce':    ETHNonce(d['nonce']),
 					'data':     HexStr(d['data']) }
-			if o['data'] and not o['to']:
-				self.token_addr = TokenAddr(etx.creates.hex())
+			if o['data'] and not o['to']: # token- or contract-creating transaction
+				o['token_addr'] = TokenAddr(etx.creates.hex()) # NB: could be a non-token contract address
+				self.disable_fee_check = True
 			txid = CoinTxID(etx.hash.hex())
 			assert txid == self.coin_txid,"txid in tx.hex doesn't match value in MMGen transaction file"
 		else:
 			d = json.loads(self.hex)
 			o = {   'from':     CoinAddr(d['from']),
-					'to':       CoinAddr(d['to']) if d['to'] else Str(''),
+					'to':       CoinAddr(d['to']) if d['to'] else Str(''), # NB: for token, 'to' is sendto address
 					'amt':      ETHAmt(d['amt']),
 					'gasPrice': ETHAmt(d['gasPrice']),
 					'startGas': ETHAmt(d['startGas']),
@@ -120,8 +116,6 @@ class EthereumMMGenTX(MMGenTX):
 					'chainId':  Int(d['chainId']),
 					'data':     HexStr(d['data']) }
 		self.tx_gas = o['startGas'] # approximate, but better than nothing
-		self.usr_contract_data = o['data']
-		if o['data'] and not o['to']: self.disable_fee_check = True
 		self.fee = self.fee_rel2abs(o['gasPrice'].toWei())
 		self.txobj = o
 		return d # 'token_addr','decimals' required by subclass
@@ -129,7 +123,7 @@ class EthereumMMGenTX(MMGenTX):
 	def get_nonce(self):
 		return ETHNonce(int(g.rpch.parity_nextNonce('0x'+self.inputs[0].addr),16))
 
-	def make_txobj(self): # create_raw
+	def make_txobj(self): # called by create_raw()
 		chain_id_method = ('parity_chainId','eth_chainId')['eth_chainId' in g.rpch.caps]
 		self.txobj = {
 			'from': self.inputs[0].addr,
@@ -147,12 +141,12 @@ class EthereumMMGenTX(MMGenTX):
 	# thus removing an attack vector
 	def create_raw(self):
 		assert len(self.inputs) == 1,'Transaction has more than one input!'
-		o_ok = (0,1) if self.usr_contract_data else (1,)
 		o_num = len(self.outputs)
-		assert o_num in o_ok,'Transaction has invalid number of outputs!'.format(o_num)
+		o_ok = 0 if self.usr_contract_data else 1
+		assert o_num == o_ok,'Transaction has {} output{} (should have {})'.format(o_num,suf(o_num),o_ok)
 		self.make_txobj()
-		ol = {k: (v.decode() if issubclass(type(v),bytes) else str(v)) for k,v in self.txobj.items()}
-		self.hex = json.dumps(ol)
+		odict = { k: str(v) for k,v in self.txobj.items() if k != 'token_to' }
+		self.hex = json.dumps(odict)
 		self.update_txid()
 
 	def del_output(self,idx): pass
@@ -221,9 +215,7 @@ class EthereumMMGenTX(MMGenTX):
 		abs_fee = self.process_fee_spec(tx_fee,None,on_fail='return')
 		if abs_fee == False:
 			return False
-		elif self.disable_fee_check:
-			return abs_fee
-		elif abs_fee > g.proto.max_tx_fee:
+		elif not self.disable_fee_check and (abs_fee > g.proto.max_tx_fee):
 			m = '{} {c}: {} fee too large (maximum fee: {} {c})'
 			msg(m.format(abs_fee.hl(),desc,g.proto.max_tx_fee.hl(),c=g.coin))
 			return False
@@ -240,9 +232,9 @@ class EthereumMMGenTX(MMGenTX):
 
 	def format_view_body(self,blockcount,nonmm_str,max_mmwid,enl,terse,sort):
 		m = {}
-		for k in ('in','out'):
-			if len(getattr(self,k+'puts')):
-				m[k] = getattr(self,k+'puts')[0].mmid if len(getattr(self,k+'puts')) else ''
+		for k in ('inputs','outputs'):
+			if len(getattr(self,k)):
+				m[k] = getattr(self,k)[0].mmid if len(getattr(self,k)) else ''
 				m[k] = ' ' + m[k].hl() if m[k] else ' ' + MMGenID.hlc(nonmm_str)
 		fs = """From:      {}{f_mmid}
 				To:        {}{t_mmid}
@@ -252,50 +244,56 @@ class EthereumMMGenTX(MMGenTX):
 				Nonce:     {}
 				Data:      {d}
 				\n""".replace('\t','')
-		keys = ('from',('to','token_to')['token_to' in self.txobj],'amt','nonce')
 		ld = len(self.txobj['data'])
-		return fs.format(   *((self.txobj[k] if self.txobj[k] != '' else Str('None')).hl() for k in keys),
+		return fs.format(   *((self.txobj[k] if self.txobj[k] != '' else Str('None')).hl() for k in self.fmt_keys),
 							d='{}... ({} bytes)'.format(self.txobj['data'][:40],ld//2) if ld else Str('None'),
 							c=g.dcoin if len(self.outputs) else '',
 							g=yellow(str(self.txobj['gasPrice'].to_unit('Gwei',show_decimal=True))),
 							G=yellow(str(self.txobj['startGas'].toKwei())),
-							t_mmid=m['out'] if len(self.outputs) else '',
-							f_mmid=m['in'])
+							t_mmid=m['outputs'] if len(self.outputs) else '',
+							f_mmid=m['inputs'])
 
 	def format_view_abs_fee(self):
 		fee = self.fee_rel2abs(self.txobj['gasPrice'].toWei())
-		note = ' (max)' if self.usr_contract_data else ''
+		note = ' (max)' if self.txobj['data'] else ''
 		return fee.hl() + note
 
 	def format_view_rel_fee(self,terse): return ''
 	def format_view_verbose_footer(self): return '' # TODO
 
-	def set_g_token(self):
-		die(2,"Transaction object mismatch.  Have you forgotten to include the '--token' option?")
+	def resolve_g_token_from_tx_file(self):
+		die(2,"The '--token' option must be specified for token transaction files")
 
 	def final_inputs_ok_msg(self,change_amt):
 		m = "Transaction leaves {} {} in the sender's account"
 		chg = '0' if (self.outputs and self.outputs[0].is_chg) else change_amt
 		return m.format(ETHAmt(chg).hl(),g.coin)
 
-	def do_sign(self,d,wif,tx_num_str):
-		d_in = {'to':       bytes.fromhex(d['to']),
-				'startgas': d['startGas'].toWei(),
-				'gasprice': d['gasPrice'].toWei(),
-				'value':    d['amt'].toWei() if d['amt'] else 0,
-				'nonce':    d['nonce'],
-				'data':     bytes.fromhex(d['data'])}
+	def do_sign(self,wif,tx_num_str):
+		o = self.txobj
+		o_conv = {
+			'to':       bytes.fromhex(o['to']),
+			'startgas': o['startGas'].toWei(),
+			'gasprice': o['gasPrice'].toWei(),
+			'value':    o['amt'].toWei() if o['amt'] else 0,
+			'nonce':    o['nonce'],
+			'data':     bytes.fromhex(o['data']) }
 
 		from .pyethereum.transactions import Transaction
-
-		etx = Transaction(**d_in).sign(wif,d['chainId'])
-		assert etx.sender.hex() == d['from'],(
+		etx = Transaction(**o_conv).sign(wif,o['chainId'])
+		assert etx.sender.hex() == o['from'],(
 			'Sender address recovered from signature does not match true sender')
+
 		from . import rlp
 		self.hex = rlp.encode(etx).hex()
 		self.coin_txid = CoinTxID(etx.hash.hex())
-		if d['data']:
-			self.token_addr = TokenAddr(etx.creates.hex())
+
+		if o['data']:
+			if o['to']:
+				assert self.txobj['token_addr'] == TokenAddr(etx.creates.hex()),'Token address mismatch'
+			else: # token- or contract-creating transaction
+				self.txobj['token_addr'] = TokenAddr(etx.creates.hex())
+
 		assert self.check_sigs(),'Signature check failed'
 
 	def sign(self,tx_num_str,keys): # return True or False; don't exit or raise exception
@@ -310,42 +308,47 @@ class EthereumMMGenTX(MMGenTX):
 		msg_r('Signing transaction{}...'.format(tx_num_str))
 
 		try:
-			self.do_sign(self.txobj,keys[0].sec.wif,tx_num_str)
+			self.do_sign(keys[0].sec.wif,tx_num_str)
 			msg('OK')
 			return True
 		except Exception as e:
+			m = "{!r}: transaction signing failed!"
+			msg(m.format(e.args[0]))
 			if g.traceback:
 				import traceback
 				ymsg('\n'+''.join(traceback.format_exception(*sys.exc_info())))
-			m = "{!r}: transaction signing failed!"
-			msg(m.format(e.args[0]))
 			return False
 
-	def is_in_mempool(self):
-		return '0x'+self.coin_txid in [x['hash'] for x in g.rpch.parity_pendingTransactions()]
-
-	def is_in_wallet(self):
-		d = g.rpch.eth_getTransactionReceipt('0x'+self.coin_txid)
-		if d and 'blockNumber' in d and d['blockNumber'] is not None:
-			return 1 + int(g.rpch.eth_blockNumber(),16) - int(d['blockNumber'],16)
-		return False
-
 	def get_status(self,status=False):
-		if self.is_in_mempool():
+
+		class r(object): pass
+
+		def is_in_mempool():
+			if not 'full_node' in g.rpch.caps:
+				return False
+			return '0x'+self.coin_txid in [x['hash'] for x in g.rpch.parity_pendingTransactions()]
+
+		def is_in_wallet():
+			d = g.rpch.eth_getTransactionReceipt('0x'+self.coin_txid)
+			if d and 'blockNumber' in d and d['blockNumber'] is not None:
+				r.confs = 1 + int(g.rpch.eth_blockNumber(),16) - int(d['blockNumber'],16)
+				r.exec_status = int(d['status'],16)
+				return True
+			return False
+
+		if is_in_mempool():
 			msg('Transaction is in mempool' if status else 'Warning: transaction is in mempool!')
 			return
 
-		confs = self.is_in_wallet()
-		if confs is not False:
-			if self.usr_contract_data:
-				exec_status = type(self).get_exec_status(self.coin_txid)
-				if exec_status == 0:
-					msg('Contract failed to execute!')
-				else:
-					msg('Contract successfully executed with status {}'.format(exec_status))
-			die(0,'Transaction has {} confirmation{}'.format(confs,suf(confs,'s')))
-
 		if status:
+			if is_in_wallet():
+				if self.txobj['data']:
+					cd = capfirst(self.contract_desc)
+					if r.exec_status == 0:
+						msg('{} failed to execute!'.format(cd))
+					else:
+						msg('{} successfully executed with status {}'.format(cd,r.exec_status))
+				die(0,'Transaction has {} confirmation{}'.format(r.confs,suf(r.confs)))
 			die(1,'Transaction is neither in mempool nor blockchain!')
 
 	def send(self,prompt_user=True,exit_on_fail=False):
@@ -357,7 +360,7 @@ class EthereumMMGenTX(MMGenTX):
 
 		fee = self.fee_rel2abs(self.txobj['gasPrice'].toWei())
 
-		if not self.disable_fee_check and fee > g.proto.max_tx_fee:
+		if not self.disable_fee_check and (fee > g.proto.max_tx_fee):
 			die(2,'Transaction fee ({}) greater than {} max_tx_fee ({} {})!'.format(
 				fee,g.proto.name.capitalize(),g.proto.max_tx_fee,g.coin))
 
@@ -383,91 +386,123 @@ class EthereumMMGenTX(MMGenTX):
 			self.add_blockcount()
 			return True
 
+	def get_cmdline_input_addrs(self):
+		ret = []
+		if opt.inputs:
+			from mmgen.tw import TrackingWallet
+			r = TrackingWallet().data_root # must create new instance here
+			m = 'Address {!r} not in tracking wallet'
+			for i in opt.inputs.split(','):
+				if is_mmgen_id(i):
+					for addr in r:
+						if r[addr]['mmid'] == i:
+							ret.append(addr)
+							break
+					else:
+						raise UserAddressNotInWallet(m.format(i))
+				elif is_coin_addr(i):
+					if not i in r:
+						raise UserAddressNotInWallet(m.format(i))
+					ret.append(i)
+				else:
+					die(1,"'{}': not an MMGen ID or coin address".format(i))
+		return ret
+
+	def print_contract_addr(self):
+		if 'token_addr' in self.txobj:
+			msg('Contract address: {}'.format(self.txobj['token_addr'].hl()))
+
 class EthereumTokenMMGenTX(EthereumMMGenTX):
-	desc   = 'Ethereum token transaction'
+	desc = 'Ethereum token transaction'
+	contract_desc = 'token contract'
 	tx_gas = ETHAmt(52000,'wei')
 	start_gas = ETHAmt(60000,'wei')
+	fmt_keys = ('from','token_to','amt','nonce')
 	fee_is_approximate = True
 
+	def __init__(self,*args,**kwargs):
+		if not kwargs.get('offline'):
+			from mmgen.tw import TrackingWallet
+			self.decimals = TrackingWallet().get_param('decimals')
+			from .contract import Token
+			self.token_obj = Token(g.token,self.decimals)
+		EthereumMMGenTX.__init__(self,*args,**kwargs)
+
 	def update_change_output(self,change_amt):
 		if self.outputs[0].is_chg:
 			self.update_output_amt(0,self.inputs[0].amt)
 
-	def check_sufficient_funds(self,inputs_sum,sel_unspent):
-		eth_bal = ETHAmt(int(g.rpch.eth_getBalance('0x'+sel_unspent[0].addr),16),'wei')
+	# token transaction, so check both eth and token balances
+	# TODO: add test with insufficient funds
+	def precheck_sufficient_funds(self,inputs_sum,sel_unspent):
+		eth_bal = self.twuo.wallet.get_eth_balance(sel_unspent[0].addr)
 		if eth_bal == 0: # we don't know the fee yet
 			msg('This account has no ether to pay for the transaction fee!')
 			return False
-		if self.send_amt > inputs_sum:
-			msg(self.msg_low_coin.format(self.send_amt-inputs_sum,g.dcoin))
-			return False
-		return True
+		return super().precheck_sufficient_funds(inputs_sum,sel_unspent)
 
 	def final_inputs_ok_msg(self,change_amt):
+		token_bal   = ( ETHAmt('0') if self.outputs[0].is_chg else
+						self.inputs[0].amt - self.outputs[0].amt )
 		m = "Transaction leaves ≈{} {} and {} {} in the sender's account"
-		if self.outputs[0].is_chg:
-			send_acct_tbal = '0'
-		else:
-			from .contract import Token
-			send_acct_tbal = Token(g.token).balance(self.inputs[0].addr) - self.outputs[0].amt
-		return m.format(ETHAmt(change_amt).hl(),g.coin,ETHAmt(send_acct_tbal).hl(),g.dcoin)
+		return m.format( change_amt.hl(), g.coin, token_bal.hl(), g.dcoin )
 
 	def get_change_amt(self): # here we know the fee
-		eth_bal = ETHAmt(int(g.rpch.eth_getBalance('0x'+self.inputs[0].addr),16),'wei')
+		eth_bal = self.twuo.wallet.get_eth_balance(self.inputs[0].addr)
 		return eth_bal - self.fee
 
-	def set_g_token(self):
+	def resolve_g_token_from_tx_file(self):
 		g.dcoin = self.dcoin
 		if is_hex_str(self.hex): return # for txsend we can leave g.token uninitialized
 		d = json.loads(self.hex)
 		if g.token.upper() == self.dcoin:
 			g.token = d['token_addr']
 		elif g.token != d['token_addr']:
-			m1 = "'{p}': invalid --token parameter for {t} Ethereum token transaction file\n"
+			m1 = "'{p}': invalid --token parameter for {t} {n} token transaction file\n"
 			m2 = "Please use '--token={t}'"
-			die(1,(m1+m2).format(p=g.token,t=self.dcoin))
+			die(1,(m1+m2).format(p=g.token,t=self.dcoin,n=capfirst(g.proto.name)))
 
-	def make_txobj(self):
+	def make_txobj(self): # called by create_raw()
 		super(EthereumTokenMMGenTX,self).make_txobj()
-		from .contract import Token
-		t = Token(g.token)
-		o = t.txcreate( self.inputs[0].addr,
-						self.outputs[0].addr,
-						(self.inputs[0].amt if self.outputs[0].is_chg else self.outputs[0].amt),
-						self.start_gas,
-						self.usr_rel_fee or self.fee_abs2rel(self.fee,to_unit='eth'))
-		self.txobj['token_addr'] = self.token_addr = t.addr
-		self.txobj['decimals']   = t.decimals()
+		t = self.token_obj
+		o = self.txobj
+		o['token_addr'] = t.addr
+		o['decimals'] = t.decimals()
+		o['token_to'] = o['to']
+		o['data'] = t.create_data(o['token_to'],o['amt'])
 
 	def check_txfile_hex_data(self):
 		d = super(EthereumTokenMMGenTX,self).check_txfile_hex_data()
 		o = self.txobj
-		from .contract import Token
-		if self.check_sigs(): # online, from rlp
-			rpc_init()
+
+		if self.check_sigs(): # online, from rlp and wallet
 			o['token_addr'] = TokenAddr(o['to'])
-			o['amt']        = Token(o['token_addr']).transferdata2amt(o['data'])
-		else:                # offline, from json
+			o['decimals'] = self.decimals
+		else:                 # offline, from json
 			o['token_addr'] = TokenAddr(d['token_addr'])
-			o['decimals']   = Int(d['decimals'])
-			t = Token(o['token_addr'],o['decimals'])
-			self.usr_contract_data = o['data'] = t.create_data(o['to'],o['amt'])
+			o['decimals'] = Int(d['decimals'])
+
+		from .contract import Token
+		t = self.token_obj = Token(o['token_addr'],o['decimals'])
+
+		if self.check_sigs(): # online, from rlp - 'amt' was eth amt, now token amt
+			o['amt'] = t.transferdata2amt(o['data'])
+		else:                 # offline, from json - 'amt' is token amt
+			o['data'] = t.create_data(o['to'],o['amt'])
+
+		o['token_to'] = type(t).transferdata2sendaddr(o['data'])
 
 	def format_view_body(self,*args,**kwargs):
-		if self.usr_contract_data:
-			from .contract import Token
-			self.txobj['token_to'] = Token.transferdata2sendaddr(self.usr_contract_data)
 		return 'Token:     {d} {c}\n{r}'.format(
 			d=self.txobj['token_addr'].hl(),
 			c=blue('(' + g.dcoin + ')'),
 			r=super(EthereumTokenMMGenTX,self).format_view_body(*args,**kwargs))
 
-	def do_sign(self,d,wif,tx_num_str):
-		from .contract import Token
-		d = self.txobj
-		t = Token(d['token_addr'],decimals=d['decimals'])
-		tx_in = t.txcreate(d['from'],d['to'],d['amt'],self.start_gas,d['gasPrice'],nonce=d['nonce'])
-		(self.hex,self.coin_txid) = t.txsign(tx_in,wif,d['from'],chain_id=d['chainId'])
+	def do_sign(self,wif,tx_num_str):
+		o = self.txobj
+		t = self.token_obj
+		tx_in = t.make_tx_in(o['from'],o['to'],o['amt'],self.start_gas,o['gasPrice'],nonce=o['nonce'])
+		(self.hex,self.coin_txid) = t.txsign(tx_in,wif,o['from'],chain_id=o['chainId'])
 		assert self.check_sigs(),'Signature check failed'
 
 class EthereumMMGenBumpTX(EthereumMMGenTX,MMGenBumpTX):

mmgen/exception.py

@@ -38,6 +38,7 @@ class BadTwComment(Exception):            mmcode = 2
 class RPCFailure(Exception):              mmcode = 3
 class BadTxSizeEstimate(Exception):       mmcode = 3
 class MaxInputSizeExceeded(Exception):    mmcode = 3
+class WalletFileError(Exception):         mmcode = 3
 
 # 4: red hl, 'MMGen Fatal Error' + exception + message
 class BadMMGenTxID(Exception):            mmcode = 4

mmgen/globalvars.py

@@ -98,6 +98,7 @@ class g(object):
 	rpc_password         = ''
 	rpc_fail_on_command  = ''
 	rpch                 = None # global RPC handle
+	use_cached_balances  = False
 
 	# regtest:
 	bob                  = False

mmgen/main_addrimport.py

@@ -24,7 +24,7 @@ import time
 
 from mmgen.common import *
 from mmgen.addr import AddrList,KeyAddrList
-from mmgen.obj import TwLabel
+from mmgen.obj import TwLabel,is_coin_addr
 
 ai_msgs = lambda k: {
 	'rescan': """
@@ -81,12 +81,6 @@ def import_mmgen_list(infile):
 			rdie(2,'Segwit is not active on this chain. Cannot import Segwit addresses')
 	return al
 
-try:
-	rpc_init()
-except UnrecognizedTokenSymbol as e:
-	m = "When importing addresses for a new token, the token must be specified by address, not symbol."
-	raise type(e)('{}\n{}'.format(e.args[0],m))
-
 if len(cmd_args) == 1:
 	infile = cmd_args[0]
 	check_infile(infile)
@@ -111,6 +105,12 @@ err_msg = None
 from mmgen.tw import TrackingWallet
 tw = TrackingWallet(mode='w')
 
+if g.token:
+	if not is_coin_addr(g.token):
+		m = "When importing addresses for a new token, the token must be specified by address, not symbol."
+		raise InvalidTokenAddress('{!r}: invalid token address\n{}'.format(m))
+	sym = tw.addr2sym(g.token) # check for presence in wallet or blockchain; raises exception on failure
+
 if opt.rescan and not 'rescan' in tw.caps:
 	msg("'--rescan' ignored: not supported by {}".format(type(tw).__name__))
 	opt.rescan = False
@@ -127,6 +127,9 @@ def import_address(addr,label,rescan):
 	except Exception as e:
 		global err_msg
 		err_msg = e.args[0]
+	if g.token and not tw.get_token_param(g.token,'symbol'):
+		tw.set_token_param(g.token,'symbol',sym)
+		tw.set_token_param(g.token,'decimals',tw.token_obj.decimals())
 
 w_n_of_m = len(str(al.num_addrs)) * 2 + 2
 w_mmid = 1 if opt.addrlist or opt.address else len(str(max(al.idxs()))) + 13
@@ -183,4 +186,4 @@ if opt.batch:
 	ret = tw.batch_import_address(arg_list)
 	msg('OK: {} addresses imported'.format(len(ret)))
 
-tw.write()
+del tw

mmgen/main_autosign.py

@@ -187,7 +187,7 @@ def sign_tx_file(txfile,signed_txs):
 		g.token = tmp_tx.dcoin
 		g.dcoin = tmp_tx.dcoin or g.coin
 
-		tx = mmgen.tx.MMGenTX(txfile)
+		tx = mmgen.tx.MMGenTX(txfile,offline=True)
 
 		if g.proto.sign_mode == 'daemon':
 			rpc_init(reinit=True)
@@ -200,6 +200,8 @@ def sign_tx_file(txfile,signed_txs):
 			return False
 	except Exception as e:
 		msg('An error occurred: {}'.format(e.args[0]))
+		if g.debug or g.traceback:
+			print_stack_trace('AUTOSIGN {}'.format(txfile))
 		return False
 	except:
 		return False

mmgen/main_split.py

@@ -87,6 +87,8 @@ transaction reconfirmed before the timelock expires. Use at your own risk.
 
 cmd_args = opts.init(opts_data,add_opts=['tx_fee','tx_fee_adj','comment_file'])
 
+die(1,'This command is disabled')
+
 opt.other_coin = opt.other_coin.upper() if opt.other_coin else g.proto.forks[-1][2].upper()
 if opt.other_coin.lower() not in [e[2] for e in g.proto.forks if e[3] == True]:
 	die(1,"'{}': not a replayable fork of {} chain".format(opt.other_coin,g.coin))

mmgen/main_tool.py

@@ -71,6 +71,7 @@ opts_data = {
 -t, --type=t          Specify address type (valid options: 'legacy',
                       'compressed', 'segwit', 'bech32', 'zcash_z')
 -v, --verbose         Produce more verbose output
+-X, --cached-balances Use cached balances (Ethereum only)
 """,
 	'notes': """
 
@@ -90,6 +91,8 @@ Type '{pn} help <command>' for help on a particular command
 
 cmd_args = opts.init(opts_data,add_opts=['hidden_incog_input_params','in_fmt','use_old_ed25519'])
 
+g.use_cached_balances = opt.cached_balances
+
 if len(cmd_args) < 1: opts.usage()
 cmd = cmd_args.pop(0)
 

mmgen/main_txbump.py

@@ -142,6 +142,10 @@ if g.proto.base_proto == 'Bitcoin':
 
 if not opt.yes:
 	tx.add_comment()   # edits an existing comment
+
+from mmgen.tw import TwUnspentOutputs
+tx.twuo = TwUnspentOutputs(minconf=opt.minconf)
+
 tx.create_raw()        # creates tx.hex, tx.txid
 tx.add_timestamp()
 tx.add_blockcount()
@@ -152,6 +156,8 @@ if not silent:
 	msg(green('\nREPLACEMENT TRANSACTION:'))
 	msg_r(tx.format_view(terse=True))
 
+del tx.twuo.wallet
+
 if seed_files or kl or kal:
 	if txsign(tx,seed_files,kl,kal):
 		tx.write_to_file(ask_write=False)

mmgen/main_txcreate.py

@@ -54,6 +54,7 @@ opts_data = {
 -v, --verbose         Produce more verbose output
 -V, --vsize-adj=   f  Adjust transaction's estimated vsize by factor 'f'
 -y, --yes             Answer 'yes' to prompts, suppress non-essential output
+-X, --cached-balances Use cached balances (Ethereum only)
 """,
 		'notes': '\n{}{}',
 	},
@@ -71,6 +72,8 @@ opts_data = {
 
 cmd_args = opts.init(opts_data)
 
+g.use_cached_balances = opt.cached_balances
+
 rpc_init()
 
 from mmgen.tx import MMGenTX

mmgen/main_txdo.py

@@ -78,6 +78,7 @@ opts_data = {
                        wallet is scanned for subseeds.
 -v, --verbose          Produce more verbose output
 -V, --vsize-adj=     f Adjust transaction's estimated vsize by factor 'f'
+-X, --cached-balances  Use cached balances (Ethereum only)
 -y, --yes              Answer 'yes' to prompts, suppress non-essential output
 -z, --show-hash-presets Show information on available hash presets
 """,
@@ -108,6 +109,8 @@ column below:
 
 cmd_args = opts.init(opts_data)
 
+g.use_cached_balances = opt.cached_balances
+
 rpc_init()
 
 from mmgen.tx import *
@@ -127,7 +130,6 @@ if txsign(tx,seed_files,kl,kal):
 	tx.write_to_file(ask_write=False)
 	tx.send(exit_on_fail=True)
 	tx.write_to_file(ask_overwrite=False,ask_write=False)
-	if hasattr(tx,'token_addr'):
-		msg('Contract address: {}'.format(tx.token_addr.hl()))
+	tx.print_contract_addr()
 else:
 	die(2,'Transaction could not be signed')

mmgen/main_txsend.py

@@ -68,6 +68,4 @@ if not opt.yes:
 
 tx.send(exit_on_fail=True)
 tx.write_to_file(ask_overwrite=False,ask_write=False)
-
-if hasattr(tx,'token_addr'):
-	msg('Contract address: {}'.format(tx.token_addr.hl()))
+tx.print_contract_addr()

mmgen/main_txsign.py

@@ -114,7 +114,7 @@ for tx_num,tx_file in enumerate(tx_files,1):
 	if len(tx_files) > 1:
 		msg('\nTransaction #{} of {}:'.format(tx_num,len(tx_files)))
 		tx_num_str = ' #{}'.format(tx_num)
-	tx = MMGenTX(tx_file)
+	tx = MMGenTX(tx_file,offline=True)
 
 	if tx.marked_signed():
 		msg('Transaction is already signed!'); continue

mmgen/opts.py

@@ -82,7 +82,7 @@ def opt_postproc_initializations():
 		init_color(num_colors=('auto',256)[bool(g.force_256_color)])
 
 	g.coin = g.coin.upper() # allow user to use lowercase
-	g.dcoin = g.coin
+	g.dcoin = g.coin # the display coin; for ERC20 tokens, g.dcoin is set to the token symbol
 
 def set_data_dir_root():
 	g.data_dir_root = os.path.normpath(os.path.expanduser(opt.data_dir)) if opt.data_dir else \
@@ -338,7 +338,6 @@ def init(opts_data,add_opts=[],opt_filter=None,parse_only=False):
 		opt.verbose,opt.quiet = (True,None)
 	if g.debug_opts: opt_postproc_debug()
 
-	g.altcoin_data_dir = os.path.join(g.data_dir_root,'altcoins')
 	warn_altcoins(altcoin_trust_level)
 
 	# We don't need this data anymore
@@ -348,7 +347,7 @@ def init(opts_data,add_opts=[],opt_filter=None,parse_only=False):
 
 def opt_is_tx_fee(val,desc):
 	from mmgen.tx import MMGenTX
-	tx = MMGenTX()
+	tx = MMGenTX(offline=True)
 	# TODO: size is just a guess; do this check after parsing tx file
 	ret = tx.process_fee_spec(val,224,on_fail='return')
 	# Non-standard startgas: disable fee checking

mmgen/rpc.py

@@ -21,11 +21,9 @@ rpc.py:  Cryptocoin RPC library for the MMGen suite
 """
 
 import http.client,base64,json
-
 from decimal import Decimal
 
 from mmgen.common import *
-from mmgen.obj import MMGenObject
 
 def dmsg_rpc(s):
 	if g.debug_rpc: msg(s)
@@ -82,6 +80,9 @@ class CoinDaemonRPCConnection(MMGenObject):
 	# With on_fail='return', returns 'rpcfail',(resp_object,(die_args))
 	def request(self,cmd,*args,**kwargs):
 
+		if g.debug:
+			print_stack_trace('RPC REQUEST {}\n  args: {!r}\n  kwargs: {!r}'.format(cmd,args,kwargs))
+
 		if g.rpc_fail_on_command == cmd:
 			cmd = 'badcommand_' + cmd
 

mmgen/tool.py

@@ -554,7 +554,7 @@ class MMGenToolCmdFile(MMGenToolCmdBase):
 
 		sep = '—'*77+'\n'
 		return sep.join(
-			[MMGenTX(fn).format_view(terse=terse,sort=tx_sort) for fn in flist.names()]
+			[MMGenTX(fn,offline=True).format_view(terse=terse,sort=tx_sort) for fn in flist.names()]
 		).rstrip()
 
 class MMGenToolCmdFileCrypt(MMGenToolCmdBase):
@@ -772,6 +772,7 @@ class MMGenToolCmdRPC(MMGenToolCmdBase):
 				die(1,m.format(mmgen_addrs))
 			usr_addr_list = [MMGenID('{}:{}'.format(a[0],i)) for i in AddrIdxList(a[1])]
 
+		rpc_init()
 		from mmgen.tw import TwAddrList
 		al = TwAddrList(usr_addr_list,minconf,showempty,showbtcaddrs,all_labels)
 		if not al:
@@ -793,7 +794,9 @@ class MMGenToolCmdRPC(MMGenToolCmdBase):
 		twuo.do_sort(sort,reverse=reverse)
 		twuo.age_fmt = age_fmt
 		twuo.show_mmid = show_mmid
-		return twuo.format_for_printing(color=True) if wide else twuo.format_for_display()
+		ret = twuo.format_for_printing(color=True) if wide else twuo.format_for_display()
+		del twuo.wallet
+		return ret
 
 	def add_label(self,mmgen_or_coin_addr:str,label:str):
 		"add descriptive label for address in tracking wallet"

mmgen/tw.py

@@ -20,6 +20,8 @@
 tw: Tracking wallet methods for the MMGen suite
 """
 
+import json
+from mmgen.exception import *
 from mmgen.common import *
 from mmgen.obj import *
 from mmgen.tx import is_mmgen_id
@@ -79,7 +81,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 """.strip().format(g.proj_name.lower())
 	}
 
-	def __init__(self,minconf=1):
+	def __init__(self,minconf=1,addrs=[]):
 		self.unspent      = self.MMGenTwOutputList()
 		self.fmt_display  = ''
 		self.fmt_print    = ''
@@ -88,12 +90,16 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 		self.group        = False
 		self.show_mmid    = True
 		self.minconf      = minconf
-		self.get_unspent_data()
+		self.addrs        = addrs
 		self.age_fmt      = 'days'
 		self.sort_key     = 'age'
-		self.do_sort()
 		self.disp_prec    = self.get_display_precision()
 
+		self.wallet = TrackingWallet('w')
+		self.get_unspent_data()
+		self.do_sort()
+
+
 	@property
 	def age_fmt(self):
 		return self._age_fmt
@@ -121,7 +127,9 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 		# 4. include_unsafe (boolean, optional, default=true) Include outputs that are not safe to spend
 		# 5. query_options  (json object, optional) JSON with query options
 
-		return g.rpch.listunspent(self.minconf)
+		# for now, self.addrs is just an empty list for Bitcoin and friends
+		add_args = (9999999,self.addrs) if self.addrs else ()
+		return g.rpch.listunspent(self.minconf,*add_args)
 
 	def get_unspent_data(self):
 		if g.bogus_wallet_data: # for debugging purposes only
@@ -326,6 +334,8 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 				else:
 					if action == 'a_addr_delete':
 						fs = "Removing {} #{} from tracking wallet.  Is this what you want?"
+					elif action == 'a_balance_refresh':
+						fs = "Refreshing tracking wallet {} #{}.  Is this what you want?"
 					if keypress_confirm(fs.format(self.item_desc,n)):
 						return n
 
@@ -356,11 +366,19 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 			elif action == 'd_redraw': pass
 			elif action == 'd_reverse': self.unspent.reverse(); self.reverse = not self.reverse
 			elif action == 'a_quit': msg(''); return self.unspent
+			elif action == 'a_balance_refresh':
+				idx = self.get_idx_from_user(action)
+				if idx:
+					e = self.unspent[idx-1]
+					bal = self.wallet.get_balance(e.addr,force_rpc=True)
+					self.get_unspent_data()
+					self.do_sort()
+					oneshot_msg = yellow('{} balance for account #{} refreshed\n\n'.format(g.dcoin,idx))
 			elif action == 'a_lbl_add':
 				idx,lbl = self.get_idx_from_user(action)
 				if idx:
 					e = self.unspent[idx-1]
-					if TrackingWallet(mode='w').add_label(e.twmmid,lbl,addr=e.addr):
+					if self.wallet.add_label(e.twmmid,lbl,addr=e.addr):
 						self.get_unspent_data()
 						self.do_sort()
 						a = 'added to' if lbl else 'removed from'
@@ -371,7 +389,7 @@ watch-only wallet using '{}-addrimport' and then re-run this program.
 				idx = self.get_idx_from_user(action)
 				if idx:
 					e = self.unspent[idx-1]
-					if TrackingWallet(mode='w').remove_address(e.addr):
+					if self.wallet.remove_address(e.addr):
 						self.get_unspent_data()
 						self.do_sort()
 						oneshot_msg = yellow("{} #{} removed\n\n".format(capfirst(self.item_desc),idx))
@@ -398,7 +416,7 @@ class TwAddrList(MMGenDict):
 	def __new__(cls,*args,**kwargs):
 		return MMGenDict.__new__(altcoin_subclass(cls,'tw','TwAddrList'),*args,**kwargs)
 
-	def __init__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels):
+	def __init__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None):
 
 		def check_dup_mmid(acct_labels):
 			mmid_prev,err = None,False
@@ -530,16 +548,146 @@ class TwAddrList(MMGenDict):
 
 class TrackingWallet(MMGenObject):
 
+	caps = ('rescan','batch')
+	data_key = 'addresses'
+	use_tw_file = False
+	aggressive_sync = False
+
 	def __new__(cls,*args,**kwargs):
 		return MMGenObject.__new__(altcoin_subclass(cls,'tw','TrackingWallet'))
 
-	mode = 'r'
-	caps = ('rescan','batch')
+	def __init__(self,mode='r',no_rpc=False):
 
-	def __init__(self,mode='r'):
-		m = "'{}': invalid 'mode' parameter for {} constructor"
-		assert mode in ('r','w'),m.format(mode,type(self).__name__)
+		if g.debug:
+			print_stack_trace('TW INIT {!r} {!r}'.format(mode,self))
+
+		assert mode in ('r','w'),"{!r}: wallet mode must be 'r' or 'w'".format(self)
 		self.mode = mode
+		self.desc = self.base_desc = '{} tracking wallet'.format(capfirst(g.proto.name))
+
+		if self.use_tw_file:
+			self.init_from_wallet_file()
+		else:
+			self.init_empty()
+
+		if self.data['coin'] != g.coin:
+			m = 'Tracking wallet coin ({}) does not match current coin ({})!'
+			raise WalletFileError(m.format(self.data['coin'],g.coin))
+
+		self.conv_types(self.data[self.data_key])
+		self.rpc_init()
+		self.cur_balances = {} # cache balances to prevent repeated lookups per program invocation
+
+	def init_empty(self):
+		self.data = { 'coin': g.coin, 'addresses': {} }
+
+	def init_from_wallet_file(self):
+
+		tw_dir = (
+			os.path.join(g.data_dir,g.proto.data_subdir) if g.coin == 'BTC' else
+			os.path.join(g.data_dir_root,'altcoins',g.coin.lower(),g.proto.data_subdir) )
+		self.tw_fn = os.path.join(tw_dir,'tracking-wallet.json')
+
+		check_or_create_dir(tw_dir)
+
+		try:
+			self.orig_data = get_data_from_file(self.tw_fn,quiet=True)
+			self.data = json.loads(self.orig_data)
+		except:
+			try: os.stat(self.tw_fn)
+			except:
+				self.orig_data = ''
+				self.init_empty()
+				self.force_write()
+			else:
+				m = "File '{}' exists but does not contain valid json data"
+				raise WalletFileError(m.format(self.tw_fn))
+		else:
+			self.upgrade_wallet_maybe()
+
+		# ensure that wallet file is written when user exits via KeyboardInterrupt:
+		if self.mode == 'w':
+			import atexit
+			def del_tw(tw):
+				dmsg('Running exit handler del_tw() for {!r}'.format(tw))
+				del tw
+			atexit.register(del_tw,self)
+
+	# TrackingWallet instances must be explicitly destroyed with 'del tw', 'del twuo.wallet'
+	# and the like to ensure the instance is deleted and wallet is written before global
+	# vars are destroyed by interpreter at shutdown.
+	# This is especially important, as exceptions are ignored within __del__():
+	#     /usr/share/doc/python3.6-doc/html/reference/datamodel.html#object.__del__
+	# This code can only be debugged by examining the program output.  Since no exceptions
+	# are raised, errors will not be caught by the test suite.
+	def __del__(self):
+		if g.debug:
+			print_stack_trace('TW DEL {!r}'.format(self))
+
+		if self.mode == 'w':
+			self.write()
+		elif g.debug:
+			msg('read-only wallet, doing nothing')
+
+	def upgrade_wallet_maybe(self): pass
+
+	@staticmethod
+	def conv_types(ad):
+		for k,v in ad.items():
+			if k in ('params','coin'): continue
+			v['mmid'] = TwMMGenID(v['mmid'],on_fail='raise')
+			v['comment'] = TwComment(v['comment'],on_fail='raise')
+
+	def rpc_init(self):
+		rpc_init()
+
+	@property
+	def data_root(self):
+		return self.data[self.data_key]
+
+	@property
+	def data_root_desc(self):
+		return self.data_key
+
+	def cache_balance(self,addr,bal,session_cache,data_root,force=False):
+		if force or addr not in session_cache:
+			session_cache[addr] = str(bal)
+			if addr in data_root:
+				data_root[addr]['balance'] = str(bal)
+				if self.aggressive_sync:
+					self.write()
+
+	def get_cached_balance(self,addr,session_cache,data_root):
+		if addr in session_cache:
+			return g.proto.coin_amt(session_cache[addr])
+		if not g.use_cached_balances:
+			return None
+		if addr in data_root and 'balance' in data_root[addr]:
+			return g.proto.coin_amt(data_root[addr]['balance'])
+
+	def get_balance(self,addr,force_rpc=False):
+		ret = None if force_rpc else self.get_cached_balance(addr,self.cur_balances,self.data_root)
+		if ret == None:
+			ret = self.rpc_get_balance(addr)
+			self.cache_balance(addr,ret,self.cur_balances,self.data_root)
+		return ret
+
+	def rpc_get_balance(self,addr):
+		raise NotImplementedError('not implemented')
+
+	@property
+	def sorted_list(self):
+		return sorted(
+			[ { 'addr':x[0],
+				'mmid':x[1]['mmid'],
+				'comment':x[1]['comment'] }
+					for x in self.data_root.items() if x[0] not in ('params','coin') ],
+			key=lambda x: x['mmid'].sort_key+x['addr'] )
+
+	@property
+	def mmid_ordered_dict(self):
+		from collections import OrderedDict
+		return OrderedDict([(x['mmid'],{'addr':x['addr'],'comment':x['comment']}) for x in self.sorted_list])
 
 	@write_mode
 	def import_address(self,addr,label,rescan):
@@ -549,11 +697,38 @@ class TrackingWallet(MMGenObject):
 	def batch_import_address(self,arg_list):
 		return g.rpch.importaddress(arg_list,batch=True)
 
+	def force_write(self):
+		mode_save = self.mode
+		self.mode = 'w'
+		self.write()
+		self.mode = mode_save
+
 	@write_mode
-	def write(self): pass
+	def write_changed(self,data):
+		write_data_to_file(
+			self.tw_fn,data,
+			desc='{} data'.format(self.base_desc),
+			ask_overwrite=False,ignore_opt_outdir=True,quiet=True,
+			check_data=True,cmp_data=self.orig_data)
+		self.orig_data = data
+
+	def write(self): # use 'check_data' to check wallet hasn't been altered by another program
+		if not self.use_tw_file:
+			dmsg("'use_tw_file' is False, doing nothing")
+			return
+		dmsg('write(): checking if {} data has changed'.format(self.desc))
+		wdata = json.dumps(self.data)
+
+		if self.orig_data != wdata:
+			if g.debug:
+				print_stack_trace('TW DATA CHANGED {!r}'.format(self))
+				print_diff(self.orig_data,wdata,from_json=True)
+			self.write_changed(wdata)
+		elif g.debug:
+			msg('Data is unchanged\n')
 
 	def is_in_wallet(self,addr):
-		return addr in TwAddrList([],0,True,True,True).coinaddr_list()
+		return addr in TwAddrList([],0,True,True,True,wallet=self).coinaddr_list()
 
 	@write_mode
 	def set_label(self,coinaddr,lbl):

mmgen/tx.py

@@ -82,6 +82,7 @@ def mmaddr2coinaddr(mmaddr,ad_w,ad_f):
 	return CoinAddr(coin_addr)
 
 def segwit_is_active(exit_on_error=False):
+	rpc_init()
 	d = g.rpch.getblockchaininfo()
 	if d['chain'] == 'regtest':
 		return True
@@ -306,6 +307,7 @@ class MMGenTX(MMGenObject):
 	view_sort_orders = ('addr','raw')
 	dfl_view_sort_order = 'addr'
 
+	msg_wallet_low_coin = 'Wallet has insufficient funds for this transaction ({} {} needed)'
 	msg_low_coin = 'Selected outputs insufficient to fund this transaction ({} {} needed)'
 	msg_no_change_output = """
 ERROR: No change address specified.  If you wish to create a transaction with
@@ -318,7 +320,7 @@ inputs must be supplied to '{pnl}-txsign' in a file with the '--keys-from-file'
 option.
 Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_name.lower())
 
-	def __init__(self,filename=None,metadata_only=False,caller=None,quiet_open=False):
+	def __init__(self,filename=None,metadata_only=False,caller=None,quiet_open=False,offline=False):
 		self.inputs      = MMGenTxInputList()
 		self.outputs     = MMGenTxOutputList()
 		self.send_amt    = g.proto.coin_amt('0')  # total amt minus change
@@ -416,6 +418,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		self.hex = HexStr(g.rpch.createrawtransaction(i,o))
 		self.update_txid()
 
+	def print_contract_addr(self): pass
+
 	# returns true if comment added or changed
 	def add_comment(self,infile=None):
 		if infile:
@@ -1276,7 +1280,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		if not self.chain and not self.inputs[0].addr.is_for_chain('testnet'):
 			self.chain = 'mainnet'
 
-		if self.dcoin: self.set_g_token()
+		if self.dcoin:
+			self.resolve_g_token_from_tx_file()
 
 	def process_cmd_arg(self,arg,ad_f,ad_w):
 
@@ -1320,7 +1325,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 			check_infile(a)
 			ad_f.add(AddrList(a))
 
-		ad_w = AddrData(source='tw')
+		ad_w = AddrData(source='tw',wallet=self.twuo.wallet)
 
 		self.process_cmd_args(cmd_args,ad_f,ad_w)
 
@@ -1338,9 +1343,13 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 						return selected
 					msg('Unspent output number must be <= {}'.format(len(unspent)))
 
-	def check_sufficient_funds(self,inputs_sum,foo):
-		if self.send_amt > inputs_sum:
-			msg(self.msg_low_coin.format(self.send_amt-inputs_sum,g.coin))
+	# we don't know fee yet, so perform preliminary check with fee == 0
+	def precheck_sufficient_funds(self,inputs_sum,sel_unspent):
+		if self.twuo.total < self.send_amt:
+			msg(self.msg_wallet_low_coin.format(self.send_amt-inputs_sum,g.dcoin))
+			return False
+		if inputs_sum < self.send_amt:
+			msg(self.msg_low_coin.format(self.send_amt-inputs_sum,g.dcoin))
 			return False
 		return True
 
@@ -1380,17 +1389,21 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 
 		return set(sel_nums) # silently discard duplicates
 
-	def get_inputs_from_user(self,tw):
+	def get_cmdline_input_addrs(self):
+		# Bitcoin full node, call doesn't go to the network, so just call listunspent with addrs=[]
+		return []
+
+	def get_inputs_from_user(self):
 
 		while True:
-			us_f = ('select_unspent','select_unspent_cmdline')[bool(opt.inputs)]
-			sel_nums = getattr(self,us_f)(tw.unspent)
+			us_f = self.select_unspent_cmdline if opt.inputs else self.select_unspent
+			sel_nums = us_f(self.twuo.unspent)
 
-			msg('Selected output{}: {}'.format(suf(sel_nums,'s'),' '.join(map(str,sel_nums))))
-			sel_unspent = tw.MMGenTwOutputList([tw.unspent[i-1] for i in sel_nums])
+			msg('Selected output{}: {}'.format(suf(sel_nums),' '.join(map(str,sel_nums))))
+			sel_unspent = self.twuo.MMGenTwOutputList([self.twuo.unspent[i-1] for i in sel_nums])
 
 			inputs_sum = sum(s.amt for s in sel_unspent)
-			if not self.check_sufficient_funds(inputs_sum,sel_unspent):
+			if not self.precheck_sufficient_funds(inputs_sum,sel_unspent):
 				continue
 
 			non_mmaddrs = [i for i in sel_unspent if i.twmmid.type == 'non-mmgen']
@@ -1406,7 +1419,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 
 			change_amt = self.get_change_amt()
 
-			if change_amt >= 0: # TODO: show both ETH and token amts remaining
+			if change_amt >= 0:
 				p = self.final_inputs_ok_msg(change_amt)
 				if opt.yes or keypress_confirm(p+'. OK?',default_yes=True):
 					if opt.yes: msg(p)
@@ -1426,17 +1439,20 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 
 		if opt.comment_file: self.add_comment(opt.comment_file)
 
-		if not do_info: self.get_outputs_from_cmdline(cmd_args)
+		twuo_addrs = self.get_cmdline_input_addrs()
+
+		from mmgen.tw import TwUnspentOutputs
+		self.twuo = TwUnspentOutputs(minconf=opt.minconf,addrs=twuo_addrs)
+
+		if not do_info:
+			self.get_outputs_from_cmdline(cmd_args)
 
 		do_license_msg()
 
-		from mmgen.tw import TwUnspentOutputs
-		tw = TwUnspentOutputs(minconf=opt.minconf)
-
 		if not opt.inputs:
-			tw.view_and_sort(self)
+			self.twuo.view_and_sort(self)
 
-		tw.display_total()
+		self.twuo.display_total()
 
 		if do_info: sys.exit(0)
 
@@ -1446,7 +1462,7 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 			('Unknown','{} {}'.format(self.send_amt.hl(),g.dcoin))[bool(self.send_amt)]
 		))
 
-		change_amt = self.get_inputs_from_user(tw)
+		change_amt = self.get_inputs_from_user()
 
 		self.update_change_output(change_amt)
 		self.update_send_amt(change_amt)
@@ -1479,6 +1495,8 @@ Selected non-{pnm} inputs: {{}}""".strip().format(pnm=g.proj_name,pnl=g.proj_nam
 		if not opt.yes:
 			self.view_with_prompt('View decoded transaction?')
 
+		del self.twuo
+
 class MMGenBumpTX(MMGenTX):
 
 	def __new__(cls,*args,**kwargs):

test/ref/ethereum/tracking-wallet-v1.json

@@ -0,0 +1 @@
+{"coin":"ETH","e704b6cfd9f0edb2e6cfbd0c913438d37ede7b35":{"mmid":"98831F3A:E:1","comment":""},"b7d06382a998817a16ba487a99636bf8cae29d9d":{"mmid":"98831F3A:E:2","comment":""},"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef":{"mmid":"eth:deadbeefdeadbeefdeadbeefdeadbeefdeadbeef","comment":""}}

test/ref/ethereum/tracking-wallet-v2.json

@@ -0,0 +1 @@
+{"coin":"ETH","accounts":{"e704b6cfd9f0edb2e6cfbd0c913438d37ede7b35":{"mmid":"98831F3A:E:1","comment":""},"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef":{"mmid":"eth:deadbeefdeadbeefdeadbeefdeadbeefdeadbeef","comment":""}},"tokens":{"d5f051401ca478b34c80d0b5a119e437dc6d9df5":{"6e0fbe42e1343309b3ccb9068dbad6132f86c96f":{"mmid":"98831F3A:E:12","comment":""},"b72268fa55a57fe838a745b27c90f0edb415af61":{"mmid":"98831F3A:E:13","comment":""}},"3dd0864668c36d27b53a98137764c99f9fd5b7b2":{"ca2b705adf43151ce61e01c653424e9790e9eb84":{"mmid":"98831F3A:E:22","comment":""},"7227a8fd728b74208255c6a8a09cb9fb66b1230c":{"mmid":"98831F3A:E:23","comment":""}}}}

test/test_py_d/ts_ethdev.py

@@ -53,11 +53,13 @@ except:
 
 if re.match(r'\b0.5.1\b',solc_ver): # Raspbian Stretch
 	vbal1 = '1.2288337'
+	vbal1a = 'TODO'
 	vbal2 = '99.997085083'
 	vbal3 = '1.23142165'
 	vbal4 = '127.0287837'
 elif solc_ver == '' or re.match(r'\b0.5.3\b',solc_ver): # Ubuntu Bionic
 	vbal1 = '1.2288487'
+	vbal1a = '1.22627465'
 	vbal2 = '99.997092733'
 	vbal3 = '1.23142915'
 	vbal4 = '127.0287987'
@@ -77,32 +79,44 @@ bals = {
 			(burn_addr + '\s+Non-MMGen',amt1)],
 	'8': [  ('98831F3A:E:1','0'),
 			('98831F3A:E:2','23.45495'),
-			('98831F3A:E:11',vbal1,'a'),
+			('98831F3A:E:11',vbal1,'a1'),
 			('98831F3A:E:12','99.99895'),
 			('98831F3A:E:21','2.345'),
 			(burn_addr + '\s+Non-MMGen',amt1)],
 	'9': [  ('98831F3A:E:1','0'),
 			('98831F3A:E:2','23.45495'),
-			('98831F3A:E:11',vbal1,'a'),
+			('98831F3A:E:11',vbal1,'a1'),
+			('98831F3A:E:12',vbal2),
+			('98831F3A:E:21','2.345'),
+			(burn_addr + '\s+Non-MMGen',amt1)],
+	'10': [ ('98831F3A:E:1','0'),
+			('98831F3A:E:2','23.0218'),
+			('98831F3A:E:3','0.4321'),
+			('98831F3A:E:11',vbal1,'a1'),
 			('98831F3A:E:12',vbal2),
 			('98831F3A:E:21','2.345'),
 			(burn_addr + '\s+Non-MMGen',amt1)]
 }
+
 token_bals = {
 	'1': [  ('98831F3A:E:11','1000','1.234')],
-	'2': [  ('98831F3A:E:11','998.76544',vbal3,'a'),
+	'2': [  ('98831F3A:E:11','998.76544',vbal3,'a1'),
 			('98831F3A:E:12','1.23456','0')],
-	'3': [  ('98831F3A:E:11','110.654317776666555545',vbal1,'a'),
+	'3': [  ('98831F3A:E:11','110.654317776666555545',vbal1,'a1'),
 			('98831F3A:E:12','1.23456','0')],
-	'4': [  ('98831F3A:E:11','110.654317776666555545',vbal1,'a'),
+	'4': [  ('98831F3A:E:11','110.654317776666555545',vbal1,'a1'),
 			('98831F3A:E:12','1.23456','0'),
 			(burn_addr + '\s+Non-MMGen',amt2,amt1)],
-	'5': [  ('98831F3A:E:11','110.654317776666555545',vbal1,'a'),
+	'5': [  ('98831F3A:E:11','110.654317776666555545',vbal1,'a1'),
 			('98831F3A:E:12','1.23456','99.99895'),
 			(burn_addr + '\s+Non-MMGen',amt2,amt1)],
-	'6': [  ('98831F3A:E:11','110.654317776666555545',vbal1,'a'),
+	'6': [  ('98831F3A:E:11','110.654317776666555545',vbal1,'a1'),
 			('98831F3A:E:12','0',vbal2),
 			('98831F3A:E:13','1.23456','0'),
+			(burn_addr + '\s+Non-MMGen',amt2,amt1)],
+	'7': [  ('98831F3A:E:11','67.444317776666555545',vbal1a,'a2'),
+			('98831F3A:E:12','43.21',vbal2),
+			('98831F3A:E:13','1.23456','0'),
 			(burn_addr + '\s+Non-MMGen',amt2,amt1)]
 }
 token_bals_getbalance = {
@@ -120,12 +134,15 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 	tmpdir_nums = [22]
 	cmd_group = (
 		('setup',               'Ethereum Parity dev mode tests for coin {} (start parity)'.format(g.coin)),
+		('wallet_upgrade1',     'upgrading the tracking wallet (v1 -> v2)'),
+		('wallet_upgrade2',     'upgrading the tracking wallet (v2 -> v3)'),
 		('addrgen',             'generating addresses'),
 		('addrimport',          'importing addresses'),
 		('addrimport_dev_addr', "importing Parity dev address 'Ox00a329c..'"),
 
 		('txcreate1',           'creating a transaction (spend from dev address to address :1)'),
 		('txsign1',             'signing the transaction'),
+		('tx_status0',          'getting the transaction status'),
 		('txsign1_ni',          'signing the transaction (non-interactive)'),
 		('txsend1',             'sending the transaction'),
 		('bal1',                'the {} balance'.format(g.coin)),
@@ -147,6 +164,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 
 		('txsign4',             'signing the transaction'),
 		('txsend4',             'sending the transaction'),
+		('tx_status1a',         'getting the transaction status'),
 		('bal4',                'the {} balance'.format(g.coin)),
 
 		('txcreate5',           'creating a transaction (fund burn address)'),
@@ -192,6 +210,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		('token_txcreate1',     'creating a token transaction'),
 		('token_txsign1',       'signing the transaction'),
 		('token_txsend1',       'sending the transaction'),
+		('tx_status3',          'getting the transaction status'),
 		('token_bal2',          'the {} balance and token balance'.format(g.coin)),
 
 		('token_txcreate2',     'creating a token transaction (to burn address)'),
@@ -232,6 +251,19 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		('token_listaddresses1','listaddresses --token=mm1'),
 		('token_listaddresses2','listaddresses --token=mm1 showempty=1'),
 
+		('twview_cached_balances','twview (cached balances)'),
+		('token_twview_cached_balances','token twview (cached balances)'),
+		('txcreate_cached_balances','txcreate (cached balances)'),
+		('token_txcreate_cached_balances','token txcreate (cached balances)'),
+
+		('txdo_cached_balances',     'txdo (cached balances)'),
+		('txcreate_refresh_balances','refreshing balances'),
+		('bal10',                    'the {} balance'.format(g.coin)),
+
+		('token_txdo_cached_balances',     'token txdo (cached balances)'),
+		('token_txcreate_refresh_balances','refreshing token balances'),
+		('token_bal7',                     'the token balance'),
+
 		('twview1','twview'),
 		('twview2','twview wide=1'),
 		('twview3','twview wide=1 sort=age (ignored)'),
@@ -247,10 +279,12 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		('edit_label2','adding label to addr #{} in {} tracking wallet (lat+cyr+gr)'.format(del_addrs[1],g.coin)),
 		('edit_label3','removing label from addr #{} in {} tracking wallet'.format(del_addrs[0],g.coin)),
 
+		('token_edit_label1','adding label to addr #{} in {} token tracking wallet'.format(del_addrs[0],g.coin)),
+
 		('remove_addr1','removing addr #{} from {} tracking wallet'.format(del_addrs[0],g.coin)),
 		('remove_addr2','removing addr #{} from {} tracking wallet'.format(del_addrs[1],g.coin)),
-		('remove_token_addr1','removing addr #{} from {} token tracking wallet'.format(del_addrs[0],g.coin)),
-		('remove_token_addr2','removing addr #{} from {} token tracking wallet'.format(del_addrs[1],g.coin)),
+		('token_remove_addr1','removing addr #{} from {} token tracking wallet'.format(del_addrs[0],g.coin)),
+		('token_remove_addr2','removing addr #{} from {} token tracking wallet'.format(del_addrs[1],g.coin)),
 
 		('stop',                'stopping parity'),
 	)
@@ -300,6 +334,25 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 			die(1,'No parity executable found!')
 		return 'ok'
 
+	def wallet_upgrade(self,src_file):
+		if g.coin == 'ETC':
+			msg('skipping test {!r} for ETC'.format(self.test_name))
+			return 'skip'
+		src_dir = joinpath(ref_dir,'ethereum')
+		dest_dir = joinpath(self.tr.data_dir,'altcoins',g.coin.lower())
+		w_from = joinpath(src_dir,src_file)
+		w_to = joinpath(dest_dir,'tracking-wallet.json')
+		os.makedirs(dest_dir,mode=0o750,exist_ok=True)
+		dest = shutil.copy2(w_from,w_to)
+		assert dest == w_to, dest
+		t = self.spawn('mmgen-tool', self.eth_args + ['twview'])
+		t.read()
+		os.unlink(w_to)
+		return t
+
+	def wallet_upgrade1(self): return self.wallet_upgrade('tracking-wallet-v1.json')
+	def wallet_upgrade2(self): return self.wallet_upgrade('tracking-wallet-v2.json')
+
 	def addrgen(self,addrs='1-3,11-13,21-23'):
 		from mmgen.addr import MMGenAddrType
 		t = self.spawn('mmgen-addrgen', self.eth_args + [dfl_words_file,addrs])
@@ -315,7 +368,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 			t.read()
 			t.req_exit_val = 2
 			return t
-		if g.debug: t.expect("Type uppercase 'YES' to confirm: ",'YES\n')
+#		if g.debug: t.expect("Type uppercase 'YES' to confirm: ",'YES\n')
 		t.expect('Importing')
 		t.expect(expect)
 		t.read()
@@ -332,16 +385,17 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 	def addrimport_burn_addr(self):
 		return self.addrimport_one_addr(addr=burn_addr)
 
-	def txcreate(self,args=[],menu=[],acct='1',non_mmgen_inputs=0,
+	def txcreate(self,args=[],menu=[],acct='1',non_mmgen_inputs=0,caller='txcreate',
 						interactive_fee = '50G',
 						eth_fee_res     = None,
 						fee_res_fs      = '0.00105 {} (50 gas price in Gwei)',
-						fee_desc        = 'gas price' ):
+						fee_desc        = 'gas price',
+						no_read         = False):
 		fee_res = fee_res_fs.format(g.coin)
-		t = self.spawn('mmgen-txcreate', self.eth_args + ['-B'] + args)
+		t = self.spawn('mmgen-'+caller, self.eth_args + ['-B'] + args)
 		t.expect(r'add \[l\]abel, .*?:.','p', regex=True)
 		t.written_to_file('Account balances listing')
-		return self.txcreate_ui_common( t, menu=menu,
+		t = self.txcreate_ui_common( t, menu=menu, caller=caller,
 										input_sels_prompt = 'to spend from',
 										inputs            = acct,
 										file_desc         = 'Ethereum transaction',
@@ -352,6 +406,9 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 										fee_desc          = fee_desc,
 										eth_fee_res       = eth_fee_res,
 										add_comment       = tx_label_jp )
+		if not no_read:
+			t.read()
+		return t
 
 	def txsign(self,ni=False,ext='{}.rawtx',add_args=[]):
 		ext = ext.format('-α' if g.debug_utf8 else '')
@@ -359,7 +416,8 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		write_to_file(keyfile,dfl_privkey+'\n')
 		txfile = self.get_file_with_ext(ext,no_dot=True)
 		t = self.spawn( 'mmgen-txsign',
-						self.eth_args
+						['--outdir={}'.format(self.tmpdir),'--coin='+g.coin,'--quiet']
+						+ ['--rpc-host=bad_host'] # ETH signing must work without RPC
 						+ add_args
 						+ ([],['--yes'])[ni]
 						+ ['-k', keyfile, txfile, dfl_words_file] )
@@ -371,16 +429,18 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		if not bogus_send: os.environ['MMGEN_BOGUS_SEND'] = ''
 		t = self.spawn('mmgen-txsend', self.eth_args + add_args + [txfile])
 		if not bogus_send: os.environ['MMGEN_BOGUS_SEND'] = '1'
-		txid = self.txsend_ui_common(t,quiet=True,bogus_send=bogus_send,has_label=True)
+		txid = self.txsend_ui_common(t,quiet=not g.debug,bogus_send=bogus_send,has_label=True)
 		return t
 
 	def txcreate1(self):
 		# valid_keypresses = EthereumTwUnspentOutputs.key_mappings.keys()
-		menu = ['a','d','r','M','D','e','m','m'] # include one invalid keypress, 'D'
+		menu = ['a','d','r','M','X','e','m','m'] # include one invalid keypress, 'X'
 		args = ['98831F3A:E:1,123.456']
 		return self.txcreate(args=args,menu=menu,acct='1',non_mmgen_inputs=1)
 
 	def txsign1(self):    return self.txsign(add_args=['--use-internal-keccak-module'])
+	def tx_status0(self):
+		return self.tx_status(ext='{}.sigtx',expect_str='neither in mempool nor blockchain',exit_val=1)
 	def txsign1_ni(self): return self.txsign(ni=True)
 	def txsend1(self):    return self.txsend()
 	def bal1(self):       return self.bal(n='1')
@@ -399,17 +459,23 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 	def txsend3(self): return self.txsend(ext='2.345,50000]{}.sigtx')
 	def bal3(self):    return self.bal(n='3')
 
-	def tx_status(self,ext,expect_str):
+	def tx_status(self,ext,expect_str,expect_str2='',add_args=[],exit_val=0):
 		ext = ext.format('-α' if g.debug_utf8 else '')
 		txfile = self.get_file_with_ext(ext,no_dot=True)
-		t = self.spawn('mmgen-txsend', self.eth_args + ['--status',txfile])
+		t = self.spawn('mmgen-txsend', self.eth_args + add_args + ['--status',txfile])
 		t.expect(expect_str)
+		if expect_str2:
+			t.expect(expect_str2)
 		t.read()
+		t.req_exit_val = exit_val
 		return t
 
 	def tx_status1(self):
 		return self.tx_status(ext='2.345,50000]{}.sigtx',expect_str='has 1 confirmation')
 
+	def tx_status1a(self):
+		return self.tx_status(ext='2.345,50000]{}.sigtx',expect_str='has 2 confirmations')
+
 	def txcreate4(self):
 		args = ['98831F3A:E:2,23.45495']
 		interactive_fee='40G'
@@ -445,7 +511,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		t = self.spawn('mmgen-tool', self.eth_args + ['twview','wide=1'])
 		for b in bals[n]:
 			addr,amt,adj = b if len(b) == 3 else b + (False,)
-			if adj and g.coin == 'ETC': amt = str(Decimal(amt) + self.bal_corr)
+			if adj and g.coin == 'ETC': amt = str(Decimal(amt) + Decimal(adj[1]) * self.bal_corr)
 			pat = r'{}\s+{}\s'.format(addr,amt.replace('.',r'\.'))
 			t.expect(pat,regex=True)
 		t.read()
@@ -455,7 +521,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		t = self.spawn('mmgen-tool', self.eth_args + ['--token=mm1','twview','wide=1'])
 		for b in token_bals[n]:
 			addr,_amt1,_amt2,adj = b if len(b) == 4 else b + (False,)
-			if adj and g.coin == 'ETC': _amt2 = str(Decimal(_amt2) + self.bal_corr)
+			if adj and g.coin == 'ETC': _amt2 = str(Decimal(_amt2) + Decimal(adj[1]) * self.bal_corr)
 			pat = r'{}\s+{}\s+{}\s'.format(addr,_amt1.replace('.',r'\.'),_amt2.replace('.',r'\.'))
 			t.expect(pat,regex=True)
 		t.read()
@@ -510,7 +576,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		imsg("Compiling solidity token contract '{}' with 'solc'".format(token_data['symbol']))
 		try: os.mkdir(odir)
 		except: pass
-		cmd = ['scripts/create-token.py','--coin='+g.coin,'--outdir='+odir] + cmd_args + [dfl_addr_chk]
+		cmd = ['scripts/traceback_run.py','scripts/create-token.py','--coin='+g.coin,'--outdir='+odir] + cmd_args + [dfl_addr_chk]
 		imsg("Executing: {}".format(' '.join(cmd)))
 		subprocess.check_output(cmd,stderr=subprocess.STDOUT)
 		imsg("ERC20 token '{}' compiled".format(token_data['symbol']))
@@ -551,7 +617,9 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 			t = self.spawn('mmgen-txsend', self.eth_args + [txfile],no_msg=True)
 
 		os.environ['MMGEN_BOGUS_SEND'] = '1'
-		txid = self.txsend_ui_common(t,caller=mmgen_cmd,quiet=True,bogus_send=False)
+		txid = self.txsend_ui_common(t,caller=mmgen_cmd,
+			quiet = mmgen_cmd == 'txdo' or not g.debug,
+			bogus_send=False)
 		addr = t.expect_getend('Contract address: ')
 		from mmgen.altcoins.eth.tx import EthereumMMGenTX as etx
 		assert etx.get_exec_status(txid,True) != 0,(
@@ -648,12 +716,14 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 
 	def token_txcreate(self,args=[],token='',inputs='1',fee='50G'):
 		t = self.spawn('mmgen-txcreate', self.eth_args + ['--token='+token,'-B','--tx-fee='+fee] + args)
-		return self.txcreate_ui_common( t,
+		t = self.txcreate_ui_common( t,
 										menu              = [],
 										inputs            = inputs,
 										input_sels_prompt = 'to spend from',
 										file_desc         = 'Ethereum token transaction',
 										add_comment       = tx_label_lat_cyr_gr)
+		t.read()
+		return t
 	def token_txsign(self,ext='',token=''):
 		return self.txsign(ni=True,ext=ext,add_args=['--token='+token])
 	def token_txsend(self,ext='',token=''):
@@ -665,6 +735,14 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		return self.token_txsign(ext='1.23456,50000]{}.rawtx',token='mm1')
 	def token_txsend1(self):
 		return self.token_txsend(ext='1.23456,50000]{}.sigtx',token='mm1')
+
+	def tx_status3(self):
+		return self.tx_status(
+			ext='1.23456,50000]{}.sigtx',
+			add_args=['--token=mm1'],
+			expect_str='successfully executed',
+			expect_str2='has 1 confirmation')
+
 	def token_bal2(self):
 		return self.token_bal(n='2')
 
@@ -745,6 +823,62 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 	def token_listaddresses2(self):
 		return self.listaddresses(args=['--token=mm1'],tool_args=['showempty=1'])
 
+	def twview_cached_balances(self):
+		return self.twview(args=['--cached-balances'])
+	def token_twview_cached_balances(self):
+		return self.twview(args=['--token=mm1','--cached-balances'])
+
+	def txcreate_cached_balances(self):
+		args = ['--tx-fee=20G','--cached-balances','98831F3A:E:3,0.1276']
+		return self.txcreate(args=args,acct='2')
+	def token_txcreate_cached_balances(self):
+		args=['--cached-balances','--tx-fee=12G','98831F3A:E:12,1.2789']
+		return self.token_txcreate(args=args,token='mm1')
+
+	def txdo_cached_balances(self,
+			acct = '2',
+			fee_res_fs = '0.00105 {} (50 gas price in Gwei)',
+			add_args = ['98831F3A:E:3,0.4321']):
+		args = ['--tx-fee=20G','--cached-balances'] + add_args + [dfl_words_file]
+		os.environ['MMGEN_BOGUS_SEND'] = ''
+		t = self.txcreate(args=args,acct=acct,caller='txdo',fee_res_fs=fee_res_fs,no_read=True)
+		os.environ['MMGEN_BOGUS_SEND'] = '1'
+		self._do_confirm_send(t,quiet=not g.debug,sure=False)
+		t.read()
+		return t
+
+	def txcreate_refresh_balances(self,
+			bals=['2','3'],
+			args=['-B','--cached-balances','-i'],
+			total= '1000126.14829832312345678',adj_total=True,total_coin=g.coin):
+		if g.coin == 'ETC' and adj_total:
+			total = str(Decimal(total) + self.bal_corr)
+		t = self.spawn('mmgen-txcreate', self.eth_args + args)
+		for n in bals:
+			t.expect('[R]efresh balance:\b','R')
+			t.expect(' main menu): ',n)
+			t.expect('Is this what you want? (y/N): ','y')
+		t.expect('[R]efresh balance:\b','q')
+		t.expect('Total unspent: {} {}'.format(total,total_coin))
+		t.read()
+		return t
+
+	def bal10(self): return self.bal(n='10')
+
+	def token_txdo_cached_balances(self):
+		return self.txdo_cached_balances(
+					acct='1',
+					fee_res_fs='0.0026 {} (50 gas price in Gwei)',
+					add_args=['--token=mm1','98831F3A:E:12,43.21'])
+
+	def token_txcreate_refresh_balances(self):
+		return self.txcreate_refresh_balances(
+					bals=['1','2'],
+					args=['--token=mm1','-B','--cached-balances','-i'],
+					total='1000',adj_total=False,total_coin='MM1')
+
+	def token_bal7(self): return self.token_bal(n='7')
+
 	def twview1(self):
 		return self.twview()
 	def twview2(self):
@@ -767,11 +901,19 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 
 	def edit_label(self,out_num,args=[],action='l',label_text=None):
 		t = self.spawn('mmgen-txcreate', self.eth_args + args + ['-B','-i'])
-		p1,p2 = ('emove address:\b','return to main menu): ')
+		p1,p2 = ('efresh balance:\b','return to main menu): ')
 		p3,r3 = (p2,label_text+'\n') if label_text is not None else ('(y/N): ','y')
 		p4,r4 = (('(y/N): ',),('y',)) if label_text == '' else ((),())
-		for p,r in zip((p1,p1,p2,p3)+p4+(p1,p1),('M',action,out_num+'\n',r3)+r4+('M','q')):
+		for p,r in zip((p1,p1,p2,p3)+p4,('M',action,out_num+'\n',r3)+r4):
 			t.expect(p,r)
+		m = (   'Account #{} removed' if action == 'D' else
+				'Label added to account #{}' if label_text else
+				'Label removed from account #{}' )
+		t.expect(m.format(out_num))
+		for p,r in zip((p1,p1),('M','q')):
+			t.expect(p,r)
+		t.expect('Total unspent:')
+		t.read()
 		return t
 
 	def edit_label1(self):
@@ -781,14 +923,17 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 	def edit_label3(self):
 		return self.edit_label(out_num=del_addrs[0],label_text='')
 
+	def token_edit_label1(self):
+		return self.edit_label(out_num='1',label_text='Token label #1',args=['--token=mm1'])
+
 	def remove_addr1(self):
-		return self.edit_label(out_num=del_addrs[0],action='R')
+		return self.edit_label(out_num=del_addrs[0],action='D')
 	def remove_addr2(self):
-		return self.edit_label(out_num=del_addrs[1],action='R')
-	def remove_token_addr1(self):
-		return self.edit_label(out_num=del_addrs[0],args=['--token=mm1'],action='R')
-	def remove_token_addr2(self):
-		return self.edit_label(out_num=del_addrs[1],args=['--token=mm1'],action='R')
+		return self.edit_label(out_num=del_addrs[1],action='D')
+	def token_remove_addr1(self):
+		return self.edit_label(out_num=del_addrs[0],args=['--token=mm1'],action='D')
+	def token_remove_addr2(self):
+		return self.edit_label(out_num=del_addrs[1],args=['--token=mm1'],action='D')
 
 	def stop(self):
 		self.spawn('',msg_only=True)

test/test_py_d/ts_shared.py

@@ -235,7 +235,8 @@ class TestSuiteShared(object):
 		t.written_to_file('Encrypted secret keys',oo=True)
 		return t
 
-	def _do_confirm_send(self,t,quiet=False,confirm_send=True):
-		t.expect('Are you sure you want to broadcast this')
+	def _do_confirm_send(self,t,quiet=False,confirm_send=True,sure=True):
+		if sure:
+			t.expect('Are you sure you want to broadcast this')
 		m = ('YES, I REALLY WANT TO DO THIS','YES')[quiet]
 		t.expect("'{}' to confirm: ".format(m),('',m)[confirm_send]+'\n')