fixes and cleanups throughout

mmgen/addrlist.py

@@ -20,7 +20,7 @@
 addrlist.py: Address list classes for the MMGen suite
 """
 
-from .util import qmsg,qmsg_r,suf,make_chksum_N,Msg
+from .util import qmsg,qmsg_r,suf,make_chksum_N,Msg,die
 from .objmethods import MMGenObject,Hilite,InitErrors
 from .obj import MMGenListItem,ListItemAttr,MMGenDict,TwComment,WalletPassword
 from .key import PrivKey

mmgen/base_proto/ethereum/__init__.py

@@ -0,0 +1,5 @@
+async def erigon_sleep(self):
+	from ...globalvars import g
+	if self.proto.network == 'regtest' and g.daemon_id == 'erigon':
+		import asyncio
+		await asyncio.sleep(5)

mmgen/base_proto/ethereum/contract.py

@@ -17,12 +17,13 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-altcoins.base_proto.ethereum.contract: Ethereum contract and token classes
+altcoins.base_proto.ethereum.contract: Ethereum ERC20 token classes
 """
 
 from decimal import Decimal
 from . import rlp
 
+from . import erigon_sleep
 from ...util import msg,pp_msg
 from ...globalvars import g
 from ...base_obj import AsyncInit
@@ -33,7 +34,7 @@ from ...amt import ETHAmt
 def parse_abi(s):
 	return [s[:8]] + [s[8+x*64:8+(x+1)*64] for x in range(len(s[8:])//64)]
 
-class TokenBase(MMGenObject): # ERC20
+class TokenCommon(MMGenObject):
 
 	def create_method_id(self,sig):
 		return self.keccak_256(sig.encode()).hexdigest()[:8]
@@ -51,9 +52,7 @@ class TokenBase(MMGenObject): # ERC20
 				method_sig,
 				'\n  '.join(parse_abi(data)) ))
 		ret = await self.rpc.call('eth_call',{ 'to': '0x'+self.addr, 'data': '0x'+data },'pending')
-		if self.proto.network == 'regtest' and g.daemon_id == 'erigon': # ERIGON
-			import asyncio
-			await asyncio.sleep(5)
+		await erigon_sleep(self)
 		if toUnit:
 			return int(ret,16) * self.base_unit
 		else:
@@ -119,21 +118,25 @@ class TokenBase(MMGenObject): # ERC20
 			chain_id = None if res == None else int(res,16)
 
 		tx = Transaction(**tx_in).sign(key,chain_id)
-		hex_tx = rlp.encode(tx).hex()
-		coin_txid = CoinTxID(tx.hash.hex())
+
 		if tx.sender.hex() != from_addr:
 			die(3,f'Sender address {from_addr!r} does not match address of key {tx.sender.hex()!r}!')
+
 		if g.debug:
 			msg('TOKEN DATA:')
 			pp_msg(tx.to_dict())
 			msg('PARSED ABI DATA:\n  {}'.format(
 				'\n  '.join(parse_abi(tx.data.hex())) ))
-		return hex_tx,coin_txid
+
+		return (
+			rlp.encode(tx).hex(),
+			CoinTxID(tx.hash.hex())
+		)
 
 # The following are used for token deployment only:
 
-	async def txsend(self,hex_tx):
-		return (await self.rpc.call('eth_sendRawTransaction','0x'+hex_tx)).replace('0x','',1)
+	async def txsend(self,txhex):
+		return (await self.rpc.call('eth_sendRawTransaction','0x'+txhex)).replace('0x','',1)
 
 	async def transfer(   self,from_addr,to_addr,amt,key,start_gas,gasPrice,
 					method_sig='transfer(address,uint256)',
@@ -145,10 +148,10 @@ class TokenBase(MMGenObject): # ERC20
 					nonce = int(await self.rpc.call('eth_getTransactionCount','0x'+from_addr,'pending'),16),
 					method_sig = method_sig,
 					from_addr2 = from_addr2 )
-		(hex_tx,coin_txid) = await self.txsign(tx_in,key,from_addr)
-		return await self.txsend(hex_tx)
+		(txhex,coin_txid) = await self.txsign(tx_in,key,from_addr)
+		return await self.txsend(txhex)
 
-class Token(TokenBase):
+class Token(TokenCommon):
 
 	def __init__(self,proto,addr,decimals,rpc=None):
 		if type(self).__name__ == 'Token':
@@ -161,7 +164,7 @@ class Token(TokenBase):
 		self.base_unit = Decimal('10') ** -self.decimals
 		self.rpc = rpc
 
-class TokenResolve(TokenBase,metaclass=AsyncInit):
+class TokenResolve(TokenCommon,metaclass=AsyncInit):
 
 	async def __init__(self,proto,rpc,addr):
 		from ...util import get_keccak

mmgen/cfg.py

@@ -28,6 +28,7 @@ import sys,os,re
 from collections import namedtuple
 
 from .globalvars import *
+from .exception import CfgFileParseError
 from .util import *
 
 def cfg_file(id_str):
@@ -192,17 +193,7 @@ class CfgFileSampleSys(CfgFileSample):
 		else:
 			# self.fn is used for error msgs only, so file need not exist on filesystem
 			self.fn = os.path.join(os.path.dirname(__file__),'data',self.fn_base)
-			# Resource will be unpacked and then cleaned up if necessary, see:
-			#    https://docs.python.org/3/library/importlib.html:
-			#        Note: This module provides functionality similar to pkg_resources Basic
-			#        Resource Access without the performance overhead of that package.
-			#    https://importlib-resources.readthedocs.io/en/latest/migration.html
-			#    https://setuptools.readthedocs.io/en/latest/pkg_resources.html
-			try:
-				from importlib.resources import files # Python 3.9
-			except ImportError:
-				from importlib_resources import files
-			self.data = files('mmgen').joinpath('data',self.fn_base).read_text().splitlines()
+			self.data = g.get_mmgen_data_file(self.fn_base).splitlines()
 
 	def make_metadata(self):
 		return [f'# Version {self.cur_ver} {self.computed_chksum}']

mmgen/common.py

@@ -21,7 +21,6 @@ common.py:  Common imports for all MMGen scripts
 """
 
 import sys,os
-from .exception import *
 from .globalvars import *
 import mmgen.opts as opts
 from .opts import opt

mmgen/daemon.py

@@ -586,6 +586,15 @@ class bitcoin_core_daemon(CoinDaemon):
 	def stop_cmd(self):
 		return self.cli_cmd('stop')
 
+	def set_label_args(self,rpc,coinaddr,lbl):
+		if 'label_api' in rpc.caps:
+			return ('setlabel',coinaddr,lbl)
+		else:
+			# NOTE: this works because importaddress() removes the old account before
+			# associating the new account with the address.
+			# RPC args: addr,label,rescan[=true],p2sh[=none]
+			return ('importaddress',coinaddr,lbl,False)
+
 class bitcoin_cash_node_daemon(bitcoin_core_daemon):
 	daemon_data = _dd('Bitcoin Cash Node', 24000000, '24.0.0')
 	exec_fn = 'bitcoind-bchn'
@@ -598,6 +607,11 @@ class bitcoin_cash_node_daemon(bitcoin_core_daemon):
 	cfg_file_hdr = '# Bitcoin Cash Node config file\n'
 	nonstd_datadir = True
 
+	def set_label_args(self,rpc,coinaddr,lbl):
+		# bitcoin-{abc,bchn} 'setlabel' RPC is broken, so use old 'importaddress' method to set label
+		# Broken behavior: new label is set OK, but old label gets attached to another address
+		return ('importaddress',coinaddr,lbl,False)
+
 class litecoin_core_daemon(bitcoin_core_daemon):
 	daemon_data = _dd('Litecoin Core', 180100, '0.18.1')
 	exec_fn = 'litecoind'

mmgen/devtools.py

@@ -49,12 +49,17 @@ if os.getenv('MMGEN_DEBUG') or os.getenv('MMGEN_TEST_SUITE') or os.getenv('MMGEN
 
 	class MMGenObject(object):
 
+		def print_stack_trace(self,*args,**kwargs):
+			print_stack_trace(*args,**kwargs)
+
 		# Pretty-print any object subclassed from MMGenObject, recursing into sub-objects - WIP
 		def pmsg(self):
 			print(self.pfmt())
+
 		def pdie(self):
 			print(self.pfmt())
 			sys.exit(1)
+
 		def pfmt(self,lvl=0,id_list=[]):
 			scalars = (str,int,float,Decimal)
 			def do_list(out,e,lvl=0,is_dict=False):

mmgen/fileutil.py

@@ -276,7 +276,7 @@ def get_words_from_file(infile,desc,quiet=False):
 def get_data_from_file(infile,desc='data',dash=False,silent=False,binary=False,quiet=False):
 
 	from .opts import opt
-	if not opt.quiet and not silent and not quiet and desc:
+	if not (opt.quiet or silent or quiet):
 		qmsg(f'Getting {desc} from file {infile!r}')
 
 	with _open_or_die(
@@ -294,8 +294,8 @@ def get_data_from_file(infile,desc='data',dash=False,silent=False,binary=False,q
 
 	return data
 
-def _mmgen_decrypt_file_maybe(fn,desc='',quiet=False,silent=False):
-	d = get_data_from_file(fn,desc,binary=True,quiet=quiet,silent=silent)
+def _mmgen_decrypt_file_maybe(fn,desc='data',quiet=False,silent=False):
+	d = get_data_from_file(fn,desc=desc,binary=True,quiet=quiet,silent=silent)
 	from .crypto import mmenc_ext
 	have_enc_ext = get_extension(fn) == mmenc_ext
 	if have_enc_ext or not is_utf8(d):
@@ -305,8 +305,8 @@ def _mmgen_decrypt_file_maybe(fn,desc='',quiet=False,silent=False):
 		d = mmgen_decrypt_retry(d,desc)
 	return d
 
-def get_lines_from_file(fn,desc='',trim_comments=False,quiet=False,silent=False):
-	dec = _mmgen_decrypt_file_maybe(fn,desc,quiet=quiet,silent=silent)
+def get_lines_from_file(fn,desc='data',trim_comments=False,quiet=False,silent=False):
+	dec = _mmgen_decrypt_file_maybe(fn,desc=desc,quiet=quiet,silent=silent)
 	ret = dec.decode().splitlines()
 	if trim_comments:
 		ret = strip_comments(ret)

mmgen/globalvars.py

@@ -53,6 +53,8 @@ class GlobalContext(Lockable):
 	email     = '<mmgen@tuta.io>'
 	Cdates    = '2013-2022'
 
+	is_txprog = prog_name == 'mmgen-regtest' or prog_name.startswith('mmgen-tx')
+
 	stdin_tty = sys.stdin.isatty()
 	stdout = sys.stdout
 	stderr = sys.stderr
@@ -311,24 +313,28 @@ class GlobalContext(Lockable):
 			if name[:11] == 'MMGEN_DEBUG':
 				os.environ[name] = '1'
 
-	def _get_importlib_resources_files(self):
+	def get_mmgen_data_file(self,filename):
 		"""
 		this is an expensive import, so do only when required
 		"""
+		# Resource will be unpacked and then cleaned up if necessary, see:
+		#    https://docs.python.org/3/library/importlib.html:
+		#        Note: This module provides functionality similar to pkg_resources Basic
+		#        Resource Access without the performance overhead of that package.
+		#    https://importlib-resources.readthedocs.io/en/latest/migration.html
+		#    https://setuptools.readthedocs.io/en/latest/pkg_resources.html
 		try:
 			from importlib.resources import files # Python 3.9
 		except ImportError:
 			from importlib_resources import files
-		return files
+		return files('mmgen').joinpath('data',filename).read_text()
 
 	@property
 	def version(self):
-		files = self._get_importlib_resources_files()
-		return files('mmgen').joinpath('data','version').read_text().strip()
+		return self.get_mmgen_data_file('version').strip()
 
 	@property
 	def release_date(self):
-		files = self._get_importlib_resources_files()
-		return files('mmgen').joinpath('data','release_date').read_text().strip()
+		return self.get_mmgen_data_file('release_date').strip()
 
 g = GlobalContext()

mmgen/help.py

@@ -43,6 +43,10 @@ def help_notes_func(proto,po,k):
 			from .seedsplit import MasterShareIdx
 			return MasterShareIdx
 
+		def test_py_log_file():
+			from test.test_py_d.common import log_file
+			return log_file
+
 		def tool_help():
 			from .tool.help import main_help
 			return main_help()

mmgen/main_autosign.py

@@ -25,6 +25,7 @@ from subprocess import run,PIPE,DEVNULL
 from stat import *
 
 from .common import *
+from .color import red
 
 mountpoint   = '/mnt/tx'
 tx_dir       = '/mnt/tx/tx'
@@ -232,8 +233,9 @@ async def sign():
 		if signed_txs and not opt.no_summary:
 			print_summary(signed_txs)
 		if fails:
-			rmsg('\nFailed transactions:')
-			rmsg('  ' + '\n  '.join(sorted(fails)) + '\n')
+			msg('')
+			rmsg('Failed transactions:')
+			msg('  ' + '\n  '.join(red(s) for s in sorted(fails)) + '\n') # avoid the 'less' NL color bug
 		return False if fails else True
 	else:
 		msg('No unsigned transactions')

mmgen/main_txbump.py

@@ -42,7 +42,7 @@ opts_data = {
 -d, --outdir=        d Specify an alternate directory 'd' for output
 -e, --echo-passphrase  Print passphrase to screen when typing it
 -f, --tx-fee=        f Transaction fee, as a decimal {cu} amount or as
-                       {fu} (an integer followed by {fl}).
+                       {fu} (an integer followed by {fl!r}).
                        See FEE SPECIFICATION below.
 -H, --hidden-incog-input-params=f,o  Read hidden incognito data from file
                       'f' at offset 'o' (comma-separated)

mmgen/main_txcreate.py

@@ -40,7 +40,7 @@ opts_data = {
 -E, --fee-estimate-mode=M Specify the network fee estimate mode.  Choices:
                       {fe_all}.  Default: {fe_dfl!r}
 -f, --tx-fee=      f  Transaction fee, as a decimal {cu} amount or as
-                      {fu} (an integer followed by {fl}).
+                      {fu} (an integer followed by {fl!r}).
                       See FEE SPECIFICATION below.  If omitted, fee will be
                       calculated using network fee estimation.
 -g, --tx-gas=      g  Specify start gas amount in Wei (ETH only)

mmgen/main_txdo.py

@@ -44,7 +44,7 @@ opts_data = {
 -E, --fee-estimate-mode=M Specify the network fee estimate mode.  Choices:
                        {fe_all}.  Default: {fe_dfl!r}
 -f, --tx-fee=        f Transaction fee, as a decimal {cu} amount or as
-                       {fu} (an integer followed by {fl}).
+                       {fu} (an integer followed by {fl!r}).
                        See FEE SPECIFICATION below.  If omitted, fee will be
                        calculated using network fee estimation.
 -g, --tx-gas=        g Specify start gas amount in Wei (ETH only)

mmgen/obj.py

@@ -173,6 +173,7 @@ class MMGenListItem(MMGenObject):
 	valid_attrs = set()
 	valid_attrs_extra = set()
 	invalid_attrs = {
+		'print_stack_trace',
 		'pfmt',
 		'pmsg',
 		'pdie',

mmgen/opts.py

@@ -21,7 +21,7 @@ opts.py:  MMGen-specific options processing after generic processing by share.Op
 """
 import sys,os,stat
 
-from .exception import UserOptError
+from .exception import UserOptError,CfgFileParseError
 from .globalvars import g
 from .base_obj import Lockable
 
@@ -419,37 +419,6 @@ def init(
 
 	return po.cmd_args
 
-# DISABLED
-def opt_is_tx_fee(key,val,desc): # 'key' must remain a placeholder
-
-	# contract data or non-standard startgas: disable fee checking
-	if hasattr(opt,'contract_data') and opt.contract_data:
-		return
-	if hasattr(opt,'tx_gas') and opt.tx_gas:
-		return
-
-	from .tx import MMGenTX
-	from .protocol import init_proto_from_opts
-	tx = MMGenTX.New(init_proto_from_opts())
-	# Size of 224 is just a ball-park figure to eliminate the most extreme cases at startup
-	# This check will be performed again once we know the true size
-	ret = tx.feespec2abs(val,224)
-
-	if ret == False:
-		raise UserOptError('{!r}: invalid {}\n(not a {} amount or {} specification)'.format(
-			val,
-			desc,
-			tx.proto.coin.upper(),
-			tx.rel_fee_desc ))
-
-	if ret > tx.proto.max_tx_fee:
-		raise UserOptError('{!r}: invalid {}\n({} > max_tx_fee ({} {}))'.format(
-			val,
-			desc,
-			ret.fmt(fs='1.1'),
-			tx.proto.max_tx_fee,
-			tx.proto.coin.upper() ))
-
 def check_usr_opts(usr_opts): # Raises an exception if any check fails
 
 	def opt_splits(val,sep,n,desc):

mmgen/protocol.py

@@ -67,6 +67,10 @@ class CoinProtocol(MMGenObject):
 				'regtest': '_rt',
 			}[network]
 
+			if 'tx' not in self.mmcaps and g.is_txprog:
+				from .util import die
+				die(1,f'Command {g.prog_name!r} not supported for coin {self.coin}')
+
 			if hasattr(self,'chain_names'):
 				self.chain_name = self.chain_names[0] # first chain name is default
 			else:

mmgen/rpc.py

@@ -303,6 +303,7 @@ class RPCClient(MMGenObject):
 			try:
 				socket.create_connection((host,port),timeout=1).close()
 			except:
+				from .exception import SocketError
 				raise SocketError(f'Unable to connect to {host}:{port}')
 
 		self.http_hdrs = { 'Content-Type': 'application/json' }
@@ -552,7 +553,7 @@ class BitcoinRPCClient(RPCClient,metaclass=AsyncInit):
 
 		fn = self.get_daemon_cfg_fn()
 		try:
-			lines = get_lines_from_file(fn,'',silent=not opt.verbose)
+			lines = get_lines_from_file(fn,'daemon config file',silent=not opt.verbose)
 		except:
 			vmsg(f'Warning: {fn!r} does not exist or is unreadable')
 			return dict((k,None) for k in req_keys)
@@ -571,7 +572,7 @@ class BitcoinRPCClient(RPCClient,metaclass=AsyncInit):
 
 	def get_daemon_auth_cookie(self):
 		fn = self.get_daemon_auth_cookie_fn()
-		return get_lines_from_file(fn,'')[0] if os.access(fn,os.R_OK) else ''
+		return get_lines_from_file(fn,'cookie',quiet=True)[0] if os.access(fn,os.R_OK) else ''
 
 	@staticmethod
 	def make_host_path(wallet):

mmgen/twctl.py

@@ -240,16 +240,7 @@ class TrackingWallet(MMGenObject,metaclass=AsyncInit):
 
 	@write_mode
 	async def set_label(self,coinaddr,lbl):
-		# bitcoin-{abc,bchn} 'setlabel' RPC is broken, so use old 'importaddress' method to set label
-		# broken behavior: new label is set OK, but old label gets attached to another address
-		if 'label_api' in self.rpc.caps and self.proto.coin != 'BCH':
-			args = ('setlabel',coinaddr,lbl)
-		else:
-			# NOTE: this works because importaddress() removes the old account before
-			# associating the new account with the address.
-			# RPC args: addr,label,rescan[=true],p2sh[=none]
-			args = ('importaddress',coinaddr,lbl,False)
-
+		args = self.rpc.daemon.set_label_args( self.rpc, coinaddr, lbl )
 		try:
 			return await self.rpc.call(*args)
 		except Exception as e:

mmgen/txfile.py

@@ -87,7 +87,7 @@ class MMGenTxFile:
 					tx.label = MMGenTxLabel(comment)
 
 			desc = 'number of lines' # four required lines
-			metadata,tx.hex,inputs_data,outputs_data = tx_data
+			( metadata, tx.serialized, inputs_data, outputs_data ) = tx_data
 			assert len(metadata) < 100,'invalid metadata length' # rough check
 			metadata = metadata.split()
 
@@ -123,7 +123,7 @@ class MMGenTxFile:
 
 			desc = 'transaction file hex data'
 			tx.check_txfile_hex_data()
-			desc = 'Ethereum transaction file hex or json data'
+			desc = 'Ethereum RLP or JSON data'
 			tx.parse_txfile_hex_data()
 			desc = 'inputs data'
 			tx.inputs  = eval_io_data(inputs_data,'inputs')

mmgen/util.py

@@ -164,7 +164,7 @@ def fmt(s,indent='',strip_char=None):
 	"de-indent multiple lines of text, or indent with specified string"
 	return indent + ('\n'+indent).join([l.strip(strip_char) for l in s.strip().splitlines()]) + '\n'
 
-def fmt_list(l,fmt='dfl',indent=''):
+def fmt_list(iterable,fmt='dfl',indent=''):
 	"pretty-format a list"
 	sep,lq,rq = {
 		'utf8':      ("“, ”",      "“",    "”"),
@@ -175,7 +175,7 @@ def fmt_list(l,fmt='dfl',indent=''):
 		'min':       (",",         "'",    "'"),
 		'col':       ('\n'+indent, indent, '' ),
 	}[fmt]
-	return lq + sep.join(l) + rq
+	return lq + sep.join(iterable) + rq
 
 def list_gen(*data):
 	"""
@@ -354,24 +354,22 @@ def capfirst(s): # different from str.capitalize() - doesn't downcase any uc in
 def decode_timestamp(s):
 #	tz_save = open('/etc/timezone').read().rstrip()
 	os.environ['TZ'] = 'UTC'
-	ts = time.strptime(s,'%Y%m%d_%H%M%S')
-	t = time.mktime(ts)
 #	os.environ['TZ'] = tz_save
-	return int(t)
+	return int(time.mktime( time.strptime(s,'%Y%m%d_%H%M%S') ))
 
 def make_timestamp(secs=None):
-	t = int(secs) if secs else time.time()
-	return '{:04d}{:02d}{:02d}_{:02d}{:02d}{:02d}'.format(*time.gmtime(t)[:6])
+	return '{:04d}{:02d}{:02d}_{:02d}{:02d}{:02d}'.format(*time.gmtime(
+		int(secs) if secs else time.time() )[:6])
 
 def make_timestr(secs=None):
-	t = int(secs) if secs else time.time()
-	return '{}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}'.format(*time.gmtime(t)[:6])
+	return '{}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}'.format(*time.gmtime(
+		int(secs) if secs else time.time() )[:6])
 
 def secs_to_dhms(secs):
-	dsecs = secs // 3600
+	hrs = secs // 3600
 	return '{}{:02d}:{:02d}:{:02d} h/m/s'.format(
-		('{} day{}, '.format(dsecs//24,suf(dsecs//24)) if dsecs > 24 else ''),
-		dsecs % 24,
+		('{} day{}, '.format(hrs//24,suf(hrs//24)) if hrs > 24 else ''),
+		hrs % 24,
 		(secs // 60) % 60,
 		secs % 60
 	)

scripts/exec_wrapper.py

@@ -92,9 +92,8 @@ exec_wrapper_tracemalloc_setup()
 try:
 	sys.argv.pop(0)
 	exec_wrapper_execed_file = sys.argv[0]
-	with open(sys.argv[0]) as fp:
-		text = fp.read()
-	exec(text)
+	with open(exec_wrapper_execed_file) as fp:
+		exec(fp.read())
 except SystemExit as e:
 	if e.code != 0 and not os.getenv('EXEC_WRAPPER_NO_TRACEBACK'):
 		exec_wrapper_write_traceback()

scripts/tx-v2-to-v3.py

@@ -26,7 +26,8 @@ cmd_args = opts.init(opts_data)
 
 from mmgen.tx import *
 
-if len(cmd_args) != 1: opts.usage()
+if len(cmd_args) != 1:
+	opts.usage()
 
 tx = MMGenTX(cmd_args[0],quiet_open=True)
 tx.write_to_file(ask_tty=False,ask_overwrite=not opt.quiet,ask_write=not opt.quiet)

setup.cfg

@@ -41,14 +41,15 @@ install_requires =
 
 packages =
 	mmgen
-	mmgen.share
-	mmgen.proto
-	mmgen.tool
 	mmgen.base_proto
 	mmgen.base_proto.ethereum
 	mmgen.base_proto.ethereum.pyethereum
 	mmgen.base_proto.ethereum.rlp
 	mmgen.base_proto.ethereum.rlp.sedes
+	mmgen.base_proto.ethereum.tx
+	mmgen.proto
+	mmgen.share
+	mmgen.tool
 
 scripts =
 	cmds/mmgen-addrgen

test/overlay/__init__.py

@@ -37,15 +37,15 @@ def overlay_setup(repo_root):
 		shutil.rmtree(overlay_dir,ignore_errors=True)
 		for d in (
 				'mmgen',
-				'mmgen.data',
-				'mmgen.share',
-				'mmgen.tool',
-				'mmgen.proto',
 				'mmgen.base_proto',
 				'mmgen.base_proto.ethereum',
 				'mmgen.base_proto.ethereum.pyethereum',
 				'mmgen.base_proto.ethereum.rlp',
-				'mmgen.base_proto.ethereum.rlp.sedes' ):
+				'mmgen.base_proto.ethereum.rlp.sedes',
+				'mmgen.data',
+				'mmgen.proto',
+				'mmgen.share',
+				'mmgen.tool' ):
 			process_srcdir(d)
 
 	return overlay_dir

test/overlay/fakemods/crypto.py

@@ -6,6 +6,7 @@ if os.getenv('MMGEN_TEST_SUITE_DETERMINISTIC'):
 	add_user_random_orig = add_user_random
 
 	import sys
+	from hashlib import sha256
 	fake_rand_h = sha256('.'.join(sys.argv).encode())
 
 	def fake_urandom(n):

test/test.py

@@ -74,15 +74,18 @@ import sys,os,time
 
 from include.tests_header import repo_root
 from test.overlay import get_overlay_dir,overlay_setup
+
 overlay_dir = get_overlay_dir(repo_root)
 sys.path.insert(0,overlay_dir)
 
-try: os.unlink(os.path.join(repo_root,'my.err'))
-except: pass
+if not (len(sys.argv) == 2 and sys.argv[1] == 'clean'):
+	'hack: overlay must be set up before mmgen mods are imported'
+	overlay_setup(repo_root)
 
 from mmgen.common import *
-from test.include.common import *
-from test.test_py_d.common import *
+
+try: os.unlink(os.path.join(repo_root,'my.err'))
+except: pass
 
 g.quiet = False # if 'quiet' was set in config file, disable here
 os.environ['MMGEN_QUIET'] = '0' # for this script and spawned scripts
@@ -92,7 +95,7 @@ opts_data = {
 	'text': {
 		'desc': 'Test suite for the MMGen suite',
 		'usage':'[options] [command(s) or metacommand(s)]',
-		'options': f"""
+		'options': """
 -h, --help           Print this help message
 --, --longhelp       Print help message for long options (common options)
 -A, --no-daemon-autostart Don't start and stop daemons automatically
@@ -112,7 +115,7 @@ opts_data = {
 -g, --list-current-cmd-groups List command groups for current configuration
 -n, --names          Display command names instead of descriptions
 -N, --no-timings     Suppress display of timing information
--o, --log            Log commands to file {log_file!r}
+-o, --log            Log commands to file {lf!r}
 -O, --pexpect-spawn  Use pexpect.spawn instead of popen_spawn (much slower,
                      kut does real terminal emulation)
 -p, --pause          Pause between tests, resuming on keypress
@@ -136,15 +139,23 @@ opts_data = {
 If no command is given, the whole test suite is run.
 """
 	},
+	'code': {
+		'options': lambda proto,help_notes,s: s.format(
+				lf = help_notes('test_py_log_file')
+			)
+	}
 }
 
-data_dir = get_data_dir() # include/common.py
-
 # we need some opt values before running opts.init, so parse without initializing:
-_uopts = opts.init(opts_data,parse_only=True).user_opts
+po = opts.init(opts_data,parse_only=True)
+
+from test.include.common import *
+from test.test_py_d.common import *
+
+data_dir = get_data_dir() # include/common.py
 
 # step 1: delete data_dir symlink in ./test;
-if not ('resume' in _uopts or 'skip_deps' in _uopts):
+if not ('resume' in po.user_opts or 'skip_deps' in po.user_opts):
 	try: os.unlink(data_dir)
 	except: pass
 
@@ -773,7 +784,8 @@ class TestSuiteRunner(object):
 			start_test_daemons(network_id,remove_datadir=True)
 			self.daemons_started = True
 
-		os.environ['MMGEN_BOGUS_WALLET_DATA'] = '' # zero this here, so test group doesn't have to
+		os.environ['MMGEN_BOGUS_WALLET_DATA'] = '' # zero this here, so test groups don't have to
+
 		self.ts = self.gm.gm_init_group(self,gname,self.spawn_wrapper)
 		self.ts_clsname = type(self.ts).__name__
 
@@ -796,7 +808,6 @@ class TestSuiteRunner(object):
 		self.start_time = time.time()
 		self.daemons_started = False
 		gname_save = None
-		overlay_setup(repo_root)
 		if usr_args:
 			for arg in usr_args:
 				if arg in self.gm.cmd_groups:

test/test_py_d/ts_ethdev.py

@@ -842,7 +842,6 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		usr_addrs = [tool_cmd(cmdname='gen_addr',proto=self.proto).gen_addr(addr,dfl_words_file) for addr in usr_mmaddrs]
 
 		from mmgen.base_proto.ethereum.contract import TokenResolve
-		from mmgen.base_proto.ethereum.tx import EthereumMMGenTX as etx
 		async def do_transfer(rpc):
 			for i in range(2):
 				tk = await TokenResolve(

test/test_py_d/ts_main.py

@@ -197,6 +197,9 @@ class TestSuiteMain(TestSuiteBase,TestSuiteShared):
 			self.tx_fee     = {'btc':'0.0001','bch':'0.001','ltc':'0.01'}[self.proto.coin.lower()]
 			self.txbump_fee = {'btc':'123s','bch':'567s','ltc':'12345s'}[self.proto.coin.lower()]
 
+		self.unspent_data_file = joinpath('test','trash','unspent.json')
+		os.environ['MMGEN_BOGUS_WALLET_DATA'] = self.unspent_data_file
+
 	def _get_addrfile_checksum(self,display=False):
 		addrfile = self.get_file_with_ext('addrs')
 		silence()
@@ -327,16 +330,9 @@ class TestSuiteMain(TestSuiteBase,TestSuiteShared):
 		return self.walletchk(wf,pf,wcls=wcls,dfl_wallet=dfl_wallet)
 
 	def _write_fake_data_to_file(self,d):
-		unspent_data_file = joinpath(self.tmpdir,'unspent.json')
-		write_data_to_file(unspent_data_file,d,'Unspent outputs',quiet=True,ignore_opt_outdir=True)
-		os.environ['MMGEN_BOGUS_WALLET_DATA'] = unspent_data_file
-		bwd_msg = f'MMGEN_BOGUS_WALLET_DATA={unspent_data_file}'
-		if opt.print_cmdline:
-			msg(bwd_msg)
-		if opt.log:
-			self.tr.log_fd.write(bwd_msg + ' ')
+		write_data_to_file(self.unspent_data_file,d,'Unspent outputs',quiet=True,ignore_opt_outdir=True)
 		if opt.verbose or opt.exact_output:
-			sys.stderr.write(f'Fake transaction wallet data written to file {unspent_data_file!r}\n')
+			sys.stderr.write(f'Fake transaction wallet data written to file {self.unspent_data_file!r}\n')
 
 	def _create_fake_unspent_entry(self,coinaddr,al_id=None,idx=None,lbl=None,non_mmgen=False,segwit=False):
 		if 'S' not in self.proto.mmtypes: segwit = False

test/test_py_d/ts_misc.py

@@ -69,16 +69,15 @@ class TestSuiteHelp(TestSuiteBase):
 
 	def helpscreens(self,arg='--help',scripts=(),expect='USAGE:.*OPTIONS:'):
 
-		scripts = scripts or tuple(s.replace('mmgen-','') for s in os.listdir('cmds'))
+		scripts = list(scripts) or [s.replace('mmgen-','') for s in os.listdir('cmds')]
 
-		if self.test_name.endswith('helpscreens'):
-			skip = (
-				['regtest','xmrwallet'] if self.proto.base_coin == 'ETH' else
-				['regtest'] if self.proto.base_coin == 'XMR' else
-				[] )
-			scripts = sorted( set(scripts) - set(skip) )
+		if 'tx' not in self.proto.mmcaps:
+			scripts = [s for s in scripts if not (s == 'regtest' or s.startswith('tx'))]
 
-		for s in scripts:
+		if self.proto.coin not in ('BTC','XMR') and 'xmrwallet' in scripts:
+			scripts.remove('xmrwallet')
+
+		for s in sorted(scripts):
 			t = self.spawn(f'mmgen-{s}',[arg],extra_desc=f'(mmgen-{s})')
 			t.expect(expect,regex=True)
 			t.read()