test suite: run scripts from overlay tree

Rationale of this commit: to relocate some ugly test-related code from the MMGen
package tree to the test tree, as well as to enable deterministic testing
(implemented in the next commit).

The overlay tree is a symlinked mirror of the MMGen package dir with a few
monkey-patched modules.

The monkey-patching is conditional, so the modules are certain to get tested in
their unpatched state as well.

MANIFEST.in

@@ -19,6 +19,8 @@ include test/ref/monero/*
 include test/ref/ethereum/bin/mm1/*
 include test/ref/ethereum/bin/mm2/*
 include test/misc/*.py
+include test/overlay/*.py
+include test/overlay/fakemods/*.py
 
 include test/test-release.sh
 

mmgen/globalvars.py

@@ -122,7 +122,6 @@ class GlobalContext(Lockable):
 	enable_erigon = False
 
 	# test suite:
-	bogus_wallet_data    = ''
 	bogus_send           = False
 	debug_utf8           = False
 	traceback            = False
@@ -209,7 +208,6 @@ class GlobalContext(Lockable):
 		'MMGEN_TEST_SUITE_REGTEST',
 		'MMGEN_TEST_SUITE_POPEN_SPAWN',
 		'MMGEN_TERMINAL_WIDTH',
-		'MMGEN_BOGUS_WALLET_DATA',
 		'MMGEN_BOGUS_SEND',
 		'MMGEN_DEBUG',
 		'MMGEN_DEBUG_OPTS',

mmgen/rpc.py

@@ -772,7 +772,4 @@ async def rpc_init(proto,backend=None,daemon=None,ignore_daemon_version=False):
 			RPC client chain:   {rpc.chain}
 			""",indent='  ').rstrip())
 
-	if g.bogus_wallet_data:
-		rpc.blockcount = 1000000
-
 	return rpc

mmgen/tw.py

@@ -44,28 +44,6 @@ def get_tw_label(proto,s):
 	except:
 		return None
 
-_date_formatter = {
-	'days':      lambda rpc,secs: (rpc.cur_date - secs) // 86400,
-	'date':      lambda rpc,secs: '{}-{:02}-{:02}'.format(*time.gmtime(secs)[:3])[2:],
-	'date_time': lambda rpc,secs: '{}-{:02}-{:02} {:02}:{:02}'.format(*time.gmtime(secs)[:5]),
-}
-
-if os.getenv('MMGEN_BOGUS_WALLET_DATA'):
-	# 1831006505 (09 Jan 2028) = projected time of block 1000000
-	_date_formatter['days'] = lambda rpc,secs: (1831006505 - secs) // 86400
-	async def _set_dates(rpc,us):
-		for o in us:
-			o.date = 1831006505 - int(9.7 * 60 * (o.confs - 1))
-else:
-	async def _set_dates(rpc,us):
-		if rpc.proto.base_proto != 'Bitcoin':
-			return
-		if us and us[0].date is None:
-			# 'blocktime' differs from 'time', is same as getblockheader['time']
-			dates = [o['blocktime'] for o in await rpc.gathered_call('gettransaction',[(o.txid,) for o in us])]
-			for idx,o in enumerate(us):
-				o.date = dates[idx]
-
 class TwUnspentOutputs(MMGenObject,metaclass=AsyncInit):
 
 	def __new__(cls,proto,*args,**kwargs):
@@ -173,10 +151,8 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, add [l]abel:
 		return await self.rpc.call('listunspent',self.minconf,*add_args)
 
 	async def get_unspent_data(self,sort_key=None,reverse_sort=False):
-		if g.bogus_wallet_data: # for debugging and test suite
-			us_raw = json.loads(get_data_from_file(g.bogus_wallet_data),parse_float=Decimal)
-		else:
-			us_raw = await self.get_unspent_rpc()
+
+		us_raw = await self.get_unspent_rpc()
 
 		if not us_raw:
 			die(0,fmt(f"""
@@ -263,10 +239,20 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, add [l]abel:
 		dc = namedtuple('display_constants',['col1_w','mmid_w','addr_w','btaddr_w','label_w','tx_w','txdots'])
 		return dc(col1_w,mmid_w,addr_w,btaddr_w,label_w,tx_w,txdots)
 
+	@staticmethod
+	async def set_dates(rpc,us):
+		if rpc.proto.base_proto != 'Bitcoin':
+			return
+		if us and us[0].date is None:
+			# 'blocktime' differs from 'time', is same as getblockheader['time']
+			dates = [o['blocktime'] for o in await rpc.gathered_call('gettransaction',[(o.txid,) for o in us])]
+			for idx,o in enumerate(us):
+				o.date = dates[idx]
+
 	async def format_for_display(self):
 		unsp = self.unspent
 		if self.age_fmt in self.age_fmts_date_dependent:
-			await _set_dates(self.rpc,unsp)
+			await self.set_dates(self.rpc,unsp)
 		self.set_term_columns()
 
 		c = getattr(self,'display_constants',None)
@@ -344,7 +330,7 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, add [l]abel:
 		return self.fmt_display
 
 	async def format_for_printing(self,color=False,show_confs=True):
-		await _set_dates(self.rpc,self.unspent)
+		await self.set_dates(self.rpc,self.unspent)
 		addr_w = max(len(i.addr) for i in self.unspent)
 		mmid_w = max(len(('',i.twmmid)[i.twmmid.type=='mmgen']) for i in self.unspent) or 12 # DEADBEEF:S:1
 		amt_w = self.proto.coin_amt.max_prec + 5
@@ -544,12 +530,20 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, add [l]abel:
 		elif age_fmt == 'block':
 			return self.rpc.blockcount - (o.confs - 1)
 		else:
-			return _date_formatter[age_fmt](self.rpc,o.date)
+			return self.date_formatter[age_fmt](self.rpc,o.date)
+
+	date_formatter = {
+		'days':      lambda rpc,secs: (rpc.cur_date - secs) // 86400,
+		'date':      lambda rpc,secs: '{}-{:02}-{:02}'.format(*time.gmtime(secs)[:3])[2:],
+		'date_time': lambda rpc,secs: '{}-{:02}-{:02} {:02}:{:02}'.format(*time.gmtime(secs)[:5]),
+	}
+
 
 class TwAddrList(MMGenDict,metaclass=AsyncInit):
 	has_age = True
 	age_fmts = TwUnspentOutputs.age_fmts
 	age_disp = TwUnspentOutputs.age_disp
+	date_formatter = TwUnspentOutputs.date_formatter
 
 	def __new__(cls,proto,*args,**kwargs):
 		return MMGenDict.__new__(altcoin_subclass(cls,proto,'tw'),*args,**kwargs)
@@ -665,7 +659,9 @@ class TwAddrList(MMGenDict,metaclass=AsyncInit):
 
 		mmids = sorted(self,key=sort_algo,reverse=bool(sort and 'reverse' in sort))
 		if show_age:
-			await _set_dates(self.rpc,[o for o in mmids if hasattr(o,'confs')])
+			await TwUnspentOutputs.set_dates(
+				self.rpc,
+				[o for o in mmids if hasattr(o,'confs')] )
 
 		def gen_output():
 

scripts/exec_wrapper.py

@@ -13,20 +13,24 @@ def exec_wrapper_get_colors():
 			(lambda s,n=n:f'\033[{n};1m{s}\033[0m' )
 		for n in (31,32,33,34) ])
 
-def exec_wrapper_init():
+def exec_wrapper_init(): # don't change: name is used to test if script is running under exec_wrapper
 
-	sys.path[0] = 'test' if os.path.dirname(sys.argv[1]) == 'test' else '.'
+	if os.path.dirname(sys.argv[1]) == 'test': # scripts in ./test do overlay setup themselves
+		sys.path[0] = 'test'
+	else:
+		from test.overlay import overlay_setup
+		sys.path[0] = overlay_setup()
 
 	os.environ['MMGEN_TRACEBACK'] = '1'
 	os.environ['PYTHONPATH'] = '.'
 	if 'TMUX' in os.environ:
 		del os.environ['TMUX']
 
-	of = 'my.err'
-	try: os.unlink(of)
-	except: pass
-
-	return of
+	if not os.getenv('EXEC_WRAPPER_NO_TRACEBACK'):
+		try:
+			os.unlink('my.err')
+		except:
+			pass
 
 def exec_wrapper_write_traceback():
 	import traceback,re
@@ -43,9 +47,15 @@ def exec_wrapper_write_traceback():
 	c = exec_wrapper_get_colors()
 	sys.stdout.write('{}{}'.format(c.yellow(''.join(lines)),c.red(exc)))
 
-	open(exec_wrapper_traceback_file,'w').write(''.join(lines+[exc]))
+	open('my.err','w').write(''.join(lines+[exc]))
+
+def exec_wrapper_end_msg():
+	if os.getenv('EXEC_WRAPPER_SPAWN'):
+		c = exec_wrapper_get_colors()
+		# write to stdout to ensure script output gets to terminal first
+		sys.stdout.write(c.blue('Runtime: {:0.5f} secs\n'.format(time.time() - exec_wrapper_tstart)))
 
-exec_wrapper_traceback_file = exec_wrapper_init() # sets sys.path[0]
+exec_wrapper_init() # sets sys.path[0]
 exec_wrapper_tstart = time.time()
 
 try:
@@ -53,13 +63,14 @@ try:
 	exec_wrapper_execed_file = sys.argv[0]
 	exec(open(sys.argv[0]).read())
 except SystemExit as e:
-	if e.code != 0:
+	if e.code != 0 and not os.getenv('EXEC_WRAPPER_NO_TRACEBACK'):
 		exec_wrapper_write_traceback()
+	else:
+		exec_wrapper_end_msg()
 	sys.exit(e.code)
 except Exception as e:
 	exec_wrapper_write_traceback()
 	retval = e.mmcode if hasattr(e,'mmcode') else e.code if hasattr(e,'code') else 1
 	sys.exit(retval)
 
-c = exec_wrapper_get_colors()
-sys.stderr.write(c.blue('Runtime: {:0.5f} secs\n'.format(time.time() - exec_wrapper_tstart)))
+exec_wrapper_end_msg()

test/gentest.py

@@ -21,10 +21,10 @@ test/gentest.py:  Cryptocoin key/address generation tests for the MMGen suite
 """
 
 import sys,os
-pn = os.path.dirname(sys.argv[0])
-os.chdir(os.path.join(pn,os.pardir))
-sys.path.__setitem__(0,os.path.abspath(os.curdir))
-os.environ['MMGEN_TEST_SUITE'] = '1'
+
+from include.tests_header import repo_root
+from test.overlay import overlay_setup
+sys.path.insert(0,overlay_setup())
 
 # Import these _after_ local path's been added to sys.path
 from mmgen.common import *

test/objtest.py

@@ -20,10 +20,11 @@
 test/objtest.py:  Test MMGen data objects
 """
 
-import sys,os
-pn = os.path.dirname(sys.argv[0])
-os.chdir(os.path.join(pn,os.pardir))
-sys.path.__setitem__(0,os.path.abspath(os.curdir))
+import sys,os,re
+
+from include.tests_header import repo_root
+from test.overlay import overlay_setup
+sys.path.insert(0,overlay_setup())
 
 os.environ['MMGEN_TEST_SUITE'] = '1'
 

test/overlay/__init__.py

@@ -0,0 +1,42 @@
+import sys,os
+
+def overlay_setup():
+
+	def process_srcdir(d):
+		srcdir = os.path.join(repo_root,*d.split('.'))
+		destdir = os.path.join(overlay_dir,*d.split('.'))
+		os.makedirs(destdir)
+		for fn in os.listdir(srcdir):
+			if (
+				fn.endswith('.py') or
+				d == 'mmgen.data' or
+				d == 'mmgen' and fn.startswith('secp256k1')
+			):
+				if fn in fakemods:
+					os.symlink(
+						os.path.join(fakemod_dir,fn),
+						os.path.join(destdir,fn) )
+					link_fn = fn.removesuffix('.py') + '_orig.py'
+				else:
+					link_fn = fn
+				os.symlink(
+					os.path.join(srcdir,fn),
+					os.path.join(destdir,link_fn) )
+
+	repo_root = os.path.dirname(os.path.abspath(os.path.dirname(sys.argv[0])))
+	overlay_dir = os.path.join(repo_root,'test','overlay','tree')
+	fakemod_dir = os.path.join(repo_root,'test','overlay','fakemods')
+	fakemods  = os.listdir(fakemod_dir)
+	if not os.path.exists(os.path.join(overlay_dir,'mmgen','main.py')):
+		for d in (
+				'mmgen',
+				'mmgen.data',
+				'mmgen.share',
+				'mmgen.altcoins',
+				'mmgen.altcoins.eth',
+				'mmgen.altcoins.eth.pyethereum',
+				'mmgen.altcoins.eth.rlp',
+				'mmgen.altcoins.eth.rlp.sedes' ):
+			process_srcdir(d)
+
+	return overlay_dir

test/overlay/fakemods/rpc.py

@@ -0,0 +1,17 @@
+from .rpc_orig import *
+
+if os.getenv('MMGEN_BOGUS_WALLET_DATA'):
+
+	rpc_init_orig = rpc_init
+
+	async def rpc_init(proto,backend=None,daemon=None,ignore_daemon_version=False):
+
+		ret = await rpc_init_orig(
+			proto = proto,
+			backend = backend,
+			daemon = daemon,
+			ignore_daemon_version = ignore_daemon_version )
+
+		ret.blockcount = 1000000
+
+		return ret

test/overlay/fakemods/tw.py

@@ -0,0 +1,15 @@
+from .tw_orig import *
+
+if os.getenv('MMGEN_BOGUS_WALLET_DATA'):
+	# 1831006505 (09 Jan 2028) = projected time of block 1000000
+	TwUnspentOutputs.date_formatter['days'] = lambda rpc,secs: (1831006505 - secs) // 86400
+
+	async def fake_set_dates(foo,rpc,us):
+		for o in us:
+			o.date = 1831006505 - int(9.7 * 60 * (o.confs - 1))
+
+	async def fake_get_unspent_rpc(foo):
+		return json.loads(get_data_from_file(os.getenv('MMGEN_BOGUS_WALLET_DATA')),parse_float=Decimal)
+
+	TwUnspentOutputs.set_dates = fake_set_dates
+	TwUnspentOutputs.get_unspent_rpc = fake_get_unspent_rpc

test/test.py

@@ -73,6 +73,9 @@ def create_shm_dir(data_dir,trash_dir):
 import sys,os,time
 
 from include.tests_header import repo_root
+from test.overlay import overlay_setup
+overlay_dir = overlay_setup()
+sys.path.insert(0,overlay_dir)
 
 try: os.unlink(os.path.join(repo_root,'my.err'))
 except: pass
@@ -124,7 +127,6 @@ opts_data = {
 -T, --pexpect-timeout=T Set the timeout for pexpect
 -v, --verbose        Produce more verbose output
 -W, --no-dw-delete   Don't remove default wallet from data dir after dw tests are done
--x, --exec-wrapper   Run the command inside the '{ew}' script
 -X, --exit-after=C   Exit after command 'C'
 -y, --segwit         Generate and use Segwit addresses
 -Y, --segwit-random  Generate and use a random mix of Segwit and Legacy addrs
@@ -424,7 +426,7 @@ def do_between():
 def list_tmpdirs():
 	return {k:cfgs[k]['tmpdir'] for k in cfgs}
 
-def clean(usr_dirs=None):
+def clean(usr_dirs=None,clean_overlay=True):
 	if opt.skip_deps:
 		return
 	all_dirs = list_tmpdirs()
@@ -444,6 +446,10 @@ def clean(usr_dirs=None):
 	cleandir(trash_dir)
 	iqmsg(green(f'Cleaned directories {data_dir!r} {trash_dir!r}'))
 
+	if clean_overlay:
+		cleandir(overlay_dir)
+		iqmsg(green(f'Cleaned directory {os.path.relpath(overlay_dir)!r}'))
+
 def create_tmp_dirs(shm_dir):
 	if g.platform == 'win':
 		for cfg in sorted(cfgs):
@@ -478,10 +484,6 @@ def set_environ_for_spawned_scripts():
 	if not opt.buf_keypress:
 		os.environ['MMGEN_DISABLE_HOLD_PROTECT'] = '1'
 
-	# If test.py itself is running under exec_wrapper, the spawned script shouldn't be, so disable this:
-	if os.getenv('MMGEN_TRACEBACK') and not opt.exec_wrapper:
-		os.environ['MMGEN_TRACEBACK'] = ''
-
 	os.environ['MMGEN_NO_LICENSE'] = '1'
 	os.environ['MMGEN_MIN_URANDCHARS'] = '3'
 	os.environ['MMGEN_BOGUS_SEND'] = '1'
@@ -674,7 +676,7 @@ class TestSuiteRunner(object):
 
 		args = [cmd] + passthru_opts + self.ts.extra_spawn_args + args
 
-		if opt.exec_wrapper and not no_exec_wrapper:
+		if not no_exec_wrapper:
 			args = ['scripts/exec_wrapper.py'] + args
 
 		if g.platform == 'win':
@@ -717,8 +719,13 @@ class TestSuiteRunner(object):
 
 		os.environ['MMGEN_FORCE_COLOR'] = '1' if self.ts.color else ''
 
+		env = { 'EXEC_WRAPPER_SPAWN':'1' }
+		if 'exec_wrapper_init' in globals(): # Python 3.9: OR the dicts
+			env.update({ 'EXEC_WRAPPER_NO_TRACEBACK':'1' })
+		env.update(os.environ)
+
 		from test.include.pexpect import MMGenPexpect
-		return MMGenPexpect(args,no_output=no_output)
+		return MMGenPexpect( args, no_output=no_output, env=env )
 
 	def end_msg(self):
 		t = int(time.time() - self.start_time)
@@ -732,7 +739,7 @@ class TestSuiteRunner(object):
 		ts_cls = CmdGroupMgr().load_mod(gname)
 
 		if do_clean:
-			clean(ts_cls.tmpdir_nums)
+			clean(ts_cls.tmpdir_nums,clean_overlay=False)
 
 		for k in ('segwit','segwit_random','bech32'):
 			if getattr(opt,k):

test/tooltest.py

@@ -24,6 +24,9 @@ import sys,os,binascii
 from subprocess import run,PIPE
 
 from include.tests_header import repo_root
+from test.overlay import overlay_setup
+sys.path.insert(0,overlay_setup())
+
 from mmgen.common import *
 from test.include.common import *
 
@@ -130,7 +133,8 @@ if not opt.system:
 	os.environ['PYTHONPATH'] = repo_root
 	mmgen_cmd = os.path.relpath(os.path.join(repo_root,'cmds',mmgen_cmd))
 
-spawn_cmd = [mmgen_cmd]
+spawn_cmd = ['scripts/exec_wrapper.py',mmgen_cmd]
+
 if opt.coverage:
 	d,f = init_coverage()
 	spawn_cmd = ['python3','-m','trace','--count','--coverdir='+d,'--file='+f] + spawn_cmd

test/tooltest2.py

@@ -28,6 +28,9 @@ from subprocess import run,PIPE
 from decimal import Decimal
 
 from include.tests_header import repo_root
+from test.overlay import overlay_setup
+sys.path.insert(0,overlay_setup())
+
 from mmgen.common import *
 from test.include.common import *
 from mmgen.wallet import is_bip39_mnemonic,is_mmgen_mnemonic
@@ -819,6 +822,7 @@ async def run_test(gid,cmd_name):
 
 	def fork_cmd(cmd_name,args,out,opts):
 		cmd = (
+			['scripts/exec_wrapper.py'] +
 			list(tool_cmd) +
 			(opts or []) +
 			[cmd_name] + args