From 96a250b51d0d0ebb44ed0fc5a84d12750b2e2f05 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 3 Oct 2021 17:40:02 +0000 Subject: [PATCH] test suite: run scripts from overlay tree Rationale of this commit: to relocate some ugly test-related code from the MMGen package tree to the test tree, as well as to enable deterministic testing (implemented in the next commit). The overlay tree is a symlinked mirror of the MMGen package dir with a few monkey-patched modules. The monkey-patching is conditional, so the modules are certain to get tested in their unpatched state as well. --- MANIFEST.in | 2 ++ mmgen/globalvars.py | 2 -- mmgen/rpc.py | 3 -- mmgen/tw.py | 56 +++++++++++++++++------------------- scripts/exec_wrapper.py | 35 ++++++++++++++-------- test/gentest.py | 8 +++--- test/objtest.py | 9 +++--- test/overlay/__init__.py | 42 +++++++++++++++++++++++++++ test/overlay/fakemods/rpc.py | 17 +++++++++++ test/overlay/fakemods/tw.py | 15 ++++++++++ test/test.py | 25 ++++++++++------ test/tooltest.py | 6 +++- test/tooltest2.py | 4 +++ 13 files changed, 159 insertions(+), 65 deletions(-) create mode 100644 test/overlay/__init__.py create mode 100644 test/overlay/fakemods/rpc.py create mode 100644 test/overlay/fakemods/tw.py diff --git a/MANIFEST.in b/MANIFEST.in index 0377fc8d..f22a0e95 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -19,6 +19,8 @@ include test/ref/monero/* include test/ref/ethereum/bin/mm1/* include test/ref/ethereum/bin/mm2/* include test/misc/*.py +include test/overlay/*.py +include test/overlay/fakemods/*.py include test/test-release.sh diff --git a/mmgen/globalvars.py b/mmgen/globalvars.py index f4f942af..737a045e 100755 --- a/mmgen/globalvars.py +++ b/mmgen/globalvars.py @@ -122,7 +122,6 @@ class GlobalContext(Lockable): enable_erigon = False # test suite: - bogus_wallet_data = '' bogus_send = False debug_utf8 = False traceback = False @@ -209,7 +208,6 @@ class GlobalContext(Lockable): 'MMGEN_TEST_SUITE_REGTEST', 'MMGEN_TEST_SUITE_POPEN_SPAWN', 'MMGEN_TERMINAL_WIDTH', - 'MMGEN_BOGUS_WALLET_DATA', 'MMGEN_BOGUS_SEND', 'MMGEN_DEBUG', 'MMGEN_DEBUG_OPTS', diff --git a/mmgen/rpc.py b/mmgen/rpc.py index e0a722b4..3e74a284 100755 --- a/mmgen/rpc.py +++ b/mmgen/rpc.py @@ -772,7 +772,4 @@ async def rpc_init(proto,backend=None,daemon=None,ignore_daemon_version=False): RPC client chain: {rpc.chain} """,indent=' ').rstrip()) - if g.bogus_wallet_data: - rpc.blockcount = 1000000 - return rpc diff --git a/mmgen/tw.py b/mmgen/tw.py index d779426d..74292cb1 100755 --- a/mmgen/tw.py +++ b/mmgen/tw.py @@ -44,28 +44,6 @@ def get_tw_label(proto,s): except: return None -_date_formatter = { - 'days': lambda rpc,secs: (rpc.cur_date - secs) // 86400, - 'date': lambda rpc,secs: '{}-{:02}-{:02}'.format(*time.gmtime(secs)[:3])[2:], - 'date_time': lambda rpc,secs: '{}-{:02}-{:02} {:02}:{:02}'.format(*time.gmtime(secs)[:5]), -} - -if os.getenv('MMGEN_BOGUS_WALLET_DATA'): - # 1831006505 (09 Jan 2028) = projected time of block 1000000 - _date_formatter['days'] = lambda rpc,secs: (1831006505 - secs) // 86400 - async def _set_dates(rpc,us): - for o in us: - o.date = 1831006505 - int(9.7 * 60 * (o.confs - 1)) -else: - async def _set_dates(rpc,us): - if rpc.proto.base_proto != 'Bitcoin': - return - if us and us[0].date is None: - # 'blocktime' differs from 'time', is same as getblockheader['time'] - dates = [o['blocktime'] for o in await rpc.gathered_call('gettransaction',[(o.txid,) for o in us])] - for idx,o in enumerate(us): - o.date = dates[idx] - class TwUnspentOutputs(MMGenObject,metaclass=AsyncInit): def __new__(cls,proto,*args,**kwargs): @@ -173,10 +151,8 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, add [l]abel: return await self.rpc.call('listunspent',self.minconf,*add_args) async def get_unspent_data(self,sort_key=None,reverse_sort=False): - if g.bogus_wallet_data: # for debugging and test suite - us_raw = json.loads(get_data_from_file(g.bogus_wallet_data),parse_float=Decimal) - else: - us_raw = await self.get_unspent_rpc() + + us_raw = await self.get_unspent_rpc() if not us_raw: die(0,fmt(f""" @@ -263,10 +239,20 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, add [l]abel: dc = namedtuple('display_constants',['col1_w','mmid_w','addr_w','btaddr_w','label_w','tx_w','txdots']) return dc(col1_w,mmid_w,addr_w,btaddr_w,label_w,tx_w,txdots) + @staticmethod + async def set_dates(rpc,us): + if rpc.proto.base_proto != 'Bitcoin': + return + if us and us[0].date is None: + # 'blocktime' differs from 'time', is same as getblockheader['time'] + dates = [o['blocktime'] for o in await rpc.gathered_call('gettransaction',[(o.txid,) for o in us])] + for idx,o in enumerate(us): + o.date = dates[idx] + async def format_for_display(self): unsp = self.unspent if self.age_fmt in self.age_fmts_date_dependent: - await _set_dates(self.rpc,unsp) + await self.set_dates(self.rpc,unsp) self.set_term_columns() c = getattr(self,'display_constants',None) @@ -344,7 +330,7 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, add [l]abel: return self.fmt_display async def format_for_printing(self,color=False,show_confs=True): - await _set_dates(self.rpc,self.unspent) + await self.set_dates(self.rpc,self.unspent) addr_w = max(len(i.addr) for i in self.unspent) mmid_w = max(len(('',i.twmmid)[i.twmmid.type=='mmgen']) for i in self.unspent) or 12 # DEADBEEF:S:1 amt_w = self.proto.coin_amt.max_prec + 5 @@ -544,12 +530,20 @@ Actions: [q]uit view, [p]rint to file, pager [v]iew, [w]ide view, add [l]abel: elif age_fmt == 'block': return self.rpc.blockcount - (o.confs - 1) else: - return _date_formatter[age_fmt](self.rpc,o.date) + return self.date_formatter[age_fmt](self.rpc,o.date) + + date_formatter = { + 'days': lambda rpc,secs: (rpc.cur_date - secs) // 86400, + 'date': lambda rpc,secs: '{}-{:02}-{:02}'.format(*time.gmtime(secs)[:3])[2:], + 'date_time': lambda rpc,secs: '{}-{:02}-{:02} {:02}:{:02}'.format(*time.gmtime(secs)[:5]), + } + class TwAddrList(MMGenDict,metaclass=AsyncInit): has_age = True age_fmts = TwUnspentOutputs.age_fmts age_disp = TwUnspentOutputs.age_disp + date_formatter = TwUnspentOutputs.date_formatter def __new__(cls,proto,*args,**kwargs): return MMGenDict.__new__(altcoin_subclass(cls,proto,'tw'),*args,**kwargs) @@ -665,7 +659,9 @@ class TwAddrList(MMGenDict,metaclass=AsyncInit): mmids = sorted(self,key=sort_algo,reverse=bool(sort and 'reverse' in sort)) if show_age: - await _set_dates(self.rpc,[o for o in mmids if hasattr(o,'confs')]) + await TwUnspentOutputs.set_dates( + self.rpc, + [o for o in mmids if hasattr(o,'confs')] ) def gen_output(): diff --git a/scripts/exec_wrapper.py b/scripts/exec_wrapper.py index 6f349300..c0d5eee0 100755 --- a/scripts/exec_wrapper.py +++ b/scripts/exec_wrapper.py @@ -13,20 +13,24 @@ def exec_wrapper_get_colors(): (lambda s,n=n:f'\033[{n};1m{s}\033[0m' ) for n in (31,32,33,34) ]) -def exec_wrapper_init(): +def exec_wrapper_init(): # don't change: name is used to test if script is running under exec_wrapper - sys.path[0] = 'test' if os.path.dirname(sys.argv[1]) == 'test' else '.' + if os.path.dirname(sys.argv[1]) == 'test': # scripts in ./test do overlay setup themselves + sys.path[0] = 'test' + else: + from test.overlay import overlay_setup + sys.path[0] = overlay_setup() os.environ['MMGEN_TRACEBACK'] = '1' os.environ['PYTHONPATH'] = '.' if 'TMUX' in os.environ: del os.environ['TMUX'] - of = 'my.err' - try: os.unlink(of) - except: pass - - return of + if not os.getenv('EXEC_WRAPPER_NO_TRACEBACK'): + try: + os.unlink('my.err') + except: + pass def exec_wrapper_write_traceback(): import traceback,re @@ -43,9 +47,15 @@ def exec_wrapper_write_traceback(): c = exec_wrapper_get_colors() sys.stdout.write('{}{}'.format(c.yellow(''.join(lines)),c.red(exc))) - open(exec_wrapper_traceback_file,'w').write(''.join(lines+[exc])) + open('my.err','w').write(''.join(lines+[exc])) -exec_wrapper_traceback_file = exec_wrapper_init() # sets sys.path[0] +def exec_wrapper_end_msg(): + if os.getenv('EXEC_WRAPPER_SPAWN'): + c = exec_wrapper_get_colors() + # write to stdout to ensure script output gets to terminal first + sys.stdout.write(c.blue('Runtime: {:0.5f} secs\n'.format(time.time() - exec_wrapper_tstart))) + +exec_wrapper_init() # sets sys.path[0] exec_wrapper_tstart = time.time() try: @@ -53,13 +63,14 @@ try: exec_wrapper_execed_file = sys.argv[0] exec(open(sys.argv[0]).read()) except SystemExit as e: - if e.code != 0: + if e.code != 0 and not os.getenv('EXEC_WRAPPER_NO_TRACEBACK'): exec_wrapper_write_traceback() + else: + exec_wrapper_end_msg() sys.exit(e.code) except Exception as e: exec_wrapper_write_traceback() retval = e.mmcode if hasattr(e,'mmcode') else e.code if hasattr(e,'code') else 1 sys.exit(retval) -c = exec_wrapper_get_colors() -sys.stderr.write(c.blue('Runtime: {:0.5f} secs\n'.format(time.time() - exec_wrapper_tstart))) +exec_wrapper_end_msg() diff --git a/test/gentest.py b/test/gentest.py index 3eca8618..25d8e0ca 100755 --- a/test/gentest.py +++ b/test/gentest.py @@ -21,10 +21,10 @@ test/gentest.py: Cryptocoin key/address generation tests for the MMGen suite """ import sys,os -pn = os.path.dirname(sys.argv[0]) -os.chdir(os.path.join(pn,os.pardir)) -sys.path.__setitem__(0,os.path.abspath(os.curdir)) -os.environ['MMGEN_TEST_SUITE'] = '1' + +from include.tests_header import repo_root +from test.overlay import overlay_setup +sys.path.insert(0,overlay_setup()) # Import these _after_ local path's been added to sys.path from mmgen.common import * diff --git a/test/objtest.py b/test/objtest.py index 7e902ce1..ac90159e 100755 --- a/test/objtest.py +++ b/test/objtest.py @@ -20,10 +20,11 @@ test/objtest.py: Test MMGen data objects """ -import sys,os -pn = os.path.dirname(sys.argv[0]) -os.chdir(os.path.join(pn,os.pardir)) -sys.path.__setitem__(0,os.path.abspath(os.curdir)) +import sys,os,re + +from include.tests_header import repo_root +from test.overlay import overlay_setup +sys.path.insert(0,overlay_setup()) os.environ['MMGEN_TEST_SUITE'] = '1' diff --git a/test/overlay/__init__.py b/test/overlay/__init__.py new file mode 100644 index 00000000..fb720833 --- /dev/null +++ b/test/overlay/__init__.py @@ -0,0 +1,42 @@ +import sys,os + +def overlay_setup(): + + def process_srcdir(d): + srcdir = os.path.join(repo_root,*d.split('.')) + destdir = os.path.join(overlay_dir,*d.split('.')) + os.makedirs(destdir) + for fn in os.listdir(srcdir): + if ( + fn.endswith('.py') or + d == 'mmgen.data' or + d == 'mmgen' and fn.startswith('secp256k1') + ): + if fn in fakemods: + os.symlink( + os.path.join(fakemod_dir,fn), + os.path.join(destdir,fn) ) + link_fn = fn.removesuffix('.py') + '_orig.py' + else: + link_fn = fn + os.symlink( + os.path.join(srcdir,fn), + os.path.join(destdir,link_fn) ) + + repo_root = os.path.dirname(os.path.abspath(os.path.dirname(sys.argv[0]))) + overlay_dir = os.path.join(repo_root,'test','overlay','tree') + fakemod_dir = os.path.join(repo_root,'test','overlay','fakemods') + fakemods = os.listdir(fakemod_dir) + if not os.path.exists(os.path.join(overlay_dir,'mmgen','main.py')): + for d in ( + 'mmgen', + 'mmgen.data', + 'mmgen.share', + 'mmgen.altcoins', + 'mmgen.altcoins.eth', + 'mmgen.altcoins.eth.pyethereum', + 'mmgen.altcoins.eth.rlp', + 'mmgen.altcoins.eth.rlp.sedes' ): + process_srcdir(d) + + return overlay_dir diff --git a/test/overlay/fakemods/rpc.py b/test/overlay/fakemods/rpc.py new file mode 100644 index 00000000..5ba7f25f --- /dev/null +++ b/test/overlay/fakemods/rpc.py @@ -0,0 +1,17 @@ +from .rpc_orig import * + +if os.getenv('MMGEN_BOGUS_WALLET_DATA'): + + rpc_init_orig = rpc_init + + async def rpc_init(proto,backend=None,daemon=None,ignore_daemon_version=False): + + ret = await rpc_init_orig( + proto = proto, + backend = backend, + daemon = daemon, + ignore_daemon_version = ignore_daemon_version ) + + ret.blockcount = 1000000 + + return ret diff --git a/test/overlay/fakemods/tw.py b/test/overlay/fakemods/tw.py new file mode 100644 index 00000000..8b48dfac --- /dev/null +++ b/test/overlay/fakemods/tw.py @@ -0,0 +1,15 @@ +from .tw_orig import * + +if os.getenv('MMGEN_BOGUS_WALLET_DATA'): + # 1831006505 (09 Jan 2028) = projected time of block 1000000 + TwUnspentOutputs.date_formatter['days'] = lambda rpc,secs: (1831006505 - secs) // 86400 + + async def fake_set_dates(foo,rpc,us): + for o in us: + o.date = 1831006505 - int(9.7 * 60 * (o.confs - 1)) + + async def fake_get_unspent_rpc(foo): + return json.loads(get_data_from_file(os.getenv('MMGEN_BOGUS_WALLET_DATA')),parse_float=Decimal) + + TwUnspentOutputs.set_dates = fake_set_dates + TwUnspentOutputs.get_unspent_rpc = fake_get_unspent_rpc diff --git a/test/test.py b/test/test.py index abced954..14bc817c 100755 --- a/test/test.py +++ b/test/test.py @@ -73,6 +73,9 @@ def create_shm_dir(data_dir,trash_dir): import sys,os,time from include.tests_header import repo_root +from test.overlay import overlay_setup +overlay_dir = overlay_setup() +sys.path.insert(0,overlay_dir) try: os.unlink(os.path.join(repo_root,'my.err')) except: pass @@ -124,7 +127,6 @@ opts_data = { -T, --pexpect-timeout=T Set the timeout for pexpect -v, --verbose Produce more verbose output -W, --no-dw-delete Don't remove default wallet from data dir after dw tests are done --x, --exec-wrapper Run the command inside the '{ew}' script -X, --exit-after=C Exit after command 'C' -y, --segwit Generate and use Segwit addresses -Y, --segwit-random Generate and use a random mix of Segwit and Legacy addrs @@ -424,7 +426,7 @@ def do_between(): def list_tmpdirs(): return {k:cfgs[k]['tmpdir'] for k in cfgs} -def clean(usr_dirs=None): +def clean(usr_dirs=None,clean_overlay=True): if opt.skip_deps: return all_dirs = list_tmpdirs() @@ -444,6 +446,10 @@ def clean(usr_dirs=None): cleandir(trash_dir) iqmsg(green(f'Cleaned directories {data_dir!r} {trash_dir!r}')) + if clean_overlay: + cleandir(overlay_dir) + iqmsg(green(f'Cleaned directory {os.path.relpath(overlay_dir)!r}')) + def create_tmp_dirs(shm_dir): if g.platform == 'win': for cfg in sorted(cfgs): @@ -478,10 +484,6 @@ def set_environ_for_spawned_scripts(): if not opt.buf_keypress: os.environ['MMGEN_DISABLE_HOLD_PROTECT'] = '1' - # If test.py itself is running under exec_wrapper, the spawned script shouldn't be, so disable this: - if os.getenv('MMGEN_TRACEBACK') and not opt.exec_wrapper: - os.environ['MMGEN_TRACEBACK'] = '' - os.environ['MMGEN_NO_LICENSE'] = '1' os.environ['MMGEN_MIN_URANDCHARS'] = '3' os.environ['MMGEN_BOGUS_SEND'] = '1' @@ -674,7 +676,7 @@ class TestSuiteRunner(object): args = [cmd] + passthru_opts + self.ts.extra_spawn_args + args - if opt.exec_wrapper and not no_exec_wrapper: + if not no_exec_wrapper: args = ['scripts/exec_wrapper.py'] + args if g.platform == 'win': @@ -717,8 +719,13 @@ class TestSuiteRunner(object): os.environ['MMGEN_FORCE_COLOR'] = '1' if self.ts.color else '' + env = { 'EXEC_WRAPPER_SPAWN':'1' } + if 'exec_wrapper_init' in globals(): # Python 3.9: OR the dicts + env.update({ 'EXEC_WRAPPER_NO_TRACEBACK':'1' }) + env.update(os.environ) + from test.include.pexpect import MMGenPexpect - return MMGenPexpect(args,no_output=no_output) + return MMGenPexpect( args, no_output=no_output, env=env ) def end_msg(self): t = int(time.time() - self.start_time) @@ -732,7 +739,7 @@ class TestSuiteRunner(object): ts_cls = CmdGroupMgr().load_mod(gname) if do_clean: - clean(ts_cls.tmpdir_nums) + clean(ts_cls.tmpdir_nums,clean_overlay=False) for k in ('segwit','segwit_random','bech32'): if getattr(opt,k): diff --git a/test/tooltest.py b/test/tooltest.py index 0b11d46c..de571b59 100755 --- a/test/tooltest.py +++ b/test/tooltest.py @@ -24,6 +24,9 @@ import sys,os,binascii from subprocess import run,PIPE from include.tests_header import repo_root +from test.overlay import overlay_setup +sys.path.insert(0,overlay_setup()) + from mmgen.common import * from test.include.common import * @@ -130,7 +133,8 @@ if not opt.system: os.environ['PYTHONPATH'] = repo_root mmgen_cmd = os.path.relpath(os.path.join(repo_root,'cmds',mmgen_cmd)) -spawn_cmd = [mmgen_cmd] +spawn_cmd = ['scripts/exec_wrapper.py',mmgen_cmd] + if opt.coverage: d,f = init_coverage() spawn_cmd = ['python3','-m','trace','--count','--coverdir='+d,'--file='+f] + spawn_cmd diff --git a/test/tooltest2.py b/test/tooltest2.py index 9adbb857..bd183c3c 100755 --- a/test/tooltest2.py +++ b/test/tooltest2.py @@ -28,6 +28,9 @@ from subprocess import run,PIPE from decimal import Decimal from include.tests_header import repo_root +from test.overlay import overlay_setup +sys.path.insert(0,overlay_setup()) + from mmgen.common import * from test.include.common import * from mmgen.wallet import is_bip39_mnemonic,is_mmgen_mnemonic @@ -819,6 +822,7 @@ async def run_test(gid,cmd_name): def fork_cmd(cmd_name,args,out,opts): cmd = ( + ['scripts/exec_wrapper.py'] + list(tool_cmd) + (opts or []) + [cmd_name] + args