From 08fc25d8632069f419934d26fa2d71e36d0cda35 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 3 Oct 2021 17:40:03 +0000 Subject: [PATCH] deterministic testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Goal: for each test run to produce reproducible output, allowing us to check for possible stochastic behavior in the scripts, as well as output-related regressions (for example, garbage or improperly formatted output produced by a bad format string) that might not be detected by the test scripts. In practice, bugginess of the pexpect module and the non-deterministic behavior of Bitcoin Core’s regtest implementation preclude completely identical output from test run to test run, but the differences are small enough to result in an easily reviewable diff. Enable this feature by setting the MMGEN_TEST_SUITE_DETERMINISTIC environment variable or running test/test-release.sh with the -D switch. Examples: $ script -c 'test/test-release.sh -FDv quick' -O run1 $ script -c 'test/test-release.sh -FDv quick' -O run2 # (optionally remove control characters from output files) $ diff -u run1 run2 > diff $ export MMGEN_TEST_SUITE_DETERMINISTIC=1 $ script -c 'test/test.py -ne main' -O run1 $ script -c 'test/test.py -ne main' -O run2 # (optionally remove control characters from output files) $ diff -u run1 run2 > diff --- scripts/exec_wrapper.py | 2 +- test/gentest.py | 3 ++- test/include/common.py | 6 +++++- test/objtest.py | 2 ++ test/overlay/fakemods/crypto.py | 21 +++++++++++++++++++++ test/overlay/fakemods/tw.py | 16 ++++++++++++++++ test/overlay/fakemods/util.py | 11 +++++++++++ test/test-release.sh | 11 +++++++++-- test/test.py | 7 ++++++- test/unit_tests_d/ut_scrypt.py | 2 +- 10 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 test/overlay/fakemods/crypto.py create mode 100644 test/overlay/fakemods/util.py diff --git a/scripts/exec_wrapper.py b/scripts/exec_wrapper.py index c0d5eee0..3e85e538 100755 --- a/scripts/exec_wrapper.py +++ b/scripts/exec_wrapper.py @@ -50,7 +50,7 @@ def exec_wrapper_write_traceback(): open('my.err','w').write(''.join(lines+[exc])) def exec_wrapper_end_msg(): - if os.getenv('EXEC_WRAPPER_SPAWN'): + if os.getenv('EXEC_WRAPPER_SPAWN') and not os.getenv('MMGEN_TEST_SUITE_DETERMINISTIC'): c = exec_wrapper_get_colors() # write to stdout to ensure script output gets to terminal first sys.stdout.write(c.blue('Runtime: {:0.5f} secs\n'.format(time.time() - exec_wrapper_tstart))) diff --git a/test/gentest.py b/test/gentest.py index 25d8e0ca..b00df354 100755 --- a/test/gentest.py +++ b/test/gentest.py @@ -333,7 +333,8 @@ def speed_test(kg,ag,rounds): vmsg(f'\nkey: {sec.wif}\naddr: {addr}\n') qmsg( f'\rRound {i+1}/{rounds} ' + - f'\n{rounds} addresses generated in {time.time()-start:.2f} seconds' + f'\n{rounds} addresses generated' + + ('' if g.test_suite_deterministic else ' in {time.time()-start:.2f} seconds') ) def dump_test(kg,ag,fh): diff --git a/test/include/common.py b/test/include/common.py index 6f479dcc..f8e5b1de 100755 --- a/test/include/common.py +++ b/test/include/common.py @@ -67,7 +67,11 @@ ref_kafile_pass = 'kafile password' ref_kafile_hash_preset = '1' def getrand(n): - return os.urandom(n) + if g.test_suite_deterministic: + from mmgen.crypto import fake_urandom + return fake_urandom(n) + else: + return os.urandom(n) def getrandnum(n): return int(getrand(n).hex(),16) diff --git a/test/objtest.py b/test/objtest.py index ac90159e..68fb49c3 100755 --- a/test/objtest.py +++ b/test/objtest.py @@ -97,6 +97,8 @@ def run_test(test,arg,input_data,arg1,exc_name): try: if not opt.super_silent: arg_disp = repr(arg_copy[0] if type(arg_copy) == tuple else arg_copy) + if g.test_suite_deterministic and isinstance(arg_copy,dict): + arg_disp = re.sub(r'object at 0x[0-9a-f]+','object at [SCRUBBED]',arg_disp) msg_r((green if input_data=='good' else orange)(f'{arg_disp+":":<22}')) cls = globals()[test] diff --git a/test/overlay/fakemods/crypto.py b/test/overlay/fakemods/crypto.py new file mode 100644 index 00000000..9e042351 --- /dev/null +++ b/test/overlay/fakemods/crypto.py @@ -0,0 +1,21 @@ +from .crypto_orig import * + +if os.getenv('MMGEN_TEST_SUITE_DETERMINISTIC'): + get_random_orig = get_random + add_user_random_orig = add_user_random + + fake_rand_h = sha256('.'.join(sys.argv).encode()) + def fake_urandom(n): + + def gen(rounds): + for i in range(rounds): + fake_rand_h.update(b'foo') + yield fake_rand_h.digest() + + return b''.join(gen(int(n/32)+1))[:n] + + def get_random(length): + return fake_urandom(len(get_random_orig(length))) + + def add_user_random(rand_bytes,desc): + return fake_urandom(len(add_user_random_orig(rand_bytes,desc))) diff --git a/test/overlay/fakemods/tw.py b/test/overlay/fakemods/tw.py index 8b48dfac..ed3848c1 100644 --- a/test/overlay/fakemods/tw.py +++ b/test/overlay/fakemods/tw.py @@ -1,5 +1,21 @@ from .tw_orig import * +if os.getenv('MMGEN_TEST_SUITE_DETERMINISTIC'): + def _time_gen(): + """ add a minute to each successive time value """ + for i in range(1000000): + yield 1321009871 + (i*60) + + _time_iter = _time_gen() + + TwUnspentOutputs.date_formatter = { + 'days': lambda rpc,secs: (next(_time_iter) - secs) // 86400, + 'date': lambda rpc,secs: '{}-{:02}-{:02}'.format(*time.gmtime(next(_time_iter))[:3])[2:], + 'date_time': lambda rpc,secs: '{}-{:02}-{:02} {:02}:{:02}'.format(*time.gmtime(next(_time_iter))[:5]), + } + + TwAddrList.date_formatter = TwUnspentOutputs.date_formatter + if os.getenv('MMGEN_BOGUS_WALLET_DATA'): # 1831006505 (09 Jan 2028) = projected time of block 1000000 TwUnspentOutputs.date_formatter['days'] = lambda rpc,secs: (1831006505 - secs) // 86400 diff --git a/test/overlay/fakemods/util.py b/test/overlay/fakemods/util.py new file mode 100644 index 00000000..f3aa8248 --- /dev/null +++ b/test/overlay/fakemods/util.py @@ -0,0 +1,11 @@ +from .util_orig import * + +if os.getenv('MMGEN_TEST_SUITE_DETERMINISTIC'): + make_timestamp_orig = make_timestamp + make_timestr_orig = make_timestr + + def make_timestamp(secs=None): + return make_timestamp_orig(1321009871) + + def make_timestr(secs=None): + return make_timestr_orig(1321009871) diff --git a/test/test-release.sh b/test/test-release.sh index 7f059027..901f9e06 100755 --- a/test/test-release.sh +++ b/test/test-release.sh @@ -40,7 +40,7 @@ quick_tests='dep misc obj color unit hash ref altref alts xmr eth autosign btc b qskip_tests='btc_tn bch bch_rt ltc ltc_rt' PROGNAME=$(basename $0) -while getopts hAbCfFi:I:lNOps:tvV OPT +while getopts hAbCDfFi:I:lNOps:tvV OPT do case "$OPT" in h) printf " %-16s Test MMGen release\n" "${PROGNAME}:" @@ -49,6 +49,7 @@ do echo " -A Skip tests requiring altcoin modules or daemons" echo " -b Buffer keypresses for all invocations of 'test/test.py'" echo " -C Run tests in coverage mode" + echo " -D Run tests in deterministic mode" echo " -f Speed up the tests by using fewer rounds" echo " -F Reduce rounds even further" echo " -i BRANCH Create and install Python package from cloned BRANCH, then" @@ -114,6 +115,8 @@ do gentest_py="$python $gentest_py" mmgen_tool="$python $mmgen_tool" mmgen_keygen="$python $mmgen_keygen" ;& + D) export MMGEN_TEST_SUITE_DETERMINISTIC=1 + export MMGEN_DISABLE_COLOR=1 ;; f) FAST=1 rounds=10 rounds_min=3 rounds_mid=25 rounds_max=50 unit_tests_py+=" --fast" ;; F) FAST=1 rounds=3 rounds_min=1 rounds_mid=3 rounds_max=5 unit_tests_py+=" --fast" ;; i) INSTALL=$OPTARG ;; @@ -595,5 +598,9 @@ elapsed=$(($(date +%s)-start_time)) elapsed_fmt=$(printf %02d:%02d $((elapsed/60)) $((elapsed%60))) [ "$LIST_CMDS" ] || { - echo -e "${GREEN}All OK. Total elapsed time: $elapsed_fmt$RESET" + if [ "$MMGEN_TEST_SUITE_DETERMINISTIC" ]; then + echo -e "${GREEN}All OK" + else + echo -e "${GREEN}All OK. Total elapsed time: $elapsed_fmt$RESET" + fi } diff --git a/test/test.py b/test/test.py index 14bc817c..ca1bb114 100755 --- a/test/test.py +++ b/test/test.py @@ -168,7 +168,7 @@ def add_cmdline_opts(): # add_cmdline_opts() -opts.UserOpts._reset_ok += ('skip_deps','no_daemon_autostart') +opts.UserOpts._reset_ok += ('skip_deps','no_daemon_autostart','names','no_timings') # step 2: opts.init will create new data_dir in ./test (if not 'resume' or 'skip_deps'): usr_args = opts.init(opts_data) @@ -186,6 +186,11 @@ if not (opt.resume or opt.skip_deps): check_segwit_opts() +if g.test_suite_deterministic: + opt.no_timings = True + init_color(num_colors=0) + os.environ['MMGEN_DISABLE_COLOR'] = '1' + if opt.profile: opt.names = True diff --git a/test/unit_tests_d/ut_scrypt.py b/test/unit_tests_d/ut_scrypt.py index fbb5f0b0..171f5b29 100755 --- a/test/unit_tests_d/ut_scrypt.py +++ b/test/unit_tests_d/ut_scrypt.py @@ -52,7 +52,7 @@ class unit_test(object): st = time.time() ret = scrypt_hash_passphrase(pw,salt,hp).hex() t = time.time() - st - vmsg(f' {t:0.4f} secs') + vmsg('' if g.test_suite_deterministic else f' {t:0.4f} secs') assert ret == res, ret if opt.quiet: