deterministic testing

Goal: for each test run to produce reproducible output, allowing us to check
for possible stochastic behavior in the scripts, as well as output-related
regressions (for example, garbage or improperly formatted output produced by a
bad format string) that might not be detected by the test scripts.

In practice, bugginess of the pexpect module and the non-deterministic behavior
of Bitcoin Core’s regtest implementation preclude completely identical output
from test run to test run, but the differences are small enough to result in an
easily reviewable diff.

Enable this feature by setting the MMGEN_TEST_SUITE_DETERMINISTIC environment
variable or running test/test-release.sh with the -D switch.

Examples:

    $ script -c 'test/test-release.sh -FDv quick' -O run1
    $ script -c 'test/test-release.sh -FDv quick' -O run2
    # (optionally remove control characters from output files)
    $ diff -u run1 run2 > diff

    $ export MMGEN_TEST_SUITE_DETERMINISTIC=1
    $ script -c 'test/test.py -ne main' -O run1
    $ script -c 'test/test.py -ne main' -O run2
    # (optionally remove control characters from output files)
    $ diff -u run1 run2 > diff

scripts/exec_wrapper.py

@@ -50,7 +50,7 @@ def exec_wrapper_write_traceback():
 	open('my.err','w').write(''.join(lines+[exc]))
 
 def exec_wrapper_end_msg():
-	if os.getenv('EXEC_WRAPPER_SPAWN'):
+	if os.getenv('EXEC_WRAPPER_SPAWN') and not os.getenv('MMGEN_TEST_SUITE_DETERMINISTIC'):
 		c = exec_wrapper_get_colors()
 		# write to stdout to ensure script output gets to terminal first
 		sys.stdout.write(c.blue('Runtime: {:0.5f} secs\n'.format(time.time() - exec_wrapper_tstart)))

test/gentest.py

@@ -333,7 +333,8 @@ def speed_test(kg,ag,rounds):
 		vmsg(f'\nkey:  {sec.wif}\naddr: {addr}\n')
 	qmsg(
 		f'\rRound {i+1}/{rounds} ' +
-		f'\n{rounds} addresses generated in {time.time()-start:.2f} seconds'
+		f'\n{rounds} addresses generated' +
+		('' if g.test_suite_deterministic else ' in {time.time()-start:.2f} seconds')
 	)
 
 def dump_test(kg,ag,fh):

test/include/common.py

@@ -67,7 +67,11 @@ ref_kafile_pass = 'kafile password'
 ref_kafile_hash_preset = '1'
 
 def getrand(n):
-	return os.urandom(n)
+	if g.test_suite_deterministic:
+		from mmgen.crypto import fake_urandom
+		return fake_urandom(n)
+	else:
+		return os.urandom(n)
 
 def getrandnum(n):
 	return int(getrand(n).hex(),16)

test/objtest.py

@@ -97,6 +97,8 @@ def run_test(test,arg,input_data,arg1,exc_name):
 	try:
 		if not opt.super_silent:
 			arg_disp = repr(arg_copy[0] if type(arg_copy) == tuple else arg_copy)
+			if g.test_suite_deterministic and isinstance(arg_copy,dict):
+				arg_disp = re.sub(r'object at 0x[0-9a-f]+','object at [SCRUBBED]',arg_disp)
 			msg_r((green if input_data=='good' else orange)(f'{arg_disp+":":<22}'))
 		cls = globals()[test]
 

test/overlay/fakemods/crypto.py

@@ -0,0 +1,21 @@
+from .crypto_orig import *
+
+if os.getenv('MMGEN_TEST_SUITE_DETERMINISTIC'):
+	get_random_orig = get_random
+	add_user_random_orig = add_user_random
+
+	fake_rand_h = sha256('.'.join(sys.argv).encode())
+	def fake_urandom(n):
+
+		def gen(rounds):
+			for i in range(rounds):
+				fake_rand_h.update(b'foo')
+				yield fake_rand_h.digest()
+
+		return b''.join(gen(int(n/32)+1))[:n]
+
+	def get_random(length):
+		return fake_urandom(len(get_random_orig(length)))
+
+	def add_user_random(rand_bytes,desc):
+		return fake_urandom(len(add_user_random_orig(rand_bytes,desc)))

test/overlay/fakemods/tw.py

@@ -1,5 +1,21 @@
 from .tw_orig import *
 
+if os.getenv('MMGEN_TEST_SUITE_DETERMINISTIC'):
+	def _time_gen():
+		""" add a minute to each successive time value """
+		for i in range(1000000):
+			yield 1321009871 + (i*60)
+
+	_time_iter = _time_gen()
+
+	TwUnspentOutputs.date_formatter = {
+		'days':      lambda rpc,secs: (next(_time_iter) - secs) // 86400,
+		'date':      lambda rpc,secs: '{}-{:02}-{:02}'.format(*time.gmtime(next(_time_iter))[:3])[2:],
+		'date_time': lambda rpc,secs: '{}-{:02}-{:02} {:02}:{:02}'.format(*time.gmtime(next(_time_iter))[:5]),
+	}
+
+	TwAddrList.date_formatter = TwUnspentOutputs.date_formatter
+
 if os.getenv('MMGEN_BOGUS_WALLET_DATA'):
 	# 1831006505 (09 Jan 2028) = projected time of block 1000000
 	TwUnspentOutputs.date_formatter['days'] = lambda rpc,secs: (1831006505 - secs) // 86400

test/overlay/fakemods/util.py

@@ -0,0 +1,11 @@
+from .util_orig import *
+
+if os.getenv('MMGEN_TEST_SUITE_DETERMINISTIC'):
+	make_timestamp_orig = make_timestamp
+	make_timestr_orig = make_timestr
+
+	def make_timestamp(secs=None):
+		return make_timestamp_orig(1321009871)
+
+	def make_timestr(secs=None):
+		return make_timestr_orig(1321009871)

test/test-release.sh

@@ -40,7 +40,7 @@ quick_tests='dep misc obj color unit hash ref altref alts xmr eth autosign btc b
 qskip_tests='btc_tn bch bch_rt ltc ltc_rt'
 
 PROGNAME=$(basename $0)
-while getopts hAbCfFi:I:lNOps:tvV OPT
+while getopts hAbCDfFi:I:lNOps:tvV OPT
 do
 	case "$OPT" in
 	h)  printf "  %-16s Test MMGen release\n" "${PROGNAME}:"
@@ -49,6 +49,7 @@ do
 		echo   "           -A      Skip tests requiring altcoin modules or daemons"
 		echo   "           -b      Buffer keypresses for all invocations of 'test/test.py'"
 		echo   "           -C      Run tests in coverage mode"
+		echo   "           -D      Run tests in deterministic mode"
 		echo   "           -f      Speed up the tests by using fewer rounds"
 		echo   "           -F      Reduce rounds even further"
 		echo   "           -i BRANCH Create and install Python package from cloned BRANCH, then"
@@ -114,6 +115,8 @@ do
 		gentest_py="$python $gentest_py"
 		mmgen_tool="$python $mmgen_tool"
 		mmgen_keygen="$python $mmgen_keygen" ;&
+	D)  export MMGEN_TEST_SUITE_DETERMINISTIC=1
+		export MMGEN_DISABLE_COLOR=1 ;;
 	f)  FAST=1 rounds=10 rounds_min=3 rounds_mid=25 rounds_max=50 unit_tests_py+=" --fast" ;;
 	F)  FAST=1 rounds=3 rounds_min=1 rounds_mid=3 rounds_max=5 unit_tests_py+=" --fast" ;;
 	i)  INSTALL=$OPTARG ;;
@@ -595,5 +598,9 @@ elapsed=$(($(date +%s)-start_time))
 elapsed_fmt=$(printf %02d:%02d $((elapsed/60)) $((elapsed%60)))
 
 [ "$LIST_CMDS" ] || {
-	echo -e "${GREEN}All OK.  Total elapsed time: $elapsed_fmt$RESET"
+	if [ "$MMGEN_TEST_SUITE_DETERMINISTIC" ]; then
+		echo -e "${GREEN}All OK"
+	else
+		echo -e "${GREEN}All OK.  Total elapsed time: $elapsed_fmt$RESET"
+	fi
 }

test/test.py

@@ -168,7 +168,7 @@ def add_cmdline_opts():
 
 # add_cmdline_opts()
 
-opts.UserOpts._reset_ok += ('skip_deps','no_daemon_autostart')
+opts.UserOpts._reset_ok += ('skip_deps','no_daemon_autostart','names','no_timings')
 
 # step 2: opts.init will create new data_dir in ./test (if not 'resume' or 'skip_deps'):
 usr_args = opts.init(opts_data)
@@ -186,6 +186,11 @@ if not (opt.resume or opt.skip_deps):
 
 check_segwit_opts()
 
+if g.test_suite_deterministic:
+	opt.no_timings = True
+	init_color(num_colors=0)
+	os.environ['MMGEN_DISABLE_COLOR'] = '1'
+
 if opt.profile:
 	opt.names = True
 

test/unit_tests_d/ut_scrypt.py

@@ -52,7 +52,7 @@ class unit_test(object):
 				st = time.time()
 				ret = scrypt_hash_passphrase(pw,salt,hp).hex()
 				t = time.time() - st
-				vmsg(f'  {t:0.4f} secs')
+				vmsg('' if g.test_suite_deterministic else f'  {t:0.4f} secs')
 				assert ret == res, ret
 
 		if opt.quiet: