lots of minor fixes and cleanups throughout

MANIFEST.in

@@ -14,12 +14,13 @@ include test/ref/zcash/*
 include test/ref/monero/*
 include test/misc/*.py
 
+include test/test-release.sh
+
 include mmgen/altcoins/eth/rlp/LICENSE
 include mmgen/altcoins/eth/pyethereum/LICENSE
 
 include scripts/compute-file-chksum.py
 include scripts/create-token.py
-include scripts/test-release.sh
 include scripts/uninstall-mmgen.py
 
 prune test/ref/__db*

README.md

@@ -105,8 +105,8 @@ the more prosaic 2048-word [BIP39 wordlist][bw] used in most wallets today.
   “signature” of your transactions.
 - **[Full control over transaction fees][M]:** Fees are specified as absolute or
   sat/byte amounts and can be adjusted interactively, letting you round fees to
-  improve anonymity.  Network fee estimation is available. [RBF][R] and [fee
-  bumping][B] are supported.
+  improve anonymity.  Network fee estimation, [RBF][R] and [fee bumping][B] are
+  supported.
 - **Support for six wallet formats:** three encrypted (native wallet,
   brainwallet, incognito wallet) and three unencrypted (mnemonic, mmseed,
   hexseed).
@@ -119,7 +119,7 @@ the more prosaic 2048-word [BIP39 wordlist][bw] used in most wallets today.
 - **[Transaction autosigning][X]:** This feature puts your offline signing
   machine into “hands-off” mode, allowing you to transact directly from cold
   storage securely and conveniently.  Additional LED signaling support is
-  provided for RPi and Armbian-based boards.
+  provided for Raspbian and Armbian platforms.
 - **[Password generation][G]:** MMGen can be used to generate and manage your
   online passwords.  Passwords are identified by arbitrarily chosen strings like
   “alice@github” or “bob@reddit”.

mmgen/main_autosign.py

@@ -130,11 +130,12 @@ def check_daemons_running():
 
 	for coin in coins:
 		g.proto = CoinProtocol(coin,g.testnet)
-		if g.proto.sign_mode != 'daemon': continue
+		if g.proto.sign_mode != 'daemon':
+			continue
 		vmsg('Checking {} daemon'.format(coin))
 		try:
 			rpc_init(reinit=True)
-			g.rpch.getbalance()
+			g.rpch.getblockcount()
 		except SystemExit as e:
 			if e.code != 0:
 				fs = '{} daemon not running or not listening on port {}'

mmgen/regtest.py

@@ -24,8 +24,8 @@ import os,time,shutil
 from subprocess import run,PIPE
 from mmgen.common import *
 
-data_dir     = os.path.join(g.data_dir_root,'regtest',g.coin.lower())
-daemon_dir   = os.path.join(data_dir,'regtest')
+data_dir     = os.path.abspath(os.path.join(g.data_dir_root,'regtest',g.coin.lower()))
+daemon_dir   = os.path.abspath(os.path.join(data_dir,'regtest'))
 rpc_ports    = { 'btc':8552, 'bch':8553, 'b2x':8554, 'ltc':8555 }
 rpc_port     = rpc_ports[g.coin.lower()]
 rpc_user     = 'bobandalice'
@@ -76,7 +76,7 @@ def start_cmd(*args,**kwargs):
 	if 'pipe_stdout_only' in kwargs and kwargs['pipe_stdout_only']: ip = ep = None
 	return run(cmd,stdin=ip,stdout=op,stderr=ep)
 
-def test_daemon():
+def get_daemon_state():
 	cp = start_cmd('cli','getblockcount',quiet=True)
 	err = process_output(cp,silent=True)[1]
 	if "error: couldn't connect" in err or "error: Could not connect" in err:
@@ -88,7 +88,7 @@ def test_daemon():
 
 def wait_for_daemon(state,silent=False,nonl=False):
 	for i in range(200):
-		ret = test_daemon()
+		ret = get_daemon_state()
 		if not silent:
 			if opt.verbose: msg('returning state '+ret)
 			else: gmsg_r('.')
@@ -171,47 +171,11 @@ def cli(*args):
 	Msg_r(cp.stdout.decode())
 	msg_r(cp.stderr.decode())
 
-def fork(coin):
-	coin = coin.upper()
-	from mmgen.protocol import CoinProtocol
-	forks = CoinProtocol(coin,False).forks
-	if not [f for f in forks if f[2] == g.coin.lower() and f[3] == True]:
-		die(1,"Coin {} is not a replayable fork of coin {}".format(g.coin,coin))
-
-	gmsg('Creating fork from coin {} to coin {}'.format(coin,g.coin))
-	source_data_dir = os.path.join(g.data_dir_root,'regtest',coin.lower())
-
-	try: os.stat(source_data_dir)
-	except: die(1,"Source directory '{}' does not exist!".format(source_data_dir))
-
-	# stop the other daemon
-	global rpc_port,data_dir
-	rpc_port_save,data_dir_save = rpc_port,data_dir
-	rpc_port = rpc_ports[coin.lower()]
-	data_dir = os.path.join(g.data_dir_root,'regtest',coin.lower())
-	if test_daemon() != 'stopped':
-		stop_and_wait(silent=True,stop_silent=True)
-	rpc_port,data_dir = rpc_port_save,data_dir_save
-
-	try: os.makedirs(data_dir)
-	except: pass
-
-	# stop our daemon
-	if test_daemon() != 'stopped':
-		stop_and_wait(silent=True,stop_silent=True)
-
-	create_data_dir()
-	os.rmdir(data_dir)
-	shutil.copytree(source_data_dir,data_dir,symlinks=True)
-	start_and_wait('miner',reindex=True,silent=True)
-	stop_and_wait(silent=True,stop_silent=True)
-	gmsg('Fork {} successfully created'.format(g.coin))
-
 def setup():
 	try: os.makedirs(data_dir)
 	except: pass
 
-	if test_daemon() != 'stopped':
+	if get_daemon_state() != 'stopped':
 		stop_and_wait(silent=True,stop_silent=True)
 	create_data_dir()
 
@@ -233,10 +197,11 @@ def setup():
 	gmsg('Setup complete')
 
 def get_current_user_win(quiet=False):
-	if test_daemon() == 'stopped': return None
+	if get_daemon_state() == 'stopped': return None
 	logfile = os.path.join(daemon_dir,'debug.log')
 	for ss in ('Wallet completed loading in','Using wallet wallet'):
-		o = start_cmd('grep',ss,logfile,quiet=True).stdout.splitlines()
+		cp = run(['grep',ss,logfile],stdout=PIPE)
+		o = cp.stdout.splitlines()
 		if o:
 			last_line = o[-1].decode()
 			break
@@ -257,7 +222,7 @@ def get_current_user_win(quiet=False):
 		return None
 
 def get_current_user_unix(quiet=False):
-	cp = start_cmd('pgrep','-af','{}.*--rpcport={}.*'.format(g.proto.daemon_name,rpc_port),quiet=True)
+	cp = run(['pgrep','-af','{}.*--rpcport={}.*'.format(g.proto.daemon_name,rpc_port)],stdout=PIPE)
 	cmdline = cp.stdout.decode()
 	if not cmdline:
 		return None
@@ -276,9 +241,9 @@ def user(user=None,quiet=False):
 	if user==None:
 		get_current_user()
 		return True
-	if test_daemon() == 'busy':
+	if get_daemon_state() == 'busy':
 		wait_for_daemon('ready')
-	if test_daemon() == 'ready':
+	if get_daemon_state() == 'ready':
 		if user == get_current_user(quiet=True):
 			if not quiet: msg('{} is already the current user for coin {}'.format(user.capitalize(),g.coin))
 			return True
@@ -292,7 +257,7 @@ def user(user=None,quiet=False):
 	gmsg('done')
 
 def stop(silent=False,ignore_noconnect_error=True):
-	if test_daemon() != 'stopped' and not silent:
+	if get_daemon_state() != 'stopped' and not silent:
 		gmsg('Stopping {} regtest daemon for coin {}'.format(g.proto.name,g.coin))
 	cp = start_cmd('cli','stop')
 	err = process_output(cp)[1]
@@ -316,7 +281,7 @@ def generate(blocks=1,silent=False):
 		else:
 			rdie(1,'Error getting new address:\n{}'.format(err))
 
-	if test_daemon() == 'stopped':
+	if get_daemon_state() == 'stopped':
 		die(1,'Regtest daemon is not running')
 
 	wait_for_daemon('ready',silent=True)
@@ -333,3 +298,39 @@ def generate(blocks=1,silent=False):
 		rdie(1,'Error generating blocks')
 
 	gmsg('Mined {} block{}'.format(blocks,suf(blocks)))
+
+def fork(coin):
+	coin = coin.upper()
+	from mmgen.protocol import CoinProtocol
+	forks = CoinProtocol(coin,False).forks
+	if not [f for f in forks if f[2] == g.coin.lower() and f[3] == True]:
+		die(1,"Coin {} is not a replayable fork of coin {}".format(g.coin,coin))
+
+	gmsg('Creating fork from coin {} to coin {}'.format(coin,g.coin))
+	source_data_dir = os.path.join(g.data_dir_root,'regtest',coin.lower())
+
+	try: os.stat(source_data_dir)
+	except: die(1,"Source directory '{}' does not exist!".format(source_data_dir))
+
+	# stop the other daemon
+	global rpc_port,data_dir
+	rpc_port_save,data_dir_save = rpc_port,data_dir
+	rpc_port = rpc_ports[coin.lower()]
+	data_dir = os.path.join(g.data_dir_root,'regtest',coin.lower())
+	if get_daemon_state() != 'stopped':
+		stop_and_wait(silent=True,stop_silent=True)
+	rpc_port,data_dir = rpc_port_save,data_dir_save
+
+	try: os.makedirs(data_dir)
+	except: pass
+
+	# stop our daemon
+	if get_daemon_state() != 'stopped':
+		stop_and_wait(silent=True,stop_silent=True)
+
+	create_data_dir()
+	os.rmdir(data_dir)
+	shutil.copytree(source_data_dir,data_dir,symlinks=True)
+	start_and_wait('miner',reindex=True,silent=True)
+	stop_and_wait(silent=True,stop_silent=True)
+	gmsg('Fork {} successfully created'.format(g.coin))

mmgen/rpc.py

@@ -192,7 +192,6 @@ class CoinDaemonRPCConnection(MMGenObject):
 		'estimatesmartfee',
 		'getaddressesbyaccount',
 		'getaddressesbylabel',
-		'getbalance',
 		'getblock',
 		'getblockchaininfo',
 		'getblockcount',

mmgen/util.py

@@ -778,15 +778,21 @@ def do_license_msg(immed=False):
 	msg('')
 
 def get_daemon_cfg_options(cfg_keys):
-	cfg_file = os.path.join(g.proto.daemon_data_dir,g.proto.name+'.conf')
+
+	# Use dirname() to remove 'bob' or 'alice' component
+	cfg_dir = os.path.dirname(g.data_dir) if g.regtest else g.proto.daemon_data_dir
+	cfg_file = os.path.join(cfg_dir,g.proto.name+'.conf' )
+
 	try:
-		lines = get_lines_from_file(cfg_file,'',silent=bool(opt.quiet))
+		lines = get_lines_from_file(cfg_file,'',silent=not opt.verbose)
 		kv_pairs = [l.split('=') for l in lines]
 		cfg = {k:v for k,v in kv_pairs if k in cfg_keys}
 	except:
 		vmsg("Warning: '{}' does not exist or is unreadable".format(cfg_file))
 		cfg = {}
+
 	for k in set(cfg_keys) - set(cfg.keys()): cfg[k] = ''
+
 	return cfg
 
 def get_coin_daemon_auth_cookie():

setup.py

@@ -33,6 +33,13 @@ from distutils.core import setup,Extension
 from distutils.command.build_ext import build_ext
 from distutils.command.install_data import install_data
 
+cwd = os.getcwd()
+
+def copy_owner(a,b):
+	st = os.stat(a)
+	try: os.chown(b,st.st_uid,st.st_gid,follow_symlinks=False)
+	except: pass
+
 # install extension module in repository after building
 class my_build_ext(build_ext):
 	def build_extension(self,ext):
@@ -44,6 +51,7 @@ class my_build_ext(build_ext):
 		os.chmod(ext_src,0o755)
 		print('copying {} to {}'.format(ext_src,ext_dest))
 		copy2(ext_src,ext_dest)
+		copy_owner(cwd,ext_dest)
 
 class my_install_data(install_data):
 	def run(self):
@@ -59,7 +67,6 @@ module1 = Extension(
 	include_dirs = ['/usr/local/include',r'C:\msys64\mingw64\include',r'C:\msys64\usr\include'],
 	)
 
-
 from mmgen.globalvars import g
 setup(
 		name         = 'mmgen',
@@ -109,6 +116,7 @@ setup(
 			'mmgen.tool',
 			'mmgen.tw',
 			'mmgen.tx',
+			'mmgen.txsign',
 			'mmgen.util',
 
 			'mmgen.altcoins.__init__',
@@ -151,7 +159,6 @@ setup(
 			'mmgen.main_txsend',
 			'mmgen.main_txsign',
 			'mmgen.main_wallet',
-			'mmgen.txsign',
 
 			'mmgen.share.__init__',
 			'mmgen.share.Opts',

test/test-release.sh

@@ -34,9 +34,9 @@ mmgen_tool='cmds/mmgen-tool'
 mmgen_keygen='cmds/mmgen-keygen'
 python='python3'
 rounds=100 rounds_min=20 rounds_mid=250 rounds_max=500
-monero_addrs='3,99,2,22-24,101-104'
+xmr_addrs='3,99,2,22-24,101-104'
 
-dfl_tests='misc obj color unit hash ref alts monero eth autosign btc btc_tn btc_rt bch bch_rt ltc ltc_rt tool tool2 gen'
+dfl_tests='misc obj color unit hash ref alts xmr eth autosign btc btc_tn btc_rt bch bch_rt ltc ltc_rt tool tool2 gen'
 extra_tests='autosign_minimal autosign_live ltc_tn bch_tn'
 
 PROGNAME=$(basename $0)
@@ -67,7 +67,7 @@ do
 		echo   "     unit     - unit tests"
 		echo   "     hash     - internal hash function implementations"
 		echo   "     alts     - operations for all supported gen-only altcoins"
-		echo   "     monero   - operations for Monero"
+		echo   "     xmr      - operations for Monero"
 		echo   "     eth      - operations for Ethereum"
 		echo   "     autosign - autosign"
 		echo   "     btc      - bitcoin"
@@ -99,8 +99,8 @@ do
 		gentest_py="$python $gentest_py"
 		mmgen_tool="$python $mmgen_tool"
 		mmgen_keygen="$python $mmgen_keygen" ;&
-	f)  FAST=1 rounds=10 rounds_min=3 rounds_mid=25 rounds_max=50 monero_addrs='3,23' unit_tests_py+=" --fast" ;;
-	F)  FAST=1 rounds=3 rounds_min=1 rounds_mid=3 rounds_max=5 monero_addrs='3,23' unit_tests_py+=" --fast" ;;
+	f)  FAST=1 rounds=10 rounds_min=3 rounds_mid=25 rounds_max=50 xmr_addrs='3,23' unit_tests_py+=" --fast" ;;
+	F)  FAST=1 rounds=3 rounds_min=1 rounds_mid=3 rounds_max=5 xmr_addrs='3,23' unit_tests_py+=" --fast" ;;
 	i)  INSTALL=1 ;;
 	I)  INSTALL_ONLY=1 ;;
 	l)  echo -e "Default tests:\n  $dfl_tests"
@@ -109,7 +109,7 @@ do
 	O)  test_py+=" --pexpect-spawn" ;;
 	p)  PAUSE=1 ;;
 	R)  NO_TMPFILE_REMOVAL=1 ;;
-	t)  TESTING=1 ;;
+	t)  LIST_CMDS=1 ;;
 	v)  EXACT_OUTPUT=1 test_py+=" --exact-output" ;&
 	V)  VERBOSE=1 [ "$EXACT_OUTPUT" ] || test_py+=" --verbose"
 		unit_tests_py="${unit_tests_py/--quiet/--verbose}"
@@ -188,12 +188,19 @@ do_test() {
 	for test in "${tests_arr[@]}"; do
 		[ -z "$test" ] && continue
 		test_disp=$YELLOW${test/\#/$RESET$MAGENTA\#}$RESET
-		[ "${test:0:1}" == '#' ] && { echo -e "$test_disp"; continue; }
+		[ "${test:0:1}" == '#' ] && {
+			[ "$LIST_CMDS" ] || echo -e "$test_disp"
+			continue
+		}
 		let n+=1
 		echo $skips | grep -q "\<$n\>" && continue
-		echo -e "${GREEN}Running:$RESET $test_disp"
+		if [ "$LIST_CMDS" ]; then
+			echo $test
+		else
+			echo -e "${GREEN}Running:$RESET $test_disp"
+		fi
 #		continue
-		[ "$TESTING" ] || eval "$test" || {
+		[ "$LIST_CMDS" ] || eval "$test" || {
 			echo -e $RED"test-release.sh: test '$CUR_TEST' failed at command '$test'"$RESET
 			exit 1
 		}
@@ -205,7 +212,7 @@ s_misc='Testing various subsystems'
 t_misc="
 	$altcoin_py
 "
-f_misc='Miscellaneous tests complete'
+f_misc='Miscellaneous tests completed'
 
 i_obj='Data object'
 s_obj='Testing data objects'
@@ -217,17 +224,17 @@ t_obj="
 	$objtest_py --coin=eth
 	$objattrtest_py
 "
-f_obj='Data object test complete'
+f_obj='Data object tests completed'
 
 i_color='Color'
-s_color='Running color'
+s_color='Testing terminal colors'
 t_color="$colortest_py"
-f_color='Color tests complete'
+f_color='Terminal color tests completed'
 
 i_unit='Unit'
-s_unit='Running unit'
+s_unit='The bitcoin and bitcoin-abc mainnet daemons must be running for the following tests'
 t_unit="$unit_tests_py"
-f_unit='Unit tests run complete'
+f_unit='You may stop the bitcoin and bitcoin-abc mainnet daemons if you wish'
 
 i_hash='Internal hash function implementations'
 s_hash='Testing internal hash function implementations'
@@ -236,12 +243,12 @@ t_hash="
 	$python test/hashfunc.py sha512 $rounds_max # native sha512 - not used by MMGen
 	$python test/hashfunc.py keccak $rounds_max
 "
-f_hash='Hash function tests complete'
+f_hash='Hash function tests completed'
 
 [ "$ARM32" ] && t_hash_skip='2' # gmpy produces invalid init constants
 [ "$MSYS2" ] && t_hash_skip='2 3' # 2:py_long_long issues, 3:no pysha3 for keccak reference
 
-i_ref='Miscellaneous reference data tests'
+i_ref='Miscellaneous reference data'
 s_ref='The following tests will test some generated values against reference data'
 t_ref="
 	$scrambletest_py
@@ -307,32 +314,34 @@ else
 fi
 mkdir -p $TMPDIR
 
-i_monero='Monero'
-s_monero='Testing key-address file generation and wallet creation and sync operations for Monero'
-s_monero='The monerod (mainnet) daemon must be running for the following tests'
-t_monero="
+i_xmr='Monero'
+s_xmr='Testing key-address file generation and wallet creation and sync operations for Monero'
+s_xmr='The monerod (mainnet) daemon must be running for the following tests'
+t_xmr="
 	mmgen-walletgen -q -r0 -p1 -Llabel --outdir $TMPDIR -o words
-	$mmgen_keygen -q --accept-defaults --use-internal-keccak-module --outdir $TMPDIR --coin=xmr $TMPDIR/*.mmwords $monero_addrs
+	$mmgen_keygen -q --accept-defaults --use-internal-keccak-module --outdir $TMPDIR --coin=xmr $TMPDIR/*.mmwords $xmr_addrs
 	cs1=\$(mmgen-tool -q --accept-defaults --coin=xmr keyaddrfile_chksum $TMPDIR/*-XMR*.akeys)
-	$mmgen_keygen -q --use-old-ed25519 --accept-defaults --outdir $TMPDIR --coin=xmr $TMPDIR/*.mmwords $monero_addrs
+	$mmgen_keygen -q --use-old-ed25519 --accept-defaults --outdir $TMPDIR --coin=xmr $TMPDIR/*.mmwords $xmr_addrs
 	cs2=\$(mmgen-tool -q --accept-defaults --coin=xmr keyaddrfile_chksum $TMPDIR/*-XMR*.akeys)
 	[ \"\$cs1\" == \"\$cs2\" ]
 "
-f_monero='Monero tests completed'
+f_xmr='You may stop the Monero mainnet daemon if you wish'
+
+mmgen_tool_xmr="$mmgen_tool -q --accept-defaults --outdir $TMPDIR"
 
 [ "$MSYS2" ] || { # password file descriptor issues, cannot use popen_spawn()
-	t_monero+="
-$mmgen_tool -q --accept-defaults --outdir $TMPDIR keyaddrlist2monerowallets $TMPDIR/*-XMR*.akeys addrs=23
-$mmgen_tool -q --accept-defaults --outdir $TMPDIR keyaddrlist2monerowallets $TMPDIR/*-XMR*.akeys addrs=103-200
+	t_xmr+="
+$mmgen_tool_xmr keyaddrlist2monerowallets $TMPDIR/*-XMR*.akeys addrs=23
+$mmgen_tool_xmr keyaddrlist2monerowallets $TMPDIR/*-XMR*.akeys addrs=103-200
 rm $TMPDIR/*-MoneroWallet*
-$mmgen_tool -q --accept-defaults --outdir $TMPDIR keyaddrlist2monerowallets $TMPDIR/*-XMR*.akeys
-$mmgen_tool -q --accept-defaults --outdir $TMPDIR syncmonerowallets $TMPDIR/*-XMR*.akeys addrs=3
-$mmgen_tool -q --accept-defaults --outdir $TMPDIR syncmonerowallets $TMPDIR/*-XMR*.akeys addrs=23-29
-$mmgen_tool -q --accept-defaults --outdir $TMPDIR syncmonerowallets $TMPDIR/*-XMR*.akeys
+$mmgen_tool_xmr keyaddrlist2monerowallets $TMPDIR/*-XMR*.akeys
+$mmgen_tool_xmr syncmonerowallets $TMPDIR/*-XMR*.akeys addrs=3
+$mmgen_tool_xmr syncmonerowallets $TMPDIR/*-XMR*.akeys addrs=23-29
+$mmgen_tool_xmr syncmonerowallets $TMPDIR/*-XMR*.akeys
 	"
 }
 
-[ "$monero_addrs" == '3,23' ] && t_monero_skip='4 8 13'
+[ "$xmr_addrs" == '3,23' ] && t_xmr_skip='4 8 13'
 
 i_eth='Ethereum'
 s_eth='Testing transaction and tracking wallet operations for Ethereum and Ethereum Classic'
@@ -345,19 +354,19 @@ f_eth='Ethereum tests completed'
 i_autosign='Autosign'
 s_autosign='The bitcoin, bitcoin-abc and litecoin mainnet and testnet daemons must be running for the following test'
 t_autosign="$test_py autosign"
-f_autosign='Autosign test complete'
+f_autosign='Autosign test completed'
 
 i_autosign_minimal='Autosign Minimal'
 s_autosign_minimal='The bitcoin mainnet and testnet daemons must be running for the following test'
 t_autosign_minimal="$test_py autosign_minimal"
-f_autosign_minimal='Autosign Minimal test complete'
+f_autosign_minimal='Autosign Minimal test completed'
 
 i_autosign_live='Autosign Live'
 s_autosign_live="The bitcoin mainnet and testnet daemons must be running for the following test\n"
 s_autosign_live+="${YELLOW}Mountpoint, '/etc/fstab' and removable device must be configured "
 s_autosign_live+="as described in 'mmgen-autosign --help'${RESET}"
 t_autosign_live="$test_py autosign_live"
-f_autosign_live='Autosign Live test complete'
+f_autosign_live='Autosign Live test completed'
 
 i_btc='Bitcoin mainnet'
 s_btc='The bitcoin (mainnet) daemon must both be running for the following tests'
@@ -385,25 +394,23 @@ s_btc_rt="The following tests will test MMGen's regtest (Bob and Alice) mode"
 t_btc_rt="$test_py regtest"
 f_btc_rt='Regtest (Bob and Alice) mode tests for BTC completed'
 
-i_bch='Bitcoin cash (BCH)'
-s_bch='The bitcoin cash daemon (Bitcoin ABC) must both be running for the following tests'
+i_bch='Bcash (BCH) mainnet'
+s_bch='The bitcoin-abc mainnet daemon must both be running for the following tests'
 t_bch="$test_py --coin=bch --exclude regtest"
 f_bch='You may stop the Bitcoin ABC daemon if you wish'
 
-i_bch_tn='Bitcoin cash (BCH) testnet'
+i_bch_tn='Bcash (BCH) testnet'
 s_bch_tn='The bitcoin-abc testnet daemon must both be running for the following tests'
-t_bch_tn="
-	$test_py --coin=bch --testnet=1 --exclude regtest
-"
-f_bch_tn='You may stop the bitcoin-abc testnet daemon if you wish'
+t_bch_tn="$test_py --coin=bch --testnet=1 --exclude regtest"
+f_bch_tn='Bcash (BCH) testnet tests completed'
 
-i_bch_rt='Bitcoin cash (BCH) regtest'
+i_bch_rt='Bcash (BCH) regtest'
 s_bch_rt="The following tests will test MMGen's regtest (Bob and Alice) mode"
 t_bch_rt="$test_py --coin=bch regtest"
-f_bch_rt='Regtest (Bob and Alice) mode tests for BCH completed'
+f_bch_tn='You may stop the bitcoin-abc testnet daemon if you wish'
 
 i_ltc='Litecoin'
-s_ltc='The litecoin daemon must both be running for the following tests'
+s_ltc='The litecoin mainnet daemon must both be running for the following tests'
 t_ltc="
 	$test_py --coin=ltc --exclude regtest
 	$test_py --coin=ltc --segwit
@@ -491,7 +498,7 @@ t_gen="
 "
 f_gen='gentest tests completed'
 
-[ -d .git -a -n "$INSTALL"  -a -z "$TESTING" ] && {
+[ -d .git -a -n "$INSTALL"  -a -z "$LIST_CMDS" ] && {
 	check
 	uninstall
 	install
@@ -507,12 +514,16 @@ prompt_skip() {
 
 run_tests() {
 	for t in $1; do
-		eval echo -e "'\n'"\${GREEN}'###' Running $(echo \$i_$t) tests\$RESET
-		eval echo -e $(echo \$s_$t)
+		if [ "$LIST_CMDS" ]; then
+			eval echo -e '\\n#' $(echo \$i_$t) "\($t\)"
+		else
+			eval echo -e "'\n'"\${GREEN}'###' Running $(echo \$i_$t) tests\$RESET
+			eval echo -e $(echo \$s_$t)
+		fi
 		[ "$PAUSE" ] && prompt_skip && continue
 		CUR_TEST=$t
 		do_test $t
-		eval echo -e \$GREEN$(echo \$f_$t)\$RESET
+		[ "$LIST_CMDS" ] || eval echo -e $(echo \$f_$t)
 	done
 }
 
@@ -526,7 +537,7 @@ tests=$dfl_tests
 [ "$*" ] && tests="$*"
 
 check_args
-echo "Running tests: $tests"
+[ "$LIST_CMDS" ] || echo "Running tests: $tests"
 START=$(date +%s)
 run_tests "$tests"
 TIME=$(($(date +%s)-START))
@@ -534,4 +545,4 @@ MS=$(printf %02d:%02d $((TIME/60)) $((TIME%60)))
 
 [ "$NO_TMPFILE_REMOVAL" ] || rm -rf /tmp/mmgen-test-release-*
 
-echo -e "${GREEN}All OK.  Total elapsed time: $MS$RESET"
+[ "$LIST_CMDS" ] || echo -e "${GREEN}All OK.  Total elapsed time: $MS$RESET"

test/test.py

@@ -84,6 +84,7 @@ g.quiet = False # if 'quiet' was set in config file, disable here
 os.environ['MMGEN_QUIET'] = '0' # for this script and spawned scripts
 
 opts_data = {
+	'sets': [('list_current_cmd_groups',True,'list_cmd_groups',True)],
 	'text': {
 		'desc': 'Test suite for the MMGen suite',
 		'usage':'[options] [command(s) or metacommand(s)]',
@@ -102,7 +103,8 @@ opts_data = {
 -e, --exact-output   Show the exact output of the MMGen script(s) being run
 -G, --exclude-groups=G Exclude the specified command groups (comma-separated)
 -l, --list-cmds      List and describe the commands in the test suite
--L, --list-cmd-groups Output a list of command groups, with no descriptions
+-L, --list-cmd-groups Output a list of command groups with descriptions
+-g, --list-current-cmd-groups List command groups for current configuration
 -n, --names          Display command names instead of descriptions
 -o, --log            Log commands to file {lf}
 -O, --pexpect-spawn  Use pexpect.spawn instead of popen_spawn (much slower,
@@ -145,7 +147,7 @@ if not ('resume' in _uopts or 'skip_deps' in _uopts):
 	try: os.unlink(data_dir)
 	except: pass
 
-sys.argv = [sys.argv[0]] + ['--data-dir='+data_dir] + sys.argv[1:]
+sys.argv.insert(1,'--data-dir=' + data_dir)
 
 # step 2: opts.init will create new data_dir in ./test (if not 'resume' or 'skip_deps'):
 usr_args = opts.init(opts_data)
@@ -155,6 +157,8 @@ trash_dir = os.path.join('test','trash')
 if not ('resume' in _uopts or 'skip_deps' in _uopts):
 	shm_dir = create_shm_dir(data_dir,trash_dir)
 
+network_id = g.coin.lower() + ('_tn' if g.testnet else '')
+
 check_segwit_opts()
 
 if opt.profile: opt.names = True
@@ -536,10 +540,23 @@ class CmdGroupMgr(object):
 		return cls(trunner,cfgs,spawn_prog)
 
 	def list_cmd_groups(self):
+		ginfo = []
 		for gname in self.cmd_groups:
 			clsname,kwargs = self.cmd_groups[gname]
 			cls = self.load_mod(gname,kwargs['modname'] if 'modname' in kwargs else None)
-			msg('{:17} - {}'.format(gname,cls.__doc__))
+			ginfo.append((gname,cls))
+
+		if opt.list_current_cmd_groups:
+			exclude = (opt.exclude_groups or '').split(',')
+			ginfo = [g for g in ginfo
+						if network_id in g[1].networks
+							and not g[0] in exclude
+							and g[0] in self.dfl_groups + tuple(usr_args) ]
+
+		for name,cls in ginfo:
+			msg('{:17} - {}'.format(name,cls.__doc__))
+
+		Die(0,'\n'+' '.join(e[0] for e in ginfo))
 
 	def find_cmd_in_groups(self,cmd,group=None):
 		"""
@@ -602,9 +619,9 @@ class TestSuiteRunner(object):
 
 		passthru_opts = ['--{}{}'.format(k.replace('_','-'),
 							'=' + getattr(opt,k) if getattr(opt,k) != True else '')
-								for k in self.ts.passthru_opts if getattr(opt,k)]
+								for k in ('data_dir',) + self.ts.passthru_opts if getattr(opt,k)]
 
-		args = [cmd] + passthru_opts + ['--data-dir='+self.data_dir] + args
+		args = [cmd] + passthru_opts + self.ts.extra_spawn_args + args
 
 		if opt.traceback:
 			args = ['scripts/traceback_run.py'] + args
@@ -902,7 +919,6 @@ if not opt.skip_deps: # do this before list cmds exit, so we stay in sync with s
 
 if opt.list_cmd_groups:
 	CmdGroupMgr().list_cmd_groups()
-	Die(0,'\n'+' '.join(CmdGroupMgr.cmd_groups))
 elif opt.list_cmds:
 	list_cmds()
 

test/test_py_d/ts_autosign.py

@@ -194,6 +194,7 @@ class TestSuiteAutosign(TestSuiteBase):
 			opts = ['--coins='+','.join(coins)]
 			led_files = {   'opi': ('/sys/class/leds/orangepi:red:status/brightness',),
 							'rpi': ('/sys/class/leds/led0/brightness','/sys/class/leds/led0/trigger') }
+
 			for k in ('opi','rpi'):
 				if os.path.exists(led_files[k][0]):
 					led_support = k
@@ -209,15 +210,17 @@ class TestSuiteAutosign(TestSuiteBase):
 				omsg(purple("Running autosign test with '--led'"))
 				do_autosign_live(opts,mountpoint,led_opts=['--led'],gen_wallet=False)
 				omsg(purple("Running autosign test with '--stealth-led'"))
-				return do_autosign_live(opts,mountpoint,led_opts=['--stealth-led'],gen_wallet=False)
+				ret = do_autosign_live(opts,mountpoint,led_opts=['--stealth-led'],gen_wallet=False)
 			else:
-				return do_autosign_live(opts,mountpoint)
+				ret = do_autosign_live(opts,mountpoint)
 		else:
 			mountpoint = self.tmpdir
 			opts = ['--no-insert-check','--mountpoint='+mountpoint,'--coins='+','.join(coins)]
 			try: os.mkdir(joinpath(mountpoint,'tx'))
 			except: pass
-			return do_autosign(opts,mountpoint)
+			ret = do_autosign(opts,mountpoint)
+
+		return ret
 
 class TestSuiteAutosignMinimal(TestSuiteAutosign):
 	'autosigning with BTC, ETH and ETC'

test/test_py_d/ts_base.py

@@ -29,6 +29,7 @@ from test.test_py_d.common import *
 class TestSuiteBase(object):
 	'initializer class for the test.py test suite'
 	passthru_opts = ()
+	extra_spawn_args = []
 	networks = ()
 	segwit_opts_ok = False
 

test/test_py_d/ts_main.py

@@ -144,6 +144,8 @@ class TestSuiteMain(TestSuiteBase,TestSuiteShared):
 	)
 
 	def __init__(self,trunner,cfgs,spawn):
+		if g.coin.lower() not in self.networks:
+			return
 		rpc_init()
 		self.lbl_id = ('account','label')['label_api' in g.rpch.caps]
 		if g.coin in ('BTC','BCH','LTC'):

test/test_py_d/ts_misc.py

@@ -29,6 +29,7 @@ from mmgen.seed import SeedSource
 
 class TestSuiteHelp(TestSuiteBase):
 	'help, info and usage screens'
+	networks = ('btc','ltc','bch','eth')
 	tmpdir_nums = []
 	passthru_opts = ('coin','testnet')
 	cmd_group = (

test/test_py_d/ts_ref_3seed.py

@@ -32,8 +32,7 @@ from test.test_py_d.ts_wallet import TestSuiteWalletConv
 
 class TestSuiteRef3Seed(TestSuiteBase,TestSuiteShared):
 	'saved wallet files for 128-, 192- and 256-bit seeds + generated filename checks'
-	networks = ('btc','btc_tn','ltc','ltc_tn')
-	passthru_opts = ('coin','testnet')
+	networks = ('btc',)
 	mmtypes = (None,)
 	tmpdir_nums = [6,7,8]
 	addr_idx_list_in = '1010,500-501,31-33,1,33,500,1011'
@@ -189,6 +188,8 @@ class TestSuiteRef3Seed(TestSuiteBase,TestSuiteShared):
 
 class TestSuiteRef3Addr(TestSuiteRef3Seed):
 	'generated reference address, key and password files for 128-, 192- and 256-bit seeds'
+	networks = ('btc','btc_tn','ltc','ltc_tn')
+	passthru_opts = ('coin','testnet')
 	tmpdir_nums = [26,27,28]
 	shared_deps = ['mmdat',pwfile]
 

test/test_py_d/ts_ref_altcoin.py

@@ -88,7 +88,8 @@ class TestSuiteRefAltcoin(TestSuiteRef,TestSuiteBase):
 			coin,token = ('eth','mm1') if k == 'mm1' else (k,None)
 			ref_subdir = self._get_ref_subdir_by_coin(coin)
 			for tn in (False,True):
-				if tn and coin == 'etc': continue
+				if tn and coin == 'etc':
+					continue
 				g.testnet = tn
 				init_coin(coin)
 				fn = TestSuiteRef.sources['ref_tx_file'][token or coin][bool(tn)]

test/test_py_d/ts_regtest.py

@@ -20,9 +20,8 @@
 ts_regtest.py: Regtest tests for the test.py test suite
 """
 
-import os
+import os,json
 from decimal import Decimal
-from ast import literal_eval
 from mmgen.globalvars import g
 from mmgen.opts import opt
 from mmgen.util import die,gmsg,write_data_to_file
@@ -626,7 +625,8 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 		disable_debug()
 		ret = self.spawn('mmgen-regtest',['show_mempool']).read()
 		restore_debug()
-		return literal_eval(ret.split('\n')[0]) # allow for extra output by handler at end
+		m = re.search(r'(\[\s*"[a-f0-9]{64}"\s*])',ret) # allow for extra output by handler at end
+		return json.loads(m.group(1))
 
 	def get_mempool1(self):
 		mp = self._get_mempool()

test/unit_tests.py

@@ -43,7 +43,7 @@ If no test is specified, all available tests are run
 	}
 }
 
-sys.argv = [sys.argv[0]] + ['--skip-cfg-file'] + sys.argv[1:]
+sys.argv.insert(1,'--skip-cfg-file')
 cmd_args = opts.init(opts_data)
 
 def exit_msg():