From a0eb2871f9cd328e4e09ca33d85c249db9fa9c0d Mon Sep 17 00:00:00 2001
From: The MMGen Project <mmgen@tuta.io>
Date: Sat, 27 Jul 2024 09:42:44 +0000
Subject: [PATCH] minor fixes and cleanups throughout

---
 mmgen/daemon.py                        |  1 +
 mmgen/wallet/base.py                   |  2 +-
 test/clean.py                          | 23 +++++++++++++++--------
 test/cmdtest.py                        |  6 ++++--
 test/cmdtest_py_d/common.py            |  4 ++--
 test/cmdtest_py_d/ct_input.py          | 14 ++++++++++++--
 test/cmdtest_py_d/ct_main.py           |  4 ++--
 test/cmdtest_py_d/ct_misc.py           |  4 +++-
 test/include/common.py                 |  1 +
 test/misc/term_ni.py                   | 21 +++++++++++++--------
 test/scrambletest.py                   |  3 ++-
 test/test-release.d/cfg.sh             |  2 +-
 test/test-release.sh                   | 17 ++++++++++-------
 test/tooltest2.py                      |  5 +++--
 test/unit_tests.py                     |  4 ++--
 test/unit_tests_d/ut_daemon.py         | 13 +++++++++----
 test/unit_tests_d/ut_rpc.py            | 10 ++++++++--
 test/unit_tests_d/ut_tx_deserialize.py |  4 ++--
 18 files changed, 91 insertions(+), 47 deletions(-)

diff --git a/mmgen/daemon.py b/mmgen/daemon.py
index 8a03c67c..0e5f3ca9 100755
--- a/mmgen/daemon.py
+++ b/mmgen/daemon.py
@@ -215,6 +215,7 @@ class Daemon(Lockable):
 				))
 			if self.wait:
 				self.wait_for_state('stopped')
+				time.sleep(0.3) # race condition
 			return ret
 		else:
 			if not (quiet or silent):
diff --git a/mmgen/wallet/base.py b/mmgen/wallet/base.py
index 388d20e7..111e8ed2 100755
--- a/mmgen/wallet/base.py
+++ b/mmgen/wallet/base.py
@@ -65,7 +65,7 @@ class wallet(MMGenObject,metaclass=WalletMeta):
 			self._deformat_retry()
 			self._decrypt_retry()
 
-		msg('Valid {a} for Seed ID {b}{c}{d}'.format(
+		self.cfg._util.qmsg('Valid {a} for Seed ID {b}{c}{d}'.format(
 			a = self.desc,
 			b = self.seed.sid.hl(),
 			c = f' (seed length {self.seed.bitlen})' if self.seed.bitlen != 256 else '',
diff --git a/test/clean.py b/test/clean.py
index 9ed13cc9..a10575a5 100755
--- a/test/clean.py
+++ b/test/clean.py
@@ -14,6 +14,10 @@ test/clean.py: Clean the test directory
 
 import sys, os
 
+repo_root = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), os.pardir)))
+os.chdir(repo_root)
+sys.path[0] = repo_root
+
 from mmgen.cfg import Config
 
 opts_data = {
@@ -29,10 +33,6 @@ opts_data = {
 
 cfg = Config(opts_data=opts_data)
 
-repo_root = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), os.pardir)))
-os.chdir(repo_root)
-sys.path[0] = repo_root
-
 from test.overlay import get_overlay_tree_dir
 overlay_tree_dir = get_overlay_tree_dir(repo_root)
 if os.path.exists(overlay_tree_dir):
@@ -46,8 +46,15 @@ set_globals(cfg)
 
 from test.include.cfg import clean_cfgs
 
-data_dir = Config.test_datadir
-trash_dir = os.path.join('test', 'trash')
-trash_dir2 = os.path.join('test', 'trash2')
+extra_dirs = [
+	Config.test_datadir,
+	os.path.join('test', 'trash'),
+	os.path.join('test', 'trash2')
+]
 
-clean(clean_cfgs, extra_dirs=[data_dir, trash_dir, trash_dir2])
+clean(clean_cfgs, extra_dirs=extra_dirs)
+
+for d in extra_dirs:
+	if os.path.exists(d) and not os.path.isdir(d):
+		print(f'Removing non-directory ‘{d}’')
+		os.unlink(d)
diff --git a/test/cmdtest.py b/test/cmdtest.py
index 9e074942..4ebc67a3 100755
--- a/test/cmdtest.py
+++ b/test/cmdtest.py
@@ -83,7 +83,7 @@ except ImportError:
 
 from mmgen.cfg import Config,gc
 from mmgen.color import red,yellow,green,blue,cyan,gray,nocolor,init_color
-from mmgen.util import msg, Msg, bmsg, die, suf, make_timestr
+from mmgen.util import msg, Msg, rmsg, bmsg, die, suf, make_timestr
 
 from test.include.common import (
 	set_globals,
@@ -1008,7 +1008,9 @@ if __name__ == '__main__':
 			Msg(red(str(e)))
 			Msg(blue('cmdtest.py: spawned script exited with error'))
 		raise
-	except Exception:
+	except Exception as e:
+		if type(e).__name__ == 'TestSuiteException':
+			rmsg('TEST ERROR: ' + str(e))
 		# if cmdtest.py itself is running under exec_wrapper, re-raise so exec_wrapper can handle exception:
 		if os.getenv('MMGEN_EXEC_WRAPPER') or not os.getenv('MMGEN_IGNORE_TEST_PY_EXCEPTION'):
 			raise
diff --git a/test/cmdtest_py_d/common.py b/test/cmdtest_py_d/common.py
index 30aded23..20eb3e88 100755
--- a/test/cmdtest_py_d/common.py
+++ b/test/cmdtest_py_d/common.py
@@ -22,7 +22,7 @@ test.cmdtest_py_d.common: Shared routines and data for the cmdtest.py test suite
 
 import sys,os
 
-from mmgen.color import green,blue
+from mmgen.color import green, blue, gray
 from mmgen.util import msg
 
 from ..include.common import cfg,getrand,text_jp,text_zh,ascii_cyr_gr,lat_cyr_gr
@@ -76,7 +76,7 @@ def ok_msg():
 	sys.stderr.write(green('\nOK\n') if cfg.exact_output or cfg.verbose else ' OK\n')
 
 def skip(name,reason=None):
-	msg('Skipping {}{}'.format( name, f' ({reason})' if reason else '' ))
+	msg(gray('Skipping {}{}'.format( name, f' ({reason})' if reason else '' )))
 	return 'skip'
 
 def confirm_continue():
diff --git a/test/cmdtest_py_d/ct_input.py b/test/cmdtest_py_d/ct_input.py
index 89216f32..1bf2c593 100755
--- a/test/cmdtest_py_d/ct_input.py
+++ b/test/cmdtest_py_d/ct_input.py
@@ -63,7 +63,8 @@ class CmdTestInput(CmdTestBase):
 		('line_input_insert',             'line_input() [inserted text]'),
 		('line_input_insert_term1',       'line_input() [inserted text, term]'),
 		('line_input_insert_term2',       'line_input() [inserted text, term, no hold protect]'),
-		('line_input_edit_term',          'line_input() [inserted + edited text, term, utf8]'),
+		('line_input_edit_term',          'line_input() [edited text, term, utf8]'),
+		('line_input_edit_term_insert',   'line_input() [inserted + edited text, term, utf8]'),
 		('line_input_erase_term',         'line_input() [inserted + erased text, term]'),
 	),
 	'password': (
@@ -351,6 +352,15 @@ class CmdTestInput(CmdTestBase):
 			True)
 
 	def line_input_edit_term(self):
+		return self._line_input(
+			['prompt> ', True, '', True],
+			'\b\bφυφυ\b\bβαρ',
+			'φυβαρ',
+			True)
+
+	def line_input_edit_term_insert(self):
+		if self.skip_for_mac('readline text buffer issues'):
+			return 'skip'
 		return self._line_input(
 			['prompt> ', True, 'φυφυ', True],
 			'\b\bβαρ',
@@ -408,7 +418,7 @@ class CmdTestInput(CmdTestBase):
 			'Type a number.*: ',
 			('\n' if enter_for_dfl else str(mne.entry_modes.index(entry_mode)+1)),
 			regex = True )
-		t.expect('Using entry mode (\S+)',regex=True)
+		t.expect(r'Using entry mode (\S+)', regex=True)
 		mode = strip_ansi_escapes(t.p.match.group(1)).lower()
 		assert mode == mne.em.name.lower(), f'{mode} != {mne.em.name.lower()}'
 		stealth_mnemonic_entry(t,mne,mn,entry_mode=entry_mode,pad_entry=pad_entry)
diff --git a/test/cmdtest_py_d/ct_main.py b/test/cmdtest_py_d/ct_main.py
index a2253659..fa2f1845 100755
--- a/test/cmdtest_py_d/ct_main.py
+++ b/test/cmdtest_py_d/ct_main.py
@@ -23,7 +23,7 @@ test.cmdtest_py_d.ct_main: Basic operations tests for the cmdtest.py test suite
 import sys,os
 
 from mmgen.util import msg,msg_r,async_run,capfirst,get_extension,die
-from mmgen.color import green,cyan
+from mmgen.color import green, cyan, gray
 from mmgen.fileutil import get_data_from_file,write_data_to_file
 from mmgen.wallet import get_wallet_cls
 from mmgen.wallet.mmgen import wallet as MMGenWallet
@@ -671,7 +671,7 @@ class CmdTestMain(CmdTestBase,CmdTestShared):
 
 	def txbump(self,txfile,prepend_args=[],seed_args=[]):
 		if not self.proto.cap('rbf'):
-			msg('Skipping RBF')
+			msg(gray('Skipping RBF'))
 			return 'skip'
 		args = prepend_args + ['--quiet','--outdir='+self.tmpdir,txfile] + seed_args
 		t = self.spawn('mmgen-txbump',args)
diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py
index 4776b00d..d861757f 100755
--- a/test/cmdtest_py_d/ct_misc.py
+++ b/test/cmdtest_py_d/ct_misc.py
@@ -22,7 +22,7 @@ test.cmdtest_py_d.ct_misc: Miscellaneous test groups for the cmdtest.py test sui
 
 import sys,os,re,time
 
-from mmgen.util import ymsg
+from mmgen.util import ymsg, die
 
 from ..include.common import cfg,start_test_daemons,stop_test_daemons,imsg
 from .common import get_file_with_ext,dfl_words_file
@@ -131,6 +131,8 @@ class CmdTestMisc(CmdTestBase):
 				t.expect('foo')
 			except pexpect.TIMEOUT:
 				imsg('[input not echoed - OK]')
+			else:
+				die('TestSuiteException', 'Terminal echoed in noecho mode!')
 			t.send('x')
 
 		if self.skip_for_win('no termios support'):
diff --git a/test/include/common.py b/test/include/common.py
index 16feeee1..5fdd403e 100755
--- a/test/include/common.py
+++ b/test/include/common.py
@@ -290,6 +290,7 @@ def test_daemons_ops(*network_ids,op,remove_datadir=False):
 		for network_id in network_ids:
 			d = CoinDaemon(cfg,network_id,test_suite=True)
 			if remove_datadir:
+				d.wait = True
 				d.stop(silent=True)
 				d.remove_datadir()
 			ret = d.cmd(op,silent=silent)
diff --git a/test/misc/term_ni.py b/test/misc/term_ni.py
index c69e9de8..f5f5d99e 100755
--- a/test/misc/term_ni.py
+++ b/test/misc/term_ni.py
@@ -7,6 +7,7 @@ sys.path[0] = os.curdir
 from mmgen.cfg import Config
 cfg = Config()
 
+from mmgen.util import msg
 from mmgen.term import init_term,get_term
 init_term(cfg)
 term = get_term()
@@ -16,16 +17,20 @@ if sys.argv[1] == 'echo':
 	from mmgen.ui import line_input
 	from mmgen.term import get_char_raw
 
-	term.init(noecho=True)
-	line_input( cfg, 'noecho> ' )
-	get_char_raw()
+	def test_noecho():
+		term.init(noecho=True)
+		ret = line_input(cfg, 'noecho> ')
+		msg(f'==> [{ret.upper()}]')
+		get_char_raw()
 
-	term.set('echo')
-	line_input( cfg, 'echo> ' )
+	def test_echo():
+		term.set('echo')
+		ret = line_input(cfg, 'echo> ')
+		msg(f'==> [{ret.upper()}]')
 
-	term.set('noecho')
-	line_input( cfg, 'noecho> ' )
-	get_char_raw()
+	test_noecho()
+	test_echo()
+	test_noecho()
 
 elif sys.argv[1] == 'cleanup':
 
diff --git a/test/scrambletest.py b/test/scrambletest.py
index b515c7b7..78972737 100755
--- a/test/scrambletest.py
+++ b/test/scrambletest.py
@@ -32,6 +32,7 @@ except ImportError:
 
 from mmgen.cfg import Config
 from mmgen.util import msg,msg_r,bmsg,die
+from mmgen.color import gray
 
 opts_data = {
 	'text': {
@@ -133,7 +134,7 @@ def do_coin_tests():
 			tuple(bitcoin_data.items()) +
 			tuple(altcoin_data.items() if not cfg.no_altcoin else []) ):
 		if tname == 'zec_zcash_z' and sys.platform == 'win32':
-			msg("Skipping 'zec_zcash_z' test for Windows platform")
+			msg(gray("Skipping 'zec_zcash_z' test for Windows platform"))
 			continue
 		coin,mmtype = tname.split('_',1) if '_' in tname else (tname,None)
 		type_arg = ' --type='+mmtype if mmtype else ''
diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh
index 1e9c313f..ff609c35 100755
--- a/test/test-release.d/cfg.sh
+++ b/test/test-release.d/cfg.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet
 # Copyright (C)2013-2024 The MMGen Project <mmgen@tuta.io>
diff --git a/test/test-release.sh b/test/test-release.sh
index 22fa0541..2b2e4b00 100755
--- a/test/test-release.sh
+++ b/test/test-release.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet
 # Copyright (C)2013-2024 The MMGen Project <mmgen@tuta.io>
@@ -116,6 +116,10 @@ print_ver_hash() {
 	python3 -m pip freeze | grep "^$repo\>" | sed 's/.*sha256=//' | cut -c 1-12
 }
 
+do_typescript() {
+	script -O "$1" -c "$2"
+}
+
 install_package() {
 	echo -e "${BLUE}Installing package$YELLOW $repo$RESET"
 	rm -rf build dist *.egg-info
@@ -165,7 +169,7 @@ do_reexec() {
 		target_dir="$orig_cwd/.clone-test"
 		clone_dir=$target_dir
 	else # TYPESCRIPT=1
-		script -O "$orig_cwd/$typescript_file" -c "$exec_prog"
+		do_typescript "$orig_cwd/$typescript_file" "$exec_prog"
 		return
 	fi
 
@@ -194,7 +198,7 @@ do_reexec() {
 		echo -n 'Building sdist...'
 		eval "python3 -m build --no-isolation --sdist --config-setting=quiet $STDOUT_DEVNULL"
 		echo -e "done\n${BLUE}Unpacking sdist archive to $YELLOW$target_dir$RESET"
-		tar -C $target_dir -axf dist/*.tar.gz
+		tar -C $target_dir -zxf dist/*.tar.gz
 		cd $target_dir/${repo//-/[-_]}-*
 		echo -e "${BLUE}cd -> $YELLOW$PWD$RESET"
 		if [ "$clone_dir" ]; then rm -rf $clone_dir; fi
@@ -207,7 +211,7 @@ do_reexec() {
 	echo -e "\n${BLUE}Executing test runner: ${CYAN}test/test-release $ORIG_ARGS$RESET\n"
 
 	if [ "$TYPESCRIPT" ]; then
-		script -O "$orig_cwd/$typescript_file" -c "$exec_prog"
+		do_typescript "$orig_cwd/$typescript_file" "$exec_prog"
 	else
 		eval $exec_prog
 	fi
@@ -232,6 +236,7 @@ elif [ "$(uname -m)" == 'aarch64' ]; then
 	ARM64=1
 elif [ "$MSYSTEM" ] && uname -a | grep -qi 'msys'; then
 	MSYS2=1
+	DISTRO='MSYS2'
 fi
 
 [ "$ARM32" -o "$ARM64" ] && {
@@ -239,9 +244,7 @@ fi
 	HTTP_LONG_TIMEOUT='MMGEN_HTTP_TIMEOUT=300 '
 }
 
-if [ "$MSYS2" ]; then
-	DISTRO='MSYS2'
-else
+if [ -e '/etc/os-release' ]; then
 	DISTRO=$(grep '^ID=' '/etc/os-release' | cut -c 4-)
 	[ "$DISTRO" ] || {
 		echo 'Unable to determine distro from /etc/os-release. Aborting'
diff --git a/test/tooltest2.py b/test/tooltest2.py
index f6c34b85..5a9c8797 100755
--- a/test/tooltest2.py
+++ b/test/tooltest2.py
@@ -210,7 +210,8 @@ def run_test(cls, gid, cmd_name):
 		if 'fmt=xmrseed' in args and cfg.no_altcoin:
 			if not skipping:
 				qmsg('')
-			qmsg(('' if n else '\n') + gray(f'Skipping altcoin test {cmd_name} {args}'))
+			skip_msg = f'Skipping altcoin test {cmd_name} {args}'
+			qmsg(('' if n else '\n') + gray(skip_msg if len(skip_msg) <= 100 else skip_msg[:97] + '...'))
 			skipping = True
 			continue
 		else:
@@ -228,7 +229,7 @@ def run_test(cls, gid, cmd_name):
 			cmd_out = fork_cmd(cmd_name,args,opts,stdin_input)
 		else:
 			if stdin_input and sys.platform == 'win32':
-				msg('Skipping for MSWin - no os.fork()')
+				msg(gray('Skipping for MSWin - no os.fork()'))
 				continue
 			method = getattr(cls(cfg,cmdname=cmd_name,proto=proto,mmtype=mmtype),cmd_name)
 			cmd_out = call_method(cls, method, cmd_name, args, mmtype, stdin_input)
diff --git a/test/unit_tests.py b/test/unit_tests.py
index bd306b19..7f3ce985 100755
--- a/test/unit_tests.py
+++ b/test/unit_tests.py
@@ -34,13 +34,13 @@ if not os.getenv('MMGEN_DEVTOOLS'):
 	init_dev()
 
 from mmgen.cfg import Config,gc
-from mmgen.color import green,gray,brown,orange
+from mmgen.color import green, gray, brown, orange, yellow, red
 from mmgen.util import msg,gmsg,ymsg,Msg
 
 from test.include.common import set_globals,end_msg
 
 def die(ev,s):
-	msg(s)
+	msg((red if ev > 1 else yellow)(s))
 	sys.exit(ev)
 
 opts_data = {
diff --git a/test/unit_tests_d/ut_daemon.py b/test/unit_tests_d/ut_daemon.py
index b1545bcf..ba862f0e 100755
--- a/test/unit_tests_d/ut_daemon.py
+++ b/test/unit_tests_d/ut_daemon.py
@@ -7,11 +7,11 @@ test.unit_tests_d.ut_daemon: unit test for the MMGen suite's Daemon class
 from subprocess import run,PIPE
 from collections import namedtuple
 
-from mmgen.color import orange
+from mmgen.color import orange, red
 from mmgen.util import fmt_list
 from mmgen.daemon import CoinDaemon
 
-from ..include.common import cfg,qmsg,qmsg_r,vmsg
+from ..include.common import cfg, qmsg, qmsg_r, vmsg, msg
 
 def test_flags():
 	d = CoinDaemon(cfg,'eth')
@@ -67,8 +67,13 @@ class unit_tests:
 		qmsg_r(message)
 		args = ['python3', f'test/{args_in[0]}-coin-daemons.py'] + list(args_in[1:]) + self.daemon_ctrl_args
 		vmsg('\n' + orange(f"Running '{' '.join(args)}':"))
-		pipe = None if cfg.verbose else PIPE
-		run( args, stdout=pipe, stderr=pipe, check=True )
+		cp = run(args, stdout=PIPE, stderr=PIPE, text=True)
+		if cp.returncode != 0:
+			if cp.stdout:
+				msg(cp.stdout)
+			if cp.stderr:
+				msg(red(cp.stderr))
+			return False
 		qmsg('OK')
 		return True
 
diff --git a/test/unit_tests_d/ut_rpc.py b/test/unit_tests_d/ut_rpc.py
index 9f65de51..1f831b68 100755
--- a/test/unit_tests_d/ut_rpc.py
+++ b/test/unit_tests_d/ut_rpc.py
@@ -20,7 +20,7 @@ from ..include.common import cfg,qmsg,vmsg
 async def cfg_file_auth_test(cfg, d, bad_auth=False):
 	m = 'missing credentials' if bad_auth else f'credentials from {d.cfg_file}'
 	qmsg(cyan(f'\n  Testing authentication with {m}:'))
-	time.sleep(0.1) # race condition
+	d.stop()
 	d.remove_datadir() # removes cookie file to force authentication from cfg file
 	os.makedirs(d.network_datadir)
 
@@ -45,7 +45,9 @@ async def cfg_file_auth_test(cfg, d, bad_auth=False):
 		rpc = await rpc_init(cfg, d.proto)
 		assert rpc.auth.user == 'ut_rpc', f'{rpc.auth.user}: user is not ut_rpc!'
 
-	d.stop()
+	if not cfg.no_daemon_stop:
+		d.stop()
+		d.remove_datadir()
 
 async def print_daemon_info(rpc):
 
@@ -117,8 +119,11 @@ async def run_test(network_ids, test_cf_auth=False, daemon_ids=None, cfg_in=None
 
 	async def do_test(d, cfg):
 
+		d.wait = True
+
 		if not cfg.no_daemon_stop:
 			d.stop()
+			d.remove_datadir()
 
 		if not cfg.no_daemon_autostart:
 			d.remove_datadir()
@@ -132,6 +137,7 @@ async def run_test(network_ids, test_cf_auth=False, daemon_ids=None, cfg_in=None
 
 		if not cfg.no_daemon_stop:
 			d.stop()
+			d.remove_datadir()
 
 		if test_cf_auth and sys.platform != 'win32':
 			await cfg_file_auth_test(cfg, d)
diff --git a/test/unit_tests_d/ut_tx_deserialize.py b/test/unit_tests_d/ut_tx_deserialize.py
index a32c62c7..db92d092 100755
--- a/test/unit_tests_d/ut_tx_deserialize.py
+++ b/test/unit_tests_d/ut_tx_deserialize.py
@@ -98,7 +98,7 @@ async def do_mmgen_ref(daemons,fns,name,desc):
 			tx_hex   = tx.serialized,
 			desc     = fn,
 			n        = n+1 )
-	stop_test_daemons(*daemons)
+	stop_test_daemons(*daemons, remove_datadir=True)
 	Msg('OK')
 	return True
 
@@ -133,7 +133,7 @@ class unit_tests:
 				desc = e[0]
 
 		Msg('OK')
-		stop_test_daemons('btc')
+		stop_test_daemons('btc', remove_datadir=True)
 		return True
 
 	async def mmgen_ref(self,name,ut):