cmdtest.py: various cleanups (4 files)

test/cmdtest_d/base.py

@@ -38,6 +38,7 @@ class CmdTestBase:
 	need_daemon = False
 	platform_skip = ()
 	tmpdir_nums = []
+	skip_cmds = ()
 	test_name = None
 	is_helper = False
 

test/cmdtest_d/include/group_mgr.py

@@ -13,6 +13,7 @@ cmdtest_d.include.group_mgr: Command group manager for the MMGen Wallet cmdtest
 """
 
 import sys, os, time
+from collections import namedtuple
 
 from mmgen.color import yellow, green, cyan
 from mmgen.util import Msg, die
@@ -102,34 +103,33 @@ class CmdGroupMgr:
 		without touching 'cmd_list'
 		"""
 
-		cls = self.load_mod(gname, modname)
-		cdata = []
-
 		def get_shared_deps(cmdname, tmpdir_idx):
 			"""
 			shared_deps are "implied" dependencies for all cmds in cmd_group that don't appear in
 			the cmd_group data or cmds' argument lists.  Supported only for 3seed tests at present.
 			"""
-			if not hasattr(cls, 'shared_deps'):
-				return []
-
 			return [k for k, v in cfgs[str(tmpdir_idx)]['dep_generators'].items()
-						if k in cls.shared_deps and v != cmdname]
+				if k in cls.shared_deps and v != cmdname] if hasattr(cls, 'shared_deps') else []
+
+		cls = self.load_mod(gname, modname)
 
 		if not 'cmd_group' in cls.__dict__ and hasattr(cls, 'cmd_group_in'):
 			cls.cmd_group = self.create_cmd_group(cls, sg_name)
 
-		for a, b in cls.cmd_group:
-			if is3seed:
-				for n, (i, j) in enumerate(zip(cls.tmpdir_nums, (128, 192, 256))):
-					k = f'{a}_{n+1}'
-					if hasattr(cls, 'skip_cmds') and k in cls.skip_cmds:
-						continue
-					cdata.append((k, (i, f'{b} ({j}-bit)', [[[] + get_shared_deps(k, i), i]])))
-			elif full_data:
-				cdata.append((a, b))
-			else:
-				cdata.append((a, (cls.tmpdir_nums[0], b, [[[], cls.tmpdir_nums[0]]])))
+		def gen_cdata():
+			cgd = namedtuple('cmd_group_data', ['tmpdir_num', 'desc', 'dpy_list'])
+			for a, b in cls.cmd_group:
+				if is3seed:
+					for n, (i, j) in enumerate(zip(cls.tmpdir_nums, [128, 192, 256])):
+						k = f'{a}_{n + 1}'
+						if not k in cls.skip_cmds:
+							yield (k, cgd(i, f'{b} ({j}-bit)', [[get_shared_deps(k, i), i]]))
+				elif full_data:
+					yield (a, cgd(*b))
+				else:
+					yield (a, cgd(cls.tmpdir_nums[0], b, [[[], cls.tmpdir_nums[0]]]))
+
+		cdata = tuple(gen_cdata()) # cannot use dict() here because of repeated keys
 
 		if add_dpy:
 			self.dpy_data.update(dict(cdata))
@@ -192,9 +192,7 @@ class CmdGroupMgr:
 			name_w = max(len(name) for name in ginfo)
 			for name, cls in ginfo.items():
 				if not cls.is_helper:
-					yield '  {} - {}'.format(
-						yellow(name.ljust(name_w)),
-						(cls.__doc__.strip() if cls.__doc__ else cls.__name__))
+					yield '  {} - {}'.format(yellow(name.ljust(name_w)), cls.__doc__.strip())
 					if 'cmd_subgroups' in cls.__dict__:
 						subgroups = {k:v for k, v in cls.cmd_subgroups.items() if not k.startswith('_')}
 						max_w = max(len(k) for k in subgroups)
@@ -211,7 +209,7 @@ class CmdGroupMgr:
 		and return it as a string.  Loads modules but alters no global variables.
 		"""
 		if group:
-			if not group in [e[0] for e in self.cmd_groups]:
+			if not group in self.cmd_groups:
 				die(1, f'{group!r}: unrecognized group')
 			groups = [self.cmd_groups[group]]
 		else:

test/cmdtest_d/include/runner.py

@@ -13,10 +13,11 @@ test.cmdtest_d.include.runner: test runner for the MMGen Wallet cmdtest suite
 """
 
 import sys, os, time, asyncio
+from collections import namedtuple
 
 from mmgen.cfg import gc
 from mmgen.color import red, yellow, green, blue, cyan, gray, nocolor
-from mmgen.util import msg, Msg, rmsg, ymsg, bmsg, die, suf, make_timestr, isAsync
+from mmgen.util import msg, Msg, rmsg, ymsg, bmsg, die, suf, make_timestr, isAsync, capfirst
 
 from ...include.common import (
 	cmdtest_py_log_fn,
@@ -30,7 +31,7 @@ from ...include.common import (
 )
 
 from .common import get_file_with_ext, confirm_continue
-from .cfg import cfgs, cmd_groups_dfl
+from .cfg import cfgs
 from .group_mgr import CmdGroupMgr
 
 def format_args(args):
@@ -141,7 +142,8 @@ class CmdTestRunner:
 
 		self.exit_val = exit_val
 
-		desc = self.tg.test_name if self.cfg.names else self.gm.dpy_data[self.tg.test_name][1]
+		desc = self.tg.test_name if self.cfg.names else self.gm.dpy_data[self.tg.test_name].desc
+
 		if extra_desc:
 			desc += ' ' + extra_desc
 
@@ -347,8 +349,9 @@ class CmdTestRunner:
 								except Exception as e: # allow calling of functions not in cmd_group
 									if isinstance(e, KeyError) and e.args[0] == cmdname:
 										func = getattr(self.tg, cmdname)
-										ret = asyncio.run(func()) if isAsync(func) else func()
-										self.process_retval(cmdname, ret)
+										self.process_retval(
+											cmdname,
+											asyncio.run(func()) if isAsync(func) else func())
 									else:
 										raise
 								self.do_between()
@@ -385,16 +388,16 @@ class CmdTestRunner:
 
 		fns = []
 		if force_delete or not root:
-			# does cmd produce a needed dependency(ies)?
-			ret = self.get_num_exts_for_cmd(cmd)
-			if ret:
-				for ext in ret[1]:
-					fn = get_file_with_ext(cfgs[ret[0]]['tmpdir'], ext, delete=build)
-					if fn:
+			# does cmd produce a required dependency(ies)?
+			if deps := self.get_cmd_deps(cmd):
+				for ext in deps.exts:
+					if fn := get_file_with_ext(cfgs[deps.cfgnum]['tmpdir'], ext, delete=build):
 						if force_delete:
 							os.unlink(fn)
-						else: fns.append(fn)
-					else: rerun = True
+						else:
+							fns.append(fn)
+					else:
+						rerun = True
 
 		fdeps = self.generate_file_deps(cmd)
 		cdeps = self.generate_cmd_deps(fdeps)
@@ -419,10 +422,8 @@ class CmdTestRunner:
 					self.run_test(cmd)
 				if not root:
 					self.do_between()
-		else:
-			# If prog produces multiple files:
-			if cmd not in self.rebuild_list or rerun is True:
-				self.rebuild_list[cmd] = (rerun, fns[0] if fns else '') # FIX
+		elif rerun or cmd not in self.rebuild_list:
+			self.rebuild_list[cmd] = 'rebuild' if rerun and fns else 'build' if rerun else 'OK'
 
 		return rerun
 
@@ -432,9 +433,9 @@ class CmdTestRunner:
 			sys.exit(0)
 
 		if self.tg.full_data:
-			d = [(str(num), ext) for exts, num in self.gm.dpy_data[cmd][2] for ext in exts]
+			d = [(num, ext) for exts, num in self.gm.dpy_data[cmd].dpy_list for ext in exts]
 			# delete files depended on by this cmd
-			arg_list = [get_file_with_ext(cfgs[num]['tmpdir'], ext) for num, ext in d]
+			arg_list = [get_file_with_ext(cfgs[str(num)]['tmpdir'], ext) for num, ext in d]
 
 			# remove shared_deps from arg list
 			if hasattr(self.tg, 'shared_deps'):
@@ -456,7 +457,7 @@ class CmdTestRunner:
 		self.tg.test_name = cmd # NB: Do not remove, this needs to be set twice
 
 		if self.tg.full_data:
-			tmpdir_num = self.gm.dpy_data[cmd][0]
+			tmpdir_num = self.gm.dpy_data[cmd].tmpdir_num
 			self.tg.tmpdir_num = tmpdir_num
 			for k in (test_cfg := cfgs[str(tmpdir_num)]):
 				if k in self.gm.cfg_attrs:
@@ -532,33 +533,31 @@ class CmdTestRunner:
 		self.check_needs_rerun(cmd)
 
 		w = max(map(len, self.rebuild_list)) + 1
-		for cmd in self.rebuild_list:
-			c = self.rebuild_list[cmd]
-			m = 'Rebuild' if (c[0] and c[1]) else 'Build' if c[0] else 'OK'
-			omsg('cmd {:<{w}} {}'.format(cmd+':', m, w=w))
+		for cmd, desc in self.rebuild_list.items():
+			omsg('cmd {:<{w}} {}'.format(cmd+':', capfirst(desc), w=w))
 
 	def generate_file_deps(self, cmd):
-		return [(str(n), e) for exts, n in self.gm.dpy_data[cmd][2] for e in exts]
+		return [(str(n), e) for exts, n in self.gm.dpy_data[cmd].dpy_list for e in exts]
 
 	def generate_cmd_deps(self, fdeps):
 		return [cfgs[str(n)]['dep_generators'][ext] for n, ext in fdeps]
 
-	def get_num_exts_for_cmd(self, cmd):
+	def get_cmd_deps(self, cmd):
 		try:
-			num = str(self.gm.dpy_data[cmd][0])
+			self.gm.dpy_data[cmd]
 		except KeyError:
 			qmsg_r(f'Missing dependency {cmd!r}')
 			if gname := self.gm.find_cmd_in_groups(cmd):
 				kwargs = self.gm.cmd_groups[gname].params | {'add_dpy': True}
 				self.gm.create_group(gname, None, **kwargs)
-				num = str(self.gm.dpy_data[cmd][0])
 				qmsg(f' found in group {gname!r}')
 			else:
 				qmsg(' not found in any command group!')
 				raise
-		dgl = cfgs[num]['dep_generators']
-		if cmd in dgl.values():
-			exts = [k for k in dgl if dgl[k] == cmd]
-			return (num, exts)
+		num = str(self.gm.dpy_data[cmd].tmpdir_num)
+		dep_gens = cfgs[num]['dep_generators']
+		if cmd in dep_gens.values():
+			cd = namedtuple('cmd_deps', ['cfgnum', 'exts'])
+			return cd(num, [k for k in dep_gens if dep_gens[k] == cmd])
 		else:
 			return None

test/cmdtest_d/main.py

@@ -241,13 +241,13 @@ class CmdTestMain(CmdTestBase, CmdTestShared):
 			(3, 'tx signing with inputs and outputs from two wallets', [[['mmdat'], 1], [['mmdat', 'rawtx'], 3]])
 			),
 		('walletgen14',
-			(14, 'wallet generation (14)', [[['del_dw_run'], 15]], 14)
+			(14, 'wallet generation (14)', [[['del_dw_run'], 15]])
 		),
 		('addrgen14',
 			(14, 'address generation (14)', [[['mmdat'], 14]])
 		),
 		('keyaddrgen14',
-			(14, 'key-address file generation (14)', [[['mmdat'], 14]], 14)
+			(14, 'key-address file generation (14)', [[['mmdat'], 14]])
 		),
 		('walletgen4',
 			(4, 'wallet generation (4) (brainwallet)', [[['del_dw_run'], 15]])
@@ -310,7 +310,7 @@ class CmdTestMain(CmdTestBase, CmdTestShared):
 			])
 		), # must go after txsign4
 		('walletgen5',
-			(20, 'wallet generation (5)', [[['del_dw_run'], 15]], 20)
+			(20, 'wallet generation (5)', [[['del_dw_run'], 15]])
 		),
 		('addrgen5',
 			(20, 'address generation (5)', [[['mmdat'], 20]])
@@ -322,7 +322,7 @@ class CmdTestMain(CmdTestBase, CmdTestShared):
 			(20, 'transaction signing with bad vsize', [[['mmdat', 'rawtx'], 20]])
 		),
 		('walletgen6',
-			(21, 'wallet generation (6)', [[['del_dw_run'], 15]], 21)
+			(21, 'wallet generation (6)', [[['del_dw_run'], 15]])
 		),
 		('addrgen6',
 			(21, 'address generation (6)', [[['mmdat'], 21]])