various build and testing fixes

MANIFEST.in

@@ -14,6 +14,8 @@ include test/ref/ethereum_classic/*
 include test/ref/dash/*
 include test/ref/zcash/*
 include test/ref/monero/*
+include test/ref/ethereum/bin/mm1/*
+include test/ref/ethereum/bin/mm2/*
 include test/misc/*.py
 
 include test/test-release.sh

mmgen/base_obj.py

@@ -31,6 +31,7 @@ class AttrCtrl:
 	"""
 	_lock = False
 	_use_class_attr = False
+	_skip_type_check = ()
 
 	def lock(self):
 		self._lock = True
@@ -50,7 +51,11 @@ class AttrCtrl:
 
 			ref_val = getattr(type(self),name) if self._use_class_attr else getattr(self,name)
 
-			if (ref_val is not None) and not isinstance(value,type(ref_val)):
+			if (
+				(name not in self._skip_type_check)
+				and (ref_val is not None)
+				and not isinstance(value,type(ref_val))
+			):
 				do_error(name,value,ref_val)
 
 		return object.__setattr__(self,name,value)

mmgen/daemon.py

@@ -208,8 +208,7 @@ class Daemon(MMGenObject):
 
 	def remove_datadir(self):
 		if self.state == 'stopped':
-			import shutil
-			shutil.rmtree(self.datadir,ignore_errors=True)
+			run(['/bin/rm','-rf',self.datadir])
 		else:
 			msg(f'Cannot remove {self.datadir!r} - daemon is not stopped')
 

mmgen/globalvars.py

@@ -239,6 +239,7 @@ class GlobalContext(Lockable):
 	}
 	if platform == 'win':
 		autoset_opts['rpc_backend'].choices.remove('aiohttp')
+		_skip_type_check = ('stdout','stderr')
 
 	auto_typeset_opts = {
 		'seed_len': int,

mmgen/util.py

@@ -207,6 +207,9 @@ def check_or_create_dir(path):
 	try:
 		os.listdir(path)
 	except:
+		if os.getenv('MMGEN_TEST_SUITE'):
+			from subprocess import run
+			run(['/bin/rm','-rf',path])
 		try:
 			os.makedirs(path,0o700)
 		except:

setup.py

@@ -33,6 +33,7 @@ have_msys2 = subprocess.check_output(['uname','-s']).strip()[:7] == b'MSYS_NT'
 from distutils.core import setup,Extension
 from distutils.command.build_ext import build_ext
 from distutils.command.install_data import install_data
+from distutils.command.build_py import build_py
 
 cwd = os.getcwd()
 
@@ -66,9 +67,13 @@ class my_install_data(install_data):
 	def run(self):
 		for f in 'mmgen.cfg','mnemonic.py','mn_wordlist.c':
 			os.chmod(os.path.join('data_files',f),0o644)
-		link_or_copy('test','start-coin-daemons.py','stop-coin-daemons.py')
 		install_data.run(self)
 
+class my_build_py(build_py):
+	def run(self):
+		link_or_copy('test','start-coin-daemons.py','stop-coin-daemons.py')
+		build_py.run(self)
+
 module1 = Extension(
 	name         = 'mmgen.secp256k1',
 	sources      = ['extmod/secp256k1mod.c'],
@@ -88,7 +93,11 @@ setup(
 		license      = 'GNU GPL v3',
 		platforms    = 'Linux, MS Windows, Raspberry Pi/Raspbian, Orange Pi/Armbian',
 		keywords     = g.keywords,
-		cmdclass     = { 'build_ext': my_build_ext, 'install_data': my_install_data },
+		cmdclass     = {
+			'build_ext': my_build_ext,
+			'build_py': my_build_py,
+			'install_data': my_install_data,
+		},
 		ext_modules  = [module1],
 		data_files = [('share/mmgen', [
 				'data_files/mmgen.cfg',     # source files must have 0644 mode

test/include/common.py

@@ -171,23 +171,27 @@ def iqmsg(s):
 def iqmsg_r(s):
 	if not opt.quiet: omsg_r(s)
 
-def start_test_daemons(*network_ids):
+def start_test_daemons(*network_ids,remove_datadir=False):
 	if not opt.no_daemon_autostart:
-		return test_daemons_ops(*network_ids,op='start')
+		return test_daemons_ops(*network_ids,op='start',remove_datadir=remove_datadir)
 
 def stop_test_daemons(*network_ids):
 	if not opt.no_daemon_stop:
 		return test_daemons_ops(*network_ids,op='stop')
 
-def restart_test_daemons(*network_ids):
+def restart_test_daemons(*network_ids,remove_datadir=False):
 	stop_test_daemons(*network_ids)
-	return start_test_daemons(*network_ids)
+	return start_test_daemons(*network_ids,remove_datadir=remove_datadir)
 
-def test_daemons_ops(*network_ids,op):
+def test_daemons_ops(*network_ids,op,remove_datadir=False):
 	if not opt.no_daemon_autostart:
 		from mmgen.daemon import CoinDaemon
 		silent = not opt.verbose and not getattr(opt,'exact_output',False)
 		for network_id in network_ids:
 			if network_id.lower() not in CoinDaemon.network_ids: # silently ignore invalid IDs
 				continue
-			CoinDaemon(network_id,test_suite=True).cmd(op,silent=silent)
+			d = CoinDaemon(network_id,test_suite=True)
+			if remove_datadir:
+				d.stop(silent=True)
+				d.remove_datadir()
+			d.cmd(op,silent=silent)

test/misc/term.py

@@ -117,7 +117,7 @@ def tt_get_char(raw=False,one_char=False,sleep=0,immed_chars=''):
 def tt_urand():
 	cmsg('Testing _get_random_data_from_user():')
 	from mmgen.crypto import _get_random_data_from_user
-	ret = _get_random_data_from_user(10,desc='data',test_suite=True).decode()
+	ret = _get_random_data_from_user(10,desc='data').decode()
 	msg('USER ENTROPY (user input + keystroke timings):\n\n{}'.format(fmt(ret,'  ')))
 	times = ret.splitlines()[1:]
 	avg_prec = sum(len(t.split('.')[1]) for t in times) // len(times)
@@ -132,7 +132,7 @@ def tt_txview():
 	cmsg('Testing tx.view_with_prompt() (try each viewing option)')
 	from mmgen.tx import MMGenTX
 	fn = 'test/ref/0B8D5A[15.31789,14,tl=1320969600].rawtx'
-	tx = MMGenTX(fn,offline=True)
+	tx = MMGenTX.Unsigned(filename=fn,quiet_open=True)
 	while True:
 		tx.view_with_prompt('View data for transaction?',pause=False)
 		if not keypress_confirm('Continue testing transaction view?',default_yes=True):

test/test-release.sh

@@ -351,11 +351,11 @@ i_xmr='Monero'
 s_xmr='Testing key-address file generation and wallet creation and sync operations for Monero'
 s_xmr='The monerod (mainnet) daemon must be running for the following tests'
 t_xmr="
-	mmgen-walletgen -q -r0 -p1 -Llabel --outdir $TMPDIR -o words
+	cmds/mmgen-walletgen -q -r0 -p1 -Llabel --outdir $TMPDIR -o words
 	$mmgen_keygen -q --accept-defaults --use-internal-keccak-module --outdir $TMPDIR --coin=xmr $TMPDIR/*.mmwords $xmr_addrs
-	cs1=\$(mmgen-tool -q --accept-defaults --coin=xmr keyaddrfile_chksum $TMPDIR/*-XMR*.akeys)
+	cs1=\$(cmds/mmgen-tool -q --accept-defaults --coin=xmr keyaddrfile_chksum $TMPDIR/*-XMR*.akeys)
 	$mmgen_keygen -q --use-old-ed25519 --accept-defaults --outdir $TMPDIR --coin=xmr $TMPDIR/*.mmwords $xmr_addrs
-	cs2=\$(mmgen-tool -q --accept-defaults --coin=xmr keyaddrfile_chksum $TMPDIR/*-XMR*.akeys)
+	cs2=\$(cmds/mmgen-tool -q --accept-defaults --coin=xmr keyaddrfile_chksum $TMPDIR/*-XMR*.akeys)
 	[ \"\$cs1\" == \"\$cs2\" ]
 	test/start-coin-daemons.py xmr
 	$mmgen_tool_xmr keyaddrlist2monerowallets $TMPDIR/*-XMR*.akeys addrs=23

test/test.py

@@ -588,7 +588,7 @@ class CmdGroupMgr(object):
 			ginfo = [g for g in ginfo
 						if network_id in g[1].networks
 							and not g[0] in exclude
-							and g[0] in self.cmd_groups_dfl + tuple(usr_args) ]
+							and g[0] in tuple(self.cmd_groups_dfl) + tuple(usr_args) ]
 
 		for name,cls in ginfo:
 			msg('{:17} - {}'.format(name,cls.__doc__))
@@ -989,7 +989,7 @@ if opt.pause:
 
 set_environ_for_spawned_scripts()
 if network_id not in ('eth','etc'):
-	start_test_daemons(network_id)
+	start_test_daemons(network_id,remove_datadir=True)
 
 try:
 	tr = TestSuiteRunner(data_dir,trash_dir)

test/test_py_d/ts_ethdev.py

@@ -330,7 +330,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 			from shutil import copytree
 			for d in ('mm1','mm2'):
 				copytree(os.path.join(srcdir,d),os.path.join(self.tmpdir,d))
-		restart_test_daemons(self.proto.coin)
+		start_test_daemons(self.proto.coin,remove_datadir=True)
 		return 'ok'
 
 	def wallet_upgrade(self,src_file):
@@ -388,7 +388,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 						fee_desc        = 'gas price',
 						no_read         = False,
 						tweaks          = [] ):
-		fee_res = r'\D{}\D.* {c} (.*\D{}\D.* gas price in Gwei)'.format(*fee_res_data,c=self.proto.coin)
+		fee_res = r'\D{}\D.*{c} .*\D{}\D.*gas price in Gwei'.format(*fee_res_data,c=self.proto.coin)
 		t = self.spawn('mmgen-'+caller, self.eth_args + ['-B'] + args)
 		t.expect(r'add \[l\]abel, .*?:.','p', regex=True)
 		t.written_to_file('Account balances listing')
@@ -911,7 +911,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 			t.expect(' main menu): ',n)
 			t.expect('Is this what you want? (y/N): ','y')
 		t.expect('[R]efresh balance:\b','q')
-		t.expect('Total unspent: .*\D{}\D.*{}'.format(total,total_coin),regex=True)
+		t.expect('Total unspent:.*\D{}\D.*{}'.format(total,total_coin),regex=True)
 		t.read()
 		return t
 

test/test_py_d/ts_main.py

@@ -148,7 +148,7 @@ class TestSuiteMain(TestSuiteBase,TestSuiteShared):
 
 	def __init__(self,trunner,cfgs,spawn):
 		TestSuiteBase.__init__(self,trunner,cfgs,spawn)
-		if self.proto.coin.lower() not in self.networks:
+		if trunner == None or self.proto.coin.lower() not in self.networks:
 			return
 		self.rpc = run_session(rpc_init(self.proto))
 		self.lbl_id = ('account','label')['label_api' in self.rpc.caps]

test/test_py_d/ts_ref.py

@@ -281,10 +281,12 @@ class TestSuiteRef(TestSuiteBase,TestSuiteShared):
 
 	def ref_tool_decrypt(self):
 		f = joinpath(ref_dir,ref_enc_fn)
-		disable_debug()
+		if not g.debug_utf8:
+			disable_debug()
 		dec_file = joinpath(self.tmpdir,'famous.txt')
 		t = self.spawn('mmgen-tool', ['-q','decrypt',f,'outfile='+dec_file,'hash_preset=1'])
-		restore_debug()
+		if not g.debug_utf8:
+			restore_debug()
 		t.passphrase('user data',tool_enc_passwd)
 		t.written_to_file('Decrypted data')
 		dec_txt = read_from_file(dec_file)

test/test_py_d/ts_regtest.py

@@ -734,14 +734,16 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 
 	@staticmethod
 	def _gen_pairs(n):
-		disable_debug()
+		if not g.debug_utf8:
+			disable_debug()
 		from subprocess import run,PIPE
 		ret = [run(['python3',joinpath('cmds','mmgen-tool'),'--regtest=1'] +
 					(['--type=compressed'],[])[i==0] +
 					['-r0','randpair'],
 					stdout=PIPE,check=True
 				).stdout.decode().split() for i in range(n)]
-		restore_debug()
+		if not g.debug_utf8:
+			restore_debug()
 		return ret
 
 	def bob_pre_import(self):

test/unit_tests_d/ut_rpc.py

@@ -63,8 +63,8 @@ class init_test:
 def run_test(coin,auth):
 	proto = init_proto(coin,network=('mainnet','regtest')[coin=='eth']) # FIXME CoinDaemon's network handling broken
 	d = CoinDaemon(network_id=coin,test_suite=True)
-	if auth:
-		d.remove_datadir()
+	d.stop()
+	d.remove_datadir()
 	d.start()
 
 	for backend in g.autoset_opts['rpc_backend'].choices:

test/unit_tests_d/ut_tx.py

@@ -15,12 +15,11 @@ class unit_tests:
 
 	def tx(self,name,ut):
 		qmsg('  Testing transaction objects')
-		proto = init_proto('btc')
 		d = CoinDaemon('btc',test_suite=True)
 		d.start()
-		proto.rpc_port = d.rpc_port
 
 		async def do():
+			proto = init_proto('btc')
 			tx = MMGenTX.New(proto=proto)
 			tx.rpc = await rpc_init(proto=proto)
 
@@ -46,6 +45,8 @@ class unit_tests:
 			f = MMGenTxFile(tx)
 
 			fn_gen = f.make_filename()
+			if g.debug_utf8:
+				fn_gen = fn_gen.replace('-α','')
 			assert fn_gen == fn, f'{fn_gen} != {fn}'
 
 			text = f.format()

test/unit_tests_d/ut_tx_deserialize.py

@@ -127,9 +127,10 @@ class unit_test(object):
 					n        = n+1 )
 			Msg('OK')
 
-		start_test_daemons('btc','btc_tn') # ,'bch')
+		start_test_daemons('btc',remove_datadir=True)
+		start_test_daemons('btc_tn')
 		run_session(test_core_vectors())
 		run_session(test_mmgen_txs())
-		stop_test_daemons('btc','btc_tn') # ,'bch')
+		stop_test_daemons('btc','btc_tn')
 
 		return True