test-release.sh, gentest.py, tooltest.py: cleanups

test/gentest.py

@@ -89,21 +89,24 @@ if not 1 <= len(cmd_args) <= 2: opts.usage()
 
 addr_type = MMGenAddrType(opt.type or g.proto.dfl_mmtype)
 
+from collections import namedtuple
+ep = namedtuple('external_prog_output',['wif','addr','vk'])
+
 from subprocess import run,PIPE,DEVNULL
 def get_cmd_output(cmd,input=None):
 	return run(cmd,input=input,stdout=PIPE,stderr=DEVNULL).stdout.decode().splitlines()
 
 def ethkey_sec2addr(sec):
 	o = get_cmd_output(['ethkey','info',sec])
-	return (o[0].split()[1],o[-1].split()[1])
+	return ep(o[0].split()[1],o[-1].split()[1],None)
 
 def keyconv_sec2addr(sec):
 	o = get_cmd_output(['keyconv','-C',g.coin,sec.wif])
-	return (o[1].split()[1],o[0].split()[1])
+	return ep(o[1].split()[1],o[0].split()[1],None)
 
 def zcash_mini_sec2addr(sec):
 	o = get_cmd_output(['zcash-mini','-key','-simple'],input=(sec.wif+'\n').encode())
-	return (o[1],o[0],o[-1])
+	return ep(o[1],o[0],o[-1])
 
 def pycoin_sec2addr(sec):
 	coin = ci.external_tests['testnet']['pycoin'][g.coin] if g.testnet else g.coin
@@ -120,11 +123,11 @@ def pycoin_sec2addr(sec):
 			addr = network.address.for_p2pkh_wit(hash160_c)
 	else:
 		addr = key.address()
-	return (key.wif(),addr)
+	return ep(key.wif(),addr,None)
 
 def moneropy_sec2addr(sec):
 	sk_t,vk_t,addr_t = mp_acc.account_from_spend_key(sec) # VERY slow!
-	return (sk_t,addr_t,vk_t)
+	return ep(sk_t,addr_t,vk_t)
 
 # pycoin/networks/all.py pycoin/networks/legacy_networks.py
 def init_external_prog():
@@ -150,7 +153,7 @@ def init_external_prog():
 		try:
 			from pycoin.networks.registry import network_for_netcode
 		except:
-			raise ImportError("Unable to import pycoin.networks.registry Is pycoin installed and up-to-date?")
+			raise ImportError("Unable to import pycoin.networks.registry.  Is pycoin installed on your system?")
 		ext_sec2addr = pycoin_sec2addr
 		ext_prog = 'pycoin'
 	elif test_support('moneropy'):
@@ -172,15 +175,22 @@ def init_external_prog():
 	b_desc = ext_prog
 	b = 'ext'
 
-def test_equal(a_addr,b_addr,sec,wif,a,b):
+def test_equal(a_addr,b_addr,in_bytes,sec,wif,a,b):
 	if a_addr != b_addr:
-		qmsg_r(red('\nERROR: Values do not match!'))
-		die(3,"""
-	  sec key   : {}
-	  WIF key   : {}
-	  {a:10}: {}
-	  {b:10}: {}
-	""".format(sec,wif,a_addr,b_addr,pnm=g.proj_name,a=kg_a.desc,b=b_desc).rstrip())
+		fs = """
+  {i:{w}}: {}
+  {s:{w}}: {}
+  {W:{w}}: {}
+  {a:{w}}: {}
+  {b:{w}}: {}
+			"""
+		die(3,
+			red('\nERROR: Values do not match!')
+			+ fs.format(
+				in_bytes.hex(), sec, wif, a_addr, b_addr,
+				i='input', s='sec key', W='WIF key', a=kg_a.desc, b=b_desc,
+				w=max(len(e) for e in (kg_a.desc,b_desc)) + 1
+		).rstrip())
 
 def compare_test():
 	for k in ('segwit','compressed'):
@@ -195,7 +205,7 @@ def compare_test():
 	global last_t
 	last_t = time.time()
 	A = kg_a.desc
-	B = ext_prog if b == 'ext' else kg_b.desc
+	B = b_desc
 	if A == B:
 		msg('skipping - generation methods A and B are the same ({})'.format(A))
 		return
@@ -208,24 +218,26 @@ def compare_test():
 			qmsg_r('\rRound {}/{} '.format(i+1,trounds))
 			last_t = time.time()
 		sec = PrivKey(in_bytes,compressed=addr_type.compressed,pubkey_type=addr_type.pubkey_type)
-		ph = kg_a.to_pubhex(sec)
-		a_addr = ag.to_addr(ph)
-		a_vk = ag.to_viewkey(ph) if 'viewkey' in addr_type.extra_attrs else None
+		a_ph = kg_a.to_pubhex(sec)
+		a_addr = ag.to_addr(a_ph)
+		a_vk = None
 		if b == 'ext':
-			if 'viewkey' in addr_type.extra_attrs:
-				b_wif,b_addr,b_vk = ext_sec2addr(sec)
-				test_equal(a_vk,b_vk,sec,sec.wif,a,b)
-			else:
-				b_wif,b_addr = ext_sec2addr(sec)
-			test_equal(sec.wif,b_wif,sec,sec.wif,a,b)
+			ret = ext_sec2addr(sec)
+			tinfo = (in_bytes,sec,sec.wif,a,ext_prog)
+			test_equal(sec.wif,ret.wif,*tinfo)
+			test_equal(a_addr,ret.addr,*tinfo)
+			if ret.vk:
+				a_vk = ag.to_viewkey(a_ph)
+				test_equal(a_vk,ret.vk,*tinfo)
 		else:
 			b_addr = ag.to_addr(kg_b.to_pubhex(sec))
+			tinfo = (in_bytes,sec,sec.wif,a,b)
+			test_equal(a_addr,b_addr,*tinfo)
 		vmsg(ct_fs.format(b=in_bytes.hex(),k=sec.wif,v=a_vk,a=a_addr))
-		test_equal(a_addr,b_addr,sec,sec.wif,a,ext_prog if b == 'ext' else b)
 		qmsg_r('\rRound {}/{} '.format(n+1,trounds))
 
 	ct_fs   = ( '\ninput:    {b}\n%-9s {k}\naddr:     {a}\n',
-				'\ninput:    {b}\n%-9s {k}\nvkey:     {v}\naddr:     {a}\n')[
+				'\ninput:    {b}\n%-9s {k}\nviewkey:  {v}\naddr:     {a}\n')[
 					'viewkey' in addr_type.extra_attrs] % (addr_type.wif_label + ':')
 
 	# test some important private key edge cases:
@@ -278,7 +290,8 @@ def dump_test():
 			die(2,'\nInvalid {}net WIF address in dump file: {}'.format(('main','test')[g.testnet],wif))
 		b_addr = ag.to_addr(kg_a.to_pubhex(sec))
 		vmsg('\nwif: {}\naddr: {}\n'.format(wif,b_addr))
-		test_equal(a_addr,b_addr,sec,wif,3,a)
+		tinfo = (bytes.fromhex(sec),sec,wif,3,a)
+		test_equal(a_addr,b_addr,*tinfo)
 	qmsg(green(('\n','')[bool(opt.verbose)] + 'OK'))
 
 # begin execution

test/test-release.sh

@@ -222,7 +222,7 @@ i_hash='Internal hash function implementations'
 s_hash='Testing internal hash function implementations'
 t_hash="
 	$python test/hashfunc.py sha256 $rounds_max
-	$python test/hashfunc.py sha512 $rounds_max # native sha512 not used by MMGen
+	$python test/hashfunc.py sha512 $rounds_max # native sha512 - not used by MMGen
 	$python test/hashfunc.py keccak $rounds_max
 "
 f_hash='Hash function tests complete'
@@ -278,7 +278,7 @@ t_alts="
 		$gentest_py --coin=zec 2:ext $rounds
 		$gentest_py --coin=zec --type=zcash_z 2:ext $rounds_mid
 		$gentest_py --all 2:zcash_mini $rounds_mid
-		$gentest_py --all 2:moneropy $rounds_min
+		$gentest_py --all 2:moneropy $rounds_mid # very slow, be patient!
 	"
 }
 
@@ -449,23 +449,25 @@ f_tool='tooltest tests completed'
 i_gen='Gentest'
 s_gen="The following tests will run '$gentest_py' on mainnet and testnet for all supported coins"
 t_gen="
-	$gentest_py -q 2 $REFDIR/btcwallet.dump
-	$gentest_py -q --type=segwit 2 $REFDIR/btcwallet-segwit.dump
-	$gentest_py -q --type=bech32 2 $REFDIR/btcwallet-bech32.dump
-	$gentest_py -q 1:2 $rounds
-	$gentest_py -q --type=segwit 1:2 $rounds
-	$gentest_py -q --type=bech32 1:2 $rounds
-	$gentest_py -q --testnet=1 2 $REFDIR/btcwallet-testnet.dump
-	$gentest_py -q --testnet=1 1:2 $rounds
-	$gentest_py -q --testnet=1 --type=segwit 1:2 $rounds
-	$gentest_py -q --coin=ltc 2 $REFDIR/litecoin/ltcwallet.dump
-	$gentest_py -q --coin=ltc --type=segwit 2 $REFDIR/litecoin/ltcwallet-segwit.dump
-	$gentest_py -q --coin=ltc --type=bech32 2 $REFDIR/litecoin/ltcwallet-bech32.dump
-	$gentest_py -q --coin=ltc 1:2 $rounds
-	$gentest_py -q --coin=ltc --type=segwit 1:2 $rounds
-	$gentest_py -q --coin=ltc --testnet=1 2 $REFDIR/litecoin/ltcwallet-testnet.dump
-	$gentest_py -q --coin=ltc --testnet=1 1:2 $rounds
-	$gentest_py -q --coin=ltc --testnet=1 --type=segwit 1:2 $rounds
+	# wallet dumps:
+	$gentest_py 2 $REFDIR/btcwallet.dump
+	$gentest_py --type=segwit 2 $REFDIR/btcwallet-segwit.dump
+	$gentest_py --type=bech32 2 $REFDIR/btcwallet-bech32.dump
+	$gentest_py --testnet=1 2 $REFDIR/btcwallet-testnet.dump
+	$gentest_py --coin=ltc 2 $REFDIR/litecoin/ltcwallet.dump
+	$gentest_py --coin=ltc --type=segwit 2 $REFDIR/litecoin/ltcwallet-segwit.dump
+	$gentest_py --coin=ltc --type=bech32 2 $REFDIR/litecoin/ltcwallet-bech32.dump
+	$gentest_py --coin=ltc --testnet=1 2 $REFDIR/litecoin/ltcwallet-testnet.dump
+	# libsecp256k1 vs python-ecdsa:
+	$gentest_py 1:2 $rounds
+	$gentest_py --type=segwit 1:2 $rounds
+	$gentest_py --type=bech32 1:2 $rounds
+	$gentest_py --testnet=1 1:2 $rounds
+	$gentest_py --testnet=1 --type=segwit 1:2 $rounds
+	$gentest_py --coin=ltc 1:2 $rounds
+	$gentest_py --coin=ltc --type=segwit 1:2 $rounds
+	$gentest_py --coin=ltc --testnet=1 1:2 $rounds
+	$gentest_py --coin=ltc --testnet=1 --type=segwit 1:2 $rounds
 "
 f_gen='gentest tests completed'
 

test/tooltest.py

@@ -182,17 +182,17 @@ if opt.list_names:
 
 from mmgen.tx import is_wif,is_coin_addr
 
-msg_w = 33
+msg_w = 35
 def test_msg(m):
 	m2 = 'Testing {}'.format(m)
 	msg_r(green(m2+'\n') if opt.verbose else '{:{w}}'.format(m2,w=msg_w+8))
 
-maybe_compressed = ('','compressed')['C' in g.proto.mmtypes]
-maybe_segwit     = ('','segwit')['S' in g.proto.mmtypes]
-maybe_bech32     = ('','bech32')['B' in g.proto.mmtypes]
-maybe_type_compressed = ([],['--type=compressed'])['C' in g.proto.mmtypes]
-maybe_type_segwit     = ([],['--type=segwit'])['S' in g.proto.mmtypes]
-maybe_type_bech32     = ([],['--type=bech32'])['B' in g.proto.mmtypes]
+compressed = ('','compressed')['C' in g.proto.mmtypes]
+segwit     = ('','segwit')['S' in g.proto.mmtypes]
+bech32     = ('','bech32')['B' in g.proto.mmtypes]
+type_compressed_arg = ([],['--type=compressed'])['C' in g.proto.mmtypes]
+type_segwit_arg     = ([],['--type=segwit'])['S' in g.proto.mmtypes]
+type_bech32_arg     = ([],['--type=bech32'])['B' in g.proto.mmtypes]
 
 class MMGenToolTestUtils(object):
 
@@ -291,38 +291,38 @@ class MMGenToolTestCmds(object):
 
 	# Cryptocoin
 	def randwif(self,name):
-		for n,k in enumerate(['',maybe_compressed]):
+		for n,k in enumerate(['',compressed]):
 			ao = ['--type='+k] if k else []
 			ret = tu.run_cmd_out(name,add_opts=ao,Return=True,fn_idx=n+1)
 			ok_or_die(ret,is_wif,'WIF key')
 	def randpair(self,name):
-		for n,k in enumerate(['',maybe_compressed,maybe_segwit,maybe_bech32]):
+		for n,k in enumerate(['',compressed,segwit,bech32]):
 			ao = ['--type='+k] if k else []
-			wif,addr = tu.run_cmd_out(name,add_opts=ao,Return=True,fn_idx=n+1).split()
+			wif,addr = tu.run_cmd_out(name,add_opts=ao,Return=True,fn_idx=n+1,literal=True).split()
 			ok_or_die(wif,is_wif,'WIF key',skip_ok=True)
 			ok_or_die(addr,is_coin_addr,'Coin address')
 	def wif2addr(self,name,f1,f2,f3,f4):
-		for n,f,k,m in (
-			(1,f1,'',''),
-			(2,f2,'',maybe_compressed),
-			(3,f3,maybe_segwit,''),
-			(4,f4,maybe_bech32,'')
+		for n,f,k in (
+			(1,f1,''),
+			(2,f2,compressed),
+			(3,f3,segwit),
+			(4,f4,bech32)
 			):
 			ao = ['--type='+k] if k else []
 			wif = read_from_file(f).split()[0]
-			tu.run_cmd_out(name,wif,add_opts=ao,fn_idx=n,extra_msg=m)
+			tu.run_cmd_out(name,wif,add_opts=ao,fn_idx=n)
 	def wif2hex(self,name,f1,f2,f3,f4):
 		for n,f,m in (
 			(1,f1,''),
-			(2,f2,maybe_compressed),
-			(3,f3,'{} for {}'.format(maybe_compressed,maybe_segwit)),
-			(4,f4,'{} for {}'.format(maybe_compressed,maybe_bech32))
+			(2,f2,compressed),
+			(3,f3,'{} for {}'.format(compressed or 'uncompressed',segwit or 'p2pkh')),
+			(4,f4,'{} for {}'.format(compressed or 'uncompressed',bech32 or 'p2pkh'))
 			):
 			wif = read_from_file(f).split()[0]
 			tu.run_cmd_out(name,wif,fn_idx=n,extra_msg=m)
 	def privhex2addr(self,name,f1,f2,f3,f4):
 		keys = [read_from_file(f).rstrip() for f in (f1,f2,f3,f4)]
-		for n,k in enumerate(('',maybe_compressed,maybe_segwit,maybe_bech32)):
+		for n,k in enumerate(('',compressed,segwit,bech32)):
 			ao = ['--type='+k] if k else []
 			ret = tu.run_cmd(name,[keys[n]],add_opts=ao).rstrip()
 			iaddr = read_from_tmpfile(cfg,'randpair{}.out'.format(n+1)).split()[-1]
@@ -330,34 +330,34 @@ class MMGenToolTestCmds(object):
 			cmp_or_die(iaddr,ret)
 			ok()
 	def hex2wif(self,name,f1,f2,f3,f4):
-		for n,fi,fo,k in ((1,f1,f2,''),(2,f3,f4,maybe_compressed)):
+		for n,fi,fo,k in ((1,f1,f2,''),(2,f3,f4,compressed)):
 			ao = ['--type='+k] if k else []
 			ret = tu.run_cmd_chk(name,fi,fo,add_opts=ao)
 	def addr2pubhash(self,name,f1,f2,f3,f4):
 		for n,f,m,ao in (
 			(1,f1,'',[]),
-			(2,f2,'from {}'.format(maybe_compressed),[]),
-			(4,f4,'',maybe_type_bech32),
+			(2,f2,'from {}'.format(compressed or 'uncompressed'),[]),
+			(4,f4,'',type_bech32_arg),
 			):
 			addr = read_from_file(f).split()[-1]
 			tu.run_cmd_out(name,addr,fn_idx=n,add_opts=ao,extra_msg=m)
 	def pubhash2addr(self,name,f1,f2,f3,f4,f5,f6,f7,f8):
 		for n,fi,fo,m,ao in (
 			(1,f1,f2,'',[]),
-			(2,f3,f4,'from {}'.format(maybe_compressed),[]),
-			(4,f7,f8,'',maybe_type_bech32)
+			(2,f3,f4,'from {}'.format(compressed or 'uncompressed'),[]),
+			(4,f7,f8,'',type_bech32_arg)
 			):
 			tu.run_cmd_chk(name,fi,fo,add_opts=ao,extra_msg=m)
 	def privhex2pubhex(self,name,f1,f2,f3): # from Hex2wif
 		addr = read_from_file(f3).strip()
-		tu.run_cmd_out(name,addr,add_opts=maybe_type_compressed,fn_idx=3) # what about uncompressed?
+		tu.run_cmd_out(name,addr,add_opts=type_compressed_arg,fn_idx=3) # what about uncompressed?
 	def pubhex2redeem_script(self,name,f1,f2,f3): # from above
 		addr = read_from_file(f3).strip()
-		tu.run_cmd_out(name,addr,add_opts=maybe_type_segwit,fn_idx=3)
+		tu.run_cmd_out(name,addr,add_opts=type_segwit_arg,fn_idx=3)
 		type_save = opt.type
 		opt.type = 'segwit'
 		rs = read_from_tmpfile(cfg,'privhex2pubhex3.out').strip()
-		tu.run_cmd_out('pubhex2addr',rs,add_opts=maybe_type_segwit,fn_idx=3,hush=True)
+		tu.run_cmd_out('pubhex2addr',rs,add_opts=type_segwit_arg,fn_idx=3,hush=True)
 		opt.type = type_save
 		addr1 = read_from_tmpfile(cfg,'pubhex2addr3.out').strip()
 		addr2 = read_from_tmpfile(cfg,'randpair3.out').split()[1]
@@ -365,16 +365,16 @@ class MMGenToolTestCmds(object):
 		ok()
 	def wif2redeem_script(self,name,f1,f2,f3): # compare output with above
 		wif = read_from_file(f3).split()[0]
-		ret1 = tu.run_cmd_out(name,wif,add_opts=maybe_type_segwit,fn_idx=3,Return=True)
+		ret1 = tu.run_cmd_out(name,wif,add_opts=type_segwit_arg,fn_idx=3,Return=True)
 		ret2 = read_from_tmpfile(cfg,'pubhex2redeem_script3.out').strip()
 		cmp_or_die(ret1,ret2)
 		ok()
 	def wif2segwit_pair(self,name,f1,f2): # does its own checking, so just run
 		wif = read_from_file(f2).split()[0]
-		tu.run_cmd_out(name,wif,add_opts=maybe_type_segwit,fn_idx=2)
+		tu.run_cmd_out(name,wif,add_opts=type_segwit_arg,fn_idx=2)
 	def pubhex2addr(self,name,f1,f2,f3):
 		addr = read_from_file(f3).strip()
-		tu.run_cmd_out(name,addr,add_opts=maybe_type_segwit,fn_idx=3)
+		tu.run_cmd_out(name,addr,add_opts=type_segwit_arg,fn_idx=3)
 
 	def pipetest(self,name,f1,f2,f3):
 		wif = read_from_file(f3).split()[0]