mmgen-regtest: improvements + additional tests in test suite

MANIFEST.in

@@ -7,5 +7,6 @@ include scripts/bitcoind-walletunlock.py
 include scripts/compute-file-chksum.py
 include scripts/deinstall.sh
 include scripts/tx-old2new.py
+include scripts/test-release.sh
 
 prune test/ref/__db*

mmgen/globalvars.py

@@ -127,7 +127,7 @@ class g(object):
 		('tx_id','info'),
 		('tx_id','terse_info'),
 		('aug1hf','rbf'), # TODO: remove in 0.9.4
-		('batch','rescan')
+		('batch','rescan') # still incompatible as of Core 0.15.0
 	)
 	cfg_file_opts = (
 		'color','debug','hash_preset','http_timeout','no_license','rpc_host','rpc_port',

mmgen/main_regtest.py

@@ -28,9 +28,9 @@ opts_data = lambda: {
 	'sets': ( ('yes', True, 'quiet', True), ),
 	'options': """
 -h, --help          Print this help message
--m, --mixed         Create Bob and Alice's wallets with mixed address types
--e, --empty         Don't fund Bob and Alice's wallets on setup
 --, --longhelp      Print help message for long options (common options)
+-e, --empty         Don't fund Bob and Alice's wallets on setup
+-m, --mixed         Create Bob and Alice's wallets with mixed address types
 -q, --quiet         Produce quieter output
 -v, --verbose       Produce more verbose output
 """,

mmgen/regtest.py

@@ -149,7 +149,7 @@ def create_mmgen_wallet(user):
 	p.wait()
 
 def create_mmgen_addrs(user,addr_type):
-	gmsg('Creating MMGen addresses for user {} (type: {})'.format(user.capitalize(),addr_type))
+	gmsg('Creating MMGen addresses for user {} (type {})'.format(user.capitalize(),addr_type))
 	suf = ('-'+addr_type,'')[addr_type=='L']
 	try: os.unlink(mmaddrs(user).format(suf))
 	except: pass
@@ -164,11 +164,10 @@ def create_mmgen_addrs(user,addr_type):
 	p.wait()
 
 def import_mmgen_addrs(user,addr_type):
-	gmsg_r('Importing MMGen addresses for user {} (type: {})'.format(user.capitalize(),addr_type))
+	gmsg_r('Importing MMGen addresses for user {} (type {})'.format(user.capitalize(),addr_type))
 	suf = ('-'+addr_type,'')[addr_type=='L']
-	p = start_cmd('python','mmgen-addrimport',
-			'--{}'.format(user),'--data-dir='+g.data_dir,
-				'-q','--batch',mmaddrs(user).format(suf))
+	p = start_cmd('python','mmgen-addrimport','-q','--batch',
+			'--{}'.format(user),'--data-dir='+g.data_dir,mmaddrs(user).format(suf))
 	err = process_output(p)[1]
 	if not 'addresses imported' in err:
 		rdie(1,'Error importing MMGen addresses')
@@ -183,25 +182,17 @@ def stop_and_wait(silent=False,nonl=False,stop_silent=False,ignore_noconnect_err
 	stop(silent=stop_silent,ignore_noconnect_error=ignore_noconnect_error)
 	wait_for_daemon('stopped',silent=silent,nonl=nonl)
 
-def setup_wallet(user,addr_type):
+def setup_wallet(user,funded,stop=True):
 	gmsg_r("Setting up {}'s tracking wallet".format(user.capitalize()))
 	start_and_wait(user)
 	create_mmgen_wallet(user)
-	create_mmgen_addrs(user,addr_type)
-	import_mmgen_addrs(user,addr_type)
-	stop_and_wait(stop_silent=True)
-
-def setup_mixed_wallet(user):
-	gmsg_r("Setting up {}'s wallet (mixed address types)".format(user.capitalize()))
-	start_and_wait(user)
-	create_mmgen_wallet(user)
-	create_mmgen_addrs(user,'L')
-	create_mmgen_addrs(user,'C')
-	create_mmgen_addrs(user,'S')
-	import_mmgen_addrs(user,'L'); msg('')
-	import_mmgen_addrs(user,'C'); msg('')
-	import_mmgen_addrs(user,'S'); msg('')
-	stop_and_wait(silent=True,stop_silent=True)
+	mmtypes = ([funded],['L','C','S'])[bool(opt.mixed)]
+	for mmtype in mmtypes:
+		create_mmgen_addrs(user,mmtype)
+	for mmtype in mmtypes:
+		import_mmgen_addrs(user,mmtype); msg('')
+	if stop:
+		stop_and_wait(silent=True,stop_silent=True)
 
 def fund_wallet(user,amt):
 	gmsg('Sending {} BTC to {}'.format(amt,user.capitalize()))
@@ -220,25 +211,18 @@ def setup():
 	if test_daemon() != 'stopped':
 		stop_and_wait(silent=True,stop_silent=True)
 	create_data_dir()
-	gmsg_r('Starting setup')
 
-	start_and_wait('orig')
+	gmsg('Starting setup')
 
-	generate(432,silent=True)
-
-	stop_and_wait(silent=True,stop_silent=True)
-
-	if opt.mixed:
-		setup_mixed_wallet('bob')
-		setup_mixed_wallet('alice')
-	else:
-		setup_wallet('bob','C')
-		setup_wallet('alice','S')
+	setup_wallet('alice','S')
+	setup_wallet('bob','C')
 
 	if opt.empty:
-		msg("'--empty' selected: skipping funding of wallets")
+		ymsg("'--empty' selected: skipping funding of wallets")
 	else:
-		start_and_wait('orig',silent=True)
+		gmsg_r('Funding wallets')
+		start_and_wait('orig')
+		generate(432,silent=True)
 		fund_wallet('bob',init_amt)
 		fund_wallet('alice',init_amt)
 		generate(1)
@@ -283,10 +267,10 @@ def user(user=None,quiet=False):
 			return True
 		gmsg_r('Switching to user {}'.format(user.capitalize()))
 		stop_and_wait(silent=False,nonl=True,stop_silent=True)
-		start_and_wait(user,silent=False,nonl=True)
+		start_and_wait(user,nonl=True)
 	else:
 		gmsg_r('Starting regtest daemon with current user {}'.format(user.capitalize()))
-		start_and_wait(user,silent=False,nonl=True)
+		start_and_wait(user,nonl=True)
 	gmsg('done')
 
 def stop(silent=False,ignore_noconnect_error=True):

scripts/test-release.sh

@@ -32,37 +32,31 @@ function install {
 
 function do_test {
 	set +x
-	for i in "${CMDS[@]}"; do
+	for i in "$@"; do
 		echo -e "\n${GREEN}Running:$RESET $YELLOW$i$RESET"
 		eval "$i"
 	done
 }
 
-check
-(install)
-
-eval "cd .test-release/pydist/mmgen-*"
-
-CMDS=(
-	'test/test.py -On'
+T1=('test/test.py -On'
 	'test/test.py -On --segwit dfl_wallet main ref ref_other'
-	'test/test.py -On --segwit-random dfl_wallet main'
-)
-do_test
-
-CMDS=('test/test.py -On regtest')
-do_test
-
-# tooltest tests both segwit and non-segwit
-CMDS=(
-	'test/tooltest.py'
-	"test/gentest.py -q 2 $REFDIR/btcwallet.dump"
+	'test/test.py -On --segwit-random dfl_wallet main')
+T2=('test/test.py -On regtest')
+T3=('test/tooltest.py') # tooltest tests both segwit and non-segwit
+T4=("test/gentest.py -q 2 $REFDIR/btcwallet.dump"
 	"test/gentest.py -q --testnet=1 2 $REFDIR/btcwallet-testnet.dump"
 	'test/gentest.py -q 1:2 10'
 	'test/gentest.py -q --segwit 1:2 10'
 #	"scripts/tx-old2new.py -S $REFDIR/tx_*raw >/dev/null 2>&1"
-	"scripts/compute-file-chksum.py $REFDIR/*testnet.rawtx >/dev/null 2>&1"
-)
-do_test
+	"scripts/compute-file-chksum.py $REFDIR/*testnet.rawtx >/dev/null 2>&1")
+
+check
+(install)
+eval "cd .test-release/pydist/mmgen-*"
+
+do_test "${T1[@]}"
+do_test "${T2[@]}"
+do_test "${T3[@]}"
+do_test "${T4[@]}"
 
 echo -e "\n${GREEN}All OK$RESET"

test/test.py

@@ -538,20 +538,38 @@ cmd_group['conv_out'] = ( # writing
 )
 
 cmd_group['regtest'] = (
-	('regtest_setup',       'regtest (Bob and Alice) mode setup'),
-	('regtest_alice_bal1',  "Alice's balance"),
-	('regtest_bob_bal1',    "Bob's balance"),
-	('regtest_bob_split',   "splitting Bob's funds"),
-	('regtest_generate',    'mining a block'),
-	('regtest_bob_bal2',    "Bob's balance"),
-	('regtest_bob_rbf_send','sending funds to Alice (RBF)'),
-	('regtest_get_mempool1','mempool (before RBF bump)'),
-	('regtest_bob_rbf_bump1','bumping RBF transaction'),
-	('regtest_get_mempool2','mempool (after RBF bump)'),
-	('regtest_generate',    'mining a block'),
-	('regtest_bob_bal3',    "Bob's balance"),
-	('regtest_alice_bal2',  "Alice's balance"),
-	('regtest_stop',        'stopping regtest daemon'),
+	('regtest_setup',              'regtest (Bob and Alice) mode setup'),
+	('regtest_bob_bal1',           "Bob's balance"),
+	('regtest_bob_split1',         "splitting Bob's funds"),
+	('regtest_generate',           'mining a block'),
+	('regtest_bob_bal2',           "Bob's balance"),
+	('regtest_bob_rbf_send',       'sending funds to Alice (RBF)'),
+	('regtest_get_mempool1',       'mempool (before RBF bump)'),
+	('regtest_bob_rbf_bump',       'bumping RBF transaction'),
+	('regtest_get_mempool2',       'mempool (after RBF bump)'),
+	('regtest_generate',           'mining a block'),
+	('regtest_bob_bal3',           "Bob's balance"),
+	('regtest_bob_pre_import',     'sending to non-imported address'),
+	('regtest_generate',           'mining a block'),
+	('regtest_bob_import_addr',    'importing non-MMGen address with --rescan'),
+	('regtest_bob_bal4',           "Bob's balance (after import with rescan)"),
+	('regtest_bob_import_list',    'importing flat address list'),
+	('regtest_bob_split2',         "splitting Bob's funds"),
+	('regtest_generate',           'mining a block'),
+	('regtest_bob_bal5',           "Bob's balance"),
+	('regtest_bob_send_non_mmgen', 'sending funds to Alice (from non-MMGen addrs)'),
+	('regtest_generate',           'mining a block'),
+	('regtest_bob_bal6',           "Bob's balance"),
+	('regtest_alice_bal2',         "Alice's balance"),
+	('regtest_alice_add_label1',   'adding a label'),
+	('regtest_alice_chk_label1',   'the label'),
+	('regtest_alice_add_label2',   'adding a label'),
+	('regtest_alice_chk_label2',   'the label'),
+	('regtest_alice_edit_label1',  'editing a label'),
+	('regtest_alice_chk_label3',   'the label'),
+	('regtest_alice_remove_label1','removing a label'),
+	('regtest_alice_chk_label4',   'the label'),
+	('regtest_stop',               'stopping regtest daemon'),
 )
 
 cmd_list = OrderedDict()
@@ -1858,23 +1876,16 @@ class MMGenTestSuite(object):
 		os.environ['MMGEN_TEST_SUITE'] = '' # mnemonic is piped to stdin, so stop being a terminal
 		t = MMGenExpect(name,'mmgen-regtest',['-m','--data-dir='+data_dir,'setup'])
 		os.environ['MMGEN_TEST_SUITE'] = '1'
-		t.expect('Mined')
-		t.expect('Setting up')
-		t.expect('Creating')
-		t.expect('Creating')
-		t.expect('Importing')
-		t.expect('Importing')
-		t.expect('Importing')
-		t.expect('Setting up')
-		t.expect('Creating')
-		t.expect('Creating')
-		t.expect('Importing')
-		t.expect('Importing')
-		t.expect('Importing')
-		t.expect('Sending')
-		t.expect('Sending')
-		t.expect('Mined')
-		t.expect('Setup complete')
+		t.expect('Starting setup')
+
+		for user in ('alice','bob'):
+			t.expect('Setting up')
+			for i in range(4): t.expect('Creating')
+			for i in range(3): t.expect('Importing')
+
+		for s in ('Funding','Mined','Sending','Sending','Mined','Setup complete'):
+			t.expect(s)
+
 		t.ok()
 
 	def regtest_user_bal(self,name,user,bal):
@@ -1886,7 +1897,7 @@ class MMGenTestSuite(object):
 		return self.regtest_user_bal(name,'alice','500')
 
 	def regtest_alice_bal2(self,name):
-		return self.regtest_user_bal(name,'alice','600')
+		return self.regtest_user_bal(name,'alice','986.9995799')
 
 	def regtest_bob_bal1(self,name):
 		return self.regtest_user_bal(name,'bob','500')
@@ -1897,6 +1908,15 @@ class MMGenTestSuite(object):
 	def regtest_bob_bal3(self,name):
 		return self.regtest_user_bal(name,'bob','399.9998214')
 
+	def regtest_bob_bal4(self,name):
+		return self.regtest_user_bal(name,'bob','399.9998079')
+
+	def regtest_bob_bal5(self,name):
+		return self.regtest_user_bal(name,'bob','399.9996799')
+
+	def regtest_bob_bal6(self,name):
+		return self.regtest_user_bal(name,'bob','13')
+
 	def regtest_user_txdo(self,name,user,fee,outputs_cl,outputs_prompt,extra_args=[],no_send=False):
 		os.environ['MMGEN_BOGUS_SEND'] = ''
 		t = MMGenExpect(name,'mmgen-txdo',
@@ -1918,7 +1938,7 @@ class MMGenTestSuite(object):
 		t.read()
 		t.ok()
 
-	def regtest_bob_split(self,name):
+	def regtest_bob_split1(self,name):
 		from mmgen.regtest import sids
 		outputs_cl = [sids['bob']+':C:1,100', sids['bob']+':L:2,200',sids['bob']+':S:2']
 		return self.regtest_user_txdo(name,'bob','20s',outputs_cl,'1')
@@ -1931,6 +1951,14 @@ class MMGenTestSuite(object):
 			sids['bob']+':S:2']
 		return self.regtest_user_txdo(name,'bob','10s',outputs_cl,'3',extra_args=['--rbf'])
 
+	def regtest_bob_send_non_mmgen(self,name):
+		from mmgen.regtest import sids
+		fn = os.path.join(cfg['tmpdir'],'non-mmgen.keys')
+		outputs_cl = [
+			'2N8w8qTupvd9L9wLFbrn6UhdfF1gadDAmFD,10', # sids['alice']:S:2
+			'2NF4y3y4CEjQCcssjX2BDLHT88XHn8z53JS']    # sids['alice']:S:3
+		return self.regtest_user_txdo(name,'bob','0.0001',outputs_cl,'3-9',extra_args=['--keys-from-file='+fn])
+
 	def regtest_user_txbump(self,name,user,txfile,fee,red_op,no_send=False):
 		os.environ['MMGEN_BOGUS_SEND'] = ''
 		t = MMGenExpect(name,'mmgen-txbump',
@@ -1947,7 +1975,7 @@ class MMGenTestSuite(object):
 		t.read()
 		t.ok()
 
-	def regtest_bob_rbf_bump1(self,name):
+	def regtest_bob_rbf_bump(self,name):
 		txfile = get_file_with_ext(',10].sigtx',cfg['tmpdir'],delete=False,no_dot=True)
 		return self.regtest_user_txbump(name,'bob',txfile,'60s','c')
 
@@ -1977,6 +2005,87 @@ class MMGenTestSuite(object):
 			rdie(2,'TX in mempool has not changed!  RBF bump failed')
 		ok()
 
+	def regtest_user_import(self,name,user,args):
+		t = MMGenExpect(name,'mmgen-addrimport',['--quiet','--'+user]+args)
+		t.read()
+		t.ok()
+
+	@staticmethod
+	def gen_pairs(n):
+		return [subprocess.check_output(
+					['python','mmgen-tool','--testnet=1','-r0','randpair','compressed={}'.format((i+1)%2)]).split()
+						for i in range(n)]
+
+	def regtest_bob_pre_import(self,name):
+		pairs = self.gen_pairs(5)
+		write_to_tmpfile(cfg,'non-mmgen.keys','\n'.join([a[0] for a in pairs])+'\n')
+		write_to_tmpfile(cfg,'non-mmgen.addrs','\n'.join([a[1] for a in pairs])+'\n')
+		return self.regtest_user_txdo(name,'bob','10s',[pairs[0][1]],'3')
+
+	def regtest_bob_import_addr(self,name):
+		addr = read_from_tmpfile(cfg,'non-mmgen.addrs').split()[0]
+		return self.regtest_user_import(name,'bob',['--rescan','--address='+addr])
+
+	def regtest_bob_import_list(self,name):
+		fn = os.path.join(cfg['tmpdir'],'non-mmgen.addrs')
+		return self.regtest_user_import(name,'bob',['--addrlist',fn])
+
+	def regtest_bob_split2(self,name):
+		addrs = read_from_tmpfile(cfg,'non-mmgen.addrs').split()
+		amts = (a for a in (1.12345678,2.87654321,3.33443344,4.00990099,5.43214321))
+		outputs1 = ['{},{}'.format(a,amts.next()) for a in addrs]
+		from mmgen.regtest import sids
+		outputs2 = [sids['bob']+':C:2,6', sids['bob']+':L:3,7',sids['bob']+':S:3']
+		return self.regtest_user_txdo(name,'bob','20s',outputs1+outputs2,'1-2')
+
+	def regtest_user_add_label(self,name,user,addr,label):
+		t = MMGenExpect(name,'mmgen-tool',['--'+user,'add_label',addr,label])
+		t.expect('Added label.*in tracking wallet',regex=True)
+		t.ok()
+
+	def regtest_user_remove_label(self,name,user,addr):
+		t = MMGenExpect(name,'mmgen-tool',['--'+user,'remove_label',addr])
+		t.expect('Removed label.*in tracking wallet',regex=True)
+		t.ok()
+
+	def regtest_alice_add_label1(self,name):
+		return self.regtest_user_add_label(name,'alice','9304C211:S:1','Original Label')
+
+	def regtest_alice_add_label2(self,name):
+		return self.regtest_user_add_label(name,'alice','9304C211:S:1','Replacement Label')
+
+	def regtest_alice_remove_label1(self,name):
+		return self.regtest_user_remove_label(name,'alice','9304C211:S:1')
+
+	def regtest_user_chk_label(self,name,user,addr,label):
+		t = MMGenExpect(name,'mmgen-tool',['--'+user,'listaddresses','all_labels=1'])
+		t.expect('{}\s+\S{{30}}\S+\s+{}\s+'.format(addr,label),regex=True)
+		t.ok()
+
+	def regtest_alice_chk_label1(self,name):
+		return self.regtest_user_chk_label(name,'alice','9304C211:S:1','Original Label')
+
+	def regtest_alice_chk_label2(self,name):
+		return self.regtest_user_chk_label(name,'alice','9304C211:S:1','Replacement Label')
+
+	def regtest_alice_chk_label3(self,name):
+		return self.regtest_user_chk_label(name,'alice','9304C211:S:1','Edited Label')
+
+	def regtest_alice_chk_label4(self,name):
+		return self.regtest_user_chk_label(name,'alice','9304C211:S:1','-')
+
+	def regtest_user_edit_label(self,name,user,output,label):
+		t = MMGenExpect(name,'mmgen-txcreate',['-B','--'+user,'-i'])
+		t.expect(r"'q'=quit view, .*?:.",'M',regex=True)
+		t.expect(r"'q'=quit view, .*?:.",'l',regex=True)
+		t.expect(r"Enter unspent.*return to main menu\):.",output+'\n',regex=True)
+		t.expect(r"Enter label text.*return to main menu\):.",label+'\n',regex=True)
+		t.expect(r"'q'=quit view, .*?:.",'q',regex=True)
+		t.ok()
+
+	def regtest_alice_edit_label1(self,name):
+		return self.regtest_user_edit_label(name,'alice','3','Edited Label')
+
 	def regtest_stop(self,name):
 		t = MMGenExpect(name,'mmgen-regtest',['stop'])
 		t.ok()