tw.ctl, regtest, cmdtest regtest: cleanups

mmgen/proto/btc/regtest.py

@@ -25,6 +25,7 @@ from ...util import msg,gmsg,die,capfirst,suf
 from ...protocol import init_proto
 from ...rpc import rpc_init,json_encoder
 from ...objmethods import MMGenObject
+from ...daemon import CoinDaemon
 
 def create_data_dir(cfg,data_dir):
 	try:
@@ -63,9 +64,9 @@ class MMGenRegtest(MMGenObject):
 	coins        = ('btc','bch','ltc')
 	usr_cmds     = ('setup','generate','send','start','stop', 'state', 'balances','mempool','cli','wallet_cli')
 
-	hdseed = 'beadcafe' * 8
-	miner_wif = 'cTyMdQ2BgfAsjopRVZrj7AoEGp97pKfrC2NkqLuwHr4KHfPNAKwp'
-	miner_addrs = {
+	bdb_hdseed = 'beadcafe' * 8
+	bdb_miner_wif = 'cTyMdQ2BgfAsjopRVZrj7AoEGp97pKfrC2NkqLuwHr4KHfPNAKwp'
+	bdb_miner_addrs = {
 		# cTyMdQ2BgfAsjopRVZrj7AoEGp97pKfrC2NkqLuwHr4KHfPNAKwp hdseed=1
 		'btc': 'bcrt1qaq8t3pakcftpk095tnqfv5cmmczysls024atnd',
 		'ltc': 'rltc1qaq8t3pakcftpk095tnqfv5cmmczysls05c8zyn',
@@ -77,17 +78,36 @@ class MMGenRegtest(MMGenObject):
 		self.coin = coin.lower()
 		assert self.coin in self.coins, f'{coin!r}: invalid coin for regtest'
 
-		from ...daemon import CoinDaemon
-		self.proto = init_proto( cfg, self.coin, regtest=True, need_amt=True )
-		self.d = CoinDaemon(cfg,self.coin+'_rt',test_suite=cfg.test_suite)
-		self.miner_addr = self.miner_addrs[self.coin]
+		self.proto = init_proto(cfg, self.coin, regtest=True, need_amt=True)
+		self.d = CoinDaemon(
+			cfg,
+			self.coin + '_rt',
+			test_suite = cfg.test_suite)
+
+	# Caching creates problems (broken pipe) when recreating + loading wallets,
+	# so reinstantiate with every call:
+	@property
+	async def rpc(self):
+		return await rpc_init(self.cfg, self.proto, backend=None, daemon=self.d)
+
+	@property
+	async def miner_addr(self):
+		if not hasattr(self,'_miner_addr'):
+			self._miner_addr = self.bdb_miner_addrs[self.coin]
+		return self._miner_addr
+
+	@property
+	async def miner_wif(self):
+		if not hasattr(self,'_miner_wif'):
+			self._miner_wif = self.bdb_miner_wif
+		return self._miner_wif
 
 	def create_hdseed_wif(self):
 		from ...tool.api import tool_api
 		t = tool_api(self.cfg)
 		t.init_coin(self.proto.coin,self.proto.network)
 		t.addrtype = 'compressed' if self.proto.coin == 'BCH' else 'bech32'
-		return t.hex2wif(self.hdseed)
+		return t.hex2wif(self.bdb_hdseed)
 
 	async def generate(self,blocks=1,silent=False):
 
@@ -101,7 +121,7 @@ class MMGenRegtest(MMGenObject):
 		out = await self.rpc_call(
 			'generatetoaddress',
 			blocks,
-			self.miner_addr,
+			await self.miner_addr,
 			wallet = 'miner' )
 
 		if len(out) != blocks:
@@ -110,6 +130,14 @@ class MMGenRegtest(MMGenObject):
 		if not silent:
 			gmsg(f'Mined {blocks} block{suf(blocks)}')
 
+	async def create_wallet(self,user):
+		return await (await self.rpc).icall(
+			'createwallet',
+			wallet_name     = user,
+			blank           = True,
+			no_keys         = user != 'miner',
+			load_on_startup = False)
+
 	async def setup(self):
 
 		try:
@@ -126,15 +154,9 @@ class MMGenRegtest(MMGenObject):
 
 		self.d.start(silent=True)
 
-		rpc = await rpc_init(self.cfg,self.proto,backend=None,daemon=self.d)
 		for user in ('miner','bob','alice'):
 			gmsg(f'Creating {capfirst(user)}’s tracking wallet')
-			await rpc.icall(
-				'createwallet',
-				wallet_name     = user,
-				blank           = True,
-				no_keys         = user != 'miner',
-				load_on_startup = False )
+			await self.create_wallet(user)
 
 		# BCH and LTC daemons refuse to set HD seed with empty blockchain ("in IBD" error),
 		# so generate a block:
@@ -143,11 +165,11 @@ class MMGenRegtest(MMGenObject):
 		# Unfortunately, we don’t get deterministic output with BCH and LTC even with fixed
 		# hdseed, as their 'sendtoaddress' calls produce non-deterministic TXIDs due to random
 		# input ordering and fee estimation.
-		await rpc.call(
+		await (await self.rpc).call(
 			'sethdseed',
 			True,
 			self.create_hdseed_wif(),
-			wallet = 'miner' )
+			wallet = 'miner')
 
 		# Broken litecoind can only mine 431 blocks in regtest mode, so generate just enough
 		# blocks to fund the test suite
@@ -175,8 +197,7 @@ class MMGenRegtest(MMGenObject):
 	async def rpc_call(self,*args,wallet=None,start_daemon=True):
 		if start_daemon and self.d.state == 'stopped':
 			await self.start_daemon()
-		rpc = await rpc_init(self.cfg,self.proto,backend=None,daemon=self.d)
-		return await rpc.call(*args,wallet=wallet)
+		return await (await self.rpc).call(*args, wallet=wallet)
 
 	async def start(self):
 		if self.d.state == 'stopped':

mmgen/proto/btc/rpc.py

@@ -47,8 +47,16 @@ class CallSigs:
 
 	class bitcoin_core:
 
-		@classmethod
-		def createwallet(cls,wallet_name,no_keys=True,blank=True,passphrase='',load_on_startup=True):
+		def __init__(self, cfg):
+			self.cfg = cfg
+
+		def createwallet(
+				self,
+				wallet_name,
+				no_keys         = True,
+				blank           = True,
+				passphrase      = '',
+				load_on_startup = True):
 			"""
 			Quirk: when --datadir is specified (even if standard), wallet is created directly in
 			datadir, otherwise in datadir/wallets
@@ -64,8 +72,7 @@ class CallSigs:
 				load_on_startup # 7. load_on_startup
 			)
 
-		@classmethod
-		def gettransaction(cls,txid,include_watchonly,verbose):
+		def gettransaction(self, txid, include_watchonly, verbose):
 			return (
 				'gettransaction',
 				txid,               # 1. transaction id
@@ -76,8 +83,13 @@ class CallSigs:
 
 	class litecoin_core(bitcoin_core):
 
-		@classmethod
-		def createwallet(cls,wallet_name,no_keys=True,blank=True,passphrase='',load_on_startup=True):
+		def createwallet(
+				self,
+				wallet_name,
+				no_keys         = True,
+				blank           = True,
+				passphrase      = '',
+				load_on_startup = True):
 			return (
 				'createwallet',
 				wallet_name,    # 1. wallet_name
@@ -85,8 +97,7 @@ class CallSigs:
 				blank,          # 3. blank (no keys or seed)
 			)
 
-		@classmethod
-		def gettransaction(cls,txid,include_watchonly,verbose):
+		def gettransaction(self, txid, include_watchonly, verbose):
 			return (
 				'gettransaction',
 				txid,               # 1. transaction id
@@ -113,7 +124,7 @@ class BitcoinRPCClient(RPCClient,metaclass=AsyncInit):
 
 		self.proto = proto
 		self.daemon = daemon
-		self.call_sigs = getattr(CallSigs,daemon.id,None)
+		self.call_sigs = getattr(CallSigs,daemon.id)(cfg)
 
 		super().__init__(
 			cfg  = cfg,

mmgen/proto/btc/tw/ctl.py

@@ -27,12 +27,12 @@ class BitcoinTwCtl(TwCtl):
 		raise NotImplementedError('not implemented')
 
 	@write_mode
-	async def import_address(self,addr,label): # rescan is True by default, so set to False
-		return await self.rpc.call('importaddress',addr,label,False)
+	async def import_address(self, addr, label, rescan=False): # rescan is True by default, so set to False
+		return await self.rpc.call('importaddress', addr, label, rescan)
 
 	@write_mode
-	def batch_import_address(self,arg_list):   # rescan is True by default, so set to False
-		return self.rpc.batch_call('importaddress',[a+(False,) for a in arg_list])
+	async def batch_import_address(self,arg_list):
+		return await self.rpc.batch_call('importaddress', arg_list)
 
 	@write_mode
 	async def remove_address(self,addr):

mmgen/proto/eth/tw/ctl.py

@@ -92,7 +92,7 @@ class EthereumTwCtl(TwCtl):
 		pass
 
 	@write_mode
-	async def import_address(self,addr,label):
+	async def import_address(self,addr,label,rescan=False):
 		r = self.data_root
 		if addr in r:
 			if not r[addr]['mmid'] and label.mmid:

mmgen/tw/ctl.py

@@ -316,7 +316,7 @@ class TwCtl(MMGenObject,metaclass=AsyncInit):
 
 		if batch:
 			msg_r(f'Batch importing {len(out)} address{suf(data,"es")}...')
-			ret = await self.batch_import_address((a,b) for a,b,c in out)
+			ret = await self.batch_import_address((a, b, False) for a, b, c in out)
 			msg(f'done\n{len(ret)} addresses imported')
 		else:
 			if gather: # this seems to provide little performance benefit

test/cmdtest.py

@@ -73,7 +73,7 @@ def create_shm_dir(data_dir,trash_dir):
 
 	return shm_dir
 
-import sys,os,time
+import sys, os, time, asyncio
 
 if sys.argv[-1] == 'clean':
 	os.environ['MMGEN_TEST_SUITE'] = '1'
@@ -95,7 +95,7 @@ else:
 
 from mmgen.cfg import Config,gc
 from mmgen.color import red,yellow,green,blue,cyan,gray,nocolor,init_color
-from mmgen.util import msg,Msg,bmsg,die,suf,make_timestr,async_run
+from mmgen.util import msg, Msg, bmsg, die, suf, make_timestr
 
 from test.include.common import (
 	set_globals,
@@ -820,7 +820,7 @@ class CmdTestRunner:
 									if isinstance(e,KeyError) and e.args[0] == cmdname:
 										ret = getattr(self.tg,cmdname)()
 										if type(ret).__name__ == 'coroutine':
-											async_run(ret)
+											asyncio.run(ret)
 									else:
 										raise
 								do_between()
@@ -946,7 +946,7 @@ class CmdTestRunner:
 
 		ret = getattr(self.tg,cmd)(*arg_list) # run the test
 		if type(ret).__name__ == 'coroutine':
-			ret = async_run(ret)
+			ret = asyncio.run(ret)
 		self.process_retval(cmd,ret)
 
 		if cfg.profile:

test/cmdtest_py_d/ct_regtest.py

@@ -23,6 +23,7 @@ test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite
 import sys,os,json,time,re
 from decimal import Decimal
 
+from mmgen.proto.btc.regtest import MMGenRegtest
 from mmgen.color import yellow
 from mmgen.util import msg_r,die,gmsg,capfirst,fmt_list
 from mmgen.protocol import init_proto
@@ -463,15 +464,16 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared):
 		self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True )
 		coin = self.proto.coin.lower()
 
+		gldict = globals()
 		for k in rt_data:
-			setattr( sys.modules[__name__], k, rt_data[k][coin] if coin in rt_data[k] else None )
+			gldict[k] = rt_data[k][coin] if coin in rt_data[k] else None
+
+		self.rt = MMGenRegtest(cfg, self.proto.coin)
 
 		if self.proto.coin == 'BTC':
 			self.test_rbf = True # tests are non-coin-dependent, so run just once for BTC
 			if cfg.test_suite_deterministic:
 				self.deterministic = True
-				self.miner_addr = 'bcrt1qaq8t3pakcftpk095tnqfv5cmmczysls024atnd' # regtest.create_hdseed()
-				self.miner_wif = 'cTyMdQ2BgfAsjopRVZrj7AoEGp97pKfrC2NkqLuwHr4KHfPNAKwp'
 
 		self.spawn_env['MMGEN_BOGUS_SEND'] = ''
 		self.write_to_tmpfile('wallet_password',rt_pw)
@@ -508,7 +510,9 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared):
 			rmtree(joinpath(self.tr.data_dir,'regtest'))
 		except:
 			pass
-		t = self.spawn('mmgen-regtest',['-n','setup'])
+		t = self.spawn(
+			'mmgen-regtest',
+			['--setup-no-stop-daemon', 'setup'])
 		for s in ('Starting','Creating','Creating','Creating','Mined','Setup complete'):
 			t.expect(s)
 		return t
@@ -626,20 +630,20 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared):
 	def addrimport_alice(self):
 		return self.addrimport('alice',batch=False,quiet=False)
 
-	def bob_import_miner_addr(self):
+	async def bob_import_miner_addr(self):
 		if not self.deterministic:
 			return 'skip'
 		return self.spawn(
 			'mmgen-addrimport',
-			[ '--bob', '--rescan', '--quiet', f'--address={self.miner_addr}' ] )
+			[ '--bob', '--rescan', '--quiet', f'--address={await self.rt.miner_addr}' ] )
 
-	def fund_wallet_deterministic(self,addr,utxo_nums,skip_passphrase=False):
+	async def fund_wallet_deterministic(self, addr, utxo_nums, skip_passphrase=False):
 		"""
 		the deterministic funding method using specific inputs
 		"""
 		if not self.deterministic:
 			return 'skip'
-		self.write_to_tmpfile('miner.key',f'{self.miner_wif}\n')
+		self.write_to_tmpfile('miner.key',f'{await self.rt.miner_wif}\n')
 		keyfile = joinpath(self.tmpdir,'miner.key')
 
 		return self.user_txdo(
@@ -667,18 +671,16 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared):
 		if not self.deterministic:
 			return 'skip'
 		self.spawn('',msg_only=True)
-		from mmgen.proto.btc.regtest import MMGenRegtest
-		rt = MMGenRegtest(cfg,self.proto.coin)
-		await rt.stop()
+		await self.rt.stop()
 		from shutil import rmtree
 		imsg('Deleting Bob’s old tracking wallet')
-		rmtree(os.path.join(rt.d.datadir,'regtest','wallets','bob'),ignore_errors=True)
-		rt.init_daemon()
-		rt.d.start(silent=True)
+		rmtree(os.path.join(self.rt.d.datadir, 'regtest', 'wallets', 'bob'), ignore_errors=True)
+		self.rt.init_daemon()
+		self.rt.d.start(silent=True)
 		imsg('Creating Bob’s new tracking wallet')
-		await rt.rpc_call('createwallet','bob',True,True,None,False,False,False)
-		await rt.stop()
-		await rt.start()
+		await self.rt.create_wallet('bob')
+		await self.rt.stop()
+		await self.rt.start()
 		return 'ok'
 
 	def addrimport_bob2(self):
@@ -1429,9 +1431,7 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared):
 		imsg('Unloading Carol’s tracking wallet')
 		t = self.spawn('mmgen-regtest',['cli','unloadwallet','carol'])
 		t.ok()
-		from mmgen.rpc import rpc_init
-		rpc = await rpc_init(cfg,self.proto)
-		wdir = joinpath(rpc.daemon.network_datadir,'wallets','carol')
+		wdir = joinpath((await self.rt.rpc).daemon.network_datadir, 'wallets', 'carol')
 		from shutil import rmtree
 		imsg('Deleting Carol’s tracking wallet')
 		rmtree(wdir)