Daemon: handle path and RPC port selection logic in class

mmgen/daemon.py

@@ -31,18 +31,17 @@ class Daemon(MMGenObject):
 
 	subclasses_must_implement = ('state','stop_cmd')
 
-	network_ids = ('btc','btc_tn','bch','bch_tn','ltc','ltc_tn','xmr','eth','etc')
+	network_ids = ('btc','btc_tn','btc_rt','bch','bch_tn','bch_rt','ltc','ltc_tn','ltc_rt','xmr','eth','etc')
 
-	cd = namedtuple('coin_data',['coin','coind_exec','cli_exec','conf_file','dfl_rpc','dfl_rpc_tn'])
+	cd = namedtuple('daemon_data',['coin','coind_exec','cli_exec','conf_file','dfl_rpc','dfl_rpc_tn','dfl_rpc_rt'])
 	daemon_ids = {
-		'btc': cd('Bitcoin',         'bitcoind',    'bitcoin-cli', 'bitcoin.conf',  8333,18333),
-		'bch': cd('Bcash',           'bitcoind-abc','bitcoin-cli', 'bitcoin.conf',  8442,18442), # MMGen RPC dfls
-		'ltc': cd('Litecoin',        'litecoind',   'litecoin-cli','litecoin.conf', 9333,19335),
-		'xmr': cd('Monero',          'monerod',     'monerod',     'bitmonero.conf',18082,28082),
-		'eth': cd('Ethereum',        'parity',      'parity',      'parity.conf',   8545,8545),
-		'etc': cd('Ethereum Classic','parity',      'parity',      'parity.conf',   8545,8545)
+		'btc': cd('Bitcoin',         'bitcoind',    'bitcoin-cli', 'bitcoin.conf',  8333,18333,18444),
+		'bch': cd('Bcash',           'bitcoind-abc','bitcoin-cli', 'bitcoin.conf',  8442,18442,18553),# MMGen RPC dfls
+		'ltc': cd('Litecoin',        'litecoind',   'litecoin-cli','litecoin.conf', 9333,19335,19446),
+		'xmr': cd('Monero',          'monerod',     'monerod',     'bitmonero.conf',18082,None,None),
+		'eth': cd('Ethereum',        'parity',      'parity',      'parity.conf',   8545,None,None),
+		'etc': cd('Ethereum Classic','parity',      'parity',      'parity.conf',   8545,None,None)
 	}
-	port_shift = 1000
 
 	debug = False
 	wait = True
@@ -63,20 +62,27 @@ class Daemon(MMGenObject):
 	usr_cli_args = []
 	usr_shared_args = []
 
-	usr_rpc_port = None
-
-	def __new__(cls,network_id,datadir=None,rpc_port=None,desc='test suite daemon'):
-
+	def __new__(cls,network_id,test_suite=False):
 		network_id = network_id.lower()
 		assert network_id in cls.network_ids, '{!r}: invalid network ID'.format(network_id)
 
-		if not datadir: # hack for throwaway instances
-			datadir = '/tmp/foo'
-		assert os.path.isabs(datadir), '{!r}: invalid datadir (not an absolute path)'.format(datadir)
+		if test_suite:
+			rel_datadir = os.path.join('test','daemons')
+			desc = 'test suite daemon'
+		elif not network_id.endswith('_rt'):
+			raise RuntimeError('only test suite and regtest supported')
 
 		if network_id.endswith('_tn'):
 			daemon_id = network_id[:-3]
 			network = 'testnet'
+		elif network_id.endswith('_rt'):
+			daemon_id = network_id[:-3]
+			network = 'regtest'
+			desc = 'regtest daemon'
+			if test_suite:
+				rel_datadir = os.path.join('test','data_dir','regtest')
+			else:
+				rel_datadir = os.path.join(g.data_dir_root,'regtest')
 		else:
 			daemon_id = network_id
 			network = 'mainnet'
@@ -86,25 +92,32 @@ class Daemon(MMGenObject):
 			else EthereumDaemon if daemon_id in ('eth','etc')
 			else BitcoinDaemon )
 
+		if test_suite:
+			me.datadir = os.path.abspath(os.path.join(os.getcwd(),rel_datadir,daemon_id))
+			me.port_shift = 1237
+		else:
+			me.datadir = os.path.join(rel_datadir,daemon_id)
+			me.port_shift = 0
+
 		me.network_id = network_id
 		me.daemon_id = daemon_id
 		me.network = network
-		me.datadir = datadir
-		me.platform = g.platform
 		me.desc = desc
-		me.usr_rpc_port = rpc_port
+		me.platform = g.platform
 		return me
 
-	def __init__(self,network_id,datadir=None,rpc_port=None,desc='test suite daemon'):
+	def __init__(self,network_id,test_suite=False):
 
 		self.pidfile = '{}/{}-daemon.pid'.format(self.datadir,self.network)
 
 		for k in self.daemon_ids[self.daemon_id]._fields:
 			setattr(self,k,getattr(self.daemon_ids[self.daemon_id],k))
 
-		self.rpc_port = self.usr_rpc_port or (
-			(self.dfl_rpc,self.dfl_rpc_tn)[self.network=='testnet'] + self.port_shift
-		)
+		self.rpc_port = {
+				'mainnet': self.dfl_rpc,
+				'testnet': self.dfl_rpc_tn,
+				'regtest': self.dfl_rpc_rt,
+			}[self.network] + self.port_shift
 
 		self.net_desc = '{} {}'.format(self.coin,self.network)
 		self.subclass_init()

mmgen/main_autosign.py

@@ -136,7 +136,7 @@ def check_daemons_running():
 			continue
 		if g.test_suite:
 			g.proto.daemon_data_dir = 'test/daemons/' + coin.lower()
-			g.rpc_port = Daemon(get_network_id(coin,g.testnet)).rpc_port
+			g.rpc_port = Daemon(get_network_id(coin,g.testnet),test_suite=True).rpc_port
 		vmsg('Checking {} daemon'.format(coin))
 		try:
 			rpc_init(reinit=True)
@@ -199,7 +199,7 @@ def sign_tx_file(txfile,signed_txs):
 		if g.proto.sign_mode == 'daemon':
 			if g.test_suite:
 				g.proto.daemon_data_dir = 'test/daemons/' + g.coin.lower()
-				g.rpc_port = Daemon(get_network_id(g.coin,g.testnet)).rpc_port
+				g.rpc_port = Daemon(get_network_id(g.coin,g.testnet),test_suite=True).rpc_port
 			rpc_init(reinit=True)
 
 		if txsign(tx,wfs,None,None):

mmgen/main_regtest.py

@@ -79,5 +79,4 @@ elif cmd_args[0] not in MMGenRegtest.usr_cmds:
 	die(1,'{!r}: invalid command'.format(cmd_args[0]))
 elif cmd_args[0] not in ('cli','balances'):
 	check_num_args()
-
 MMGenRegtest(g.coin).cmd(cmd_args)

mmgen/opts.py

@@ -318,10 +318,10 @@ def init(opts_data,add_opts=[],opt_filter=None,parse_only=False):
 		g.proto = CoinProtocol(g.coin,g.testnet)
 		g.rpc_host = 'localhost'
 		g.data_dir = os.path.join(g.data_dir_root,'regtest',g.coin.lower(),('alice','bob')[g.bob])
-		from mmgen.regtest import MMGenRegtest as rt
-		g.rpc_port = rt.rpc_ports[g.coin.lower()]
-		g.rpc_user = rt.rpc_user
-		g.rpc_password = rt.rpc_password
+		from mmgen.regtest import MMGenRegtest
+		g.rpc_user = MMGenRegtest.rpc_user
+		g.rpc_password = MMGenRegtest.rpc_password
+		g.rpc_port = MMGenRegtest(g.coin).d.rpc_port
 
 	check_or_create_dir(g.data_dir) # g.data_dir is finalized, so now we can do this
 
@@ -537,7 +537,7 @@ def check_opts(usr_opts): # Returns false if any check fails
 		elif key in ('bob','alice'):
 			m = "Regtest (Bob and Alice) mode not set up yet.  Run '{}-regtest setup' to initialize."
 			from mmgen.regtest import MMGenRegtest
-			try: os.stat(os.path.join(MMGenRegtest(g.coin).data_dir,'regtest','debug.log'))
+			try: os.stat(os.path.join(MMGenRegtest(g.coin).d.datadir,'regtest','debug.log'))
 			except: die(1,m.format(g.proj_name.lower()))
 		elif key == 'locktime':
 			if not opt_is_int(val,desc): return False

mmgen/regtest.py

@@ -25,11 +25,6 @@ from subprocess import run,PIPE
 from mmgen.common import *
 from mmgen.daemon import Daemon
 
-# To enforce MMGen policy that all testing must ignore the user's ~/.bitcoin
-# dir, locate the daemon datadir under MMGen data_dir:
-def dfl_data_dir(coin):
-	return os.path.abspath(os.path.join(g.data_dir_root,'regtest',coin.lower()))
-
 def create_data_dir(data_dir):
 	try: os.stat(os.path.join(data_dir,'regtest'))
 	except: pass
@@ -73,7 +68,6 @@ class RegtestDaemon(MMGenObject): # mixin class
 
 class MMGenRegtest(MMGenObject):
 
-	rpc_ports = { 'btc':8552, 'bch':8553, 'ltc':8555 }
 	rpc_user     = 'bobandalice'
 	rpc_password = 'hodltothemoon'
 	users        = ('bob','alice','miner')
@@ -83,23 +77,22 @@ class MMGenRegtest(MMGenObject):
 		'setup','generate','send','stop',
 		'balances','mempool','cli' )
 
-	def __init__(self,coin,datadir=None):
+	def __init__(self,coin):
 		self.coin = coin.lower()
-		self.data_dir = datadir or dfl_data_dir(self.coin)
-		self.rpc_port = self.rpc_ports[self.coin]
+		self.test_suite = os.getenv('MMGEN_TEST_SUITE_REGTEST')
+		self.d = Daemon(self.coin+'_rt',test_suite=self.test_suite)
 
 		assert self.coin in self.coins,'{!r}: invalid coin for regtest'.format(user)
-		assert os.path.isabs(self.data_dir), '{!r}: invalid datadir (not an absolute path)'.format(datadir)
 
 	def setup(self):
 
-		try: os.makedirs(self.data_dir)
+		try: os.makedirs(self.d.datadir)
 		except: pass
 
 		if self.daemon_state() != 'stopped':
 			self.stop_daemon()
 
-		create_data_dir(self.data_dir)
+		create_data_dir(self.d.datadir)
 
 		gmsg('Starting {} regtest setup'.format(self.coin))
 
@@ -134,7 +127,7 @@ class MMGenRegtest(MMGenObject):
 
 		assert user is None or user in self.users,'{!r}: invalid user for regtest'.format(user)
 
-		d = Daemon(self.coin,self.data_dir,desc='regtest daemon',rpc_port=self.rpc_port)
+		d = Daemon(self.coin+'_rt',test_suite=self.test_suite)
 
 		type(d).generate = RegtestDaemon.generate
 
@@ -163,7 +156,7 @@ class MMGenRegtest(MMGenObject):
 				msg(err)
 
 	def current_user_unix(self,quiet=False):
-		cmd = ['pgrep','-af','{}.*--rpcport={}.*'.format(g.proto.daemon_name,self.rpc_port)]
+		cmd = ['pgrep','-af','{}.*--rpcport={}.*'.format(g.proto.daemon_name,self.d.rpc_port)]
 		cmdout = run(cmd,stdout=PIPE).stdout.decode()
 		if cmdout:
 			for k in self.users:
@@ -176,7 +169,7 @@ class MMGenRegtest(MMGenObject):
 		if self.daemon_state() == 'stopped':
 			return None
 
-		debug_logfile = os.path.join(self.data_dir,'regtest','debug.log')
+		debug_logfile = os.path.join(self.d.datadir,'regtest','debug.log')
 		fd = os.open(debug_logfile,os.O_RDONLY|os.O_BINARY)
 		file_size = os.fstat(fd).st_size
 
@@ -283,10 +276,10 @@ class MMGenRegtest(MMGenObject):
 
 		gmsg('Creating fork from coin {} to coin {}'.format(coin,g.coin))
 
-		source_rt = MMGenRegtest(coin,dfl_data_dir(coin))
+		source_rt = MMGenRegtest(coin)
 
-		try: os.stat(source_rt.data_dir)
-		except: die(1,"Source directory '{}' does not exist!".format(source_rt.data_dir))
+		try: os.stat(source_rt.d.datadir)
+		except: die(1,"Source directory '{}' does not exist!".format(source_rt.d.datadir))
 
 		# stop the source daemon
 		if source_rt.daemon_state() != 'stopped':
@@ -296,14 +289,12 @@ class MMGenRegtest(MMGenObject):
 		if self.daemon_state() != 'stopped':
 			self.stop_daemon()
 
-		data_dir = dfl_data_dir(g.coin)
-
-		try: os.makedirs(data_dir)
+		try: os.makedirs(self.d.datadir)
 		except: pass
 
-		create_data_dir(data_dir)
-		os.rmdir(data_dir)
-		shutil.copytree(source_data_dir,data_dir,symlinks=True)
+		create_data_dir(self.d.datadir)
+		os.rmdir(self.d.datadir)
+		shutil.copytree(source_data_dir,self.d.datadir,symlinks=True)
 		self.start_daemon('miner',reindex=True)
 		self.stop_daemon()
 

test/common.py

@@ -175,10 +175,8 @@ def test_daemons_ops(*network_ids,op):
 	if opt.no_daemon_autostart:
 		return
 	from mmgen.daemon import Daemon
-	repo_root = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]),os.pardir)))
 	silent = not opt.verbose and not (hasattr(opt,'exact_output') and opt.exact_output)
 	for network_id in network_ids:
 		if network_id not in Daemon.network_ids: # silently ignore invalid IDs
 			continue
-		datadir = '{}/test/daemons/{}'.format(repo_root,network_id.replace('_tn',''))
-		Daemon(network_id,datadir).cmd(op,silent=silent)
+		Daemon(network_id,test_suite=True).cmd(op,silent=silent)

test/start-coin-daemons.py

@@ -64,13 +64,12 @@ if 'eth' in ids and 'etc' in ids:
 
 for network_id in ids:
 	network_id = network_id.lower()
-	coin = network_id.replace('_tn','')
 	if opt.regtest_user:
-		datadir = '{}/test/data_dir/regtest/{}'.format(repo_root,coin)
-		d = MMGenRegtest(network_id,datadir).test_daemon(opt.regtest_user)
+		d = MMGenRegtest(network_id).test_daemon(opt.regtest_user)
 	else:
-		datadir = '{}/test/daemons/{}'.format(repo_root,coin)
-		d = Daemon(network_id,datadir)
+		if network_id.endswith('_rt'):
+			continue
+		d = Daemon(network_id,test_suite=True)
 	d.debug = opt.debug
 	d.wait = not opt.no_wait
 	if opt.get_state:

test/test-release.sh

@@ -327,7 +327,7 @@ t_xmr="
 "
 f_xmr='Monero tests completed'
 
-mmgen_tool_xmr="$mmgen_tool --rpc-port=19082 -q --accept-defaults --outdir $TMPDIR"
+mmgen_tool_xmr="$mmgen_tool --rpc-port=19319 -q --accept-defaults --outdir $TMPDIR"
 
 [ "$MSYS2" ] || { # password file descriptor issues, cannot use popen_spawn()
 	t_xmr+="

test/test.py

@@ -156,7 +156,7 @@ network_id = get_network_id(get_coin(),bool(_uopts.get('testnet')))
 
 sys.argv.insert(1,'--data-dir=' + data_dir)
 sys.argv.insert(1,'--daemon-data-dir=test/daemons/' + get_coin())
-sys.argv.insert(1,'--rpc-port={}'.format(Daemon(network_id).rpc_port))
+sys.argv.insert(1,'--rpc-port={}'.format(Daemon(network_id,test_suite=True).rpc_port))
 
 # step 2: opts.init will create new data_dir in ./test (if not 'resume' or 'skip_deps'):
 usr_args = opts.init(opts_data)

test/test_py_d/ts_ethdev.py

@@ -300,7 +300,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 
 	def __init__(self,trunner,cfgs,spawn):
 		from mmgen.daemon import Daemon
-		self.rpc_port = Daemon(g.coin).rpc_port
+		self.rpc_port = Daemon(g.coin,test_suite=True).rpc_port
 		os.environ['MMGEN_BOGUS_WALLET_DATA'] = ''
 		return TestSuiteBase.__init__(self,trunner,cfgs,spawn)
 

test/test_py_d/ts_ref_altcoin.py

@@ -97,7 +97,7 @@ class TestSuiteRefAltcoin(TestSuiteRef,TestSuiteBase):
 					start_test_daemons(network_id)
 					extra_opts += [
 						'--daemon-data-dir=test/daemons/bch',
-						'--rpc-port={}'.format(Daemon(network_id).rpc_port) ]
+						'--rpc-port={}'.format(Daemon(network_id,test_suite=True).rpc_port) ]
 				g.testnet = tn
 				init_coin(coin)
 				fn = TestSuiteRef.sources['ref_tx_file'][token or coin][bool(tn)]

test/test_py_d/ts_regtest.py

@@ -231,6 +231,9 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 	usr_subsids = { 'bob': {}, 'alice': {} }
 
 	def __init__(self,trunner,cfgs,spawn):
+		os.environ['MMGEN_TEST_SUITE_REGTEST'] = '1'
+		from mmgen.regtest import MMGenRegtest
+		rt = MMGenRegtest(g.coin)
 		coin = g.coin.lower()
 		for k in rt_data:
 			globals()[k] = rt_data[k][coin] if coin in rt_data[k] else None

test/unit_tests_d/ut_tx_deserialize.py

@@ -112,11 +112,11 @@ class unit_test(object):
 			for n,(coin,tn,fn) in enumerate(fns):
 				init_coin(coin,tn)
 				g.proto.daemon_data_dir = 'test/daemons/' + g.coin.lower()
-				g.rpc_port = Daemon(coin + ('','_tn')[tn]).rpc_port
+				g.rpc_port = Daemon(coin + ('','_tn')[tn],test_suite=True).rpc_port
 				rpc_init(reinit=True)
 				test_tx(MMGenTX(fn).hex,fn,n+1)
 			init_coin('btc',False)
-			g.rpc_port = Daemon('btc').rpc_port
+			g.rpc_port = Daemon('btc',test_suite=True).rpc_port
 			rpc_init(reinit=True)
 			Msg('OK')