From 5e43ae2106231c6a7379468bbeee8fffd1144b7a Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 6 Mar 2024 11:05:22 +0000 Subject: [PATCH] tw.ctl, regtest, cmdtest regtest: cleanups --- mmgen/proto/btc/regtest.py | 61 ++++++++++++++++++++++----------- mmgen/proto/btc/rpc.py | 29 +++++++++++----- mmgen/proto/btc/tw/ctl.py | 8 ++--- mmgen/proto/eth/tw/ctl.py | 2 +- mmgen/tw/ctl.py | 2 +- test/cmdtest.py | 8 ++--- test/cmdtest_py_d/ct_regtest.py | 40 ++++++++++----------- 7 files changed, 91 insertions(+), 59 deletions(-) diff --git a/mmgen/proto/btc/regtest.py b/mmgen/proto/btc/regtest.py index 39483670..8fd360a9 100755 --- a/mmgen/proto/btc/regtest.py +++ b/mmgen/proto/btc/regtest.py @@ -25,6 +25,7 @@ from ...util import msg,gmsg,die,capfirst,suf from ...protocol import init_proto from ...rpc import rpc_init,json_encoder from ...objmethods import MMGenObject +from ...daemon import CoinDaemon def create_data_dir(cfg,data_dir): try: @@ -63,9 +64,9 @@ class MMGenRegtest(MMGenObject): coins = ('btc','bch','ltc') usr_cmds = ('setup','generate','send','start','stop', 'state', 'balances','mempool','cli','wallet_cli') - hdseed = 'beadcafe' * 8 - miner_wif = 'cTyMdQ2BgfAsjopRVZrj7AoEGp97pKfrC2NkqLuwHr4KHfPNAKwp' - miner_addrs = { + bdb_hdseed = 'beadcafe' * 8 + bdb_miner_wif = 'cTyMdQ2BgfAsjopRVZrj7AoEGp97pKfrC2NkqLuwHr4KHfPNAKwp' + bdb_miner_addrs = { # cTyMdQ2BgfAsjopRVZrj7AoEGp97pKfrC2NkqLuwHr4KHfPNAKwp hdseed=1 'btc': 'bcrt1qaq8t3pakcftpk095tnqfv5cmmczysls024atnd', 'ltc': 'rltc1qaq8t3pakcftpk095tnqfv5cmmczysls05c8zyn', @@ -77,17 +78,36 @@ class MMGenRegtest(MMGenObject): self.coin = coin.lower() assert self.coin in self.coins, f'{coin!r}: invalid coin for regtest' - from ...daemon import CoinDaemon - self.proto = init_proto( cfg, self.coin, regtest=True, need_amt=True ) - self.d = CoinDaemon(cfg,self.coin+'_rt',test_suite=cfg.test_suite) - self.miner_addr = self.miner_addrs[self.coin] + self.proto = init_proto(cfg, self.coin, regtest=True, need_amt=True) + self.d = CoinDaemon( + cfg, + self.coin + '_rt', + test_suite = cfg.test_suite) + + # Caching creates problems (broken pipe) when recreating + loading wallets, + # so reinstantiate with every call: + @property + async def rpc(self): + return await rpc_init(self.cfg, self.proto, backend=None, daemon=self.d) + + @property + async def miner_addr(self): + if not hasattr(self,'_miner_addr'): + self._miner_addr = self.bdb_miner_addrs[self.coin] + return self._miner_addr + + @property + async def miner_wif(self): + if not hasattr(self,'_miner_wif'): + self._miner_wif = self.bdb_miner_wif + return self._miner_wif def create_hdseed_wif(self): from ...tool.api import tool_api t = tool_api(self.cfg) t.init_coin(self.proto.coin,self.proto.network) t.addrtype = 'compressed' if self.proto.coin == 'BCH' else 'bech32' - return t.hex2wif(self.hdseed) + return t.hex2wif(self.bdb_hdseed) async def generate(self,blocks=1,silent=False): @@ -101,7 +121,7 @@ class MMGenRegtest(MMGenObject): out = await self.rpc_call( 'generatetoaddress', blocks, - self.miner_addr, + await self.miner_addr, wallet = 'miner' ) if len(out) != blocks: @@ -110,6 +130,14 @@ class MMGenRegtest(MMGenObject): if not silent: gmsg(f'Mined {blocks} block{suf(blocks)}') + async def create_wallet(self,user): + return await (await self.rpc).icall( + 'createwallet', + wallet_name = user, + blank = True, + no_keys = user != 'miner', + load_on_startup = False) + async def setup(self): try: @@ -126,15 +154,9 @@ class MMGenRegtest(MMGenObject): self.d.start(silent=True) - rpc = await rpc_init(self.cfg,self.proto,backend=None,daemon=self.d) for user in ('miner','bob','alice'): gmsg(f'Creating {capfirst(user)}’s tracking wallet') - await rpc.icall( - 'createwallet', - wallet_name = user, - blank = True, - no_keys = user != 'miner', - load_on_startup = False ) + await self.create_wallet(user) # BCH and LTC daemons refuse to set HD seed with empty blockchain ("in IBD" error), # so generate a block: @@ -143,11 +165,11 @@ class MMGenRegtest(MMGenObject): # Unfortunately, we don’t get deterministic output with BCH and LTC even with fixed # hdseed, as their 'sendtoaddress' calls produce non-deterministic TXIDs due to random # input ordering and fee estimation. - await rpc.call( + await (await self.rpc).call( 'sethdseed', True, self.create_hdseed_wif(), - wallet = 'miner' ) + wallet = 'miner') # Broken litecoind can only mine 431 blocks in regtest mode, so generate just enough # blocks to fund the test suite @@ -175,8 +197,7 @@ class MMGenRegtest(MMGenObject): async def rpc_call(self,*args,wallet=None,start_daemon=True): if start_daemon and self.d.state == 'stopped': await self.start_daemon() - rpc = await rpc_init(self.cfg,self.proto,backend=None,daemon=self.d) - return await rpc.call(*args,wallet=wallet) + return await (await self.rpc).call(*args, wallet=wallet) async def start(self): if self.d.state == 'stopped': diff --git a/mmgen/proto/btc/rpc.py b/mmgen/proto/btc/rpc.py index 33b08ebe..caf346be 100755 --- a/mmgen/proto/btc/rpc.py +++ b/mmgen/proto/btc/rpc.py @@ -47,8 +47,16 @@ class CallSigs: class bitcoin_core: - @classmethod - def createwallet(cls,wallet_name,no_keys=True,blank=True,passphrase='',load_on_startup=True): + def __init__(self, cfg): + self.cfg = cfg + + def createwallet( + self, + wallet_name, + no_keys = True, + blank = True, + passphrase = '', + load_on_startup = True): """ Quirk: when --datadir is specified (even if standard), wallet is created directly in datadir, otherwise in datadir/wallets @@ -64,8 +72,7 @@ class CallSigs: load_on_startup # 7. load_on_startup ) - @classmethod - def gettransaction(cls,txid,include_watchonly,verbose): + def gettransaction(self, txid, include_watchonly, verbose): return ( 'gettransaction', txid, # 1. transaction id @@ -76,8 +83,13 @@ class CallSigs: class litecoin_core(bitcoin_core): - @classmethod - def createwallet(cls,wallet_name,no_keys=True,blank=True,passphrase='',load_on_startup=True): + def createwallet( + self, + wallet_name, + no_keys = True, + blank = True, + passphrase = '', + load_on_startup = True): return ( 'createwallet', wallet_name, # 1. wallet_name @@ -85,8 +97,7 @@ class CallSigs: blank, # 3. blank (no keys or seed) ) - @classmethod - def gettransaction(cls,txid,include_watchonly,verbose): + def gettransaction(self, txid, include_watchonly, verbose): return ( 'gettransaction', txid, # 1. transaction id @@ -113,7 +124,7 @@ class BitcoinRPCClient(RPCClient,metaclass=AsyncInit): self.proto = proto self.daemon = daemon - self.call_sigs = getattr(CallSigs,daemon.id,None) + self.call_sigs = getattr(CallSigs,daemon.id)(cfg) super().__init__( cfg = cfg, diff --git a/mmgen/proto/btc/tw/ctl.py b/mmgen/proto/btc/tw/ctl.py index 72ce1538..eeef202c 100755 --- a/mmgen/proto/btc/tw/ctl.py +++ b/mmgen/proto/btc/tw/ctl.py @@ -27,12 +27,12 @@ class BitcoinTwCtl(TwCtl): raise NotImplementedError('not implemented') @write_mode - async def import_address(self,addr,label): # rescan is True by default, so set to False - return await self.rpc.call('importaddress',addr,label,False) + async def import_address(self, addr, label, rescan=False): # rescan is True by default, so set to False + return await self.rpc.call('importaddress', addr, label, rescan) @write_mode - def batch_import_address(self,arg_list): # rescan is True by default, so set to False - return self.rpc.batch_call('importaddress',[a+(False,) for a in arg_list]) + async def batch_import_address(self,arg_list): + return await self.rpc.batch_call('importaddress', arg_list) @write_mode async def remove_address(self,addr): diff --git a/mmgen/proto/eth/tw/ctl.py b/mmgen/proto/eth/tw/ctl.py index 2ae2ca65..63a45705 100755 --- a/mmgen/proto/eth/tw/ctl.py +++ b/mmgen/proto/eth/tw/ctl.py @@ -92,7 +92,7 @@ class EthereumTwCtl(TwCtl): pass @write_mode - async def import_address(self,addr,label): + async def import_address(self,addr,label,rescan=False): r = self.data_root if addr in r: if not r[addr]['mmid'] and label.mmid: diff --git a/mmgen/tw/ctl.py b/mmgen/tw/ctl.py index 3967762f..89067f49 100755 --- a/mmgen/tw/ctl.py +++ b/mmgen/tw/ctl.py @@ -316,7 +316,7 @@ class TwCtl(MMGenObject,metaclass=AsyncInit): if batch: msg_r(f'Batch importing {len(out)} address{suf(data,"es")}...') - ret = await self.batch_import_address((a,b) for a,b,c in out) + ret = await self.batch_import_address((a, b, False) for a, b, c in out) msg(f'done\n{len(ret)} addresses imported') else: if gather: # this seems to provide little performance benefit diff --git a/test/cmdtest.py b/test/cmdtest.py index bbd53a8f..4b13935f 100755 --- a/test/cmdtest.py +++ b/test/cmdtest.py @@ -73,7 +73,7 @@ def create_shm_dir(data_dir,trash_dir): return shm_dir -import sys,os,time +import sys, os, time, asyncio if sys.argv[-1] == 'clean': os.environ['MMGEN_TEST_SUITE'] = '1' @@ -95,7 +95,7 @@ else: from mmgen.cfg import Config,gc from mmgen.color import red,yellow,green,blue,cyan,gray,nocolor,init_color -from mmgen.util import msg,Msg,bmsg,die,suf,make_timestr,async_run +from mmgen.util import msg, Msg, bmsg, die, suf, make_timestr from test.include.common import ( set_globals, @@ -820,7 +820,7 @@ class CmdTestRunner: if isinstance(e,KeyError) and e.args[0] == cmdname: ret = getattr(self.tg,cmdname)() if type(ret).__name__ == 'coroutine': - async_run(ret) + asyncio.run(ret) else: raise do_between() @@ -946,7 +946,7 @@ class CmdTestRunner: ret = getattr(self.tg,cmd)(*arg_list) # run the test if type(ret).__name__ == 'coroutine': - ret = async_run(ret) + ret = asyncio.run(ret) self.process_retval(cmd,ret) if cfg.profile: diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index d9d697e0..a49ce5b6 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -23,6 +23,7 @@ test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite import sys,os,json,time,re from decimal import Decimal +from mmgen.proto.btc.regtest import MMGenRegtest from mmgen.color import yellow from mmgen.util import msg_r,die,gmsg,capfirst,fmt_list from mmgen.protocol import init_proto @@ -463,15 +464,16 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared): self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) coin = self.proto.coin.lower() + gldict = globals() for k in rt_data: - setattr( sys.modules[__name__], k, rt_data[k][coin] if coin in rt_data[k] else None ) + gldict[k] = rt_data[k][coin] if coin in rt_data[k] else None + + self.rt = MMGenRegtest(cfg, self.proto.coin) if self.proto.coin == 'BTC': self.test_rbf = True # tests are non-coin-dependent, so run just once for BTC if cfg.test_suite_deterministic: self.deterministic = True - self.miner_addr = 'bcrt1qaq8t3pakcftpk095tnqfv5cmmczysls024atnd' # regtest.create_hdseed() - self.miner_wif = 'cTyMdQ2BgfAsjopRVZrj7AoEGp97pKfrC2NkqLuwHr4KHfPNAKwp' self.spawn_env['MMGEN_BOGUS_SEND'] = '' self.write_to_tmpfile('wallet_password',rt_pw) @@ -508,7 +510,9 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared): rmtree(joinpath(self.tr.data_dir,'regtest')) except: pass - t = self.spawn('mmgen-regtest',['-n','setup']) + t = self.spawn( + 'mmgen-regtest', + ['--setup-no-stop-daemon', 'setup']) for s in ('Starting','Creating','Creating','Creating','Mined','Setup complete'): t.expect(s) return t @@ -626,20 +630,20 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared): def addrimport_alice(self): return self.addrimport('alice',batch=False,quiet=False) - def bob_import_miner_addr(self): + async def bob_import_miner_addr(self): if not self.deterministic: return 'skip' return self.spawn( 'mmgen-addrimport', - [ '--bob', '--rescan', '--quiet', f'--address={self.miner_addr}' ] ) + [ '--bob', '--rescan', '--quiet', f'--address={await self.rt.miner_addr}' ] ) - def fund_wallet_deterministic(self,addr,utxo_nums,skip_passphrase=False): + async def fund_wallet_deterministic(self, addr, utxo_nums, skip_passphrase=False): """ the deterministic funding method using specific inputs """ if not self.deterministic: return 'skip' - self.write_to_tmpfile('miner.key',f'{self.miner_wif}\n') + self.write_to_tmpfile('miner.key',f'{await self.rt.miner_wif}\n') keyfile = joinpath(self.tmpdir,'miner.key') return self.user_txdo( @@ -667,18 +671,16 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared): if not self.deterministic: return 'skip' self.spawn('',msg_only=True) - from mmgen.proto.btc.regtest import MMGenRegtest - rt = MMGenRegtest(cfg,self.proto.coin) - await rt.stop() + await self.rt.stop() from shutil import rmtree imsg('Deleting Bob’s old tracking wallet') - rmtree(os.path.join(rt.d.datadir,'regtest','wallets','bob'),ignore_errors=True) - rt.init_daemon() - rt.d.start(silent=True) + rmtree(os.path.join(self.rt.d.datadir, 'regtest', 'wallets', 'bob'), ignore_errors=True) + self.rt.init_daemon() + self.rt.d.start(silent=True) imsg('Creating Bob’s new tracking wallet') - await rt.rpc_call('createwallet','bob',True,True,None,False,False,False) - await rt.stop() - await rt.start() + await self.rt.create_wallet('bob') + await self.rt.stop() + await self.rt.start() return 'ok' def addrimport_bob2(self): @@ -1429,9 +1431,7 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared): imsg('Unloading Carol’s tracking wallet') t = self.spawn('mmgen-regtest',['cli','unloadwallet','carol']) t.ok() - from mmgen.rpc import rpc_init - rpc = await rpc_init(cfg,self.proto) - wdir = joinpath(rpc.daemon.network_datadir,'wallets','carol') + wdir = joinpath((await self.rt.rpc).daemon.network_datadir, 'wallets', 'carol') from shutil import rmtree imsg('Deleting Carol’s tracking wallet') rmtree(wdir)