diff --git a/mmgen/base_proto/bitcoin/tw/ctl.py b/mmgen/base_proto/bitcoin/tw/ctl.py index e6db771b..722c1516 100755 --- a/mmgen/base_proto/bitcoin/tw/ctl.py +++ b/mmgen/base_proto/bitcoin/tw/ctl.py @@ -14,7 +14,7 @@ base_proto.bitcoin.twctl: Bitcoin base protocol tracking wallet control class from ....globalvars import g from ....tw.ctl import TrackingWallet -from ....util import msg,msg_r,rmsg,vmsg,die,suf,fmt_list,write_mode +from ....util import msg,msg_r,rmsg,vmsg,die,suf,fmt,fmt_list,write_mode,keypress_confirm class BitcoinTrackingWallet(TrackingWallet): @@ -153,3 +153,21 @@ class BitcoinTrackingWallet(TrackingWallet): msg('Address has no balance' if len(coin_addrs) == 1 else 'Addresses have no balances' ) return True + + async def twimport_check_and_create_wallet(self,info_msg): + + if await self.rpc.check_or_create_daemon_wallet(wallet_create=False): + die(3, + f'Existing {self.rpc.daemon.desc} wallet detected!\n' + + 'It must be moved, or backed up and securely deleted, before running this command' ) + + msg('\n'+fmt(info_msg.strip(),indent=' ')) + + if not keypress_confirm('Continue?'): + msg('Exiting at user request') + return False + + if not await self.rpc.check_or_create_daemon_wallet(wallet_create=True): + die(3,'Wallet could not be created') + + return True diff --git a/mmgen/base_proto/ethereum/tw/ctl.py b/mmgen/base_proto/ethereum/tw/ctl.py index a7ccbeba..57697a94 100755 --- a/mmgen/base_proto/ethereum/tw/ctl.py +++ b/mmgen/base_proto/ethereum/tw/ctl.py @@ -79,9 +79,7 @@ class EthereumTrackingWallet(TrackingWallet): @write_mode async def batch_import_address(self,args_list): - for arg_list in args_list: - await self.import_address(*arg_list) - return args_list + return [await self.import_address(*a) for a in args_list] @write_mode async def import_address(self,addr,label): @@ -221,3 +219,6 @@ class EthereumTokenTrackingWallet(EthereumTrackingWallet): 'decimals': t.decimals } } + + async def twimport_check_and_create_wallet(self,info_msg): + raise NotImplementedError('method not implemented for Ethereum') diff --git a/mmgen/main_tool.py b/mmgen/main_tool.py index 5c21c545..71b51445 100755 --- a/mmgen/main_tool.py +++ b/mmgen/main_tool.py @@ -164,6 +164,8 @@ mods = { 'rescan_address', 'rescan_blockchain', 'resolve_address', + 'twexport', + 'twimport', 'twview', 'txhist', ), diff --git a/mmgen/tool/rpc.py b/mmgen/tool/rpc.py index 5ecd1e81..736df585 100755 --- a/mmgen/tool/rpc.py +++ b/mmgen/tool/rpc.py @@ -191,3 +191,15 @@ class tool_cmd(tool_cmd_base): from ..tw.ctl import TrackingWallet ret = await (await TrackingWallet(self.proto,mode='w')).rescan_blockchain(start_block,stop_block) return True + + async def twexport(self,include_amts=True): + "export the tracking wallet to JSON format" + from ..tw.ctl import TrackingWallet + tw = await TrackingWallet( self.proto ) + return await tw.twexport( include_amts=include_amts ) + + async def twimport(self,filename:str,ignore_checksum=False,batch=False): + "restore the tracking wallet from a JSON dump created by ‘twexport’" + from ..tw.ctl import TrackingWallet + tw = await TrackingWallet( self.proto, mode='i', rpc_ignore_wallet=True ) + return await tw.twimport( filename, ignore_checksum=ignore_checksum, batch=batch ) diff --git a/mmgen/tw/ctl.py b/mmgen/tw/ctl.py index 53c35378..75fd05a7 100755 --- a/mmgen/tw/ctl.py +++ b/mmgen/tw/ctl.py @@ -28,18 +28,24 @@ from ..util import ( msg, msg_r, qmsg, + ymsg, dmsg, suf, write_mode, base_proto_subclass, - die ) + die, + make_timestamp, + make_chksum_8 ) from ..base_obj import AsyncInit from ..objmethods import MMGenObject from ..obj import TwComment,get_obj from ..addr import CoinAddr,is_mmgen_id,is_coin_addr -from ..rpc import rpc_init +from ..rpc import rpc_init,json_encoder from .common import TwMMGenID,TwLabel +def json_dump(data): + return json.dumps( data, cls=json_encoder, separators=(',', ':'), sort_keys=True ) + class TrackingWallet(MMGenObject,metaclass=AsyncInit): caps = ('rescan','batch') @@ -47,6 +53,7 @@ class TrackingWallet(MMGenObject,metaclass=AsyncInit): use_tw_file = False aggressive_sync = False importing = False + dump_fn_pfx = 'mmgen-tracking-wallet-dump' def __new__(cls,proto,*args,**kwargs): return MMGenObject.__new__(base_proto_subclass(cls,proto,'tw','ctl')) @@ -340,3 +347,106 @@ class TrackingWallet(MMGenObject,metaclass=AsyncInit): for d in out: await do_import(*d) msg('Address import completed OK') + + async def twimport(self,filename,ignore_checksum=False,batch=False): + + info_msg = """ + This utility will create a new tracking wallet, import the addresses from + the JSON dump into it and update their balances. The operation typically + takes just a few minutes. + """ + + def check_chksum(d): + chksum = make_chksum_8( json_dump(d['data']).encode() ).lower() + if chksum != d['checksum']: + if ignore_checksum: + ymsg(f'Warning: ignoring incorrect checksum {chksum}') + else: + die(3,'File checksum incorrect! ({} != {})'.format(chksum,d['checksum'])) + + def check_mappings_chksum(d): + mappings_json = json_dump([e[:2] for e in d['entries']]) + mappings_chksum = make_chksum_8(mappings_json.encode()).lower() + if mappings_chksum != d['mappings_checksum']: + die(3,f'ERROR: Mappings checksum incorrect! ({mappings_chksum} != {d["mappings_checksum"]})') + + def check_network(d): + coin,network = d['network'].split('_') + if coin != self.proto.coin.lower(): + die(2,f'Coin in wallet dump is {coin.upper()}, but configured coin is {self.proto.coin}') + if network != self.proto.network: + die(2,f'Network in wallet dump is {network}, but configured network is {self.proto.network}') + + def get_and_verify_data(): + from ..fileutil import get_data_from_file + d = json.loads(get_data_from_file(filename,quiet=True)) + check_chksum(d) + check_mappings_chksum(d['data']) + check_network(d['data']) + return d + + if not await self.twimport_check_and_create_wallet(info_msg): + return True + + d = get_and_verify_data() + + _d1 = namedtuple('import_data',d['data']['entries_keys']) + entries = [_d1(*e) for e in d['data']['entries']] + + _d2 = namedtuple('import_data',['addr','twmmid','comment']) + await self.import_address_common( + [_d2(e.address, e.mmgen_id, e.comment) for e in entries], + batch = batch ) + + await self.rescan_addresses([e.address for e in entries]) + + return True + + async def twexport(self,include_amts=True): + + coin = self.proto.coin_id.lower() + network = self.proto.network + + from .addrs import TwAddrList + al = await TwAddrList( + proto = self.proto, + usr_addr_list = None, + minconf = 0, + showempty = True, + showbtcaddrs = True, + all_labels = False ) + + keys = ['mmgen_id','address'] + (['amount'] if include_amts else []) + ['comment'] + + entries = sorted( + ( + [(v['lbl'].mmid, v['addr'], v['amt'], v['lbl'].comment) for v in al.values()] if include_amts else + [(v['lbl'].mmid, v['addr'], v['lbl'].comment) for v in al.values()] + ), + key = lambda x: x[0].sort_key ) + + mappings_json = json_dump([e[:2] for e in entries]) + + data = { + 'id': 'mmgen_tracking_wallet', + 'version': 1, + 'network': f'{coin}_{network}', + 'blockheight': self.rpc.blockcount, + 'time': make_timestamp(), + 'mappings_checksum': make_chksum_8(mappings_json.encode()).lower(), + 'entries_keys': keys, + 'entries': entries, + 'num_entries': len(entries), + } + if include_amts: + data['value'] = al.total + + chksum = make_chksum_8( json_dump(data).encode() ).lower() + + from ..fileutil import write_data_to_file + write_data_to_file( + outfile = f'{self.dump_fn_pfx}-{coin}-{network}.json', + data = json_dump( { 'checksum': chksum, 'data': data } ), + desc = f'tracking wallet JSON data' ) + + return True diff --git a/test/test_py_d/ts_ethdev.py b/test/test_py_d/ts_ethdev.py index b6d103b2..8188436b 100755 --- a/test/test_py_d/ts_ethdev.py +++ b/test/test_py_d/ts_ethdev.py @@ -1017,7 +1017,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared): return self.token_addrimport('token_addr2','21-23',expect='3/3') def token_addrimport_batch(self): - return self.token_addrimport('token_addr1','11-13',expect='OK: 3',extra_args=['--batch']) + return self.token_addrimport('token_addr1','11-13',expect='3 addresses',extra_args=['--batch']) def token_addrimport_sym(self): return self.addrimport( diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index 857eb6ca..fe71e443 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -209,6 +209,13 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared): ('bob_rescan_blockchain_gb', 'rescanning the blockchain (Genesis block)'), ('bob_rescan_blockchain_one','rescanning the blockchain (single block)'), ('bob_rescan_blockchain_ss', 'rescanning the blockchain (range of blocks)'), + ('bob_twexport', 'exporting a tracking wallet to JSON'), + ('carol_twimport', 'importing a tracking wallet JSON dump'), + ('carol_delete_wallet', 'unloading and deleting Carol’s tracking wallet'), + ('bob_twexport_noamt', 'exporting a tracking wallet to JSON (include_amts=0)'), + ('carol_twimport_nochksum', 'importing a tracking wallet JSON dump (ignore_checksum=1)'), + ('carol_delete_wallet', 'unloading and deleting Carol’s tracking wallet'), + ('carol_twimport_batch', 'importing a tracking wallet JSON dump (batch=1)'), ('bob_split2', "splitting Bob's funds"), ('bob_0conf0_getbalance', "Bob's balance (unconfirmed, minconf=0)"), ('bob_0conf1_getbalance', "Bob's balance (unconfirmed, minconf=1)"), @@ -398,7 +405,7 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared): def addrgen_bob(self): return self.addrgen('bob') def addrgen_alice(self): return self.addrgen('alice') - def addrimport(self,user,sid=None,addr_range='1-5',num_addrs=5,mmtypes=[]): + def addrimport(self,user,sid=None,addr_range='1-5',num_addrs=5,mmtypes=[],batch=True,quiet=True): id_strs = { 'legacy':'', 'compressed':'-C', 'segwit':'-S', 'bech32':'-B' } if not sid: sid = self._user_sid(user) from mmgen.addr import MMGenAddrType @@ -412,19 +419,26 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared): self._add_comments_to_addr_file(addrfile,addrfile,use_labels=True) t = self.spawn( 'mmgen-addrimport', - ['--quiet', '--'+user, '--batch', addrfile], - extra_desc=f'({desc})' ) + args = ( + (['--quiet'] if quiet else []) + + ['--'+user] + + (['--batch'] if batch else []) + + [addrfile] ), + extra_desc = f'({desc})' ) if g.debug: t.expect("Type uppercase 'YES' to confirm: ",'YES\n') t.expect('Importing') - t.expect(f'{num_addrs} addresses imported') - ok_msg() + if batch: + t.expect(f'{num_addrs} addresses imported') + else: + t.expect(f'import completed OK') + t.ok() t.skip_ok = True return t def addrimport_bob(self): return self.addrimport('bob') - def addrimport_alice(self): return self.addrimport('alice') + def addrimport_alice(self): return self.addrimport('alice',batch=False,quiet=False) def bob_import_miner_addr(self): if not self.deterministic: @@ -964,6 +978,44 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared): def bob_rescan_blockchain_ss(self): return self.bob_rescan_blockchain(['start_block=300','stop_block=302'],'300-302') + def bob_twexport(self,add_args=[]): + t = self.spawn('mmgen-tool',['--bob',f'--outdir={self.tmpdir}','twexport'] + add_args) + t.written_to_file('JSON data') + return t + + def bob_twexport_noamt(self): + return self.bob_twexport(add_args=['include_amts=0']) + + def carol_twimport(self,add_args=[]): + from mmgen.tw.ctl import TrackingWallet as twcls + fn = joinpath(self.tmpdir,f'{twcls.dump_fn_pfx}-{self.proto.coin.lower()}-regtest.json') + t = self.spawn('mmgen-tool',['--carol','twimport',fn] + add_args) + t.expect('(y/N): ','y') + if 'batch=true' in add_args: + t.expect('{} addresses imported'.format(15 if self.proto.coin == 'BCH' else 25)) + else: + t.expect('import completed OK') + t.expect('Found 3 unspent outputs') + return t + + def carol_twimport_nochksum(self): + return self.carol_twimport(add_args=['ignore_checksum=true']) + + def carol_twimport_batch(self): + return self.carol_twimport(add_args=['batch=true']) + + async def carol_delete_wallet(self): + imsg(f'Unloading Carol’s tracking wallet') + t = self.spawn('mmgen-regtest',['cli','unloadwallet','carol']) + t.ok() + from mmgen.rpc import rpc_init + rpc = await rpc_init(self.proto) + wdir = joinpath(rpc.daemon.network_datadir,'wallets','carol') + from shutil import rmtree + imsg(f'Deleting Carol’s tracking wallet') + rmtree(wdir) + return 'silent' + def bob_split2(self): addrs = self.read_from_tmpfile('non-mmgen.addrs').split() amts = (1.12345678,2.87654321,3.33443344,4.00990099,5.43214321)