new commands: mmgen-tool twexport, mmgen-tool twimport

- create a compact JSON dump of the MMGen tracking wallet with `twexport`

- recreate a working tracking wallet with updated balances from the dump in
  just a few minutes with `twimport`

- the import operation leverages the `scantxoutset` RPC call to selectively
  rescan only required blocks, reducing block rescan time to typically just
  a few seconds

- to recover historical transactions for viewing with `mmgen-tool txhist`,
  a full blockchain rescan may be performed on the recreated wallet using
  `mmgen-tool rescan_blockchain`

- supported coins: BTC, BCH, LTC

mmgen/base_proto/bitcoin/tw/ctl.py

@@ -14,7 +14,7 @@ base_proto.bitcoin.twctl: Bitcoin base protocol tracking wallet control class
 
 from ....globalvars import g
 from ....tw.ctl import TrackingWallet
-from ....util import msg,msg_r,rmsg,vmsg,die,suf,fmt_list,write_mode
+from ....util import msg,msg_r,rmsg,vmsg,die,suf,fmt,fmt_list,write_mode,keypress_confirm
 
 class BitcoinTrackingWallet(TrackingWallet):
 
@@ -153,3 +153,21 @@ class BitcoinTrackingWallet(TrackingWallet):
 			msg('Address has no balance' if len(coin_addrs) == 1 else
 				'Addresses have no balances' )
 			return True
+
+	async def twimport_check_and_create_wallet(self,info_msg):
+
+		if await self.rpc.check_or_create_daemon_wallet(wallet_create=False):
+			die(3,
+				f'Existing {self.rpc.daemon.desc} wallet detected!\n' +
+				'It must be moved, or backed up and securely deleted, before running this command' )
+
+		msg('\n'+fmt(info_msg.strip(),indent='  '))
+
+		if not keypress_confirm('Continue?'):
+			msg('Exiting at user request')
+			return False
+
+		if not await self.rpc.check_or_create_daemon_wallet(wallet_create=True):
+			die(3,'Wallet could not be created')
+
+		return True

mmgen/base_proto/ethereum/tw/ctl.py

@@ -79,9 +79,7 @@ class EthereumTrackingWallet(TrackingWallet):
 
 	@write_mode
 	async def batch_import_address(self,args_list):
-		for arg_list in args_list:
-			await self.import_address(*arg_list)
-		return args_list
+		return [await self.import_address(*a) for a in args_list]
 
 	@write_mode
 	async def import_address(self,addr,label):
@@ -221,3 +219,6 @@ class EthereumTokenTrackingWallet(EthereumTrackingWallet):
 				'decimals': t.decimals
 			}
 		}
+
+	async def twimport_check_and_create_wallet(self,info_msg):
+		raise NotImplementedError('method not implemented for Ethereum')

mmgen/main_tool.py

@@ -164,6 +164,8 @@ mods = {
 		'rescan_address',
 		'rescan_blockchain',
 		'resolve_address',
+		'twexport',
+		'twimport',
 		'twview',
 		'txhist',
 	),

mmgen/tool/rpc.py

@@ -191,3 +191,15 @@ class tool_cmd(tool_cmd_base):
 		from ..tw.ctl import TrackingWallet
 		ret = await (await TrackingWallet(self.proto,mode='w')).rescan_blockchain(start_block,stop_block)
 		return True
+
+	async def twexport(self,include_amts=True):
+		"export the tracking wallet to JSON format"
+		from ..tw.ctl import TrackingWallet
+		tw = await TrackingWallet( self.proto )
+		return await tw.twexport( include_amts=include_amts )
+
+	async def twimport(self,filename:str,ignore_checksum=False,batch=False):
+		"restore the tracking wallet from a JSON dump created by ‘twexport’"
+		from ..tw.ctl import TrackingWallet
+		tw = await TrackingWallet( self.proto, mode='i', rpc_ignore_wallet=True )
+		return await tw.twimport( filename, ignore_checksum=ignore_checksum, batch=batch )

mmgen/tw/ctl.py

@@ -28,18 +28,24 @@ from ..util import (
 	msg,
 	msg_r,
 	qmsg,
+	ymsg,
 	dmsg,
 	suf,
 	write_mode,
 	base_proto_subclass,
-	die )
+	die,
+	make_timestamp,
+	make_chksum_8 )
 from ..base_obj import AsyncInit
 from ..objmethods import MMGenObject
 from ..obj import TwComment,get_obj
 from ..addr import CoinAddr,is_mmgen_id,is_coin_addr
-from ..rpc import rpc_init
+from ..rpc import rpc_init,json_encoder
 from .common import TwMMGenID,TwLabel
 
+def json_dump(data):
+	return json.dumps( data, cls=json_encoder, separators=(',', ':'), sort_keys=True )
+
 class TrackingWallet(MMGenObject,metaclass=AsyncInit):
 
 	caps = ('rescan','batch')
@@ -47,6 +53,7 @@ class TrackingWallet(MMGenObject,metaclass=AsyncInit):
 	use_tw_file = False
 	aggressive_sync = False
 	importing = False
+	dump_fn_pfx = 'mmgen-tracking-wallet-dump'
 
 	def __new__(cls,proto,*args,**kwargs):
 		return MMGenObject.__new__(base_proto_subclass(cls,proto,'tw','ctl'))
@@ -340,3 +347,106 @@ class TrackingWallet(MMGenObject,metaclass=AsyncInit):
 				for d in out:
 					await do_import(*d)
 			msg('Address import completed OK')
+
+	async def twimport(self,filename,ignore_checksum=False,batch=False):
+
+		info_msg = """
+			This utility will create a new tracking wallet, import the addresses from
+			the JSON dump into it and update their balances.  The operation typically
+			takes just a few minutes.
+		"""
+
+		def check_chksum(d):
+			chksum = make_chksum_8( json_dump(d['data']).encode() ).lower()
+			if chksum != d['checksum']:
+				if ignore_checksum:
+					ymsg(f'Warning: ignoring incorrect checksum {chksum}')
+				else:
+					die(3,'File checksum incorrect! ({} != {})'.format(chksum,d['checksum']))
+
+		def check_mappings_chksum(d):
+			mappings_json = json_dump([e[:2] for e in d['entries']])
+			mappings_chksum = make_chksum_8(mappings_json.encode()).lower()
+			if mappings_chksum != d['mappings_checksum']:
+				die(3,f'ERROR: Mappings checksum incorrect! ({mappings_chksum} != {d["mappings_checksum"]})')
+
+		def check_network(d):
+			coin,network = d['network'].split('_')
+			if coin != self.proto.coin.lower():
+				die(2,f'Coin in wallet dump is {coin.upper()}, but configured coin is {self.proto.coin}')
+			if network != self.proto.network:
+				die(2,f'Network in wallet dump is {network}, but configured network is {self.proto.network}')
+
+		def get_and_verify_data():
+			from ..fileutil import get_data_from_file
+			d = json.loads(get_data_from_file(filename,quiet=True))
+			check_chksum(d)
+			check_mappings_chksum(d['data'])
+			check_network(d['data'])
+			return d
+
+		if not await self.twimport_check_and_create_wallet(info_msg):
+			return True
+
+		d = get_and_verify_data()
+
+		_d1 = namedtuple('import_data',d['data']['entries_keys'])
+		entries = [_d1(*e) for e in d['data']['entries']]
+
+		_d2 = namedtuple('import_data',['addr','twmmid','comment'])
+		await self.import_address_common(
+			[_d2(e.address, e.mmgen_id, e.comment) for e in entries],
+			batch = batch )
+
+		await self.rescan_addresses([e.address for e in entries])
+
+		return True
+
+	async def twexport(self,include_amts=True):
+
+		coin = self.proto.coin_id.lower()
+		network = self.proto.network
+
+		from .addrs import TwAddrList
+		al = await TwAddrList(
+			proto = self.proto,
+			usr_addr_list = None,
+			minconf = 0,
+			showempty = True,
+			showbtcaddrs = True,
+			all_labels = False )
+
+		keys = ['mmgen_id','address'] + (['amount'] if include_amts else []) + ['comment']
+
+		entries = sorted(
+			(
+				[(v['lbl'].mmid, v['addr'], v['amt'], v['lbl'].comment) for v in al.values()] if include_amts else
+				[(v['lbl'].mmid, v['addr'],           v['lbl'].comment) for v in al.values()]
+			),
+			key = lambda x: x[0].sort_key )
+
+		mappings_json = json_dump([e[:2] for e in entries])
+
+		data = {
+			'id': 'mmgen_tracking_wallet',
+			'version': 1,
+			'network': f'{coin}_{network}',
+			'blockheight': self.rpc.blockcount,
+			'time': make_timestamp(),
+			'mappings_checksum': make_chksum_8(mappings_json.encode()).lower(),
+			'entries_keys': keys,
+			'entries': entries,
+			'num_entries': len(entries),
+		}
+		if include_amts:
+			data['value'] = al.total
+
+		chksum = make_chksum_8( json_dump(data).encode() ).lower()
+
+		from ..fileutil import write_data_to_file
+		write_data_to_file(
+			outfile = f'{self.dump_fn_pfx}-{coin}-{network}.json',
+			data = json_dump( { 'checksum': chksum, 'data': data } ),
+			desc = f'tracking wallet JSON data' )
+
+		return True

test/test_py_d/ts_ethdev.py

@@ -1017,7 +1017,7 @@ class TestSuiteEthdev(TestSuiteBase,TestSuiteShared):
 		return self.token_addrimport('token_addr2','21-23',expect='3/3')
 
 	def token_addrimport_batch(self):
-		return self.token_addrimport('token_addr1','11-13',expect='OK: 3',extra_args=['--batch'])
+		return self.token_addrimport('token_addr1','11-13',expect='3 addresses',extra_args=['--batch'])
 
 	def token_addrimport_sym(self):
 		return self.addrimport(

test/test_py_d/ts_regtest.py

@@ -209,6 +209,13 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 		('bob_rescan_blockchain_gb', 'rescanning the blockchain (Genesis block)'),
 		('bob_rescan_blockchain_one','rescanning the blockchain (single block)'),
 		('bob_rescan_blockchain_ss', 'rescanning the blockchain (range of blocks)'),
+		('bob_twexport',             'exporting a tracking wallet to JSON'),
+		('carol_twimport',           'importing a tracking wallet JSON dump'),
+		('carol_delete_wallet',      'unloading and deleting Carol’s tracking wallet'),
+		('bob_twexport_noamt',       'exporting a tracking wallet to JSON (include_amts=0)'),
+		('carol_twimport_nochksum',  'importing a tracking wallet JSON dump (ignore_checksum=1)'),
+		('carol_delete_wallet',      'unloading and deleting Carol’s tracking wallet'),
+		('carol_twimport_batch',     'importing a tracking wallet JSON dump (batch=1)'),
 		('bob_split2',               "splitting Bob's funds"),
 		('bob_0conf0_getbalance',    "Bob's balance (unconfirmed, minconf=0)"),
 		('bob_0conf1_getbalance',    "Bob's balance (unconfirmed, minconf=1)"),
@@ -398,7 +405,7 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 	def addrgen_bob(self):   return self.addrgen('bob')
 	def addrgen_alice(self): return self.addrgen('alice')
 
-	def addrimport(self,user,sid=None,addr_range='1-5',num_addrs=5,mmtypes=[]):
+	def addrimport(self,user,sid=None,addr_range='1-5',num_addrs=5,mmtypes=[],batch=True,quiet=True):
 		id_strs = { 'legacy':'', 'compressed':'-C', 'segwit':'-S', 'bech32':'-B' }
 		if not sid: sid = self._user_sid(user)
 		from mmgen.addr import MMGenAddrType
@@ -412,19 +419,26 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 				self._add_comments_to_addr_file(addrfile,addrfile,use_labels=True)
 			t = self.spawn(
 				'mmgen-addrimport',
-				['--quiet', '--'+user, '--batch', addrfile],
-				extra_desc=f'({desc})' )
+				args = (
+					(['--quiet'] if quiet else []) +
+					['--'+user] +
+					(['--batch'] if batch else []) +
+					[addrfile] ),
+				extra_desc = f'({desc})' )
 			if g.debug:
 				t.expect("Type uppercase 'YES' to confirm: ",'YES\n')
 			t.expect('Importing')
-			t.expect(f'{num_addrs} addresses imported')
-			ok_msg()
+			if batch:
+				t.expect(f'{num_addrs} addresses imported')
+			else:
+				t.expect(f'import completed OK')
+			t.ok()
 
 		t.skip_ok = True
 		return t
 
 	def addrimport_bob(self):   return self.addrimport('bob')
-	def addrimport_alice(self): return self.addrimport('alice')
+	def addrimport_alice(self): return self.addrimport('alice',batch=False,quiet=False)
 
 	def bob_import_miner_addr(self):
 		if not self.deterministic:
@@ -964,6 +978,44 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 	def bob_rescan_blockchain_ss(self):
 		return self.bob_rescan_blockchain(['start_block=300','stop_block=302'],'300-302')
 
+	def bob_twexport(self,add_args=[]):
+		t = self.spawn('mmgen-tool',['--bob',f'--outdir={self.tmpdir}','twexport'] + add_args)
+		t.written_to_file('JSON data')
+		return t
+
+	def bob_twexport_noamt(self):
+		return self.bob_twexport(add_args=['include_amts=0'])
+
+	def carol_twimport(self,add_args=[]):
+		from mmgen.tw.ctl import TrackingWallet as twcls
+		fn = joinpath(self.tmpdir,f'{twcls.dump_fn_pfx}-{self.proto.coin.lower()}-regtest.json')
+		t = self.spawn('mmgen-tool',['--carol','twimport',fn] + add_args)
+		t.expect('(y/N): ','y')
+		if 'batch=true' in add_args:
+			t.expect('{} addresses imported'.format(15 if self.proto.coin == 'BCH' else 25))
+		else:
+			t.expect('import completed OK')
+		t.expect('Found 3 unspent outputs')
+		return t
+
+	def carol_twimport_nochksum(self):
+		return self.carol_twimport(add_args=['ignore_checksum=true'])
+
+	def carol_twimport_batch(self):
+		return self.carol_twimport(add_args=['batch=true'])
+
+	async def carol_delete_wallet(self):
+		imsg(f'Unloading Carol’s tracking wallet')
+		t = self.spawn('mmgen-regtest',['cli','unloadwallet','carol'])
+		t.ok()
+		from mmgen.rpc import rpc_init
+		rpc = await rpc_init(self.proto)
+		wdir = joinpath(rpc.daemon.network_datadir,'wallets','carol')
+		from shutil import rmtree
+		imsg(f'Deleting Carol’s tracking wallet')
+		rmtree(wdir)
+		return 'silent'
+
 	def bob_split2(self):
 		addrs = self.read_from_tmpfile('non-mmgen.addrs').split()
 		amts = (1.12345678,2.87654321,3.33443344,4.00990099,5.43214321)