mmgen-addrimport: reimplement --rescan using `scantxoutset`

Details:

    $ mmgen-addrimport --help

mmgen/base_proto/bitcoin/tw/ctl.py

@@ -28,12 +28,12 @@ class BitcoinTrackingWallet(TrackingWallet):
 		raise NotImplementedError('not implemented')
 
 	@write_mode
-	async def import_address(self,addr,label,rescan):
-		return await self.rpc.call('importaddress',addr,label,rescan,timeout=(False,3600)[rescan])
+	async def import_address(self,addr,label): # rescan is True by default, so set to False
+		return await self.rpc.call('importaddress',addr,label,False)
 
 	@write_mode
-	def batch_import_address(self,arg_list):
-		return self.rpc.batch_call('importaddress',arg_list)
+	def batch_import_address(self,arg_list):   # rescan is True by default, so set to False
+		return self.rpc.batch_call('importaddress',[a+(False,) for a in arg_list])
 
 	@write_mode
 	async def remove_address(self,addr):

mmgen/base_proto/ethereum/tw/ctl.py

@@ -84,7 +84,7 @@ class EthereumTrackingWallet(TrackingWallet):
 		return args_list
 
 	@write_mode
-	async def import_address(self,addr,label,foo):
+	async def import_address(self,addr,label):
 		r = self.data_root
 		if addr in r:
 			if not r[addr]['mmid'] and label.mmid:

mmgen/data/version

@@ -1 +1 @@
-13.2.dev2
+13.2.dev3

mmgen/globalvars.py

@@ -193,7 +193,6 @@ class GlobalContext(Lockable):
 		('label','keep_label'),
 		('tx_id','info'),
 		('tx_id','terse_info'),
-		('batch','rescan'), # TODO: still incompatible?
 	)
 	cfg_file_opts = (
 		'color',

mmgen/main_addrimport.py

@@ -20,7 +20,7 @@
 mmgen-addrimport: Import addresses into a MMGen coin daemon tracking wallet
 """
 
-import time
+from collections import namedtuple
 
 from .common import *
 from .addrlist import AddrList,KeyAddrList
@@ -38,15 +38,28 @@ opts_data = {
 -l, --addrlist     Address source is a flat list of non-MMGen coin addresses
 -k, --keyaddr-file Address source is a key-address file
 -q, --quiet        Suppress warnings
--r, --rescan       Rescan the blockchain.  Required if address to import is
-                   in the blockchain and has a balance.  Rescanning is slow.
+-r, --rescan       Update address balances by selectively rescanning the
+                   blockchain for unspent outputs that include the imported
+                   address(es).  Required if any of the imported addresses
+                   are already in the blockchain and have a balance.
 -t, --token-addr=A Import addresses for ERC20 token with address 'A'
 """,
-	'notes': """\n
-This command can also be used to update the comment fields of addresses
-already in the tracking wallet.
+	'notes': """
 
-The --batch and --rescan options cannot be used together.
+This command can also be used to update the comment fields or balances of
+addresses already in the tracking wallet.
+
+Rescanning now uses the ‘scantxoutset’ RPC call and a selective scan of
+blocks containing the relevant UTXOs for much faster performance than the
+previous implementation.  The rescan operation typically takes around two
+minutes total, independent of the number of addresses imported.
+
+Bear in mind that the UTXO scan will not find historical transactions: to add
+them to the tracking wallet, you must perform a full or partial rescan of the
+blockchain with the ‘mmgen-tool rescan_blockchain’ utility.  A full rescan of
+the blockchain may take up to several hours.
+
+It’s recommended to use ‘--rpc-backend=aio’ with ‘--rescan’.
 """
 	}
 }
@@ -116,32 +129,22 @@ def check_opts(tw):
 
 	return batch,rescan
 
-async def import_addr(tw,addr,label,rescan,msg_fmt,msg_args):
+async def import_address(args):
 	try:
-		task = asyncio.create_task(tw.import_address(addr,label,rescan))
-		if rescan:
-			start = time.time()
-			while True:
-				if task.done():
-					break
-				msg_r(('\r{} '+msg_fmt).format(
-					secs_to_hms(int(time.time()-start)),
-					*msg_args ))
-				await asyncio.sleep(0.5)
-			await task
-			msg('\nOK')
-		else:
-			await task
-			qmsg(msg_fmt.format(*msg_args) + ' - OK')
+		res = await args.tw.import_address( args.addr, args.lbl )
+		qmsg(args.msg)
+		return res
 	except Exception as e:
-		die(2,f'\nImport of address {addr!r} failed: {e.args[0]!r}')
+		die(2,f'\nImport of address {args.addr!r} failed: {e.args[0]!r}')
 
-def make_args_list(tw,al,batch,rescan):
+def gen_args_list(tw,al,batch):
 
-	fs = '{:%s} {:34} {:%s}' % (
+	fs = '{:%s} {:34} {:%s} - OK' % (
 		len(str(al.num_addrs)) * 2 + 2,
 		1 if opt.addrlist or opt.address else len(str(max(al.idxs()))) + 13 )
 
+	ad = namedtuple('args_list_data',['addr','lbl','tw','msg'])
+
 	for num,e in enumerate(al.data,1):
 		if e.idx:
 			label = f'{al.al_id}:{e.idx}' + (' ' + e.label if e.label else '')
@@ -151,10 +154,13 @@ def make_args_list(tw,al,batch,rescan):
 			add_msg = 'non-'+g.proj_name
 
 		if batch:
-			yield (e.addr,TwLabel(proto,label),False)
+			yield ad( e.addr, TwLabel(proto,label), None, None )
 		else:
-			msg_args = ( f'{num}/{al.num_addrs}:', e.addr, '('+add_msg+')' )
-			yield (tw,e.addr,TwLabel(proto,label),rescan,fs,msg_args)
+			yield ad(
+				addr = e.addr,
+				lbl  = TwLabel(proto,label),
+				tw   = tw,
+				msg  = fs.format(f'{num}/{al.num_addrs}:', e.addr, f'({add_msg})') )
 
 async def main():
 	from .tw.ctl import TrackingWallet
@@ -187,18 +193,17 @@ async def main():
 
 	batch,rescan = check_opts(tw)
 
-	args_list = make_args_list(tw,al,batch,rescan)
+	args_list = list(gen_args_list(tw,al,batch))
 
 	if batch:
-		ret = await tw.batch_import_address(list(args_list))
+		ret = await tw.batch_import_address([ (a.addr,a.lbl) for a in args_list ])
 		msg(f'OK: {len(ret)} addresses imported')
-	elif rescan:
-		for arg_list in args_list:
-			await import_addr(*arg_list)
 	else:
-		tasks = [import_addr(*arg_list) for arg_list in args_list]
-		await asyncio.gather(*tasks)
-		msg('OK')
+		await asyncio.gather(*(import_address(a) for a in args_list))
+		msg('Address import completed OK')
+
+	if rescan:
+		await tw.rescan_addresses({a.addr for a in args_list})
 
 	del tw
 

test/test_py_d/ts_regtest.py

@@ -202,6 +202,7 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 		('bob_bal4',                 "Bob's balance (after import)"),
 		('bob_import_list',          'importing flat address list'),
 		('bob_import_list_rescan',   'importing flat address list with --rescan'),
+		('bob_import_list_rescan_aio','importing flat address list with --rescan (aiohttp backend)'),
 		('bob_resolve_addr',         'resolving an address in the tracking wallet'),
 		('bob_rescan_addr',          'rescanning an address'),
 		('bob_rescan_blockchain_all','rescanning the blockchain (full rescan)'),
@@ -907,14 +908,19 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 			t.expect("Type uppercase 'YES' to confirm: ",'YES\n')
 		t.expect(f'Importing {nAddr} address')
 		if '--rescan' in args:
-			for i in range(nAddr):
-				t.expect('OK')
+			if not '--quiet' in args:
+				t.expect('Continue? (Y/n): ','y')
+			t.expect('Rescanning block')
 		return t
 
 	def bob_import_addr(self):
 		addr = self.read_from_tmpfile('non-mmgen.addrs').split()[0]
 		return self.user_import('bob',['--quiet','--address='+addr],nAddr=1)
 
+	def bob_import_list_rescan_aio(self):
+		addrfile = joinpath(self.tmpdir,'non-mmgen.addrs')
+		return self.user_import('bob',['--rescan','--rpc-backend=aio','--addrlist',addrfile],nAddr=5)
+
 	def bob_resolve_addr(self):
 		mmaddr = '{}:C:1'.format(self._user_sid('bob'))
 		t = self.spawn('mmgen-tool',['--bob','resolve_address',mmaddr])