From e7ac7fd206a71a01559ee5710319559af2cf9b3b Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 26 May 2022 16:07:22 +0000 Subject: [PATCH] mmgen-addrimport: reimplement --rescan using `scantxoutset` Details: $ mmgen-addrimport --help --- mmgen/base_proto/bitcoin/tw/ctl.py | 8 +-- mmgen/base_proto/ethereum/tw/ctl.py | 2 +- mmgen/data/version | 2 +- mmgen/globalvars.py | 1 - mmgen/main_addrimport.py | 79 +++++++++++++++-------------- test/test_py_d/ts_regtest.py | 10 +++- 6 files changed, 56 insertions(+), 46 deletions(-) diff --git a/mmgen/base_proto/bitcoin/tw/ctl.py b/mmgen/base_proto/bitcoin/tw/ctl.py index a48957bf..dff829f2 100755 --- a/mmgen/base_proto/bitcoin/tw/ctl.py +++ b/mmgen/base_proto/bitcoin/tw/ctl.py @@ -28,12 +28,12 @@ class BitcoinTrackingWallet(TrackingWallet): raise NotImplementedError('not implemented') @write_mode - async def import_address(self,addr,label,rescan): - return await self.rpc.call('importaddress',addr,label,rescan,timeout=(False,3600)[rescan]) + async def import_address(self,addr,label): # rescan is True by default, so set to False + return await self.rpc.call('importaddress',addr,label,False) @write_mode - def batch_import_address(self,arg_list): - return self.rpc.batch_call('importaddress',arg_list) + def batch_import_address(self,arg_list): # rescan is True by default, so set to False + return self.rpc.batch_call('importaddress',[a+(False,) for a in arg_list]) @write_mode async def remove_address(self,addr): diff --git a/mmgen/base_proto/ethereum/tw/ctl.py b/mmgen/base_proto/ethereum/tw/ctl.py index c97f6116..a7ccbeba 100755 --- a/mmgen/base_proto/ethereum/tw/ctl.py +++ b/mmgen/base_proto/ethereum/tw/ctl.py @@ -84,7 +84,7 @@ class EthereumTrackingWallet(TrackingWallet): return args_list @write_mode - async def import_address(self,addr,label,foo): + async def import_address(self,addr,label): r = self.data_root if addr in r: if not r[addr]['mmid'] and label.mmid: diff --git a/mmgen/data/version b/mmgen/data/version index 369af71f..bf588547 100644 --- a/mmgen/data/version +++ b/mmgen/data/version @@ -1 +1 @@ -13.2.dev2 +13.2.dev3 diff --git a/mmgen/globalvars.py b/mmgen/globalvars.py index ee059841..c2195a13 100755 --- a/mmgen/globalvars.py +++ b/mmgen/globalvars.py @@ -193,7 +193,6 @@ class GlobalContext(Lockable): ('label','keep_label'), ('tx_id','info'), ('tx_id','terse_info'), - ('batch','rescan'), # TODO: still incompatible? ) cfg_file_opts = ( 'color', diff --git a/mmgen/main_addrimport.py b/mmgen/main_addrimport.py index c4dbeadb..77f47ae6 100755 --- a/mmgen/main_addrimport.py +++ b/mmgen/main_addrimport.py @@ -20,7 +20,7 @@ mmgen-addrimport: Import addresses into a MMGen coin daemon tracking wallet """ -import time +from collections import namedtuple from .common import * from .addrlist import AddrList,KeyAddrList @@ -38,15 +38,28 @@ opts_data = { -l, --addrlist Address source is a flat list of non-MMGen coin addresses -k, --keyaddr-file Address source is a key-address file -q, --quiet Suppress warnings --r, --rescan Rescan the blockchain. Required if address to import is - in the blockchain and has a balance. Rescanning is slow. +-r, --rescan Update address balances by selectively rescanning the + blockchain for unspent outputs that include the imported + address(es). Required if any of the imported addresses + are already in the blockchain and have a balance. -t, --token-addr=A Import addresses for ERC20 token with address 'A' """, - 'notes': """\n -This command can also be used to update the comment fields of addresses -already in the tracking wallet. + 'notes': """ -The --batch and --rescan options cannot be used together. +This command can also be used to update the comment fields or balances of +addresses already in the tracking wallet. + +Rescanning now uses the ‘scantxoutset’ RPC call and a selective scan of +blocks containing the relevant UTXOs for much faster performance than the +previous implementation. The rescan operation typically takes around two +minutes total, independent of the number of addresses imported. + +Bear in mind that the UTXO scan will not find historical transactions: to add +them to the tracking wallet, you must perform a full or partial rescan of the +blockchain with the ‘mmgen-tool rescan_blockchain’ utility. A full rescan of +the blockchain may take up to several hours. + +It’s recommended to use ‘--rpc-backend=aio’ with ‘--rescan’. """ } } @@ -116,32 +129,22 @@ def check_opts(tw): return batch,rescan -async def import_addr(tw,addr,label,rescan,msg_fmt,msg_args): +async def import_address(args): try: - task = asyncio.create_task(tw.import_address(addr,label,rescan)) - if rescan: - start = time.time() - while True: - if task.done(): - break - msg_r(('\r{} '+msg_fmt).format( - secs_to_hms(int(time.time()-start)), - *msg_args )) - await asyncio.sleep(0.5) - await task - msg('\nOK') - else: - await task - qmsg(msg_fmt.format(*msg_args) + ' - OK') + res = await args.tw.import_address( args.addr, args.lbl ) + qmsg(args.msg) + return res except Exception as e: - die(2,f'\nImport of address {addr!r} failed: {e.args[0]!r}') + die(2,f'\nImport of address {args.addr!r} failed: {e.args[0]!r}') -def make_args_list(tw,al,batch,rescan): +def gen_args_list(tw,al,batch): - fs = '{:%s} {:34} {:%s}' % ( + fs = '{:%s} {:34} {:%s} - OK' % ( len(str(al.num_addrs)) * 2 + 2, 1 if opt.addrlist or opt.address else len(str(max(al.idxs()))) + 13 ) + ad = namedtuple('args_list_data',['addr','lbl','tw','msg']) + for num,e in enumerate(al.data,1): if e.idx: label = f'{al.al_id}:{e.idx}' + (' ' + e.label if e.label else '') @@ -151,10 +154,13 @@ def make_args_list(tw,al,batch,rescan): add_msg = 'non-'+g.proj_name if batch: - yield (e.addr,TwLabel(proto,label),False) + yield ad( e.addr, TwLabel(proto,label), None, None ) else: - msg_args = ( f'{num}/{al.num_addrs}:', e.addr, '('+add_msg+')' ) - yield (tw,e.addr,TwLabel(proto,label),rescan,fs,msg_args) + yield ad( + addr = e.addr, + lbl = TwLabel(proto,label), + tw = tw, + msg = fs.format(f'{num}/{al.num_addrs}:', e.addr, f'({add_msg})') ) async def main(): from .tw.ctl import TrackingWallet @@ -187,18 +193,17 @@ async def main(): batch,rescan = check_opts(tw) - args_list = make_args_list(tw,al,batch,rescan) + args_list = list(gen_args_list(tw,al,batch)) if batch: - ret = await tw.batch_import_address(list(args_list)) + ret = await tw.batch_import_address([ (a.addr,a.lbl) for a in args_list ]) msg(f'OK: {len(ret)} addresses imported') - elif rescan: - for arg_list in args_list: - await import_addr(*arg_list) else: - tasks = [import_addr(*arg_list) for arg_list in args_list] - await asyncio.gather(*tasks) - msg('OK') + await asyncio.gather(*(import_address(a) for a in args_list)) + msg('Address import completed OK') + + if rescan: + await tw.rescan_addresses({a.addr for a in args_list}) del tw diff --git a/test/test_py_d/ts_regtest.py b/test/test_py_d/ts_regtest.py index 2c810d87..ef857bb1 100755 --- a/test/test_py_d/ts_regtest.py +++ b/test/test_py_d/ts_regtest.py @@ -202,6 +202,7 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared): ('bob_bal4', "Bob's balance (after import)"), ('bob_import_list', 'importing flat address list'), ('bob_import_list_rescan', 'importing flat address list with --rescan'), + ('bob_import_list_rescan_aio','importing flat address list with --rescan (aiohttp backend)'), ('bob_resolve_addr', 'resolving an address in the tracking wallet'), ('bob_rescan_addr', 'rescanning an address'), ('bob_rescan_blockchain_all','rescanning the blockchain (full rescan)'), @@ -907,14 +908,19 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared): t.expect("Type uppercase 'YES' to confirm: ",'YES\n') t.expect(f'Importing {nAddr} address') if '--rescan' in args: - for i in range(nAddr): - t.expect('OK') + if not '--quiet' in args: + t.expect('Continue? (Y/n): ','y') + t.expect('Rescanning block') return t def bob_import_addr(self): addr = self.read_from_tmpfile('non-mmgen.addrs').split()[0] return self.user_import('bob',['--quiet','--address='+addr],nAddr=1) + def bob_import_list_rescan_aio(self): + addrfile = joinpath(self.tmpdir,'non-mmgen.addrs') + return self.user_import('bob',['--rescan','--rpc-backend=aio','--addrlist',addrfile],nAddr=5) + def bob_resolve_addr(self): mmaddr = '{}:C:1'.format(self._user_sid('bob')) t = self.spawn('mmgen-tool',['--bob','resolve_address',mmaddr])