From c587ab39989faa0f647800211b7b87cd922762cf Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 6 Mar 2024 11:05:23 +0000 Subject: [PATCH] support descriptor wallets for BTC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For BTC, MMGen Wallet now creates and uses a descriptor wallet as its tracking wallet. For LTC and BCH, the legacy Berkeley DB wallet is used by default as before. While the legacy BDB wallet continues to be fully supported, BTC users are advised to upgrade their tracking wallet to a descriptor wallet to avoid support issues with future versions of Bitcoin Core. Upgrade a legacy tracking wallet as follows: $ bitcoin-cli migratewallet mmgen-tracking-wallet Alternatively, you may dump your tracking wallet to JSON and restore it as a descriptor wallet using the ‘mmgen-tool’ commands ‘twexport’, ‘twimport’ and ‘rescan_blockchain’ (see the help screens for those commands for details). Testing (add the -e option to see script output): # descriptor wallet: $ test/cmdtest.py regtest # Berkeley DB wallet: $ test/cmdtest.py regtest_legacy --- mmgen/data/version | 2 +- mmgen/main_regtest.py | 3 ++- mmgen/proto/btc/daemon.py | 3 ++- mmgen/proto/btc/regtest.py | 32 ++++++++++++++++++++----------- mmgen/proto/btc/rpc.py | 11 ++++++++++- mmgen/proto/btc/tw/bal.py | 13 +++++++++---- mmgen/proto/btc/tw/ctl.py | 17 ++++++++++++++-- test/cmdtest_py_d/cfg.py | 1 + test/cmdtest_py_d/ct_automount.py | 6 +++--- test/cmdtest_py_d/ct_regtest.py | 11 +++++++++-- test/test-release.d/cfg.sh | 5 ++++- 11 files changed, 77 insertions(+), 27 deletions(-) diff --git a/mmgen/data/version b/mmgen/data/version index 59b4bb6c..2b75a6e7 100644 --- a/mmgen/data/version +++ b/mmgen/data/version @@ -1 +1 @@ -14.1.dev17 +14.1.dev18 diff --git a/mmgen/main_regtest.py b/mmgen/main_regtest.py index 5a61850d..61abe7bc 100755 --- a/mmgen/main_regtest.py +++ b/mmgen/main_regtest.py @@ -32,6 +32,7 @@ opts_data = { 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) +-b, --bdb-wallet Create and use a legacy Berkeley DB coin daemon wallet -e, --empty Don't fund Bob and Alice's wallets on setup -n, --setup-no-stop-daemon Don't stop daemon after setup is finished -q, --quiet Produce quieter output @@ -83,6 +84,6 @@ elif cmd_args[0] not in ('cli','wallet_cli','balances'): check_num_args() async def main(): - await MMGenRegtest(cfg,cfg.coin).cmd(cmd_args) + await MMGenRegtest(cfg, cfg.coin, bdb_wallet=cfg.bdb_wallet).cmd(cmd_args) async_run(main()) diff --git a/mmgen/proto/btc/daemon.py b/mmgen/proto/btc/daemon.py index 12f1ea3a..8ede8a14 100755 --- a/mmgen/proto/btc/daemon.py +++ b/mmgen/proto/btc/daemon.py @@ -31,6 +31,7 @@ class bitcoin_core_daemon(CoinDaemon): 'linux': [gc.home_dir,'.bitcoin'], 'win32': [os.getenv('APPDATA'),'Bitcoin'] } + avail_opts = ('no_daemonize', 'online', 'bdb_wallet') def init_datadir(self): if self.network == 'regtest' and not self.test_suite: @@ -78,7 +79,7 @@ class bitcoin_core_daemon(CoinDaemon): ['--daemon', self.platform == 'linux' and not self.opt.no_daemonize], ['--fallbackfee=0.0002', self.coin == 'BTC' and self.network == 'regtest'], ['--usecashaddr=0', self.coin == 'BCH'], - ['--deprecatedrpc=create_bdb', self.coin == 'BTC'], + ['--deprecatedrpc=create_bdb', self.coin == 'BTC' and self.opt.bdb_wallet], ['--mempoolreplacement=1', self.coin == 'LTC'], ['--txindex=1', self.coin == 'LTC' or self.network == 'regtest'], ['--addresstype=bech32', self.coin == 'LTC' and self.network == 'regtest'], diff --git a/mmgen/proto/btc/regtest.py b/mmgen/proto/btc/regtest.py index 8fd360a9..e47a0f9d 100755 --- a/mmgen/proto/btc/regtest.py +++ b/mmgen/proto/btc/regtest.py @@ -73,16 +73,19 @@ class MMGenRegtest(MMGenObject): 'bch': 'n2fxhNx27GhHAWQhyuZ5REcBNrJqCJsJ12', } - def __init__(self,cfg,coin): + def __init__(self, cfg, coin, bdb_wallet=False): self.cfg = cfg self.coin = coin.lower() + self.bdb_wallet = bdb_wallet + assert self.coin in self.coins, f'{coin!r}: invalid coin for regtest' self.proto = init_proto(cfg, self.coin, regtest=True, need_amt=True) self.d = CoinDaemon( cfg, self.coin + '_rt', - test_suite = cfg.test_suite) + test_suite = cfg.test_suite, + opts = ['bdb_wallet'] if bdb_wallet else None) # Caching creates problems (broken pipe) when recreating + loading wallets, # so reinstantiate with every call: @@ -93,13 +96,17 @@ class MMGenRegtest(MMGenObject): @property async def miner_addr(self): if not hasattr(self,'_miner_addr'): - self._miner_addr = self.bdb_miner_addrs[self.coin] + self._miner_addr = ( + self.bdb_miner_addrs[self.coin] if self.bdb_wallet else + await self.rpc_call('getnewaddress', wallet='miner')) return self._miner_addr @property async def miner_wif(self): if not hasattr(self,'_miner_wif'): - self._miner_wif = self.bdb_miner_wif + self._miner_wif = ( + self.bdb_miner_wif if self.bdb_wallet else + await self.rpc_call('dumpprivkey', (await self.miner_addr), wallet='miner')) return self._miner_wif def create_hdseed_wif(self): @@ -118,6 +125,7 @@ class MMGenRegtest(MMGenObject): self.d.wait_for_state('ready') + # very slow with descriptor wallet and large block count - 'generatetodescriptor' no better out = await self.rpc_call( 'generatetoaddress', blocks, @@ -134,8 +142,9 @@ class MMGenRegtest(MMGenObject): return await (await self.rpc).icall( 'createwallet', wallet_name = user, - blank = True, + blank = user != 'miner' or self.bdb_wallet, no_keys = user != 'miner', + descriptors = not self.bdb_wallet, load_on_startup = False) async def setup(self): @@ -165,11 +174,12 @@ class MMGenRegtest(MMGenObject): # Unfortunately, we don’t get deterministic output with BCH and LTC even with fixed # hdseed, as their 'sendtoaddress' calls produce non-deterministic TXIDs due to random # input ordering and fee estimation. - await (await self.rpc).call( - 'sethdseed', - True, - self.create_hdseed_wif(), - wallet = 'miner') + if self.bdb_wallet: + await (await self.rpc).call( + 'sethdseed', + True, + self.create_hdseed_wif(), + wallet = 'miner') # Broken litecoind can only mine 431 blocks in regtest mode, so generate just enough # blocks to fund the test suite @@ -255,7 +265,7 @@ class MMGenRegtest(MMGenObject): gmsg(f'Creating fork from coin {coin} to coin {proto.coin}') - source_rt = MMGenRegtest( self.cfg, coin ) + source_rt = MMGenRegtest(self.cfg, coin, bdb_wallet=self.bdb_wallet) try: os.stat(source_rt.d.datadir) diff --git a/mmgen/proto/btc/rpc.py b/mmgen/proto/btc/rpc.py index caf346be..93c56cab 100755 --- a/mmgen/proto/btc/rpc.py +++ b/mmgen/proto/btc/rpc.py @@ -56,6 +56,7 @@ class CallSigs: no_keys = True, blank = True, passphrase = '', + descriptors = True, load_on_startup = True): """ Quirk: when --datadir is specified (even if standard), wallet is created directly in @@ -68,7 +69,7 @@ class CallSigs: blank, # 3. blank (no keys or seed) passphrase, # 4. passphrase (empty string for non-encrypted) False, # 5. avoid_reuse (track address reuse) - False, # 6. descriptors (native descriptor wallet) + descriptors, # 6. descriptors (native descriptor wallet) load_on_startup # 7. load_on_startup ) @@ -89,6 +90,7 @@ class CallSigs: no_keys = True, blank = True, passphrase = '', + descriptors = True, load_on_startup = True): return ( 'createwallet', @@ -201,6 +203,12 @@ class BitcoinRPCClient(RPCClient,metaclass=AsyncInit): else: self.wallet_path = f'/wallet/{self.twname}' + @property + async def walletinfo(self): + if not hasattr(self,'_walletinfo'): + self._walletinfo = await self.call('getwalletinfo') + return self._walletinfo + def set_auth(self): """ MMGen's credentials override coin daemon's @@ -343,6 +351,7 @@ class BitcoinRPCClient(RPCClient,metaclass=AsyncInit): 'getrawtransaction', 'gettransaction', 'importaddress', # address (address or script) label rescan p2sh (Add P2SH version of the script) + 'importdescriptors', # like above, but for descriptor wallets 'listaccounts', 'listlabels', 'listunspent', diff --git a/mmgen/proto/btc/tw/bal.py b/mmgen/proto/btc/tw/bal.py index 73f08a47..cb3f917d 100755 --- a/mmgen/proto/btc/tw/bal.py +++ b/mmgen/proto/btc/tw/bal.py @@ -20,6 +20,7 @@ class BitcoinTwGetBalance(TwGetBalance): async def __init__(self, cfg, proto, minconf, quiet): self.rpc = await rpc_init(cfg, proto) + self.walletinfo = await self.rpc.walletinfo await super().__init__(cfg, proto, minconf, quiet) start_labels = ('TOTAL','Non-MMGen','Non-wallet') @@ -53,21 +54,25 @@ class BitcoinTwGetBalance(TwGetBalance): self.data['TOTAL'][col_key] += amt self.data[label][col_key] += amt - if d['spendable']: + if d['spendable']: # TODO: use 'solvable' for descriptor wallets? self.data[label]['spendable'] += amt def format(self, color): def gen_spendable_warning(): - for k,v in self.data.items(): - if v['spendable']: - yield red(f'Warning: this wallet contains PRIVATE KEYS for {k} outputs!') + if check_spendable: + for k,v in self.data.items(): + if v['spendable']: + yield red(f'Warning: this wallet contains PRIVATE KEYS for {k} outputs!') if color: from ....color import red else: from ....color import nocolor as red + desc_wallet = self.walletinfo.get('descriptors') + check_spendable = not desc_wallet or (desc_wallet and self.walletinfo['private_keys_enabled']) + warning = '\n'.join(gen_spendable_warning()) return super().format(color) + ('\n' if warning else '') + warning diff --git a/mmgen/proto/btc/tw/ctl.py b/mmgen/proto/btc/tw/ctl.py index eeef202c..fa47cad2 100755 --- a/mmgen/proto/btc/tw/ctl.py +++ b/mmgen/proto/btc/tw/ctl.py @@ -28,11 +28,24 @@ class BitcoinTwCtl(TwCtl): @write_mode async def import_address(self, addr, label, rescan=False): # rescan is True by default, so set to False - return await self.rpc.call('importaddress', addr, label, rescan) + if (await self.rpc.walletinfo).get('descriptors'): + return await self.batch_import_address([(addr, label, rescan)]) + else: + return await self.rpc.call('importaddress', addr, label, rescan) @write_mode async def batch_import_address(self,arg_list): - return await self.rpc.batch_call('importaddress', arg_list) + if (await self.rpc.walletinfo).get('descriptors'): + from ....contrib.descriptors import descsum_create + return await self.rpc.call( + 'importdescriptors', + [{ + 'desc': descsum_create(f'addr({addr})'), + 'label': label, + 'timestamp': 0 if rescan else 'now', + } for addr, label, rescan in arg_list]) + else: + return await self.rpc.batch_call('importaddress', arg_list) @write_mode async def remove_address(self,addr): diff --git a/test/cmdtest_py_d/cfg.py b/test/cmdtest_py_d/cfg.py index 0fb604d4..51afe850 100755 --- a/test/cmdtest_py_d/cfg.py +++ b/test/cmdtest_py_d/cfg.py @@ -45,6 +45,7 @@ cmd_groups_dfl = { cmd_groups_extra = { 'dev': ('CmdTestDev',{'modname':'misc'}), + 'regtest_legacy': ('CmdTestRegtestBDBWallet', {'modname':'regtest'}), 'autosign_btc': ('CmdTestAutosignBTC',{'modname':'autosign'}), 'autosign_live': ('CmdTestAutosignLive',{'modname':'autosign'}), 'autosign_live_simulate': ('CmdTestAutosignLiveSimulate',{'modname':'autosign'}), diff --git a/test/cmdtest_py_d/ct_automount.py b/test/cmdtest_py_d/ct_automount.py index eaeefb04..e1ad5f84 100755 --- a/test/cmdtest_py_d/ct_automount.py +++ b/test/cmdtest_py_d/ct_automount.py @@ -15,11 +15,11 @@ import os, time from pathlib import Path from .ct_autosign import CmdTestAutosignThreaded -from .ct_regtest import CmdTestRegtest, rt_pw +from .ct_regtest import CmdTestRegtestBDBWallet, rt_pw from .common import get_file_with_ext from ..include.common import cfg -class CmdTestAutosignAutomount(CmdTestAutosignThreaded, CmdTestRegtest): +class CmdTestAutosignAutomount(CmdTestAutosignThreaded, CmdTestRegtestBDBWallet): 'automounted transacting operations via regtest mode' networks = ('btc', 'bch', 'ltc') @@ -75,7 +75,7 @@ class CmdTestAutosignAutomount(CmdTestAutosignThreaded, CmdTestRegtest): self.coins = [cfg.coin.lower()] CmdTestAutosignThreaded.__init__(self, trunner, cfgs, spawn) - CmdTestRegtest.__init__(self, trunner, cfgs, spawn) + CmdTestRegtestBDBWallet.__init__(self, trunner, cfgs, spawn) if trunner == None: return diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index a49ce5b6..4eea47b1 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -175,6 +175,7 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared): deterministic = False test_rbf = False proto = None # pylint + bdb_wallet = False cmd_group_in = ( ('setup', 'regtest (Bob and Alice) mode setup'), @@ -468,7 +469,9 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared): for k in rt_data: gldict[k] = rt_data[k][coin] if coin in rt_data[k] else None - self.rt = MMGenRegtest(cfg, self.proto.coin) + self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' + + self.rt = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) if self.proto.coin == 'BTC': self.test_rbf = True # tests are non-coin-dependent, so run just once for BTC @@ -512,7 +515,8 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared): pass t = self.spawn( 'mmgen-regtest', - ['--setup-no-stop-daemon', 'setup']) + (['--bdb-wallet'] if self.use_bdb_wallet else []) + + ['--setup-no-stop-daemon', 'setup']) for s in ('Starting','Creating','Creating','Creating','Mined','Setup complete'): t.expect(s) return t @@ -1980,3 +1984,6 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared): return 'ok' else: return self.spawn('mmgen-regtest',['stop']) + +class CmdTestRegtestBDBWallet(CmdTestRegtest): + bdb_wallet = True diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index 078e7f4b..c5c7c435 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -186,7 +186,10 @@ init_tests() { " d_btc_rt="overall operations using the regtest network (Bitcoin)" - t_btc_rt="- $cmdtest_py regtest" + t_btc_rt=" + - $cmdtest_py regtest + - $cmdtest_py regtest_legacy + " d_bch="overall operations with emulated RPC data (Bitcoin Cash Node)" t_bch="- $cmdtest_py --coin=bch --exclude regtest"