support descriptor wallets for BTC

For BTC, MMGen Wallet now creates and uses a descriptor wallet as its tracking
wallet.  For LTC and BCH, the legacy Berkeley DB wallet is used by default as
before.

While the legacy BDB wallet continues to be fully supported, BTC users are
advised to upgrade their tracking wallet to a descriptor wallet to avoid
support issues with future versions of Bitcoin Core.

Upgrade a legacy tracking wallet as follows:

    $ bitcoin-cli migratewallet mmgen-tracking-wallet

Alternatively, you may dump your tracking wallet to JSON and restore it as a
descriptor wallet using the ‘mmgen-tool’ commands ‘twexport’, ‘twimport’ and
‘rescan_blockchain’ (see the help screens for those commands for details).

Testing (add the -e option to see script output):

    # descriptor wallet:
    $ test/cmdtest.py regtest

    # Berkeley DB wallet:
    $ test/cmdtest.py regtest_legacy

mmgen/data/version

@@ -1 +1 @@
-14.1.dev17
+14.1.dev18

mmgen/main_regtest.py

@@ -32,6 +32,7 @@ opts_data = {
 		'options': """
 -h, --help          Print this help message
 --, --longhelp      Print help message for long options (common options)
+-b, --bdb-wallet    Create and use a legacy Berkeley DB coin daemon wallet
 -e, --empty         Don't fund Bob and Alice's wallets on setup
 -n, --setup-no-stop-daemon  Don't stop daemon after setup is finished
 -q, --quiet         Produce quieter output
@@ -83,6 +84,6 @@ elif cmd_args[0] not in ('cli','wallet_cli','balances'):
 	check_num_args()
 
 async def main():
-	await MMGenRegtest(cfg,cfg.coin).cmd(cmd_args)
+	await MMGenRegtest(cfg, cfg.coin, bdb_wallet=cfg.bdb_wallet).cmd(cmd_args)
 
 async_run(main())

mmgen/proto/btc/daemon.py

@@ -31,6 +31,7 @@ class bitcoin_core_daemon(CoinDaemon):
 		'linux': [gc.home_dir,'.bitcoin'],
 		'win32': [os.getenv('APPDATA'),'Bitcoin']
 	}
+	avail_opts = ('no_daemonize', 'online', 'bdb_wallet')
 
 	def init_datadir(self):
 		if self.network == 'regtest' and not self.test_suite:
@@ -78,7 +79,7 @@ class bitcoin_core_daemon(CoinDaemon):
 			['--daemon',               self.platform == 'linux' and not self.opt.no_daemonize],
 			['--fallbackfee=0.0002',   self.coin == 'BTC' and self.network == 'regtest'],
 			['--usecashaddr=0',        self.coin == 'BCH'],
-			['--deprecatedrpc=create_bdb', self.coin == 'BTC'],
+			['--deprecatedrpc=create_bdb', self.coin == 'BTC' and self.opt.bdb_wallet],
 			['--mempoolreplacement=1', self.coin == 'LTC'],
 			['--txindex=1',            self.coin == 'LTC' or self.network == 'regtest'],
 			['--addresstype=bech32',   self.coin == 'LTC' and self.network == 'regtest'],

mmgen/proto/btc/regtest.py

@@ -73,16 +73,19 @@ class MMGenRegtest(MMGenObject):
 		'bch': 'n2fxhNx27GhHAWQhyuZ5REcBNrJqCJsJ12',
 	}
 
-	def __init__(self,cfg,coin):
+	def __init__(self, cfg, coin, bdb_wallet=False):
 		self.cfg = cfg
 		self.coin = coin.lower()
+		self.bdb_wallet = bdb_wallet
+
 		assert self.coin in self.coins, f'{coin!r}: invalid coin for regtest'
 
 		self.proto = init_proto(cfg, self.coin, regtest=True, need_amt=True)
 		self.d = CoinDaemon(
 			cfg,
 			self.coin + '_rt',
-			test_suite = cfg.test_suite)
+			test_suite = cfg.test_suite,
+			opts       = ['bdb_wallet'] if bdb_wallet else None)
 
 	# Caching creates problems (broken pipe) when recreating + loading wallets,
 	# so reinstantiate with every call:
@@ -93,13 +96,17 @@ class MMGenRegtest(MMGenObject):
 	@property
 	async def miner_addr(self):
 		if not hasattr(self,'_miner_addr'):
-			self._miner_addr = self.bdb_miner_addrs[self.coin]
+			self._miner_addr = (
+				self.bdb_miner_addrs[self.coin] if self.bdb_wallet else
+				await self.rpc_call('getnewaddress', wallet='miner'))
 		return self._miner_addr
 
 	@property
 	async def miner_wif(self):
 		if not hasattr(self,'_miner_wif'):
-			self._miner_wif = self.bdb_miner_wif
+			self._miner_wif = (
+				self.bdb_miner_wif if self.bdb_wallet else
+				await self.rpc_call('dumpprivkey', (await self.miner_addr), wallet='miner'))
 		return self._miner_wif
 
 	def create_hdseed_wif(self):
@@ -118,6 +125,7 @@ class MMGenRegtest(MMGenObject):
 
 		self.d.wait_for_state('ready')
 
+		# very slow with descriptor wallet and large block count - 'generatetodescriptor' no better
 		out = await self.rpc_call(
 			'generatetoaddress',
 			blocks,
@@ -134,8 +142,9 @@ class MMGenRegtest(MMGenObject):
 		return await (await self.rpc).icall(
 			'createwallet',
 			wallet_name     = user,
-			blank           = True,
+			blank           = user != 'miner' or self.bdb_wallet,
 			no_keys         = user != 'miner',
+			descriptors     = not self.bdb_wallet,
 			load_on_startup = False)
 
 	async def setup(self):
@@ -165,11 +174,12 @@ class MMGenRegtest(MMGenObject):
 		# Unfortunately, we don’t get deterministic output with BCH and LTC even with fixed
 		# hdseed, as their 'sendtoaddress' calls produce non-deterministic TXIDs due to random
 		# input ordering and fee estimation.
-		await (await self.rpc).call(
-			'sethdseed',
-			True,
-			self.create_hdseed_wif(),
-			wallet = 'miner')
+		if self.bdb_wallet:
+			await (await self.rpc).call(
+				'sethdseed',
+				True,
+				self.create_hdseed_wif(),
+				wallet = 'miner')
 
 		# Broken litecoind can only mine 431 blocks in regtest mode, so generate just enough
 		# blocks to fund the test suite
@@ -255,7 +265,7 @@ class MMGenRegtest(MMGenObject):
 
 		gmsg(f'Creating fork from coin {coin} to coin {proto.coin}')
 
-		source_rt = MMGenRegtest( self.cfg, coin )
+		source_rt = MMGenRegtest(self.cfg, coin, bdb_wallet=self.bdb_wallet)
 
 		try:
 			os.stat(source_rt.d.datadir)

mmgen/proto/btc/rpc.py

@@ -56,6 +56,7 @@ class CallSigs:
 				no_keys         = True,
 				blank           = True,
 				passphrase      = '',
+				descriptors     = True,
 				load_on_startup = True):
 			"""
 			Quirk: when --datadir is specified (even if standard), wallet is created directly in
@@ -68,7 +69,7 @@ class CallSigs:
 				blank,          # 3. blank (no keys or seed)
 				passphrase,     # 4. passphrase (empty string for non-encrypted)
 				False,          # 5. avoid_reuse (track address reuse)
-				False,          # 6. descriptors (native descriptor wallet)
+				descriptors,    # 6. descriptors (native descriptor wallet)
 				load_on_startup # 7. load_on_startup
 			)
 
@@ -89,6 +90,7 @@ class CallSigs:
 				no_keys         = True,
 				blank           = True,
 				passphrase      = '',
+				descriptors     = True,
 				load_on_startup = True):
 			return (
 				'createwallet',
@@ -201,6 +203,12 @@ class BitcoinRPCClient(RPCClient,metaclass=AsyncInit):
 		else:
 			self.wallet_path = f'/wallet/{self.twname}'
 
+	@property
+	async def walletinfo(self):
+		if not hasattr(self,'_walletinfo'):
+			self._walletinfo = await self.call('getwalletinfo')
+		return self._walletinfo
+
 	def set_auth(self):
 		"""
 		MMGen's credentials override coin daemon's
@@ -343,6 +351,7 @@ class BitcoinRPCClient(RPCClient,metaclass=AsyncInit):
 		'getrawtransaction',
 		'gettransaction',
 		'importaddress', # address (address or script) label rescan p2sh (Add P2SH version of the script)
+		'importdescriptors', # like above, but for descriptor wallets
 		'listaccounts',
 		'listlabels',
 		'listunspent',

mmgen/proto/btc/tw/bal.py

@@ -20,6 +20,7 @@ class BitcoinTwGetBalance(TwGetBalance):
 
 	async def __init__(self, cfg, proto, minconf, quiet):
 		self.rpc = await rpc_init(cfg, proto)
+		self.walletinfo = await self.rpc.walletinfo
 		await super().__init__(cfg, proto, minconf, quiet)
 
 	start_labels = ('TOTAL','Non-MMGen','Non-wallet')
@@ -53,21 +54,25 @@ class BitcoinTwGetBalance(TwGetBalance):
 			self.data['TOTAL'][col_key] += amt
 			self.data[label][col_key] += amt
 
-			if d['spendable']:
+			if d['spendable']: # TODO: use 'solvable' for descriptor wallets?
 				self.data[label]['spendable'] += amt
 
 	def format(self, color):
 
 		def gen_spendable_warning():
-			for k,v in self.data.items():
-				if v['spendable']:
-					yield red(f'Warning: this wallet contains PRIVATE KEYS for {k} outputs!')
+			if check_spendable:
+				for k,v in self.data.items():
+					if v['spendable']:
+						yield red(f'Warning: this wallet contains PRIVATE KEYS for {k} outputs!')
 
 		if color:
 			from ....color import red
 		else:
 			from ....color import nocolor as red
 
+		desc_wallet = self.walletinfo.get('descriptors')
+		check_spendable = not desc_wallet or (desc_wallet and self.walletinfo['private_keys_enabled'])
+
 		warning = '\n'.join(gen_spendable_warning())
 
 		return super().format(color) + ('\n' if warning else '') + warning

mmgen/proto/btc/tw/ctl.py

@@ -28,11 +28,24 @@ class BitcoinTwCtl(TwCtl):
 
 	@write_mode
 	async def import_address(self, addr, label, rescan=False): # rescan is True by default, so set to False
-		return await self.rpc.call('importaddress', addr, label, rescan)
+		if (await self.rpc.walletinfo).get('descriptors'):
+			return await self.batch_import_address([(addr, label, rescan)])
+		else:
+			return await self.rpc.call('importaddress', addr, label, rescan)
 
 	@write_mode
 	async def batch_import_address(self,arg_list):
-		return await self.rpc.batch_call('importaddress', arg_list)
+		if (await self.rpc.walletinfo).get('descriptors'):
+			from ....contrib.descriptors import descsum_create
+			return await self.rpc.call(
+				'importdescriptors',
+				[{
+					'desc': descsum_create(f'addr({addr})'),
+					'label': label,
+					'timestamp': 0 if rescan else 'now',
+				} for addr, label, rescan in arg_list])
+		else:
+			return await self.rpc.batch_call('importaddress', arg_list)
 
 	@write_mode
 	async def remove_address(self,addr):

test/cmdtest_py_d/cfg.py

@@ -45,6 +45,7 @@ cmd_groups_dfl = {
 
 cmd_groups_extra = {
 	'dev':                    ('CmdTestDev',{'modname':'misc'}),
+	'regtest_legacy':         ('CmdTestRegtestBDBWallet', {'modname':'regtest'}),
 	'autosign_btc':           ('CmdTestAutosignBTC',{'modname':'autosign'}),
 	'autosign_live':          ('CmdTestAutosignLive',{'modname':'autosign'}),
 	'autosign_live_simulate': ('CmdTestAutosignLiveSimulate',{'modname':'autosign'}),

test/cmdtest_py_d/ct_automount.py

@@ -15,11 +15,11 @@ import os, time
 from pathlib import Path
 
 from .ct_autosign import CmdTestAutosignThreaded
-from .ct_regtest import CmdTestRegtest, rt_pw
+from .ct_regtest import CmdTestRegtestBDBWallet, rt_pw
 from .common import get_file_with_ext
 from ..include.common import cfg
 
-class CmdTestAutosignAutomount(CmdTestAutosignThreaded, CmdTestRegtest):
+class CmdTestAutosignAutomount(CmdTestAutosignThreaded, CmdTestRegtestBDBWallet):
 	'automounted transacting operations via regtest mode'
 
 	networks = ('btc', 'bch', 'ltc')
@@ -75,7 +75,7 @@ class CmdTestAutosignAutomount(CmdTestAutosignThreaded, CmdTestRegtest):
 		self.coins = [cfg.coin.lower()]
 
 		CmdTestAutosignThreaded.__init__(self, trunner, cfgs, spawn)
-		CmdTestRegtest.__init__(self, trunner, cfgs, spawn)
+		CmdTestRegtestBDBWallet.__init__(self, trunner, cfgs, spawn)
 
 		if trunner == None:
 			return

test/cmdtest_py_d/ct_regtest.py

@@ -175,6 +175,7 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared):
 	deterministic = False
 	test_rbf = False
 	proto = None # pylint
+	bdb_wallet = False
 
 	cmd_group_in = (
 		('setup',                   'regtest (Bob and Alice) mode setup'),
@@ -468,7 +469,9 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared):
 		for k in rt_data:
 			gldict[k] = rt_data[k][coin] if coin in rt_data[k] else None
 
-		self.rt = MMGenRegtest(cfg, self.proto.coin)
+		self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC'
+
+		self.rt = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet)
 
 		if self.proto.coin == 'BTC':
 			self.test_rbf = True # tests are non-coin-dependent, so run just once for BTC
@@ -512,7 +515,8 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared):
 			pass
 		t = self.spawn(
 			'mmgen-regtest',
-			['--setup-no-stop-daemon', 'setup'])
+			(['--bdb-wallet'] if self.use_bdb_wallet else [])
+			+ ['--setup-no-stop-daemon', 'setup'])
 		for s in ('Starting','Creating','Creating','Creating','Mined','Setup complete'):
 			t.expect(s)
 		return t
@@ -1980,3 +1984,6 @@ class CmdTestRegtest(CmdTestBase,CmdTestShared):
 			return 'ok'
 		else:
 			return self.spawn('mmgen-regtest',['stop'])
+
+class CmdTestRegtestBDBWallet(CmdTestRegtest):
+	bdb_wallet = True

test/test-release.d/cfg.sh

@@ -186,7 +186,10 @@ init_tests() {
 	"
 
 	d_btc_rt="overall operations using the regtest network (Bitcoin)"
-	t_btc_rt="- $cmdtest_py regtest"
+	t_btc_rt="
+		- $cmdtest_py regtest
+		- $cmdtest_py regtest_legacy
+	"
 
 	d_bch="overall operations with emulated RPC data (Bitcoin Cash Node)"
 	t_bch="- $cmdtest_py --coin=bch --exclude regtest"