mmgen-txcreate: automatic change address selection

Instead of selecting a specific change address, i.e. '01ABCDEF:B:7', users can
now omit the address index, specifying '01ABCDEF:B' instead, and the script
will automatically choose the first unused address in the tracking wallet
matching the requested seed ID and address type.

This feature is entirely opt-in.  Existing behavior continues to be supported
without alteration.

Sample invocations:

    # old invocation:
    $ mmgen-txcreate -q 35N9FntsNwy98TmjFHyCpsBVDVUs5wDPfB,0.123 01ABCDEF:S:7

    # new invocation:
    $ mmgen-txcreate -q 35N9FntsNwy98TmjFHyCpsBVDVUs5wDPfB,0.123 01ABCDEF:S

Testing/demo:

    # run the regtest test partially, leaving the coin daemon running:
    $ test/test.py -Dn regtest.view

    # run the 'auto_chg' test group, displaying script output:
    $ test/test.py -DSAe regtest.auto_chg

    # view the addresses in Bob’s tracking wallet, verifying that the first
    # unused ones in each grouping were chosen as change outputs:
    $ PYTHONPATH=. MMGEN_TEST_SUITE=1 cmds/mmgen-tool --bob listaddresses interactive=1

    # When finished, gracefully shut down the daemon:
    $ test/stop-coin-daemons.py btc_rt

mmgen/data/version

@@ -1 +1 @@
-13.3.dev21
+13.3.dev22

mmgen/help.py

@@ -157,10 +157,14 @@ EXAMPLES:
 
     $ {g.prog_name} {sample_addr},0.123 01ABCDEF:{mmtype}:7
 
+  Same as above, but select the change address automatically:
+
+    $ {g.prog_name} {sample_addr},0.123 01ABCDEF:{mmtype}
+
   Same as above, but reduce verbosity and specify fee of 20 satoshis
   per byte:
 
-    $ {g.prog_name} -q -f 20s {sample_addr},0.123 01ABCDEF:{mmtype}:7
+    $ {g.prog_name} -q -f 20s {sample_addr},0.123 01ABCDEF:{mmtype}
 
   Send entire balance of selected inputs minus fee to an external {proto.name}
   address:
@@ -178,7 +182,10 @@ All addresses on the command line can be either {proto.name} addresses or MMGen
 IDs in the form <seed ID>:<address type letter>:<index>.
 
 Outputs are specified in the form <address>,<amount>, with the change output
-specified by address only.
+specified by address only.  Alternatively, the change output may be an
+addrlist ID in the form <seed ID>:<address type letter>, in which case the
+first unused address in the tracking wallet matching the requested ID will
+be automatically selected as the change output.
 
 If the transaction fee is not specified on the command line (see FEE
 SPECIFICATION below), it will be calculated dynamically using network fee
@@ -189,7 +196,9 @@ Network-estimated fees will be multiplied by the value of --tx-fee-adj, if
 specified.
 
 To send the value of all inputs (minus TX fee) to a single output, specify
-a single address with no amount on the command line.
+a single address with no amount on the command line.  Alternatively, an
+addrlist ID may be specified, and the address will be chosen automatically
+as described above for the change output.
 """
 
 		def txsign():

mmgen/main_txcreate.py

@@ -27,7 +27,7 @@ opts_data = {
 	'sets': [('yes', True, 'quiet', True)],
 	'text': {
 		'desc': f'Create a transaction with outputs to specified coin or {g.proj_name} addresses',
-		'usage':   '[opts]  <addr,amt> ... [change addr] [addr file] ...',
+		'usage':   '[opts]  [<addr,amt> ...] <change addr or addrlist ID> [addr file ...]',
 		'options': """
 -h, --help            Print this help message
 --, --longhelp        Print help message for long options (common options)

mmgen/main_txdo.py

@@ -28,7 +28,7 @@ opts_data = {
 	'sets': [('yes', True, 'quiet', True)],
 	'text': {
 		'desc': f'Create, sign and send an {g.proj_name} transaction',
-		'usage':   '[opts]  <addr,amt> ... [change addr] [addr file] ... [seed source] ...',
+		'usage':   '[opts]  [<addr,amt> ...] <change addr or addrlist ID> [addr file ...] [seed source ...]',
 		'options': """
 -h, --help             Print this help message
 --, --longhelp         Print help message for long options (common options)

mmgen/tw/addresses.py

@@ -249,6 +249,52 @@ class TwAddresses(TwView):
 		else: # addr not in tracking wallet
 			return None
 
+	def get_change_address(self,al_id):
+		"""
+		Get lowest-indexed unused address in tracking wallet for requested AddrListID.
+		Return values on failure:
+		    None:  no addresses in wallet with requested AddrListID
+		    False: no unused addresses in wallet with requested AddrListID
+		"""
+
+		def get_start():
+			"""
+			bisecting algorithm to find first entry with requested al_id
+
+			Since 'btc' > 'F' and pre_target sorts below the first twmmid of the al_id
+			stringwise, we can just search on raw twmmids.
+			"""
+			pre_target = al_id + ':0'
+			bot = 0
+			top = len(data) - 1
+			n = top >> 1
+
+			while True:
+
+				if bot == top:
+					return bot if data[bot].al_id == al_id else None
+
+				if data[n].twmmid < pre_target:
+					bot = n + 1
+				else:
+					top = n
+
+				n = (top + bot) >> 1
+
+		self.reverse = False
+		self.do_sort('twmmid')
+
+		data = self.data
+		start = get_start()
+
+		if start is not None:
+			for d in data[start:]:
+				if d.al_id == al_id:
+					if not d.recvd:
+						return d
+				else:
+					return False
+
 	class action(TwView.action):
 
 		def s_amt(self,parent):

mmgen/tx/new.py

@@ -18,7 +18,7 @@ from .base import Base
 from ..color import pink,yellow
 from ..obj import get_obj,MMGenList
 from ..util import msg,qmsg,fmt,die,suf,remove_dups,get_extension
-from ..addr import is_mmgen_id,CoinAddr,is_coin_addr
+from ..addr import is_mmgen_id,CoinAddr,is_coin_addr,AddrListID,is_addrlist_id
 
 def mmaddr2coinaddr(mmaddr,ad_w,ad_f,proto):
 
@@ -69,6 +69,7 @@ class New(Base):
 		ERROR: No change address specified.  If you wish to create a transaction with
 		only one output, specify a single output address with no {} amount
 	"""
+	chg_autoselected = False
 
 	def update_output_amt(self,idx,amt):
 		o = self.outputs[idx]._asdict()
@@ -180,6 +181,18 @@ class New(Base):
 			coin_addr = mmaddr2coinaddr(addr,ad_w,ad_f,self.proto)
 		elif is_coin_addr(self.proto,addr):
 			coin_addr = CoinAddr(self.proto,addr)
+		elif is_addrlist_id(self.proto,addr):
+			if self.proto.base_proto_coin != 'BTC':
+				die(2,f'Change addresses not supported for {self.proto.name} protocol')
+			from ..tw.addresses import TwAddresses
+			res = (await TwAddresses(self.proto,get_data=True)).get_change_address(addr)
+			if res:
+				coin_addr = res.addr
+				self.chg_autoselected = True
+			else:
+				die(2,'Tracking wallet contains no {t}addresses from address list {a!r}'.format(
+					t = ('unused ','')[res is None],
+					a = addr ))
 		else:
 			die(2,f'{addr}: invalid {err_desc} {{!r}}'.format(f'{addr},{amt}' if amt else addr))
 
@@ -236,9 +249,20 @@ class New(Base):
 		self.check_dup_addrs('outputs')
 
 		if self.chg_output is not None:
-			if len(self.outputs) > 1:
+			if self.chg_autoselected:
+				self.confirm_autoselected_addr(self.chg_output)
+			elif len(self.outputs) > 1:
 				await self.warn_chg_addr_used(self.chg_output)
 
+	def confirm_autoselected_addr(self,chg):
+		from ..ui import keypress_confirm
+		if not keypress_confirm(
+				'Using {a} as {b} address. OK?'.format(
+					a = chg.mmid.hl(),
+					b = 'single output' if len(self.outputs) == 1 else 'change' ),
+				default_yes = True ):
+			die(1,'Exiting at user request')
+
 	async def warn_chg_addr_used(self,chg):
 		from ..tw.addresses import TwAddresses
 		if (await TwAddresses(self.proto,get_data=True)).is_used(chg.addr):

test/test_py_d/ts_regtest.py

@@ -164,6 +164,7 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 		('subgroup.txhist',     ['main']),
 		('subgroup.label',      ['main']),
 		('subgroup.view',       ['label']),
+		('subgroup.auto_chg',   ['view']),
 		('stop',                'stopping regtest daemon'),
 	)
 	cmd_subgroups = {
@@ -346,6 +347,18 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 		('alice_txcreate_info',           'txcreate -i'),
 		('alice_txcreate_info_term',      'txcreate -i (pexpect_spawn)'),
 	),
+	'auto_chg': (
+		'automatic change address selection',
+		('bob_split3',        'splitting Bob’s funds'),
+		('generate',          'mining a block'),
+		('bob_auto_chg1',     'creating an automatic change address transaction (C)'),
+		('bob_auto_chg2',     'creating an automatic change address transaction (B)'),
+		('bob_auto_chg3',     'creating an automatic change address transaction (S)'),
+		('bob_auto_chg4',     'creating an automatic change address transaction (single address)'),
+		('bob_auto_chg_bad1', 'error handling for auto change address transaction (bad ID FFFFFFFF:C)'),
+		('bob_auto_chg_bad2', 'error handling for auto change address transaction (bad ID 00000000:C)'),
+		('bob_auto_chg_bad3', 'error handling for auto change address transaction (no unused addresses)'),
+	),
 	}
 
 	def __init__(self,trunner,cfgs,spawn):
@@ -1394,6 +1407,66 @@ class TestSuiteRegtest(TestSuiteBase,TestSuiteShared):
 			msgfile = os.path.join(self.tmpdir,'signatures.json')
 		)
 
+	def bob_split3(self):
+		if not self.proto.cap('segwit'):
+			return 'skip'
+		sid = self._user_sid('bob')
+		return self.user_txdo(
+			user = 'bob',
+			fee = '23s',
+			outputs_cl = [sid+':C:5,0.0135', sid+':L:4'],
+			outputs_list = '1' )
+
+	def _bob_auto_chg(self,al_id,include_dest=True):
+		dest = [self.burn_addr+',0.01'] if include_dest else []
+		t = self.spawn(
+			'mmgen-txcreate',
+			['-d',self.tr.trash_dir,'-B','--bob', al_id] + dest)
+		return self.txcreate_ui_common(t,
+			menu            = [],
+			inputs          = '1',
+			interactive_fee = '20s',
+			auto_chg_al_id  = al_id )
+
+	def bob_auto_chg1(self):
+		return self._bob_auto_chg(self._user_sid('bob') + ':C')
+
+	def bob_auto_chg2(self):
+		if not self.proto.cap('segwit'):
+			return 'skip'
+		return self._bob_auto_chg(self._user_sid('bob') + ':B')
+
+	def bob_auto_chg3(self):
+		if not self.proto.cap('segwit'):
+			return 'skip'
+		return self._bob_auto_chg(self._user_sid('bob') + ':S')
+
+	def bob_auto_chg4(self):
+		return self._bob_auto_chg( self._user_sid('bob') + ':C', include_dest=False )
+
+	def _bob_auto_chg_bad(self,al_id,expect):
+		t = self.spawn(
+			'mmgen-txcreate',
+			['-d',self.tr.trash_dir,'-B','--bob', self.burn_addr+',0.01', al_id] )
+		t.req_exit_val = 2
+		t.expect(expect)
+		return t
+
+	def bob_auto_chg_bad1(self):
+		return self._bob_auto_chg_bad(
+			'FFFFFFFF:C',
+			'contains no addresses' )
+
+	def bob_auto_chg_bad2(self):
+		return self._bob_auto_chg_bad(
+			'00000000:C',
+			'contains no addresses' )
+
+	def bob_auto_chg_bad3(self):
+		return self._bob_auto_chg_bad(
+			self._user_sid('bob') + ':L',
+			'contains no unused addresses' )
+
 	def stop(self):
 		if opt.no_daemon_stop:
 			self.spawn('',msg_only=True)

test/test_py_d/ts_shared.py

@@ -47,7 +47,8 @@ class TestSuiteShared(object):
 			view              = 't',
 			save              = True,
 			tweaks            = [],
-			used_chg_addr_resp = None ):
+			used_chg_addr_resp = None,
+			auto_chg_al_id    = None ):
 
 		txdo = (caller or self.test_name)[:4] == 'txdo'
 
@@ -58,6 +59,9 @@ class TestSuiteShared(object):
 		if used_chg_addr_resp is not None:
 			t.expect('reuse harms your privacy.*:.*',used_chg_addr_resp,regex=True)
 
+		if auto_chg_al_id is not None:
+			t.expect(fr'Using .*{auto_chg_al_id}:\d+\D.* as.*address','y',regex=True)
+
 		pat = expect_pat
 		for choice in menu + ['q']:
 			t.expect(pat,choice,regex=True)