tx.file: new transaction file format

- the new format is plain JSON, readable with tools such as `jq`.  Filenames
  and extensions for raw, signed and sent transactions remain unchanged

- reading/writing the legacy format continues to be supported, but new
  transactions cannot be written to it.  This means users who upgrade MMGen
  Wallet to this commit on their online computer(s) must upgrade it on their
  offline signing device(s) as well: upgraded offline installations can
  interoperate with non-upgraded online installations, but not vice-versa

- no additional action is required: this change is transparent to the user

Testing:

    $ test/unit_tests.py tx.txfile
    $ test/cmdtest.py -n main
    $ test/test-release.sh -A autosign

mmgen/data/version

@@ -1 +1 @@
-15.1.dev4
+15.1.dev5

mmgen/tx/base.py

@@ -92,6 +92,7 @@ class Base(MMGenObject):
 		Non-{gc.proj_name} addresses found in inputs:
 		    {{}}
 	"""
+	file_format = 'json'
 
 	class Input(MMGenTxIO):
 		scriptPubKey = ListItemAttr(HexStr)

mmgen/tx/file.py

@@ -19,10 +19,15 @@
 """
 tx.file: Transaction file operations for the MMGen suite
 """
-import os
+
+import os, json
 
 from ..util import ymsg,make_chksum_6,die
 from ..obj import MMGenObject,HexStr,MMGenTxID,CoinTxID,MMGenTxComment
+from ..rpc import json_encoder
+
+def json_dumps(data):
+	return json.dumps(data, separators = (',', ':'), cls=json_encoder)
 
 def get_proto_from_coin_id(tx, coin_id, chain):
 	coin, tokensym = coin_id.split(':') if ':' in coin_id else (coin_id, None)
@@ -49,6 +54,21 @@ def eval_io_data(tx, data, desc):
 	return io_list(parent=tx, data=[io(tx.proto,**d) for d in data])
 
 class MMGenTxFile(MMGenObject):
+	data_label = 'MMGenTransaction'
+	attrs = {
+		'chain': None,
+		'txid': MMGenTxID,
+		'send_amt': 'skip',
+		'timestamp': None,
+		'blockcount': None,
+		'serialized': None,
+	}
+	extra_attrs = {
+		'locktime': None,
+		'comment': MMGenTxComment,
+		'coin_txid': CoinTxID,
+		'sent_timestamp': None,
+	}
 
 	def __init__(self,tx):
 		self.tx       = tx
@@ -62,7 +82,38 @@ class MMGenTxFile(MMGenObject):
 		if len(data) > tx.cfg.max_tx_file_size:
 			die('MaxFileSizeExceeded',
 				f'Transaction file size exceeds limit ({tx.cfg.max_tx_file_size} bytes)')
-		return self.parse_data_legacy(data, metadata_only)
+		return (self.parse_data_json if data[0] == '{' else self.parse_data_legacy)(data, metadata_only)
+
+	def parse_data_json(self, data, metadata_only):
+		tx = self.tx
+		tx.file_format = 'json'
+		outer_data = json.loads(data)
+		data = outer_data[self.data_label]
+		if outer_data['chksum'] != make_chksum_6(json_dumps(data)):
+			chk = make_chksum_6(json_dumps(data))
+			die(3, f'{self.data_label}: invalid checksum for TxID {data["txid"]} ({chk} != {outer_data["chksum"]})')
+
+		tx.proto = get_proto_from_coin_id(tx, data['coin_id'], data['chain'])
+
+		for k, v in self.attrs.items():
+			if v != 'skip':
+				setattr(tx, k, v(data[k]) if v else data[k])
+
+		if metadata_only:
+			return
+
+		for k, v in self.extra_attrs.items():
+			if k in data:
+				setattr(tx, k, v(data[k]) if v else data[k])
+
+		for k in ('inputs', 'outputs'):
+			setattr(tx, k, eval_io_data(tx, data[k], k))
+
+		tx.check_txfile_hex_data()
+
+		tx.parse_txfile_serialized_data() # Ethereum RLP or JSON data
+
+		assert tx.proto.coin_amt(data['send_amt']) == tx.send_amt, f'{data["send_amt"]} != {tx.send_amt}'
 
 	def parse_data_legacy(self, data, metadata_only):
 		tx = self.tx
@@ -209,7 +260,20 @@ class MMGenTxFile(MMGenObject):
 
 			return '\n'.join([make_chksum_6(' '.join(lines))] + lines) + '\n'
 
-		fmt_data = format_data_legacy()
+		def format_data_json():
+			data = json_dumps({
+					'coin_id': coin_id
+				} | {
+					k: getattr(tx, k) for k in self.attrs
+				} | {
+					'inputs':  [e._asdict() for e in tx.inputs],
+					'outputs': [e._asdict() for e in tx.outputs]
+				} | {
+					k: getattr(tx, k) for k in self.extra_attrs if getattr(tx, k)
+				})
+			return '{{"{}":{},"chksum":"{}"}}'.format(self.data_label, data, make_chksum_6(data))
+
+		fmt_data = {'json': format_data_json, 'legacy': format_data_legacy}[tx.file_format]()
 
 		if len(fmt_data) > tx.cfg.max_tx_file_size:
 			die( 'MaxFileSizeExceeded', f'Transaction file size exceeds limit ({tx.cfg.max_tx_file_size} bytes)' )

test/ref/data_dir/altcoins/etc/tracking-wallet.json

@@ -0,0 +1 @@
+{"coin":"ETC","network":"MAINNET","accounts":{"65f56389a1c702ab62a32c13d43d87a734b5e11f":{"mmid":"98831F3A:E:12","comment":"","balance":"99.997088092"},"d4ab1fecf420cbdc3d551c1935acdeaa1fb5b181":{"mmid":"98831F3A:E:13","comment":"","balance":"0"}},"tokens":{"492934308e98b590a626666b703a6ddf2120e85e":{"params":{"symbol":"MM1","decimals":18},"65f56389a1c702ab62a32c13d43d87a734b5e11f":{"mmid":"98831F3A:E:12","comment":"","balance":"43.21"},"d4ab1fecf420cbdc3d551c1935acdeaa1fb5b181":{"mmid":"98831F3A:E:13","comment":"","balance":"1.23456"}}}}

test/ref/tx/0A869F[1.23456,32].regtest.asubtx

@@ -0,0 +1 @@
+{"MMGenTransaction":{"coin_id":"BTC","chain":"regtest","txid":"0A869F","send_amt":"1.23456","timestamp":"20241011_142209","blockcount":395,"serialized":"0200000000010116ee956bfc0e1e902f2fdf9c92e4cfeb129856efb30bb94563792fdac83258eb0000000017160014e665d2746915d246d8ee50fcbd092778107657a6fdffffff0200ca5b07000000001976a914000000000000000000000000000000000000000088ace094df9c0b0000001976a9142d1fd5841f0c217662f01a97a1fdf34de1d233f888ac024730440220278901a54dd4834ead1b3896a97eedb546a61f1df239a0e807f11e4aa00ee2ac022033a50b5e5e76835aa2eaa14df2ff5f8985d617dffc66d0f9841b5a2943077599012103fba670aa3ae2be0a8b65b3eca71760aff9026a81d61046c0e1c2fb237140381700000000","inputs":[{"txid":"eb5832c8da2f796345b90bb3ef569812ebcfe4929cdf2f2f901e0efc6b95ee16","vout":0,"scriptPubKey":"a91487d16a346909c7aeb90ca2ac8cfeb24784f4193887","comment":"","amt":"500","addr":"2N5dN5ady9UzR3qYdRSmTPUWpHhAmVbHoRD","confs":2,"mmid":"DE05F9A1:S:1","sequence":4294967293}],"outputs":[{"addr":"mfWxJ45yp2SFn7UciZyNpvDKrzbhyfKrY8","amt":"1.23456","is_chg":false},{"addr":"mjdYqZ8VzXawMSu2hFPYn18EC8Y6WEs3BC","amt":"498.76538592","is_chg":true,"mmid":"DE05F9A1:C:5"}],"comment":"This one\u2019s worth a comment","coin_txid":"a7b9b1a3607334e8ff5dc0bc9902bbfb7493b8070aa020748ecb237a9aca9785","sent_timestamp":"20241011_142211"},"chksum":"e22da6"}

test/ref/tx/7A8157[6.65227,34].rawtx

@@ -0,0 +1 @@
+{"MMGenTransaction":{"coin_id":"BTC","chain":"mainnet","txid":"7A8157","send_amt":"6.65227","timestamp":"20241010_154753","blockcount":1000000,"serialized":"02000000011a7537162621fc98a191fdb96e1dbcfa290935ff7661a3d9e60525f3d1b703a30000000000fdffffff0388e1b913000000001976a91471402298f14e70277ab549ccb6a5a34eb862faa888ac70adec130000000017a914a7c818791e3c19d62459901e7a94c566d1f397e08738eb94af000000001976a9147819b0332bb615808965de645d488d242c82194f88ac00000000","inputs":[{"vout":0,"txid":"a303b7d1f32505e6d9a36176ff350929fabc1d6eb9fd91a198fc21261637751a","scriptPubKey":"76a91429d8b6a52a93730cb125db5984debd38528b0d7c88ac","comment":"House purchase","amt":"36.11009344","addr":"14pGKP3YbVg1XHLh523QxR2xHuA3iy9p1U","confs":316150,"mmid":"10175B2D:L:5","sequence":4294967293}],"outputs":[{"addr":"1BKpBsAb4fF8NkhWni17aEGTNJsCQJ3yCZ","amt":"3.30949","is_chg":false,"mmid":"10175B2D:L:12"},{"addr":"3GzAUTwc8j8MuKsVs6qfCbNKVeuLgXmt6Q","amt":"3.34278","is_chg":false},{"addr":"1Bx2rscjX7w8PwFuY3Nn2GyJUZXcXam1mE","amt":"29.45772344","is_chg":true,"mmid":"10175B2D:L:99"}]},"chksum":"95d646"}

test/ref/tx/91060A-BCH[1.23456].regtest.arawtx

@@ -0,0 +1 @@
+{"MMGenTransaction":{"coin_id":"BCH","chain":"regtest","txid":"91060A","send_amt":"1.23456","timestamp":"20241011_143137","blockcount":395,"serialized":"0200000001f5615164c22c221122649c7076b30a815a25ed125fb89b8c15902e56c939b64f0100000000ffffffff0200ca5b07000000001976a914000000000000000000000000000000000000000088ac008adf9c0b0000001976a914e9bd3f5df1e2621e8d69fa5a7fb4d48f12a1f5ac88ac00000000","inputs":[{"txid":"4fb639c9562e90158c9bb85f12ed255a810ab376709c642211222cc2645161f5","vout":1,"scriptPubKey":"76a914c330f16a6618930f9de900813855135da8f5f89788ac","comment":"","amt":"500","addr":"bchreg:qrpnput2vcvfxruaayqgzwz4zdw63a0cjujruft7h6","confs":2,"mmid":"D1432478:L:1","sequence":4294967295}],"outputs":[{"addr":"bchreg:qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqha9s37tt","amt":"1.23456","is_chg":false},{"addr":"bchreg:qr5m606a783xy85dd8a95la56j839g044s7gcgjq60","amt":"498.76535808","is_chg":true,"mmid":"D1432478:C:5"}]},"chksum":"41eb4f"}

test/ref/tx/C09D73-LTC[981.73747,2000].testnet.rawtx

@@ -0,0 +1 @@
+{"MMGenTransaction":{"coin_id":"LTC","chain":"testnet","txid":"C09D73","send_amt":"981.73747","timestamp":"20241011_142938","blockcount":1000000,"serialized":"02000000023088a0bfd661b1102f25d4963cb44027855166bed946f4286c509969c8b298130300000000fdffffff395fc6e667bc2c84d199b618f6b309facd975837c983ad44f2425ed94bc26a7c0100000000fdffffff04e0d5e949020000001976a914cfceadbbbcc160a35e0cff9e90039451495cb57588ace0eeb56e0700000017a914714a56e6b30326420f358d5bf130caaee27609358778bafc220d0000001976a914bed52df4b7fdfedae9470239c1a80e17c8b400b488ac5f150922660000001976a914bc5d121e221457b98343fe69fa7e2e855b740fb488ac00000000","inputs":[{"vout":3,"txid":"1398b2c86999506c28f446d9be6651852740b43c96d4252f10b161d6bfa08830","scriptPubKey":"76a91497c85adaaf5bd12acde85b79311c1f5712bff23b88ac","comment":"","amt":"1484.89524648","addr":"muMWLvUo85eFpgkprNUFiZCfoWGKo2duj9","confs":821544,"mmid":"D59AC957:L:2","sequence":4294967293},{"vout":1,"txid":"7c6ac24bd95e42f244ad83c9375897cdfa09b3f618b699d1842cbc67e6c65f39","scriptPubKey":"76a91479db077835922a8c8da6cea87a6e4ea5f527fdc588ac","comment":"\u6240\u4ee5\uff0c\u6211\u5011\u975e\u5e38\u9700\u8981\u9019\u6a23\u4e00\u7a2e\u96fb\u5b50\u652f\u4ed8\u7cfb\u7d71\uff0c\u5b83\u57fa\u65bc\u5bc6\u78bc\u5b78\u539f\u7406\u800c\u4e0d\u57fa\u65bc\u4fe1\u7528\uff0c\u4f7f\u5f97\u4efb\u4f55\u9054","amt":"3883.42907183","addr":"mrdGSgNP1NVCJxPHCzYHDGQz9LcSezrNHJ","confs":775261,"mmid":"5E75A2D0:L:5","sequence":4294967293}],"outputs":[{"addr":"mzTjm3gKT44wAWHo1bJzqrQ6A7BoHs8L3P","amt":"98.29996","is_chg":false,"mmid":"5E75A2D0:L:12"},{"addr":"QWw1XAvBMcnUUvUUgYHQFx9KLWUnjrTnPq","amt":"319.22188","is_chg":false},{"addr":"mxuz4UTTehBrH6GYUNJYJYoy31UaaXAD8W","amt":"564.21563","is_chg":false,"mmid":"D59AC957:L:1022"},{"addr":"mxgvpnhfdZQEMNEdCkvYoT7WiLfKARa8Qa","amt":"4386.57684831","is_chg":true,"mmid":"D59AC957:L:1023"}]},"chksum":"bad2aa"}

test/ref/tx/D850C6-MM1[43.21,50000].subtx

@@ -0,0 +1 @@
+{"MMGenTransaction":{"coin_id":"ETC:MM1","chain":"classic","txid":"D850C6","send_amt":"43.21","timestamp":"20241011_144051","blockcount":20,"serialized":"f8a902850ba43b740082ea6094492934308e98b590a626666b703a6ddf2120e85e80b844a9059cbb00000000000000000000000065f56389a1c702ab62a32c13d43d87a734b5e11f00000000000000000000000000000000000000000000000257a8c21048a1000046a064093d1ddd4ee3bbb3f88e27388272bf62d744ef77d03a307a68ed674d671eeba07d7332481c857c9f030386c68e95b4d8e688abe77b938d93d62d83c8c61b2269","inputs":[{"comment":"","amt":"110.654317776666555545","addr":"1f5c9ee4a60d4a3c8b89eec978438bb6b53abf4a","confs":0,"mmid":"98831F3A:E:11"}],"outputs":[{"addr":"65f56389a1c702ab62a32c13d43d87a734b5e11f","amt":"43.21","is_chg":false,"mmid":"98831F3A:E:12"}],"comment":"\u5fc5\u8981\u306a\u306e\u306f\u3001\u4fe1\u7528\u3067\u306f\u306a\u304f\u6697\u53f7\u5316\u3055\u308c\u305f\u8a3c\u660e\u306b\u57fa\u3065\u304f\u96fb\u5b50\u53d6\u5f15\u30b7\u30b9\u30c6\u30e0\u3067\u3042\u308a\u3001\u3053\u308c\u306b\u3088\u308a\u5e0c\u671b\u3059\u308b\u4e8c\u8005\u304c\u4fe1\u7528\u3067\u304d\u308b\u7b2c\u4e09\u8005\u6a5f\u95a2\u3092\u4ecb\u3055\u305a\u306b\u76f4\u63a5\u53d6\u5f15\u3067\u304d\u308b\u3088\u3046","coin_txid":"61f624d8a279e566055294e6a3d17f4c7438e80f50a509e145df150b0447b53c","sent_timestamp":"20241011_144051"},"chksum":"974b90"}

test/unit_tests_d/ut_tx.py

@@ -11,10 +11,11 @@ from mmgen.tx import NewTX,CompletedTX,UnsignedTX
 from mmgen.tx.file import MMGenTxFile
 from mmgen.daemon import CoinDaemon
 from mmgen.protocol import init_proto
+from mmgen.cfg import Config
 
 from ..include.common import cfg, qmsg, vmsg
 
-async def do_txfile_test(desc,fns):
+async def do_txfile_test(desc, fns, cfg=cfg, check=False):
 	qmsg(f'  Testing CompletedTX initializer ({desc})')
 	for fn in fns:
 		qmsg(f'     parsing: {os.path.basename(fn)}')
@@ -31,14 +32,18 @@ async def do_txfile_test(desc,fns):
 
 		assert fn_gen == os.path.basename(fn), f'{fn_gen} != {fn}'
 
-		text = f.format()
+		if check:
+			text = f.format()
+			with open(fpath) as fh:
+				text_chk = fh.read()
+			assert text == text_chk, f'\nformatted text:\n{text}\n  !=\noriginal file:\n{text_chk}'
 
 	qmsg('  OK')
 	return True
 
 class unit_tests:
 
-	altcoin_deps = ('txfile_alt',)
+	altcoin_deps = ('txfile_alt', 'txfile_alt_legacy')
 
 	async def tx(self,name,ut):
 		qmsg('  Testing NewTX initializer')
@@ -53,9 +58,33 @@ class unit_tests:
 		qmsg('  OK')
 		return True
 
-	async def txfile(self,name,ut):
+	async def txfile(self, name, ut):
 		return await do_txfile_test(
 			'Bitcoin',
+			(
+				'tx/7A8157[6.65227,34].rawtx',
+				'tx/BB3FD2[7.57134314,123].sigtx',
+				'tx/0A869F[1.23456,32].regtest.asubtx',
+			),
+			check = True
+		)
+
+	async def txfile_alt(self, name, ut):
+		return await do_txfile_test(
+			'altcoins',
+			(
+				'tx/C09D73-LTC[981.73747,2000].testnet.rawtx',
+				'tx/91060A-BCH[1.23456].regtest.arawtx',
+				'tx/D850C6-MM1[43.21,50000].subtx', # token tx
+			),
+			# token resolved by tracking wallet under data_dir:
+			cfg = Config({'data_dir': 'test/ref/data_dir'}),
+			check = True
+		)
+
+	async def txfile_legacy(self, name, ut):
+		return await do_txfile_test(
+			'Bitcoin - legacy file format',
 			(
 				'0B8D5A[15.31789,14,tl=1320969600].rawtx',
 				'542169[5.68152,34].sigtx',
@@ -64,9 +93,9 @@ class unit_tests:
 			)
 		)
 
-	async def txfile_alt(self,name,ut):
+	async def txfile_alt_legacy(self, name, ut):
 		return await do_txfile_test(
-			'altcoins',
+			'altcoins - legacy file format',
 			(
 				'460D4D-BCH[10.19764,tl=1320969600].rawtx',
 				'ethereum/5881D2-MM1[1.23456,50000].rawtx',