mmnode-feeview: display improvements, support BCH, add test

Testing/demo:

    $ test/test.py -e regtest.feeview
    $ test/test.py -e --coin=ltc regtest.feeview
    $ test/test.py -e --coin=bch regtest.feeview

mmgen_node_tools/data/version

@@ -1 +1 @@
-3.1.dev10
+3.1.dev11

mmgen_node_tools/main_feeview.py

@@ -52,7 +52,8 @@ opts.init({
 -h, --help            Print this help message
 --, --longhelp        Print help message for long options (common options)
 -c, --include-current Include current bracket’s TXs in cumulative MB value
--d, --detail          Same as --ranges --show-mb-col --precision=6
+-d, --outdir=D        Write log data to directory 'D'
+-D, --detail          Same as --ranges --show-mb-col --precision=6
 -e, --show-empty      Show all fee brackets, including empty ones
 -i, --ignore-below=B  Ignore fee brackets with less than 'B' bytes of TXs
 -l, --log             Log JSON-RPC mempool data to 'mempool.json'
@@ -100,18 +101,24 @@ class fee_bracket:
 def log(data,fn):
 	import json
 	from mmgen.rpc import json_encoder
-	open(fn,'w').write(json.dumps(data,cls=json_encoder,sort_keys=True,indent=4))
+	from mmgen.fileutil import write_data_to_file
+	write_data_to_file(
+		outfile = fn,
+		data = json.dumps(data,cls=json_encoder,sort_keys=True,indent=4),
+		desc = 'mempool',
+		ask_overwrite = False )
 
 def create_data(coin_amt,mempool):
 	out = [fee_bracket(fee_brackets[i],fee_brackets[i-1] if i else 0) for i in range(len(fee_brackets))]
 
 	# populate fee brackets:
+	size_key = 'size' if proto.coin == 'BCH' else 'vsize'
 	for tx in mempool.values():
 		fee = coin_amt(tx['fees']['base']).to_unit('satoshi')
-		vsize = tx['vsize']
+		size = tx[size_key]
 		for bracket in out:
-			if fee / vsize < bracket.top:
-				bracket.tx_bytes += vsize
+			if fee / size < bracket.top:
+				bracket.tx_bytes += size
 				break
 
 	# remove empty top brackets:
@@ -132,19 +139,23 @@ def create_data(coin_amt,mempool):
 
 	return out
 
-def gen_header(host,blockcount):
-	yield('MEMPOOL FEE STRUCTURE ({})\n{} UTC\nBlock {}'.format(
-		host,
-		make_timestr(),
-		blockcount,
-		))
+def gen_header(host,mempool,blockcount):
+
+	yield(fmt(f"""
+		Mempool Fee Structure
+		Date:     {make_timestr()} UTC
+		Host:     {host}
+		Network:  {proto.coin.upper()} {proto.network.upper()}
+		Block:    {blockcount}
+		TX count: {len(mempool)}
+		""")).strip()
 
 	if opt.show_empty:
 		yield('Displaying all fee brackets')
 	elif opt.ignore_below:
-		yield('Ignoring fee brackets with less than {} bytes ({})'.format(
+		yield('Ignoring fee brackets with less than {:,} bytes ({})'.format(
 			ignore_below,
-			int2bytespec(ignore_below,'MB','0.6'),
+			int2bytespec(ignore_below,'MB','0.6',strip=True,add_space=True),
 			))
 
 	if opt.include_current:
@@ -187,6 +198,7 @@ def gen_body(data):
 async def main():
 
 	from mmgen.protocol import init_proto_from_opts
+	global proto
 	proto = init_proto_from_opts(need_amt=True)
 
 	from mmgen.rpc import rpc_init
@@ -199,7 +211,10 @@ async def main():
 
 	data = create_data(proto.coin_amt,mempool)
 	stdout_or_pager(
-		'\n'.join(gen_header(c.host,await c.call('getblockcount'))) + '\n' +
+		'\n'.join(gen_header(
+			c.host,
+			mempool,
+			await c.call('getblockcount') )) + '\n\n' +
 		'\n'.join(gen_body(data)) + '\n' )
 
 async_run(main())

setup.cfg

@@ -23,7 +23,7 @@ python_requires = >=3.7
 include_package_data = True
 
 install_requires =
-	mmgen>=13.3.dev11
+	mmgen>=13.3.dev12
 
 packages =
 	mmgen_node_tools

test/test-release.d/cfg.sh

@@ -11,7 +11,7 @@
 # Testing status
 #  mmnode-addrbal             OK
 #  mmnode-blocks-info         OK
-#  mmnode-feeview             -
+#  mmnode-feeview             OK
 #  mmnode-halving-calculator  OK
 #  mmnode-netrate             -
 #  mmnode-peerblocks          OK

test/test_py_d/ts_regtest.py

@@ -47,6 +47,7 @@ class TestSuiteRegtest(TestSuiteBase):
 		('subgroup.fund_addrbal',       []),
 		('subgroup.addrbal',            ['fund_addrbal']),
 		('subgroup.blocks_info',        ['addrbal']),
+		('subgroup.feeview',            []),
 		('stop',                        'stopping regtest daemon'),
 	)
 	cmd_subgroups = {
@@ -84,6 +85,18 @@ class TestSuiteRegtest(TestSuiteBase):
 		('blocks_info3',              "blocks-info +100"),
 		('blocks_info4',              "blocks-info --miner-info --fields=all --stats=all +1"),
 	),
+	'feeview': (
+		"'mmnode-feeview' script",
+		('feeview_setup',             'setting up feeview test'),
+		('feeview1',                  "'mmnode-feeview'"),
+		('feeview2',                  "'mmnode-feeview --columns=40 --include-current'"),
+		('feeview3',                  "'mmnode-feeview --precision=6'"),
+		('feeview4',                  "'mmnode-feeview --detail'"),
+		('feeview5',                  "'mmnode-feeview --show-empty --log'"),
+		('feeview6',                  "'mmnode-feeview --ignore-below=1MB'"),
+		('feeview7',                  "'mmnode-feeview --ignore-below=20kB'"),
+		('feeview8',                  "'mmnode-feeview' (empty mempool)"),
+	),
 	}
 
 	def __init__(self,trunner,cfgs,spawn):
@@ -247,6 +260,128 @@ class TestSuiteRegtest(TestSuiteBase):
 			'Next diff adjust: 2016'
 		])
 
+	async def feeview_setup(self):
+
+		def create_pairs(nPairs):
+
+			from mmgen.tool.api import tool_api
+			from collections import namedtuple
+
+			t = tool_api()
+			t.init_coin(self.proto.coin,self.proto.network)
+			t.addrtype = 'compressed' if self.proto.coin == 'BCH' else 'bech32'
+			wp = namedtuple('wifaddrpair',['wif','addr'])
+
+			def gen():
+				for n in range(1,nPairs+1):
+					wif = t.hex2wif(f'{n:064x}')
+					yield wp( wif, t.wif2addr(wif) )
+
+			return list(gen())
+
+		def gen_fees(n_in,low,high):
+
+			# very approximate tx size estimation:
+			ibytes,wbytes,obytes = (148,0,34) if self.proto.coin == 'BCH' else (43,108,31)
+			x = (ibytes + (wbytes//4) + (obytes * nPairs)) * self.proto.coin_amt(self.proto.coin_amt.satoshi)
+
+			n = n_in - 1
+			vmax = high - low
+
+			for i in range(n_in):
+				yield (low + (i/n)**6 * vmax) * x
+
+		async def do_tx(inputs,outputs,wif):
+			tx_hex = await r.rpc_call( 'createrawtransaction', inputs, outputs )
+			tx = await r.rpc_call( 'signrawtransactionwithkey', tx_hex, [wif], [], self.proto.sighash_type )
+			assert tx['complete'] == True
+			return tx['hex']
+
+		async def do_tx1():
+			us = await r.rpc_call('listunspent',wallet='miner')
+			fee = self.proto.coin_amt('0.001')
+			outputs = {p.addr:tx1_amt for p in pairs[:nTxs]}
+			outputs.update({burn_addr: us[0]['amount'] - (tx1_amt*nTxs) - fee})
+			return await do_tx(
+				[{ 'txid': us[0]['txid'], 'vout': 0 }],
+				outputs,
+				r.miner_wif )
+
+		async def do_tx2(tx,pairno):
+			fee = fees[pairno]
+			outputs = {p.addr:tx2_amt for p in pairs}
+			outputs.update({burn_addr: tx1_amt - (tx2_amt*len(pairs)) - fee})
+			return await do_tx(
+				[{ 'txid': tx['txid'], 'vout': pairno }],
+				outputs,
+				pairs[pairno].wif )
+
+		async def do_txs(tx_in):
+			for pairno in range(nTxs):
+				tx_hex = await do_tx2(tx_in,pairno)
+				await r.rpc_call('sendrawtransaction',tx_hex)
+
+		self.spawn('',msg_only=True)
+
+		r = self.regtest
+		nPairs = 100
+		nTxs = 25
+		tx1_amt = self.proto.coin_amt('{:0.4f}'.format(24 / (nTxs+1))) # 25 BTC subsidy
+		tx2_amt = self.proto.coin_amt('0.0001')
+
+		imsg(f'Creating {nPairs} key-address pairs')
+		pairs = create_pairs(nPairs+1)
+		burn_addr = pairs.pop()[1]
+
+		imsg(f'Creating funding transaction with {nTxs} outputs of value {tx1_amt} {self.proto.coin}')
+		tx1_hex = await do_tx1()
+
+		imsg(f'Relaying funding transaction')
+		await r.rpc_call('sendrawtransaction',tx1_hex)
+
+		imsg(f'Mining a block')
+		await r.generate(1,silent=True)
+
+		imsg(f'Generating fees for mempool transactions')
+		fees = list(gen_fees(nTxs,2,120))
+
+		imsg(f'Creating and relaying {nTxs} mempool transactions with {nPairs} outputs each')
+		await do_txs(await r.rpc_call('decoderawtransaction',tx1_hex))
+
+		return 'ok'
+
+	def _feeview(self,args,expect_list=[]):
+		t = self.spawn('mmnode-feeview',args)
+		if expect_list:
+			t.match_expect_list(expect_list)
+		return t
+
+	def feeview1(self):
+		return self._feeview([])
+
+	def feeview2(self):
+		return self._feeview(['--columns=40','--include-current'])
+
+	def feeview3(self):
+		return self._feeview(['--precision=6'])
+
+	def feeview4(self):
+		return self._feeview(['--detail'])
+
+	def feeview5(self):
+		return self._feeview(['--show-empty','--log',f'--outdir={self.tmpdir}'])
+
+	def feeview6(self):
+		return self._feeview(['--ignore-below=1MB'])
+
+	def feeview7(self):
+		return self._feeview(['--ignore-below=4kB'])
+
+	async def feeview8(self):
+		imsg('Clearing mempool')
+		await self.regtest.generate(1,silent=True)
+		return self._feeview([])
+
 	def stop(self):
 		if opt.no_daemon_stop:
 			self.spawn('',msg_only=True)