From 1a9184debc2f09205abd13a406619e368d7c54dc Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 20 Jul 2024 16:45:56 +0000 Subject: [PATCH 01/51] version bump --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 15a2799..49cbc5f 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.3.0 +3.4.dev0 diff --git a/setup.cfg b/setup.cfg index 1945c49..def5f65 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet==14.1.0 + mmgen-wallet>=14.1.0 pyyaml yahooquery From c9d6d8f0475f003a7a86f7d5d005ed94495d5bbd Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 22 Sep 2024 16:10:08 +0000 Subject: [PATCH 02/51] Version 3.4.0 - released concurrently with MMGen Wallet v15.0.0 - adds macOS support --- README.md | 6 +++--- mmgen_node_tools/data/keywords | 2 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 16 +++++++++++++--- test/cmdtest_py_d/cfg.py | 7 ++++++- test/cmdtest_py_d/ct_main.py | 2 +- test/init.sh | 13 ++++++++++--- 7 files changed, 35 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f72051f..2751da7 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ If installing as user (without venv), make sure that `~/.local/bin` is in `PATH` > $ python3 -m pip install --no-deps yahooquery > ``` -#### Linux: +#### Linux, macOS: -> Install the [required MMGen Wallet packages][7] for your Linux distribution. +> Install some [required packages][7] with your package manager and pip. ### Stable version: @@ -100,5 +100,5 @@ Donate: [5]: https://github.com/mmgen/mmgen-wallet/wiki/MMGen-Signing-Keys [6]: https://github.com/mmgen/mmgen-wallet/ -[7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Linux +[7]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-Wallet-on-Linux-or-macOS [8]: https://github.com/mmgen/mmgen-wallet/wiki/Install-MMGen-on-Microsoft-Windows#a_m diff --git a/mmgen_node_tools/data/keywords b/mmgen_node_tools/data/keywords index 3c3c563..f26b502 100644 --- a/mmgen_node_tools/data/keywords +++ b/mmgen_node_tools/data/keywords @@ -1 +1 @@ -Bitcoin, BTC, Ethereum, ETH, Monero, XMR, ERC20, cryptocurrency, wallet, BIP32, cold storage, offline, online, spending, open-source, command-line, Python, Linux, Bitcoin Core, bitcoind, hd, deterministic, hierarchical, secure, anonymous, Electrum, seed, mnemonic, brainwallet, Scrypt, utility, script, scriptable, blockchain, raw, transaction, permissionless, console, terminal, curses, ansi, color, tmux, remote, client, daemon, RPC, json, entropy, xterm, rxvt, PowerShell, MSYS, MSYS2, MinGW, MinGW64, MSWin, Armbian, Raspbian, Raspberry Pi, Orange Pi, BCash, BCH, Litecoin, LTC, altcoin, ZEC, Zcash, DASH, Dashpay, SHA256Compress, monerod, EMC, Emercoin, token, deploy, contract, gas, fee, smart contract, solidity, Parity, OpenEthereum, testnet, devmode, Kovan +Bitcoin, BTC, Ethereum, ETH, Monero, XMR, ERC20, cryptocurrency, wallet, cold storage, offline, signing, online, security, privacy, spending, financial, investment, open-source, command-line, Python, Linux, Microsoft Windows, macOS, Bitcoin Core, BIP32, BIP39, BIP44, BIP69, BIP125, bitcoind, hd, deterministic, hierarchical, secure, anonymous, Electrum, seed, mnemonic, brainwallet, Scrypt, utility, script, scriptable, blockchain, raw, transaction, permissionless, console, terminal, curses, ansi, color, tmux, remote, client, daemon, RPC, json, entropy, xterm, rxvt, MSYS2, MSWin, Armbian, Raspbian, Raspberry Pi, Orange Pi, Rock Pi, BCash, Bitcoin Cash Node, BCH, Litecoin, LTC, altcoin, ZEC, Zcash, SHA256Compress, monerod, token, deploy, contract, gas, fee, smart contract, solidity, Parity, OpenEthereum, testnet, devmode, regtest diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 49cbc5f..1809198 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.4.dev0 +3.4.0 diff --git a/setup.cfg b/setup.cfg index def5f65..9a0ab04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,19 +8,29 @@ author = The MMGen Project author_email = mmgen@tuta.io url = https://github.com/mmgen/mmgen-node-tools license = GNU GPL v3 -platforms = Linux, Armbian, Raspbian, MS Windows +platforms = Linux, Armbian, Raspbian, MS Windows, MacOS keywords = file: mmgen_node_tools/data/keywords project_urls = Website = https://mmgen.org Bug Tracker = https://github.com/mmgen/mmgen-node-tools/issues classifiers = - Programming Language :: Python :: 3 License :: OSI Approved :: GNU General Public License v3 (GPLv3) Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows + Operating System :: MacOS Environment :: Console + Programming Language :: Python :: 3 + Programming Language :: C + Framework :: AsyncIO + Framework :: aiohttp Topic :: Office/Business :: Financial Topic :: Security :: Cryptography + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Utilities + Intended Audience :: Developers + Intended Audience :: End Users/Desktop + Intended Audience :: Financial and Insurance Industry + Intended Audience :: System Administrators Development Status :: 5 - Production/Stable [options] @@ -28,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=14.1.0 + mmgen-wallet==15.0.0 pyyaml yahooquery diff --git a/test/cmdtest_py_d/cfg.py b/test/cmdtest_py_d/cfg.py index 96bfe9f..ef52880 100755 --- a/test/cmdtest_py_d/cfg.py +++ b/test/cmdtest_py_d/cfg.py @@ -30,4 +30,9 @@ cfgs = { } def fixup_cfgs(): - pass + import os + + for k in cfgs: + cfgs[k]['tmpdir'] = os.path.join('test', 'tmp', str(k)) + +fixup_cfgs() diff --git a/test/cmdtest_py_d/ct_main.py b/test/cmdtest_py_d/ct_main.py index 7c38bcf..d52a2e3 100755 --- a/test/cmdtest_py_d/ct_main.py +++ b/test/cmdtest_py_d/ct_main.py @@ -96,4 +96,4 @@ class CmdTestMain(CmdTestBase): def peerblocks3(self): return self.peerblocks2( ['--columns=80'], - pexpect_spawn = sys.platform != 'win32' ) + pexpect_spawn = sys.platform == 'linux') diff --git a/test/init.sh b/test/init.sh index 51e4e9e..a379adf 100755 --- a/test/init.sh +++ b/test/init.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # mmgen = Multi-Mode GENerator, a command-line cryptocurrency wallet # Copyright (C)2013-2022 The MMGen Project @@ -18,6 +18,12 @@ trap 'echo -e "${GREEN}Exiting at user request$RESET"; exit' INT trap 'echo -e "${RED}Node Tools test suite initialization exited with error (line $BASH_LINENO) $RESET"' ERR umask 0022 +for i in '-c' '-f'; do + stat $i %i / >/dev/null 2>&1 && stat_fmt_opt=$i +done + +[ "$stat_fmt_opt" ] || { echo 'No suitable ‘stat’ binary found. Cannot proceed'; exit; } + STDOUT_DEVNULL='>/dev/null' STDERR_DEVNULL='2>/dev/null' @@ -77,6 +83,7 @@ create_test_links() { test/overlay/__init__.py symbolic test/overlay/fakemods/mmgen symbolic test/__init__.py symbolic + test/clean.py symbolic test/cmdtest.py hard test/unit_tests.py hard test/test-release.sh symbolic @@ -87,7 +94,7 @@ create_test_links() { while read path type; do [ "$path" ] || continue pfx=$(echo $path | sed -r 's/[^/]//g' | sed 's/\//..\//g') - symlink_arg=$(if [ $type == 'symbolic' ]; then echo --symbolic; fi) + symlink_arg=$(if [ $type == 'symbolic' ]; then echo -s; fi) target="$wallet_repo/$path" if [ ! -e "$target" ]; then echo "Target path $target is missing! Cannot proceed" @@ -99,7 +106,7 @@ create_test_links() { [ "$VERBOSE" ] && printf "$fs" "Deleting" "symbolic link:" $path $target rm -rf $path elif [ -e $path ]; then - if [ "$(stat --printf=%i $path)" -ne "$(stat --printf=%i $target)" ]; then + if [ "$(stat $stat_fmt_opt %i $path)" -ne "$(stat $stat_fmt_opt %i $target)" ]; then [ "$VERBOSE" ] && printf "$fs" "Deleting" "stale hard link:" $path "?" rm -rf $path fi From 92fdfc047e213aed42fdb7a71c29aef32707404f Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 29 Sep 2024 14:29:10 +0000 Subject: [PATCH 03/51] Support BCH cashaddr format --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 6 +++--- setup.cfg | 2 +- test/cmdtest_py_d/ct_regtest.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1809198..3f8a888 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.4.0 +3.5.dev1 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 7af2b77..e78a53e 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -38,7 +38,7 @@ def do_output(proto,addr_data,blk_hdrs): indent = ' ' * (col1w + 2) for n,(addr,unspents) in enumerate(addr_data.items(),1): - Msg(f'\n{n:{col1w}}) Address: {addr.hl()}') + Msg(f'\n{n:{col1w}}) Address: {addr.hl(addr.view_pref)}') if unspents: heights = { u['height'] for u in unspents } @@ -93,7 +93,7 @@ def do_output_tabular(proto,addr_data,blk_hdrs): if unspents: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(width=max_addrw,color=True), + a = addr.fmt(addr.view_pref, width=max_addrw, color=True), u = red(str(len(unspents)).rjust(5)), b = unspents[0]['height'], t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), @@ -104,7 +104,7 @@ def do_output_tabular(proto,addr_data,blk_hdrs): else: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(width=max_addrw,color=True), + a = addr.fmt(addr.view_pref, width=max_addrw, color=True), u = ' -', b = '-', t = '', diff --git a/setup.cfg b/setup.cfg index 9a0ab04..875bb01 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet==15.0.0 + mmgen-wallet>=15.1.dev1 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index c89f8fa..ee59b31 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -112,7 +112,7 @@ class CmdTestRegtest(CmdTestBase): if cfg._proto.testnet: die(2,'--testnet and --regtest options incompatible with regtest test suite') self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) - self.addrs = gen_addrs(self.proto,'regtest',[1,2,3,4,5]) + self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto,'regtest',[1,2,3,4,5])] self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' self.regtest = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) From cb42eaa8cdc602be0d3718d73331b94745558d36 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 8 Oct 2024 13:19:15 +0000 Subject: [PATCH 04/51] update for MMGen Wallet 15.1.dev3 --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 2 +- setup.cfg | 2 +- test/cmdtest_py_d/ct_misc.py | 2 +- test/cmdtest_py_d/ct_regtest.py | 93 +++++++++++++++++---------------- 5 files changed, 53 insertions(+), 48 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 3f8a888..ecc3a64 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev1 +3.5.dev2 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index c3f7325..351cb1e 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -234,7 +234,7 @@ from mmgen.util import fmt_list,fmt_dict from mmgen.cfg import Config from . import Ticker -gcfg = Config( opts_data=opts_data, do_post_init=True ) +gcfg = Config(opts_data=opts_data, caller_post_init=True) Ticker.make_cfg(gcfg) diff --git a/setup.cfg b/setup.cfg index 875bb01..9b82f69 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev1 + mmgen-wallet>=15.1.dev3 pyyaml yahooquery diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_py_d/ct_misc.py index 8127e1d..6ac7d37 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_py_d/ct_misc.py @@ -50,7 +50,7 @@ class CmdTestHelp(CmdTestBase): return t def longhelpscreens(self): - return self.helpscreens(arg='--longhelp',expect='USAGE:.*LONG OPTIONS:') + return self.helpscreens(arg='--longhelp',expect='USAGE:.*GLOBAL OPTIONS:') class CmdTestScripts(CmdTestBase): 'scripts not requiring a coin daemon' diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index ee59b31..0807a16 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -35,7 +35,6 @@ class CmdTestRegtest(CmdTestBase): 'various operations via regtest mode' networks = ('btc','ltc','bch') passthru_opts = ('coin',) - extra_spawn_args = ['--regtest=1'] tmpdir_nums = [1] color = True deterministic = False @@ -177,57 +176,57 @@ class CmdTestRegtest(CmdTestBase): def sendto2(self): return self.sendto(self.addrs[0],'0.234') def sendto3(self): return self.sendto(self.addrs[1],'0.345') - def addrbal(self,args,expect_list): - t = self.spawn('mmnode-addrbal',args) + def addrbal(self, args, expect_list): + t = self.spawn('mmnode-addrbal', args2 + args) t.match_expect_list(expect_list) return t def addrbal_single(self): return self.addrbal( - args2 + [self.addrs[0]], + [self.addrs[0]], [ f'Balance: 0.357 {cfg.coin}', '2 unspent outputs in 2 blocks', - '394','0.123', - '395','0.234' + '394', '0.123', + '395', '0.234' ]) def addrbal_multiple(self): return self.addrbal( - args2 + [self.addrs[1],self.addrs[0]], + [self.addrs[1], self.addrs[0]], [ - '396','0.345', - '394','0.123', - '395','0.234' + '396', '0.345', + '394', '0.123', + '395', '0.234' ]) def addrbal_multiple_tabular1(self): return self.addrbal( - args2 + ['--tabular',self.addrs[1],self.addrs[0]], + ['--tabular', self.addrs[1], self.addrs[0]], [ - self.addrs[1] + ' 1 396','0.345', - self.addrs[0] + ' 2 395','0.357' + self.addrs[1] + ' 1 396', '0.345', + self.addrs[0] + ' 2 395', '0.357' ]) def addrbal_multiple_tabular2(self): return self.addrbal( - args2 + ['--tabular','--first-block',self.addrs[1],self.addrs[0]], + ['--tabular', '--first-block', self.addrs[1], self.addrs[0]], [ - self.addrs[1] + ' 1 396','396','0.345', - self.addrs[0] + ' 2 394','395','0.357' + self.addrs[1] + ' 1 396', '396', '0.345', + self.addrs[0] + ' 2 394', '395', '0.357' ]) def addrbal_nobal1(self): return self.addrbal( - args2 + [self.addrs[2]], ['Address has no balance'] ) + [self.addrs[2]], ['Address has no balance']) def addrbal_nobal2(self): return self.addrbal( - args2 + [self.addrs[2],self.addrs[3]], ['Addresses have no balances'] ) + [self.addrs[2], self.addrs[3]], ['Addresses have no balances']) def addrbal_nobal3(self): return self.addrbal( - args2 + [self.addrs[4],self.addrs[0],self.addrs[3]], + [self.addrs[4], self.addrs[0], self.addrs[3]], [ 'No balance', '2 unspent outputs in 2 blocks', @@ -237,7 +236,7 @@ class CmdTestRegtest(CmdTestBase): def addrbal_nobal3_tabular1(self): return self.addrbal( - args2 + ['--tabular',self.addrs[4],self.addrs[0],self.addrs[3]], + ['--tabular', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - -', self.addrs[0] + ' 2 395','0.357', @@ -246,7 +245,7 @@ class CmdTestRegtest(CmdTestBase): def addrbal_nobal3_tabular2(self): return self.addrbal( - args2 + ['--tabular','--first-block',self.addrs[4],self.addrs[0],self.addrs[3]], + ['--tabular', '--first-block', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - - -', self.addrs[0] + ' 2 394','395','0.357', @@ -254,39 +253,45 @@ class CmdTestRegtest(CmdTestBase): ]) def blocks_info(self,args,expect_list): - t = self.spawn('mmnode-blocks-info',args) + t = self.spawn('mmnode-blocks-info', args1 + args) t.match_expect_list(expect_list) return t def blocks_info1(self): - return self.blocks_info( args1 + ['--help'], ['USAGE:','OPTIONS:']) + return self.blocks_info( + ['--help'], + ['USAGE:','OPTIONS:']) def blocks_info2(self): - return self.blocks_info( args1, [ - 'Current height: 396', - ]) + return self.blocks_info( + [], + ['Current height: 396']) def blocks_info3(self): - return self.blocks_info( args1 + ['+100'], [ - 'Range: 297-396', - 'Current height: 396', - 'Next diff adjust: 2016' - ]) + return self.blocks_info( + ['+100'], + [ + 'Range: 297-396', + 'Current height: 396', + 'Next diff adjust: 2016' + ]) def blocks_info4(self): n1,i1,o1,n2,i2,o2 = (2,1,3,6,3,9) if cfg.coin == 'BCH' else (2,1,4,6,3,12) - return self.blocks_info( args1 + ['--miner-info','--fields=all','--stats=all','+3'], [ - 'Averages', - f'nTx: {n1}', - f'Inputs: {i1}', - f'Outputs: {o1}', - 'Totals', - f'nTx: {n2}', - f'Inputs: {i2}', - f'Outputs: {o2}', - 'Current height: 396', - 'Next diff adjust: 2016' - ]) + return self.blocks_info( + ['--miner-info', '--fields=all', '--stats=all', '+3'], + [ + 'Averages', + f'nTx: {n1}', + f'Inputs: {i1}', + f'Outputs: {o1}', + 'Totals', + f'nTx: {n2}', + f'Inputs: {i2}', + f'Outputs: {o2}', + 'Current height: 396', + 'Next diff adjust: 2016' + ]) async def feeview_setup(self): @@ -380,7 +385,7 @@ class CmdTestRegtest(CmdTestBase): return 'ok' def _feeview(self,args,expect_list=[]): - t = self.spawn('mmnode-feeview',args) + t = self.spawn('mmnode-feeview', args1 + args) if expect_list: t.match_expect_list(expect_list) return t From dee57d88869a8c9c6c22bbf01bc07de35de7545e Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 18 Oct 2024 10:33:47 +0000 Subject: [PATCH 05/51] update for MMGen Wallet CoinAmt changes --- mmgen_node_tools/BlocksInfo.py | 27 +++++++++++++++++++-------- mmgen_node_tools/main_addrbal.py | 4 ++-- test/cmdtest_py_d/ct_regtest.py | 9 +++++---- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 40cc6e5..26e6cfa 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -23,6 +23,7 @@ mmgen_node_tools.BlocksInfo: Display information about a block or range of block import re,json from collections import namedtuple from time import strftime,gmtime +from decimal import Decimal from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int from mmgen.rpc import json_encoder @@ -241,14 +242,14 @@ class BlocksInfo: 'tf': lambda arg: '{:.8f}'.format(arg * from_satoshi), 'su': lambda arg: str(arg * from_satoshi).rstrip('0').rstrip('.'), 'fe': lambda arg: str(arg), - 'di': lambda arg: '{:.2e}'.format(arg), + 'di': lambda arg: '{:.2e}'.format(Decimal(arg)), } if self.cfg.coin == 'BCH': self.fmt_funcs.update({ 'su': lambda arg: str(arg).rstrip('0').rstrip('.'), - 'fe': lambda arg: str(int(arg * to_satoshi)), - 'tf': lambda arg: '{:.8f}'.format(arg), + 'fe': lambda arg: str(int(Decimal(arg) * to_satoshi)), + 'tf': lambda arg: '{:.8f}'.format(Decimal(arg)), }) self.fnames = tuple( @@ -547,7 +548,7 @@ class BlocksInfo: else: sample_blks = min(min_sample_blks,self.tip) start_hdr = await c.call('getblockheader',await c.call('getblockhash',self.tip-sample_blks)) - diff_adj = float(tip_hdr['difficulty'] / start_hdr['difficulty']) + diff_adj = Decimal(tip_hdr['difficulty']) / Decimal(start_hdr['difficulty']) time1 = rel_hdr['time'] - start_hdr['time'] time2 = tip_hdr['time'] - rel_hdr['time'] bdi = ((time1 * diff_adj) + time2) / sample_blks @@ -570,17 +571,26 @@ class BlocksInfo: 'sample_blks': ('{}', sample_blks) } ), - ('Cur difficulty: {}', 'cur_diff', '{:.2e}', tip_hdr['difficulty']), + ('Cur difficulty: {}', 'cur_diff', '{:.2e}', Decimal(tip_hdr['difficulty'])), ('Est. diff adjust: {}%', 'est_diff_adjust_pct', '{:+.2f}', ((600 / bdi) - 1) * 100), )) + def sum_field_avg(self, field): + return self.sum_field_total(field) // len(self.res) + + def sum_field_total(self, field): + if isinstance(getattr(self.res[0], field), str): + return sum(Decimal(getattr(block, field)) for block in self.res) + else: + return sum(getattr(block, field) for block in self.res) + async def create_col_avg_stats(self): def gen(): for field in self.fnames: if field in self.avg_stats_skip: yield ( field, ('{}','') ) else: - ret = sum(getattr(block,field) for block in self.res) // len(self.res) + ret = self.sum_field_avg(field) func = self.fields[field].fmt_func yield ( field, ( (self.fmt_funcs[func] if func else '{}'), ret )) if not self.header_printed: @@ -590,9 +600,10 @@ class BlocksInfo: def avg_stats_data(self,data,spec_conv,spec_val): coin = self.rpc.proto.coin + return data( hdr = 'Averages for processed blocks:', - func = lambda field: sum(getattr(block,field) for block in self.res) // len(self.res), + func = self.sum_field_avg, spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}' }, spec_convs = { 'interval': spec_conv(0, lambda arg: secs_to_ms(arg)), @@ -616,7 +627,7 @@ class BlocksInfo: coin = self.rpc.proto.coin return data( hdr = 'Totals for processed blocks:', - func = lambda field: sum(getattr(block,field) for block in self.res), + func = self.sum_field_total, spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}', 'reward': f' {coin}' }, spec_convs = { 'interval': spec_conv(0, lambda arg: secs_to_dhms(arg)), diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index e78a53e..7dd37a3 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -44,7 +44,7 @@ def do_output(proto,addr_data,blk_hdrs): heights = { u['height'] for u in unspents } Msg('{}Balance: {}'.format( indent, - proto.coin_amt(sum(u['amount'] for u in unspents)).hl2(unit=True,fs='{:,}') )), + sum(proto.coin_amt(u['amount']) for u in unspents).hl2(unit=True, fs='{:,}'))), Msg('{}{} unspent output{} in {} block{}'.format( indent, red(str(len(unspents))), @@ -99,7 +99,7 @@ def do_output_tabular(proto,addr_data,blk_hdrs): t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), B = unspents[-1]['height'], T = make_timestr( blk_hdrs[unspents[-1]['height']]['time'] ), - A = proto.coin_amt(sum(u['amount'] for u in unspents)).fmt(color=True,iwidth=7,prec=8) + A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(color=True, iwidth=7, prec=8) )) else: Msg(fs.format( diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_py_d/ct_regtest.py index 0807a16..f1b2774 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_py_d/ct_regtest.py @@ -13,6 +13,7 @@ test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite """ import sys,os +from decimal import Decimal from mmgen.util import msg_r,die,gmsg from mmgen.protocol import init_proto @@ -316,13 +317,13 @@ class CmdTestRegtest(CmdTestBase): # very approximate tx size estimation: ibytes,wbytes,obytes = (148,0,34) if self.proto.coin == 'BCH' else (43,108,31) - x = (ibytes + (wbytes//4) + (obytes * nPairs)) * self.proto.coin_amt(self.proto.coin_amt.satoshi) + x = (ibytes + (wbytes//4) + (obytes * nPairs)) * self.proto.coin_amt.satoshi n = n_in - 1 vmax = high - low for i in range(n_in): - yield (low + (i/n)**6 * vmax) * x + yield Decimal(low + (i/n)**6 * vmax) * x async def do_tx(inputs,outputs,wif): tx_hex = await r.rpc_call( 'createrawtransaction', inputs, outputs ) @@ -335,14 +336,14 @@ class CmdTestRegtest(CmdTestBase): tx_input = us[7] # 25 BTC in coinbase -- us[0] could have < 25 BTC fee = self.proto.coin_amt('0.001') outputs = {p.addr:tx1_amt for p in pairs[:nTxs]} - outputs.update({burn_addr: tx_input['amount'] - (tx1_amt*nTxs) - fee}) + outputs.update({burn_addr: self.proto.coin_amt(tx_input['amount']) - (tx1_amt*nTxs) - fee}) return await do_tx( [{ 'txid': tx_input['txid'], 'vout': 0 }], outputs, await r.miner_wif) async def do_tx2(tx,pairno): - fee = fees[pairno] + fee = self.proto.coin_amt(fees[pairno], from_decimal=True) outputs = {p.addr:tx2_amt for p in pairs} outputs.update({burn_addr: tx1_amt - (tx2_amt*len(pairs)) - fee}) return await do_tx( From b9b289c56db227cfc42e541dffc7767bce129c78 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 18 Oct 2024 10:33:48 +0000 Subject: [PATCH 06/51] directory rename: - test/cmdtest_py_d -> test/cmdtest_d --- MANIFEST.in | 2 +- test/{cmdtest_py_d => cmdtest_d}/cfg.py | 2 +- test/{cmdtest_py_d => cmdtest_d}/ct_main.py | 2 +- test/{cmdtest_py_d => cmdtest_d}/ct_misc.py | 2 +- test/{cmdtest_py_d => cmdtest_d}/ct_regtest.py | 2 +- test/init.sh | 4 ++-- test/test-release.d/cfg.sh | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) rename test/{cmdtest_py_d => cmdtest_d}/cfg.py (93%) rename test/{cmdtest_py_d => cmdtest_d}/ct_main.py (96%) rename test/{cmdtest_py_d => cmdtest_d}/ct_misc.py (99%) rename test/{cmdtest_py_d => cmdtest_d}/ct_regtest.py (99%) diff --git a/MANIFEST.in b/MANIFEST.in index 370da07..1fe78fa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,6 @@ include mmgen_node_tools/data/* include test/init.sh include test/test-release.d/*.sh include test/unit_tests_d/*.py -include test/cmdtest_py_d/*.py +include test/cmdtest_d/*.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/test/cmdtest_py_d/cfg.py b/test/cmdtest_d/cfg.py similarity index 93% rename from test/cmdtest_py_d/cfg.py rename to test/cmdtest_d/cfg.py index ef52880..c28ce5d 100755 --- a/test/cmdtest_py_d/cfg.py +++ b/test/cmdtest_d/cfg.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_py_d.cfg: configuration data for cmdtest.py +test.cmdtest_d.cfg: configuration data for cmdtest.py """ cmd_groups_altcoin = [] diff --git a/test/cmdtest_py_d/ct_main.py b/test/cmdtest_d/ct_main.py similarity index 96% rename from test/cmdtest_py_d/ct_main.py rename to test/cmdtest_d/ct_main.py index d52a2e3..3009d2f 100755 --- a/test/cmdtest_py_d/ct_main.py +++ b/test/cmdtest_d/ct_main.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-wallet """ -cmdtest_py_d.ct_main: Basic operations tests for the cmdtest.py test suite +cmdtest_d.ct_main: Basic operations tests for the cmdtest.py test suite """ import sys,time diff --git a/test/cmdtest_py_d/ct_misc.py b/test/cmdtest_d/ct_misc.py similarity index 99% rename from test/cmdtest_py_d/ct_misc.py rename to test/cmdtest_d/ct_misc.py index 6ac7d37..15b109e 100755 --- a/test/cmdtest_py_d/ct_misc.py +++ b/test/cmdtest_d/ct_misc.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_py_d.ct_misc: Miscellaneous test groups for the cmdtest.py test suite +test.cmdtest_d.ct_misc: Miscellaneous test groups for the cmdtest.py test suite """ import os,shutil diff --git a/test/cmdtest_py_d/ct_regtest.py b/test/cmdtest_d/ct_regtest.py similarity index 99% rename from test/cmdtest_py_d/ct_regtest.py rename to test/cmdtest_d/ct_regtest.py index f1b2774..24f19b9 100755 --- a/test/cmdtest_py_d/ct_regtest.py +++ b/test/cmdtest_d/ct_regtest.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_py_d.ct_regtest: Regtest tests for the cmdtest.py test suite +test.cmdtest_d.ct_regtest: Regtest tests for the cmdtest.py test suite """ import sys,os diff --git a/test/init.sh b/test/init.sh index a379adf..8f77c82 100755 --- a/test/init.sh +++ b/test/init.sh @@ -87,8 +87,8 @@ create_test_links() { test/cmdtest.py hard test/unit_tests.py hard test/test-release.sh symbolic - test/cmdtest_py_d/common.py symbolic - test/cmdtest_py_d/ct_base.py symbolic + test/cmdtest_d/common.py symbolic + test/cmdtest_d/ct_base.py symbolic cmds/mmgen-regtest symbolic ' while read path type; do diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index b371e72..7a393df 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -42,7 +42,7 @@ init_tests() { t_lint=" - $pylint --errors-only mmgen_node_tools - $pylint --errors-only test - - $pylint --errors-only --disable=relative-beyond-top-level test/cmdtest_py_d + - $pylint --errors-only --disable=relative-beyond-top-level test/cmdtest_d " d_unit="low-level subsystems" From 30772b369999c6b5deae7d4ea653d46cc09680ed Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 18 Oct 2024 10:33:48 +0000 Subject: [PATCH 07/51] Update for MMGen Wallet 15.1.dev6 --- MANIFEST.in | 2 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/init.sh | 8 +++++++- test/{unit_tests_d => modtest_d}/ut_BlocksInfo.py | 0 test/{unit_tests_d => modtest_d}/ut_dep.py | 0 test/test-release.d/cfg.sh | 12 ++++++------ 7 files changed, 16 insertions(+), 10 deletions(-) rename test/{unit_tests_d => modtest_d}/ut_BlocksInfo.py (100%) rename test/{unit_tests_d => modtest_d}/ut_dep.py (100%) diff --git a/MANIFEST.in b/MANIFEST.in index 1fe78fa..0443a99 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ include mmgen_node_tools/data/* include test/init.sh include test/test-release.d/*.sh -include test/unit_tests_d/*.py +include test/modtest_d/*.py include test/cmdtest_d/*.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index ecc3a64..f19fa82 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev2 +3.5.dev3 diff --git a/setup.cfg b/setup.cfg index 9b82f69..edbad57 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev3 + mmgen-wallet>=15.1.dev6 pyyaml yahooquery diff --git a/test/init.sh b/test/init.sh index 8f77c82..f6a6f00 100755 --- a/test/init.sh +++ b/test/init.sh @@ -77,6 +77,10 @@ create_dir_links() { done } +delete_old_stuff() { + rm -rf test/unit_tests.py +} + create_test_links() { paths=' test/include symbolic @@ -85,7 +89,7 @@ create_test_links() { test/__init__.py symbolic test/clean.py symbolic test/cmdtest.py hard - test/unit_tests.py hard + test/modtest.py hard test/test-release.sh symbolic test/cmdtest_d/common.py symbolic test/cmdtest_d/ct_base.py symbolic @@ -121,6 +125,8 @@ create_test_links() { becho 'Initializing MMGen Node Tools Test Suite' +delete_old_stuff + check_mmgen_repo || die "MMGen Wallet repository not found at $wallet_repo!" build_mmgen_extmod diff --git a/test/unit_tests_d/ut_BlocksInfo.py b/test/modtest_d/ut_BlocksInfo.py similarity index 100% rename from test/unit_tests_d/ut_BlocksInfo.py rename to test/modtest_d/ut_BlocksInfo.py diff --git a/test/unit_tests_d/ut_dep.py b/test/modtest_d/ut_dep.py similarity index 100% rename from test/unit_tests_d/ut_dep.py rename to test/modtest_d/ut_dep.py diff --git a/test/test-release.d/cfg.sh b/test/test-release.d/cfg.sh index 7a393df..a34c324 100755 --- a/test/test-release.d/cfg.sh +++ b/test/test-release.d/cfg.sh @@ -18,7 +18,7 @@ # mmnode-ticker OK # mmnode-txfind - -all_tests='unit lint misc scripts btc btc_rt bch_rt ltc_rt' +all_tests='mod lint misc scripts btc btc_rt bch_rt ltc_rt' groups_desc=" default - All tests minus the extra tests @@ -29,10 +29,10 @@ groups_desc=" " init_groups() { - dfl_tests='unit misc scripts btc btc_rt bch_rt ltc_rt' + dfl_tests='mod misc scripts btc btc_rt bch_rt ltc_rt' extra_tests='lint' - noalt_tests='unit misc scripts btc btc_rt' - quick_tests='unit misc scripts btc btc_rt' + noalt_tests='mod misc scripts btc btc_rt' + quick_tests='mod misc scripts btc btc_rt' qskip_tests='lint bch_rt ltc_rt' } @@ -45,8 +45,8 @@ init_tests() { - $pylint --errors-only --disable=relative-beyond-top-level test/cmdtest_d " - d_unit="low-level subsystems" - t_unit="- $unit_tests_py" + d_mod="low-level subsystems" + t_mod="- $modtest_py" d_misc="miscellaneous features" t_misc="- $cmdtest_py helpscreens" From bec7df1d6b5b3ea342fc28292ceaf83dc7498054 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 3 Jan 2025 16:42:11 +0300 Subject: [PATCH 08/51] support Nix and NixOS Quick Start for BTC: $ git clone https://github.com/mmgen/mmgen-wallet $ git clone https://github.com/mmgen/mmgen-node-tools $ cd mmgen-node-tools $ test/init.sh $ nix-shell --pure nix Enable altcoins and additional packages: # From the mmgen-node-tools repository root: $ mkdir -p ~/.mmgen $ cp ../mmgen-wallet/nix/user-packages.nix ~/.mmgen # ... edit ~/.mmgen/user-packages.nix as required ... $ nix-shell --pure nix For NixOS installation and other information, see: nix/README.node-tools ../mmgen-wallet/nix/README --- mmgen_node_tools/data/version | 2 +- nix/README.node-tools | 11 ++++++++ nix/default.nix | 6 +++++ nix/node-tools-packages.nix | 12 +++++++++ nix/shell.nix | 1 + nix/yahooquery-noversioning.patch | 45 +++++++++++++++++++++++++++++++ nix/yahooquery.nix | 44 ++++++++++++++++++++++++++++++ setup.cfg | 2 +- 8 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 nix/README.node-tools create mode 100644 nix/default.nix create mode 100644 nix/node-tools-packages.nix create mode 100644 nix/shell.nix create mode 100644 nix/yahooquery-noversioning.patch create mode 100644 nix/yahooquery.nix diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index f19fa82..cf8acd2 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev3 +3.5.dev4 diff --git a/nix/README.node-tools b/nix/README.node-tools new file mode 100644 index 0000000..9308181 --- /dev/null +++ b/nix/README.node-tools @@ -0,0 +1,11 @@ +Nix configuration directory for the MMGen Node Tools suite + +Usage is as described in ‘nix/README’ in the mmgen-wallet repository, with the +following differences: + + a) all commands are executed from the repository root of mmgen-node-tools + instead of mmgen-wallet + + b) for NixOS, complete the steps as described up until the rebuild step. + Copy the contents of this directory to ‘/etc/nixos/mmgen-project’ (this + will overwrite ‘default.nix’), and continue with the rebuild step. diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..7b4b214 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,6 @@ +import ( + if builtins.pathExists ./merged-packages.nix then + ./merged-packages.nix + else + ../../mmgen-wallet/nix/merged-packages.nix + ) { add_pkgs_path = ./node-tools-packages.nix; } diff --git a/nix/node-tools-packages.nix b/nix/node-tools-packages.nix new file mode 100644 index 0000000..19792c7 --- /dev/null +++ b/nix/node-tools-packages.nix @@ -0,0 +1,12 @@ +{ pkgs, python }: + +{ + system-packages = with pkgs; { + cacert = cacert; # ticker (curl) + }; + + python-packages = with python.pkgs; { + yahooquery = (pkgs.callPackage ./yahooquery.nix {}); # ticker + pyyaml = pyyaml; # ticker + }; +} diff --git a/nix/shell.nix b/nix/shell.nix new file mode 100644 index 0000000..01991d2 --- /dev/null +++ b/nix/shell.nix @@ -0,0 +1 @@ +import ../../mmgen-wallet/nix/shell.nix { add_pkgs_path = ./node-tools-packages.nix; } diff --git a/nix/yahooquery-noversioning.patch b/nix/yahooquery-noversioning.patch new file mode 100644 index 0000000..4bd3297 --- /dev/null +++ b/nix/yahooquery-noversioning.patch @@ -0,0 +1,45 @@ +diff --git a/pyproject.toml b/pyproject.toml +index 9d3fb29..399c215 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -10,24 +10,24 @@ readme = "README.md" + + [tool.poetry.dependencies] + python = ">=3.8.1,<4.0" +-requests = "^2.31.0" +-pandas = "^2.0.3" +-requests-futures = "^1.0.1" +-tqdm = "^4.65.0" +-lxml = "^4.9.3" +-selenium = {version = "^4.10.0", optional = true} +-beautifulsoup4 = "^4.12.2" ++requests = ">=2.31.0" ++pandas = ">=2.0.3" ++requests-futures = ">=1.0.1" ++tqdm = ">=4.65.0" ++lxml = ">=4.9.3" ++selenium = {version = ">=4.10.0", optional = true} ++beautifulsoup4 = ">=4.12.2" + + [tool.poetry.dev-dependencies] +-pytest = "^7.4.0" +-isort = "^5.0.0" +-flake8 = "^6.0.0" +-mypy = "^1.4.1" +-pytest-cov = "^4.1.0" +-black = "^23.7.0" +-pre-commit = "^3.3.3" +-ipython = "^8.0.0" +-mkdocs-material = "^9.1.18" ++pytest = ">=7.4.0" ++isort = ">=5.0.0" ++flake8 = ">=6.0.0" ++mypy = ">=1.4.1" ++pytest-cov = ">=4.1.0" ++black = ">=23.7.0" ++pre-commit = ">=3.3.3" ++ipython = ">=8.0.0" ++mkdocs-material = ">=9.1.18" + + [build-system] + requires = ["poetry-core>=1.0.0"] diff --git a/nix/yahooquery.nix b/nix/yahooquery.nix new file mode 100644 index 0000000..8b1535d --- /dev/null +++ b/nix/yahooquery.nix @@ -0,0 +1,44 @@ +{ + lib, + pkgs, + fetchFromGitHub, +}: + +with pkgs.python312.pkgs; + +buildPythonPackage rec { + pname = "yahooquery"; + version = "2.3.7"; + pyproject = true; + + disabled = pythonOlder "3.8.1"; + + src = fetchFromGitHub { + owner = "dpguthrie"; + repo = "yahooquery"; + rev = "refs/tags/v${version}"; + hash = "sha256-Iyuni1SoTB6f7nNFhN5A8Gnv9kV78frjpqvvW8qd+/M="; + }; + + patches = [ ./yahooquery-noversioning.patch ]; + + build-system = [ poetry-core ]; + + dependencies = [ + requests # ^2.31.0 + pandas # ^2.0.3 + requests-futures # ^1.0.1 + tqdm # ^4.65.0 + lxml # ^4.9.3 + selenium # {version = ^4.10.0, optional = true} + beautifulsoup4 # ^4.12.2 + ]; + + doCheck = false; # skip tests + + meta = with lib; { + description = "Python wrapper for an unofficial Yahoo Finance API"; + homepage = "https://yahooquery.dpguthrie.com"; + license = licenses.mit; + }; +} diff --git a/setup.cfg b/setup.cfg index edbad57..f63dd91 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev6 + mmgen-wallet>=15.1.dev9 pyyaml yahooquery From 7cb2fc5b08722b3a2ac1436fb9ce2d7b4fd12852 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Jan 2025 11:06:28 +0000 Subject: [PATCH 09/51] nix/shell.nix: add `repo` arg; test/init.sh: add `-c` option --- mmgen_node_tools/data/version | 2 +- nix/shell.nix | 5 ++++- setup.cfg | 2 +- test/init.sh | 17 ++++++++++++++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index cf8acd2..c087ce9 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev4 +3.5.dev5 diff --git a/nix/shell.nix b/nix/shell.nix index 01991d2..acfdee3 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -1 +1,4 @@ -import ../../mmgen-wallet/nix/shell.nix { add_pkgs_path = ./node-tools-packages.nix; } +import ../../mmgen-wallet/nix/shell.nix { + repo = "mmgen-node-tools"; + add_pkgs_path = ./node-tools-packages.nix; +} diff --git a/setup.cfg b/setup.cfg index f63dd91..db836e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev9 + mmgen-wallet>=15.1.dev10 pyyaml yahooquery diff --git a/test/init.sh b/test/init.sh index f6a6f00..15eeddb 100755 --- a/test/init.sh +++ b/test/init.sh @@ -28,15 +28,17 @@ STDOUT_DEVNULL='>/dev/null' STDERR_DEVNULL='2>/dev/null' PROGNAME=$(basename $0) -while getopts hv OPT +while getopts hcv OPT do case "$OPT" in h) printf " %-16s Initialize the MMGen Node Tools test suite\n" "${PROGNAME}:" echo " USAGE: $PROGNAME" echo " OPTIONS: '-h' Print this help message" + echo " -c Create links from mmgen-wallet ‘cmds’ subdirectory" echo " -v Be more verbose" exit ;; v) VERBOSE=1 STDOUT_DEVNULL='' STDERR_DEVNULL='' ;; + c) CMD_LINKS=1 ;; *) exit ;; esac done @@ -123,6 +125,17 @@ create_test_links() { done <<<$paths } +create_cmd_links() { + [ "$VERBOSE" ] && becho 'Creating links to mmgen-wallet repo ‘cmds’ subdirectory' + ( + filenames=$(cd $wallet_repo/cmds && ls) + cd cmds + for filename in $filenames; do + [ -e $filename ] || ln -s "../$wallet_repo/cmds/$filename" + done + ) +} + becho 'Initializing MMGen Node Tools Test Suite' delete_old_stuff @@ -137,6 +150,8 @@ create_dir_links create_test_links +[ "$CMD_LINKS" ] && create_cmd_links + [ "$VERBOSE" ] && becho 'OK' true From b62ff991fe030448c52440f5ccf722b070ed2d41 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 6 Feb 2025 10:07:57 +0000 Subject: [PATCH 10/51] update for MMGen Wallet v15.1.dev14 --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 6 +++--- setup.cfg | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index c087ce9..426110c 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev5 +3.5.dev6 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 7dd37a3..7fafac1 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -142,10 +142,10 @@ async def main(req_addrs): addr = re.match('addr\((.*?)\)',unspent['desc'])[1] addr_data[addr].append(unspent) else: - from mmgen.proto.btc.tx.base import scriptPubKey2addr + from mmgen.proto.btc.tx.base import decodeScriptPubKey for unspent in sorted(res['unspents'],key=lambda x: x['height']): - addr = scriptPubKey2addr( proto, unspent['scriptPubKey'] )[0] - addr_data[addr].append(unspent) + ds = decodeScriptPubKey(proto, unspent['scriptPubKey']) + addr_data[ds.addr].append(unspent) good_addrs = len([v for v in addr_data.values() if v]) diff --git a/setup.cfg b/setup.cfg index db836e8..e140080 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev10 + mmgen-wallet>=15.1.dev14 pyyaml yahooquery From 966c1716505f63829bb70c063560e0d3b1cff67f Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 15 Mar 2025 18:23:16 +0000 Subject: [PATCH 11/51] update for MMGen Wallet v15.1.dev20 --- README.md | 2 +- mmgen_node_tools/Ticker.py | 14 ++++++++------ mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 8 ++++---- mmgen_node_tools/main_feeview.py | 2 +- mmgen_node_tools/main_halving_calculator.py | 8 ++++---- setup.cfg | 2 +- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2751da7..89c43b2 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Code repository: Code repository mirrors: [Github](https://github.com/mmgen/mmgen-node-tools) | [Gitlab](https://gitlab.com/mmgen/mmgen-node-tools) | -[Gitflic](https://gitflic.ru/project/mmgen/mmgen-node-tools) +[Codeberg](https://codeberg.org/mmgen/mmgen-node-tools) [Keybase](https://keybase.io/mmgen) | [Twitter](https://twitter.com/TheMMGenProject) | [Reddit](https://www.reddit.com/user/mmgen-py) | diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bae27c2..252ba6c 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -799,7 +799,7 @@ class Ticker: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: - fmt_func = lambda t,now: time.strftime('%F %X',time.gmtime(t)) + fmt_func = lambda t,now: time.strftime('%F %X', time.gmtime(t)) d = self.data max_w = 0 @@ -820,15 +820,17 @@ class Ticker: msg(str(e)) pass else: - t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( (min(t,min_t) if min_t else t), now ) - max_w = max(len(t_fmt),max_w) + t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( + (min(t,min_t) if min_t else t), + now = now) + max_w = max(len(t_fmt), max_w) self.upd_w = max_w def init_prec(self): - exp = [(a.id, self.prices[a.id]['usd-us-dollar'].adjusted() ) for a in self.usr_col_assets] - self.uprec = { k: max(0,v+4) + cfg.add_prec for k,v in exp } - self.uwid = { k: 12 + max(0, abs(v)-6) + cfg.add_prec for k,v in exp } + exp = [(a.id, self.prices[a.id]['usd-us-dollar'].adjusted()) for a in self.usr_col_assets] + self.uprec = {k: max(0, v+4) + cfg.add_prec for k, v in exp} + self.uwid = {k: 12 + max(0, abs(v)-6) + cfg.add_prec for k, v in exp} def get_id(self,asset): if asset.id: diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 426110c..aa6a660 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev6 +3.5.dev7 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 7fafac1..ecf2b1a 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -60,7 +60,7 @@ def do_output(proto,addr_data,blk_hdrs): make_timestr( blk_hdrs[u['height']]['time'] ), CoinTxID(u['txid']).hl(), red(str(u['vout']).rjust(4)), - proto.coin_amt(u['amount']).fmt(color=True,iwidth=6,prec=8) + proto.coin_amt(u['amount']).fmt(6, color=True, prec=8) )) else: Msg(f'{indent}No balance') @@ -93,18 +93,18 @@ def do_output_tabular(proto,addr_data,blk_hdrs): if unspents: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(addr.view_pref, width=max_addrw, color=True), + a = addr.fmt(addr.view_pref, max_addrw, color=True), u = red(str(len(unspents)).rjust(5)), b = unspents[0]['height'], t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), B = unspents[-1]['height'], T = make_timestr( blk_hdrs[unspents[-1]['height']]['time'] ), - A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(color=True, iwidth=7, prec=8) + A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(7, color=True, prec=8) )) else: Msg(fs.format( n = str(n) + ')', - a = addr.fmt(addr.view_pref, width=max_addrw, color=True), + a = addr.fmt(addr.view_pref, max_addrw, color=True), u = ' -', b = '-', t = '', diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 7bee773..16591b8 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -87,7 +87,7 @@ if cfg.ignore_below: ignore_below = parse_bytespec(cfg.ignore_below) precision = ( - check_int_between(cfg.precision,min_prec,max_prec,'--precision arg') + check_int_between(cfg.precision, min_prec, max_prec, desc='--precision arg') if cfg.precision else dfl_prec ) from mmgen.term import get_terminal_size diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 1648d76..caeb178 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -171,10 +171,10 @@ async def main(): c = date(t), d = ' P' if n > nhist else '' if n < nhist else ' E', e = f'{bdr:8.5f}', - f = proto.coin_amt(sub,from_unit='satoshi').fmt(iwidth=2,prec=8), - g = proto.coin_amt(mined,from_unit='satoshi').fmt(iwidth=8,prec=8), - h = proto.coin_amt(total_mined,from_unit='satoshi').fmt(iwidth=8,prec=8) - ) for n,sub,blk,mined,total_mined,bdr,t in gen_data()) + f = proto.coin_amt(sub, from_unit='satoshi').fmt(2, prec=8), + g = proto.coin_amt(mined, from_unit='satoshi').fmt(8, prec=8), + h = proto.coin_amt(total_mined, from_unit='satoshi').fmt(8, prec=8) + ) for n, sub, blk, mined, total_mined, bdr, t in gen_data()) ) if cfg.list: diff --git a/setup.cfg b/setup.cfg index e140080..b43abad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev14 + mmgen-wallet>=15.1.dev20 pyyaml yahooquery From d63cb9817a249618931f526e9d5029bfe45adb0d Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 23 Mar 2025 10:12:56 +0000 Subject: [PATCH 12/51] update for MMGen Wallet v15.1.dev23 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_d/ct_regtest.py | 4 ++-- test/init.sh | 2 ++ 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index aa6a660..1e3f513 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev7 +3.5.dev8 diff --git a/setup.cfg b/setup.cfg index b43abad..c67cb47 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev20 + mmgen-wallet>=15.1.dev23 pyyaml yahooquery diff --git a/test/cmdtest_d/ct_regtest.py b/test/cmdtest_d/ct_regtest.py index 24f19b9..f043641 100755 --- a/test/cmdtest_d/ct_regtest.py +++ b/test/cmdtest_d/ct_regtest.py @@ -105,8 +105,8 @@ class CmdTestRegtest(CmdTestBase): ), } - def __init__(self,trunner,cfgs,spawn): - CmdTestBase.__init__(self,trunner,cfgs,spawn) + def __init__(self, cfg, trunner, cfgs, spawn): + CmdTestBase.__init__(self, cfg, trunner, cfgs, spawn) if trunner == None: return if cfg._proto.testnet: diff --git a/test/init.sh b/test/init.sh index 15eeddb..94671f9 100755 --- a/test/init.sh +++ b/test/init.sh @@ -95,6 +95,8 @@ create_test_links() { test/test-release.sh symbolic test/cmdtest_d/common.py symbolic test/cmdtest_d/ct_base.py symbolic + test/cmdtest_d/runner.py symbolic + test/cmdtest_d/group_mgr.py symbolic cmds/mmgen-regtest symbolic ' while read path type; do From 4688ee94c0dfa1a73165e72a520182e724e47c47 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 29 Mar 2025 12:12:35 +0000 Subject: [PATCH 13/51] update test suite for MMGen Wallet v15.1.dev25 --- README.md | 3 +- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_d/{ => include}/cfg.py | 2 +- test/cmdtest_d/{ct_main.py => main.py} | 6 ++-- test/cmdtest_d/{ct_misc.py => misc.py} | 6 ++-- test/cmdtest_d/{ct_regtest.py => regtest.py} | 10 +++---- test/init.sh | 31 ++++++++++++-------- 8 files changed, 34 insertions(+), 28 deletions(-) rename test/cmdtest_d/{ => include}/cfg.py (93%) rename test/cmdtest_d/{ct_main.py => main.py} (94%) rename test/cmdtest_d/{ct_misc.py => misc.py} (98%) rename test/cmdtest_d/{ct_regtest.py => regtest.py} (98%) diff --git a/README.md b/README.md index 89c43b2..a536cce 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,8 @@ $ python3 -m pip install --upgrade mmgen-node-tools ### Development version: -Install the latest development version of [MMGen Wallet][6] for your platform: +First install the latest development version of [MMGen Wallet][6] for your +platform. Then perform the following steps: ```bash $ git clone https://github.com/mmgen/mmgen-node-tools diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1e3f513..0602083 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev8 +3.5.dev9 diff --git a/setup.cfg b/setup.cfg index c67cb47..905e659 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev23 + mmgen-wallet>=15.1.dev25 pyyaml yahooquery diff --git a/test/cmdtest_d/cfg.py b/test/cmdtest_d/include/cfg.py similarity index 93% rename from test/cmdtest_d/cfg.py rename to test/cmdtest_d/include/cfg.py index c28ce5d..5a4263d 100755 --- a/test/cmdtest_d/cfg.py +++ b/test/cmdtest_d/include/cfg.py @@ -9,7 +9,7 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_d.cfg: configuration data for cmdtest.py +test.cmdtest_d.include.cfg: configuration data for cmdtest.py """ cmd_groups_altcoin = [] diff --git a/test/cmdtest_d/ct_main.py b/test/cmdtest_d/main.py similarity index 94% rename from test/cmdtest_d/ct_main.py rename to test/cmdtest_d/main.py index 3009d2f..6e35b7d 100755 --- a/test/cmdtest_d/ct_main.py +++ b/test/cmdtest_d/main.py @@ -9,13 +9,13 @@ # https://gitlab.com/mmgen/mmgen-wallet """ -cmdtest_d.ct_main: Basic operations tests for the cmdtest.py test suite +cmdtest_d.main: Basic operations tests for the cmdtest.py test suite """ -import sys,time +import sys, time from ..include.common import cfg -from .ct_base import CmdTestBase +from .base import CmdTestBase class CmdTestMain(CmdTestBase): 'basic operations with fake RPC data' diff --git a/test/cmdtest_d/ct_misc.py b/test/cmdtest_d/misc.py similarity index 98% rename from test/cmdtest_d/ct_misc.py rename to test/cmdtest_d/misc.py index 15b109e..1a8175e 100755 --- a/test/cmdtest_d/ct_misc.py +++ b/test/cmdtest_d/misc.py @@ -9,13 +9,13 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_d.ct_misc: Miscellaneous test groups for the cmdtest.py test suite +test.cmdtest_d.misc: Miscellaneous test groups for the cmdtest.py test suite """ -import os,shutil +import os, shutil from ..include.common import cfg -from .ct_base import CmdTestBase +from .base import CmdTestBase refdir = os.path.join('test','ref','ticker') diff --git a/test/cmdtest_d/ct_regtest.py b/test/cmdtest_d/regtest.py similarity index 98% rename from test/cmdtest_d/ct_regtest.py rename to test/cmdtest_d/regtest.py index f043641..3b77836 100755 --- a/test/cmdtest_d/ct_regtest.py +++ b/test/cmdtest_d/regtest.py @@ -9,18 +9,18 @@ # https://gitlab.com/mmgen/mmgen-node-tools """ -test.cmdtest_d.ct_regtest: Regtest tests for the cmdtest.py test suite +test.cmdtest_d.regtest: Regtest tests for the cmdtest.py test suite """ -import sys,os +import sys, os from decimal import Decimal -from mmgen.util import msg_r,die,gmsg +from mmgen.util import msg_r, die, gmsg from mmgen.protocol import init_proto from mmgen.proto.btc.regtest import MMGenRegtest -from ..include.common import cfg,imsg,stop_test_daemons,joinpath -from .ct_base import CmdTestBase +from ..include.common import cfg, imsg, stop_test_daemons, joinpath +from .base import CmdTestBase args1 = ['--bob'] args2 = ['--bob','--rpc-backend=http'] diff --git a/test/init.sh b/test/init.sh index 94671f9..134f4f3 100755 --- a/test/init.sh +++ b/test/init.sh @@ -81,23 +81,28 @@ create_dir_links() { delete_old_stuff() { rm -rf test/unit_tests.py + rm -rf test/cmdtest_d/common.py + rm -rf test/cmdtest_d/ct_base.py + rm -rf test/cmdtest_d/group_mgr.py + rm -rf test/cmdtest_d/runner.py } create_test_links() { paths=' - test/include symbolic - test/overlay/__init__.py symbolic - test/overlay/fakemods/mmgen symbolic - test/__init__.py symbolic - test/clean.py symbolic - test/cmdtest.py hard - test/modtest.py hard - test/test-release.sh symbolic - test/cmdtest_d/common.py symbolic - test/cmdtest_d/ct_base.py symbolic - test/cmdtest_d/runner.py symbolic - test/cmdtest_d/group_mgr.py symbolic - cmds/mmgen-regtest symbolic + test/include symbolic + test/overlay/__init__.py symbolic + test/overlay/fakemods/mmgen symbolic + test/__init__.py symbolic + test/clean.py symbolic + test/cmdtest.py hard + test/modtest.py hard + test/test-release.sh symbolic + test/cmdtest_d/base.py symbolic + test/cmdtest_d/include/common.py symbolic + test/cmdtest_d/include/runner.py symbolic + test/cmdtest_d/include/group_mgr.py symbolic + test/cmdtest_d/include/pexpect.py symbolic + cmds/mmgen-regtest symbolic ' while read path type; do [ "$path" ] || continue From ab93f54d541d4ee6511f9bf37e703d50fc2745f5 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 30 Mar 2025 10:18:52 +0000 Subject: [PATCH 14/51] update test suite for MMGen Wallet v15.1.dev26 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 0602083..de6f13c 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev9 +3.5.dev10 diff --git a/setup.cfg b/setup.cfg index 905e659..0ab2734 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev25 + mmgen-wallet>=15.1.dev26 pyyaml yahooquery From 106f201300d7dadb3a70e979db9373e9f6ca0e34 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 24 May 2025 10:10:24 +0000 Subject: [PATCH 15/51] update for MMGen Wallet v15.1.dev39 --- MANIFEST.in | 1 + mmgen_node_tools/BlocksInfo.py | 2 +- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_feeview.py | 2 +- setup.cfg | 3 +-- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0443a99..77b53ac 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,5 +6,6 @@ include test/init.sh include test/test-release.d/*.sh include test/modtest_d/*.py include test/cmdtest_d/*.py +include test/cmdtest_d/include/cfg.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 26e6cfa..f19089a 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -26,7 +26,7 @@ from time import strftime,gmtime from decimal import Decimal from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int -from mmgen.rpc import json_encoder +from mmgen.rpc.util import json_encoder class RangeParser: diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index de6f13c..70beee3 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev10 +3.5.dev11 diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 16591b8..e7b9952 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -103,7 +103,7 @@ class fee_bracket: def log(data,fn): import json - from mmgen.rpc import json_encoder + from mmgen.rpc.util import json_encoder from mmgen.fileutil import write_data_to_file write_data_to_file( cfg = cfg, diff --git a/setup.cfg b/setup.cfg index 0ab2734..cd27742 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,6 @@ project_urls = Website = https://mmgen.org Bug Tracker = https://github.com/mmgen/mmgen-node-tools/issues classifiers = - License :: OSI Approved :: GNU General Public License v3 (GPLv3) Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows Operating System :: MacOS @@ -38,7 +37,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev26 + mmgen-wallet>=15.1.dev39 pyyaml yahooquery From 5a24cbfc0a50d03203710c387763a1b893f3d5f3 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 24 Sep 2025 10:45:50 +0000 Subject: [PATCH 16/51] Version 3.5.0 - released concurrently with MMGen Wallet v16.0.0 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 70beee3..1545d96 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.dev11 +3.5.0 diff --git a/setup.cfg b/setup.cfg index cd27742..9bca61c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.9 include_package_data = True install_requires = - mmgen-wallet>=15.1.dev39 + mmgen-wallet==16.0.0 pyyaml yahooquery From cab8be016707b6886cbacf56666611b9a3fdad98 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:37 +0000 Subject: [PATCH 17/51] pyproject.toml: add ruff data --- pyproject.toml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 81515d2..25af44f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,41 @@ requires = [ ] build-backend = "setuptools.build_meta" +[tool.ruff] +line-length = 106 +indent-width = 4 + +[tool.ruff.format] +quote-style = "single" +indent-style = "tab" + +[tool.ruff.lint] +ignore = [ + "E401", # multiple imports per line + "E701", # multiple statements per line + "E721", # use isinstance() + "E731", # lambda instead of def + "E402", # module import not top of file + "E722", # bare except + "E713", # membership 'not in' + "E741", # ambiguous variable name +] + +[tool.ruff.lint.per-file-ignores] +"test/include/common.py" = [ "F821" ] # undefined name 'cfg' +"test/misc/input_func.py" = [ "F401" ] # imported but unused +"test/modtest_d/cashaddr.py" = [ "F841" ] # assigned to but never used +"test/modtest_d/dep.py" = [ "F401" ] # imported but unused +"test/modtest_d/testdep.py" = [ "F401" ] # imported but unused +"test/modtest_d/obj.py" = [ "F841" ] # assigned to but never used +"test/objtest_d/*" = [ "F401" ] # imported but unused +"test/objattrtest_d/*" = [ "F401" ] # imported but unused +"test/overlay/fakemods/*" = [ "F403", "F405" ] # `import *` used +"test/*.py" = [ "F401" ] # imported but unused +"test/colortest.py" = [ "F403", "F405" ] # `import *` used +"test/tooltest2.py" = [ "F403", "F405" ] # `import *` used +"test/overlay/tree/*" = [ "ALL" ] + [tool.pylint.format] indent-string = "\t" indent-after-paren = 2 From 5c9d7053017f463193b9b5aa9dfb036f1844d697 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:41 +0000 Subject: [PATCH 18/51] test suite: lint, whitespace --- test/cmdtest_d/main.py | 2 +- test/cmdtest_d/misc.py | 4 ++-- test/cmdtest_d/regtest.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/cmdtest_d/main.py b/test/cmdtest_d/main.py index 6e35b7d..f1881f8 100755 --- a/test/cmdtest_d/main.py +++ b/test/cmdtest_d/main.py @@ -41,7 +41,7 @@ class CmdTestMain(CmdTestBase): def peerblocks(self,args,expect_list=None,pexpect_spawn=False): t = self.spawn( - f'mmnode-peerblocks', + 'mmnode-peerblocks', args, pexpect_spawn = pexpect_spawn ) if cfg.exact_output: # disable echoing of input diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 1a8175e..8a39462 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -32,7 +32,7 @@ class CmdTestHelp(CmdTestBase): color = True def version(self): - t = self.spawn(f'mmnode-netrate',['--version']) + t = self.spawn('mmnode-netrate', ['--version']) t.expect('MMNODE-NETRATE version') return t @@ -108,7 +108,7 @@ class CmdTestScripts(CmdTestBase): def ticker(self, args=[], expect_list=None, cached=True, exit_val=None): t = self.spawn( - f'mmnode-ticker', + 'mmnode-ticker', (['--cached-data'] if cached else []) + self.ticker_args + args, exit_val = exit_val) if expect_list: diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index 3b77836..4792dee 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -107,7 +107,7 @@ class CmdTestRegtest(CmdTestBase): def __init__(self, cfg, trunner, cfgs, spawn): CmdTestBase.__init__(self, cfg, trunner, cfgs, spawn) - if trunner == None: + if trunner is None: return if cfg._proto.testnet: die(2,'--testnet and --regtest options incompatible with regtest test suite') @@ -328,7 +328,7 @@ class CmdTestRegtest(CmdTestBase): async def do_tx(inputs,outputs,wif): tx_hex = await r.rpc_call( 'createrawtransaction', inputs, outputs ) tx = await r.rpc_call( 'signrawtransactionwithkey', tx_hex, [wif], [], self.proto.sighash_type ) - assert tx['complete'] == True + assert tx['complete'] return tx['hex'] async def do_tx1(): @@ -371,13 +371,13 @@ class CmdTestRegtest(CmdTestBase): imsg(f'Creating funding transaction with {nTxs} outputs of value {tx1_amt} {self.proto.coin}') tx1_hex = await do_tx1() - imsg(f'Relaying funding transaction') + imsg('Relaying funding transaction') await r.rpc_call('sendrawtransaction',tx1_hex) - imsg(f'Mining a block') + imsg('Mining a block') await r.generate(1,silent=True) - imsg(f'Generating fees for mempool transactions') + imsg('Generating fees for mempool transactions') fees = list(gen_fees(nTxs,2,120)) imsg(f'Creating and relaying {nTxs} mempool transactions with {nPairs} outputs each') From 4d1f4577a7758c0c028166d7f59538ec53d74d8d Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:41 +0000 Subject: [PATCH 19/51] lint, whitespace --- mmgen_node_tools/BlocksInfo.py | 2 +- mmgen_node_tools/PollDisplay.py | 2 +- mmgen_node_tools/Sound.py | 2 +- mmgen_node_tools/Ticker.py | 23 ++++++++++++--------- mmgen_node_tools/Util.py | 1 - mmgen_node_tools/main_addrbal.py | 2 +- mmgen_node_tools/main_halving_calculator.py | 5 ++--- 7 files changed, 19 insertions(+), 18 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index f19089a..d5eac9e 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -361,7 +361,7 @@ class BlocksInfo: if p.debug: msg(repr(self.range_data(first,last,from_tip,nblocks,step))) if nblocks: - if first == None: + if first is None: first = self.tip - nblocks + 1 last = first + nblocks - 1 diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index 72bedac..1de8a1a 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -52,7 +52,7 @@ class PollDisplay: count += 1 async def process_input(): - if self.input == None: + if self.input is None: sys.exit(1) elif self.input == 'q': msg('') diff --git a/mmgen_node_tools/Sound.py b/mmgen_node_tools/Sound.py index 88d3b02..df9f8ae 100755 --- a/mmgen_node_tools/Sound.py +++ b/mmgen_node_tools/Sound.py @@ -33,7 +33,7 @@ def timespec2secs(ts): mul = { 's': 1, 'm': 60, 'h': 60*60, 'd': 60*60*24 } pat = r'^([0-9]+)([smhd]*)$' m = re.match(pat,ts) - if m == None: + if m is None: die(2,"'%s': invalid time specifier" % ts) a,b = m.groups() return int(a) * (mul[b] if b else 1) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 252ba6c..6c8bc08 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -19,13 +19,13 @@ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other # Possible alternatives: # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR -import sys,os,re,time,datetime,json,yaml,random -from subprocess import run,PIPE,CalledProcessError +import sys, os, re, time, datetime, json, yaml, random +from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.color import red,yellow,green,blue,orange,gray -from mmgen.util import msg,msg_r,Msg,Msg_r,die,Die,suf,fmt,fmt_list,fmt_dict,list_gen +from mmgen.color import red, yellow, green, blue, orange, gray +from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -89,9 +89,11 @@ class DataSource: msg('') from .Misc import curl_exit_codes msg(red(curl_exit_codes[e.returncode])) - msg(red('Command line:\n {}'.format( ' '.join((repr(i) if ' ' in i else i) for i in e.cmd) ))) + msg(red('Command line:\n {}'.format( + ' '.join((repr(i) if ' ' in i else i) for i in e.cmd)))) from mmgen.exception import MMGenCalledProcessError - raise MMGenCalledProcessError(f'Subprocess returned non-zero exit status {e.returncode}') + raise MMGenCalledProcessError( + f'Subprocess returned non-zero exit status {e.returncode}') def get_data(self): @@ -327,15 +329,16 @@ class DataSource: def postprocess_data(self,data): def gen(): keys = set() - for key,val in data.items(): + d = {} + for key, val in data.items(): if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$",key): date = '{}-{:>02}-{:>02}'.format(*m[2].split(', ')) if (sym := m[1]) in keys: d[date] = val else: keys.add(sym) - d = {date:val} - yield (sym,d) + d = {date: val} + yield (sym, d) return dict(gen()) def assets_list_gen(cfg_in): @@ -936,7 +939,7 @@ class Ticker: def fmt_row(self,d,amt=None,amt_fmt=None): def fmt_pct(n): - return gray(' --') if n == None else (red,green)[n>=0](f'{n:+7.2f}') + return gray(' --') if n is None else (red,green)[n>=0](f'{n:+7.2f}') p = self.prices[d['id']] diff --git a/mmgen_node_tools/Util.py b/mmgen_node_tools/Util.py index 9060f68..1d5426b 100755 --- a/mmgen_node_tools/Util.py +++ b/mmgen_node_tools/Util.py @@ -20,7 +20,6 @@ mmgen_node_tools.Util: utility functions for MMGen node tools """ import time -from mmgen.util import suf def get_hms(t=None,utc=False,no_secs=False): secs = t or time.time() diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index ecf2b1a..747d0bb 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -14,7 +14,7 @@ mmnode-addrbal: Get balances for arbitrary addresses in the blockchain import sys -from mmgen.obj import CoinTxID,Int +from mmgen.obj import CoinTxID from mmgen.cfg import Config from mmgen.util import msg,Msg,die,suf,make_timestr,async_run from mmgen.color import red diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index caeb178..50916ab 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -21,7 +21,6 @@ mmnode-halving-calculator: Estimate date(s) of future block subsidy halving(s) """ import time -from decimal import Decimal from mmgen.cfg import Config from mmgen.util import async_run @@ -148,8 +147,8 @@ async def main(): b = 'BLOCK', c = 'DATE', d = '', - e = f'BDI (mins)', - f = f'SUBSIDY ({proto.coin})', + e = 'BDI (mins)', + f = 'SUBSIDY ({proto.coin})', g = f'MINED ({proto.coin})', h = f'TOTAL MINED ({proto.coin})' ) From bfb2dd839aab1d221e38d6e6532dca20f2904632 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:41 +0000 Subject: [PATCH 20/51] setup.cfg: bump requirements --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9bca61c..9af335b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,11 +33,11 @@ classifiers = Development Status :: 5 - Production/Stable [options] -python_requires = >=3.9 +python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet==16.0.0 + mmgen-wallet>=16.0.0 pyyaml yahooquery From 37c74e361c0f695a43b3f08b4b85cc2106b92665 Mon Sep 17 00:00:00 2001 From: "MMGen@trixie" Date: Wed, 1 Oct 2025 15:26:42 +0000 Subject: [PATCH 21/51] use match statement where practicable --- mmgen_node_tools/BlocksInfo.py | 102 ++++++++++++++++++--------------- mmgen_node_tools/Ticker.py | 21 +++---- 2 files changed, 66 insertions(+), 57 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index d5eac9e..42ccc23 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -197,18 +197,18 @@ class BlocksInfo: return parse_cs_uarg(self.cfg.stats.lower(),self.all_stats,self.dfl_stats,'stat') def parse_cmd_args(): # => (block_list, first, last, step) - if not cmd_args: - return (None,self.tip,self.tip,None) - elif len(cmd_args) == 1: - r = self.parse_rangespec(cmd_args[0]) - return ( - list(range(r.first,r.last+1,r.step)) if r.step else None, - r.first, - r.last, - r.step - ) - else: - return ([self.conv_blkspec(a) for a in cmd_args],None,None,None) + match cmd_args: + case [] | None: + return (None, self.tip, self.tip, None) + case [arg]: + r = self.parse_rangespec(arg) + return ( + list(range(r.first, r.last+1, r.step)) if r.step else None, + r.first, + r.last, + r.step) + case [*args]: + return ([self.conv_blkspec(a) for a in args], None, None, None) self.cfg = cfg self.rpc = rpc @@ -323,25 +323,29 @@ class BlocksInfo: repl = (name if add_name else '') + ':' + (fill_char if name in fill else '') yield (ls + self.fields[name].fs.replace(':',repl) + rs) - def conv_blkspec(self,arg): - if str(arg).lower() == 'cur': - return self.tip - elif is_int(arg): - if int(arg) < 0: - die(1,f'{arg}: block number must be non-negative') - elif int(arg) > self.tip: - die(1,f'{arg}: requested block height greater than current chain tip!') - else: - return int(arg) - else: - die(1,f'{arg}: invalid block specifier') + def conv_blkspec(self, arg): + match arg: + case str() if arg.lower() == 'cur': + return self.tip + case x if is_int(x): + match int(arg): + case x if x < 0: + die(1, f'{x}: block number must be non-negative') + case x if x > self.tip: + die(1, f'{x}: requested block height greater than current chain tip!') + case x: + return x + case _: + die(1, f'{arg}: invalid block specifier') - def check_nblocks(self,arg): - if arg <= 0: - die(1,'nBlocks must be a positive integer') - elif arg > self.tip: - die(1, f"'{arg}': nBlocks must be less than current chain height") - return arg + def check_nblocks(self, arg): + match arg: + case x if x <= 0: + die(1, 'nBlocks must be a positive integer') + case x if x > self.tip: + die(1, f'{arg}: nBlocks must be less than current chain height') + case _: + return arg def parse_rangespec(self,arg): @@ -487,20 +491,21 @@ class BlocksInfo: def fmt_stat_item(self,fs,s): return fs.format(s) if type(fs) == str else fs(s) - async def output_stats(self,res,sname): + async def output_stats(self, res, sname): def gen(data): for d in data: - if len(d) == 2: - yield (indent+d[0]).format(**{k:self.fmt_stat_item(*v) for k,v in d[1].items()}) - elif len(d) == 4: - yield (indent+d[0]).format(self.fmt_stat_item(d[2],d[3])) - elif type(d) == str: - yield d - else: - assert False, f'{d}: invalid stats data' + match d: + case [a, b]: + yield (indent + a).format(**{k:self.fmt_stat_item(*v) for k, v in b.items()}) + case [a, _, b, c]: + yield (indent + a).format(self.fmt_stat_item(b, c)) + case str(): + yield d + case _: + assert False, f'{d}: invalid stats data' - foo,data = await res + foo, data = await res indent = '' if sname in self.noindent_stats else ' ' Msg('\n'.join(gen(data))) @@ -755,13 +760,16 @@ class JSONBlocksInfo(BlocksInfo): def gen(data): for d in data: - if len(d) == 2: - for k,v in d[1].items(): - yield (k,self.fmt_stat_item(*v)) - elif len(d) == 4: - yield (d[1],self.fmt_stat_item(d[2],d[3])) - elif type(d) != str: - assert False, f'{d}: invalid stats data' + match d: + case [_, a]: + for k, v in a.items(): + yield (k, self.fmt_stat_item(*v)) + case [_, a, b, c]: + yield (a, self.fmt_stat_item(b, c)) + case str(): + pass + case _: + assert False, f'{d}: invalid stats data' varname,data = await res Msg_r(', "{}_data": {}'.format( varname, json.dumps(dict(gen(data)),cls=json_encoder) )) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 6c8bc08..95e03f2 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -125,16 +125,17 @@ class DataSource: else: die(1,self.rate_limit_errmsg(elapsed)) - if data_type == 'json': - try: - data = json.loads(data_in) - except: - self.json_data_error_msg(data_in) - die(2,'Retrieved data is not valid JSON, exiting') - json_text = data_in - elif data_type == 'python': - data = data_in - json_text = json.dumps(data_in) + match data_type: + case 'json': + try: + data = json.loads(data_in) + except: + self.json_data_error_msg(data_in) + die(2,'Retrieved data is not valid JSON, exiting') + json_text = data_in + case 'python': + data = data_in + json_text = json.dumps(data_in) if not data: if use_cached_data: From fe45fbaa239ae6c28ed056cd1c2f0977550bad45 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 1 Oct 2025 15:26:42 +0000 Subject: [PATCH 22/51] minor cleanups --- mmgen_node_tools/main_blocks_info.py | 4 ++-- mmgen_node_tools/main_halving_calculator.py | 4 ++-- test/modtest_d/__init__.py | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100755 test/modtest_d/__init__.py diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index c2bbf59..18136c4 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -164,7 +164,7 @@ async def main(): cls = JSONBlocksInfo if cfg.json else BlocksInfo - m = cls( cfg, cfg._args, await rpc_init(cfg,ignore_wallet=True) ) + m = cls(cfg, cfg._args, await rpc_init(cfg, ignore_wallet=True)) if m.fnames and not cfg.no_header: m.print_header() @@ -178,4 +178,4 @@ async def main(): m.finalize_output() -async_run(main()) +async_run(cfg, main) diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 50916ab..8f0a3a3 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -67,7 +67,7 @@ async def main(): proto = cfg._proto from mmgen.rpc import rpc_init - c = await rpc_init( cfg, proto, ignore_wallet=True ) + c = await rpc_init(cfg, proto, ignore_wallet=True) tip = await c.call('getblockcount') assert tip > 1, 'block tip must be > 1' @@ -181,4 +181,4 @@ async def main(): else: print_current_stats() -async_run(main()) +async_run(cfg, main) diff --git a/test/modtest_d/__init__.py b/test/modtest_d/__init__.py new file mode 100755 index 0000000..86190b5 --- /dev/null +++ b/test/modtest_d/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +""" +test.modtest_d: shared data for module tests for the MMGen Node Tools suite +""" + +altcoin_tests = [] From 1f12baac4c246f812942e4817dfd8541d5ee8913 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 1 Oct 2025 15:26:42 +0000 Subject: [PATCH 23/51] update for MMGen Wallet v16.1.dev3 --- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_addrbal.py | 2 +- mmgen_node_tools/main_feeview.py | 2 +- mmgen_node_tools/main_netrate.py | 2 +- mmgen_node_tools/main_peerblocks.py | 8 ++++---- mmgen_node_tools/main_txfind.py | 2 +- setup.cfg | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1545d96..229c4d8 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.5.0 +3.6.dev0 diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 747d0bb..37f0482 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -167,6 +167,6 @@ if len(cfg._args) < 1: die(1,'This command requires at least one coin address argument') try: - async_run(main(cfg._args)) + async_run(cfg, main, args=[cfg._args]) except KeyboardInterrupt: sys.stderr.write('\n') diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index e7b9952..3211f7f 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -220,4 +220,4 @@ async def main(): await c.call('getblockcount') )) + '\n\n' + '\n'.join(gen_body(data)) + '\n' ) -async_run(main()) +async_run(cfg, main) diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index 52b509d..b273ac6 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -67,6 +67,6 @@ async def main(): rs,ss,ts = (r,s,t) try: - async_run(main()) + async_run(cfg, main) except KeyboardInterrupt: sys.stderr.write('\n') diff --git a/mmgen_node_tools/main_peerblocks.py b/mmgen_node_tools/main_peerblocks.py index 6ebb5a3..35292ea 100755 --- a/mmgen_node_tools/main_peerblocks.py +++ b/mmgen_node_tools/main_peerblocks.py @@ -31,10 +31,10 @@ opts_data = { } } -async def main(): +from mmgen.cfg import Config +cfg = Config(opts_data=opts_data) - from mmgen.cfg import Config - cfg = Config(opts_data=opts_data) +async def main(): from mmgen.rpc import rpc_init rpc = await rpc_init(cfg,ignore_wallet=True) @@ -48,4 +48,4 @@ async def main(): await peers.run(rpc) from mmgen.util import async_run -async_run(main()) +async_run(cfg, main) diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index 4a54430..2411f36 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -92,4 +92,4 @@ msgs = msg_data['quiet' if cfg.quiet else 'normal'] if len(cfg._args) != 1: die(1,'One transaction ID must be specified') -sys.exit(async_run(main(cfg._args[0]))) +sys.exit(async_run(cfg, main, args=[cfg._args[0]])) diff --git a/setup.cfg b/setup.cfg index 9af335b..29481b2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.0.0 + mmgen-wallet>=16.1.dev3 pyyaml yahooquery From 0bad23b77bd2327937ec82ef7e8d4dba127e44be Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 3 Oct 2025 10:31:49 +0000 Subject: [PATCH 24/51] update for MMGen Wallet v16.1.dev4 --- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/cmdtest_d/include/cfg.py | 12 ++++++++---- test/cmdtest_d/misc.py | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 229c4d8..9f32e9d 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev0 +3.6.dev1 diff --git a/setup.cfg b/setup.cfg index 29481b2..d48a8c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.1.dev3 + mmgen-wallet>=16.1.dev4 pyyaml yahooquery diff --git a/test/cmdtest_d/include/cfg.py b/test/cmdtest_d/include/cfg.py index 5a4263d..08b6263 100755 --- a/test/cmdtest_d/include/cfg.py +++ b/test/cmdtest_d/include/cfg.py @@ -12,13 +12,17 @@ test.cmdtest_d.include.cfg: configuration data for cmdtest.py """ +from collections import namedtuple + cmd_groups_altcoin = [] +gd = namedtuple('cmd_groups_data', ['clsname', 'params']) + cmd_groups_dfl = { - 'main': ('CmdTestMain',{}), - 'helpscreens': ('CmdTestHelp',{'modname':'misc','full_data':True}), - 'scripts': ('CmdTestScripts',{'modname':'misc'}), - 'regtest': ('CmdTestRegtest',{}), + 'main': gd('CmdTestMain', {}), + 'helpscreens': gd('CmdTestHelp', {'modname': 'misc', 'full_data': True}), + 'scripts': gd('CmdTestScripts', {'modname': 'misc'}), + 'regtest': gd('CmdTestRegtest', {}), } cmd_groups_extra = {} diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 8a39462..9528b76 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -125,7 +125,7 @@ class CmdTestScripts(CmdTestBase): if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') - ret = t.expect(['proxy host could not be resolved', 'ProxyError']) + ret = t.expect(['proxy host could not be resolved', 'unexpected keyword']) t.exit_val = 1 if ret else 3 return t From 0b79ef719b3712b64571b63dbd3177ab247aa8b3 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 4 Oct 2025 09:56:55 +0000 Subject: [PATCH 25/51] whitespace, minor changes (16 files) --- mmgen_node_tools/BlocksInfo.py | 402 +++++++-------- mmgen_node_tools/PeerBlocks.py | 55 +- mmgen_node_tools/PollDisplay.py | 12 +- mmgen_node_tools/Sound.py | 32 +- mmgen_node_tools/Ticker.py | 485 ++++++++---------- mmgen_node_tools/Util.py | 60 ++- mmgen_node_tools/main_addrbal.py | 66 ++- mmgen_node_tools/main_blocks_info.py | 19 +- mmgen_node_tools/main_feeview.py | 65 ++- mmgen_node_tools/main_halving_calculator.py | 56 +- mmgen_node_tools/main_netrate.py | 21 +- mmgen_node_tools/main_peerblocks.py | 7 +- mmgen_node_tools/main_ticker.py | 23 +- mmgen_node_tools/main_txfind.py | 17 +- test/cmdtest_d/regtest.py | 8 +- .../fakemods/mmgen_node_tools/PeerBlocks.py | 8 +- 16 files changed, 629 insertions(+), 707 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index 42ccc23..a59bf50 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -20,58 +20,57 @@ mmgen_node_tools.BlocksInfo: Display information about a block or range of blocks """ -import re,json +import re, json from collections import namedtuple -from time import strftime,gmtime +from time import strftime, gmtime from decimal import Decimal -from mmgen.util import msg,Msg,Msg_r,die,suf,secs_to_ms,secs_to_dhms,is_int +from mmgen.util import msg, Msg, Msg_r, die, suf, secs_to_ms, secs_to_dhms, is_int from mmgen.rpc.util import json_encoder class RangeParser: debug = False - def __init__(self,caller,arg): + def __init__(self, caller, arg): self.caller = caller self.arg = self.orig_arg = arg - def parse(self,target): - ret = getattr(self,'parse_'+target)() + def parse(self, target): + ret = getattr(self, 'parse_'+target)() if self.debug: msg(f'arg after parse({target}): {self.arg}') return ret def finalize(self): if self.arg: - die(1,f'{self.orig_arg!r}: invalid range specifier') + die(1, f'{self.orig_arg!r}: invalid range specifier') def parse_from_tip(self): - m = re.match(r'-([0-9]+)(.*)',self.arg) + m = re.match(r'-([0-9]+)(.*)', self.arg) if m: - res,self.arg = (m[1],m[2]) + res, self.arg = (m[1], m[2]) return self.caller.check_nblocks(int(res)) def parse_abs_range(self): - m = re.match(r'([^+-]+)(-([^+-]+)){0,1}(.*)',self.arg) + m = re.match(r'([^+-]+)(-([^+-]+)){0,1}(.*)', self.arg) if m: if self.debug: msg(f'abs_range parse: first={m[1]}, last={m[3]}') self.arg = m[4] return ( self.caller.conv_blkspec(m[1]), - self.caller.conv_blkspec(m[3]) if m[3] else None - ) - return (None,None) + self.caller.conv_blkspec(m[3]) if m[3] else None) + return (None, None) def parse_add(self): - m = re.match(r'\+([0-9*]+)(.*)',self.arg) + m = re.match(r'\+([0-9*]+)(.*)', self.arg) if m: - res,self.arg = (m[1],m[2]) + res, self.arg = (m[1], m[2]) if res.strip('*') != res: - die(1,f"'+{res}': malformed nBlocks specifier") + die(1, f"'+{res}': malformed nBlocks specifier") if len(res) > 30: - die(1,f"'+{res}': overly long nBlocks specifier") + die(1, f"'+{res}': overly long nBlocks specifier") return self.caller.check_nblocks(eval(res)) # res is only digits plus '*', so eval safe class BlocksInfo: @@ -81,33 +80,33 @@ class BlocksInfo: total_solve_time = 0 header_printed = False - bf = namedtuple('block_info_fields',['fmt_func','src','fs','hdr1','hdr2','key1','key2']) + bf = namedtuple('block_info_fields', ['fmt_func', 'src', 'fs', 'hdr1', 'hdr2', 'key1', 'key2']) # bh=getblockheader, bs=getblockstats, lo=local fields = { - 'block': bf( None, 'bh', '{:<6}', '', 'Block', 'height', None ), - 'hash': bf( None, 'bh', '{:<64}', '', 'Hash', 'hash', None ), - 'date': bf( 'da', 'bh', '{:<19}', '', 'Date', 'time', None ), - 'interval': bf( 'td', 'lo', '{:>8}', 'Solve', 'Time ', 'interval', None ), - 'subsidy': bf( 'su', 'bs', '{:<5}', 'Sub-', 'sidy', 'subsidy', None ), - 'totalfee': bf( 'tf', 'bs', '{:>10}', '', 'Total Fee', 'totalfee', None ), - 'size': bf( None, 'bs', '{:>7}', '', 'Size', 'total_size', None ), - 'weight': bf( None, 'bs', '{:>7}', '', 'Weight', 'total_weight', None ), - 'fee90': bf( 'fe', 'bs', '{:>3}', '90%', 'Fee', 'feerate_percentiles', 4 ), - 'fee75': bf( 'fe', 'bs', '{:>3}', '75%', 'Fee', 'feerate_percentiles', 3 ), - 'fee50': bf( 'fe', 'bs', '{:>3}', '50%', 'Fee', 'feerate_percentiles', 2 ), - 'fee25': bf( 'fe', 'bs', '{:>3}', '25%', 'Fee', 'feerate_percentiles', 1 ), - 'fee10': bf( 'fe', 'bs', '{:>3}', '10%', 'Fee', 'feerate_percentiles', 0 ), - 'fee_max': bf( 'fe', 'bs', '{:>5}', 'Max', 'Fee', 'maxfeerate', None ), - 'fee_avg': bf( 'fe', 'bs', '{:>3}', 'Avg', 'Fee', 'avgfeerate', None ), - 'fee_min': bf( 'fe', 'bs', '{:>3}', 'Min', 'Fee', 'minfeerate', None ), - 'nTx': bf( None, 'bh', '{:>5}', '', ' nTx ', 'nTx', None ), - 'inputs': bf( None, 'bs', '{:>5}', 'In- ', 'puts', 'ins', None ), - 'outputs': bf( None, 'bs', '{:>5}', 'Out-', 'puts', 'outs', None ), - 'utxo_inc': bf( None, 'bs', '{:>6}', ' UTXO', ' Incr', 'utxo_increase', None ), - 'version': bf( None, 'bh', '{:<8}', '', 'Version', 'versionHex', None ), - 'difficulty': bf( 'di', 'bh', '{:<8}', 'Diffi-','culty', 'difficulty', None ), - 'miner': bf( None, 'lo', '{:<5}', '', 'Miner', 'miner', None ), - } + 'block': bf(None, 'bh', '{:<6}', '', 'Block', 'height', None), + 'hash': bf(None, 'bh', '{:<64}', '', 'Hash', 'hash', None), + 'date': bf('da', 'bh', '{:<19}', '', 'Date', 'time', None), + 'interval': bf('td', 'lo', '{:>8}', 'Solve', 'Time ', 'interval', None), + 'subsidy': bf('su', 'bs', '{:<5}', 'Sub-', 'sidy', 'subsidy', None), + 'totalfee': bf('tf', 'bs', '{:>10}', '', 'Total Fee', 'totalfee', None), + 'size': bf(None, 'bs', '{:>7}', '', 'Size', 'total_size', None), + 'weight': bf(None, 'bs', '{:>7}', '', 'Weight', 'total_weight', None), + 'fee90': bf('fe', 'bs', '{:>3}', '90%', 'Fee', 'feerate_percentiles', 4), + 'fee75': bf('fe', 'bs', '{:>3}', '75%', 'Fee', 'feerate_percentiles', 3), + 'fee50': bf('fe', 'bs', '{:>3}', '50%', 'Fee', 'feerate_percentiles', 2), + 'fee25': bf('fe', 'bs', '{:>3}', '25%', 'Fee', 'feerate_percentiles', 1), + 'fee10': bf('fe', 'bs', '{:>3}', '10%', 'Fee', 'feerate_percentiles', 0), + 'fee_max': bf('fe', 'bs', '{:>5}', 'Max', 'Fee', 'maxfeerate', None), + 'fee_avg': bf('fe', 'bs', '{:>3}', 'Avg', 'Fee', 'avgfeerate', None), + 'fee_min': bf('fe', 'bs', '{:>3}', 'Min', 'Fee', 'minfeerate', None), + 'nTx': bf(None, 'bh', '{:>5}', '', ' nTx ', 'nTx', None), + 'inputs': bf(None, 'bs', '{:>5}', 'In- ', 'puts', 'ins', None), + 'outputs': bf(None, 'bs', '{:>5}', 'Out-', 'puts', 'outs', None), + 'utxo_inc': bf(None, 'bs', '{:>6}', ' UTXO', ' Incr', 'utxo_increase', None), + 'version': bf(None, 'bh', '{:<8}', '', 'Version', 'versionHex', None), + 'difficulty': bf('di', 'bh', '{:<8}', 'Diffi-','culty', 'difficulty', None), + 'miner': bf(None, 'lo', '{:<5}', '', 'Miner', 'miner', None)} + dfl_fields = ( 'block', 'date', @@ -121,8 +120,8 @@ class BlocksInfo: 'fee10', 'fee_avg', 'fee_min', - 'version', - ) + 'version') + fixed_fields = ( 'block', # until ≈ 09/01/2028 (block 1000000) 'hash', @@ -131,36 +130,34 @@ class BlocksInfo: 'weight', # until ≈ 2.5x block size increase 'version', 'subsidy', # until ≈ 01/04/2028 (increases by 1 digit per halving until 9th halving [max 10 digits]) - 'difficulty', # until 1.00e+100 (i.e. never) - ) + 'difficulty') # until 1.00e+100 (i.e. never) # column width adjustment data: - fs_lsqueeze = ('totalfee','inputs','outputs','nTx') + fs_lsqueeze = ('totalfee', 'inputs', 'outputs', 'nTx') fs_rsqueeze = () fs_groups = ( - ('fee10','fee25','fee50','fee75','fee90','fee_avg','fee_min','fee_max'), - ) + ('fee10', 'fee25', 'fee50', 'fee75', 'fee90', 'fee_avg', 'fee_min', 'fee_max')) fs_lsqueeze2 = ('interval',) - all_stats = ['col_avg','range','avg','mini_avg','total','diff'] - dfl_stats = ['range','mini_avg','diff'] + all_stats = ['col_avg', 'range', 'avg', 'mini_avg', 'total', 'diff'] + dfl_stats = ['range', 'mini_avg', 'diff'] noindent_stats = ['col_avg'] - avg_stats_skip = {'block', 'hash', 'date', 'version','miner'} + avg_stats_skip = {'block', 'hash', 'date', 'version', 'miner'} - range_data = namedtuple('parsed_range_data',['first','last','from_tip','nblocks','step']) + range_data = namedtuple('parsed_range_data', ['first', 'last', 'from_tip', 'nblocks', 'step']) - t_fmt = lambda self,t: f'{t/86400:.2f} days' if t > 172800 else f'{t/3600:.2f} hrs' + t_fmt = lambda self, t: f'{t/86400:.2f} days' if t > 172800 else f'{t/3600:.2f} hrs' @classmethod - def parse_cslist(cls,uarg,full_set,dfl_set,desc): + def parse_cslist(cls, uarg, full_set, dfl_set, desc): - def make_list(m,func): + def make_list(m, func): groups_lc = [set(e.lower() for e in gi.split(',')) for gi in m.groups()] for group in groups_lc: for e in group: if e not in full_set_lc: - die(1,f'{e!r}: unrecognized {desc}') + die(1, f'{e!r}: unrecognized {desc}') # display elements in order: return [e for e in full_set if e.lower() in func(groups_lc)] @@ -168,33 +165,31 @@ class BlocksInfo: dfl_set_lc = set(e.lower() for e in dfl_set) cspat = r'(\w+(?:,\w+)*)' - for pat,func in ( - ( rf'{cspat}$', lambda g: g[0] ), - ( rf'\+{cspat}$', lambda g: dfl_set_lc | g[0] ), - ( rf'\-{cspat}$', lambda g: dfl_set_lc - g[0] ), - ( rf'\+{cspat}\-{cspat}$', lambda g: ( dfl_set_lc | g[0] ) - g[1] ), - ( rf'\-{cspat}\+{cspat}$', lambda g: ( dfl_set_lc - g[0] ) | g[1] ), - ( rf'all\-{cspat}$', lambda g: full_set_lc - g[0] ) - ): - m = re.match(pat,uarg,re.ASCII|re.IGNORECASE) + for pat, func in ( + (rf'{cspat}$', lambda g: g[0]), + (rf'\+{cspat}$', lambda g: dfl_set_lc | g[0]), + (rf'\-{cspat}$', lambda g: dfl_set_lc - g[0]), + (rf'\+{cspat}\-{cspat}$', lambda g: (dfl_set_lc | g[0]) - g[1]), + (rf'\-{cspat}\+{cspat}$', lambda g: (dfl_set_lc - g[0]) | g[1]), + (rf'all\-{cspat}$', lambda g: full_set_lc - g[0])): + m = re.match(pat, uarg, re.ASCII|re.IGNORECASE) if m: - return make_list(m,func) + return make_list(m, func) else: - die(1,f'{uarg}: invalid parameter') + die(1, f'{uarg}: invalid parameter') - def __init__(self,cfg,cmd_args,rpc): + def __init__(self, cfg, cmd_args, rpc): - def parse_cs_uarg(uarg,full_set,dfl_set,desc): + def parse_cs_uarg(uarg, full_set, dfl_set, desc): return ( full_set if uarg == 'all' else [] if uarg == 'none' else - self.parse_cslist(uarg,full_set,dfl_set,desc) - ) + self.parse_cslist(uarg, full_set, dfl_set, desc)) def get_fields(): - return parse_cs_uarg(self.cfg.fields,list(self.fields),self.dfl_fields,'field') + return parse_cs_uarg(self.cfg.fields, list(self.fields), self.dfl_fields, 'field') def get_stats(): - return parse_cs_uarg(self.cfg.stats.lower(),self.all_stats,self.dfl_stats,'stat') + return parse_cs_uarg(self.cfg.stats.lower(), self.all_stats, self.dfl_stats, 'stat') def parse_cmd_args(): # => (block_list, first, last, step) match cmd_args: @@ -217,7 +212,7 @@ class BlocksInfo: from_satoshi = self.rpc.proto.coin_amt.satoshi to_satoshi = 1 / from_satoshi - self.block_list,self.first,self.last,self.step = parse_cmd_args() + self.block_list, self.first, self.last, self.step = parse_cmd_args() have_segwit = self.rpc.info('segwit_is_active') @@ -228,35 +223,33 @@ class BlocksInfo: self.stats_deps = { 'avg': set(self.fields) - self.avg_stats_skip, 'col_avg': set(self.fields) - self.avg_stats_skip, - 'mini_avg': {'interval','size'} | ({'weight'} if have_segwit else set()), - 'total': {'interval','subsidy','totalfee','nTx','inputs','outputs','utxo_inc'}, + 'mini_avg': {'interval', 'size'} | ({'weight'} if have_segwit else set()), + 'total': {'interval', 'subsidy', 'totalfee', 'nTx', 'inputs', 'outputs', 'utxo_inc'}, 'range': {}, - 'diff': {}, - } + 'diff': {}} self.fmt_funcs = { - 'da': lambda arg: strftime('%Y-%m-%d %X',gmtime(arg)), + 'da': lambda arg: strftime('%Y-%m-%d %X', gmtime(arg)), 'td': lambda arg: ( - '-{:02}:{:02}'.format(abs(arg)//60,abs(arg)%60) if arg < 0 else - ' {:02}:{:02}'.format(arg//60,arg%60) ), + '-{:02}:{:02}'.format(abs(arg)//60, abs(arg)%60) if arg < 0 else + ' {:02}:{:02}'.format(arg//60, arg%60)), 'tf': lambda arg: '{:.8f}'.format(arg * from_satoshi), 'su': lambda arg: str(arg * from_satoshi).rstrip('0').rstrip('.'), 'fe': lambda arg: str(arg), - 'di': lambda arg: '{:.2e}'.format(Decimal(arg)), - } + 'di': lambda arg: '{:.2e}'.format(Decimal(arg))} if self.cfg.coin == 'BCH': self.fmt_funcs.update({ 'su': lambda arg: str(arg).rstrip('0').rstrip('.'), 'fe': lambda arg: str(int(Decimal(arg) * to_satoshi)), - 'tf': lambda arg: '{:.8f}'.format(Decimal(arg)), - }) + 'tf': lambda arg: '{:.8f}'.format(Decimal(arg))}) self.fnames = tuple( - [f for f in self.fields if self.fields[f].src == 'bh' or f == 'interval'] if self.cfg.header_info else - get_fields() if self.cfg.fields else - self.dfl_fields - ) + [f for f in self.fields if self.fields[f].src == 'bh' or f == 'interval'] + if self.cfg.header_info + else get_fields() if self.cfg.fields + else self.dfl_fields) + if self.cfg.miner_info and 'miner' not in self.fnames: self.fnames += ('miner',) @@ -266,15 +259,15 @@ class BlocksInfo: if 'diff' in self.stats and not self.cfg.stats and self.last != self.tip: self.stats.remove('diff') - if {'avg','col_avg'} <= set(self.stats) and self.cfg.stats_only: + if {'avg', 'col_avg'} <= set(self.stats) and self.cfg.stats_only: self.stats.remove('col_avg') - if {'avg','mini_avg'} <= set(self.stats): + if {'avg', 'mini_avg'} <= set(self.stats): self.stats.remove('mini_avg') if self.cfg.full_stats: add_fnames = {fname for sname in self.stats for fname in self.stats_deps[sname]} - self.fnames = tuple(f for f in self.fields if f in {'block'} | set(self.fnames) | add_fnames ) + self.fnames = tuple(f for f in self.fields if f in {'block'} | set(self.fnames) | add_fnames) else: if 'col_avg' in self.stats and not self.fnames: self.stats.remove('col_avg') @@ -287,8 +280,7 @@ class BlocksInfo: self.bs_keys = set( [v.key1 for v in self.fvals if v.src == 'bs'] + ['total_size'] + - (['total_weight'] if have_segwit else []) - ) + (['total_weight'] if have_segwit else [])) if 'miner' in self.fnames: # capturing parens must contain only ASCII chars! @@ -302,17 +294,16 @@ class BlocksInfo: rb'([\x20-\x7e]{9,})', rb'[/^]([a-zA-Z0-9&. #/-]{5,})', rb'[/^]([_a-zA-Z0-9&. #/-]+)/', - rb'^\x03...\W{0,5}([\\_a-zA-Z0-9&. #/-]+)[/\\]', - )] + rb'^\x03...\W{0,5}([\\_a-zA-Z0-9&. #/-]+)[/\\]')] - self.block_data = namedtuple('block_data',self.fnames) - self.deps = { v.src for v in self.fvals } + self.block_data = namedtuple('block_data', self.fnames) + self.deps = {v.src for v in self.fvals} - def gen_fs(self,fnames,fill=[],fill_char='-',add_name=False): + def gen_fs(self, fnames, fill=[], fill_char='-', add_name=False): for i in range(len(fnames)): name = fnames[i] - ls = (' ','')[name in self.fs_lsqueeze + self.fs_lsqueeze2] - rs = (' ','')[name in self.fs_rsqueeze] + ls = (' ', '')[name in self.fs_lsqueeze + self.fs_lsqueeze2] + rs = (' ', '')[name in self.fs_rsqueeze] if i < len(fnames) - 1 and fnames[i+1] in self.fs_lsqueeze2: rs = '' if i: @@ -321,7 +312,7 @@ class BlocksInfo: ls = '' break repl = (name if add_name else '') + ':' + (fill_char if name in fill else '') - yield (ls + self.fields[name].fs.replace(':',repl) + rs) + yield (ls + self.fields[name].fs.replace(':', repl) + rs) def conv_blkspec(self, arg): match arg: @@ -347,22 +338,22 @@ class BlocksInfo: case _: return arg - def parse_rangespec(self,arg): + def parse_rangespec(self, arg): - p = RangeParser(self,arg) + p = RangeParser(self, arg) - from_tip = p.parse('from_tip') - first,last = (self.tip-from_tip,None) if from_tip else p.parse('abs_range') - add1 = p.parse('add') - add2 = p.parse('add') + from_tip = p.parse('from_tip') + first, last = (self.tip-from_tip, None) if from_tip else p.parse('abs_range') + add1 = p.parse('add') + add2 = p.parse('add') p.finalize() if add2 and last is not None: - die(1,f'{arg!r}: invalid range specifier') + die(1, f'{arg!r}: invalid range specifier') - nblocks,step = (add1,add2) if last is None else (None,add1) + nblocks, step = (add1, add2) if last is None else (None, add1) - if p.debug: msg(repr(self.range_data(first,last,from_tip,nblocks,step))) + if p.debug: msg(repr(self.range_data(first, last, from_tip, nblocks, step))) if nblocks: if first is None: @@ -373,12 +364,12 @@ class BlocksInfo: last = self.conv_blkspec(last or first) if p.debug: - msg(repr(self.range_data(first,last,from_tip,nblocks,step))) + msg(repr(self.range_data(first, last, from_tip, nblocks, step))) if first > last: - die(1,f'{first}-{last}: invalid block range') + die(1, f'{first}-{last}: invalid block range') - return self.range_data(first,last,from_tip,nblocks,step) + return self.range_data(first, last, from_tip, nblocks, step) async def process_blocks(self): @@ -388,7 +379,7 @@ class BlocksInfo: c = self.rpc - heights = self.block_list or range(self.first,self.last+1) + heights = self.block_list or range(self.first, self.last+1) self.hdrs = await get_hdrs(heights) if self.block_list: @@ -397,8 +388,7 @@ class BlocksInfo: else: self.first_prev_hdr = ( self.hdrs[0] if heights[0] == 0 else - await c.call('getblockheader',await c.call('getblockhash',heights[0]-1)) - ) + await c.call('getblockheader', await c.call('getblockhash', heights[0]-1))) self.t_cur = self.first_prev_hdr['time'] self.res = [] @@ -409,16 +399,16 @@ class BlocksInfo: ret = await self.process_block(self.hdrs[n]) self.res.append(ret) if self.fnames and not self.cfg.stats_only: - self.output_block(ret,n) + self.output_block(ret, n) - def output_block(self,data,n): + def output_block(self, data, n): def gen(): - for k,v in data._asdict().items(): + for k, v in data._asdict().items(): func = self.fields[k].fmt_func yield self.fmt_funcs[func](v) if func else v Msg(self.fs.format(*gen())) - async def process_block(self,hdr): + async def process_block(self, hdr): self.t_diff = hdr['time'] - self.t_cur self.t_cur = hdr['time'] @@ -426,14 +416,12 @@ class BlocksInfo: blk_data = { 'bh': hdr, - 'lo': { 'interval': self.t_diff } - } + 'lo': {'interval': self.t_diff}} if 'bs' in self.deps: bs = ( self.genesis_stats if hdr['height'] == 0 else - await self.rpc.call('getblockstats',hdr['hash'],list(self.bs_keys)) - ) + await self.rpc.call('getblockstats', hdr['hash'], list(self.bs_keys))) self.total_bytes += bs['total_size'] if 'total_weight' in bs: self.total_weight += bs['total_weight'] @@ -446,14 +434,13 @@ class BlocksInfo: for v in self.fvals: yield ( blk_data[v.src][v.key1] if v.key2 is None else - blk_data[v.src][v.key1][v.key2] - ) + blk_data[v.src][v.key1][v.key2]) return self.block_data(*gen()) - async def get_miner_string(self,H): - tx0 = (await self.rpc.call('getblock',H))['tx'][0] - bd = await self.rpc.call('getrawtransaction',tx0,1) + async def get_miner_string(self, H): + tx0 = (await self.rpc.call('getblock', H))['tx'][0] + bd = await self.rpc.call('getrawtransaction', tx0, 1) if type(bd) == tuple: return '---' else: @@ -464,13 +451,12 @@ class BlocksInfo: trmap_in = { '\\': ' ', '/': ' ', - ',': ' ', - } - trmap = { ord(a):b for a,b in trmap_in.items() } + ',': ' '} + trmap = {ord(a): b for a, b in trmap_in.items()} for pat in self.miner_pats: m = pat.search(cb) if m: - return re.sub( r'\s+', ' ', m[1].decode().strip('^').translate(trmap).strip() ) + return re.sub(r'\s+', ' ', m[1].decode().strip('^').translate(trmap).strip()) return '' def print_header(self): @@ -484,11 +470,11 @@ class BlocksInfo: yield self.fs.format(*hdr1) yield self.fs.format(*hdr2) - def process_stats(self,sname): - method = getattr(self,f'create_{sname}_stats',None) - return self.output_stats(method() if method else self.create_stats(sname),sname) + def process_stats(self, sname): + method = getattr(self, f'create_{sname}_stats', None) + return self.output_stats(method() if method else self.create_stats(sname), sname) - def fmt_stat_item(self,fs,s): + def fmt_stat_item(self, fs, s): return fs.format(s) if type(fs) == str else fs(s) async def output_stats(self, res, sname): @@ -497,7 +483,7 @@ class BlocksInfo: for d in data: match d: case [a, b]: - yield (indent + a).format(**{k:self.fmt_stat_item(*v) for k, v in b.items()}) + yield (indent + a).format(**{k: self.fmt_stat_item(*v) for k, v in b.items()}) case [a, _, b, c]: yield (indent + a).format(self.fmt_stat_item(b, c)) case str(): @@ -524,15 +510,14 @@ class BlocksInfo: 'range': ('{}', self.hdrs[-1]['height'] - self.hdrs[0]['height'] + 1), 'elapsed': (self.t_fmt, elapsed), 'nBlocks': ('{}', total_blks), - 'step': ('{}', self.step), - } - ) - if elapsed: - yield ( 'Start: {}', 'start_date', self.fmt_funcs['da'], self.hdrs[0]['time'] ) - yield ( 'End: {}', 'end_date', self.fmt_funcs['da'], self.hdrs[-1]['time'] ) - yield ( 'Avg BDI: {} min', 'avg_bdi', '{:.2f}', elapsed / nblocks / 60 ) + 'step': ('{}', self.step)}) - return ( 'range', gen() ) + if elapsed: + yield ('Start: {}', 'start_date', self.fmt_funcs['da'], self.hdrs[0]['time']) + yield ('End: {}', 'end_date', self.fmt_funcs['da'], self.hdrs[-1]['time']) + yield ('Avg BDI: {} min', 'avg_bdi', '{:.2f}', elapsed / nblocks / 60) + + return ('range', gen()) async def create_diff_stats(self): @@ -541,18 +526,17 @@ class BlocksInfo: tip_hdr = ( self.hdrs[-1] if self.hdrs[-1]['height'] == self.tip else - await c.call('getblockheader',await c.call('getblockhash',self.tip)) - ) + await c.call('getblockheader', await c.call('getblockhash', self.tip))) min_sample_blks = 432 # ≈3 days - rel_hdr = await c.call('getblockheader',await c.call('getblockhash',self.tip-rel)) + rel_hdr = await c.call('getblockheader', await c.call('getblockhash', self.tip-rel)) if rel >= min_sample_blks: sample_blks = rel - bdi = ( tip_hdr['time'] - rel_hdr['time'] ) / rel + bdi = (tip_hdr['time'] - rel_hdr['time']) / rel else: - sample_blks = min(min_sample_blks,self.tip) - start_hdr = await c.call('getblockheader',await c.call('getblockhash',self.tip-sample_blks)) + sample_blks = min(min_sample_blks, self.tip) + start_hdr = await c.call('getblockheader', await c.call('getblockhash', self.tip-sample_blks)) diff_adj = Decimal(tip_hdr['difficulty']) / Decimal(start_hdr['difficulty']) time1 = rel_hdr['time'] - start_hdr['time'] time2 = tip_hdr['time'] - rel_hdr['time'] @@ -560,7 +544,7 @@ class BlocksInfo: rem = self.rpc.proto.diff_adjust_interval - rel - return ( 'difficulty', ( + return ('difficulty', ( 'Difficulty Statistics:', ('Current height: {}', 'chain_tip', '{}', self.tip), ('Next diff adjust: {next_diff_adjust} (in {blks_remaining} block%s [{time_remaining}])' % suf(rem), @@ -593,105 +577,98 @@ class BlocksInfo: def gen(): for field in self.fnames: if field in self.avg_stats_skip: - yield ( field, ('{}','') ) + yield (field, ('{}', '')) else: ret = self.sum_field_avg(field) func = self.fields[field].fmt_func - yield ( field, ( (self.fmt_funcs[func] if func else '{}'), ret )) + yield (field, ((self.fmt_funcs[func] if func else '{}'), ret)) if not self.header_printed: self.print_header() - fs = ''.join(self.gen_fs(self.fnames,fill=self.avg_stats_skip,add_name=True)).strip() - return ('column_averages', ('Column averages:', (fs, dict(gen())) )) + fs = ''.join(self.gen_fs(self.fnames, fill=self.avg_stats_skip, add_name=True)).strip() + return ('column_averages', ('Column averages:', (fs, dict(gen())))) - def avg_stats_data(self,data,spec_conv,spec_val): + def avg_stats_data(self, data, spec_conv, spec_val): coin = self.rpc.proto.coin return data( hdr = 'Averages for processed blocks:', func = self.sum_field_avg, - spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}' }, + spec_sufs = {'subsidy': f' {coin}', 'totalfee': f' {coin}'}, spec_convs = { - 'interval': spec_conv(0, lambda arg: secs_to_ms(arg)), + 'interval': spec_conv(0, lambda arg: secs_to_ms(arg)), 'utxo_inc': spec_conv(-1, '{:<+}'), - 'mb_per_hour': spec_conv(0, '{}'), - }, + 'mb_per_hour': spec_conv(0, '{}')}, spec_vals = ( spec_val( 'mb_per_hour', 'MB/hr', 'interval', lambda values: 'bs' in self.deps, lambda values: ( '{:.4f}'.format((self.total_bytes / 10000) / (self.total_solve_time / 36)) - if self.total_solve_time else 'N/A' ), - ), - ) - ) + if self.total_solve_time else 'N/A')), + )) mini_avg_stats_data = avg_stats_data - def total_stats_data(self,data,spec_conv,spec_val): + def total_stats_data(self, data, spec_conv, spec_val): coin = self.rpc.proto.coin return data( hdr = 'Totals for processed blocks:', func = self.sum_field_total, - spec_sufs = { 'subsidy': f' {coin}', 'totalfee': f' {coin}', 'reward': f' {coin}' }, + spec_sufs = {'subsidy': f' {coin}', 'totalfee': f' {coin}', 'reward': f' {coin}'}, spec_convs = { - 'interval': spec_conv(0, lambda arg: secs_to_dhms(arg)), + 'interval': spec_conv(0, lambda arg: secs_to_dhms(arg)), 'utxo_inc': spec_conv(-1, '{:<+}'), - 'reward': spec_conv(0, self.fmt_funcs['tf']), - }, + 'reward': spec_conv(0, self.fmt_funcs['tf'])}, spec_vals = ( spec_val( 'reward', 'Reward', 'totalfee', - lambda values: {'subsidy','totalfee'} <= set(values), - lambda values: values['subsidy'] + values['totalfee'] - ), - ) - ) + lambda values: {'subsidy', 'totalfee'} <= set(values), + lambda values: values['subsidy'] + values['totalfee']), + )) - async def create_stats(self,sname): + async def create_stats(self, sname): def convert_stats_hdr(field): v = self.fields[field] - return '{} {}'.format(v.hdr1.strip(), v.hdr2.strip()).replace('- ','') if v.hdr1 else v.hdr2.strip() + return '{} {}'.format( + v.hdr1.strip(), v.hdr2.strip()).replace('- ', '') if v.hdr1 else v.hdr2.strip() - d = getattr(self,f'{sname}_stats_data')( - namedtuple('stats_data',['hdr','func','spec_sufs','spec_convs','spec_vals']), - namedtuple('spec_conv',['width_adj','conv']), - namedtuple('spec_val',['name','lbl','insert_after','condition','code']) - ) + d = getattr(self, f'{sname}_stats_data')( + namedtuple('stats_data', ['hdr', 'func', 'spec_sufs', 'spec_convs', 'spec_vals']), + namedtuple('spec_conv', ['width_adj', 'conv']), + namedtuple('spec_val', ['name', 'lbl', 'insert_after', 'condition', 'code'])) fnames = [n for n in self.fnames if n in self.stats_deps[sname]] - lbls = {n:convert_stats_hdr(n) for n in fnames} - values = {n:d.func(n) for n in fnames} - col1_w = max((len(l) for l in lbls.values()),default=0) + 2 + lbls = {n: convert_stats_hdr(n) for n in fnames} + values = {n: d.func(n) for n in fnames} + col1_w = max((len(l) for l in lbls.values()), default=0) + 2 + print(d.spec_vals) for v in d.spec_vals: + print(v) if v.condition(values): try: idx = fnames.index(v.insert_after) + 1 except: idx = 0 - fnames.insert(idx,v.name) + fnames.insert(idx, v.name) lbls[v.name] = v.lbl values[v.name] = v.code(values) def gen(): - for n,fname in enumerate(fnames): + for n, fname in enumerate(fnames): spec_conv = d.spec_convs.get(fname) yield ( '{lbl:{wid}} {{}}{suf}'.format( lbl = lbls[fname] + ':', wid = col1_w + (spec_conv.width_adj if spec_conv else 0), - suf = d.spec_sufs.get(fname) or '' - ), + suf = d.spec_sufs.get(fname) or ''), fname, spec_conv.conv if spec_conv else ( - (lambda x: self.fmt_funcs[x] if x else '{}')(self.fields[fname].fmt_func) - ), - values[fname] - ) + (lambda x: self.fmt_funcs[x] if x else '{}')(self.fields[fname].fmt_func)), + values[fname]) - return ( sname, (d.hdr,) + tuple(gen()) ) + return (sname, (d.hdr,) + tuple(gen())) - def process_stats_pre(self,i): + def process_stats_pre(self, i): if (self.fnames and not self.cfg.stats_only) or i != 0: Msg('') @@ -724,13 +701,12 @@ class BlocksInfo: 'totalfee': 0, 'txs': 1, 'utxo_increase': 1, - 'utxo_size_inc': 117 - } + 'utxo_size_inc': 117} class JSONBlocksInfo(BlocksInfo): - def __init__(self,cfg,cmd_args,rpc): - super().__init__(cfg,cmd_args,rpc) + def __init__(self, cfg, cmd_args, rpc): + super().__init__(cfg, cmd_args, rpc) if self.cfg.json_raw: self.output_block = self.output_block_raw self.fmt_stat_item = self.fmt_stat_item_raw @@ -741,22 +717,22 @@ class JSONBlocksInfo(BlocksInfo): await super().process_blocks() Msg_r(']') - def output_block_raw(self,data,n): - Msg_r( (', ','')[n==0] + json.dumps(data._asdict(),cls=json_encoder) ) + def output_block_raw(self, data, n): + Msg_r((', ', '')[n==0] + json.dumps(data._asdict(), cls=json_encoder)) - def output_block(self,data,n): + def output_block(self, data, n): def gen(): - for k,v in data._asdict().items(): + for k, v in data._asdict().items(): func = self.fields[k].fmt_func - yield ( k, (self.fmt_funcs[func](v) if func else v) ) - Msg_r( (', ','')[n==0] + json.dumps(dict(gen()),cls=json_encoder) ) + yield (k, (self.fmt_funcs[func](v) if func else v)) + Msg_r((', ', '')[n==0] + json.dumps(dict(gen()), cls=json_encoder)) def print_header(self): pass - def fmt_stat_item_raw(self,fs,s): + def fmt_stat_item_raw(self, fs, s): return s - async def output_stats(self,res,sname): + async def output_stats(self, res, sname): def gen(data): for d in data: @@ -771,10 +747,10 @@ class JSONBlocksInfo(BlocksInfo): case _: assert False, f'{d}: invalid stats data' - varname,data = await res - Msg_r(', "{}_data": {}'.format( varname, json.dumps(dict(gen(data)),cls=json_encoder) )) + varname, data = await res + Msg_r(', "{}_data": {}'.format(varname, json.dumps(dict(gen(data)), cls=json_encoder))) - def process_stats_pre(self,i): pass + def process_stats_pre(self, i): pass def finalize_output(self): Msg('}') diff --git a/mmgen_node_tools/PeerBlocks.py b/mmgen_node_tools/PeerBlocks.py index c3d7c68..3329f07 100755 --- a/mmgen_node_tools/PeerBlocks.py +++ b/mmgen_node_tools/PeerBlocks.py @@ -14,66 +14,66 @@ mmgen_node_tools.PeerBlocks: List blocks in flight, disconnect stalling nodes import asyncio from collections import namedtuple -from mmgen.util import msg,msg_r,is_int -from mmgen.term import get_term,get_terminal_size,get_char +from mmgen.util import msg, msg_r, is_int +from mmgen.term import get_term, get_terminal_size, get_char from mmgen.ui import line_input from .PollDisplay import PollDisplay -RED,RESET = ('\033[31m','\033[0m') -COLORS = ['\033[38;5;%s;1m' % c for c in list(range(247,256)) + [231]] -ERASE_ALL,CUR_HOME = ('\033[J','\033[H') -CUR_HIDE,CUR_SHOW = ('\033[?25l','\033[?25h') +RED, RESET = ('\033[31m', '\033[0m') +COLORS = ['\033[38;5;%s;1m' % c for c in list(range(247, 256)) + [231]] +ERASE_ALL, CUR_HOME = ('\033[J', '\033[H') +CUR_HIDE, CUR_SHOW = ('\033[?25l', '\033[?25h') term = None class Display(PollDisplay): poll_secs = 2 - def __init__(self,cfg): + def __init__(self, cfg): super().__init__(cfg) - global term,term_width + global term, term_width if not term: term = get_term() term.init(noecho=True) term_width = self.cfg.columns or get_terminal_size().width msg_r(CUR_HOME+ERASE_ALL+CUR_HOME) - async def get_info(self,rpc): + async def get_info(self, rpc): return await rpc.call('getpeerinfo') - def display(self,count): + def display(self, count): msg_r( CUR_HOME + (ERASE_ALL if count == 1 else '') + 'CONNECTED PEERS ({a}) {b} - poll {c}'.format( a = len(self.info), b = self.desc, - c = count ).ljust(term_width)[:term_width] + c = count).ljust(term_width)[:term_width] + '\n' + ('\n'.join(self.gen_display()) + '\n' if self.info else '') + ERASE_ALL + f"Type a peer number to disconnect, 'q' to quit, or any other key for {self.other_desc} display:" - + '\b' ) + + '\b') - async def disconnect_node(self,rpc,addr): - return await rpc.call('disconnectnode',addr) + async def disconnect_node(self, rpc, addr): + return await rpc.call('disconnectnode', addr) def get_input(self): - s = get_char(immed_chars='q0123456789',prehold_protect=False,num_bytes=1) + s = get_char(immed_chars='q0123456789', prehold_protect=False, num_bytes=1) if not is_int(s): return s with self.info_lock: msg('') term.reset() # readline required for correct operation here; without it, user must re-type first digit - ret = line_input( self.cfg, 'peer number> ', insert_txt=s, hold_protect=False ) + ret = line_input(self.cfg, 'peer number> ', insert_txt=s, hold_protect=False) term.init(noecho=True) self.enable_display = False # prevent display from updating before process_input() return ret - async def process_input(self,rpc): + async def process_input(self, rpc): ids = tuple(str(i['id']) for i in self.info) ret = False @@ -83,7 +83,7 @@ class Display(PollDisplay): from mmgen.exception import RPCFailure addr = self.info[ids.index(self.input)]['addr'] try: - await self.disconnect_node(rpc,addr) + await self.disconnect_node(rpc, addr) except RPCFailure: msg_r(f'Unable to disconnect peer {self.input} ({addr})') else: @@ -105,8 +105,8 @@ class BlocksDisplay(Display): def gen_display(self): - pd = namedtuple('peer_data',['id','blks_data','blks_width']) - bd = namedtuple('block_datum',['num','disp']) + pd = namedtuple('peer_data', ['id', 'blks_data', 'blks_width']) + bd = namedtuple('block_datum', ['num', 'disp']) def gen_block_data(): global min_height @@ -114,15 +114,15 @@ class BlocksDisplay(Display): for d in self.info: if d.get('inflight'): blocks = d['inflight'] - min_height = min(blocks) if not min_height else min(min_height,min(blocks)) - line = ' '.join(map(str,blocks))[:blks_field_width] + min_height = min(blocks) if not min_height else min(min_height, min(blocks)) + line = ' '.join(map(str, blocks))[:blks_field_width] blocks_disp = line.split() yield pd( d['id'], - [bd(blocks[i],blocks_disp[i]) for i in range(len(blocks_disp))], - len(line) ) + [bd(blocks[i], blocks_disp[i]) for i in range(len(blocks_disp))], + len(line)) else: - yield pd(d['id'],[],0) + yield pd(d['id'], [], 0) def gen_line(peer_data): for blk in peer_data.blks_data: @@ -136,7 +136,7 @@ class BlocksDisplay(Display): for peer_data in tuple(gen_block_data()): yield fs.format( peer_data.id, - ' '.join(gen_line(peer_data)) + ' ' * (blks_field_width - peer_data.blks_width) ) + ' '.join(gen_line(peer_data)) + ' ' * (blks_field_width - peer_data.blks_width)) class PeersDisplay(Display): @@ -152,5 +152,4 @@ class PeersDisplay(Display): A = id_width, b = d['addr'], B = addr_width, - c = d['subver'] - ).ljust(term_width)[:term_width] + c = d['subver']).ljust(term_width)[:term_width] diff --git a/mmgen_node_tools/PollDisplay.py b/mmgen_node_tools/PollDisplay.py index 1de8a1a..8274e40 100755 --- a/mmgen_node_tools/PollDisplay.py +++ b/mmgen_node_tools/PollDisplay.py @@ -12,7 +12,7 @@ mmgen_node_tools.PollDisplay: update and display RPC data; get input from user """ -import sys,threading +import sys, threading from mmgen.util import msg from mmgen.term import get_char @@ -22,18 +22,18 @@ class PollDisplay: input = None poll_secs = 1 - def __init__(self,cfg): + def __init__(self, cfg): self.cfg = cfg self.info_lock = threading.Lock() # self.info accessed by 2 threads self.display_kill_flag = threading.Event() def get_input(self): - return get_char(immed_chars='q',prehold_protect=False,num_bytes=1) + return get_char(immed_chars='q', prehold_protect=False, num_bytes=1) - async def process_input(self,rpc): + async def process_input(self, rpc): return True - async def run(self,rpc): + async def run(self, rpc): async def do_display(): with self.info_lock: @@ -68,7 +68,7 @@ class PollDisplay: self.display_kill_flag.set() while True: - threading.Thread(target=get_input,daemon=True).start() + threading.Thread(target=get_input, daemon=True).start() await do_display() if await process_input(): break diff --git a/mmgen_node_tools/Sound.py b/mmgen_node_tools/Sound.py index df9f8ae..557b97a 100755 --- a/mmgen_node_tools/Sound.py +++ b/mmgen_node_tools/Sound.py @@ -19,28 +19,28 @@ mmgen_node_tools.Sound: audio-related functions for MMGen node tools """ -import sys,os,time +import sys, os, time from mmgen.util import die from mmgen_node_tools.Util import do_system _alsa_config_file = '/tmp/alsa-config-' + os.path.basename(sys.argv[0]) -_dvols = { 'Master': 78, 'Speaker': 78, 'Headphone': 15, 'PCM': 190 } +_dvols = {'Master': 78, 'Speaker': 78, 'Headphone': 15, 'PCM': 190} def timespec2secs(ts): import re - mul = { 's': 1, 'm': 60, 'h': 60*60, 'd': 60*60*24 } + mul = {'s': 1, 'm': 60, 'h': 60*60, 'd': 60*60*24} pat = r'^([0-9]+)([smhd]*)$' - m = re.match(pat,ts) + m = re.match(pat, ts) if m is None: die(2,"'%s': invalid time specifier" % ts) - a,b = m.groups() + a, b = m.groups() return int(a) * (mul[b] if b else 1) def parse_repeat_spec(rs): - return [(timespec2secs(i),timespec2secs(j)) - for i,j in [a.split(':') for a in rs.split(',')]] + return [(timespec2secs(i), timespec2secs(j)) + for i, j in [a.split(':') for a in rs.split(',')]] def init_sound(): def _restore_sound(): @@ -51,33 +51,33 @@ def init_sound(): atexit.register(_restore_sound) do_system('sudo alsactl store -f ' + _alsa_config_file) -def play_sound(fn,vol,repeat_spec='',remote_host='',kill_flg=None,testing=False): +def play_sound(fn, vol, repeat_spec='', remote_host='', kill_flg=None, testing=False): if not remote_host: do_system('sudo alsactl store -f ' + _alsa_config_file) - for k in 'Master','Speaker','Headphone': - do_system(('sudo amixer -q set %s on' % k),testing) + for k in 'Master', 'Speaker', 'Headphone': + do_system(('sudo amixer -q set %s on' % k), testing) # do_system('amixer -q set Headphone off') - vols = dict([(k,int(_dvols[k] * float(vol) / 100)) for k in _dvols]) + vols = dict([(k, int(_dvols[k] * float(vol) / 100)) for k in _dvols]) for k in vols: - do_system('sudo amixer -q set %s %s' % (k,vols[k]),testing) + do_system('sudo amixer -q set %s %s' % (k, vols[k]), testing) fn = os.path.expanduser(fn) cmd = ( 'aplay -q %s' % fn, - 'ssh %s mmnode-play-sound -v%d %s' % (remote_host,vol,fn) + 'ssh %s mmnode-play-sound -v%d %s' % (remote_host, vol, fn) )[bool(remote_host)] if repeat_spec and kill_flg: - for interval,duration in parse_repeat_spec(repeat_spec): + for interval, duration in parse_repeat_spec(repeat_spec): start = time.time() while time.time() < start + duration: - do_system(cmd,testing) + do_system(cmd, testing) if kill_flg.wait(interval): if not remote_host: do_system('sudo alsactl restore -f ' + _alsa_config_file) return else: # Play once - do_system(cmd,testing) + do_system(cmd, testing) if not remote_host: do_system('sudo alsactl restore -f ' + _alsa_config_file) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 95e03f2..ebefcf3 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -29,10 +29,10 @@ from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, lis from mmgen.ui import do_pager homedir = os.getenv('HOME') -dfl_cachedir = os.path.join(homedir,'.cache','mmgen-node-tools') +dfl_cachedir = os.path.join(homedir, '.cache', 'mmgen-node-tools') cfg_fn = 'ticker-cfg.yaml' portfolio_fn = 'ticker-portfolio.yaml' -asset_tuple = namedtuple('asset_tuple',['symbol','id','source']) +asset_tuple = namedtuple('asset_tuple', ['symbol', 'id', 'source']) last_api_host = None percent_cols = { @@ -50,20 +50,19 @@ class DataSource: }, { 'fi': 'yahoospot', 'hi': 'yahoohist', - } - ] + }] @classmethod - def get_sources(cls,randomize=False): - g = random.sample(cls.source_groups,k=len(cls.source_groups)) if randomize else cls.source_groups - return {k:v for a in g for k,v in a.items()} + def get_sources(cls, randomize=False): + g = random.sample(cls.source_groups, k=len(cls.source_groups)) if randomize else cls.source_groups + return {k: v for a in g for k, v in a.items()} class base: def fetch_delay(self): global last_api_host if not gcfg.testing and last_api_host and last_api_host != self.api_host: - delay = 1 + random.randrange(1,5000) / 1000 + delay = 1 + random.randrange(1, 5000) / 1000 msg_r(f'Waiting {delay:.3f} seconds...') time.sleep(delay) msg('') @@ -72,19 +71,18 @@ class DataSource: def get_data_from_network(self): curl_cmd = list_gen( - ['curl', '--tr-encoding', '--header', 'Accept: application/json',True], + ['curl', '--tr-encoding', '--header', 'Accept: application/json', True], ['--compressed'], # adds 'Accept-Encoding: gzip' - ['--proxy', cfg.proxy, isinstance(cfg.proxy,str)], + ['--proxy', cfg.proxy, isinstance(cfg.proxy, str)], ['--silent', not cfg.verbose], - [self.api_url] - ) + [self.api_url]) if gcfg.testing: - Msg(fmt_list(curl_cmd,fmt='bare')) + Msg(fmt_list(curl_cmd, fmt='bare')) return try: - return run(curl_cmd,check=True,stdout=PIPE).stdout.decode() + return run(curl_cmd, check=True, stdout=PIPE).stdout.decode() except CalledProcessError as e: msg('') from .Misc import curl_exit_codes @@ -101,7 +99,7 @@ class DataSource: os.makedirs(cfg.cachedir) if not os.path.exists(self.json_fn): - open(self.json_fn,'w').write('{}') + open(self.json_fn, 'w').write('{}') use_cached_data = cfg.cached_data and not gcfg.download @@ -123,7 +121,7 @@ class DataSource: if gcfg.testing: return {} else: - die(1,self.rate_limit_errmsg(elapsed)) + die(1, self.rate_limit_errmsg(elapsed)) match data_type: case 'json': @@ -131,7 +129,7 @@ class DataSource: data = json.loads(data_in) except: self.json_data_error_msg(data_in) - die(2,'Retrieved data is not valid JSON, exiting') + die(2, 'Retrieved data is not valid JSON, exiting') json_text = data_in case 'python': data = data_in @@ -143,9 +141,9 @@ class DataSource: f'No cached {self.data_desc}! Run command without the --cached-data option, ' 'or use --download to retrieve data from remote host') else: - die(2,'Remote host returned no data!') + die(2, 'Remote host returned no data!') elif 'error' in data: - die(1,data['error']) + die(1, data['error']) if use_cached_data: if not cfg.quiet: @@ -162,15 +160,15 @@ class DataSource: return self.postprocess_data(data) - def json_data_error_msg(self,json_text): + def json_data_error_msg(self, json_text): pass - def postprocess_data(self,data): + def postprocess_data(self, data): return data @property def json_fn_rel(self): - return os.path.relpath(self.json_fn,start=homedir) + return os.path.relpath(self.json_fn, start=homedir) class coinpaprika(base): desc = 'CoinPaprika' @@ -185,30 +183,29 @@ class DataSource: def __init__(self): self.asset_limit = int(cfg.asset_limit or self.dfl_asset_limit) - def rate_limit_errmsg(self,elapsed): + def rate_limit_errmsg(self, elapsed): return ( f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds' + - ('' if cfg.btc_only else ', or use --cached-data or --btc') - ) + ('' if cfg.btc_only else ', or use --cached-data or --btc')) @property def api_url(self): return ( f'https://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else - f'https://{self.api_host}/v1/tickers' ) + f'https://{self.api_host}/v1/tickers') @property def json_fn(self): return os.path.join( cfg.cachedir, - 'ticker-btc.json' if cfg.btc_only else 'ticker.json' ) + 'ticker-btc.json' if cfg.btc_only else 'ticker.json') @property def timeout(self): return 0 if gcfg.test_suite else self.btc_ratelimit if cfg.btc_only else self.ratelimit - def json_data_error_msg(self,json_text): + def json_data_error_msg(self, json_text): tor_captcha_msg = f""" If you’re using Tor, the API request may have failed due to Captcha protection. A workaround for this issue is to retrieve the JSON data with a browser from @@ -223,20 +220,20 @@ class DataSource: Then invoke the program with --cached-data and without --btc """ msg(json_text[:1024] + '...') - msg(orange(fmt(tor_captcha_msg,strip_char='\t'))) + msg(orange(fmt(tor_captcha_msg, strip_char='\t'))) - def postprocess_data(self,data): + def postprocess_data(self, data): return [data] if cfg.btc_only else data @staticmethod - def parse_asset_id(s,require_label): - sym,label = (*s.split('-',1),None)[:2] + def parse_asset_id(s, require_label): + sym, label = (*s.split('-', 1), None)[:2] if require_label and not label: - die(1,f'{s!r}: asset label is missing') + die(1, f'{s!r}: asset label is missing') return asset_tuple( symbol = sym.upper(), id = (s.lower() if label else None), - source = 'cc' ) + source = 'cc') class yahoospot(base): @@ -250,12 +247,12 @@ class DataSource: json_fn_basename = 'ticker-finance.json' @staticmethod - def get_id(sym,data): + def get_id(sym, data): return sym.lower() @staticmethod - def conv_data(sym,data,btcusd): - price_usd = Decimal( data['regularMarketPrice']['raw'] ) + def conv_data(sym, data, btcusd): + price_usd = Decimal(data['regularMarketPrice']['raw']) return { 'id': sym, 'name': data['shortName'], @@ -266,15 +263,14 @@ class DataSource: 'percent_change_30d': data['pct_chg_4wks'], 'percent_change_7d': data['pct_chg_1wk'], 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, - 'last_updated': data['regularMarketTime'], - } + 'last_updated': data['regularMarketTime']} - def rate_limit_errmsg(self,elapsed): + def rate_limit_errmsg(self, elapsed): return f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds, or use --cached-data' @property def json_fn(self): - return os.path.join( cfg.cachedir, self.json_fn_basename ) + return os.path.join(cfg.cachedir, self.json_fn_basename) @property def timeout(self): @@ -282,37 +278,36 @@ class DataSource: @property def symbols(self): - return [r.symbol for r in cfg.rows if isinstance(r,tuple) and r.source == 'fi'] + return [r.symbol for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi'] def get_data_from_network(self): kwargs = { 'formatted': True, 'asynchronous': True, - 'proxies': { 'https': cfg.proxy2 }, - } + 'proxies': {'https': cfg.proxy2}} if gcfg.test_suite: - kwargs.update({ 'timeout': 1, 'retry': 0 }) + kwargs.update({'timeout': 1, 'retry': 0}) if gcfg.testing: Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format( self.symbols, - fmt_dict(kwargs,fmt='kwargs') )) + fmt_dict(kwargs, fmt='kwargs'))) return from yahooquery import Ticker - return self.process_network_data( Ticker(self.symbols,**kwargs) ) + return self.process_network_data(Ticker(self.symbols,**kwargs)) - def process_network_data(self,ticker): + def process_network_data(self, ticker): return ticker.price @staticmethod - def parse_asset_id(s,require_label): + def parse_asset_id(s, require_label): return asset_tuple( symbol = s.upper(), id = s.lower(), - source = 'fi' ) + source = 'fi') class yahoohist(yahoospot): @@ -322,17 +317,17 @@ class DataSource: period = '1y' interval = '1wk' - def process_network_data(self,ticker): + def process_network_data(self, ticker): return ticker.history( period = self.period, interval = self.interval).to_json(orient='index') - def postprocess_data(self,data): + def postprocess_data(self, data): def gen(): keys = set() d = {} for key, val in data.items(): - if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$",key): + if m := re.match(r"\('(.*?)', datetime\.date\((.*)\)\)$", key): date = '{}-{:>02}-{:>02}'.format(*m[2].split(', ')) if (sym := m[1]) in keys: d[date] = val @@ -343,12 +338,12 @@ class DataSource: return dict(gen()) def assets_list_gen(cfg_in): - for k,v in cfg_in.cfg['assets'].items(): + for k, v in cfg_in.cfg['assets'].items(): yield '' yield k.upper() for e in v: - out = e.split('-',1) - yield ' {:5s} {}'.format(out[0],out[1] if len(out) == 2 else '') + out = e.split('-', 1) + yield ' {:5s} {}'.format(out[0], out[1] if len(out) == 2 else '') def gen_data(data): """ @@ -367,10 +362,9 @@ def gen_data(data): f'The symbol {dup_sym!r} is shared by the following assets:\n' + '\n ' + '\n '.join(d['id'] for d in data['cc'] if d['symbol'] == dup_sym) + '\n\nPlease specify the asset by one of the full IDs listed above\n' + - f'instead of {dup_sym!r}' - ) + f'instead of {dup_sym!r}') - def check_assets_found(wants,found,keys=['symbol','id']): + def check_assets_found(wants, found, keys=['symbol', 'id']): error = False for k in keys: missing = wants[k] - found[k] @@ -378,36 +372,32 @@ def gen_data(data): msg( ('The following IDs were not found in source data:\n{}' if k == 'id' else 'The following symbols could not be resolved:\n{}').format( - fmt_list(missing,fmt='col',indent=' ') - )) + fmt_list(missing, fmt='col', indent=' '))) error = True if error: - die(1,'Missing data, exiting') + die(1, 'Missing data, exiting') rows_want = { - 'id': {r.id for r in cfg.rows if isinstance(r,tuple) and r.id} - {'usd-us-dollar'}, - 'symbol': {r.symbol for r in cfg.rows if isinstance(r,tuple) and r.id is None} - {'USD'}, - } + 'id': {r.id for r in cfg.rows if isinstance(r, tuple) and r.id} - {'usd-us-dollar'}, + 'symbol': {r.symbol for r in cfg.rows if isinstance(r, tuple) and r.id is None} - {'USD'}} usr_rate_assets = tuple(u.rate_asset for u in cfg.usr_rows + cfg.usr_columns if u.rate_asset) usr_rate_assets_want = { 'id': {a.id for a in usr_rate_assets if a.id}, - 'symbol': {a.symbol for a in usr_rate_assets if not a.id} - } + 'symbol': {a.symbol for a in usr_rate_assets if not a.id}} usr_assets = cfg.usr_rows + cfg.usr_columns + tuple(c for c in (cfg.query or ()) if c) usr_wants = { 'id': ( {a.id for a in usr_assets + usr_rate_assets if a.id} - - {a.id for a in usr_assets if a.rate and a.id} - {'usd-us-dollar'} ) + {a.id for a in usr_assets if a.rate and a.id} - {'usd-us-dollar'}) , 'symbol': ( {a.symbol for a in usr_assets + usr_rate_assets if not a.id} - - {a.symbol for a in usr_assets if a.rate} - {'USD'} ), - } + {a.symbol for a in usr_assets if a.rate} - {'USD'})} - found = { 'id': set(), 'symbol': set() } + found = {'id': set(), 'symbol': set()} rate_assets = {} - wants = {k:rows_want[k] | usr_wants[k] for k in ('id','symbol')} + wants = {k: rows_want[k] | usr_wants[k] for k in ('id', 'symbol')} for d in data['cc']: if d['id'] == 'btc-bitcoin': @@ -417,38 +407,37 @@ def gen_data(data): get_id = src_cls['fi'].get_id conv_func = src_cls['fi'].conv_data - for k,v in data['fi'].items(): - id = get_id(k,v) + for k, v in data['fi'].items(): + id = get_id(k, v) if wants['id']: if id in wants['id']: - if not isinstance(v,dict): + if not isinstance(v, dict): die(2, str(v)) if id in found['id']: - die(1,dup_sym_errmsg(id)) + die(1, dup_sym_errmsg(id)) if m := data['hi'].get(k): spot = v['regularMarketPrice']['raw'] hist = tuple(m.values()) v['pct_chg_1wk'], v['pct_chg_4wks'], v['pct_chg_1y'] = ( (spot / hist[-2]['close'] - 1) * 100, (spot / hist[-5]['close'] - 1) * 100, # 4 weeks ≈ 1 month - (spot / hist[0]['close'] - 1) * 100, - ) + (spot / hist[0]['close'] - 1) * 100) else: v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None - yield ( id, conv_func(id,v,btcusd) ) + yield (id, conv_func(id, v, btcusd)) found['id'].add(id) wants['id'].remove(id) if id in usr_rate_assets_want['id']: - rate_assets[k] = conv_func(id,v,btcusd) # NB: using symbol instead of ID for key + rate_assets[k] = conv_func(id, v, btcusd) # NB: using symbol instead of ID for key else: break - for k in ('id','symbol'): + for k in ('id', 'symbol'): for d in data['cc']: if wants[k]: if d[k] in wants[k]: if d[k] in found[k]: - die(1,dup_sym_errmsg(d[k])) + die(1, dup_sym_errmsg(d[k])) if not 'price_usd' in d: d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd @@ -457,8 +446,9 @@ def gen_data(data): d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] # .replace('Z','+00:00') -- Python 3.9 backport - d['last_updated'] = int(datetime.datetime.fromisoformat(d['last_updated'].replace('Z','+00:00')).timestamp()) - yield (d['id'],d) + d['last_updated'] = int(datetime.datetime.fromisoformat( + d['last_updated'].replace('Z', '+00:00')).timestamp()) + yield (d['id'], d) found[k].add(d[k]) wants[k].remove(d[k]) if d[k] in usr_rate_assets_want[k]: @@ -466,7 +456,7 @@ def gen_data(data): else: break - check_assets_found(usr_wants,found) + check_assets_found(usr_wants, found) for asset in (cfg.usr_rows + cfg.usr_columns): if asset.rate: @@ -475,14 +465,13 @@ def gen_data(data): """ _id = asset.id or f'{asset.symbol}-user-asset-{asset.symbol}'.lower() ra_rate = rate_assets[asset.rate_asset.symbol]['price_usd'] if asset.rate_asset else 1 - yield ( _id, { + yield (_id, { 'symbol': asset.symbol, 'id': _id, 'name': ' '.join(_id.split('-')[1:]), 'price_usd': ra_rate / asset.rate, 'price_btc': ra_rate / asset.rate / btcusd, - 'last_updated': None, - }) + 'last_updated': None}) yield ('usd-us-dollar', { 'symbol': 'USD', @@ -490,21 +479,20 @@ def gen_data(data): 'name': 'US Dollar', 'price_usd': Decimal(1), 'price_btc': Decimal(1) / btcusd, - 'last_updated': None, - }) + 'last_updated': None}) def main(): def update_sample_file(usr_cfg_file): - usr_data = files('mmgen_node_tools').joinpath('data',os.path.basename(usr_cfg_file)).read_text() + usr_data = files('mmgen_node_tools').joinpath('data', os.path.basename(usr_cfg_file)).read_text() sample_file = usr_cfg_file + '.sample' sample_data = open(sample_file).read() if os.path.exists(sample_file) else None if usr_data != sample_data: - os.makedirs(os.path.dirname(sample_file),exist_ok=True) + os.makedirs(os.path.dirname(sample_file), exist_ok=True) msg('{} {}'.format( - ('Updating','Creating')[sample_data is None], - sample_file )) - open(sample_file,'w').write(usr_data) + ('Updating', 'Creating')[sample_data is None], + sample_file)) + open(sample_file, 'w').write(usr_data) try: from importlib.resources import files # Python 3.9 @@ -515,19 +503,19 @@ def main(): update_sample_file(cfg_in.portfolio_file) if gcfg.portfolio and not cfg_in.portfolio: - die(1,'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format( - os.path.relpath(cfg_in.portfolio_file,start=homedir))) + die(1, 'No portfolio configured!\nTo configure a portfolio, edit the file ~/{}'.format( + os.path.relpath(cfg_in.portfolio_file, start=homedir))) if gcfg.list_ids: src_ids = ['cc'] elif gcfg.download: if not gcfg.download in DataSource.get_sources(): - die(1,f'{gcfg.download!r}: invalid data source') + die(1, f'{gcfg.download!r}: invalid data source') src_ids = [gcfg.download] else: src_ids = DataSource.get_sources(randomize=True) - src_data = { k: src_cls[k]().get_data() for k in src_ids } + src_data = {k: src_cls[k]().get_data() for k in src_ids} if gcfg.testing: return @@ -542,24 +530,24 @@ def main(): data = dict(gen_data(src_data)) (do_pager if cfg.pager else Msg_r)( - '\n'.join(getattr(Ticker,cfg.clsname)(data).gen_output()) + '\n') + '\n'.join(getattr(Ticker, cfg.clsname)(data).gen_output()) + '\n') def make_cfg(gcfg_arg): - query_tuple = namedtuple('query',['asset','to_asset']) - asset_data = namedtuple('asset_data',['symbol','id','amount','rate','rate_asset','source']) + query_tuple = namedtuple('query', ['asset', 'to_asset']) + asset_data = namedtuple('asset_data', ['symbol', 'id', 'amount', 'rate', 'rate_asset', 'source']) - def parse_asset_id(s,require_label=False): - return src_cls['fi' if re.match(fi_pat,s) else 'cc'].parse_asset_id(s,require_label) + def parse_asset_id(s, require_label=False): + return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) def get_rows_from_cfg(add_data=None): def gen(): - for n,(k,v) in enumerate(cfg_in.cfg['assets'].items()): + for n, (k, v) in enumerate(cfg_in.cfg['assets'].items()): yield k if add_data and k in add_data: v += tuple(add_data[k]) for e in v: - yield parse_asset_id(e,require_label=True) + yield parse_asset_id(e, require_label=True) return tuple(gen()) def parse_percent_cols(arg): @@ -568,17 +556,19 @@ def make_cfg(gcfg_arg): res = arg.lower().split(',') for s in res: if s not in percent_cols: - die(1,f'{arg!r}: invalid --percent-cols parameter (valid letters: {fmt_list(percent_cols)})') + die(1, '{!r}: invalid --percent-cols parameter (valid letters: {})'.format( + arg, + fmt_list(percent_cols))) return res - def parse_usr_asset_arg(key,use_cf_file=False): + def parse_usr_asset_arg(key, use_cf_file=False): """ asset_id[:rate[:rate_asset]] """ def parse_parm(s): ss = s.split(':') - assert len(ss) in (1,2,3), f'{s}: malformed argument' - asset_id,rate,rate_asset = (*ss,None,None)[:3] + assert len(ss) in (1, 2, 3), f'{s}: malformed argument' + asset_id, rate, rate_asset = (*ss, None, None)[:3] parsed_id = parse_asset_id(asset_id) return asset_data( @@ -588,19 +578,19 @@ def make_cfg(gcfg_arg): rate = ( None if rate is None else 1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else - Decimal(rate) ), + Decimal(rate)), rate_asset = parse_asset_id(rate_asset) if rate_asset else None, - source = parsed_id.source ) + source = parsed_id.source) - cl_opt = getattr(gcfg,key) + cl_opt = getattr(gcfg, key) cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] - return tuple( parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt) ) + return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) def parse_query_arg(s): """ asset_id:amount[:to_asset_id[:to_amount]] """ - def parse_query_asset(asset_id,amount): + def parse_query_asset(asset_id, amount): parsed_id = parse_asset_id(asset_id) return asset_data( symbol = parsed_id.symbol, @@ -608,21 +598,20 @@ def make_cfg(gcfg_arg): amount = None if amount is None else Decimal(amount), rate = None, rate_asset = None, - source = parsed_id.source ) + source = parsed_id.source) ss = s.split(':') - assert len(ss) in (2,3,4), f'{s}: malformed argument' - asset_id,amount,to_asset_id,to_amount = (*ss,None,None)[:4] + assert len(ss) in (2, 3, 4), f'{s}: malformed argument' + asset_id, amount, to_asset_id, to_amount = (*ss, None, None)[:4] return query_tuple( - asset = parse_query_asset(asset_id,amount), - to_asset = parse_query_asset(to_asset_id,to_amount) if to_asset_id else None - ) + asset = parse_query_asset(asset_id, amount), + to_asset = parse_query_asset(to_asset_id, to_amount) if to_asset_id else None) - def gen_uniq(obj_list,key,preload=None): - found = set([getattr(obj,key) for obj in preload if hasattr(obj,key)] if preload else ()) + def gen_uniq(obj_list, key, preload=None): + found = set([getattr(obj, key) for obj in preload if hasattr(obj, key)] if preload else ()) for obj in obj_list: - id = getattr(obj,key) + id = getattr(obj, key) if id not in found: yield obj found.add(id) @@ -632,39 +621,38 @@ def make_cfg(gcfg_arg): 'user_added', usr_rows + (tuple(asset for asset in query if asset) if query else ()) + - usr_columns ) + usr_columns) def get_portfolio_assets(ret=()): if cfg_in.portfolio and gcfg.portfolio: - ret = (parse_asset_id(e,require_label=True) for e in cfg_in.portfolio) - return ( 'portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') ) + ret = (parse_asset_id(e, require_label=True) for e in cfg_in.portfolio) + return ('portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC')) def get_portfolio(): - return {k:Decimal(v) for k,v in cfg_in.portfolio.items() if (not gcfg.btc) or k == 'btc-bitcoin'} + return {k: Decimal(v) for k, v in cfg_in.portfolio.items() + if (not gcfg.btc) or k == 'btc-bitcoin'} def parse_add_precision(arg): if not arg: return 0 s = str(arg) if not (s.isdigit() and s.isascii()): - die(1,f'{s}: invalid parameter for --add-precision (not an integer)') + die(1, f'{s}: invalid parameter for --add-precision (not an integer)') if int(s) > 30: - die(1,f'{s}: invalid parameter for --add-precision (value >30)') + die(1, f'{s}: invalid parameter for --add-precision (value >30)') return int(s) def create_rows(): rows = ( ('trade_pair',) + query if (query and query.to_asset) else - ('bitcoin',parse_asset_id('btc-bitcoin')) if gcfg.btc else - get_rows_from_cfg( add_data={'fiat':['usd-us-dollar']} if gcfg.add_columns else None ) - ) + ('bitcoin', parse_asset_id('btc-bitcoin')) if gcfg.btc else + get_rows_from_cfg(add_data={'fiat':['usd-us-dollar']} if gcfg.add_columns else None)) - for hdr,data in ( - (get_usr_assets(),) if query else - (get_usr_assets(), get_portfolio_assets()) - ): + for hdr, data in ( + (get_usr_assets(),) if query else + (get_usr_assets(), get_portfolio_assets())): if data: - uniq_data = tuple(gen_uniq(data,'symbol',preload=rows)) + uniq_data = tuple(gen_uniq(data, 'symbol', preload=rows)) if uniq_data: rows += (hdr,) + uniq_data return rows @@ -693,26 +681,25 @@ def make_cfg(gcfg_arg): 'quiet', 'verbose']) - global gcfg,cfg_in,src_cls,cfg + global gcfg, cfg_in, src_cls, cfg gcfg = gcfg_arg - src_cls = { k: getattr(DataSource,v) for k,v in DataSource.get_sources().items() } + src_cls = {k: getattr(DataSource, v) for k, v in DataSource.get_sources().items()} fi_pat = src_cls['fi'].asset_id_pat cmd_args = gcfg._args cfg_in = get_cfg_in() usr_rows = parse_usr_asset_arg('add_rows') - usr_columns = parse_usr_asset_arg('add_columns',use_cf_file=True) + usr_columns = parse_usr_asset_arg('add_columns', use_cf_file=True) query = parse_query_arg(cmd_args[0]) if cmd_args else None def get_proxy(name): - proxy = getattr(gcfg,name) + proxy = getattr(gcfg, name) return ( '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') - else (proxy or cfg_in.cfg.get(name)) - ) + else (proxy or cfg_in.cfg.get(name))) proxy = get_proxy('proxy') proxy = None if proxy == 'none' else proxy @@ -723,7 +710,7 @@ def make_cfg(gcfg_arg): usr_rows = usr_rows, usr_columns = usr_columns, query = query, - adjust = ( lambda x: (100 + x) / 100 if x else 1 )( Decimal(gcfg.adjust or 0) ), + adjust = (lambda x: (100 + x) / 100 if x else 1)(Decimal(gcfg.adjust or 0)), clsname = 'trading' if query else 'overview', btc_only = gcfg.btc or cfg_in.cfg.get('btc'), add_prec = parse_add_precision(gcfg.add_precision or cfg_in.cfg.get('add_precision')), @@ -745,17 +732,16 @@ def make_cfg(gcfg_arg): thousands_comma = gcfg.thousands_comma or cfg_in.cfg.get('thousands_comma'), update_time = gcfg.update_time or cfg_in.cfg.get('update_time'), quiet = gcfg.quiet or cfg_in.cfg.get('quiet'), - verbose = gcfg.verbose or cfg_in.cfg.get('verbose'), - ) + verbose = gcfg.verbose or cfg_in.cfg.get('verbose')) def get_cfg_in(): - ret = namedtuple('cfg_in_data',['cfg','portfolio','cfg_file','portfolio_file']) - cfg_file,portfolio_file = ( - [os.path.join(gcfg.data_dir_root,'node_tools',fn) for fn in (cfg_fn,portfolio_fn)] - ) - cfg_data,portfolio_data = ( - [yaml.safe_load(open(fn).read()) if os.path.exists(fn) else None for fn in (cfg_file,portfolio_file)] - ) + ret = namedtuple('cfg_in_data', ['cfg', 'portfolio', 'cfg_file', 'portfolio_file']) + cfg_file, portfolio_file = ( + [os.path.join(gcfg.data_dir_root, 'node_tools', fn) + for fn in (cfg_fn, portfolio_fn)]) + cfg_data, portfolio_data = ( + [yaml.safe_load(open(fn).read()) if os.path.exists(fn) else None + for fn in (cfg_file, portfolio_file)]) return ret( cfg = cfg_data or { 'assets': { @@ -765,14 +751,11 @@ def get_cfg_in(): # Pound Sterling, Euro, Swiss Franc 'fiat': [ 'gbpusd=x', 'eurusd=x', 'chfusd=x' ], # Dow Jones Industrials, Nasdaq 100, S&P 500 - 'index': [ '^dji', '^ixic', '^gspc' ], - }, - 'proxy': 'http://vpn-gw:8118' - }, - portfolio = portfolio_data, - cfg_file = cfg_file, - portfolio_file = portfolio_file, - ) + 'index': [ '^dji', '^ixic', '^gspc' ]}, + 'proxy': 'http://vpn-gw:8118'}, + portfolio = portfolio_data, + cfg_file = cfg_file, + portfolio_file = portfolio_file) class Ticker: @@ -781,43 +764,43 @@ class Ticker: offer = None to_asset = None - def __init__(self,data): + def __init__(self, data): self.comma = ',' if cfg.thousands_comma else '' - self.col1_wid = max(len('TOTAL'),( + self.col1_wid = max(len('TOTAL'), ( max(len(self.create_label(d['id'])) for d in data.values()) if cfg.name_labels else - max(len(d['symbol']) for d in data.values()) - )) + 1 + max(len(d['symbol']) for d in data.values()))) + 1 - self.rows = [row._replace(id=self.get_id(row)) if isinstance(row,tuple) else row for row in cfg.rows] - self.col_usd_prices = {k:self.data[k]['price_usd'] for k in self.col_ids} + self.rows = [row._replace(id=self.get_id(row)) if isinstance(row, tuple) else row + for row in cfg.rows] + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} - self.prices = {row.id:self.get_row_prices(row.id) - for row in self.rows if isinstance(row,tuple) and row.id in data} + self.prices = {row.id: self.get_row_prices(row.id) + for row in self.rows if isinstance(row, tuple) and row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') - def format_last_update_col(self,cross_assets=()): + def format_last_update_col(self, cross_assets=()): if cfg.elapsed: from mmgen.util2 import format_elapsed_hr fmt_func = format_elapsed_hr else: - fmt_func = lambda t,now: time.strftime('%F %X', time.gmtime(t)) + fmt_func = lambda t, now: time.strftime('%F %X', time.gmtime(t)) d = self.data max_w = 0 if cross_assets: last_updated_x = [d[a.id]['last_updated'] for a in cross_assets] - min_t = min( (int(n) for n in last_updated_x if isinstance(n,int) ), default=None ) + min_t = min((int(n) for n in last_updated_x if isinstance(n, int)), default=None) else: min_t = None for row in self.rows: - if isinstance(row,tuple): + if isinstance(row, tuple): try: - t = int( d[row.id]['last_updated'] ) + t = int(d[row.id]['last_updated']) except TypeError as e: d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) except KeyError as e: @@ -825,7 +808,7 @@ class Ticker: pass else: t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( - (min(t,min_t) if min_t else t), + (min(t, min_t) if min_t else t), now = now) max_w = max(len(t_fmt), max_w) @@ -836,7 +819,7 @@ class Ticker: self.uprec = {k: max(0, v+4) + cfg.add_prec for k, v in exp} self.uwid = {k: 12 + max(0, abs(v)-6) + cfg.add_prec for k, v in exp} - def get_id(self,asset): + def get_id(self, asset): if asset.id: return asset.id else: @@ -844,11 +827,11 @@ class Ticker: if d['symbol'] == asset.symbol: return d['id'] - def create_label(self,id): + def create_label(self, id): return self.data[id]['name'].upper() def gen_output(self): - yield 'Current time: {} UTC'.format(time.strftime('%F %X',time.gmtime(now))) + yield 'Current time: {} UTC'.format(time.strftime('%F %X', time.gmtime(now))) for asset in self.usr_col_assets: if asset.symbol != 'USD': @@ -858,17 +841,16 @@ class Ticker: self.create_label(asset.id), usdprice, self.comma, - max(2, 4-usdprice.adjusted()) ) + max(2, 4-usdprice.adjusted())) - if hasattr(self,'subhdr'): + if hasattr(self, 'subhdr'): yield self.subhdr if self.show_adj: yield ( ('Offered price differs from spot' if self.offer else 'Adjusting prices') + ' by ' - + yellow('{:+.2f}%'.format( (self.adjust-1) * 100 )) - ) + + yellow('{:+.2f}%'.format((self.adjust-1) * 100))) yield '' @@ -879,7 +861,7 @@ class Ticker: yield self.table_hdr for row in self.rows: - if isinstance(row,str): + if isinstance(row, str): yield ('-' * self.hl_wid) else: try: @@ -896,21 +878,20 @@ class Ticker: yield blue('PORTFOLIO') yield self.table_hdr yield '-' * self.hl_wid - for sym,amt in cfg.portfolio.items(): + for sym, amt in cfg.portfolio.items(): try: - yield self.fmt_row(self.data[sym],amt=amt) + yield self.fmt_row(self.data[sym], amt=amt) except KeyError: yield gray(f'(no data for {sym})') yield '-' * self.hl_wid if not cfg.btc_only: yield self.fs_num.format( lbl = 'TOTAL', pc3='', pc4='', pc1='', pc2='', upd='', amt='', - **{ k.replace('-','_'): v for k,v in self.prices['total'].items() } - ) + **{k.replace('-', '_'): v for k, v in self.prices['total'].items()}) class overview(base): - def __init__(self,data): + def __init__(self, data): self.data = data self.adjust = cfg.adjust self.show_adj = self.adjust != 1 @@ -922,25 +903,26 @@ class Ticker: self.format_last_update_col() if cfg.portfolio: - self.prices['total'] = { col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] - for row in self.rows if isinstance(row,tuple) and row.id in cfg.portfolio and row.id in data) - for col_id in self.col_ids } + self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] + for row in self.rows + if isinstance(row, tuple) and row.id in cfg.portfolio and row.id in data) + for col_id in self.col_ids} self.init_prec() self.init_fs() - def get_row_prices(self,id): + def get_row_prices(self, id): if id in self.data: d = self.data[id] - return { k: ( + return {k: ( d['price_btc'] if k == 'btc-bitcoin' else d['price_usd'] / self.col_usd_prices[k] - ) * self.adjust for k in self.col_ids } + ) * self.adjust for k in self.col_ids} - def fmt_row(self,d,amt=None,amt_fmt=None): + def fmt_row(self, d, amt=None, amt_fmt=None): def fmt_pct(n): - return gray(' --') if n is None else (red,green)[n>=0](f'{n:+7.2f}') + return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') p = self.prices[d['id']] @@ -957,53 +939,45 @@ class Ticker: pc4 = fmt_pct(d.get('percent_change_30d')), upd = d.get('last_updated_fmt'), amt = amt_fmt, - **{ k.replace('-','_'): v * (1 if amt is None else amt) for k,v in p.items() } - ) + **{k.replace('-', '_'): v * (1 if amt is None else amt) for k, v in p.items()}) def init_fs(self): - col_prec = {'usd-us-dollar':2+cfg.add_prec,'btc-bitcoin':8+cfg.add_prec } # | self.uprec # Python 3.9 - col_prec.update(self.uprec) - col_wid = {'usd-us-dollar':8+cfg.add_prec,'btc-bitcoin':12+cfg.add_prec } # """ - col_wid.update(self.uwid) + col_prec = {'usd-us-dollar': 2+cfg.add_prec, 'btc-bitcoin': 8+cfg.add_prec} | self.uprec max_row = max( - ( (k,v['btc-bitcoin']) for k,v in self.prices.items() ), - key = lambda a: a[1] - ) - widths = { k: len('{:{}.{}f}'.format( self.prices[max_row[0]][k], self.comma, col_prec[k] )) - for k in self.col_ids } + ((k, v['btc-bitcoin']) for k, v in self.prices.items()), + key = lambda a: a[1]) + widths = {k: len('{:{}.{}f}'.format(self.prices[max_row[0]][k], self.comma, col_prec[k])) + for k in self.col_ids} - fd = namedtuple('format_str_data',['fs_str','fs_num','wid']) + fd = namedtuple('format_str_data', ['fs_str', 'fs_num', 'wid']) col_fs_data = { - 'label': fd(f'{{lbl:{self.col1_wid}}}',f'{{lbl:{self.col1_wid}}}',self.col1_wid), + 'label': fd(f'{{lbl:{self.col1_wid}}}', f'{{lbl:{self.col1_wid}}}', self.col1_wid), 'pct1y': fd(' {pc3:7}', ' {pc3:7}', 8), 'pct1m': fd(' {pc4:7}', ' {pc4:7}', 8), 'pct1w': fd(' {pc1:7}', ' {pc1:7}', 8), 'pct1d': fd(' {pc2:7}', ' {pc2:7}', 8), - 'update_time': fd(' {upd}', ' {upd}', max((19 if cfg.portfolio else 0),self.upd_w) + 2), - 'amt': fd(' {amt}', ' {amt}', 21), - } -# } | { k: fd( # Python 3.9 - col_fs_data.update({ k: fd( - ' {{{}:>{}}}'.format( k.replace('-','_'), widths[k] ), - ' {{{}:{}{}.{}f}}'.format( k.replace('-','_'), widths[k], self.comma, col_prec[k] ), - widths[k]+2 - ) for k in self.col_ids - }) + 'update_time': fd(' {upd}', ' {upd}', + max((19 if cfg.portfolio else 0), self.upd_w) + 2), + 'amt': fd(' {amt}', ' {amt}', 21) + } | {k: fd( + ' {{{}:>{}}}'.format(k.replace('-', '_'), widths[k]), + ' {{{}:{}{}.{}f}}'.format(k.replace('-', '_'), widths[k], self.comma, col_prec[k]), + widths[k] + 2 + ) for k in self.col_ids} cols = ( - ['label','usd-us-dollar'] + + ['label', 'usd-us-dollar'] + [asset.id for asset in self.usr_col_assets] + - [a for a,b in ( - ( 'btc-bitcoin', not cfg.btc_only ), - ( 'pct1y', 'y' in cfg.percent_cols ), - ( 'pct1m', 'm' in cfg.percent_cols ), - ( 'pct1w', 'w' in cfg.percent_cols ), - ( 'pct1d', 'd' in cfg.percent_cols ), - ( 'update_time', cfg.update_time ), - ) if b] - ) + [a for a, b in ( + ('btc-bitcoin', not cfg.btc_only), + ('pct1y', 'y' in cfg.percent_cols), + ('pct1m', 'm' in cfg.percent_cols), + ('pct1w', 'w' in cfg.percent_cols), + ('pct1d', 'd' in cfg.percent_cols), + ('update_time', cfg.update_time)) + if b]) cols2 = list(cols) if cfg.update_time: cols2.pop() @@ -1029,17 +1003,16 @@ class Ticker: amt = ' AMOUNT', usd_us_dollar = 'USD', btc_bitcoin = ' BTC', - **{ a.id.replace('-','_'): a.symbol for a in self.usr_col_assets } - ) + **{a.id.replace('-', '_'): a.symbol for a in self.usr_col_assets}) class trading(base): - def __init__(self,data): + def __init__(self, data): self.data = data self.asset = cfg.query.asset._replace(id=self.get_id(cfg.query.asset)) self.to_asset = ( cfg.query.to_asset._replace(id=self.get_id(cfg.query.to_asset)) - if cfg.query.to_asset else None ) + if cfg.query.to_asset else None) self.col_ids = [self.asset.id] self.adjust = cfg.adjust if self.to_asset: @@ -1048,12 +1021,13 @@ class Ticker: real_price = ( self.asset.amount * data[self.asset.id]['price_usd'] - / data[self.to_asset.id]['price_usd'] - ) + / data[self.to_asset.id]['price_usd']) if self.adjust != 1: - die(1,'the --adjust option may not be combined with TO_AMOUNT in the trade specifier') + die(1, + 'the --adjust option may not be combined with TO_AMOUNT ' + 'in the trade specifier') self.adjust = self.offer / real_price - self.hl_ids = [self.asset.id,self.to_asset.id] + self.hl_ids = [self.asset.id, self.to_asset.id] else: self.hl_ids = [self.asset.id] @@ -1070,10 +1044,10 @@ class Ticker: self.init_prec() self.init_fs() - def get_row_prices(self,id): + def get_row_prices(self, id): if id in self.data: d = self.data[id] - return { k: self.col_usd_prices[self.asset.id] / d['price_usd'] for k in self.col_ids } + return {k: self.col_usd_prices[self.asset.id] / d['price_usd'] for k in self.col_ids} def init_fs(self): self.max_wid = max( @@ -1081,10 +1055,8 @@ class Ticker: v[self.asset.id] * self.asset.amount, 16 + cfg.add_prec, self.comma, - 8 + cfg.add_prec - )) - for v in self.prices.values() - ) + 8 + cfg.add_prec)) + for v in self.prices.values()) self.fs_str = '{lbl:%s} {p_spot}' % self.col1_wid self.hl_wid = self.col1_wid + self.max_wid + 1 if self.show_adj: @@ -1094,20 +1066,19 @@ class Ticker: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 - def fmt_row(self,d): + def fmt_row(self, d): id = d['id'] p = self.prices[id][self.asset.id] * self.asset.amount - p_spot = '{:{}{}.{}f}'.format( p, self.max_wid, self.comma, 8+cfg.add_prec ) + p_spot = '{:{}{}.{}f}'.format(p, self.max_wid, self.comma, 8+cfg.add_prec) p_adj = ( - '{:{}{}.{}f}'.format( p*self.adjust, self.max_wid, self.comma, 8+cfg.add_prec ) - if self.show_adj else '' ) + '{:{}{}.{}f}'.format(p*self.adjust, self.max_wid, self.comma, 8+cfg.add_prec) + if self.show_adj else '') return self.fs_str.format( lbl = self.create_label(id) if cfg.name_labels else d['symbol'], p_spot = green(p_spot) if id in self.hl_ids else p_spot, p_adj = yellow(p_adj) if id in self.hl_ids else p_adj, - upd = d.get('last_updated_fmt'), - ) + upd = d.get('last_updated_fmt')) @property def table_hdr(self): @@ -1115,12 +1086,11 @@ class Ticker: lbl = '', p_spot = '{t:>{w}}'.format( t = 'SPOT PRICE', - w = self.max_wid ), + w = self.max_wid), p_adj = '{t:>{w}}'.format( t = ('OFFERED' if self.offer else 'ADJUSTED') + ' PRICE', - w = self.max_wid ), - upd = 'UPDATED' - ) + w = self.max_wid), + upd = 'UPDATED') @property def subhdr(self): @@ -1133,9 +1103,8 @@ class Ticker: ) + ( ( ' =>' + - (' {:{}}'.format(self.offer,self.comma) if self.offer else '') + + (' {:{}}'.format(self.offer, self.comma) if self.offer else '') + ' {} ({})'.format( self.to_asset.symbol, - self.create_label(self.to_asset.id) ) - ) if self.to_asset else '' ) - ) + self.create_label(self.to_asset.id)) + ) if self.to_asset else '')) diff --git a/mmgen_node_tools/Util.py b/mmgen_node_tools/Util.py index 1d5426b..f36faad 100755 --- a/mmgen_node_tools/Util.py +++ b/mmgen_node_tools/Util.py @@ -21,53 +21,52 @@ mmgen_node_tools.Util: utility functions for MMGen node tools import time -def get_hms(t=None,utc=False,no_secs=False): +def get_hms(t=None, utc=False, no_secs=False): secs = t or time.time() - ret = (time.localtime,time.gmtime)[utc](secs) - fs,n = (('{:02}:{:02}:{:02}',6),('{:02}:{:02}',5))[no_secs] + ret = (time.localtime, time.gmtime)[utc](secs) + fs, n = (('{:02}:{:02}:{:02}', 6), ('{:02}:{:02}', 5))[no_secs] return fs.format(*ret[3:n]) -def get_day_hms(t=None,utc=False): +def get_day_hms(t=None, utc=False): secs = t or time.time() - ret = (time.localtime,time.gmtime)[utc](secs) + ret = (time.localtime, time.gmtime)[utc](secs) return '{:04}-{:02}-{:02} {:02}:{:02}:{:02}'.format(*ret[0:6]) -def do_system(cmd,testing=False,shell=False): +def do_system(cmd, testing=False, shell=False): if testing: from mmgen.util import msg msg("Would execute: '%s'" % cmd) return True else: import subprocess - return subprocess.call((cmd if shell else cmd.split()),shell,stderr=subprocess.PIPE) + return subprocess.call((cmd if shell else cmd.split()), shell, stderr=subprocess.PIPE) -def get_url(url,gzip_ok=False,proxy=None,timeout=60,verbose=False,debug=False): +def get_url(url, gzip_ok=False, proxy=None, timeout=60, verbose=False, debug=False): if debug: print('get_url():') print(' url', url) - print(' gzip_ok:',gzip_ok, 'proxy:',proxy, 'timeout:',timeout, 'verbose:',verbose) - import pycurl,io + print(' gzip_ok:', gzip_ok, 'proxy:', proxy, 'timeout:', timeout, 'verbose:', verbose) + import pycurl, io c = pycurl.Curl() c_out = io.StringIO() - c.setopt(pycurl.WRITEFUNCTION,c_out.write) - c.setopt(pycurl.TIMEOUT,timeout) - c.setopt(pycurl.FOLLOWLOCATION,True) - c.setopt(pycurl.COOKIEFILE,'') - c.setopt(pycurl.VERBOSE,verbose) + c.setopt(pycurl.WRITEFUNCTION, c_out.write) + c.setopt(pycurl.TIMEOUT, timeout) + c.setopt(pycurl.FOLLOWLOCATION, True) + c.setopt(pycurl.COOKIEFILE, '') + c.setopt(pycurl.VERBOSE, verbose) if gzip_ok: - c.setopt(pycurl.USERAGENT,'Lynx/2.8.9dev.8 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/3.4.9') + c.setopt(pycurl.USERAGENT, 'Lynx/2.8.9dev.8 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/3.4.9') c.setopt(pycurl.HTTPHEADER, [ 'Accept: text/html, text/plain, text/sgml, text/css, application/xhtml+xml, */*;q=0.01', 'Accept-Encoding: gzip', - 'Accept-Language: en'] - ) + 'Accept-Language: en']) if proxy: - c.setopt(pycurl.PROXY,proxy) - c.setopt(pycurl.URL,url) + c.setopt(pycurl.PROXY, proxy) + c.setopt(pycurl.URL, url) c.perform() text = c_out.getvalue() if text[:2] == '\x1f\x8b': # gzip magic number - c_out.seek(0,0) + c_out.seek(0, 0) import gzip with gzip.GzipFile(fileobj=c_out) as f: text = f.read() @@ -103,20 +102,19 @@ big_digits = { """ } -_bnums_c,_bpunc_c = [[l.strip('\n') + ' ' * (big_digits[m]*big_digits['n']) +_bnums_c, _bpunc_c = [[l.strip('\n') + ' ' * (big_digits[m]*big_digits['n']) for l in big_digits[k][1:].split('\n')] - for k,m in (('nums','w'),('punc','pw'))] + for k, m in (('nums', 'w'), ('punc', 'pw'))] -_bnums_n,_bpunc_n = [[[l[0+(j*w):w+(j*w)] for l in i] - for j in range(big_digits[n])] for n,w,i in - (('n',big_digits['w'],_bnums_c),('pn',big_digits['pw'],_bpunc_c))] +_bnums_n, _bpunc_n = [[[l[0+(j*w):w+(j*w)] for l in i] + for j in range(big_digits[n])] for n, w, i in + (('n', big_digits['w'], _bnums_c), ('pn', big_digits['pw'], _bpunc_c))] -def display_big_digits(s,pre='',suf=''): - s = [int((d,10,11)[(d in '.:')+(d==':')]) for d in s] +def display_big_digits(s, pre='', suf=''): + s = [int((d, 10, 11)[(d in '.:')+(d==':')]) for d in s] return pre + ('\n'+pre).join( - [''.join([(_bnums_n+_bpunc_n)[d][l] for d in s]) + suf for l in range(big_digits['h'])] - ) + [''.join([(_bnums_n+_bpunc_n)[d][l] for d in s]) + suf for l in range(big_digits['h'])]) if __name__ == '__main__': num = '2345.17' - print(display_big_digits(num,pre='+ ',suf=' +')) + print(display_big_digits(num, pre='+ ', suf=' +')) diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 37f0482..4647029 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -16,7 +16,7 @@ import sys from mmgen.obj import CoinTxID from mmgen.cfg import Config -from mmgen.util import msg,Msg,die,suf,make_timestr,async_run +from mmgen.util import msg, Msg, die, suf, make_timestr, async_run from mmgen.color import red opts_data = { @@ -32,16 +32,16 @@ opts_data = { } } -def do_output(proto,addr_data,blk_hdrs): +def do_output(proto, addr_data, blk_hdrs): col1w = len(str(len(addr_data))) indent = ' ' * (col1w + 2) - for n,(addr,unspents) in enumerate(addr_data.items(),1): + for n, (addr, unspents) in enumerate(addr_data.items(), 1): Msg(f'\n{n:{col1w}}) Address: {addr.hl(addr.view_pref)}') if unspents: - heights = { u['height'] for u in unspents } + heights = {u['height'] for u in unspents} Msg('{}Balance: {}'.format( indent, sum(proto.coin_amt(u['amount']) for u in unspents).hl2(unit=True, fs='{:,}'))), @@ -50,22 +50,21 @@ def do_output(proto,addr_data,blk_hdrs): red(str(len(unspents))), suf(unspents), red(str(len(heights))), - suf(heights) )) + suf(heights))) blk_w = len(str(unspents[-1]['height'])) - fs = '%s{:%s} {:19} {:64} {:4} {}' % (indent,max(5,blk_w)) - Msg(fs.format('Block','Date','TxID','Vout',' Amount')) + fs = '%s{:%s} {:19} {:64} {:4} {}' % (indent, max(5, blk_w)) + Msg(fs.format('Block', 'Date', 'TxID', 'Vout', ' Amount')) for u in unspents: Msg(fs.format( u['height'], - make_timestr( blk_hdrs[u['height']]['time'] ), + make_timestr(blk_hdrs[u['height']]['time']), CoinTxID(u['txid']).hl(), red(str(u['vout']).rjust(4)), - proto.coin_amt(u['amount']).fmt(6, color=True, prec=8) - )) + proto.coin_amt(u['amount']).fmt(6, color=True, prec=8))) else: Msg(f'{indent}No balance') -def do_output_tabular(proto,addr_data,blk_hdrs): +def do_output_tabular(proto, addr_data, blk_hdrs): col1w = len(str(len(addr_data))) + 1 max_addrw = max(len(addr) for addr in addr_data) @@ -75,9 +74,9 @@ def do_output_tabular(proto,addr_data,blk_hdrs): lb_w = max(len(h) for h in lb_heights) fs = ( - ' {n:>%s} {a} {u} {b:>%s} {t:19} {B:>%s} {T:19} {A}' % (col1w,max(5,fb_w),max(4,lb_w)) + ' {n:>%s} {a} {u} {b:>%s} {t:19} {B:>%s} {T:19} {A}' % (col1w, max(5, fb_w), max(4, lb_w)) if cfg.first_block else - ' {n:>%s} {a} {u} {B:>%s} {T:19} {A}' % (col1w,max(4,lb_w)) ) + ' {n:>%s} {a} {u} {B:>%s} {T:19} {A}' % (col1w, max(4, lb_w))) Msg('\n' + fs.format( n = '', @@ -87,20 +86,19 @@ def do_output_tabular(proto,addr_data,blk_hdrs): t = 'Block', B = 'Last', T = 'Block', - A = ' Amount' )) + A = ' Amount')) - for n,(addr,unspents) in enumerate(addr_data.items(),1): + for n, (addr, unspents) in enumerate(addr_data.items(), 1): if unspents: Msg(fs.format( n = str(n) + ')', a = addr.fmt(addr.view_pref, max_addrw, color=True), u = red(str(len(unspents)).rjust(5)), b = unspents[0]['height'], - t = make_timestr( blk_hdrs[unspents[0]['height']]['time'] ), + t = make_timestr(blk_hdrs[unspents[0]['height']]['time']), B = unspents[-1]['height'], - T = make_timestr( blk_hdrs[unspents[-1]['height']]['time'] ), - A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(7, color=True, prec=8) - )) + T = make_timestr(blk_hdrs[unspents[-1]['height']]['time']), + A = sum(proto.coin_amt(u['amount']) for u in unspents).fmt(7, color=True, prec=8))) else: Msg(fs.format( n = str(n) + ')', @@ -110,61 +108,61 @@ def do_output_tabular(proto,addr_data,blk_hdrs): t = '', B = '-', T = '', - A = ' -' )) + A = ' -')) async def main(req_addrs): proto = cfg._proto from mmgen.addr import CoinAddr - addrs = [CoinAddr(proto,addr) for addr in req_addrs] + addrs = [CoinAddr(proto, addr) for addr in req_addrs] from mmgen.rpc import rpc_init - rpc = await rpc_init(cfg,ignore_wallet=True) + rpc = await rpc_init(cfg, ignore_wallet=True) height = await rpc.call('getblockcount') Msg(f'{proto.coin} {proto.network.upper()} [height {height}]') from mmgen.proto.btc.misc import scantxoutset - res = await scantxoutset( cfg, rpc, [f'addr({addr})' for addr in addrs] ) + res = await scantxoutset(cfg, rpc, [f'addr({addr})' for addr in addrs]) if not res['success']: - die(1,'UTXO scanning failed or was interrupted') + die(1, 'UTXO scanning failed or was interrupted') elif not res['unspents']: msg('Address has no balance' if len(addrs) == 1 else - 'Addresses have no balances' ) + 'Addresses have no balances') else: addr_data = {k:[] for k in addrs} if 'desc' in res['unspents'][0]: import re - for unspent in sorted(res['unspents'],key=lambda x: x['height']): - addr = re.match('addr\((.*?)\)',unspent['desc'])[1] + for unspent in sorted(res['unspents'], key=lambda x: x['height']): + addr = re.match('addr\((.*?)\)', unspent['desc'])[1] addr_data[addr].append(unspent) else: from mmgen.proto.btc.tx.base import decodeScriptPubKey - for unspent in sorted(res['unspents'],key=lambda x: x['height']): + for unspent in sorted(res['unspents'], key=lambda x: x['height']): ds = decodeScriptPubKey(proto, unspent['scriptPubKey']) addr_data[ds.addr].append(unspent) good_addrs = len([v for v in addr_data.values() if v]) Msg('Total: {} in {} address{}'.format( - proto.coin_amt(res['total_amount']).hl2(unit=True,fs='{:,}'), + proto.coin_amt(res['total_amount']).hl2(unit=True, fs='{:,}'), red(str(good_addrs)), - suf(good_addrs,'es') - )) + suf(good_addrs, 'es'))) blk_heights = {i['height'] for i in res['unspents']} blk_hashes = await rpc.batch_call('getblockhash', [(h,) for h in blk_heights]) blk_hdrs = await rpc.batch_call('getblockheader', [(H,) for H in blk_hashes]) - (do_output_tabular if cfg.tabular else do_output)( proto, addr_data, dict(zip(blk_heights,blk_hdrs)) ) + (do_output_tabular if cfg.tabular else do_output)( + proto, addr_data, dict(zip(blk_heights, blk_hdrs))) -cfg = Config( opts_data=opts_data, init_opts={'rpc_backend':'aiohttp'} ) +cfg = Config(opts_data=opts_data, init_opts={'rpc_backend': 'aiohttp'}) if len(cfg._args) < 1: - die(1,'This command requires at least one coin address argument') + die(1, 'This command requires at least one coin address argument') try: async_run(cfg, main, args=[cfg._args]) diff --git a/mmgen_node_tools/main_blocks_info.py b/mmgen_node_tools/main_blocks_info.py index 18136c4..eaed15b 100755 --- a/mmgen_node_tools/main_blocks_info.py +++ b/mmgen_node_tools/main_blocks_info.py @@ -20,9 +20,9 @@ mmnode-blocks-info: Display information about a block or range of blocks """ -from mmgen.cfg import gc,Config -from mmgen.util import async_run,fmt_list -from .BlocksInfo import BlocksInfo,JSONBlocksInfo +from mmgen.cfg import gc, Config +from mmgen.util import async_run, fmt_list +from .BlocksInfo import BlocksInfo, JSONBlocksInfo opts_data = { 'sets': [ @@ -145,14 +145,13 @@ EXAMPLES: $ {p} --rpc-backend=aio -H +1000 This program requires a txindex-enabled daemon for correct operation. -""" }, +"""}, 'code': { - 'notes': lambda cfg,proto,s: s.format( + 'notes': lambda cfg, proto, s: s.format( I = proto.diff_adjust_interval, - F = fmt_list(BlocksInfo.fields,fmt='bare'), - S = fmt_list(BlocksInfo.all_stats,fmt='bare'), - p = gc.prog_name, - ) + F = fmt_list(BlocksInfo.fields, fmt='bare'), + S = fmt_list(BlocksInfo.all_stats, fmt='bare'), + p = gc.prog_name) } } @@ -172,7 +171,7 @@ async def main(): await m.process_blocks() if m.last: - for i,sname in enumerate(m.stats): + for i, sname in enumerate(m.stats): m.process_stats_pre(i) await m.process_stats(sname) diff --git a/mmgen_node_tools/main_feeview.py b/mmgen_node_tools/main_feeview.py index 3211f7f..188f60e 100755 --- a/mmgen_node_tools/main_feeview.py +++ b/mmgen_node_tools/main_feeview.py @@ -21,10 +21,10 @@ mmnode-feeview: Visualize the fee structure of a node’s mempool """ from mmgen.cfg import Config -from mmgen.util import async_run,die,fmt,make_timestr,check_int_between -from mmgen.util2 import int2bytespec,parse_bytespec +from mmgen.util import async_run, die, fmt, make_timestr, check_int_between +from mmgen.util2 import int2bytespec, parse_bytespec -min_prec,max_prec,dfl_prec = (0,6,4) +min_prec, max_prec, dfl_prec = (0, 6, 4) fee_brackets = [ 1, 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 18, @@ -42,9 +42,9 @@ fee_brackets = [ opts_data = { 'sets': [ - ('detail',True,'ranges',True), - ('detail',True,'show_mb_col',True), - ('detail',True,'precision',6), + ('detail', True, 'ranges', True), + ('detail', True, 'show_mb_col', True), + ('detail', True, 'precision', 6), ], 'text': { 'desc': 'Visualize the fee structure of a node’s mempool', @@ -83,37 +83,37 @@ cfg = Config(opts_data=opts_data) if cfg.ignore_below: if cfg.show_empty: - die(1,'Conflicting options: --ignore-below, --show-empty') + die(1, 'Conflicting options: --ignore-below, --show-empty') ignore_below = parse_bytespec(cfg.ignore_below) precision = ( check_int_between(cfg.precision, min_prec, max_prec, desc='--precision arg') - if cfg.precision else dfl_prec ) + if cfg.precision else dfl_prec) from mmgen.term import get_terminal_size width = cfg.columns or get_terminal_size().width class fee_bracket: - def __init__(self,top,bottom): + def __init__(self, top, bottom): self.top = top self.bottom = bottom self.tx_bytes = 0 self.tx_bytes_cum = 0 self.skip = False -def log(data,fn): +def log(data, fn): import json from mmgen.rpc.util import json_encoder from mmgen.fileutil import write_data_to_file write_data_to_file( cfg = cfg, outfile = fn, - data = json.dumps(data,cls=json_encoder,sort_keys=True,indent=4), + data = json.dumps(data, cls=json_encoder, sort_keys=True, indent=4), desc = 'mempool', - ask_overwrite = False ) + ask_overwrite = False) -def create_data(coin_amt,mempool): - out = [fee_bracket(fee_brackets[i],fee_brackets[i-1] if i else 0) for i in range(len(fee_brackets))] +def create_data(coin_amt, mempool): + out = [fee_bracket(fee_brackets[i], fee_brackets[i-1] if i else 0) for i in range(len(fee_brackets))] # populate fee brackets: size_key = 'size' if proto.coin == 'BCH' else 'vsize' @@ -143,7 +143,7 @@ def create_data(coin_amt,mempool): return out -def gen_header(host,mempool,blockcount): +def gen_header(host, mempool, blockcount): yield fmt(f""" Mempool Fee Structure @@ -159,27 +159,26 @@ def gen_header(host,mempool,blockcount): elif cfg.ignore_below: yield 'Ignoring fee brackets with less than {:,} bytes ({})'.format( ignore_below, - int2bytespec(ignore_below,'MB','0.6',strip=True,add_space=True), - ) + int2bytespec(ignore_below, 'MB', '0.6', strip=True, add_space=True)) if cfg.include_current: yield 'Including transactions in current fee bracket in Total MB amounts' def fmt_mb(n): - return int2bytespec(n,'MB',f'0.{precision}',print_sym=False) + return int2bytespec(n, 'MB', f'0.{precision}', print_sym=False) def gen_body(data): - tx_bytes_max = max((i.tx_bytes for i in data),default=0) - top_max = max((i.top for i in data),default=0) - bot_max = max((i.bottom for i in data),default=0) - col1_w = max(len(f'{bot_max}-{top_max}') if cfg.ranges else len(f'{top_max}'),6) + tx_bytes_max = max((i.tx_bytes for i in data), default=0) + top_max = max((i.top for i in data), default=0) + bot_max = max((i.bottom for i in data), default=0) + col1_w = max(len(f'{bot_max}-{top_max}') if cfg.ranges else len(f'{top_max}'), 6) col2_w = len(fmt_mb(tx_bytes_max)) if cfg.show_mb_col else 0 col3_w = len(fmt_mb(data[-1].tx_bytes_cum)) if data else 0 col4_w = width - col1_w - col2_w - col3_w - (4 if col2_w else 3) if cfg.show_mb_col: - fs = '{a:<%i} {b:>%i} {c:>%i} {d}' % (col1_w,col2_w,col3_w) + fs = '{a:<%i} {b:>%i} {c:>%i} {d}' % (col1_w, col2_w, col3_w) else: - fs = '{a:<%i} {c:>%i} {d}' % (col1_w,col3_w) + fs = '{a:<%i} {c:>%i} {d}' % (col1_w, col3_w) yield fs.format(a='', b='', c=f'{"Total":<{col3_w}}', d='') yield fs.format(a='sat/B', b=f'{"MB":<{col2_w}}', c=f'{"MB":<{col3_w}}', d='') @@ -188,16 +187,16 @@ def gen_body(data): if not i.skip: cum_bytes = i.tx_bytes_cum + i.tx_bytes if cfg.include_current else i.tx_bytes_cum yield fs.format( - a = '{}-{}'.format(i.bottom,i.top) if cfg.ranges else i.top, + a = '{}-{}'.format(i.bottom, i.top) if cfg.ranges else i.top, b = fmt_mb(i.tx_bytes), c = fmt_mb(cum_bytes), - d = '-' * int(col4_w * ( i.tx_bytes / tx_bytes_max )) ) + d = '-' * int(col4_w * (i.tx_bytes / tx_bytes_max))) yield fs.format( a = 'TOTAL', b = '', c = fmt_mb(data[-1].tx_bytes_cum + data[-1].tx_bytes if data else 0), - d = '' ) + d = '') async def main(): @@ -205,19 +204,19 @@ async def main(): proto = cfg._proto from mmgen.rpc import rpc_init - c = await rpc_init(cfg,ignore_wallet=True) + c = await rpc_init(cfg, ignore_wallet=True) - mempool = await c.call('getrawmempool',True) + mempool = await c.call('getrawmempool', True) if cfg.log: - log(mempool,'mempool.json') + log(mempool, 'mempool.json') - data = create_data(proto.coin_amt,mempool) + data = create_data(proto.coin_amt, mempool) cfg._util.stdout_or_pager( '\n'.join(gen_header( c.host, mempool, - await c.call('getblockcount') )) + '\n\n' + - '\n'.join(gen_body(data)) + '\n' ) + await c.call('getblockcount'))) + '\n\n' + + '\n'.join(gen_body(data)) + '\n') async_run(cfg, main) diff --git a/mmgen_node_tools/main_halving_calculator.py b/mmgen_node_tools/main_halving_calculator.py index 8f0a3a3..60c5440 100755 --- a/mmgen_node_tools/main_halving_calculator.py +++ b/mmgen_node_tools/main_halving_calculator.py @@ -28,7 +28,7 @@ from mmgen.util import async_run bdr_proj = 9.95 opts_data = { - 'sets': [('mined',True,'list',True)], + 'sets': [('mined', True, 'list', True)], 'text': { 'desc': 'Estimate date(s) of future block subsidy halving(s)', 'usage':'[opts]', @@ -41,7 +41,7 @@ opts_data = { {bdr_proj:.5f} min) -s, --sample-size=N Block range to calculate block discovery interval for next halving estimate (default: dynamically calculated) -""" } +"""} } cfg = Config(opts_data=opts_data) @@ -53,14 +53,14 @@ def date(t): return '{}-{:02}-{:02} {:02}:{:02}:{:02}'.format(*time.gmtime(t)[:6]) def dhms(t): - t,neg = (-t,'-') if t < 0 else (t,' ') + t, neg = (-t, '-') if t < 0 else (t, ' ') return f'{neg}{t//60//60//24} days, {t//60//60%24:02}:{t//60%60:02}:{t%60:02} h/m/s' def time_diff_warning(t_diff): if abs(t_diff) > 60*60: print('Warning: block tip time is {} {} clock time!'.format( dhms(abs(t_diff)), - ('behind','ahead of')[t_diff<0])) + ('behind', 'ahead of')[t_diff<0])) async def main(): @@ -72,9 +72,9 @@ async def main(): tip = await c.call('getblockcount') assert tip > 1, 'block tip must be > 1' remaining = proto.halving_interval - tip % proto.halving_interval - sample_size = int(cfg.sample_size) if cfg.sample_size else min(tip-1,max(remaining,144)) + sample_size = int(cfg.sample_size) if cfg.sample_size else min(tip-1, max(remaining, 144)) - cur,old = await c.gathered_call('getblockstats',((tip,),(tip - sample_size,))) + cur, old = await c.gathered_call('getblockstats', ((tip,), (tip - sample_size,))) clock_time = int(time.time()) time_diff_warning(clock_time - cur['time']) @@ -98,8 +98,7 @@ async def main(): f'Current block discovery interval (over last {sample_size} blocks): {bdr/60:0.2f} min\n\n' f'Current clock time (UTC): {date(clock_time)}\n' f'Est. halving date (UTC): {date(t_next)}\n' - f'Est. time until halving: {dhms(cur["time"] + t_rem - clock_time)}' - ) + f'Est. time until halving: {dhms(cur["time"] + t_rem - clock_time)}') async def print_halvings(): halving_blocknums = [i*proto.halving_interval for i in range(proto.max_halvings+1)][1:] @@ -108,13 +107,13 @@ async def main(): nhist = len(hist_halvings) nSubsidy = int(proto.start_subsidy / proto.coin_amt.satoshi) - block0_hash = await c.call('getblockhash',0) - block0_date = (await c.call('getblock',block0_hash))['time'] + block0_hash = await c.call('getblockhash', 0) + block0_date = (await c.call('getblock', block0_hash))['time'] def gen_data(): total_mined = 0 date = block0_date - for n,blk in enumerate(halving_blocknums): + for n, blk in enumerate(halving_blocknums): mined = (nSubsidy >> n) * proto.halving_interval if n == 0: mined -= nSubsidy # subtract unspendable genesis block subsidy @@ -123,13 +122,11 @@ async def main(): bdi = ( (hist_halvings[n]['time'] - date) / (proto.halving_interval * 60) if n < nhist else bdr/60 if n == nhist - else bdr_proj - ) + else bdr_proj) date = ( hist_halvings[n]['time'] if n < nhist - else t_next + int((n - nhist) * halving_secs) - ) - yield ( n, sub, blk, mined, total_mined, bdi, date ) + else t_next + int((n - nhist) * halving_secs)) + yield (n, sub, blk, mined, total_mined, bdi, date) if sub == 0: break @@ -150,8 +147,7 @@ async def main(): e = 'BDI (mins)', f = 'SUBSIDY ({proto.coin})', g = f'MINED ({proto.coin})', - h = f'TOTAL MINED ({proto.coin})' - ) + h = f'TOTAL MINED ({proto.coin})') + '\n' + fs.format( a = '-' * 7, @@ -159,22 +155,20 @@ async def main(): c = '-' * 19, d = '-' * 2, e = '-' * 10, - f = '-' * 13, + f = '-' * 17, g = '-' * 17, - h = '-' * 17 - ) + h = '-' * 17) + '\n' + '\n'.join(fs.format( - a = n + 1, - b = blk, - c = date(t), - d = ' P' if n > nhist else '' if n < nhist else ' E', - e = f'{bdr:8.5f}', - f = proto.coin_amt(sub, from_unit='satoshi').fmt(2, prec=8), - g = proto.coin_amt(mined, from_unit='satoshi').fmt(8, prec=8), - h = proto.coin_amt(total_mined, from_unit='satoshi').fmt(8, prec=8) - ) for n, sub, blk, mined, total_mined, bdr, t in gen_data()) - ) + a = n + 1, + b = blk, + c = date(t), + d = ' P' if n > nhist else '' if n < nhist else ' E', + e = f'{bdr:8.5f}', + f = proto.coin_amt(sub, from_unit='satoshi').fmt(2, prec=8), + g = proto.coin_amt(mined, from_unit='satoshi').fmt(8, prec=8), + h = proto.coin_amt(total_mined, from_unit='satoshi').fmt(8, prec=8) + ) for n, sub, blk, mined, total_mined, bdr, t in gen_data())) if cfg.list: await print_halvings() diff --git a/mmgen_node_tools/main_netrate.py b/mmgen_node_tools/main_netrate.py index b273ac6..f5cf992 100755 --- a/mmgen_node_tools/main_netrate.py +++ b/mmgen_node_tools/main_netrate.py @@ -20,7 +20,7 @@ mmnode-netrate: Bitcoin daemon network rate monitor """ -import sys,time +import sys, time from mmgen.cfg import Config from mmgen.util import async_run @@ -32,39 +32,38 @@ opts_data = { 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) -""" - } +"""} } cfg = Config(opts_data=opts_data) -ERASE_LINE,CUR_UP = '\033[K','\033[1A' +ERASE_LINE, CUR_UP = '\033[K', '\033[1A' async def main(): from mmgen.rpc import rpc_init - c = await rpc_init(cfg,ignore_wallet=True) + c = await rpc_init(cfg, ignore_wallet=True) async def get_data(): d = await c.call('getnettotals') - return [float(e) for e in (d['totalbytesrecv'],d['totalbytessent'],d['timemillis'])] + return [float(e) for e in (d['totalbytesrecv'], d['totalbytessent'], d['timemillis'])] - rs,ss,ts = (None,None,None) + rs, ss, ts = (None, None, None) while True: - r,s,t = await get_data() + r, s, t = await get_data() if rs is not None: sys.stderr.write( '\rrcvd: {:9.2f} kB/s\nsent: {:9.2f} kB/s '.format( (r-rs)/(t-ts), - (s-ss)/(t-ts) )) + (s-ss)/(t-ts))) time.sleep(2) if rs is not None: - sys.stderr.write('{}{}{}'.format(ERASE_LINE,CUR_UP,ERASE_LINE)) + sys.stderr.write('{}{}{}'.format(ERASE_LINE, CUR_UP, ERASE_LINE)) - rs,ss,ts = (r,s,t) + rs, ss, ts = (r, s, t) try: async_run(cfg, main) diff --git a/mmgen_node_tools/main_peerblocks.py b/mmgen_node_tools/main_peerblocks.py index 35292ea..6a7cb38 100755 --- a/mmgen_node_tools/main_peerblocks.py +++ b/mmgen_node_tools/main_peerblocks.py @@ -27,8 +27,7 @@ opts_data = { 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) -""" - } +"""} } from mmgen.cfg import Config @@ -37,9 +36,9 @@ cfg = Config(opts_data=opts_data) async def main(): from mmgen.rpc import rpc_init - rpc = await rpc_init(cfg,ignore_wallet=True) + rpc = await rpc_init(cfg, ignore_wallet=True) - from .PeerBlocks import BlocksDisplay,PeersDisplay + from .PeerBlocks import BlocksDisplay, PeersDisplay blocks = BlocksDisplay(cfg) peers = PeersDisplay(cfg) diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 351cb1e..c36d886 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -208,29 +208,26 @@ Customize output by editing the file To add a portfolio, edit the file ~/{pf_cfg} -""" - }, +"""}, 'code': { 'options': lambda s: s.format( - dfl_cachedir = os.path.relpath(dfl_cachedir,start=homedir), - ds = fmt_dict(DataSource.get_sources(),fmt='equal_compact'), + dfl_cachedir = os.path.relpath(dfl_cachedir, start=homedir), + ds = fmt_dict(DataSource.get_sources(), fmt='equal_compact'), al = DataSource.coinpaprika.dfl_asset_limit, - pc = fmt_list(Ticker.percent_cols,fmt='bare'), - ), + pc = fmt_list(Ticker.percent_cols, fmt='bare')), 'notes': lambda s: s.format( - assets = fmt_list(assets_list_gen(cfg_in),fmt='col',indent=' '), - cfg = os.path.relpath(cfg_in.cfg_file,start=homedir), - pf_cfg = os.path.relpath(cfg_in.portfolio_file,start=homedir), + assets = fmt_list(assets_list_gen(cfg_in), fmt='col', indent=' '), + cfg = os.path.relpath(cfg_in.cfg_file, start=homedir), + pf_cfg = os.path.relpath(cfg_in.portfolio_file, start=homedir), al = DataSource.coinpaprika.dfl_asset_limit, cc = src_cls['cc'](), - fi = src_cls['fi'](), - ) + fi = src_cls['fi']()) } } import os -from mmgen.util import fmt_list,fmt_dict +from mmgen.util import fmt_list, fmt_dict from mmgen.cfg import Config from . import Ticker @@ -238,7 +235,7 @@ gcfg = Config(opts_data=opts_data, caller_post_init=True) Ticker.make_cfg(gcfg) -from .Ticker import dfl_cachedir,homedir,DataSource,assets_list_gen,cfg_in,src_cls +from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen, cfg_in, src_cls gcfg._post_init() diff --git a/mmgen_node_tools/main_txfind.py b/mmgen_node_tools/main_txfind.py index 2411f36..0f9db5b 100755 --- a/mmgen_node_tools/main_txfind.py +++ b/mmgen_node_tools/main_txfind.py @@ -23,7 +23,7 @@ mmnode-txfind: Find a transaction in the blockchain or mempool import sys from mmgen.cfg import Config -from mmgen.util import msg,Msg,die,is_hex_str,async_run +from mmgen.util import msg, Msg, die, is_hex_str, async_run opts_data = { 'text': { @@ -48,29 +48,26 @@ msg_data = { 'normal': { 'none': 'Transaction not found in blockchain or mempool', 'block': 'Transaction is in block {b} ({c} confirmations)', - 'mem': 'Transaction is in mempool', - }, + 'mem': 'Transaction is in mempool'}, 'quiet': { 'none': 'None', 'block': '{b} {c}', - 'mem': 'mempool', - } -} + 'mem': 'mempool'}} async def main(txid): if len(txid) != 64 or not is_hex_str(txid): - die(2,f'{txid}: invalid transaction ID') + die(2, f'{txid}: invalid transaction ID') if cfg.verbose: msg(f'TxID: {txid}') from mmgen.rpc import rpc_init - c = await rpc_init(cfg,ignore_wallet=True) + c = await rpc_init(cfg, ignore_wallet=True) exitval = 0 try: tip1 = await c.call('getblockcount') - ret = await c.call('getrawtransaction',txid,True) + ret = await c.call('getrawtransaction', txid, True) tip2 = await c.call('getblockcount') except: Msg('\r' + msgs['none']) @@ -90,6 +87,6 @@ cfg = Config(opts_data=opts_data) msgs = msg_data['quiet' if cfg.quiet else 'normal'] if len(cfg._args) != 1: - die(1,'One transaction ID must be specified') + die(1, 'One transaction ID must be specified') sys.exit(async_run(cfg, main, args=[cfg._args[0]])) diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index 4792dee..a504678 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -335,19 +335,19 @@ class CmdTestRegtest(CmdTestBase): us = await r.rpc_call('listunspent',wallet='miner') tx_input = us[7] # 25 BTC in coinbase -- us[0] could have < 25 BTC fee = self.proto.coin_amt('0.001') - outputs = {p.addr:tx1_amt for p in pairs[:nTxs]} + outputs = {p.addr: tx1_amt for p in pairs[:nTxs]} outputs.update({burn_addr: self.proto.coin_amt(tx_input['amount']) - (tx1_amt*nTxs) - fee}) return await do_tx( - [{ 'txid': tx_input['txid'], 'vout': 0 }], + [{'txid': tx_input['txid'], 'vout': 0}], outputs, await r.miner_wif) async def do_tx2(tx,pairno): fee = self.proto.coin_amt(fees[pairno], from_decimal=True) - outputs = {p.addr:tx2_amt for p in pairs} + outputs = {p.addr: tx2_amt for p in pairs} outputs.update({burn_addr: tx1_amt - (tx2_amt*len(pairs)) - fee}) return await do_tx( - [{ 'txid': tx['txid'], 'vout': pairno }], + [{'txid': tx['txid'], 'vout': pairno}], outputs, pairs[pairno].wif ) diff --git a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py index 7c2fdf3..444ae08 100644 --- a/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py +++ b/test/overlay/fakemods/mmgen_node_tools/PeerBlocks.py @@ -294,15 +294,14 @@ class fake_data: 20 7303 7307 7310 7311 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 21 7310 7311 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 7317 7386 22 7316 7322 7334 7343 7344 7350 7356 7363 7374 7377 7384 7225 7317 7386 7398 7409 - """ - } + """} def make_data(): def gen_address_data(): for line in fake_data.addresses.strip().split('\n'): data = line.split(maxsplit=2) - yield (data[0], {k:v for k,v in zip(('id','addr','subver'),data)}) + yield (data[0], {k: v for k, v in zip(('id', 'addr', 'subver'), data)}) def gen_iterations_data(): for line in fake_data.iterations.strip().split('\n'): @@ -320,8 +319,7 @@ class fake_data: 'id': int(d['id']), 'addr': d['addr'], 'subver': d['subver'], - 'inflight': [int(n)+830000 for n in blocks[iter_no]], - } + 'inflight': [int(n)+830000 for n in blocks[iter_no]]} def gen_data(): for iter_no in iterations_data: From b9957f918298329ab9057a94946abccdaad9d038 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 4 Oct 2025 09:57:02 +0000 Subject: [PATCH 26/51] mmnode-ticker: support `--http-timeout` option --- mmgen_node_tools/Ticker.py | 4 ++++ mmgen_node_tools/data/version | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index ebefcf3..caee658 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -75,6 +75,7 @@ class DataSource: ['--compressed'], # adds 'Accept-Encoding: gzip' ['--proxy', cfg.proxy, isinstance(cfg.proxy, str)], ['--silent', not cfg.verbose], + ['--connect-timeout', str(gcfg.http_timeout), gcfg.http_timeout], [self.api_url]) if gcfg.testing: @@ -290,6 +291,9 @@ class DataSource: if gcfg.test_suite: kwargs.update({'timeout': 1, 'retry': 0}) + if gcfg.http_timeout: + kwargs.update({'timeout': gcfg.http_timeout}) + if gcfg.testing: Msg('\nyahooquery.Ticker(\n {},\n {}\n)'.format( self.symbols, diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 9f32e9d..6f735af 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev1 +3.6.dev2 From 06340ed521de5ba0333e2d4bf20105cde60b0e8c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sat, 4 Oct 2025 12:44:54 +0000 Subject: [PATCH 27/51] restore Nix support --- mmgen_node_tools/data/version | 2 +- nix/curl-cffi.nix | 50 +++++++++++++++++++++++++++++++ nix/node-tools-packages.nix | 4 +-- nix/use-system-libs.patch | 23 ++++++++++++++ nix/yahooquery-noversioning.patch | 45 ---------------------------- nix/yahooquery.nix | 39 ++++++++++-------------- 6 files changed, 92 insertions(+), 71 deletions(-) create mode 100644 nix/curl-cffi.nix create mode 100644 nix/use-system-libs.patch delete mode 100644 nix/yahooquery-noversioning.patch diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 6f735af..f9d163c 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev2 +3.6.dev3 diff --git a/nix/curl-cffi.nix b/nix/curl-cffi.nix new file mode 100644 index 0000000..15f9764 --- /dev/null +++ b/nix/curl-cffi.nix @@ -0,0 +1,50 @@ +{ + stdenv, + lib, + python, + buildPythonPackage, + fetchPypi, + curl-impersonate-chrome, +}: + +buildPythonPackage rec { + pname = "curl-cffi"; + # version = "0.13.0"; # uses option PROXY_CREDENTIAL_NO_REUSE, unavailable in current libcurl + version = "0.10.0"; + pyproject = true; + + src = fetchPypi { + pname = "curl_cffi"; + version = version; + # hash = "sha256-YuzZCjgr1QI3UONgbgqnyxo6i6QcFCcLjl4Unr9yxco="; # 0.13.0 + hash = "sha256-PjezUmjKWEkvVO0CCuS1DDPuDeutQUXbn3RvBO1GbrA="; # 0.10.0 + }; + + patches = [ ./use-system-libs.patch ]; + + buildInputs = [ curl-impersonate-chrome ]; + + build-system = with python.pkgs; [ + cffi + setuptools + ]; + + dependencies = with python.pkgs; [ + cffi + certifi + typing-extensions + ]; + + env = lib.optionalAttrs stdenv.cc.isGNU { + NIX_CFLAGS_COMPILE = "-Wno-error=incompatible-pointer-types"; + }; + + pythonImportsCheck = [ "curl_cffi" ]; + + meta = with lib; { + description = "Python binding for curl-impersonate via cffi"; + homepage = "https://curl-cffi.readthedocs.io"; + license = licenses.mit; + maintainers = with maintainers; [ chuangzhu ]; + }; +} diff --git a/nix/node-tools-packages.nix b/nix/node-tools-packages.nix index 19792c7..1244425 100644 --- a/nix/node-tools-packages.nix +++ b/nix/node-tools-packages.nix @@ -6,7 +6,7 @@ }; python-packages = with python.pkgs; { - yahooquery = (pkgs.callPackage ./yahooquery.nix {}); # ticker - pyyaml = pyyaml; # ticker + yahooquery = (callPackage ./yahooquery.nix {}); # ticker + pyyaml = pyyaml; # ticker }; } diff --git a/nix/use-system-libs.patch b/nix/use-system-libs.patch new file mode 100644 index 0000000..1d04487 --- /dev/null +++ b/nix/use-system-libs.patch @@ -0,0 +1,23 @@ +diff --git a/scripts/build.py b/scripts/build.py +index b705a0d..9bfcaab 100644 +--- a/scripts/build.py ++++ b/scripts/build.py +@@ -105,7 +105,6 @@ def get_curl_libraries(): + ffibuilder = FFI() + system = platform.system() + root_dir = Path(__file__).parent.parent +-download_libcurl() + + + ffibuilder.set_source( +@@ -114,9 +113,7 @@ ffibuilder.set_source( + #include "shim.h" + """, + # FIXME from `curl-impersonate` +- libraries=get_curl_libraries(), +- extra_objects=get_curl_archives(), +- library_dirs=[arch["libdir"]], ++ libraries=["curl-impersonate-chrome"], + source_extension=".c", + include_dirs=[ + str(root_dir / "include"), diff --git a/nix/yahooquery-noversioning.patch b/nix/yahooquery-noversioning.patch deleted file mode 100644 index 4bd3297..0000000 --- a/nix/yahooquery-noversioning.patch +++ /dev/null @@ -1,45 +0,0 @@ -diff --git a/pyproject.toml b/pyproject.toml -index 9d3fb29..399c215 100644 ---- a/pyproject.toml -+++ b/pyproject.toml -@@ -10,24 +10,24 @@ readme = "README.md" - - [tool.poetry.dependencies] - python = ">=3.8.1,<4.0" --requests = "^2.31.0" --pandas = "^2.0.3" --requests-futures = "^1.0.1" --tqdm = "^4.65.0" --lxml = "^4.9.3" --selenium = {version = "^4.10.0", optional = true} --beautifulsoup4 = "^4.12.2" -+requests = ">=2.31.0" -+pandas = ">=2.0.3" -+requests-futures = ">=1.0.1" -+tqdm = ">=4.65.0" -+lxml = ">=4.9.3" -+selenium = {version = ">=4.10.0", optional = true} -+beautifulsoup4 = ">=4.12.2" - - [tool.poetry.dev-dependencies] --pytest = "^7.4.0" --isort = "^5.0.0" --flake8 = "^6.0.0" --mypy = "^1.4.1" --pytest-cov = "^4.1.0" --black = "^23.7.0" --pre-commit = "^3.3.3" --ipython = "^8.0.0" --mkdocs-material = "^9.1.18" -+pytest = ">=7.4.0" -+isort = ">=5.0.0" -+flake8 = ">=6.0.0" -+mypy = ">=1.4.1" -+pytest-cov = ">=4.1.0" -+black = ">=23.7.0" -+pre-commit = ">=3.3.3" -+ipython = ">=8.0.0" -+mkdocs-material = ">=9.1.18" - - [build-system] - requires = ["poetry-core>=1.0.0"] diff --git a/nix/yahooquery.nix b/nix/yahooquery.nix index 8b1535d..3e8925f 100644 --- a/nix/yahooquery.nix +++ b/nix/yahooquery.nix @@ -1,37 +1,30 @@ { lib, - pkgs, - fetchFromGitHub, + buildPythonPackage, + fetchPypi, + python, }: -with pkgs.python312.pkgs; - buildPythonPackage rec { pname = "yahooquery"; - version = "2.3.7"; + version = "2.4.1"; pyproject = true; - disabled = pythonOlder "3.8.1"; - - src = fetchFromGitHub { - owner = "dpguthrie"; - repo = "yahooquery"; - rev = "refs/tags/v${version}"; - hash = "sha256-Iyuni1SoTB6f7nNFhN5A8Gnv9kV78frjpqvvW8qd+/M="; + src = fetchPypi { + pname = "yahooquery"; + version = version; + hash = "sha256-GQPGXq5qEtlelFAGNHkhbAeEbwE7riojkXkTUxt/rls="; }; - patches = [ ./yahooquery-noversioning.patch ]; + build-system = with python.pkgs; [ hatchling ]; - build-system = [ poetry-core ]; - - dependencies = [ - requests # ^2.31.0 - pandas # ^2.0.3 - requests-futures # ^1.0.1 - tqdm # ^4.65.0 - lxml # ^4.9.3 - selenium # {version = ^4.10.0, optional = true} - beautifulsoup4 # ^4.12.2 + propagatedBuildInputs = with python.pkgs; [ + (callPackage ./curl-cffi.nix {}) # >=0.10.0 + pandas + requests-futures + tqdm + lxml + beautifulsoup4 ]; doCheck = false; # skip tests From de833d75afb3fa313f2541eb1a71819a9c1ae141 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:47 +0000 Subject: [PATCH 28/51] whitespace, variable renames --- mmgen_node_tools/Ticker.py | 6 ++++-- test/cmdtest_d/misc.py | 14 ++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index caee658..d681a81 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -192,8 +192,10 @@ class DataSource: @property def api_url(self): return ( - f'https://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else - f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else + f'https://{self.api_host}/v1/tickers/btc-bitcoin' + if cfg.btc_only else + f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' + if self.asset_limit else f'https://{self.api_host}/v1/tickers') @property diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 9528b76..fb48282 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -106,10 +106,17 @@ class CmdTestScripts(CmdTestBase): shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) return 'ok' - def ticker(self, args=[], expect_list=None, cached=True, exit_val=None): + def ticker( + self, + args = [], + expect_list = None, + cached_data = True, + exit_val = None): t = self.spawn( 'mmnode-ticker', - (['--cached-data'] if cached else []) + self.ticker_args + args, + (['--cached-data'] if cached_data else []) + + self.ticker_args + + args, exit_val = exit_val) if expect_list: t.match_expect_list(expect_list) @@ -121,7 +128,7 @@ class CmdTestScripts(CmdTestBase): return t def ticker2(self): - t = self.ticker(cached=False) + t = self.ticker(cached_data=False) if not cfg.skipping_deps: t.expect('Creating') t.expect('Creating') @@ -137,7 +144,6 @@ class CmdTestScripts(CmdTestBase): 'BTC 23250.77 1.00000000 ETH 1659.66 0.07138094' ]) - def ticker4(self): return self.ticker( ['--widest','--add-columns=eurusd=x,inr-indian-rupee:79.5'], From 313c7af4bbf165ed65e5acd56e2461663d54bec2 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 29/51] mmgen-ticker: test caching JSON data --- MANIFEST.in | 1 + mmgen_node_tools/Ticker.py | 12 ++++++--- test/cmdtest_d/httpd/ticker.py | 27 +++++++++++++++++++ test/cmdtest_d/misc.py | 27 ++++++++++++++----- test/init.sh | 1 + .../fakemods/mmgen_node_tools/Ticker.py | 23 ++++++++++++++++ 6 files changed, 82 insertions(+), 9 deletions(-) create mode 100755 test/cmdtest_d/httpd/ticker.py create mode 100644 test/overlay/fakemods/mmgen_node_tools/Ticker.py diff --git a/MANIFEST.in b/MANIFEST.in index 77b53ac..2581100 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,5 +7,6 @@ include test/test-release.d/*.sh include test/modtest_d/*.py include test/cmdtest_d/*.py include test/cmdtest_d/include/cfg.py +include test/cmdtest_d/httpd/ticker.py include test/overlay/fakemods/mmgen_node_tools/*.py include test/ref/*/* diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index d681a81..bfe899b 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -175,6 +175,7 @@ class DataSource: desc = 'CoinPaprika' data_desc = 'cryptocurrency data' api_host = 'api.coinpaprika.com' + api_proto = 'https' ratelimit = 240 btc_ratelimit = 10 net_data_type = 'json' @@ -192,11 +193,11 @@ class DataSource: @property def api_url(self): return ( - f'https://{self.api_host}/v1/tickers/btc-bitcoin' + f'{self.api_proto}://{self.api_host}/v1/tickers/btc-bitcoin' if cfg.btc_only else - f'https://{self.api_host}/v1/tickers?limit={self.asset_limit}' + f'{self.api_proto}://{self.api_host}/v1/tickers?limit={self.asset_limit}' if self.asset_limit else - f'https://{self.api_host}/v1/tickers') + f'{self.api_proto}://{self.api_host}/v1/tickers') @property def json_fn(self): @@ -697,6 +698,11 @@ def make_cfg(gcfg_arg): cmd_args = gcfg._args cfg_in = get_cfg_in() + if gcfg.test_suite: # required for testing with overlay + from . import Ticker as this_mod + this_mod.src_cls = src_cls + this_mod.cfg_in = cfg_in + usr_rows = parse_usr_asset_arg('add_rows') usr_columns = parse_usr_asset_arg('add_columns', use_cf_file=True) query = parse_query_arg(cmd_args[0]) if cmd_args else None diff --git a/test/cmdtest_d/httpd/ticker.py b/test/cmdtest_d/httpd/ticker.py new file mode 100755 index 0000000..2d946ae --- /dev/null +++ b/test/cmdtest_d/httpd/ticker.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# +# MMGen Node Tools, terminal-based programs for Bitcoin and forkcoin nodes +# Copyright (C)2013-2025 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-node-tools + +""" +test.cmdtest_d.httpd.ticker: Ticker WSGI http server +""" + +from . import HTTPD + +class TickerServer(HTTPD): + name = 'ticker server' + port = 19900 + content_type = 'application/json' + + def make_response_body(self, method, environ): + + with open(f'test/ref/ticker/ticker.json') as fh: + text = fh.read() + + return text.encode() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index fb48282..e9d4c32 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -16,6 +16,7 @@ import os, shutil from ..include.common import cfg from .base import CmdTestBase +from .httpd.ticker import TickerServer refdir = os.path.join('test','ref','ticker') @@ -70,8 +71,9 @@ class CmdTestScripts(CmdTestBase): ), 'ticker': ( "'mmnode-ticker' script", - ('ticker1', 'ticker [--help)'), + ('ticker1', 'ticker [--help]'), ('ticker2', 'ticker (bad proxy)'), + ('ticker2a', 'ticker [--download=cc]'), ('ticker3', 'ticker [--cached-data]'), ('ticker4', 'ticker [--cached-data --wide]'), ('ticker5', 'ticker [--cached-data --wide --adjust=-0.766] (usr cfg file)'), @@ -90,9 +92,12 @@ class CmdTestScripts(CmdTestBase): ) } - @property - def ticker_args(self): - return [ f'--cachedir={self.tmpdir}', '--proxy=http://asdfzxcv:32459' ] + def __init__(self, cfg, trunner, cfgs, spawn): + if not trunner: + return + self.ticker_server = TickerServer(cfg) + self.ticker_server.start() + return super().__init__(cfg, trunner, cfgs, spawn) @property def nt_datadir(self): @@ -100,7 +105,6 @@ class CmdTestScripts(CmdTestBase): def ticker_setup(self): self.spawn('',msg_only=True) - shutil.copy2(os.path.join(refdir,'ticker.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) @@ -111,11 +115,15 @@ class CmdTestScripts(CmdTestBase): args = [], expect_list = None, cached_data = True, + add_opts = [], + use_proxy = True, exit_val = None): t = self.spawn( 'mmnode-ticker', (['--cached-data'] if cached_data else []) - + self.ticker_args + + [f'--cachedir={self.tmpdir}'] + + (['--proxy=http://asdfzxcv:32459'] if use_proxy else []) + + add_opts + args, exit_val = exit_val) if expect_list: @@ -136,6 +144,13 @@ class CmdTestScripts(CmdTestBase): t.exit_val = 1 if ret else 3 return t + def ticker2a(self): + t = self.ticker( + add_opts = ['--proxy', '', '--download=cc'], + cached_data = False, + use_proxy = False) + return t + def ticker3(self): return self.ticker( [], diff --git a/test/init.sh b/test/init.sh index 134f4f3..eec6ec3 100755 --- a/test/init.sh +++ b/test/init.sh @@ -98,6 +98,7 @@ create_test_links() { test/modtest.py hard test/test-release.sh symbolic test/cmdtest_d/base.py symbolic + test/cmdtest_d/httpd/__init__.py symbolic test/cmdtest_d/include/common.py symbolic test/cmdtest_d/include/runner.py symbolic test/cmdtest_d/include/group_mgr.py symbolic diff --git a/test/overlay/fakemods/mmgen_node_tools/Ticker.py b/test/overlay/fakemods/mmgen_node_tools/Ticker.py new file mode 100644 index 0000000..8d1ede4 --- /dev/null +++ b/test/overlay/fakemods/mmgen_node_tools/Ticker.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# MMGen Node Tools, terminal-based programs for Bitcoin and forkcoin nodes +# Copyright (C)2013-2025 The MMGen Project +# Licensed under the GNU General Public License, Version 3: +# https://www.gnu.org/licenses +# Public project repositories: +# https://github.com/mmgen/mmgen-node-tools +# https://gitlab.com/mmgen/mmgen-node-tools + +""" +fakemods.mmgen_node_tools.Ticker: fake module for Ticker class +""" + +from .Ticker_orig import * + +class overlay_fake_DataSource: + class coinpaprika: + api_host = 'localhost:19900' + api_proto = 'http' + +DataSource.coinpaprika.api_host = overlay_fake_DataSource.coinpaprika.api_host +DataSource.coinpaprika.api_proto = overlay_fake_DataSource.coinpaprika.api_proto From a8adef0be54c752c10a8f34d594ed3faaa23de0c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 30/51] mmgen-ticker: various fixes and cleanups --- mmgen_node_tools/Ticker.py | 39 +++++++++++++++++++--------------- test/cmdtest_d/httpd/ticker.py | 2 +- test/cmdtest_d/misc.py | 36 +++++++++++++++---------------- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bfe899b..9aee86c 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -25,7 +25,7 @@ from decimal import Decimal from collections import namedtuple from mmgen.color import red, yellow, green, blue, orange, gray -from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen +from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -99,18 +99,21 @@ class DataSource: if not os.path.exists(cfg.cachedir): os.makedirs(cfg.cachedir) - if not os.path.exists(self.json_fn): - open(self.json_fn, 'w').write('{}') - use_cached_data = cfg.cached_data and not gcfg.download if use_cached_data: data_type = 'json' - data_in = open(self.json_fn).read() + try: + data_in = open(self.json_fn).read() + except FileNotFoundError: + die(1, f'Cannot use cached data, because {self.json_fn_disp} does not exist') else: data_type = self.net_data_type - elapsed = int(time.time() - os.stat(self.json_fn).st_mtime) - if elapsed >= self.timeout or gcfg.testing: + try: + mtime = os.stat(self.json_fn).st_mtime + except FileNotFoundError: + mtime = 0 + if (elapsed := int(time.time() - mtime)) >= self.timeout or gcfg.testing: if gcfg.testing: msg('') self.fetch_delay() @@ -148,14 +151,14 @@ class DataSource: if use_cached_data: if not cfg.quiet: - msg(f'Using cached data from ~/{self.json_fn_rel}') + msg(f'Using cached data from {self.json_fn_disp}') else: if os.path.exists(self.json_fn): os.rename(self.json_fn, self.json_fn + '.bak') with open(self.json_fn, 'w') as fh: fh.write(json_text) if not cfg.quiet: - msg(f'JSON data cached to ~/{self.json_fn_rel}') + msg(f'JSON data cached to {self.json_fn_disp}') if gcfg.download: sys.exit(0) @@ -168,8 +171,8 @@ class DataSource: return data @property - def json_fn_rel(self): - return os.path.relpath(self.json_fn, start=homedir) + def json_fn_disp(self): + return '~/' + os.path.relpath(self.json_fn, start=homedir) class coinpaprika(base): desc = 'CoinPaprika' @@ -186,8 +189,9 @@ class DataSource: self.asset_limit = int(cfg.asset_limit or self.dfl_asset_limit) def rate_limit_errmsg(self, elapsed): + rem = self.timeout - elapsed return ( - f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds' + + f'Rate limit exceeded! Retry in {rem} second{suf(rem)}' + ('' if cfg.btc_only else ', or use --cached-data or --btc')) @property @@ -270,7 +274,8 @@ class DataSource: 'last_updated': data['regularMarketTime']} def rate_limit_errmsg(self, elapsed): - return f'Rate limit exceeded! Retry in {self.timeout-elapsed} seconds, or use --cached-data' + rem = self.timeout - elapsed + return f'Rate limit exceeded! Retry in {rem} second{suf(rem)}, or use --cached-data' @property def json_fn(self): @@ -364,10 +369,10 @@ def gen_data(data): checking for duplicates. """ - def dup_sym_errmsg(dup_sym): + def dup_sym_errmsg(data_type, dup_sym): return ( f'The symbol {dup_sym!r} is shared by the following assets:\n' + - '\n ' + '\n '.join(d['id'] for d in data['cc'] if d['symbol'] == dup_sym) + + '\n ' + '\n '.join(d['id'] for d in data[data_type].data if d['symbol'] == dup_sym) + '\n\nPlease specify the asset by one of the full IDs listed above\n' + f'instead of {dup_sym!r}') @@ -421,7 +426,7 @@ def gen_data(data): if not isinstance(v, dict): die(2, str(v)) if id in found['id']: - die(1, dup_sym_errmsg(id)) + die(1, dup_sym_errmsg('fi', id)) if m := data['hi'].get(k): spot = v['regularMarketPrice']['raw'] hist = tuple(m.values()) @@ -444,7 +449,7 @@ def gen_data(data): if wants[k]: if d[k] in wants[k]: if d[k] in found[k]: - die(1, dup_sym_errmsg(d[k])) + die(1, dup_sym_errmsg('cc', d[k])) if not 'price_usd' in d: d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd diff --git a/test/cmdtest_d/httpd/ticker.py b/test/cmdtest_d/httpd/ticker.py index 2d946ae..6131875 100755 --- a/test/cmdtest_d/httpd/ticker.py +++ b/test/cmdtest_d/httpd/ticker.py @@ -21,7 +21,7 @@ class TickerServer(HTTPD): def make_response_body(self, method, environ): - with open(f'test/ref/ticker/ticker.json') as fh: + with open('test/ref/ticker/ticker.json') as fh: text = fh.read() return text.encode() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index e9d4c32..1cde41e 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -61,19 +61,16 @@ class CmdTestScripts(CmdTestBase): color = True cmd_group_in = ( - ('subgroup.ticker_setup', []), - ('subgroup.ticker', ['ticker_setup']), + ('subgroup.ticker', []), ) cmd_subgroups = { - 'ticker_setup': ( - "setup for 'ticker' subgroup", - ('ticker_setup', 'ticker setup'), - ), 'ticker': ( "'mmnode-ticker' script", ('ticker1', 'ticker [--help]'), + ('copy_files', 'copying JSON files to cache'), + ('ticker1a', 'ticker [--download=cc] (early caching)'), + ('ticker1b', 'ticker [--download=cc] (late caching)'), ('ticker2', 'ticker (bad proxy)'), - ('ticker2a', 'ticker [--download=cc]'), ('ticker3', 'ticker [--cached-data]'), ('ticker4', 'ticker [--cached-data --wide]'), ('ticker5', 'ticker [--cached-data --wide --adjust=-0.766] (usr cfg file)'), @@ -103,7 +100,7 @@ class CmdTestScripts(CmdTestBase): def nt_datadir(self): return os.path.join( cfg.data_dir_root, 'node_tools' ) - def ticker_setup(self): + def copy_files(self): self.spawn('',msg_only=True) shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) @@ -135,20 +132,23 @@ class CmdTestScripts(CmdTestBase): t.expect('USAGE:') return t - def ticker2(self): - t = self.ticker(cached_data=False) - if not cfg.skipping_deps: - t.expect('Creating') - t.expect('Creating') - ret = t.expect(['proxy host could not be resolved', 'unexpected keyword']) - t.exit_val = 1 if ret else 3 - return t - - def ticker2a(self): + def ticker1a(self, first_run=True): t = self.ticker( add_opts = ['--proxy', '', '--download=cc'], cached_data = False, use_proxy = False) + if first_run and not cfg.skipping_deps: + t.expect('Creating') + t.expect('Creating') + return t + + def ticker1b(self): + return self.ticker1a(first_run=False) + + def ticker2(self): + t = self.ticker(cached_data=False) + ret = t.expect(['proxy host could not be resolved', 'unexpected keyword']) + t.exit_val = 1 if ret else 3 return t def ticker3(self): From 3b8aede9bad83e4187e888d7be939ae909508075 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 31/51] Ticker.gen_data(): new `process_data` class --- mmgen_node_tools/Ticker.py | 135 ++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 55 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 9aee86c..8580553 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -25,7 +25,7 @@ from decimal import Decimal from collections import namedtuple from mmgen.color import red, yellow, green, blue, orange, gray -from mmgen.util import msg, msg_r, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf +from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -389,6 +389,73 @@ def gen_data(data): if error: die(1, 'Missing data, exiting') + class process_data: + + def cc(): + nonlocal btcusd + for d in data['cc']: + if d['id'] == 'btc-bitcoin': + btcusd = Decimal(str(d['quotes']['USD']['price'])) + break + else: + raise ValueError('malformed cryptocurrency data') + for k in ('id', 'symbol'): + for d in data['cc']: + if wants[k]: + if d[k] in wants[k]: + if d[k] in found[k]: + die(1, dup_sym_errmsg('cc', d[k])) + if not 'price_usd' in d: + d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) + d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd + d['percent_change_24h'] = d['quotes']['USD']['percent_change_24h'] + d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] + d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] + d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] + d['last_updated'] = int(datetime.datetime.fromisoformat( + d['last_updated']).timestamp()) + yield (d['id'], d) + found[k].add(d[k]) + wants[k].remove(d[k]) + if d[k] in usr_rate_assets_want[k]: + rate_assets[d['symbol']] = d # NB: using symbol instead of ID for key + else: + break + + def fi(): + get_id = src_cls['fi'].get_id + conv_func = src_cls['fi'].conv_data + for k, v in data['fi'].items(): + id = get_id(k, v) + if wants['id']: + if id in wants['id']: + if not isinstance(v, dict): + die(2, str(v)) + if id in found['id']: + die(1, dup_sym_errmsg('fi', id)) + if hist := hist_close.get(k): + spot = v['regularMarketPrice']['raw'] + v['pct_chg_1wk'] = (spot / hist.close_1wk - 1) * 100 + v['pct_chg_4wks'] = (spot / hist.close_4wks - 1) * 100 # 4 weeks ≈ 1 month + v['pct_chg_1y'] = (spot / hist.close_1y - 1) * 100 + else: + v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None + yield (id, conv_func(id, v, btcusd)) + found['id'].add(id) + wants['id'].remove(id) + if id in usr_rate_assets_want['id']: # NB: using symbol instead of ID for key: + rate_assets[k] = conv_func(id, v, btcusd) + else: + break + + def hi(): + ret = namedtuple('historical_closing_prices', ['close_1wk', 'close_4wks', 'close_1y']) + nonlocal hist_close + for k, v in data['hi'].items(): + hist = tuple(v.values()) + hist_close[k] = ret(hist[-2]['close'], hist[-5]['close'], hist[0]['close']) + return () + rows_want = { 'id': {r.id for r in cfg.rows if isinstance(r, tuple) and r.id} - {'usd-us-dollar'}, 'symbol': {r.symbol for r in cfg.rows if isinstance(r, tuple) and r.id is None} - {'USD'}} @@ -411,62 +478,20 @@ def gen_data(data): wants = {k: rows_want[k] | usr_wants[k] for k in ('id', 'symbol')} - for d in data['cc']: - if d['id'] == 'btc-bitcoin': - btcusd = Decimal(str(d['quotes']['USD']['price'])) - break + btcusd = Decimal('1') # dummy + hist_close = {} - get_id = src_cls['fi'].get_id - conv_func = src_cls['fi'].conv_data + parse_fail = False + for data_type in ('cc', 'hi', 'fi'): # 'fi' depends on 'cc' and 'hi' so must go last + if data_type in data: + try: + yield from getattr(process_data, data_type)() + except Exception as e: + rmsg(f'Error in source data {data_type!r}: {e}') + parse_fail = True - for k, v in data['fi'].items(): - id = get_id(k, v) - if wants['id']: - if id in wants['id']: - if not isinstance(v, dict): - die(2, str(v)) - if id in found['id']: - die(1, dup_sym_errmsg('fi', id)) - if m := data['hi'].get(k): - spot = v['regularMarketPrice']['raw'] - hist = tuple(m.values()) - v['pct_chg_1wk'], v['pct_chg_4wks'], v['pct_chg_1y'] = ( - (spot / hist[-2]['close'] - 1) * 100, - (spot / hist[-5]['close'] - 1) * 100, # 4 weeks ≈ 1 month - (spot / hist[0]['close'] - 1) * 100) - else: - v['pct_chg_1wk'] = v['pct_chg_4wks'] = v['pct_chg_1y'] = None - yield (id, conv_func(id, v, btcusd)) - found['id'].add(id) - wants['id'].remove(id) - if id in usr_rate_assets_want['id']: - rate_assets[k] = conv_func(id, v, btcusd) # NB: using symbol instead of ID for key - else: - break - - for k in ('id', 'symbol'): - for d in data['cc']: - if wants[k]: - if d[k] in wants[k]: - if d[k] in found[k]: - die(1, dup_sym_errmsg('cc', d[k])) - if not 'price_usd' in d: - d['price_usd'] = Decimal(str(d['quotes']['USD']['price'])) - d['price_btc'] = Decimal(str(d['quotes']['USD']['price'])) / btcusd - d['percent_change_24h'] = d['quotes']['USD']['percent_change_24h'] - d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] - d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] - d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] - # .replace('Z','+00:00') -- Python 3.9 backport - d['last_updated'] = int(datetime.datetime.fromisoformat( - d['last_updated'].replace('Z', '+00:00')).timestamp()) - yield (d['id'], d) - found[k].add(d[k]) - wants[k].remove(d[k]) - if d[k] in usr_rate_assets_want[k]: - rate_assets[d['symbol']] = d # NB: using symbol instead of ID for key - else: - break + if parse_fail: + die(2, 'Invalid data encountered, exiting') check_assets_found(usr_wants, found) From e7de6890796505f2fa9499fae80c459455dc7aa2 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Sun, 12 Oct 2025 10:01:51 +0000 Subject: [PATCH 32/51] mmnode-ticker: cache network data after parsing --- mmgen_node_tools/Ticker.py | 48 ++++++++++++++++++++++++----------- mmgen_node_tools/data/version | 2 +- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 8580553..bbca8ef 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -19,7 +19,7 @@ mmgen_node_tools.Ticker: Display price information for cryptocurrency and other # Possible alternatives: # - https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,LTC&tsyms=USD,EUR -import sys, os, re, time, datetime, json, yaml, random +import os, re, time, datetime, json, yaml, random from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple @@ -149,20 +149,18 @@ class DataSource: elif 'error' in data: die(1, data['error']) + self.data = self.postprocess_data(data) + if use_cached_data: + self.json_text = None if not cfg.quiet: msg(f'Using cached data from {self.json_fn_disp}') else: - if os.path.exists(self.json_fn): - os.rename(self.json_fn, self.json_fn + '.bak') - with open(self.json_fn, 'w') as fh: - fh.write(json_text) - if not cfg.quiet: - msg(f'JSON data cached to {self.json_fn_disp}') - if gcfg.download: - sys.exit(0) + self.json_text = json_text + if cache_data(self, no_overwrite=True): + self.json_text = None - return self.postprocess_data(data) + return self def json_data_error_msg(self, json_text): pass @@ -393,14 +391,14 @@ def gen_data(data): def cc(): nonlocal btcusd - for d in data['cc']: + for d in data['cc'].data: if d['id'] == 'btc-bitcoin': btcusd = Decimal(str(d['quotes']['USD']['price'])) break else: raise ValueError('malformed cryptocurrency data') for k in ('id', 'symbol'): - for d in data['cc']: + for d in data['cc'].data: if wants[k]: if d[k] in wants[k]: if d[k] in found[k]: @@ -425,7 +423,7 @@ def gen_data(data): def fi(): get_id = src_cls['fi'].get_id conv_func = src_cls['fi'].conv_data - for k, v in data['fi'].items(): + for k, v in data['fi'].data.items(): id = get_id(k, v) if wants['id']: if id in wants['id']: @@ -451,7 +449,7 @@ def gen_data(data): def hi(): ret = namedtuple('historical_closing_prices', ['close_1wk', 'close_4wks', 'close_1y']) nonlocal hist_close - for k, v in data['hi'].items(): + for k, v in data['hi'].data.items(): hist = tuple(v.values()) hist_close[k] = ret(hist[-2]['close'], hist[-5]['close'], hist[0]['close']) return () @@ -489,10 +487,15 @@ def gen_data(data): except Exception as e: rmsg(f'Error in source data {data_type!r}: {e}') parse_fail = True + else: + cache_data(data[data_type]) if parse_fail: die(2, 'Invalid data encountered, exiting') + if gcfg.download: + return + check_assets_found(usr_wants, found) for asset in (cfg.usr_rows + cfg.usr_columns): @@ -518,6 +521,18 @@ def gen_data(data): 'price_btc': Decimal(1) / btcusd, 'last_updated': None}) +def cache_data(data_src, no_overwrite=False): + if data_src.json_text: + if os.path.exists(data_src.json_fn): + if no_overwrite: + return False + os.rename(data_src.json_fn, data_src.json_fn + '.bak') + with open(data_src.json_fn, 'w') as fh: + fh.write(data_src.json_text) + if not cfg.quiet: + msg(f'JSON data cached to {data_src.json_fn_disp}') + return True + def main(): def update_sample_file(usr_cfg_file): @@ -558,7 +573,7 @@ def main(): return if gcfg.list_ids: - do_pager('\n'.join(e['id'] for e in src_data['cc'])) + do_pager('\n'.join(e.data['id'] for e in src_data['cc'])) return global now @@ -566,6 +581,9 @@ def main(): data = dict(gen_data(src_data)) + if gcfg.download: + return + (do_pager if cfg.pager else Msg_r)( '\n'.join(getattr(Ticker, cfg.clsname)(data).gen_output()) + '\n') diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index f9d163c..4e5cebc 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev3 +3.6.dev4 From 9ffaed6a912a4a42c84f5e9675a0d70d5edd9fae Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 13 Oct 2025 14:58:54 +0000 Subject: [PATCH 33/51] mmnode-ticker: improve options setting Options set in the cfg file can now be unset on the command line, e.g.: $ mmnode-ticker --no-quiet --no-cached-data --- mmgen_node_tools/Ticker.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bbca8ef..49ce857 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -712,6 +712,12 @@ def make_cfg(gcfg_arg): rows += (hdr,) + uniq_data return rows + def get_cfg_var(name): + if name in gcfg._uopts: + return getattr(gcfg, name) + else: + return getattr(gcfg, name) or cfg_in.cfg.get(name) + cfg_tuple = namedtuple('global_cfg',[ 'rows', 'usr_rows', @@ -772,27 +778,24 @@ def make_cfg(gcfg_arg): query = query, adjust = (lambda x: (100 + x) / 100 if x else 1)(Decimal(gcfg.adjust or 0)), clsname = 'trading' if query else 'overview', - btc_only = gcfg.btc or cfg_in.cfg.get('btc'), - add_prec = parse_add_precision(gcfg.add_precision or cfg_in.cfg.get('add_precision')), - cachedir = gcfg.cachedir or cfg_in.cfg.get('cachedir') or dfl_cachedir, + btc_only = get_cfg_var('btc'), + add_prec = parse_add_precision(get_cfg_var('add_precision')), + cachedir = get_cfg_var('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), portfolio = - get_portfolio() - if cfg_in.portfolio - and (gcfg.portfolio or cfg_in.cfg.get('portfolio')) - and not query + get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query else None, - percent_cols = parse_percent_cols(gcfg.percent_cols or cfg_in.cfg.get('percent_cols')), - asset_limit = gcfg.asset_limit or cfg_in.cfg.get('asset_limit'), - cached_data = gcfg.cached_data or cfg_in.cfg.get('cached_data'), - elapsed = gcfg.elapsed or cfg_in.cfg.get('elapsed'), - name_labels = gcfg.name_labels or cfg_in.cfg.get('name_labels'), - pager = gcfg.pager or cfg_in.cfg.get('pager'), - thousands_comma = gcfg.thousands_comma or cfg_in.cfg.get('thousands_comma'), - update_time = gcfg.update_time or cfg_in.cfg.get('update_time'), - quiet = gcfg.quiet or cfg_in.cfg.get('quiet'), - verbose = gcfg.verbose or cfg_in.cfg.get('verbose')) + percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), + asset_limit = get_cfg_var('asset_limit'), + cached_data = get_cfg_var('cached_data'), + elapsed = get_cfg_var('elapsed'), + name_labels = get_cfg_var('name_labels'), + pager = get_cfg_var('pager'), + thousands_comma = get_cfg_var('thousands_comma'), + update_time = get_cfg_var('update_time'), + quiet = get_cfg_var('quiet'), + verbose = get_cfg_var('verbose')) def get_cfg_in(): ret = namedtuple('cfg_in_data', ['cfg', 'portfolio', 'cfg_file', 'portfolio_file']) From 083b29eae829d24be0714355f759d1258300da87 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 13 Oct 2025 14:59:00 +0000 Subject: [PATCH 34/51] mmnode-ticker: minor fixes and cleanups --- mmgen_node_tools/Ticker.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 49ce857..9d1c9df 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -25,7 +25,7 @@ from decimal import Decimal from collections import namedtuple from mmgen.color import red, yellow, green, blue, orange, gray -from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf +from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf, is_int from mmgen.ui import do_pager homedir = os.getenv('HOME') @@ -184,7 +184,7 @@ class DataSource: dfl_asset_limit = 2000 def __init__(self): - self.asset_limit = int(cfg.asset_limit or self.dfl_asset_limit) + self.asset_limit = int(cfg.asset_limit) if is_int(cfg.asset_limit) else self.dfl_asset_limit def rate_limit_errmsg(self, elapsed): rem = self.timeout - elapsed @@ -573,7 +573,7 @@ def main(): return if gcfg.list_ids: - do_pager('\n'.join(e.data['id'] for e in src_data['cc'])) + do_pager('\n'.join(e['id'] for e in src_data['cc'].data)) return global now @@ -606,7 +606,7 @@ def make_cfg(gcfg_arg): return tuple(gen()) def parse_percent_cols(arg): - if arg is None: + if arg is None or arg.lower() in ('none', ''): return [] res = arg.lower().split(',') for s in res: @@ -638,6 +638,8 @@ def make_cfg(gcfg_arg): source = parsed_id.source) cl_opt = getattr(gcfg, key) + if (cl_opt or '').lower() in ('', 'none'): + return () cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) @@ -718,6 +720,12 @@ def make_cfg(gcfg_arg): else: return getattr(gcfg, name) or cfg_in.cfg.get(name) + def get_proxy(name): + proxy = getattr(gcfg, name) + return ( + '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') + else (proxy or cfg_in.cfg.get(name))) + cfg_tuple = namedtuple('global_cfg',[ 'rows', 'usr_rows', @@ -749,7 +757,6 @@ def make_cfg(gcfg_arg): src_cls = {k: getattr(DataSource, v) for k, v in DataSource.get_sources().items()} fi_pat = src_cls['fi'].asset_id_pat - cmd_args = gcfg._args cfg_in = get_cfg_in() if gcfg.test_suite: # required for testing with overlay @@ -757,15 +764,15 @@ def make_cfg(gcfg_arg): this_mod.src_cls = src_cls this_mod.cfg_in = cfg_in + if cmd_args := gcfg._args: + if len(cmd_args) > 1: + die(1, 'Only one command-line argument is allowed') + query = parse_query_arg(cmd_args[0]) + else: + query = None + usr_rows = parse_usr_asset_arg('add_rows') usr_columns = parse_usr_asset_arg('add_columns', use_cf_file=True) - query = parse_query_arg(cmd_args[0]) if cmd_args else None - - def get_proxy(name): - proxy = getattr(gcfg, name) - return ( - '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') - else (proxy or cfg_in.cfg.get(name))) proxy = get_proxy('proxy') proxy = None if proxy == 'none' else proxy From 060b968ad4b88f845b9cd95fa89174a69b4b9743 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 13 Oct 2025 14:59:00 +0000 Subject: [PATCH 35/51] mmnode-ticker: display crypto assets by market cap Examples: # Display top 2000 assets by market cap: $ mmnode-ticker 2000 # Display assets 201-300 by market cap, displaying all available columns: $ mmnode-ticker --widest 201-300 # Display asset 32 by market cap: $ mmnode-ticker 32-32 Testing/demo: $ test/cmdtest.py -e scripts.ticker --- mmgen_node_tools/Ticker.py | 53 +++++++++++++++++++++++++++++---- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 25 +++++++++++----- test/cmdtest_d/misc.py | 42 ++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 13 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 9d1c9df..bf3398b 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -182,6 +182,7 @@ class DataSource: net_data_type = 'json' has_verbose = True dfl_asset_limit = 2000 + max_asset_idx = 1_000_000 def __init__(self): self.asset_limit = int(cfg.asset_limit) if is_int(cfg.asset_limit) else self.dfl_asset_limit @@ -576,6 +577,17 @@ def main(): do_pager('\n'.join(e['id'] for e in src_data['cc'].data)) return + global cfg + + if cfg.asset_range: + func = DataSource.coinpaprika.parse_asset_id + n, m = cfg.asset_range + cfg = cfg._replace(rows = + tuple(func(e['id'], require_label=False) for e in src_data['cc'].data[n-1:m]) + + tuple(['-']) + + tuple([func('btc-bitcoin', require_label=True)]) + + tuple(r for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi')) + global now now = 1659465400 if gcfg.test_suite else time.time() # 1659524400 1659445900 @@ -643,6 +655,21 @@ def make_cfg(gcfg_arg): cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) + def parse_asset_range(s): + max_idx = DataSource.coinpaprika.max_asset_idx + match s.split('-'): + case [a, b] if is_int(a) and is_int(b): + n, m = (int(a), int(b)) + case [a] if is_int(a): + n, m = (1, int(a)) + case _: + return None + if n < 1 or m < 1 or n > m: + raise ValueError(f'‘{s}’: invalid asset range specifier') + if m > max_idx: + raise ValueError(f'‘{s}’: end of range must be <= {max_idx}') + return (n, m) + def parse_query_arg(s): """ asset_id:amount[:to_asset_id[:to_amount]] @@ -731,6 +758,7 @@ def make_cfg(gcfg_arg): 'usr_rows', 'usr_columns', 'query', + 'asset_range', 'adjust', 'clsname', 'btc_only', @@ -767,8 +795,10 @@ def make_cfg(gcfg_arg): if cmd_args := gcfg._args: if len(cmd_args) > 1: die(1, 'Only one command-line argument is allowed') - query = parse_query_arg(cmd_args[0]) + asset_range = parse_asset_range(cmd_args[0]) + query = None if asset_range else parse_query_arg(cmd_args[0]) else: + asset_range = None query = None usr_rows = parse_usr_asset_arg('add_rows') @@ -783,6 +813,7 @@ def make_cfg(gcfg_arg): usr_rows = usr_rows, usr_columns = usr_columns, query = query, + asset_range = asset_range, adjust = (lambda x: (100 + x) / 100 if x else 1)(Decimal(gcfg.adjust or 0)), clsname = 'trading' if query else 'overview', btc_only = get_cfg_var('btc'), @@ -930,12 +961,14 @@ class Ticker: if self.table_hdr: yield self.table_hdr - for row in self.rows: + for n, row in enumerate(self.rows, cfg.asset_range[0] if cfg.asset_range else 1): if isinstance(row, str): + if cfg.asset_range: + return yield ('-' * self.hl_wid) else: try: - yield self.fmt_row(self.data[row.id]) + yield self.fmt_row(self.data[row.id], idx=n) except KeyError: yield gray(f'(no data for {row.id})') @@ -989,7 +1022,7 @@ class Ticker: d['price_usd'] / self.col_usd_prices[k] ) * self.adjust for k in self.col_ids} - def fmt_row(self, d, amt=None, amt_fmt=None): + def fmt_row(self, d, amt=None, amt_fmt=None, idx=None): def fmt_pct(n): return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') @@ -1002,6 +1035,7 @@ class Ticker: amt_fmt = amt_fmt.rstrip('0').rstrip('.') return self.fs_num.format( + idx = idx, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], pc1 = fmt_pct(d.get('percent_change_7d')), pc2 = fmt_pct(d.get('percent_change_24h')), @@ -1061,6 +1095,15 @@ class Ticker: self.fs_num2 = ''.join(col_fs_data[c].fs_num for c in cols2) self.hl_wid2 = sum(col_fs_data[c].wid for c in cols2) + if cfg.asset_range: + def get_col1_w(): + for n, r in enumerate(cfg.rows): + if isinstance(r, str): + return len(str(n)) + col1_w = get_col1_w() + self.fs_str = ' ' * (col1_w + 2) + self.fs_str + self.fs_num = f'{{idx:{col1_w}}}) ' + self.fs_num + @property def table_hdr(self): return self.fs_str.format( @@ -1136,7 +1179,7 @@ class Ticker: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 - def fmt_row(self, d): + def fmt_row(self, d, idx=None): id = d['id'] p = self.prices[id][self.asset.id] * self.asset.amount p_spot = '{:{}{}.{}f}'.format(p, self.max_wid, self.comma, 8+cfg.add_prec) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 4e5cebc..636c831 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev4 +3.6.dev5 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index c36d886..8efb97a 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -25,7 +25,7 @@ opts_data = { ], 'text': { 'desc': 'Display prices for cryptocurrency and other assets', - 'usage': '[opts] [TRADE_SPECIFIER]', + 'usage': '[opts] [TRADE_SPECIFIER | ASSET_RANGE]', 'options': """ -h, --help Print this help message --, --longhelp Print help message for long options (common options) @@ -71,14 +71,19 @@ opts_data = { """, 'notes': """ -The script has two display modes: ‘overview’, the default, and ‘trading’, the -latter being enabled when a TRADE_SPECIFIER argument (see below) is supplied -on the command line. +The script has three display modes: ‘overview’, enabled when no arguments are +given on the command line; ‘trading’, when a TRADE_SPECIFIER argument (see +below) is given; and ‘market cap’, when an ASSET_RANGE (see below) is given. Overview mode displays prices of all configured assets, and optionally the -user’s portfolio, while trading mode displays the price of a given quantity -of an asset in relation to other assets, optionally comparing an offered -price to the spot price. +user’s portfolio; trading mode displays the price of a given quantity of an +asset in relation to other assets, optionally comparing an offered price to +the spot price; and market cap mode lists a range of crypto assets selected +by current market cap. + +The ASSET_RANGE argument can be either an integer N, in which case the top +N assets by market cap will be displayed, or a hyphen-separated range N-M, +in which case assets from N to M by market cap will be displayed. ASSETS consist of either a symbol (e.g. ‘xmr’) or full ID (see --list-ids) consisting of symbol plus label (e.g. ‘xmr-monero’). In cases where the @@ -199,6 +204,12 @@ $ mmnode-ticker usd:2700:btc:0.123 # current spot price, at specified USDINR rate: $ mmnode-ticker -n -c inr-indian-rupee:79.5 inr:200000:btc:0.1 +# Display top 20 crypto assets by market cap, adding a Euro column: +$ mmnode-ticker -c eurusd=x 20 + +# Same as above, specifying assets using a range: +$ mmnode-ticker -c eurusd=x 1-20 + CONFIGURED ASSETS: {assets} diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 1cde41e..c2dceff 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -86,6 +86,10 @@ class CmdTestScripts(CmdTestBase): ('ticker15', 'ticker [--cached-data --wide --btc btc:2:usd:45000]'), ('ticker16', 'ticker [--cached-data --wide --elapsed -c eur,omr-omani-rial:2.59r'), ('ticker17', 'ticker [--cached-data --wide --elapsed -c bgn-bulgarian-lev:0.5113r:eur'), + ('ticker18', 'ticker [--cached-data --widest --add-columns eurusd=x 10]'), + ('ticker19', 'ticker [--cached-data 1-5]'), + ('ticker20', 'ticker [--cached-data 2-5]'), + ('ticker21', 'ticker [--cached-data 5-5]'), ) } @@ -319,3 +323,41 @@ class CmdTestScripts(CmdTestBase): 'BITCOIN 23,250.77 42,731.767 1.00000000', 'BULGARIAN LEV 0.54 1.000 0.00002340', ]) + + def ticker18(self): + return self.ticker( + ['10'], + [ + r'1\) BITCOIN 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', + r'6\) ALGORAND 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' + ], + add_opts = ['--widest', '--add-columns=eurusd=x']) + + def ticker19(self): + return self.ticker( + ['1-5'], + [ + 'USD EURUSD=X BTC ' + r'1\) BTC 23250.77 21848.7527 1.00000000', + r'5\) ADA 0.51 0.4764 0.00002180', + ], + add_opts = ['--add-columns=eurusd=x']) + + def ticker20(self): + return self.ticker( + ['2-5'], + [ + 'USD EURUSD=X BTC ' + r'2\) ETH 1659.66 1559.5846 0.07138094', + r'5\) ADA 0.51 0.4764 0.00002180', + ], + add_opts = ['--add-columns=eurusd=x']) + + def ticker21(self): + return self.ticker( + ['5-5'], + [ + 'USD EURUSD=X BTC ' + r'5\) ADA 0.51 0.4764 0.00002180', + ], + add_opts = ['--add-columns=eurusd=x']) From c1f42fc25b123c334262e8cb53b460753e1c0be2 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:14 +0000 Subject: [PATCH 36/51] mmnode-ticker: various fixes and cleanups --- mmgen_node_tools/Ticker.py | 135 +++++++++++++++++--------------- mmgen_node_tools/main_ticker.py | 4 +- test/cmdtest_d/misc.py | 6 +- 3 files changed, 81 insertions(+), 64 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index bf3398b..7122e44 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -607,12 +607,10 @@ def make_cfg(gcfg_arg): def parse_asset_id(s, require_label=False): return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) - def get_rows_from_cfg(add_data=None): + def get_rows_from_cfg(): def gen(): - for n, (k, v) in enumerate(cfg_in.cfg['assets'].items()): + for k, v in cfg_in.cfg['assets'].items(): yield k - if add_data and k in add_data: - v += tuple(add_data[k]) for e in v: yield parse_asset_id(e, require_label=True) return tuple(gen()) @@ -650,7 +648,7 @@ def make_cfg(gcfg_arg): source = parsed_id.source) cl_opt = getattr(gcfg, key) - if (cl_opt or '').lower() in ('', 'none'): + if cl_opt is None or cl_opt.lower() in ('none', ''): return () cf_opt = cfg_in.cfg.get(key,[]) if use_cf_file else [] return tuple(parse_parm(s) for s in (cl_opt.split(',') if cl_opt else cf_opt)) @@ -702,15 +700,16 @@ def make_cfg(gcfg_arg): def get_usr_assets(): return ( - 'user_added', - usr_rows + - (tuple(asset for asset in query if asset) if query else ()) + - usr_columns) + usr_rows + + (tuple(asset for asset in query if asset) if query else ()) + + usr_columns) - def get_portfolio_assets(ret=()): + def get_portfolio_assets(): if cfg_in.portfolio and gcfg.portfolio: ret = (parse_asset_id(e, require_label=True) for e in cfg_in.portfolio) - return ('portfolio', tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC')) + return tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') + else: + return () def get_portfolio(): return {k: Decimal(v) for k, v in cfg_in.portfolio.items() @@ -730,14 +729,12 @@ def make_cfg(gcfg_arg): rows = ( ('trade_pair',) + query if (query and query.to_asset) else ('bitcoin', parse_asset_id('btc-bitcoin')) if gcfg.btc else - get_rows_from_cfg(add_data={'fiat':['usd-us-dollar']} if gcfg.add_columns else None)) - + get_rows_from_cfg()) for hdr, data in ( - (get_usr_assets(),) if query else - (get_usr_assets(), get_portfolio_assets())): + ('user_uniq', get_usr_assets()), + ('portfolio_uniq', get_portfolio_assets())): if data: - uniq_data = tuple(gen_uniq(data, 'symbol', preload=rows)) - if uniq_data: + if uniq_data := tuple(gen_uniq(data, 'symbol', preload=rows)): rows += (hdr,) + uniq_data return rows @@ -787,11 +784,6 @@ def make_cfg(gcfg_arg): cfg_in = get_cfg_in() - if gcfg.test_suite: # required for testing with overlay - from . import Ticker as this_mod - this_mod.src_cls = src_cls - this_mod.cfg_in = cfg_in - if cmd_args := gcfg._args: if len(cmd_args) > 1: die(1, 'Only one command-line argument is allowed') @@ -808,6 +800,13 @@ def make_cfg(gcfg_arg): proxy = None if proxy == 'none' else proxy proxy2 = get_proxy('proxy2') + portfolio = ( + get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query + else None) + + if portfolio and asset_range: + die(1, '--portfolio not supported in market cap view') + cfg = cfg_tuple( rows = create_rows(), usr_rows = usr_rows, @@ -821,9 +820,7 @@ def make_cfg(gcfg_arg): cachedir = get_cfg_var('cachedir') or dfl_cachedir, proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), - portfolio = - get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query - else None, + portfolio = portfolio, percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), asset_limit = get_cfg_var('asset_limit'), cached_data = get_cfg_var('cached_data'), @@ -835,6 +832,8 @@ def make_cfg(gcfg_arg): quiet = get_cfg_var('quiet'), verbose = get_cfg_var('verbose')) + return (src_cls, cfg_in) + def get_cfg_in(): ret = namedtuple('cfg_in_data', ['cfg', 'portfolio', 'cfg_file', 'portfolio_file']) cfg_file, portfolio_file = ( @@ -871,7 +870,7 @@ class Ticker: self.col1_wid = max(len('TOTAL'), ( max(len(self.create_label(d['id'])) for d in data.values()) if cfg.name_labels else - max(len(d['symbol']) for d in data.values()))) + 1 + max(len(d['symbol']) for d in data.values()))) self.rows = [row._replace(id=self.get_id(row)) if isinstance(row, tuple) else row for row in cfg.rows] @@ -881,7 +880,7 @@ class Ticker: for row in self.rows if isinstance(row, tuple) and row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') - def format_last_update_col(self, cross_assets=()): + def format_last_updated_col(self, cross_assets=()): if cfg.elapsed: from mmgen.util2 import format_elapsed_hr @@ -899,19 +898,20 @@ class Ticker: min_t = None for row in self.rows: - if isinstance(row, tuple): - try: - t = int(d[row.id]['last_updated']) - except TypeError as e: - d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) - except KeyError as e: - msg(str(e)) - pass - else: - t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( - (min(t, min_t) if min_t else t), - now = now) - max_w = max(len(t_fmt), max_w) + if not isinstance(row, tuple): + continue + try: + t = int(d[row.id]['last_updated']) + except TypeError as e: + d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) + except KeyError as e: + msg(str(e)) + pass + else: + t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( + (min(t, min_t) if min_t else t), + now = now) + max_w = max(len(t_fmt), max_w) self.upd_w = max_w @@ -924,8 +924,9 @@ class Ticker: if asset.id: return asset.id else: + m = asset.symbol for d in self.data.values(): - if d['symbol'] == asset.symbol: + if m == d['symbol']: return d['id'] def create_label(self, id): @@ -961,16 +962,26 @@ class Ticker: if self.table_hdr: yield self.table_hdr - for n, row in enumerate(self.rows, cfg.asset_range[0] if cfg.asset_range else 1): - if isinstance(row, str): - if cfg.asset_range: - return - yield ('-' * self.hl_wid) - else: + if cfg.asset_range: + yield '-' * self.hl_wid + for n, row in enumerate(self.rows, cfg.asset_range[0]): + if isinstance(row, str): + break try: yield self.fmt_row(self.data[row.id], idx=n) except KeyError: yield gray(f'(no data for {row.id})') + else: + for row in self.rows: + if isinstance(row, str): + if cfg.asset_range: + return + yield ('-' * self.hl_wid) + else: + try: + yield self.fmt_row(self.data[row.id]) + except KeyError: + yield gray(f'(no data for {row.id})') yield '-' * self.hl_wid @@ -1003,7 +1014,7 @@ class Ticker: super().__init__(data) - self.format_last_update_col() + self.format_last_updated_col() if cfg.portfolio: self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] @@ -1072,9 +1083,9 @@ class Ticker: ) for k in self.col_ids} cols = ( - ['label', 'usd-us-dollar'] + - [asset.id for asset in self.usr_col_assets] + - [a for a, b in ( + ['label', 'usd-us-dollar'] + + [asset.id for asset in self.usr_col_assets] + + [a for a, b in ( ('btc-bitcoin', not cfg.btc_only), ('pct1y', 'y' in cfg.percent_cols), ('pct1m', 'm' in cfg.percent_cols), @@ -1082,6 +1093,17 @@ class Ticker: ('pct1d', 'd' in cfg.percent_cols), ('update_time', cfg.update_time)) if b]) + + if cfg.asset_range: + def get_num_w(): + for n, r in enumerate(cfg.rows): + if isinstance(r, str): + return len(str(n)) + num_w = get_num_w() + col_fs_data.update({ + 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2)}) + cols = ['idx'] + cols + cols2 = list(cols) if cfg.update_time: cols2.pop() @@ -1095,15 +1117,6 @@ class Ticker: self.fs_num2 = ''.join(col_fs_data[c].fs_num for c in cols2) self.hl_wid2 = sum(col_fs_data[c].wid for c in cols2) - if cfg.asset_range: - def get_col1_w(): - for n, r in enumerate(cfg.rows): - if isinstance(r, str): - return len(str(n)) - col1_w = get_col1_w() - self.fs_str = ' ' * (col1_w + 2) + self.fs_str - self.fs_num = f'{{idx:{col1_w}}}) ' + self.fs_num - @property def table_hdr(self): return self.fs_str.format( @@ -1152,7 +1165,7 @@ class Ticker: for a in self.usr_col_assets: self.prices[a.id]['usd-us-dollar'] = data[a.id]['price_usd'] - self.format_last_update_col(cross_assets=self.usr_col_assets) + self.format_last_updated_col(cross_assets=self.usr_col_assets) self.init_prec() self.init_fs() diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 8efb97a..c6b41b3 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -244,9 +244,9 @@ from . import Ticker gcfg = Config(opts_data=opts_data, caller_post_init=True) -Ticker.make_cfg(gcfg) +src_cls, cfg_in = Ticker.make_cfg(gcfg) -from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen, cfg_in, src_cls +from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen gcfg._post_init() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index c2dceff..b64e33c 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -338,8 +338,10 @@ class CmdTestScripts(CmdTestBase): ['1-5'], [ 'USD EURUSD=X BTC ' + '--------------------------------------------- ' r'1\) BTC 23250.77 21848.7527 1.00000000', - r'5\) ADA 0.51 0.4764 0.00002180', + r'5\) ADA 0.51 0.4764 0.00002180' + ' ---------------------------------------------' ], add_opts = ['--add-columns=eurusd=x']) @@ -348,6 +350,7 @@ class CmdTestScripts(CmdTestBase): ['2-5'], [ 'USD EURUSD=X BTC ' + '--------------------------------------------- ' r'2\) ETH 1659.66 1559.5846 0.07138094', r'5\) ADA 0.51 0.4764 0.00002180', ], @@ -358,6 +361,7 @@ class CmdTestScripts(CmdTestBase): ['5-5'], [ 'USD EURUSD=X BTC ' + '--------------------------------------------- ' r'5\) ADA 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) From 3f921d333ccba378a6309123214e0a851e707ae8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:20 +0000 Subject: [PATCH 37/51] Ticker.py: `parse_asset_id()`: unify call signature --- mmgen_node_tools/Ticker.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 7122e44..76d9d2a 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -233,7 +233,7 @@ class DataSource: return [data] if cfg.btc_only else data @staticmethod - def parse_asset_id(s, require_label): + def parse_asset_id(s, require_label=True): sym, label = (*s.split('-', 1), None)[:2] if require_label and not label: die(1, f'{s!r}: asset label is missing') @@ -314,7 +314,7 @@ class DataSource: return ticker.price @staticmethod - def parse_asset_id(s, require_label): + def parse_asset_id(s, require_label=True): return asset_tuple( symbol = s.upper(), id = s.lower(), @@ -604,7 +604,7 @@ def make_cfg(gcfg_arg): query_tuple = namedtuple('query', ['asset', 'to_asset']) asset_data = namedtuple('asset_data', ['symbol', 'id', 'amount', 'rate', 'rate_asset', 'source']) - def parse_asset_id(s, require_label=False): + def parse_asset_id(s, require_label=True): return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) def get_rows_from_cfg(): @@ -634,7 +634,7 @@ def make_cfg(gcfg_arg): ss = s.split(':') assert len(ss) in (1, 2, 3), f'{s}: malformed argument' asset_id, rate, rate_asset = (*ss, None, None)[:3] - parsed_id = parse_asset_id(asset_id) + parsed_id = parse_asset_id(asset_id, require_label=False) return asset_data( symbol = parsed_id.symbol, @@ -644,7 +644,7 @@ def make_cfg(gcfg_arg): None if rate is None else 1 / Decimal(rate[:-1]) if rate.lower().endswith('r') else Decimal(rate)), - rate_asset = parse_asset_id(rate_asset) if rate_asset else None, + rate_asset = parse_asset_id(rate_asset, require_label=False) if rate_asset else None, source = parsed_id.source) cl_opt = getattr(gcfg, key) @@ -673,7 +673,7 @@ def make_cfg(gcfg_arg): asset_id:amount[:to_asset_id[:to_amount]] """ def parse_query_asset(asset_id, amount): - parsed_id = parse_asset_id(asset_id) + parsed_id = parse_asset_id(asset_id, require_label=False) return asset_data( symbol = parsed_id.symbol, id = parsed_id.id, @@ -706,7 +706,7 @@ def make_cfg(gcfg_arg): def get_portfolio_assets(): if cfg_in.portfolio and gcfg.portfolio: - ret = (parse_asset_id(e, require_label=True) for e in cfg_in.portfolio) + ret = (parse_asset_id(e) for e in cfg_in.portfolio) return tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') else: return () From 253aa14a265f66de3b97f826c0bef4d5ff3a369d Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:20 +0000 Subject: [PATCH 38/51] Ticker.py: new `RowDict` class --- mmgen_node_tools/Ticker.py | 86 +++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 48 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 76d9d2a..ed68eda 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -42,6 +42,11 @@ percent_cols = { 'y': 'year', } +class RowDict(dict): + + def __iter__(self): + return (e for v in self.values() for e in v) + class DataSource: source_groups = [ @@ -286,7 +291,7 @@ class DataSource: @property def symbols(self): - return [r.symbol for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi'] + return [r.symbol for r in cfg.rows if r.source == 'fi'] def get_data_from_network(self): @@ -456,8 +461,8 @@ def gen_data(data): return () rows_want = { - 'id': {r.id for r in cfg.rows if isinstance(r, tuple) and r.id} - {'usd-us-dollar'}, - 'symbol': {r.symbol for r in cfg.rows if isinstance(r, tuple) and r.id is None} - {'USD'}} + 'id': {r.id for r in cfg.rows if r.id} - {'usd-us-dollar'}, + 'symbol': {r.symbol for r in cfg.rows if r.id is None} - {'USD'}} usr_rate_assets = tuple(u.rate_asset for u in cfg.usr_rows + cfg.usr_columns if u.rate_asset) usr_rate_assets_want = { 'id': {a.id for a in usr_rate_assets if a.id}, @@ -580,13 +585,16 @@ def main(): global cfg if cfg.asset_range: - func = DataSource.coinpaprika.parse_asset_id n, m = cfg.asset_range - cfg = cfg._replace(rows = - tuple(func(e['id'], require_label=False) for e in src_data['cc'].data[n-1:m]) - + tuple(['-']) - + tuple([func('btc-bitcoin', require_label=True)]) - + tuple(r for r in cfg.rows if isinstance(r, tuple) and r.source == 'fi')) + cfg = cfg._replace(rows = RowDict({ + 'asset_list': + tuple( + asset_tuple(e['symbol'], e['id'], source='cc') + for e in src_data['cc'].data[n-1:m]), + 'extra': + tuple( + [asset_tuple('BTC', 'btc-bitcoin', source='cc')] + + [r for r in cfg.rows if r.source == 'fi'])})) global now now = 1659465400 if gcfg.test_suite else time.time() # 1659524400 1659445900 @@ -607,14 +615,6 @@ def make_cfg(gcfg_arg): def parse_asset_id(s, require_label=True): return src_cls['fi' if re.match(fi_pat, s) else 'cc'].parse_asset_id(s, require_label) - def get_rows_from_cfg(): - def gen(): - for k, v in cfg_in.cfg['assets'].items(): - yield k - for e in v: - yield parse_asset_id(e, require_label=True) - return tuple(gen()) - def parse_percent_cols(arg): if arg is None or arg.lower() in ('none', ''): return [] @@ -726,16 +726,18 @@ def make_cfg(gcfg_arg): return int(s) def create_rows(): - rows = ( - ('trade_pair',) + query if (query and query.to_asset) else - ('bitcoin', parse_asset_id('btc-bitcoin')) if gcfg.btc else - get_rows_from_cfg()) + rows = RowDict( + {'trade_pair': query} if (query and query.to_asset) else + {'bitcoin': [parse_asset_id('btc-bitcoin')]} if gcfg.btc else + {k: tuple(parse_asset_id(e) for e in v) for k, v in cfg_in.cfg['assets'].items()}) for hdr, data in ( ('user_uniq', get_usr_assets()), ('portfolio_uniq', get_portfolio_assets())): if data: if uniq_data := tuple(gen_uniq(data, 'symbol', preload=rows)): - rows += (hdr,) + uniq_data + rows[hdr] = uniq_data + else: + rows[hdr] = () return rows def get_cfg_var(name): @@ -872,12 +874,10 @@ class Ticker: max(len(self.create_label(d['id'])) for d in data.values()) if cfg.name_labels else max(len(d['symbol']) for d in data.values()))) - self.rows = [row._replace(id=self.get_id(row)) if isinstance(row, tuple) else row - for row in cfg.rows] + self.rows = RowDict( + {k: tuple(row._replace(id=self.get_id(row)) for row in v) for k, v in cfg.rows.items()}) self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} - - self.prices = {row.id: self.get_row_prices(row.id) - for row in self.rows if isinstance(row, tuple) and row.id in data} + self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') def format_last_updated_col(self, cross_assets=()): @@ -898,8 +898,6 @@ class Ticker: min_t = None for row in self.rows: - if not isinstance(row, tuple): - continue try: t = int(d[row.id]['last_updated']) except TypeError as e: @@ -964,24 +962,20 @@ class Ticker: if cfg.asset_range: yield '-' * self.hl_wid - for n, row in enumerate(self.rows, cfg.asset_range[0]): - if isinstance(row, str): - break + for n, row in enumerate(self.rows['asset_list'], cfg.asset_range[0]): try: yield self.fmt_row(self.data[row.id], idx=n) except KeyError: yield gray(f'(no data for {row.id})') else: - for row in self.rows: - if isinstance(row, str): - if cfg.asset_range: - return - yield ('-' * self.hl_wid) - else: - try: - yield self.fmt_row(self.data[row.id]) - except KeyError: - yield gray(f'(no data for {row.id})') + for rows in self.rows.values(): + if rows: + yield '-' * self.hl_wid + for row in rows: + try: + yield self.fmt_row(self.data[row.id]) + except KeyError: + yield gray(f'(no data for {row.id})') yield '-' * self.hl_wid @@ -1019,7 +1013,7 @@ class Ticker: if cfg.portfolio: self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] for row in self.rows - if isinstance(row, tuple) and row.id in cfg.portfolio and row.id in data) + if row.id in cfg.portfolio and row.id in data) for col_id in self.col_ids} self.init_prec() @@ -1095,11 +1089,7 @@ class Ticker: if b]) if cfg.asset_range: - def get_num_w(): - for n, r in enumerate(cfg.rows): - if isinstance(r, str): - return len(str(n)) - num_w = get_num_w() + num_w = len(str(len(cfg.rows['asset_list']))) col_fs_data.update({ 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2)}) cols = ['idx'] + cols From 9aa4b4dcfef0d95e96513042e42cc86c6bc997ba Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 15 Oct 2025 10:14:20 +0000 Subject: [PATCH 39/51] mmnode-ticker: add `MarketCap` column Column is enabled automatically when script is invoked in market cap mode. --- mmgen_node_tools/Ticker.py | 8 ++++++-- mmgen_node_tools/data/version | 2 +- test/cmdtest_d/misc.py | 28 ++++++++++++++-------------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index ed68eda..9e598de 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -416,6 +416,7 @@ def gen_data(data): d['percent_change_7d'] = d['quotes']['USD']['percent_change_7d'] d['percent_change_30d'] = d['quotes']['USD']['percent_change_30d'] d['percent_change_1y'] = d['quotes']['USD']['percent_change_1y'] + d['market_cap'] = d['quotes']['USD']['market_cap'] d['last_updated'] = int(datetime.datetime.fromisoformat( d['last_updated']).timestamp()) yield (d['id'], d) @@ -1041,6 +1042,7 @@ class Ticker: return self.fs_num.format( idx = idx, + mcap = d.get('market_cap') / 1_000_000_000 if cfg.asset_range else None, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], pc1 = fmt_pct(d.get('percent_change_7d')), pc2 = fmt_pct(d.get('percent_change_24h')), @@ -1091,8 +1093,9 @@ class Ticker: if cfg.asset_range: num_w = len(str(len(cfg.rows['asset_list']))) col_fs_data.update({ - 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2)}) - cols = ['idx'] + cols + 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2), + 'mcap': fd('{mcap:>12}', '{mcap:12.5f}', 12)}) + cols = ['idx', 'label', 'mcap'] + cols[1:] cols2 = list(cols) if cfg.update_time: @@ -1111,6 +1114,7 @@ class Ticker: def table_hdr(self): return self.fs_str.format( lbl = '', + mcap = 'MarketCap(B)', pc1 = ' CHG_7d', pc2 = 'CHG_24h', pc3 = 'CHG_1y', diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 636c831..921f776 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev5 +3.6.dev6 diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index b64e33c..a8d259e 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -328,8 +328,8 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['10'], [ - r'1\) BITCOIN 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', - r'6\) ALGORAND 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' + r'1\) BITCOIN 444.33652 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', + r'6\) ALGORAND 2.30691 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' ], add_opts = ['--widest', '--add-columns=eurusd=x']) @@ -337,11 +337,11 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['1-5'], [ - 'USD EURUSD=X BTC ' - '--------------------------------------------- ' - r'1\) BTC 23250.77 21848.7527 1.00000000', - r'5\) ADA 0.51 0.4764 0.00002180' - ' ---------------------------------------------' + r'MarketCap\(B\) USD EURUSD=X BTC ' + '--------------------------------------------------------- ' + r'1\) BTC 444.33652 23250.77 21848.7527 1.00000000', + r'5\) ADA 17.11161 0.51 0.4764 0.00002180' + ' ---------------------------------------------------------' ], add_opts = ['--add-columns=eurusd=x']) @@ -349,10 +349,10 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['2-5'], [ - 'USD EURUSD=X BTC ' - '--------------------------------------------- ' - r'2\) ETH 1659.66 1559.5846 0.07138094', - r'5\) ADA 0.51 0.4764 0.00002180', + r'MarketCap\(B\) USD EURUSD=X BTC ' + '--------------------------------------------------------- ' + r'2\) ETH 202.15129 1659.66 1559.5846 0.07138094', + r'5\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) @@ -360,8 +360,8 @@ class CmdTestScripts(CmdTestBase): return self.ticker( ['5-5'], [ - 'USD EURUSD=X BTC ' - '--------------------------------------------- ' - r'5\) ADA 0.51 0.4764 0.00002180', + r'MarketCap\(B\) USD EURUSD=X BTC ' + '--------------------------------------------------------- ' + r'5\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) From e71ef141bffd8f44998e1510edf197f423cb2e3b Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Oct 2025 17:09:09 +0000 Subject: [PATCH 40/51] mmnode-ticker: minor cleanups --- mmgen_node_tools/Ticker.py | 20 ++++++++++---------- mmgen_node_tools/main_ticker.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 9e598de..6aef36f 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -24,7 +24,7 @@ from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.color import red, yellow, green, blue, orange, gray +from mmgen.color import red, yellow, green, blue, orange, gray, cyan from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf, is_int from mmgen.ui import do_pager @@ -39,8 +39,7 @@ percent_cols = { 'd': 'day', 'w': 'week', 'm': 'month', - 'y': 'year', -} + 'y': 'year'} class RowDict(dict): @@ -713,8 +712,8 @@ def make_cfg(gcfg_arg): return () def get_portfolio(): - return {k: Decimal(v) for k, v in cfg_in.portfolio.items() - if (not gcfg.btc) or k == 'btc-bitcoin'} + return tuple((k, Decimal(v)) for k, v in cfg_in.portfolio.items() + if (not gcfg.btc) or k == 'btc-bitcoin') def parse_add_precision(arg): if not arg: @@ -932,7 +931,7 @@ class Ticker: return self.data[id]['name'].upper() def gen_output(self): - yield 'Current time: {} UTC'.format(time.strftime('%F %X', time.gmtime(now))) + yield 'Current time: {}'.format(cyan(time.strftime('%F %X', time.gmtime(now)) + ' UTC')) for asset in self.usr_col_assets: if asset.symbol != 'USD': @@ -987,7 +986,7 @@ class Ticker: yield blue('PORTFOLIO') yield self.table_hdr yield '-' * self.hl_wid - for sym, amt in cfg.portfolio.items(): + for sym, amt in cfg.portfolio: try: yield self.fmt_row(self.data[sym], amt=amt) except KeyError: @@ -1005,16 +1004,17 @@ class Ticker: self.adjust = cfg.adjust self.show_adj = self.adjust != 1 self.usr_col_assets = [asset._replace(id=self.get_id(asset)) for asset in cfg.usr_columns] - self.col_ids = ('usd-us-dollar',) + tuple(a.id for a in self.usr_col_assets) + ('btc-bitcoin',) + self.col_ids = ('usd-us-dollar', 'btc-bitcoin') + tuple(a.id for a in self.usr_col_assets) super().__init__(data) self.format_last_updated_col() if cfg.portfolio: - self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] + pf_dict = dict(cfg.portfolio) + self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * pf_dict[row.id] for row in self.rows - if row.id in cfg.portfolio and row.id in data) + if row.id in pf_dict and row.id in data) for col_id in self.col_ids} self.init_prec() diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index c6b41b3..e533e62 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -225,7 +225,7 @@ To add a portfolio, edit the file dfl_cachedir = os.path.relpath(dfl_cachedir, start=homedir), ds = fmt_dict(DataSource.get_sources(), fmt='equal_compact'), al = DataSource.coinpaprika.dfl_asset_limit, - pc = fmt_list(Ticker.percent_cols, fmt='bare')), + pc = fmt_list(Ticker.percent_cols, fmt='fancy')), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in), fmt='col', indent=' '), cfg = os.path.relpath(cfg_in.cfg_file, start=homedir), From e6d62fd18b2d9e5885ce4452c843850871550f60 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Oct 2025 17:09:15 +0000 Subject: [PATCH 41/51] mmnode-ticker: display coin ranking in first column --- mmgen_node_tools/Ticker.py | 40 +++++++++++++++++++++++--------------- test/cmdtest_d/misc.py | 14 ++++++------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 6aef36f..a8b6595 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -865,6 +865,7 @@ class Ticker: offer = None to_asset = None + hidden_groups = ('extra',) def __init__(self, data): @@ -876,6 +877,14 @@ class Ticker: self.rows = RowDict( {k: tuple(row._replace(id=self.get_id(row)) for row in v) for k, v in cfg.rows.items()}) + + if cfg.asset_range: + self.max_rank = 0 + for group, rows in self.rows.items(): + if group not in self.hidden_groups: + for row in rows: + self.max_rank = max(self.max_rank, int(data[row.id]['rank'])) + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') @@ -931,6 +940,15 @@ class Ticker: return self.data[id]['name'].upper() def gen_output(self): + + def process_rows(rows): + yield '-' * self.hl_wid + for row in rows: + try: + yield self.fmt_row(self.data[row.id]) + except KeyError: + yield gray(f'(no data for {row.id})') + yield 'Current time: {}'.format(cyan(time.strftime('%F %X', time.gmtime(now)) + ' UTC')) for asset in self.usr_col_assets: @@ -961,21 +979,11 @@ class Ticker: yield self.table_hdr if cfg.asset_range: - yield '-' * self.hl_wid - for n, row in enumerate(self.rows['asset_list'], cfg.asset_range[0]): - try: - yield self.fmt_row(self.data[row.id], idx=n) - except KeyError: - yield gray(f'(no data for {row.id})') + yield from process_rows(self.rows['asset_list']) else: for rows in self.rows.values(): if rows: - yield '-' * self.hl_wid - for row in rows: - try: - yield self.fmt_row(self.data[row.id]) - except KeyError: - yield gray(f'(no data for {row.id})') + yield from process_rows(rows) yield '-' * self.hl_wid @@ -1028,7 +1036,7 @@ class Ticker: d['price_usd'] / self.col_usd_prices[k] ) * self.adjust for k in self.col_ids} - def fmt_row(self, d, amt=None, amt_fmt=None, idx=None): + def fmt_row(self, d, amt=None, amt_fmt=None): def fmt_pct(n): return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') @@ -1041,7 +1049,7 @@ class Ticker: amt_fmt = amt_fmt.rstrip('0').rstrip('.') return self.fs_num.format( - idx = idx, + idx = int(d['rank']) if cfg.asset_range else None, mcap = d.get('market_cap') / 1_000_000_000 if cfg.asset_range else None, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], pc1 = fmt_pct(d.get('percent_change_7d')), @@ -1091,7 +1099,7 @@ class Ticker: if b]) if cfg.asset_range: - num_w = len(str(len(cfg.rows['asset_list']))) + num_w = len(str(self.max_rank)) col_fs_data.update({ 'idx': fd(' ' * (num_w + 2), f'{{idx:{num_w}}}) ', num_w + 2), 'mcap': fd('{mcap:>12}', '{mcap:12.5f}', 12)}) @@ -1186,7 +1194,7 @@ class Ticker: self.fs_str += ' {upd}' self.hl_wid += self.upd_w + 2 - def fmt_row(self, d, idx=None): + def fmt_row(self, d): id = d['id'] p = self.prices[id][self.asset.id] * self.asset.amount p_spot = '{:{}{}.{}f}'.format(p, self.max_wid, self.comma, 8+cfg.add_prec) diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index a8d259e..3e909f4 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -329,7 +329,7 @@ class CmdTestScripts(CmdTestBase): ['10'], [ r'1\) BITCOIN 444.33652 23,250.77 21,848.7527 1.00000000 \+18.96 \+15.61 \+11.15 \+0.89', - r'6\) ALGORAND 2.30691 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' + r'33\) ALGORAND 2.30691 0.33 0.3120 0.00001428 \+16.47 \+13.57 \+9.69 \-0.82' ], add_opts = ['--widest', '--add-columns=eurusd=x']) @@ -338,10 +338,10 @@ class CmdTestScripts(CmdTestBase): ['1-5'], [ r'MarketCap\(B\) USD EURUSD=X BTC ' - '--------------------------------------------------------- ' + '---------------------------------------------------------- ' r'1\) BTC 444.33652 23250.77 21848.7527 1.00000000', - r'5\) ADA 17.11161 0.51 0.4764 0.00002180' - ' ---------------------------------------------------------' + r'8\) ADA 17.11161 0.51 0.4764 0.00002180' + ' ----------------------------------------------------------' ], add_opts = ['--add-columns=eurusd=x']) @@ -350,9 +350,9 @@ class CmdTestScripts(CmdTestBase): ['2-5'], [ r'MarketCap\(B\) USD EURUSD=X BTC ' - '--------------------------------------------------------- ' + '---------------------------------------------------------- ' r'2\) ETH 202.15129 1659.66 1559.5846 0.07138094', - r'5\) ADA 17.11161 0.51 0.4764 0.00002180', + r'8\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) @@ -362,6 +362,6 @@ class CmdTestScripts(CmdTestBase): [ r'MarketCap\(B\) USD EURUSD=X BTC ' '--------------------------------------------------------- ' - r'5\) ADA 17.11161 0.51 0.4764 0.00002180', + r'8\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) From 0a953e3ca039040b9ca4dbff501913bc39b760bb Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Thu, 16 Oct 2025 17:09:15 +0000 Subject: [PATCH 42/51] mmnode-ticker: sort output by various parameters Supported parameters: d - 1-day % change w - 1-week % change m - 1-month % change y - 1-year % change p - asset price c - market cap Examples: # Display top 50 assets by market cap, sorting by price change # in last 24 hours: $ mmnode-ticker --sort=d 50 --- mmgen_node_tools/Ticker.py | 45 ++++++++++++++++++++++- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 11 +++++- test/cmdtest_d/misc.py | 63 +++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 3 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index a8b6595..d3045d4 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -24,7 +24,7 @@ from subprocess import run, PIPE, CalledProcessError from decimal import Decimal from collections import namedtuple -from mmgen.color import red, yellow, green, blue, orange, gray, cyan +from mmgen.color import red, yellow, green, blue, orange, gray, cyan, pink from mmgen.util import msg, msg_r, rmsg, Msg, Msg_r, die, fmt, fmt_list, fmt_dict, list_gen, suf, is_int from mmgen.ui import do_pager @@ -41,6 +41,15 @@ percent_cols = { 'm': 'month', 'y': 'year'} +sp = namedtuple('sort_parameter', ['key', 'desc']) +sort_params = { + 'd': sp('percent_change_24h', '1-day % change'), + 'w': sp('percent_change_7d', '1-week % change'), + 'm': sp('percent_change_30d', '1-month % change'), + 'y': sp('percent_change_1y', '1-year % change'), + 'p': sp('price_usd', 'asset price'), + 'c': sp('market_cap', 'market cap')} + class RowDict(dict): def __iter__(self): @@ -274,6 +283,7 @@ class DataSource: 'percent_change_30d': data['pct_chg_4wks'], 'percent_change_7d': data['pct_chg_1wk'], 'percent_change_24h': data['regularMarketChangePercent']['raw'] * 100, + 'market_cap': 0, # dummy - required for sorting 'last_updated': data['regularMarketTime']} def rate_limit_errmsg(self, elapsed): @@ -752,6 +762,19 @@ def make_cfg(gcfg_arg): '' if proxy == '' else 'none' if (proxy and proxy.lower() == 'none') else (proxy or cfg_in.cfg.get(name))) + def get_sort_opt(): + match get_cfg_var('sort'): + case None: + return None + case s if s in sort_params: + return (s, True) + case s if s in ['r' + ch for ch in sort_params]: + return (s[1], False) + case s: + die(1, + f'{s!r}: invalid parameter for --sort option (must be one of {fmt_list(sort_params)})' + '\nTo reverse the sort, prefix the code letter with ‘r’') + cfg_tuple = namedtuple('global_cfg',[ 'rows', 'usr_rows', @@ -766,6 +789,7 @@ def make_cfg(gcfg_arg): 'proxy', 'proxy2', 'portfolio', + 'sort', 'percent_cols', 'asset_limit', 'cached_data', @@ -823,6 +847,7 @@ def make_cfg(gcfg_arg): proxy = proxy, proxy2 = None if proxy2 == 'none' else '' if proxy2 == '' else (proxy2 or proxy), portfolio = portfolio, + sort = get_sort_opt(), percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), asset_limit = get_cfg_var('asset_limit'), cached_data = get_cfg_var('cached_data'), @@ -869,6 +894,8 @@ class Ticker: def __init__(self, data): + global cfg + self.comma = ',' if cfg.thousands_comma else '' self.col1_wid = max(len('TOTAL'), ( @@ -885,6 +912,18 @@ class Ticker: for row in rows: self.max_rank = max(self.max_rank, int(data[row.id]['rank'])) + if cfg.sort: + code, reverse = cfg.sort + key = sort_params[code].key + sort_func = lambda row: data[row.id][key] + pf_sort_func = lambda row: data[row[0]][key] + for group in self.rows.keys(): + if group not in self.hidden_groups: + self.rows[group] = sorted(self.rows[group], key=sort_func, reverse=reverse) + if cfg.portfolio: + cfg = cfg._replace( + portfolio = sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)) + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') @@ -951,6 +990,10 @@ class Ticker: yield 'Current time: {}'.format(cyan(time.strftime('%F %X', time.gmtime(now)) + ' UTC')) + if cfg.sort: + text = sort_params[cfg.sort[0]].desc + ('' if cfg.sort[1] else ' [reversed]') + yield f'Sort order: {pink(text.upper())}' + for asset in self.usr_col_assets: if asset.symbol != 'USD': usdprice = self.data[asset.id]['price_usd'] diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 921f776..1de14a5 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev6 +3.6.dev7 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index e533e62..7be5e90 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -57,6 +57,9 @@ opts_data = { -r, --add-rows=LIST Add rows for asset specifiers in LIST (comma-separated, see ASSET SPECIFIERS below). Can also be used to supply a USD exchange rate for missing assets. +-s, --sort=P Sort output according to parameter P. Valid parameters + are {sp_codes}. See SORT PARAMETERS below. + To reverse the sort, prefix the parameter with ‘r’. -t, --testing Print command(s) to be executed to stdout and exit -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column @@ -133,6 +136,10 @@ A TRADE_SPECIFIER is a single argument in the format: a USD rate for the missing asset(s) must be supplied via the --add-columns or --add-rows options. +SORT PARAMETERS: + + {sp_fmt} + PROXY NOTE @@ -225,6 +232,7 @@ To add a portfolio, edit the file dfl_cachedir = os.path.relpath(dfl_cachedir, start=homedir), ds = fmt_dict(DataSource.get_sources(), fmt='equal_compact'), al = DataSource.coinpaprika.dfl_asset_limit, + sp_codes = fmt_list(sort_params, fmt='fancy'), pc = fmt_list(Ticker.percent_cols, fmt='fancy')), 'notes': lambda s: s.format( assets = fmt_list(assets_list_gen(cfg_in), fmt='col', indent=' '), @@ -232,6 +240,7 @@ To add a portfolio, edit the file pf_cfg = os.path.relpath(cfg_in.portfolio_file, start=homedir), al = DataSource.coinpaprika.dfl_asset_limit, cc = src_cls['cc'](), + sp_fmt = '\n '.join(f'‘{k}’ - {v.desc}' for k, v in sort_params.items()), fi = src_cls['fi']()) } } @@ -246,7 +255,7 @@ gcfg = Config(opts_data=opts_data, caller_post_init=True) src_cls, cfg_in = Ticker.make_cfg(gcfg) -from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen +from .Ticker import dfl_cachedir, homedir, DataSource, assets_list_gen, sort_params gcfg._post_init() diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 3e909f4..275117f 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -90,6 +90,14 @@ class CmdTestScripts(CmdTestBase): ('ticker19', 'ticker [--cached-data 1-5]'), ('ticker20', 'ticker [--cached-data 2-5]'), ('ticker21', 'ticker [--cached-data 5-5]'), + ('ticker22', 'ticker [--sort=rp]'), + ('ticker23', 'ticker [--sort=rp xmr:10]'), + ('ticker24', 'ticker [--sort=p]'), + ('ticker25', 'ticker [--sort=p 200]'), + ('ticker26', 'ticker [--sort=c -r algo,ada]'), + ('ticker27', 'ticker [--sort=rp -r algo,ada]'), + ('ticker28', 'ticker [--sort=d -r algo,ada]'), + ('ticker29', 'ticker [--sort=y -r algo,ada]'), ) } @@ -365,3 +373,58 @@ class CmdTestScripts(CmdTestBase): r'8\) ADA 17.11161 0.51 0.4764 0.00002180', ], add_opts = ['--add-columns=eurusd=x']) + + def ticker22(self): + return self.ticker( + [], + ['MONERO', 'ETHEREUM', 'BITCOIN', 'SILVER', 'BRENT', 'GOLD'], + add_opts = ['--name-labels', '--sort=rp']) + + def ticker23(self): + return self.ticker( + [], + ['MONERO', 'ETHEREUM', 'BITCOIN', 'SILVER', 'BRENT', 'GOLD'], + add_opts = ['--name-labels', '--sort=rp', 'xmr:10']) + + def ticker24(self): + return self.ticker( + [], + ['BITCOIN', 'ETHEREUM', 'MONERO', 'GOLD', 'BRENT', 'SILVER'], + add_opts = ['--name-labels', '--sort=p']) + + def ticker25(self): + return self.ticker( + [], + [ + r' 1\) BITCOIN', + r' 2\) ETHEREUM', + r'30\) MONERO', + r'23\) LITECOIN', + r' 8\) CARDANO', + r'33\) ALGORAND' + ], + add_opts = ['--name-labels', '--sort=p', '200']) + + def ticker26(self): + return self.ticker( + [], + ['BITCOIN', 'ETHEREUM', 'MONERO', 'CARDANO', 'ALGORAND'], + add_opts = ['--name-labels', '--sort=c', '-r', 'ada,algo']) + + def ticker27(self): + return self.ticker( + [], + ['MONERO', 'ETHEREUM', 'BITCOIN', 'S&P', 'NASDAQ', 'DOW', 'ALGORAND', 'CARDANO'], + add_opts = ['--name-labels', '--sort=rp', '--add-rows=ada-cardano,algo-algorand']) + + def ticker28(self): + return self.ticker( + [], + ['ETHEREUM', 'MONERO', 'BITCOIN', 'NASDAQ', 'S&P', 'DOW', 'CARDANO', 'ALGORAND'], + add_opts = ['--widest', '--sort=d', '-r', 'ada,algo']) + + def ticker29(self): + return self.ticker( + [], + ['ETHEREUM', 'BITCOIN', 'MONERO', 'S&P', 'DOW', 'NASDAQ', 'CARDANO', 'ALGORAND'], + add_opts = ['--widest', '-s', 'y', '-r', 'ada,algo']) From 13234e990bc70b2cd53b78faaed121fc584f8cd8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 17 Oct 2025 09:07:55 +0000 Subject: [PATCH 43/51] cmdtest.py misc.ticker: cleanups --- test/cmdtest_d/misc.py | 43 +++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 275117f..9060731 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -67,7 +67,7 @@ class CmdTestScripts(CmdTestBase): 'ticker': ( "'mmnode-ticker' script", ('ticker1', 'ticker [--help]'), - ('copy_files', 'copying JSON files to cache'), + ('copy_cache_files', 'copying JSON files to cache'), ('ticker1a', 'ticker [--download=cc] (early caching)'), ('ticker1b', 'ticker [--download=cc] (late caching)'), ('ticker2', 'ticker (bad proxy)'), @@ -106,17 +106,24 @@ class CmdTestScripts(CmdTestBase): return self.ticker_server = TickerServer(cfg) self.ticker_server.start() + self.dests = { + 'nt_datadir': os.path.join(cfg.data_dir_root, 'node_tools'), + 'cache': self.tmpdir} return super().__init__(cfg, trunner, cfgs, spawn) - @property - def nt_datadir(self): - return os.path.join( cfg.data_dir_root, 'node_tools' ) + def rm_file(self, fn, dest='nt_datadir'): + os.unlink(os.path.join(self.dests[dest], fn)) - def copy_files(self): - self.spawn('',msg_only=True) - shutil.copy2(os.path.join(refdir,'ticker-finance.json'),self.tmpdir) - shutil.copy2(os.path.join(refdir,'ticker-finance-history.json'),self.tmpdir) - shutil.copy2(os.path.join(refdir,'ticker-btc.json'),self.tmpdir) + def copy_file(self, src_fn, dest_fn=None, dest='nt_datadir'): + shutil.copy2( + os.path.join(refdir, src_fn), + os.path.join(self.dests[dest], dest_fn or src_fn)) + + def copy_cache_files(self): + self.spawn('', msg_only=True) + self.copy_file('ticker-finance.json', dest='cache') + self.copy_file('ticker-finance-history.json', dest='cache') + self.copy_file('ticker-btc.json', dest='cache') return 'ok' def ticker( @@ -186,7 +193,7 @@ class CmdTestScripts(CmdTestBase): ]) def ticker5(self): - shutil.copy2(os.path.join(refdir,'ticker-cfg.yaml'),self.nt_datadir) + self.copy_file('ticker-cfg.yaml') t = self.ticker( ['--wide','--adjust=-0.766'], [ @@ -195,7 +202,7 @@ class CmdTestScripts(CmdTestBase): r'LITECOIN 58.56 0.00251869 \+12.79 \+0.40 2022-08-02 18:25:59', r'MONERO 157.76 0.00678495 \+7.28 \+1.21' ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-cfg.yaml')) + self.rm_file('ticker-cfg.yaml') return t def ticker6(self): @@ -204,7 +211,7 @@ class CmdTestScripts(CmdTestBase): return t def ticker7(self): # demo - shutil.copy2(os.path.join(refdir,'ticker-portfolio.yaml'),self.nt_datadir) + self.copy_file('ticker-portfolio.yaml') t = self.ticker( ['--wide','--portfolio'], [ @@ -213,7 +220,7 @@ class CmdTestScripts(CmdTestBase): 'CARDANO','ALGORAND', 'PORTFOLIO','BITCOIN','ETHEREUM','MONERO','CARDANO','ALGORAND','TOTAL' ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + self.rm_file('ticker-portfolio.yaml') return t def ticker8(self): @@ -225,9 +232,7 @@ class CmdTestScripts(CmdTestBase): ]) def ticker9(self): - shutil.copy2( - os.path.join(refdir,'ticker-portfolio-bad.yaml'), - os.path.join(self.nt_datadir,'ticker-portfolio.yaml') ) + self.copy_file('ticker-portfolio-bad.yaml', 'ticker-portfolio.yaml') t = self.ticker( ['--wide','--portfolio','--elapsed','--add-rows=fake-fakecoin:0.0123','--add-precision=2'], [ @@ -236,7 +241,7 @@ class CmdTestScripts(CmdTestBase): r'FAKECOIN 81.3008 0.0034966927 -- -- --', r'\(no data for noc-nocoin\)', ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + self.rm_file('ticker-portfolio.yaml') return t def ticker10(self): @@ -287,7 +292,7 @@ class CmdTestScripts(CmdTestBase): ]) def ticker14(self): - shutil.copy2(os.path.join(refdir,'ticker-portfolio.yaml'),self.nt_datadir) + self.copy_file('ticker-portfolio.yaml') t = self.ticker( ['--btc','--wide','--portfolio','--elapsed'], [ @@ -296,7 +301,7 @@ class CmdTestScripts(CmdTestBase): 'PORTFOLIO', r'BITCOIN 28,850.44 \+6.05 -1.87 1.23456789' ]) - os.unlink(os.path.join(self.nt_datadir,'ticker-portfolio.yaml')) + self.rm_file('ticker-portfolio.yaml') return t def ticker15(self): From 8f9c4ba48ccf695b48d718c6d95a8b717ef85a4c Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Fri, 17 Oct 2025 09:08:00 +0000 Subject: [PATCH 44/51] mmnode-ticker --sort: handle missing entries in source data --- mmgen_node_tools/Ticker.py | 19 ++++++++++--------- mmgen_node_tools/data/version | 2 +- test/cmdtest_d/misc.py | 5 ++++- test/ref/ticker/ticker-cfg-bad.yaml | 11 +++++++++++ 4 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 test/ref/ticker/ticker-cfg-bad.yaml diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index d3045d4..925a4f9 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -41,14 +41,14 @@ percent_cols = { 'm': 'month', 'y': 'year'} -sp = namedtuple('sort_parameter', ['key', 'desc']) +sp = namedtuple('sort_parameter', ['key', 'sort_dfl', 'desc']) sort_params = { - 'd': sp('percent_change_24h', '1-day % change'), - 'w': sp('percent_change_7d', '1-week % change'), - 'm': sp('percent_change_30d', '1-month % change'), - 'y': sp('percent_change_1y', '1-year % change'), - 'p': sp('price_usd', 'asset price'), - 'c': sp('market_cap', 'market cap')} + 'd': sp('percent_change_24h', 0.0, '1-day % change'), + 'w': sp('percent_change_7d', 0.0, '1-week % change'), + 'm': sp('percent_change_30d', 0.0, '1-month % change'), + 'y': sp('percent_change_1y', 0.0, '1-year % change'), + 'p': sp('price_usd', Decimal(0), 'asset price'), + 'c': sp('market_cap', 0, 'market cap')} class RowDict(dict): @@ -915,8 +915,9 @@ class Ticker: if cfg.sort: code, reverse = cfg.sort key = sort_params[code].key - sort_func = lambda row: data[row.id][key] - pf_sort_func = lambda row: data[row[0]][key] + sort_dfl = sort_params[code].sort_dfl + sort_func = lambda row: data.get(row.id, {key: sort_dfl})[key] + pf_sort_func = lambda row: data.get(row[0], {key: sort_dfl})[key] for group in self.rows.keys(): if group not in self.hidden_groups: self.rows[group] = sorted(self.rows[group], key=sort_func, reverse=reverse) diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 1de14a5..6eb528d 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev7 +3.6.dev8 diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 9060731..a3edd34 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -380,10 +380,13 @@ class CmdTestScripts(CmdTestBase): add_opts = ['--add-columns=eurusd=x']) def ticker22(self): - return self.ticker( + self.copy_file('ticker-cfg-bad.yaml', 'ticker-cfg.yaml') + t = self.ticker( [], ['MONERO', 'ETHEREUM', 'BITCOIN', 'SILVER', 'BRENT', 'GOLD'], add_opts = ['--name-labels', '--sort=rp']) + self.rm_file('ticker-cfg.yaml') + return t def ticker23(self): return self.ticker( diff --git a/test/ref/ticker/ticker-cfg-bad.yaml b/test/ref/ticker/ticker-cfg-bad.yaml new file mode 100644 index 0000000..da670c6 --- /dev/null +++ b/test/ref/ticker/ticker-cfg-bad.yaml @@ -0,0 +1,11 @@ +assets: + coin1: + - btc-bitcoin + - ltc-litecoin + - eth-ethereum + - xmr-monero + - bad-badcoin + commodity: + - gc=f + - si=f + - bz=f From 2647fa1fe3d511bec6ca841101c8e1472f479551 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 20 Oct 2025 09:14:30 +0000 Subject: [PATCH 45/51] mmnode-ticker: minor cleanups --- mmgen_node_tools/Ticker.py | 30 ++++++++++++++++-------------- mmgen_node_tools/main_ticker.py | 4 ++-- test/cmdtest_d/misc.py | 2 ++ 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 925a4f9..7bcb567 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -43,10 +43,10 @@ percent_cols = { sp = namedtuple('sort_parameter', ['key', 'sort_dfl', 'desc']) sort_params = { - 'd': sp('percent_change_24h', 0.0, '1-day % change'), - 'w': sp('percent_change_7d', 0.0, '1-week % change'), - 'm': sp('percent_change_30d', 0.0, '1-month % change'), - 'y': sp('percent_change_1y', 0.0, '1-year % change'), + 'd': sp('percent_change_24h', 0.0, '1-day percent change'), + 'w': sp('percent_change_7d', 0.0, '1-week percent change'), + 'm': sp('percent_change_30d', 0.0, '1-month percent change'), + 'y': sp('percent_change_1y', 0.0, '1-year percent change'), 'p': sp('price_usd', Decimal(0), 'asset price'), 'c': sp('market_cap', 0, 'market cap')} @@ -1025,8 +1025,8 @@ class Ticker: if cfg.asset_range: yield from process_rows(self.rows['asset_list']) else: - for rows in self.rows.values(): - if rows: + for group, rows in self.rows.items(): + if rows and group not in self.hidden_groups: yield from process_rows(rows) yield '-' * self.hl_wid @@ -1082,8 +1082,10 @@ class Ticker: def fmt_row(self, d, amt=None, amt_fmt=None): - def fmt_pct(n): - return gray(' --') if n is None else (red, green)[n>=0](f'{n:+7.2f}') + def fmt_pct(d, key, wid=7): + if (n := d.get(key)) is None: + return gray(' --') + return (red, green)[n>=0](f'{n:+{wid}.2f}') p = self.prices[d['id']] @@ -1096,10 +1098,10 @@ class Ticker: idx = int(d['rank']) if cfg.asset_range else None, mcap = d.get('market_cap') / 1_000_000_000 if cfg.asset_range else None, lbl = self.create_label(d['id']) if cfg.name_labels else d['symbol'], - pc1 = fmt_pct(d.get('percent_change_7d')), - pc2 = fmt_pct(d.get('percent_change_24h')), - pc3 = fmt_pct(d.get('percent_change_1y')), - pc4 = fmt_pct(d.get('percent_change_30d')), + pc1 = fmt_pct(d, 'percent_change_7d'), + pc2 = fmt_pct(d, 'percent_change_24h'), + pc3 = fmt_pct(d, 'percent_change_1y', wid=8), + pc4 = fmt_pct(d, 'percent_change_30d'), upd = d.get('last_updated_fmt'), amt = amt_fmt, **{k.replace('-', '_'): v * (1 if amt is None else amt) for k, v in p.items()}) @@ -1117,7 +1119,7 @@ class Ticker: col_fs_data = { 'label': fd(f'{{lbl:{self.col1_wid}}}', f'{{lbl:{self.col1_wid}}}', self.col1_wid), - 'pct1y': fd(' {pc3:7}', ' {pc3:7}', 8), + 'pct1y': fd(' {pc3:8}', ' {pc3:8}', 9), 'pct1m': fd(' {pc4:7}', ' {pc4:7}', 8), 'pct1w': fd(' {pc1:7}', ' {pc1:7}', 8), 'pct1d': fd(' {pc2:7}', ' {pc2:7}', 8), @@ -1169,7 +1171,7 @@ class Ticker: mcap = 'MarketCap(B)', pc1 = ' CHG_7d', pc2 = 'CHG_24h', - pc3 = 'CHG_1y', + pc3 = ' CHG_1y', pc4 = 'CHG_30d', upd = 'UPDATED', amt = ' AMOUNT', diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index 7be5e90..d4c0055 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -40,10 +40,10 @@ opts_data = { used to supply a USD exchange rate for missing assets. -C, --cached-data Use cached data from previous network query instead of live data from server --D, --cachedir=D Read and write cached JSON data to directory ‘D’ - instead of ‘~/{dfl_cachedir}’ -d, --download=D Retrieve and cache asset data ‘D’ from network (valid options: {ds}) +-D, --cachedir=D Read and write cached JSON data to directory ‘D’ + instead of ‘~/{dfl_cachedir}’ -e, --add-precision=N Add ‘N’ digits of precision to columns -E, --elapsed Show elapsed time in UPDATED column (see --update-time) -F, --portfolio Display portfolio data diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index a3edd34..c2a4188 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -133,6 +133,7 @@ class CmdTestScripts(CmdTestBase): cached_data = True, add_opts = [], use_proxy = True, + no_msg = False, exit_val = None): t = self.spawn( 'mmnode-ticker', @@ -141,6 +142,7 @@ class CmdTestScripts(CmdTestBase): + (['--proxy=http://asdfzxcv:32459'] if use_proxy else []) + add_opts + args, + no_msg = no_msg, exit_val = exit_val) if expect_list: t.match_expect_list(expect_list) From 84e8ea65d0b86cab2896125e1f1022d4848742a8 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 20 Oct 2025 09:14:34 +0000 Subject: [PATCH 46/51] mmnode-ticker: display percent change columns in terms of non-USD assets Any crypto or finance asset may be specified. Examples: # Display percentage changes in relation to Bitcoin: $ mmnode-ticker --widest --pchg-unit=btc # In relation to Gold: $ mmnode-ticker --widest --pchg-unit=gc=f # In relation to Euros: $ mmnode-ticker --widest --pchg-unit=eurusd=x # In relation to the Nasdaq Index: $ mmnode-ticker --widest --pchg-unit=^ixic --- mmgen_node_tools/Ticker.py | 29 +++++++++++++++-- mmgen_node_tools/data/version | 2 +- mmgen_node_tools/main_ticker.py | 2 ++ test/cmdtest_d/misc.py | 38 +++++++++++++++++++++++ test/ref/ticker/ticker-cfg-sort-pchg.yaml | 14 +++++++++ 5 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 test/ref/ticker/ticker-cfg-sort-pchg.yaml diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 7bcb567..4b49b44 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -535,6 +535,10 @@ def gen_data(data): 'name': 'US Dollar', 'price_usd': Decimal(1), 'price_btc': Decimal(1) / btcusd, + 'percent_change_24h': 0.0, + 'percent_change_7d': 0.0, + 'percent_change_30d': 0.0, + 'percent_change_1y': 0.0, 'last_updated': None}) def cache_data(data_src, no_overwrite=False): @@ -742,7 +746,8 @@ def make_cfg(gcfg_arg): {k: tuple(parse_asset_id(e) for e in v) for k, v in cfg_in.cfg['assets'].items()}) for hdr, data in ( ('user_uniq', get_usr_assets()), - ('portfolio_uniq', get_portfolio_assets())): + ('portfolio_uniq', get_portfolio_assets()), + ('pchg_unit_uniq', [pchg_unit] if pchg_unit else None)): if data: if uniq_data := tuple(gen_uniq(data, 'symbol', preload=rows)): rows[hdr] = uniq_data @@ -791,6 +796,7 @@ def make_cfg(gcfg_arg): 'portfolio', 'sort', 'percent_cols', + 'pchg_unit', 'asset_limit', 'cached_data', 'elapsed', @@ -833,6 +839,9 @@ def make_cfg(gcfg_arg): if portfolio and asset_range: die(1, '--portfolio not supported in market cap view') + pchg_unit = (lambda s: parse_asset_id(s, require_label=False) if s else None)( + get_cfg_var('pchg_unit')) + cfg = cfg_tuple( rows = create_rows(), usr_rows = usr_rows, @@ -849,6 +858,7 @@ def make_cfg(gcfg_arg): portfolio = portfolio, sort = get_sort_opt(), percent_cols = parse_percent_cols(get_cfg_var('percent_cols')), + pchg_unit = pchg_unit, asset_limit = get_cfg_var('asset_limit'), cached_data = get_cfg_var('cached_data'), elapsed = get_cfg_var('elapsed'), @@ -890,7 +900,7 @@ class Ticker: offer = None to_asset = None - hidden_groups = ('extra',) + hidden_groups = ('extra', 'pchg_unit_uniq') def __init__(self, data): @@ -925,6 +935,14 @@ class Ticker: cfg = cfg._replace( portfolio = sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)) + if cfg.pchg_unit: + self.pchg_data = self.data[self.get_id(cfg.pchg_unit)] + self.pchg_factors = {k: (self.pchg_data[k] / 100) + 1 for k in ( + 'percent_change_24h', + 'percent_change_7d', + 'percent_change_30d', + 'percent_change_1y')} + self.col_usd_prices = {k: self.data[k]['price_usd'] for k in self.col_ids} self.prices = {row.id: self.get_row_prices(row.id) for row in self.rows if row.id in data} self.prices['usd-us-dollar'] = self.get_row_prices('usd-us-dollar') @@ -995,6 +1013,11 @@ class Ticker: text = sort_params[cfg.sort[0]].desc + ('' if cfg.sort[1] else ' [reversed]') yield f'Sort order: {pink(text.upper())}' + if cfg.pchg_unit: + yield 'Percent change unit: {}'.format(orange('{} ({})'.format( + self.pchg_data['symbol'], + self.pchg_data['name'].upper()))) + for asset in self.usr_col_assets: if asset.symbol != 'USD': usdprice = self.data[asset.id]['price_usd'] @@ -1085,6 +1108,8 @@ class Ticker: def fmt_pct(d, key, wid=7): if (n := d.get(key)) is None: return gray(' --') + if cfg.pchg_unit: + n = ((((n / 100) + 1) / self.pchg_factors[key]) - 1) * 100 return (red, green)[n>=0](f'{n:+{wid}.2f}') p = self.prices[d['id']] diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 6eb528d..47203f7 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev8 +3.6.dev9 diff --git a/mmgen_node_tools/main_ticker.py b/mmgen_node_tools/main_ticker.py index d4c0055..cfca2ac 100755 --- a/mmgen_node_tools/main_ticker.py +++ b/mmgen_node_tools/main_ticker.py @@ -63,6 +63,8 @@ opts_data = { -t, --testing Print command(s) to be executed to stdout and exit -T, --thousands-comma Use comma as a thousands separator -u, --update-time Include UPDATED (last update time) column +-U, --pchg-unit=A Use asset ‘A’ as unit of reference for percentage + change columns (default: USD) -v, --verbose Be more verbose -w, --wide Display most optional columns (same as -unT -p d,w) -W, --widest Display all optional columns (same as -unT -p d,w,m,y) diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index c2a4188..9112f48 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -98,6 +98,9 @@ class CmdTestScripts(CmdTestBase): ('ticker27', 'ticker [--sort=rp -r algo,ada]'), ('ticker28', 'ticker [--sort=d -r algo,ada]'), ('ticker29', 'ticker [--sort=y -r algo,ada]'), + ('ticker30', 'ticker [--cached-data --wide --pchg-unit=btc --sort=d] (cf with config file)'), + ('ticker31', 'ticker [--cached-data --wide --pchg-unit=usd] (cf with no USD)'), + ('ticker32', 'ticker [--cached-data --wide --pchg-unit=gc=f]'), ) } @@ -438,3 +441,38 @@ class CmdTestScripts(CmdTestBase): [], ['ETHEREUM', 'BITCOIN', 'MONERO', 'S&P', 'DOW', 'NASDAQ', 'CARDANO', 'ALGORAND'], add_opts = ['--widest', '-s', 'y', '-r', 'ada,algo']) + + def ticker30(self): + self.copy_file('ticker-cfg-sort-pchg.yaml', 'ticker-cfg.yaml') + t = self.ticker(add_opts=['--wide']) + chk1 = '\n'.join(t.read().splitlines()[5:-2]) + self.rm_file('ticker-cfg.yaml') + + self.copy_file('ticker-cfg-bad.yaml', 'ticker-cfg.yaml') + t = self.ticker(add_opts=['--wide', '--pchg-unit=btc', '--sort=d'], no_msg=True) + chk2 = '\n'.join(t.read().splitlines()[5:-2]) + self.rm_file('ticker-cfg.yaml') + + assert chk1 == chk2, f'\nOUTPUT 1\n{chk1}\n!= OUTPUT 2\n{chk2}\n' + return t + + def ticker31(self): + t = self.ticker(add_opts=['--wide']) + chk1 = '\n'.join(t.read().splitlines()[5:-2]) + + t = self.ticker(add_opts=['--wide', '--pchg-unit=usd'], no_msg=True) + chk2 = '\n'.join(t.read().splitlines()[6:-2]) + + assert chk1 == chk2, f'\nOUTPUT 1\n{chk1}\n!= OUTPUT 2\n{chk2}\n' + return t + + def ticker32(self): + return self.ticker( + [], + [ + 'BITCOIN', r'\+10.99', r'\+7.06', '-1.18', r'\+1.05', + 'ETHEREUM', + 'GOLD', r'\+0.00', r'\+0.00', r'\+0.00', r'\+0.00', + 'SILVER' + ], + add_opts = ['--widest', '--pchg-unit=gc=f']) diff --git a/test/ref/ticker/ticker-cfg-sort-pchg.yaml b/test/ref/ticker/ticker-cfg-sort-pchg.yaml new file mode 100644 index 0000000..4660ae4 --- /dev/null +++ b/test/ref/ticker/ticker-cfg-sort-pchg.yaml @@ -0,0 +1,14 @@ +sort: d +pchg_unit: btc + +assets: + coin1: + - btc-bitcoin + - ltc-litecoin + - eth-ethereum + - xmr-monero + - bad-badcoin + commodity: + - gc=f + - si=f + - bz=f From d02c9936ef7504e70b07e82401042ae5fc58901a Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Wed, 22 Oct 2025 10:33:16 +0000 Subject: [PATCH 47/51] mmnode-ticker: fix market cap sort with USD row --- mmgen_node_tools/Ticker.py | 1 + mmgen_node_tools/data/version | 2 +- test/cmdtest_d/misc.py | 26 +++++++++++++++++++++++++- test/ref/ticker/ticker-cfg-usd.yaml | 11 +++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 test/ref/ticker/ticker-cfg-usd.yaml diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 4b49b44..15e0e9c 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -539,6 +539,7 @@ def gen_data(data): 'percent_change_7d': 0.0, 'percent_change_30d': 0.0, 'percent_change_1y': 0.0, + 'market_cap': 0, 'last_updated': None}) def cache_data(data_src, no_overwrite=False): diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 47203f7..bd10d35 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev9 +3.6.dev10 diff --git a/test/cmdtest_d/misc.py b/test/cmdtest_d/misc.py index 9112f48..d5cb51d 100755 --- a/test/cmdtest_d/misc.py +++ b/test/cmdtest_d/misc.py @@ -101,6 +101,9 @@ class CmdTestScripts(CmdTestBase): ('ticker30', 'ticker [--cached-data --wide --pchg-unit=btc --sort=d] (cf with config file)'), ('ticker31', 'ticker [--cached-data --wide --pchg-unit=usd] (cf with no USD)'), ('ticker32', 'ticker [--cached-data --wide --pchg-unit=gc=f]'), + ('ticker33', 'ticker [--cached-data --wide --pchg-unit=btc --sort=c] (cfg file with USD)'), + ('ticker34', 'ticker [--cached-data --wide --pchg-unit=btc --sort=y] (cfg file with USD)'), + ('ticker35', 'ticker [--cached-data --wide --pchg-unit=btc --sort=p] (cfg file with USD)'), ) } @@ -475,4 +478,25 @@ class CmdTestScripts(CmdTestBase): 'GOLD', r'\+0.00', r'\+0.00', r'\+0.00', r'\+0.00', 'SILVER' ], - add_opts = ['--widest', '--pchg-unit=gc=f']) + add_opts = ['--widest', '--pchg-unit=gc=f', '--sort=c']) + + def _ticker_cur(self, sort): + self.copy_file('ticker-cfg-usd.yaml', 'ticker-cfg.yaml') + t = self.ticker( + [], + [ + 'BITCOIN 23,250.77 1.00000000 \+0.00', + 'US DOLLAR 1.00 0.00004301 -15.93', + ], + add_opts = ['--widest', '--pchg-unit=btc', f'--sort={sort}']) + self.rm_file('ticker-cfg.yaml') + return t + + def ticker33(self): + return self._ticker_cur(sort='c') + + def ticker34(self): + return self._ticker_cur(sort='y') + + def ticker35(self): + return self._ticker_cur(sort='p') diff --git a/test/ref/ticker/ticker-cfg-usd.yaml b/test/ref/ticker/ticker-cfg-usd.yaml new file mode 100644 index 0000000..c31f85f --- /dev/null +++ b/test/ref/ticker/ticker-cfg-usd.yaml @@ -0,0 +1,11 @@ +assets: + coin1: + - btc-bitcoin + - eth-ethereum + - xmr-monero + commodity: + - gc=f + - si=f + currency: + - usd-us-dollar + - eurusd=x From f5c165c36201c0b251298fd08cef9566c42b37ee Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 8 Dec 2025 14:57:31 +0000 Subject: [PATCH 48/51] whitespace, minor cleanups and fixes --- mmgen_node_tools/BlocksInfo.py | 5 +- mmgen_node_tools/Ticker.py | 3 +- test/cmdtest_d/regtest.py | 110 +++++++++++++++++---------------- 3 files changed, 61 insertions(+), 57 deletions(-) diff --git a/mmgen_node_tools/BlocksInfo.py b/mmgen_node_tools/BlocksInfo.py index a59bf50..c068017 100755 --- a/mmgen_node_tools/BlocksInfo.py +++ b/mmgen_node_tools/BlocksInfo.py @@ -353,7 +353,8 @@ class BlocksInfo: nblocks, step = (add1, add2) if last is None else (None, add1) - if p.debug: msg(repr(self.range_data(first, last, from_tip, nblocks, step))) + if p.debug: + msg(repr(self.range_data(first, last, from_tip, nblocks, step))) if nblocks: if first is None: @@ -643,9 +644,7 @@ class BlocksInfo: values = {n: d.func(n) for n in fnames} col1_w = max((len(l) for l in lbls.values()), default=0) + 2 - print(d.spec_vals) for v in d.spec_vals: - print(v) if v.condition(values): try: idx = fnames.index(v.insert_after) + 1 except: idx = 0 diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 15e0e9c..50bb85b 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -970,8 +970,7 @@ class Ticker: t = int(d[row.id]['last_updated']) except TypeError as e: d[row.id]['last_updated_fmt'] = gray('--' if 'NoneType' in str(e) else str(e)) - except KeyError as e: - msg(str(e)) + except KeyError: pass else: t_fmt = d[row.id]['last_updated_fmt'] = fmt_func( diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index a504678..b35ec7a 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -23,18 +23,18 @@ from ..include.common import cfg, imsg, stop_test_daemons, joinpath from .base import CmdTestBase args1 = ['--bob'] -args2 = ['--bob','--rpc-backend=http'] +args2 = ['--bob', '--rpc-backend=http'] -def gen_addrs(proto,network,keys): +def gen_addrs(proto, network, keys): from mmgen.tool.api import tool_api tool = tool_api(cfg) - tool.init_coin(proto.coin,'regtest') + tool.init_coin(proto.coin, 'regtest') tool.addrtype = proto.mmtypes[-1] return [tool.privhex2addr('{:064x}'.format(key)) for key in keys] class CmdTestRegtest(CmdTestBase): 'various operations via regtest mode' - networks = ('btc','ltc','bch') + networks = ('btc', 'ltc', 'bch') passthru_opts = ('coin',) tmpdir_nums = [1] color = True @@ -110,31 +110,31 @@ class CmdTestRegtest(CmdTestBase): if trunner is None: return if cfg._proto.testnet: - die(2,'--testnet and --regtest options incompatible with regtest test suite') + die(2, '--testnet and --regtest options incompatible with regtest test suite') self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) - self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto,'regtest',[1,2,3,4,5])] + self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto, 'regtest', [1, 2, 3, 4, 5])] self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' self.regtest = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) def setup(self): - stop_test_daemons(self.proto.network_id,force=True,remove_datadir=True) + stop_test_daemons(self.proto.network_id, force=True, remove_datadir=True) from shutil import rmtree try: - rmtree(joinpath(self.tr.data_dir,'regtest')) + rmtree(joinpath(self.tr.data_dir, 'regtest')) except: pass t = self.spawn( 'mmgen-regtest', (['--bdb-wallet'] if self.use_bdb_wallet else []) + ['--setup-no-stop-daemon', 'setup']) - for s in ('Starting','Creating','Creating','Creating','Mined','Setup complete'): + for s in ('Starting', 'Creating', 'Creating', 'Creating', 'Mined', 'Setup complete'): t.expect(s) return t def netrate(self, add_args, expect_str, exit_val=None): t = self.spawn('mmnode-netrate', args1 + add_args, exit_val=exit_val) - t.expect(expect_str,regex=True) + t.expect(expect_str, regex=True) return t def netrate1(self): @@ -147,35 +147,35 @@ class CmdTestRegtest(CmdTestBase): return 'ok' return t - def halving_calculator(self,add_args,expect_list): - t = self.spawn('mmnode-halving-calculator',args1+add_args) + def halving_calculator(self, add_args, expect_list): + t = self.spawn('mmnode-halving-calculator', args1+add_args) t.match_expect_list(expect_list) return t def halving_calculator1(self): - return self.halving_calculator(['--help'],['USAGE:']) + return self.halving_calculator(['--help'], ['USAGE:']) def halving_calculator2(self): - return self.halving_calculator([],['Current block: 393',f'Current block subsidy: 12.5 {cfg.coin}']) + return self.halving_calculator([], ['Current block: 393', f'Current block subsidy: 12.5 {cfg.coin}']) def halving_calculator3(self): - return self.halving_calculator(['--list'],['33 4950','0']) + return self.halving_calculator(['--list'], ['33 4950', '0']) def halving_calculator4(self): - return self.halving_calculator(['--mined'],['0 0.0000015 14949.9999835']) + return self.halving_calculator(['--mined'], ['0 0.0000015 14949.9999835']) def halving_calculator5(self): - return self.halving_calculator(['--mined','--bdr-proj=5'],['5.00000 0 0.0000015 14949.9999835']) + return self.halving_calculator(['--mined', '--bdr-proj=5'], ['5.00000 0 0.0000015 14949.9999835']) def halving_calculator6(self): - return self.halving_calculator(['--mined','--sample-size=20'],['33 4950','0 0.0000015 14949.9999835']) + return self.halving_calculator(['--mined', '--sample-size=20'], ['33 4950', '0 0.0000015 14949.9999835']) - def sendto(self,addr,amt): - return self.spawn('mmgen-regtest',['send',addr,amt]) + def sendto(self, addr, amt): + return self.spawn('mmgen-regtest', ['send', addr, amt]) - def sendto1(self): return self.sendto(self.addrs[0],'0.123') - def sendto2(self): return self.sendto(self.addrs[0],'0.234') - def sendto3(self): return self.sendto(self.addrs[1],'0.345') + def sendto1(self): return self.sendto(self.addrs[0], '0.123') + def sendto2(self): return self.sendto(self.addrs[0], '0.234') + def sendto3(self): return self.sendto(self.addrs[1], '0.345') def addrbal(self, args, expect_list): t = self.spawn('mmnode-addrbal', args2 + args) @@ -231,7 +231,7 @@ class CmdTestRegtest(CmdTestBase): [ 'No balance', '2 unspent outputs in 2 blocks', - '394','0.123','395','0.234', + '394', '0.123', '395', '0.234', 'No balance' ]) @@ -240,7 +240,7 @@ class CmdTestRegtest(CmdTestBase): ['--tabular', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - -', - self.addrs[0] + ' 2 395','0.357', + self.addrs[0] + ' 2 395', '0.357', self.addrs[3] + ' - - -', ]) @@ -249,11 +249,11 @@ class CmdTestRegtest(CmdTestBase): ['--tabular', '--first-block', self.addrs[4], self.addrs[0], self.addrs[3]], [ self.addrs[4] + ' - - - -', - self.addrs[0] + ' 2 394','395','0.357', + self.addrs[0] + ' 2 394', '395', '0.357', self.addrs[3] + ' - - - -', ]) - def blocks_info(self,args,expect_list): + def blocks_info(self, args, expect_list): t = self.spawn('mmnode-blocks-info', args1 + args) t.match_expect_list(expect_list) return t @@ -261,7 +261,7 @@ class CmdTestRegtest(CmdTestBase): def blocks_info1(self): return self.blocks_info( ['--help'], - ['USAGE:','OPTIONS:']) + ['USAGE:', 'OPTIONS:']) def blocks_info2(self): return self.blocks_info( @@ -278,7 +278,7 @@ class CmdTestRegtest(CmdTestBase): ]) def blocks_info4(self): - n1,i1,o1,n2,i2,o2 = (2,1,3,6,3,9) if cfg.coin == 'BCH' else (2,1,4,6,3,12) + n1, i1, o1, n2, i2, o2 = (2, 1, 3, 6, 3, 9) if cfg.coin == 'BCH' else (2, 1, 4, 6, 3, 12) return self.blocks_info( ['--miner-info', '--fields=all', '--stats=all', '+3'], [ @@ -302,21 +302,21 @@ class CmdTestRegtest(CmdTestBase): from collections import namedtuple t = tool_api(cfg) - t.init_coin(self.proto.coin,self.proto.network) + t.init_coin(self.proto.coin, self.proto.network) t.addrtype = 'compressed' if self.proto.coin == 'BCH' else 'bech32' - wp = namedtuple('wifaddrpair',['wif','addr']) + wp = namedtuple('wifaddrpair', ['wif', 'addr']) def gen(): - for n in range(0xfaceface,nPairs+0xfaceface): + for n in range(0xfaceface, nPairs+0xfaceface): wif = t.hex2wif(f'{n:064x}') yield wp( wif, t.wif2addr(wif) ) return list(gen()) - def gen_fees(n_in,low,high): + def gen_fees(n_in, low, high): # very approximate tx size estimation: - ibytes,wbytes,obytes = (148,0,34) if self.proto.coin == 'BCH' else (43,108,31) + ibytes, wbytes, obytes = (148, 0, 34) if self.proto.coin == 'BCH' else (43, 108, 31) x = (ibytes + (wbytes//4) + (obytes * nPairs)) * self.proto.coin_amt.satoshi n = n_in - 1 @@ -325,14 +325,20 @@ class CmdTestRegtest(CmdTestBase): for i in range(n_in): yield Decimal(low + (i/n)**6 * vmax) * x - async def do_tx(inputs,outputs,wif): - tx_hex = await r.rpc_call( 'createrawtransaction', inputs, outputs ) - tx = await r.rpc_call( 'signrawtransactionwithkey', tx_hex, [wif], [], self.proto.sighash_type ) + async def do_tx(inputs, outputs, wif): + tx_hex = await r.rpc_call('createrawtransaction', inputs, outputs) + if wif: + tx = await r.rpc_call( + 'signrawtransactionwithkey', + tx_hex, + [wif], + [], + self.proto.sighash_type) assert tx['complete'] return tx['hex'] async def do_tx1(): - us = await r.rpc_call('listunspent',wallet='miner') + us = await r.rpc_call('listunspent', wallet='miner') tx_input = us[7] # 25 BTC in coinbase -- us[0] could have < 25 BTC fee = self.proto.coin_amt('0.001') outputs = {p.addr: tx1_amt for p in pairs[:nTxs]} @@ -342,7 +348,7 @@ class CmdTestRegtest(CmdTestBase): outputs, await r.miner_wif) - async def do_tx2(tx,pairno): + async def do_tx2(tx, pairno): fee = self.proto.coin_amt(fees[pairno], from_decimal=True) outputs = {p.addr: tx2_amt for p in pairs} outputs.update({burn_addr: tx1_amt - (tx2_amt*len(pairs)) - fee}) @@ -353,10 +359,10 @@ class CmdTestRegtest(CmdTestBase): async def do_txs(tx_in): for pairno in range(nTxs): - tx_hex = await do_tx2(tx_in,pairno) - await r.rpc_call('sendrawtransaction',tx_hex) + tx_hex = await do_tx2(tx_in, pairno) + await r.rpc_call('sendrawtransaction', tx_hex) - self.spawn('',msg_only=True) + self.spawn('', msg_only=True) r = self.regtest nPairs = 100 @@ -372,20 +378,20 @@ class CmdTestRegtest(CmdTestBase): tx1_hex = await do_tx1() imsg('Relaying funding transaction') - await r.rpc_call('sendrawtransaction',tx1_hex) + await r.rpc_call('sendrawtransaction', tx1_hex) imsg('Mining a block') - await r.generate(1,silent=True) + await r.generate(1, silent=True) imsg('Generating fees for mempool transactions') - fees = list(gen_fees(nTxs,2,120)) + fees = list(gen_fees(nTxs, 2, 120)) imsg(f'Creating and relaying {nTxs} mempool transactions with {nPairs} outputs each') - await do_txs(await r.rpc_call('decoderawtransaction',tx1_hex)) + await do_txs(await r.rpc_call('decoderawtransaction', tx1_hex)) return 'ok' - def _feeview(self,args,expect_list=[]): + def _feeview(self, args, expect_list=[]): t = self.spawn('mmnode-feeview', args1 + args) if expect_list: t.match_expect_list(expect_list) @@ -395,7 +401,7 @@ class CmdTestRegtest(CmdTestBase): return self._feeview([]) def feeview2(self): - return self._feeview(['--columns=40','--include-current']) + return self._feeview(['--columns=40', '--include-current']) def feeview3(self): return self._feeview(['--precision=6']) @@ -404,7 +410,7 @@ class CmdTestRegtest(CmdTestBase): return self._feeview(['--detail']) def feeview5(self): - return self._feeview(['--show-empty','--log',f'--outdir={self.tmpdir}']) + return self._feeview(['--show-empty', '--log', f'--outdir={self.tmpdir}']) def feeview6(self): return self._feeview(['--ignore-below=1MB']) @@ -414,13 +420,13 @@ class CmdTestRegtest(CmdTestBase): async def feeview8(self): imsg('Clearing mempool') - await self.regtest.generate(1,silent=True) + await self.regtest.generate(1, silent=True) return self._feeview([]) def stop(self): if cfg.no_daemon_stop: - self.spawn('',msg_only=True) + self.spawn('', msg_only=True) msg_r('(leaving daemon running by user request)') return 'ok' else: - return self.spawn('mmgen-regtest',['stop']) + return self.spawn('mmgen-regtest', ['stop']) From e5bc33a6d6267ec320ddf4ae3511fc4590e4ec1b Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 8 Dec 2025 14:57:32 +0000 Subject: [PATCH 49/51] update for MMGen Wallet v16.1.dev21 --- mmgen_node_tools/main_addrbal.py | 4 ++-- setup.cfg | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmgen_node_tools/main_addrbal.py b/mmgen_node_tools/main_addrbal.py index 4647029..743b36e 100755 --- a/mmgen_node_tools/main_addrbal.py +++ b/mmgen_node_tools/main_addrbal.py @@ -44,7 +44,7 @@ def do_output(proto, addr_data, blk_hdrs): heights = {u['height'] for u in unspents} Msg('{}Balance: {}'.format( indent, - sum(proto.coin_amt(u['amount']) for u in unspents).hl2(unit=True, fs='{:,}'))), + sum(proto.coin_amt(u['amount']) for u in unspents).hl3(unit=True, fs='{:,}'))), Msg('{}{} unspent output{} in {} block{}'.format( indent, red(str(len(unspents))), @@ -148,7 +148,7 @@ async def main(req_addrs): good_addrs = len([v for v in addr_data.values() if v]) Msg('Total: {} in {} address{}'.format( - proto.coin_amt(res['total_amount']).hl2(unit=True, fs='{:,}'), + proto.coin_amt(res['total_amount']).hl3(unit=True, fs='{:,}'), red(str(good_addrs)), suf(good_addrs, 'es'))) diff --git a/setup.cfg b/setup.cfg index d48a8c8..91b63b2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.1.dev4 + mmgen-wallet>=16.1.dev21 pyyaml yahooquery From b97b4f5f634f2498947f4ea4b25f5ac71d08db85 Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Mon, 8 Dec 2025 14:57:32 +0000 Subject: [PATCH 50/51] update for Bitcoin Core v30.0, nixpkgs 25.11 --- mmgen_node_tools/data/version | 2 +- nix/curl-cffi.nix | 50 ----------------------------------- nix/yahooquery.nix | 2 +- test/cmdtest_d/regtest.py | 9 ++++++- 4 files changed, 10 insertions(+), 53 deletions(-) delete mode 100644 nix/curl-cffi.nix diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index bd10d35..4e5cab0 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev10 +3.6.dev11 diff --git a/nix/curl-cffi.nix b/nix/curl-cffi.nix deleted file mode 100644 index 15f9764..0000000 --- a/nix/curl-cffi.nix +++ /dev/null @@ -1,50 +0,0 @@ -{ - stdenv, - lib, - python, - buildPythonPackage, - fetchPypi, - curl-impersonate-chrome, -}: - -buildPythonPackage rec { - pname = "curl-cffi"; - # version = "0.13.0"; # uses option PROXY_CREDENTIAL_NO_REUSE, unavailable in current libcurl - version = "0.10.0"; - pyproject = true; - - src = fetchPypi { - pname = "curl_cffi"; - version = version; - # hash = "sha256-YuzZCjgr1QI3UONgbgqnyxo6i6QcFCcLjl4Unr9yxco="; # 0.13.0 - hash = "sha256-PjezUmjKWEkvVO0CCuS1DDPuDeutQUXbn3RvBO1GbrA="; # 0.10.0 - }; - - patches = [ ./use-system-libs.patch ]; - - buildInputs = [ curl-impersonate-chrome ]; - - build-system = with python.pkgs; [ - cffi - setuptools - ]; - - dependencies = with python.pkgs; [ - cffi - certifi - typing-extensions - ]; - - env = lib.optionalAttrs stdenv.cc.isGNU { - NIX_CFLAGS_COMPILE = "-Wno-error=incompatible-pointer-types"; - }; - - pythonImportsCheck = [ "curl_cffi" ]; - - meta = with lib; { - description = "Python binding for curl-impersonate via cffi"; - homepage = "https://curl-cffi.readthedocs.io"; - license = licenses.mit; - maintainers = with maintainers; [ chuangzhu ]; - }; -} diff --git a/nix/yahooquery.nix b/nix/yahooquery.nix index 3e8925f..ecb2494 100644 --- a/nix/yahooquery.nix +++ b/nix/yahooquery.nix @@ -19,7 +19,7 @@ buildPythonPackage rec { build-system = with python.pkgs; [ hatchling ]; propagatedBuildInputs = with python.pkgs; [ - (callPackage ./curl-cffi.nix {}) # >=0.10.0 + curl-cffi pandas requests-futures tqdm diff --git a/test/cmdtest_d/regtest.py b/test/cmdtest_d/regtest.py index b35ec7a..387594b 100755 --- a/test/cmdtest_d/regtest.py +++ b/test/cmdtest_d/regtest.py @@ -114,7 +114,7 @@ class CmdTestRegtest(CmdTestBase): self.proto = init_proto( cfg, self.proto.coin, network='regtest', need_amt=True ) self.addrs = [a.views[a.view_pref] for a in gen_addrs(self.proto, 'regtest', [1, 2, 3, 4, 5])] - self.use_bdb_wallet = self.bdb_wallet or self.proto.coin != 'BTC' + self.use_bdb_wallet = self.bdb_wallet and self.proto.coin != 'BTC' self.regtest = MMGenRegtest(cfg, self.proto.coin, bdb_wallet=self.use_bdb_wallet) def setup(self): @@ -334,6 +334,13 @@ class CmdTestRegtest(CmdTestBase): [wif], [], self.proto.sighash_type) + else: + tx = await r.rpc_call( + 'signrawtransactionwithwallet', + tx_hex, + None, # prevtxs + self.proto.sighash_type, + wallet = 'miner') assert tx['complete'] return tx['hex'] From f88b70c2e2413d6261dcc3d5d5d930ec778d9eff Mon Sep 17 00:00:00 2001 From: The MMGen Project Date: Tue, 27 Jan 2026 08:46:15 +0000 Subject: [PATCH 51/51] mmnode-ticker: support nested assets in `ticker-portfolio.yaml` --- mmgen_node_tools/Ticker.py | 39 ++++++++++++++------- mmgen_node_tools/data/ticker-portfolio.yaml | 10 +++++- mmgen_node_tools/data/version | 2 +- setup.cfg | 2 +- test/ref/ticker/ticker-portfolio.yaml | 9 +++-- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/mmgen_node_tools/Ticker.py b/mmgen_node_tools/Ticker.py index 50bb85b..a0adb39 100755 --- a/mmgen_node_tools/Ticker.py +++ b/mmgen_node_tools/Ticker.py @@ -720,15 +720,28 @@ def make_cfg(gcfg_arg): + usr_columns) def get_portfolio_assets(): - if cfg_in.portfolio and gcfg.portfolio: - ret = (parse_asset_id(e) for e in cfg_in.portfolio) + if portfolio: + ret = (parse_asset_id(e) for e in portfolio) return tuple(e for e in ret if (not gcfg.btc) or e.symbol == 'BTC') else: return () - def get_portfolio(): - return tuple((k, Decimal(v)) for k, v in cfg_in.portfolio.items() - if (not gcfg.btc) or k == 'btc-bitcoin') + def parse_portfolio(): + ret = {} + def add(k, v): + if gcfg.btc and k != 'btc-bitcoin': + return + if k in ret: + ret[k] += Decimal(v) + else: + ret[k] = Decimal(v) + for k, v in cfg_in.portfolio.items(): + if isinstance(v, dict): + for k2, v2 in v.items(): + add(k2, v2) + else: + add(k, v) + return ret def parse_add_precision(arg): if not arg: @@ -834,7 +847,7 @@ def make_cfg(gcfg_arg): proxy2 = get_proxy('proxy2') portfolio = ( - get_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query + parse_portfolio() if cfg_in.portfolio and get_cfg_var('portfolio') and not query else None) if portfolio and asset_range: @@ -928,13 +941,14 @@ class Ticker: key = sort_params[code].key sort_dfl = sort_params[code].sort_dfl sort_func = lambda row: data.get(row.id, {key: sort_dfl})[key] - pf_sort_func = lambda row: data.get(row[0], {key: sort_dfl})[key] + pf_sort_func = lambda row: data.get(row, {key: sort_dfl})[key] for group in self.rows.keys(): if group not in self.hidden_groups: self.rows[group] = sorted(self.rows[group], key=sort_func, reverse=reverse) if cfg.portfolio: - cfg = cfg._replace( - portfolio = sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)) + cfg = cfg._replace(portfolio = + {k: cfg.portfolio[k] + for k in sorted(cfg.portfolio, key=pf_sort_func, reverse=reverse)}) if cfg.pchg_unit: self.pchg_data = self.data[self.get_id(cfg.pchg_unit)] @@ -1061,7 +1075,7 @@ class Ticker: yield blue('PORTFOLIO') yield self.table_hdr yield '-' * self.hl_wid - for sym, amt in cfg.portfolio: + for sym, amt in cfg.portfolio.items(): try: yield self.fmt_row(self.data[sym], amt=amt) except KeyError: @@ -1086,10 +1100,9 @@ class Ticker: self.format_last_updated_col() if cfg.portfolio: - pf_dict = dict(cfg.portfolio) - self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * pf_dict[row.id] + self.prices['total'] = {col_id: sum(self.prices[row.id][col_id] * cfg.portfolio[row.id] for row in self.rows - if row.id in pf_dict and row.id in data) + if row.id in cfg.portfolio and row.id in data) for col_id in self.col_ids} self.init_prec() diff --git a/mmgen_node_tools/data/ticker-portfolio.yaml b/mmgen_node_tools/data/ticker-portfolio.yaml index fcadd30..b63e626 100644 --- a/mmgen_node_tools/data/ticker-portfolio.yaml +++ b/mmgen_node_tools/data/ticker-portfolio.yaml @@ -3,4 +3,12 @@ # Invoke `mmnode-ticker --list-ids` for a full list of supported asset IDs. btc-bitcoin: '1.23456789' eth-ethereum: '2.3456789012' -xmr-monero: '4.5678901234' +xmr-monero: '4.567890123456' + +# Nested values are supported. Values for each asset will be summed. +wallet2: + btc-bitcoin: '0.12345678' + ltc-litecoin: '1.23456789' + +exchange1: + xmr-monero: '12.345678901234' diff --git a/mmgen_node_tools/data/version b/mmgen_node_tools/data/version index 4e5cab0..6b99850 100644 --- a/mmgen_node_tools/data/version +++ b/mmgen_node_tools/data/version @@ -1 +1 @@ -3.6.dev11 +3.6.dev12 diff --git a/setup.cfg b/setup.cfg index 91b63b2..0753d4f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ python_requires = >=3.11 include_package_data = True install_requires = - mmgen-wallet>=16.1.dev21 + mmgen-wallet>=16.1.dev26 pyyaml yahooquery diff --git a/test/ref/ticker/ticker-portfolio.yaml b/test/ref/ticker/ticker-portfolio.yaml index 332257f..ca0c2b2 100644 --- a/test/ref/ticker/ticker-portfolio.yaml +++ b/test/ref/ticker/ticker-portfolio.yaml @@ -1,5 +1,10 @@ btc-bitcoin: '1.23456789' eth-ethereum: '2.345678901234567890' -xmr-monero: '4.567890123456' +xmr-monero: '3.333390123456' ada-cardano: '123.45678901' -algo-algorand: '234.5678901' + +wallet2: + algo-algorand: '234.5678901' + +exchange1: + xmr-monero: '1.2345'